cmd_doctor_test.v

1module cli23import os4import compress.zlib5import crypto.sha2566import json27import src.exchange8import src.release9import src.remote10import src.events11import src.history12import src.machine13import src.snapshot14import src.storage1516fn testsuite_begin() {17	os.setenv('HOME', os.join_path(os.temp_dir(), 'koh3-test-home-${os.getpid()}'), true)18}1920fn test_doctor_checks_cycles_and_missing_heads_without_walking_forever() {21	saves := [history.Save{ id: 'aaaaaaaaaaaa', parent: 'bbbbbbbbbbbb' },22		history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' }]23	assert doctor_history(saves, 'cccccccccccc').any(it.contains('head'))24	assert doctor_history(saves, 'aaaaaaaaaaaa').any(it.contains('cycle'))25	assert doctor_history([history.Save{ id: 'aaaaaaaaaaaa', parent: 'cccccccccccc' }],26		'aaaaaaaaaaaa').any(it.contains('missing parent'))27	assert doctor_history([]history.Save{}, '').len == 028	assert doctor_history([history.Save{ id: 'aaaaaaaaaaaa' }], '').any(it.contains('no head'))29}3031fn test_doctor_checks_off_line_paths_and_file_directory_collisions() {32	mut report := DoctorJson{}33	doctor_files('aaaaaaaaaaaa', [snapshot.FileEntry{ path: '../unsafe' },34		snapshot.FileEntry{ path: 'a' }, snapshot.FileEntry{ path: 'a/b' }], []events.Event{},35		mut report)36	assert report.problems.any(it.contains('unsafe path'))37	assert report.problems.any(it.contains('file and a directory'))38}3940fn test_doctor_reports_unsafe_save_paths_with_a_quoted_strip_command() {41	dir := os.join_path(os.temp_dir(), 'koh3_doctor_paths_${os.getpid()}')42	os.rmdir_all(dir) or {}43	os.mkdir_all(dir)!44	defer {45		os.rmdir_all(dir) or {}46	}47	store := storage.create(dir)!48	defer {49		store.close()50	}51	hash := store.put_object('unsafe'.bytes())!52	store.begin()!53	history.import_save(store, history.Save{54		id: 'badbadbadbad'55	}, [snapshot.FileEntry{56		path: "../June's file"57		hash: hash58	}])!59	history.set_head(store, 'badbadbadbad')!60	store.end()!61	mut report := DoctorJson{}62	doctor_files('badbadbadbad', history.files_of(store, 'badbadbadbad')!, []events.Event{}, mut report)63	assert report.problems == [64		"save [badbadbadbad] has unsafe path ../June's file — koh strip '../June'\\''s file'",65	]66}6768struct DoctorReadOnly {69mut:70	items map[string][]u871}7273fn (mut backend DoctorReadOnly) get(key string) !remote.Object {74	if body := backend.items[key] {75		return remote.Object{ found: true, body: body }76	}77	return remote.Object{}78}7980fn (mut backend DoctorReadOnly) list(prefix string) ![]string {81	return backend.items.keys().filter(it.starts_with('${prefix}/'))82}8384fn (mut backend DoctorReadOnly) put_if_absent(key string, body []u8) !bool {85	panic('Doctor attempted a remote write')86}8788fn (mut backend DoctorReadOnly) put_if_match(key string, body []u8, tag string) !bool {89	panic('Doctor attempted a remote write')90}9192fn (mut backend DoctorReadOnly) delete(key string) ! {93	panic('Doctor attempted a remote delete')94}9596fn test_doctor_remote_audits_unreferenced_objects_and_release_checksums_without_writes() {97	data := 'file'.bytes()98	hash := storage.hash_hex(data)99	mut backend := DoctorReadOnly{100		items: {101			'repo/head':                             'aaaaaaaaaaaa'.bytes()102			'repo/saves/aaaaaaaaaaaa':               json2.encode(exchange.SaveRecord{103				id:    'aaaaaaaaaaaa'104				files: [exchange.FileRecord{ path: 'file', hash: hash }]105			}).bytes()106			'repo/objects/${hash}':                  zlib.compress(data)!107			'repo/releases/latest':                  '1.0.0'.bytes()108			'repo/releases/1.0.0/release.json':      json2.encode(release.Record{109				repo:    'repo'110				version: '1.0.0'111				save:    'aaaaaaaaaaaa'112				files:   [release.File{113					platform: 'linux-aarch64'114					name:     'app'115					size:     data.len116					blake3:   hash117					sha256:   sha256.sum256(data).hex()118				}]119			}).bytes()120			'repo/releases/1.0.0/linux-aarch64/app': data121		}122	}123	mut report := DoctorJson{ head_id: 'aaaaaaaaaaaa' }124	doctor_remote_content(mut backend, 'repo', mut report)!125	assert report.problems.len == 0126	assert report.remote_saves == 1127	assert report.remote_releases == 1128	backend.items['repo/objects/${storage.hash_hex('unreferenced'.bytes())}'] = 'bad zlib'.bytes()129	backend.items['repo/releases/1.0.0/linux-aarch64/app'] = 'corrupt'.bytes()130	mut damaged := DoctorJson{}131	doctor_remote_content(mut backend, 'repo', mut damaged)!132	assert damaged.problems.any(it.contains('damaged remote object'))133	assert damaged.problems.any(it.contains('checksum verification'))134}135136fn test_doctor_reports_event_errors_strip_residue_and_flags() {137	mut report := DoctorJson{}138	saves := [history.Save{ id: 'aaaaaaaaaaaa' }]139	event_list := [events.Event{ id: 'strip', kind: 'strip', path: 'private' },140		events.Event{ id: 'flag', kind: 'flag', save_id: 'aaaaaaaaaaaa' },141		events.Event{ id: 'bad', kind: 'unknown' }]142	doctor_events(event_list, saves, mut report)143	doctor_files(saves[0].id, [snapshot.FileEntry{ path: 'private/file' }], event_list, mut report)144	assert report.flagged_saves == 1145	assert report.problems.any(it.contains('unknown kind'))146	assert report.problems.any(it.contains('stripped path'))147	assert report.strip_residue_paths == ['private']148}149150struct DoctorMemory {151mut:152	items map[string][]u8153}154155fn (mut backend DoctorMemory) get(key string) !remote.Object {156	if body := backend.items[key] {157		return remote.Object{ found: true, body: body }158	}159	return remote.Object{}160}161162fn (mut backend DoctorMemory) list(prefix string) ![]string {163	return backend.items.keys().filter(it.starts_with('${prefix}/'))164}165166fn (mut backend DoctorMemory) put_if_absent(key string, body []u8) !bool {167	if key in backend.items {168		return false169	}170	backend.items[key] = body171	return true172}173174fn (mut backend DoctorMemory) put_if_match(key string, body []u8, tag string) !bool {175	return false176}177178fn (mut backend DoctorMemory) delete(key string) ! {179	backend.items.delete(key)180}181182fn doctor_store(name string) !(storage.Store, string) {183	dir := os.join_path(os.temp_dir(), 'koh3_doctor_${name}_${os.getpid()}')184	os.rmdir_all(dir) or {}185	os.mkdir_all(dir)!186	return storage.create(dir)!, dir187}188189fn test_doctor_fetches_missing_parents_from_the_remote_without_writing_to_it() {190	store, dir := doctor_store('fetch')!191	defer {192		store.close()193		os.rmdir_all(dir) or {}194	}195	data := 'parent'.bytes()196	hash := storage.hash_hex(data)197	mut backend := DoctorReadOnly{198		items: {199			'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{200				id:    'aaaaaaaaaaaa'201				files: [exchange.FileRecord{ path: 'file', hash: hash }]202			}).bytes()203			'repo/objects/${hash}':    zlib.compress(data)!204		}205	}206	store.db.exec('PRAGMA foreign_keys=OFF')!207	store.begin()!208	history.import_save(store, history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' },209		[]snapshot.FileEntry{})!210	history.set_head(store, 'bbbbbbbbbbbb')!211	store.end()!212	store.db.exec('PRAGMA foreign_keys=ON')!213	missing := missing_saves(history.all_saves(store)!, 'bbbbbbbbbbbb')214	assert missing == ['aaaaaaaaaaaa']215	assert exchange.fetch_missing(store, mut backend, 'repo', missing)! == 1216	assert missing_saves(history.all_saves(store)!, 'bbbbbbbbbbbb').len == 0217	assert store.get_object(hash)! == data218	assert exchange.fetch_missing(store, mut backend, 'repo', ['cccccccccccc']) or {219		assert err.msg().contains('does not have save')220		0221	} == 0222}223224fn test_doctor_reapplies_recorded_strips_and_finds_the_only_tip() {225	store, dir := doctor_store('strip')!226	defer {227		store.close()228		os.rmdir_all(dir) or {}229	}230	secret := store.put_object('secret'.bytes())!231	kept := store.put_object('kept'.bytes())!232	store.begin()!233	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [234		snapshot.FileEntry{ path: 'keep', hash: kept },235		snapshot.FileEntry{ path: 'private/key', hash: secret },236	])!237	history.import_save(store, history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' },238		[snapshot.FileEntry{ path: 'keep', hash: kept }])!239	events.strip(store, '', 'private')!240	store.end()!241	assert only_tip(store) or { '' } == 'bbbbbbbbbbbb'242	mut report := DoctorJson{}243	doctor_files('aaaaaaaaaaaa', history.files_of(store, 'aaaaaaaaaaaa')!, events.all(store)!, mut244		report)245	assert report.strip_residue246	assert exchange.reapply_strips(store)! == 1247	assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['keep']248	assert !store.has_object(secret)!249	assert events.strip_events(store)!.len == 1250}251252fn test_doctor_restores_objects_from_working_files_and_resends_them_to_the_remote() {253	store, dir := doctor_store('objects')!254	defer {255		store.close()256		os.rmdir_all(dir) or {}257	}258	data := 'content'.bytes()259	hash := store.put_object(data)!260	store.begin()!261	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [262		snapshot.FileEntry{ path: 'app', hash: hash },263	])!264	history.set_head(store, 'aaaaaaaaaaaa')!265	store.end()!266	store.db.exec("UPDATE objects SET data = x'00'")!267	problems := store.check_objects()!268	assert problems.map(it.hash) == [hash]269	assert local_copy(store, hash) == none270	os.write_file(os.join_path(dir, 'app'), 'content')!271	store.repair_object(hash, local_copy(store, hash) or { panic('no working copy') })!272	assert store.check_objects()!.len == 0273	store.repair_object(hash, 'other'.bytes()) or {274		assert err.msg().contains('does not match')275	}276	mut backend := DoctorMemory{277		items: {278			'repo/objects/${hash}': 'damaged'.bytes()279		}280	}281	assert exchange.resend(store, mut backend, 'repo', [hash], ['aaaaaaaaaaaa'])! == 2282	assert zlib.decompress(backend.items['repo/objects/${hash}'])! == data283	assert exchange.decode_save(backend.items['repo/saves/aaaaaaaaaaaa'])!.files[0].hash == hash284}285286// An invalid strip event authorizes nothing, and a strip frees only the objects287// it removed: an unrelated unused object may be needed for recovery.288fn test_doctor_strip_repair_ignores_invalid_events_and_keeps_unrelated_objects() {289	store, dir := doctor_store('strip_safety')!290	defer {291		store.close()292		os.rmdir_all(dir) or {}293	}294	app := store.put_object('app'.bytes())!295	secret := store.put_object('secret'.bytes())!296	orphan := store.put_object('orphan'.bytes())!297	store.begin()!298	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [299		snapshot.FileEntry{ path: 'app', hash: app },300		snapshot.FileEntry{ path: 'secret', hash: secret },301	])!302	events.receive(store, events.Event{303		id:         'invalidstrip'304		kind:       events.kind_strip305		path:       'app'306		created_at: -1307		machine:    'test'308	})!309	events.strip(store, '', 'secret')!310	store.end()!311	mut report := DoctorJson{}312	doctor_files('aaaaaaaaaaaa', [snapshot.FileEntry{ path: 'app' }], events.all(store)!, mut313		report)314	assert !report.strip_residue315	assert exchange.reapply_strips(store)! == 1316	assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['app']317	assert store.has_object(app)!318	assert store.has_object(orphan)!319	assert !store.has_object(secret)!320}321322// With a remote, only a full audit that found no other head there lets Doctor323// guess, and the guess is checked again when it is written.324fn test_doctor_guesses_head_only_when_nothing_knows_better() {325	store, dir := doctor_store('only_tip')!326	defer {327		store.close()328		os.rmdir_all(dir) or {}329	}330	store.begin()!331	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, []snapshot.FileEntry{})!332	store.end()!333	mut report := DoctorJson{334		metadata_available: true335		saves:              1336		source:             'folder/repo'337		remote_head:        'bbbbbbbbbbbb'338	}339	assert !plan_repairs(store, report).any(it.action.contains('point head'))340	report.remote_head = ''341	report.checks << DoctorCheck{'remote audit', 'skipped', '--fast requested', ''}342	assert !plan_repairs(store, report).any(it.action.contains('point head'))343	report.checks = [DoctorCheck{'remote stability', 'fail', 'remote head changed during audit', ''}]344	assert !plan_repairs(store, report).any(it.action.contains('point head'))345	report.checks = []346	assert plan_repairs(store, report).any(it.action.contains('point head'))347	report.source = ''348	repairs := plan_repairs(store, report).filter(it.action.contains('point head'))349	assert repairs.len == 1350	history.set_head(store, 'cccccccccccc')!351	if _ := repairs[0].run() {352		assert false, 'a head another command set must not be replaced'353	} else {354		assert err.msg().contains('changed'), err.msg()355	}356	assert store.state(storage.head_key) or { '' } == 'cccccccccccc'357	store.remove_state(storage.head_key)!358	repairs[0].run()!359	assert store.state(storage.head_key) or { '' } == 'aaaaaaaaaaaa'360}361362fn test_doctor_reports_objects_it_restored_before_one_it_could_not() {363	store, dir := doctor_store('partial')!364	defer {365		store.close()366		os.rmdir_all(dir) or {}367	}368	kept := store.put_object('kept'.bytes())!369	lost := store.put_object('lost'.bytes())!370	store.begin()!371	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [372		snapshot.FileEntry{ path: 'kept', hash: kept },373		snapshot.FileEntry{ path: 'lost', hash: lost },374	])!375	store.end()!376	store.db.exec("UPDATE objects SET data = x'00'")!377	os.write_file(os.join_path(dir, 'kept'), 'kept')!378	if _ := repair_objects(store, [kept, lost], '', '') {379		assert false, 'an object with no verified copy must fail its repair'380	} else {381		assert err is exchange.Partway382		assert (err as exchange.Partway).done == 'restored 1 damaged or missing object'383		assert err.msg().contains(lost)384	}385	assert store.get_object(kept)! == 'kept'.bytes()386}387388fn test_doctor_refuses_a_remote_save_with_invalid_metadata() {389	store, dir := doctor_store('invalid_fetch')!390	defer {391		store.close()392		os.rmdir_all(dir) or {}393	}394	mut backend := DoctorReadOnly{395		items: {396			'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{397				id:         'aaaaaaaaaaaa'398				created_at: -1399			}).bytes()400		}401	}402	if _ := exchange.fetch_missing(store, mut backend, 'repo', ['aaaaaaaaaaaa']) {403		assert false, 'a record with invalid metadata must not be imported'404	} else {405		assert err.msg().contains('invalid metadata')406	}407	assert history.all_saves(store)!.len == 0408}409410// An empty file's Blake3 is the hash of empty bytes, so bytes that fail to411// decompress must not pass for it.412fn test_doctor_replaces_a_damaged_remote_copy_of_an_empty_file() {413	store, dir := doctor_store('empty_object')!414	defer {415		store.close()416		os.rmdir_all(dir) or {}417	}418	hash := store.put_object([]u8{})!419	mut backend := DoctorMemory{420		items: {421			'repo/objects/${hash}': 'not zlib'.bytes()422		}423	}424	assert exchange.resend(store, mut backend, 'repo', [hash], [])! == 1425	assert zlib.decompress(backend.items['repo/objects/${hash}'])!.len == 0426}427428// The remote is asked again before head is guessed: another machine may have429// offered since the audit.430fn test_doctor_rechecks_the_remote_head_before_guessing() {431	store, dir := doctor_store('remote_tip')!432	remote_dir := os.join_path(dir, 'remote')433	defer {434		store.close()435		machine.forget_remote() or {}436		os.rmdir_all(dir) or {}437	}438	store.begin()!439	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, []snapshot.FileEntry{})!440	store.end()!441	mut folder := remote.new_folder(remote_dir)!442	machine.save_remote(machine.RemoteSettings{ kind: machine.kind_folder, path: remote_dir })!443	report := DoctorJson{444		metadata_available: true445		saves:              1446		remote_repo:        'repo'447	}448	repairs := plan_repairs(store, report).filter(it.action.contains('point head'))449	assert repairs.len == 1450	folder.put_if_absent(exchange.head_key('repo'), 'bbbbbbbbbbbb'.bytes())!451	if _ := repairs[0].run() {452		assert false, 'a head offered since the audit must stop the guess'453	} else {454		assert err.msg().contains('another head'), err.msg()455	}456	assert store.state(storage.head_key) == none457}458459// An error after some objects were restored still reports them.460fn test_doctor_reports_restored_objects_when_the_remote_then_fails() {461	store, dir := doctor_store('partial_remote')!462	defer {463		store.close()464		machine.forget_remote() or {}465		os.rmdir_all(dir) or {}466	}467	kept := store.put_object('kept'.bytes())!468	lost := store.put_object('lost'.bytes())!469	store.begin()!470	history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [471		snapshot.FileEntry{ path: 'kept', hash: kept },472		snapshot.FileEntry{ path: 'lost', hash: lost },473	])!474	store.end()!475	store.db.exec("UPDATE objects SET data = x'00'")!476	os.write_file(os.join_path(dir, 'kept'), 'kept')!477	machine.save_remote(machine.RemoteSettings{478		kind: machine.kind_folder479		path: os.join_path(dir, 'unmounted')480	})!481	if _ := repair_objects(store, [kept, lost], '', '') {482		assert false, 'an unreachable remote must fail the repair'483	} else {484		assert err is exchange.Partway485		assert (err as exchange.Partway).done == 'restored 1 damaged or missing object'486		assert err.msg().contains('not there'), err.msg()487	}488}489490// A strip event Doctor cannot trust removes nothing from a fetched save either.491fn test_doctor_fetch_ignores_invalid_strip_events() {492	store, dir := doctor_store('fetch_invalid_strip')!493	defer {494		store.close()495		os.rmdir_all(dir) or {}496	}497	data := 'app'.bytes()498	hash := storage.hash_hex(data)499	mut backend := DoctorReadOnly{500		items: {501			'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{502				id:    'aaaaaaaaaaaa'503				files: [exchange.FileRecord{ path: 'app', hash: hash }]504			}).bytes()505			'repo/objects/${hash}':    zlib.compress(data)!506		}507	}508	events.receive(store, events.Event{509		id:         'invalidstrip'510		kind:       events.kind_strip511		path:       'app'512		created_at: -1513		machine:    'test'514	})!515	assert exchange.fetch_missing(store, mut backend, 'repo', ['aaaaaaaaaaaa'])! == 1516	assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['app']517}518519// A strip whose remote half was lost is sent again from the recorded event520// when the remote still holds the path; a later path that fails still reports521// the strips already sent.522fn test_doctor_sends_a_recorded_strip_again_for_remote_residue() {523	store, dir := doctor_store('remote_residue')!524	remote_dir := os.join_path(dir, 'remote')525	defer {526		store.close()527		machine.forget_remote() or {}528		os.rmdir_all(dir) or {}529	}530	secret := 'secret'.bytes()531	hash := storage.hash_hex(secret)532	mut folder := remote.new_folder(remote_dir)!533	folder.put_if_absent(exchange.save_key('repo', 'aaaaaaaaaaaa'), json2.encode(exchange.SaveRecord{534		id:    'aaaaaaaaaaaa'535		files: [exchange.FileRecord{ path: 'app', hash: hash }]536	}).bytes())!537	folder.put_if_absent(exchange.object_key('repo', hash), zlib.compress(secret)!)!538	machine.save_remote(machine.RemoteSettings{ kind: machine.kind_folder, path: remote_dir })!539	store.set_state('remote_repo', 'repo')!540	events.strip(store, '', 'app')!541	store.set_state('strip_pending:bad', 'not JSON')!542	report := DoctorJson{543		remote_repo:          'repo'544		remote_strip_residue: ['app', 'bad']545	}546	repairs := plan_repairs(store, report).filter(it.action.contains('remote still holds'))547	assert repairs.len == 1548	if _ := repairs[0].run() {549		assert false, 'an untrusted note must stop the second path'550	} else {551		assert err is exchange.Partway552		assert (err as exchange.Partway).done == 'sent 1 strip', (err as exchange.Partway).done553	}554	record := exchange.decode_save(folder.get(exchange.save_key('repo', 'aaaaaaaaaaaa'))!.body)!555	assert record.files.len == 0556	assert !folder.get(exchange.object_key('repo', hash))!.found557	assert !exchange.strip_pending(store, 'app')558}