module cli import os import compress.zlib import crypto.sha256 import json2 import src.exchange import src.release import src.remote import src.events import src.history import src.machine import src.snapshot import src.storage fn testsuite_begin() { os.setenv('HOME', os.join_path(os.temp_dir(), 'koh3-test-home-${os.getpid()}'), true) } fn test_doctor_checks_cycles_and_missing_heads_without_walking_forever() { saves := [history.Save{ id: 'aaaaaaaaaaaa', parent: 'bbbbbbbbbbbb' }, history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' }] assert doctor_history(saves, 'cccccccccccc').any(it.contains('head')) assert doctor_history(saves, 'aaaaaaaaaaaa').any(it.contains('cycle')) assert doctor_history([history.Save{ id: 'aaaaaaaaaaaa', parent: 'cccccccccccc' }], 'aaaaaaaaaaaa').any(it.contains('missing parent')) assert doctor_history([]history.Save{}, '').len == 0 assert doctor_history([history.Save{ id: 'aaaaaaaaaaaa' }], '').any(it.contains('no head')) } fn test_doctor_checks_off_line_paths_and_file_directory_collisions() { mut report := DoctorJson{} doctor_files('aaaaaaaaaaaa', [snapshot.FileEntry{ path: '../unsafe' }, snapshot.FileEntry{ path: 'a' }, snapshot.FileEntry{ path: 'a/b' }], []events.Event{}, mut report) assert report.problems.any(it.contains('unsafe path')) assert report.problems.any(it.contains('file and a directory')) } fn test_doctor_reports_unsafe_save_paths_with_a_quoted_strip_command() { dir := os.join_path(os.temp_dir(), 'koh3_doctor_paths_${os.getpid()}') os.rmdir_all(dir) or {} os.mkdir_all(dir)! defer { os.rmdir_all(dir) or {} } store := storage.create(dir)! defer { store.close() } hash := store.put_object('unsafe'.bytes())! store.begin()! history.import_save(store, history.Save{ id: 'badbadbadbad' }, [snapshot.FileEntry{ path: "../June's file" hash: hash }])! history.set_head(store, 'badbadbadbad')! store.end()! mut report := DoctorJson{} doctor_files('badbadbadbad', history.files_of(store, 'badbadbadbad')!, []events.Event{}, mut report) assert report.problems == [ "save [badbadbadbad] has unsafe path ../June's file — koh strip '../June'\\''s file'", ] } struct DoctorReadOnly { mut: items map[string][]u8 } fn (mut backend DoctorReadOnly) get(key string) !remote.Object { if body := backend.items[key] { return remote.Object{ found: true, body: body } } return remote.Object{} } fn (mut backend DoctorReadOnly) list(prefix string) ![]string { return backend.items.keys().filter(it.starts_with('${prefix}/')) } fn (mut backend DoctorReadOnly) put_if_absent(key string, body []u8) !bool { panic('Doctor attempted a remote write') } fn (mut backend DoctorReadOnly) put_if_match(key string, body []u8, tag string) !bool { panic('Doctor attempted a remote write') } fn (mut backend DoctorReadOnly) delete(key string) ! { panic('Doctor attempted a remote delete') } fn test_doctor_remote_audits_unreferenced_objects_and_release_checksums_without_writes() { data := 'file'.bytes() hash := storage.hash_hex(data) mut backend := DoctorReadOnly{ items: { 'repo/head': 'aaaaaaaaaaaa'.bytes() 'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{ id: 'aaaaaaaaaaaa' files: [exchange.FileRecord{ path: 'file', hash: hash }] }).bytes() 'repo/objects/${hash}': zlib.compress(data)! 'repo/releases/latest': '1.0.0'.bytes() 'repo/releases/1.0.0/release.json': json2.encode(release.Record{ repo: 'repo' version: '1.0.0' save: 'aaaaaaaaaaaa' files: [release.File{ platform: 'linux-aarch64' name: 'app' size: data.len blake3: hash sha256: sha256.sum256(data).hex() }] }).bytes() 'repo/releases/1.0.0/linux-aarch64/app': data } } mut report := DoctorJson{ head_id: 'aaaaaaaaaaaa' } doctor_remote_content(mut backend, 'repo', mut report)! assert report.problems.len == 0 assert report.remote_saves == 1 assert report.remote_releases == 1 backend.items['repo/objects/${storage.hash_hex('unreferenced'.bytes())}'] = 'bad zlib'.bytes() backend.items['repo/releases/1.0.0/linux-aarch64/app'] = 'corrupt'.bytes() mut damaged := DoctorJson{} doctor_remote_content(mut backend, 'repo', mut damaged)! assert damaged.problems.any(it.contains('damaged remote object')) assert damaged.problems.any(it.contains('checksum verification')) } fn test_doctor_reports_event_errors_strip_residue_and_flags() { mut report := DoctorJson{} saves := [history.Save{ id: 'aaaaaaaaaaaa' }] event_list := [events.Event{ id: 'strip', kind: 'strip', path: 'private' }, events.Event{ id: 'flag', kind: 'flag', save_id: 'aaaaaaaaaaaa' }, events.Event{ id: 'bad', kind: 'unknown' }] doctor_events(event_list, saves, mut report) doctor_files(saves[0].id, [snapshot.FileEntry{ path: 'private/file' }], event_list, mut report) assert report.flagged_saves == 1 assert report.problems.any(it.contains('unknown kind')) assert report.problems.any(it.contains('stripped path')) assert report.strip_residue_paths == ['private'] } struct DoctorMemory { mut: items map[string][]u8 } fn (mut backend DoctorMemory) get(key string) !remote.Object { if body := backend.items[key] { return remote.Object{ found: true, body: body } } return remote.Object{} } fn (mut backend DoctorMemory) list(prefix string) ![]string { return backend.items.keys().filter(it.starts_with('${prefix}/')) } fn (mut backend DoctorMemory) put_if_absent(key string, body []u8) !bool { if key in backend.items { return false } backend.items[key] = body return true } fn (mut backend DoctorMemory) put_if_match(key string, body []u8, tag string) !bool { return false } fn (mut backend DoctorMemory) delete(key string) ! { backend.items.delete(key) } fn doctor_store(name string) !(storage.Store, string) { dir := os.join_path(os.temp_dir(), 'koh3_doctor_${name}_${os.getpid()}') os.rmdir_all(dir) or {} os.mkdir_all(dir)! return storage.create(dir)!, dir } fn test_doctor_fetches_missing_parents_from_the_remote_without_writing_to_it() { store, dir := doctor_store('fetch')! defer { store.close() os.rmdir_all(dir) or {} } data := 'parent'.bytes() hash := storage.hash_hex(data) mut backend := DoctorReadOnly{ items: { 'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{ id: 'aaaaaaaaaaaa' files: [exchange.FileRecord{ path: 'file', hash: hash }] }).bytes() 'repo/objects/${hash}': zlib.compress(data)! } } store.db.exec('PRAGMA foreign_keys=OFF')! store.begin()! history.import_save(store, history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' }, []snapshot.FileEntry{})! history.set_head(store, 'bbbbbbbbbbbb')! store.end()! store.db.exec('PRAGMA foreign_keys=ON')! missing := missing_saves(history.all_saves(store)!, 'bbbbbbbbbbbb') assert missing == ['aaaaaaaaaaaa'] assert exchange.fetch_missing(store, mut backend, 'repo', missing)! == 1 assert missing_saves(history.all_saves(store)!, 'bbbbbbbbbbbb').len == 0 assert store.get_object(hash)! == data assert exchange.fetch_missing(store, mut backend, 'repo', ['cccccccccccc']) or { assert err.msg().contains('does not have save') 0 } == 0 } fn test_doctor_reapplies_recorded_strips_and_finds_the_only_tip() { store, dir := doctor_store('strip')! defer { store.close() os.rmdir_all(dir) or {} } secret := store.put_object('secret'.bytes())! kept := store.put_object('kept'.bytes())! store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [ snapshot.FileEntry{ path: 'keep', hash: kept }, snapshot.FileEntry{ path: 'private/key', hash: secret }, ])! history.import_save(store, history.Save{ id: 'bbbbbbbbbbbb', parent: 'aaaaaaaaaaaa' }, [snapshot.FileEntry{ path: 'keep', hash: kept }])! events.strip(store, '', 'private')! store.end()! assert only_tip(store) or { '' } == 'bbbbbbbbbbbb' mut report := DoctorJson{} doctor_files('aaaaaaaaaaaa', history.files_of(store, 'aaaaaaaaaaaa')!, events.all(store)!, mut report) assert report.strip_residue assert exchange.reapply_strips(store)! == 1 assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['keep'] assert !store.has_object(secret)! assert events.strip_events(store)!.len == 1 } fn test_doctor_restores_objects_from_working_files_and_resends_them_to_the_remote() { store, dir := doctor_store('objects')! defer { store.close() os.rmdir_all(dir) or {} } data := 'content'.bytes() hash := store.put_object(data)! store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [ snapshot.FileEntry{ path: 'app', hash: hash }, ])! history.set_head(store, 'aaaaaaaaaaaa')! store.end()! store.db.exec("UPDATE objects SET data = x'00'")! problems := store.check_objects()! assert problems.map(it.hash) == [hash] assert local_copy(store, hash) == none os.write_file(os.join_path(dir, 'app'), 'content')! store.repair_object(hash, local_copy(store, hash) or { panic('no working copy') })! assert store.check_objects()!.len == 0 store.repair_object(hash, 'other'.bytes()) or { assert err.msg().contains('does not match') } mut backend := DoctorMemory{ items: { 'repo/objects/${hash}': 'damaged'.bytes() } } assert exchange.resend(store, mut backend, 'repo', [hash], ['aaaaaaaaaaaa'])! == 2 assert zlib.decompress(backend.items['repo/objects/${hash}'])! == data assert exchange.decode_save(backend.items['repo/saves/aaaaaaaaaaaa'])!.files[0].hash == hash } // An invalid strip event authorizes nothing, and a strip frees only the objects // it removed: an unrelated unused object may be needed for recovery. fn test_doctor_strip_repair_ignores_invalid_events_and_keeps_unrelated_objects() { store, dir := doctor_store('strip_safety')! defer { store.close() os.rmdir_all(dir) or {} } app := store.put_object('app'.bytes())! secret := store.put_object('secret'.bytes())! orphan := store.put_object('orphan'.bytes())! store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [ snapshot.FileEntry{ path: 'app', hash: app }, snapshot.FileEntry{ path: 'secret', hash: secret }, ])! events.receive(store, events.Event{ id: 'invalidstrip' kind: events.kind_strip path: 'app' created_at: -1 machine: 'test' })! events.strip(store, '', 'secret')! store.end()! mut report := DoctorJson{} doctor_files('aaaaaaaaaaaa', [snapshot.FileEntry{ path: 'app' }], events.all(store)!, mut report) assert !report.strip_residue assert exchange.reapply_strips(store)! == 1 assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['app'] assert store.has_object(app)! assert store.has_object(orphan)! assert !store.has_object(secret)! } // With a remote, only a full audit that found no other head there lets Doctor // guess, and the guess is checked again when it is written. fn test_doctor_guesses_head_only_when_nothing_knows_better() { store, dir := doctor_store('only_tip')! defer { store.close() os.rmdir_all(dir) or {} } store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, []snapshot.FileEntry{})! store.end()! mut report := DoctorJson{ metadata_available: true saves: 1 source: 'folder/repo' remote_head: 'bbbbbbbbbbbb' } assert !plan_repairs(store, report).any(it.action.contains('point head')) report.remote_head = '' report.checks << DoctorCheck{'remote audit', 'skipped', '--fast requested', ''} assert !plan_repairs(store, report).any(it.action.contains('point head')) report.checks = [DoctorCheck{'remote stability', 'fail', 'remote head changed during audit', ''}] assert !plan_repairs(store, report).any(it.action.contains('point head')) report.checks = [] assert plan_repairs(store, report).any(it.action.contains('point head')) report.source = '' repairs := plan_repairs(store, report).filter(it.action.contains('point head')) assert repairs.len == 1 history.set_head(store, 'cccccccccccc')! if _ := repairs[0].run() { assert false, 'a head another command set must not be replaced' } else { assert err.msg().contains('changed'), err.msg() } assert store.state(storage.head_key) or { '' } == 'cccccccccccc' store.remove_state(storage.head_key)! repairs[0].run()! assert store.state(storage.head_key) or { '' } == 'aaaaaaaaaaaa' } fn test_doctor_reports_objects_it_restored_before_one_it_could_not() { store, dir := doctor_store('partial')! defer { store.close() os.rmdir_all(dir) or {} } kept := store.put_object('kept'.bytes())! lost := store.put_object('lost'.bytes())! store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [ snapshot.FileEntry{ path: 'kept', hash: kept }, snapshot.FileEntry{ path: 'lost', hash: lost }, ])! store.end()! store.db.exec("UPDATE objects SET data = x'00'")! os.write_file(os.join_path(dir, 'kept'), 'kept')! if _ := repair_objects(store, [kept, lost], '', '') { assert false, 'an object with no verified copy must fail its repair' } else { assert err is exchange.Partway assert (err as exchange.Partway).done == 'restored 1 damaged or missing object' assert err.msg().contains(lost) } assert store.get_object(kept)! == 'kept'.bytes() } fn test_doctor_refuses_a_remote_save_with_invalid_metadata() { store, dir := doctor_store('invalid_fetch')! defer { store.close() os.rmdir_all(dir) or {} } mut backend := DoctorReadOnly{ items: { 'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{ id: 'aaaaaaaaaaaa' created_at: -1 }).bytes() } } if _ := exchange.fetch_missing(store, mut backend, 'repo', ['aaaaaaaaaaaa']) { assert false, 'a record with invalid metadata must not be imported' } else { assert err.msg().contains('invalid metadata') } assert history.all_saves(store)!.len == 0 } // An empty file's Blake3 is the hash of empty bytes, so bytes that fail to // decompress must not pass for it. fn test_doctor_replaces_a_damaged_remote_copy_of_an_empty_file() { store, dir := doctor_store('empty_object')! defer { store.close() os.rmdir_all(dir) or {} } hash := store.put_object([]u8{})! mut backend := DoctorMemory{ items: { 'repo/objects/${hash}': 'not zlib'.bytes() } } assert exchange.resend(store, mut backend, 'repo', [hash], [])! == 1 assert zlib.decompress(backend.items['repo/objects/${hash}'])!.len == 0 } // The remote is asked again before head is guessed: another machine may have // offered since the audit. fn test_doctor_rechecks_the_remote_head_before_guessing() { store, dir := doctor_store('remote_tip')! remote_dir := os.join_path(dir, 'remote') defer { store.close() machine.forget_remote() or {} os.rmdir_all(dir) or {} } store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, []snapshot.FileEntry{})! store.end()! mut folder := remote.new_folder(remote_dir)! machine.save_remote(machine.RemoteSettings{ kind: machine.kind_folder, path: remote_dir })! report := DoctorJson{ metadata_available: true saves: 1 remote_repo: 'repo' } repairs := plan_repairs(store, report).filter(it.action.contains('point head')) assert repairs.len == 1 folder.put_if_absent(exchange.head_key('repo'), 'bbbbbbbbbbbb'.bytes())! if _ := repairs[0].run() { assert false, 'a head offered since the audit must stop the guess' } else { assert err.msg().contains('another head'), err.msg() } assert store.state(storage.head_key) == none } // An error after some objects were restored still reports them. fn test_doctor_reports_restored_objects_when_the_remote_then_fails() { store, dir := doctor_store('partial_remote')! defer { store.close() machine.forget_remote() or {} os.rmdir_all(dir) or {} } kept := store.put_object('kept'.bytes())! lost := store.put_object('lost'.bytes())! store.begin()! history.import_save(store, history.Save{ id: 'aaaaaaaaaaaa' }, [ snapshot.FileEntry{ path: 'kept', hash: kept }, snapshot.FileEntry{ path: 'lost', hash: lost }, ])! store.end()! store.db.exec("UPDATE objects SET data = x'00'")! os.write_file(os.join_path(dir, 'kept'), 'kept')! machine.save_remote(machine.RemoteSettings{ kind: machine.kind_folder path: os.join_path(dir, 'unmounted') })! if _ := repair_objects(store, [kept, lost], '', '') { assert false, 'an unreachable remote must fail the repair' } else { assert err is exchange.Partway assert (err as exchange.Partway).done == 'restored 1 damaged or missing object' assert err.msg().contains('not there'), err.msg() } } // A strip event Doctor cannot trust removes nothing from a fetched save either. fn test_doctor_fetch_ignores_invalid_strip_events() { store, dir := doctor_store('fetch_invalid_strip')! defer { store.close() os.rmdir_all(dir) or {} } data := 'app'.bytes() hash := storage.hash_hex(data) mut backend := DoctorReadOnly{ items: { 'repo/saves/aaaaaaaaaaaa': json2.encode(exchange.SaveRecord{ id: 'aaaaaaaaaaaa' files: [exchange.FileRecord{ path: 'app', hash: hash }] }).bytes() 'repo/objects/${hash}': zlib.compress(data)! } } events.receive(store, events.Event{ id: 'invalidstrip' kind: events.kind_strip path: 'app' created_at: -1 machine: 'test' })! assert exchange.fetch_missing(store, mut backend, 'repo', ['aaaaaaaaaaaa'])! == 1 assert history.files_of(store, 'aaaaaaaaaaaa')!.map(it.path) == ['app'] } // A strip whose remote half was lost is sent again from the recorded event // when the remote still holds the path; a later path that fails still reports // the strips already sent. fn test_doctor_sends_a_recorded_strip_again_for_remote_residue() { store, dir := doctor_store('remote_residue')! remote_dir := os.join_path(dir, 'remote') defer { store.close() machine.forget_remote() or {} os.rmdir_all(dir) or {} } secret := 'secret'.bytes() hash := storage.hash_hex(secret) mut folder := remote.new_folder(remote_dir)! folder.put_if_absent(exchange.save_key('repo', 'aaaaaaaaaaaa'), json2.encode(exchange.SaveRecord{ id: 'aaaaaaaaaaaa' files: [exchange.FileRecord{ path: 'app', hash: hash }] }).bytes())! folder.put_if_absent(exchange.object_key('repo', hash), zlib.compress(secret)!)! machine.save_remote(machine.RemoteSettings{ kind: machine.kind_folder, path: remote_dir })! store.set_state('remote_repo', 'repo')! events.strip(store, '', 'app')! store.set_state('strip_pending:bad', 'not JSON')! report := DoctorJson{ remote_repo: 'repo' remote_strip_residue: ['app', 'bad'] } repairs := plan_repairs(store, report).filter(it.action.contains('remote still holds')) assert repairs.len == 1 if _ := repairs[0].run() { assert false, 'an untrusted note must stop the second path' } else { assert err is exchange.Partway assert (err as exchange.Partway).done == 'sent 1 strip', (err as exchange.Partway).done } record := exchange.decode_save(folder.get(exchange.save_key('repo', 'aaaaaaaaaaaa'))!.body)! assert record.files.len == 0 assert !folder.get(exchange.object_key('repo', hash))!.found assert !exchange.strip_pending(store, 'app') }