migrate_test.v

1//! Migration tests, run against a fixture made by the Koh 2 binary itself.2//!3//! The fixture is why these are worth having: it is not a guess at Koh 2's4//! format, it is the format, including the inlined small files and the object5//! store. When `koh` (Koh 2) is not on the machine these tests skip rather6//! than pretend.78module migrate910import json211import os12import src.events13import src.history14import src.snapshot15import src.storage1617struct Fixture {18	dir     string19	first   string20	second  string21	message string22}2324// koh2_binary finds Koh 2: `koh2` since the cutover, `koh` before it, and never Koh 3.25fn quiet(stage string, done int, total int) {}2627fn koh2_binary() ?string {28	for name in ['koh2', 'koh'] {29		path := os.find_abs_path_of_executable(name) or { continue }30		if os.execute('${os.quoted_path(path)} --version').output.starts_with('koh v2') {31			return path32		}33	}34	return none35}3637fn sh(dir string, command string) !string {38	result := os.execute('cd ${dir} && ${command}')39	if result.exit_code != 0 {40		return error('${command} failed: ${result.output}')41	}42	return result.output43}4445fn make_fixture(koh2 string) !Fixture {46	dir := os.join_path(os.temp_dir(), 'koh3_migrate_${os.getpid()}')47	os.rmdir_all(dir) or {}48	os.mkdir_all(dir, os.MkdirParams{})!49	sh(dir, '${koh2} init')!50	// One file big enough for the object store, one small enough to be inlined.51	os.write_file(os.join_path(dir, 'big.txt'), 'x'.repeat(700))!52	os.write_file(os.join_path(dir, 'small.txt'), 'tiny\n')!53	sh(dir, '${koh2} save "first save"')!54	sh(dir, '${koh2} checkpoint "before something"')!55	os.write_file(os.join_path(dir, 'big.txt'), 'y'.repeat(700))!56	sh(dir, '${koh2} save "second save"')!5758	// Identify the first save by its message: koh2 timestamps are whole seconds,59	// so saves made in a row share one, and a filename is a hash. The head comes60	// from Koh 2's own state for the same reason.61	mut first := ''62	for name in os.ls(os.join_path(dir, '.koh', 'snapshots'))! {63		if !name.ends_with('.json') {64			continue65		}66		record :=67			json2.decode[Manifest](os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!68		if record.message == 'first save' {69			first = record.id70		}71	}72	assert first != ''73	return Fixture{74		dir:     dir75		first:   first76		second:  koh2_head(dir)77		message: 'first save'78	}79}8081// koh2_head reads the lane head Koh 2 recorded.82fn koh2_head(dir string) string {83	db := storage.connect(os.join_path(dir, '.koh', 'koh.db'), 1000) or { return '' }84	defer {85		db.close()86	}87	mut stmt := db.prepare('SELECT value FROM state WHERE key = ?') or { return '' }88	defer {89		stmt.finalize()90	}91	stmt.bind_text(1, 'head_main') or { return '' }92	if stmt.step() or { return '' } {93		return stmt.text(0)94	}95	return ''96}9798fn test_migrating_a_koh2_repo_keeps_every_save_exactly() {99	koh2 := koh2_binary() or { return }100	fixture := make_fixture(koh2)!101	defer {102		os.rmdir_all(fixture.dir) or {}103	}104	// Flag the first save, so a save-level flag has to become an event.105	sh(fixture.dir, '${koh2} flag ${fixture.first} --reason "sketchy"')!106107	// What the manifests say, read independently of the migration.108	mut manifests := map[string][]string{}109	for name in os.ls(os.join_path(fixture.dir, '.koh', 'snapshots'))! {110		if !name.ends_with('.json') {111			continue112		}113		record := json2.decode[Manifest](os.read_file(os.join_path(fixture.dir, '.koh', 'snapshots', name))!)!114		mut files := []string{}115		for file in record.files {116			files << '${file.path} ${bare_hash(file.hash)}'117		}118		manifests[record.id] = files119	}120121	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!122	// The fixture lives on main, because that is the lane koh2 starts on.123	assert report.lane == 'main'124	assert report.saves == 3125	assert report.checkpoints == 1126	assert report.flags == 1127	assert report.objects >= 3128129	// The swap happened: `.koh` is a file now and the old data is archived.130	assert os.is_file(os.join_path(fixture.dir, '.koh'))131	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))132	assert os.is_file(os.join_path(fixture.dir, '.kohignore'))133134	store := storage.open(fixture.dir)!135	defer {136		store.close()137	}138	// Objects are counted once each, not once per save that lists them.139	assert report.objects == int(store.object_count()!)140	// Every save kept its ID, its files and their hashes.141	assert history.save_count(store)! == 3142	for id, expected in manifests {143		save := history.by_id(store, id) or {144			assert false, 'save ${id} did not survive the migration'145			return146		}147		assert save.machine == 'koh2'148		mut actual := []string{}149		for file in history.files_of(store, id)! {150			actual << '${file.path} ${file.hash}'151			// Every object is present and hashes back to its key.152			if _ := store.get_object(file.hash) {153			} else {154				assert false, 'object ${file.hash} is missing or corrupt'155			}156		}157		assert actual == expected158	}159	// The checkpoint mark and the parent chain came across.160	middle := history.by_id(store, fixture.first) or { panic('the first save did not survive') }161	assert (history.head(store) or { panic('no head after migrating') }).id == fixture.second162	assert middle.message == 'first save'163164	// The flag became an event, and the working tree is exactly what Koh 3165	// expects once the carried ignore file is saved.166	assert events.flag_state(store, fixture.first)!.flagged167	assert events.flag_state(store, fixture.first)!.reason == 'sketchy'168	entries := snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))!169	mut changed := []string{}170	head_files := snapshot.index_files(history.files_of(store, fixture.second)!)171	for entry in entries {172		if entry.path !in head_files {173			changed << entry.path174		} else if head_files[entry.path].hash != entry.hash {175			changed << entry.path176		}177	}178	// The only difference is the ignore file Koh 3 just carried to the root.179	assert changed == ['.kohignore']180}181182// §13: the migration renames `kepr.build` to `koh.build`, and records the183// rename as a save, so the tree it leaves behind holds no unreviewed work.184fn test_migration_renames_the_koh_2_build_file() {185	koh2 := koh2_binary() or { return }186	fixture := make_fixture(koh2)!187	defer {188		os.rmdir_all(fixture.dir) or {}189	}190	build_file := 'command = true\noutput = out/demo\nplatforms = macos-aarch64\n'191	os.write_file(os.join_path(fixture.dir, 'kepr.build'), build_file)!192	sh(fixture.dir, '${koh2} save "add the build file"')!193194	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!195	assert report.build_renamed196	assert !os.exists(os.join_path(fixture.dir, 'kepr.build'))197	assert os.read_file(os.join_path(fixture.dir, 'koh.build'))! == build_file198199	store := storage.open(fixture.dir)!200	defer {201		store.close()202	}203	head := history.head(store) or { panic('no head after migrating') }204	assert head.message == 'migrate: rename kepr.build to koh.build'205	files := snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))!206	assert snapshot.same_tree(history.files_of(store, head.id)!, files)207}208209// A repo holding both build files is refused in the check phase, while the210// Koh 2 data is still where it was.211fn test_migration_refuses_two_build_files_without_moving_anything() {212	koh2 := koh2_binary() or { return }213	fixture := make_fixture(koh2)!214	defer {215		os.rmdir_all(fixture.dir) or {}216	}217	os.write_file(os.join_path(fixture.dir, 'kepr.build'), 'command = true\n')!218	os.write_file(os.join_path(fixture.dir, 'koh.build'), 'command = true\n')!219	sh(fixture.dir, '${koh2} save "both build files"')!220	if _ := prepare(fixture.dir, default_lane, []) {221		assert false, 'both build files must be refused'222	} else {223		assert err.msg().contains('both kepr.build and koh.build')224	}225	assert os.is_dir(os.join_path(fixture.dir, '.koh'))226	assert !os.exists(os.join_path(fixture.dir, '.koh-v2'))227}228229// A repo with no Koh 2 build file migrates exactly as it always did.230fn test_migration_without_a_build_file_is_unchanged() {231	koh2 := koh2_binary() or { return }232	fixture := make_fixture(koh2)!233	defer {234		os.rmdir_all(fixture.dir) or {}235	}236	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!237	assert !report.build_renamed238	assert !os.exists(os.join_path(fixture.dir, 'koh.build'))239	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))240	store := storage.open(fixture.dir)!241	defer {242		store.close()243	}244	head := history.head(store) or { panic('no head after migrating') }245	// The migration carries Koh 2's ignore rules to a root .kohignore, so the246	// tree differs from the imported head by exactly that file — which is what247	// the report tells the operator to save.248	saved := history.files_of(store, head.id)!249	mut unsaved := []string{}250	for file in snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))! {251		if !saved.any(it.path == file.path) {252			unsaved << file.path253		}254	}255	assert unsaved == [snapshot.ignore_file]256}257258fn test_migrate_refuses_a_tree_that_does_not_match_the_last_save() {259	koh2 := koh2_binary() or { return }260	fixture := make_fixture(koh2)!261	defer {262		os.rmdir_all(fixture.dir) or {}263	}264	os.write_file(os.join_path(fixture.dir, 'big.txt'), 'unsaved change\n')!265	if _ := prepare(fixture.dir, default_lane, []) {266		assert false, 'a dirty tree must be refused'267	} else {268		assert err.msg().contains('save with koh2 first')269	}270	// Nothing was touched: still a Koh 2 repo, no archive, no draft file.271	assert os.is_dir(os.join_path(fixture.dir, '.koh'))272	assert !os.exists(os.join_path(fixture.dir, '.koh-v2'))273	assert !os.exists(os.join_path(fixture.dir, draft_file))274}275276// A damaged manifest naming a path outside the repo must never be read there.277fn test_a_saved_path_is_only_read_inside_the_repo() {278	root := os.join_path(os.temp_dir(), 'koh3_migrate_paths_${os.getpid()}', 'repo')279	os.mkdir_all(root)!280	defer {281		os.rmdir_all(os.dir(root)) or {}282	}283	os.write_file(os.join_path(os.dir(root), 'outside.txt'), 'secret\n')!284	os.write_file(os.join_path(root, 'inside.txt'), 'fine\n')!285	assert disk_hash(root, 'inside.txt') != none286	for path in ['../outside.txt', '/etc/hosts', './inside.txt', 'a//inside.txt'] {287		assert disk_hash(root, path) == none, path288	}289}290291fn test_disk_hash_uses_the_shared_path_rule() {292	root := os.join_path(os.temp_dir(), 'koh3_migrate_rule_${os.getpid()}')293	os.mkdir_all(root)!294	defer {295		os.rmdir_all(root) or {}296	}297	for name in ['back\\slash', 'Icon\r'] {298		// Joined by hand: os.join_path turns a backslash into a separator.299		os.write_file('${root}/${name}', 'odd\n')!300		assert disk_hash(root, name) == none, name301	}302}303304fn test_unsafe_manifest_paths_are_dropped_and_named_once() {305	mut line := [306		Manifest{307			id:    'a'308			files: [ManifestFile{309				path: 'ok.txt'310			}, ManifestFile{311				path: 'Icon\r'312			}]313		},314		Manifest{315			id:    'b'316			files: [ManifestFile{317				path: 'Icon\r'318			}, ManifestFile{319				path: '../x'320			}]321		},322	]323	assert drop_unsafe_paths(mut line) == ['../x', 'Icon\\x0d']324	assert line[0].files.map(it.path) == ['ok.txt']325	assert line[1].files.len == 0326}327328// A version Koh 2 lost is named with the saves it was in; those saves come over329// without it, each flagged with the reason, rather than the repo staying stuck.330fn test_a_version_koh2_lost_is_named_and_its_saves_flagged() {331	koh2 := koh2_binary() or { return }332	fixture := make_fixture(koh2)!333	defer {334		os.rmdir_all(fixture.dir) or {}335	}336	hash := storage.hash_hex('x'.repeat(700).bytes())337	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!338	plan := prepare(fixture.dir, default_lane, [])!339	assert plan.lost.map(it.path) == ['big.txt']340	assert plan.lost[0].saves.len == 2341	run(plan, quiet)!342	store := storage.open(fixture.dir)!343	defer {344		store.close()345	}346	flags := events.latest_flags(store)!347	assert flags[fixture.first].reason.contains('Koh 2 had lost big.txt'), flags.str()348	assert 'big.txt' !in history.files_of(store, fixture.first)!.map(it.path)349}350351// Handed back with --with, found anywhere, the lost version comes home.352fn test_a_lost_version_handed_back_is_imported() {353	koh2 := koh2_binary() or { return }354	fixture := make_fixture(koh2)!355	defer {356		os.rmdir_all(fixture.dir) or {}357	}358	hash := storage.hash_hex('x'.repeat(700).bytes())359	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!360	backup := os.join_path(os.temp_dir(), 'koh3_backup_${os.getpid()}.txt')361	os.write_file(backup, 'x'.repeat(700))!362	defer {363		os.rm(backup) or {}364	}365	plan := prepare(fixture.dir, default_lane, [backup])!366	assert plan.lost.len == 0367	run(plan, quiet)!368	store := storage.open(fixture.dir)!369	defer {370		store.close()371	}372	assert 'big.txt' in history.files_of(store, fixture.first)!.map(it.path)373	assert store.get_object(hash)! == 'x'.repeat(700).bytes()374}375376// A migrate that fails clears its own draft, so the next attempt can start.377fn test_a_failed_migrate_leaves_no_draft() {378	koh2 := koh2_binary() or { return }379	fixture := make_fixture(koh2)!380	defer {381		os.rmdir_all(fixture.dir) or {}382	}383	hash := storage.hash_hex('y'.repeat(700).bytes())384	os.write_file(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]), 'damaged')!385	if _ := run(prepare(fixture.dir, default_lane, [])!, quiet) {386		assert false, 'a damaged object migrated'387	}388	assert !os.exists(os.join_path(fixture.dir, draft_file))389	assert os.is_dir(os.join_path(fixture.dir, '.koh'))390}391392// Koh 2 did not always write a manifest's files in path order (nina has one such save).393fn test_a_manifest_listing_its_files_out_of_order_migrates() {394	koh2 := koh2_binary() or { return }395	fixture := make_fixture(koh2)!396	defer {397		os.rmdir_all(fixture.dir) or {}398	}399	path := os.join_path(fixture.dir, '.koh', 'snapshots', '${fixture.first}.json')400	mut raw := json2.decode[json2.Any](os.read_file(path)!)!.as_map()401	mut files := raw['files']!.as_array()402	files.reverse_in_place()403	raw['files'] = json2.Any(files)404	os.write_file(path, json2.encode(raw))!405	run(prepare(fixture.dir, default_lane, [])!, quiet)!406	store := storage.open(fixture.dir)!407	defer {408		store.close()409	}410	assert history.files_of(store, fixture.first)!.map(it.path) == ['big.txt', 'small.txt']411}