migrate_test.v

1//! Migration tests, run against a fixture made by the Koh 2 binary itself.2//!3//! The fixture is why these are worth having: it is not a guess at Koh 2's4//! format, it is the format, including the inlined small files and the object5//! store. When `koh` (Koh 2) is not on the machine these tests skip rather6//! than pretend.78module migrate910import json211import os12import src.events13import src.history14import src.snapshot15import src.storage1617struct Fixture {18	dir     string19	first   string20	second  string21	message string22}2324// koh2_binary finds Koh 2: `koh2` since the cutover, `koh` before it, and never Koh 3.25// Global rules must not make source fixtures depend on the developer’s home.26fn testsuite_begin() {27	os.setenv('HOME', os.join_path(os.temp_dir(), 'koh3-test-home-${os.getpid()}'), true)28}2930fn quiet(stage string, done int, total int) {}3132fn koh2_binary() ?string {33	for name in ['koh2', 'koh'] {34		path := os.find_abs_path_of_executable(name) or { continue }35		if os.execute('${os.quoted_path(path)} --version').output.starts_with('koh v2') {36			return path37		}38	}39	return none40}4142fn sh(dir string, command string) !string {43	result := os.execute('cd ${dir} && ${command}')44	if result.exit_code != 0 {45		return error('${command} failed: ${result.output}')46	}47	return result.output48}4950fn make_fixture(koh2 string) !Fixture {51	dir := os.join_path(os.temp_dir(), 'koh3_migrate_${os.getpid()}')52	os.rmdir_all(dir) or {}53	os.mkdir_all(dir, os.MkdirParams{})!54	sh(dir, '${koh2} init')!55	// One file big enough for the object store, one small enough to be inlined.56	os.write_file(os.join_path(dir, 'big.txt'), 'x'.repeat(700))!57	os.write_file(os.join_path(dir, 'small.txt'), 'tiny\n')!58	sh(dir, '${koh2} save "first save"')!59	sh(dir, '${koh2} checkpoint "before something"')!60	os.write_file(os.join_path(dir, 'big.txt'), 'y'.repeat(700))!61	sh(dir, '${koh2} save "second save"')!6263	// Identify the first save by its message: koh2 timestamps are whole seconds,64	// so saves made in a row share one, and a filename is a hash. The head comes65	// from Koh 2's own state for the same reason.66	mut first := ''67	for name in os.ls(os.join_path(dir, '.koh', 'snapshots'))! {68		if !name.ends_with('.json') {69			continue70		}71		record :=72			json2.decode[Manifest](os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!73		if record.message == 'first save' {74			first = record.id75		}76	}77	assert first != ''78	return Fixture{79		dir:     dir80		first:   first81		second:  koh2_head(dir)82		message: 'first save'83	}84}8586// koh2_head reads the lane head Koh 2 recorded.87fn koh2_head(dir string) string {88	db := storage.connect(os.join_path(dir, '.koh', 'koh.db'), 1000) or { return '' }89	defer {90		db.close()91	}92	mut stmt := db.prepare('SELECT value FROM state WHERE key = ?') or { return '' }93	defer {94		stmt.finalize()95	}96	stmt.bind_text(1, 'head_main') or { return '' }97	if stmt.step() or { return '' } {98		return stmt.text(0)99	}100	return ''101}102103fn test_migrating_a_koh2_repo_keeps_every_save_exactly() {104	koh2 := koh2_binary() or { return }105	fixture := make_fixture(koh2)!106	defer {107		os.rmdir_all(fixture.dir) or {}108	}109	// Flag the first save, so a save-level flag has to become an event.110	sh(fixture.dir, '${koh2} flag ${fixture.first} --reason "sketchy"')!111112	// What the manifests say, read independently of the migration.113	mut manifests := map[string][]string{}114	for name in os.ls(os.join_path(fixture.dir, '.koh', 'snapshots'))! {115		if !name.ends_with('.json') {116			continue117		}118		record := json2.decode[Manifest](os.read_file(os.join_path(fixture.dir, '.koh', 'snapshots', name))!)!119		mut files := []string{}120		for file in record.files {121			files << '${file.path} ${bare_hash(file.hash)}'122		}123		manifests[record.id] = files124	}125126	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!127	// The fixture lives on main, because that is the lane koh2 starts on.128	assert report.lane == 'main'129	assert report.saves == 3130	assert report.checkpoints == 1131	assert report.flags == 1132	assert report.objects >= 3133134	// The swap happened: `.koh` is a file now and the old data is archived.135	assert os.is_file(os.join_path(fixture.dir, '.koh'))136	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))137	assert os.is_file(os.join_path(fixture.dir, '.kohignore'))138139	store := storage.open(fixture.dir)!140	defer {141		store.close()142	}143	// Objects are counted once each, not once per save that lists them.144	assert report.objects == int(store.object_count()!)145	// Every save kept its ID, its files and their hashes.146	assert history.save_count(store)! == 3147	for id, expected in manifests {148		save := history.by_id(store, id) or {149			assert false, 'save ${id} did not survive the migration'150			return151		}152		assert save.machine == 'koh2'153		mut actual := []string{}154		for file in history.files_of(store, id)! {155			actual << '${file.path} ${file.hash}'156			// Every object is present and hashes back to its key.157			if _ := store.get_object(file.hash) {158			} else {159				assert false, 'object ${file.hash} is missing or corrupt'160			}161		}162		assert actual == expected163	}164	// The checkpoint mark and the parent chain came across.165	middle := history.by_id(store, fixture.first) or { panic('the first save did not survive') }166	assert (history.head(store) or { panic('no head after migrating') }).id == fixture.second167	assert middle.message == 'first save'168169	// The flag became an event, and the working tree is exactly what Koh 3170	// expects once the carried ignore file is saved.171	assert events.flag_state(store, fixture.first)!.flagged172	assert events.flag_state(store, fixture.first)!.reason == 'sketchy'173	entries := snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))!174	mut changed := []string{}175	head_files := snapshot.index_files(history.files_of(store, fixture.second)!)176	for entry in entries {177		if entry.path !in head_files {178			changed << entry.path179		} else if head_files[entry.path].hash != entry.hash {180			changed << entry.path181		}182	}183	// The only difference is the ignore file Koh 3 just carried to the root.184	assert changed == ['.kohignore']185}186187// §13: the migration renames `kepr.build` to `koh.build`, and records the188// rename as a save, so the tree it leaves behind holds no unreviewed work.189fn test_migration_renames_the_koh_2_build_file() {190	koh2 := koh2_binary() or { return }191	fixture := make_fixture(koh2)!192	defer {193		os.rmdir_all(fixture.dir) or {}194	}195	build_file := 'command = true\noutput = out/demo\nplatforms = macos-aarch64\n'196	os.write_file(os.join_path(fixture.dir, 'kepr.build'), build_file)!197	sh(fixture.dir, '${koh2} save "add the build file"')!198199	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!200	assert report.build_renamed201	assert !os.exists(os.join_path(fixture.dir, 'kepr.build'))202	assert os.read_file(os.join_path(fixture.dir, 'koh.build'))! == build_file203204	store := storage.open(fixture.dir)!205	defer {206		store.close()207	}208	head := history.head(store) or { panic('no head after migrating') }209	assert head.message == 'migrate: rename kepr.build to koh.build'210	files := snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))!211	assert snapshot.same_tree(history.files_of(store, head.id)!, files)212}213214// A repo holding both build files is refused in the check phase, while the215// Koh 2 data is still where it was.216fn test_migration_refuses_two_build_files_without_moving_anything() {217	koh2 := koh2_binary() or { return }218	fixture := make_fixture(koh2)!219	defer {220		os.rmdir_all(fixture.dir) or {}221	}222	os.write_file(os.join_path(fixture.dir, 'kepr.build'), 'command = true\n')!223	os.write_file(os.join_path(fixture.dir, 'koh.build'), 'command = true\n')!224	sh(fixture.dir, '${koh2} save "both build files"')!225	if _ := prepare(fixture.dir, default_lane, []) {226		assert false, 'both build files must be refused'227	} else {228		assert err.msg().contains('both kepr.build and koh.build')229	}230	assert os.is_dir(os.join_path(fixture.dir, '.koh'))231	assert !os.exists(os.join_path(fixture.dir, '.koh-v2'))232}233234// A repo with no Koh 2 build file migrates exactly as it always did.235fn test_migration_without_a_build_file_is_unchanged() {236	koh2 := koh2_binary() or { return }237	fixture := make_fixture(koh2)!238	defer {239		os.rmdir_all(fixture.dir) or {}240	}241	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!242	assert !report.build_renamed243	assert !os.exists(os.join_path(fixture.dir, 'koh.build'))244	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))245	store := storage.open(fixture.dir)!246	defer {247		store.close()248	}249	head := history.head(store) or { panic('no head after migrating') }250	// The migration carries Koh 2's ignore rules to a root .kohignore, so the251	// tree differs from the imported head by exactly that file — which is what252	// the report tells the operator to save.253	saved := history.files_of(store, head.id)!254	mut unsaved := []string{}255	for file in snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))! {256		if !saved.any(it.path == file.path) {257			unsaved << file.path258		}259	}260	assert unsaved == [snapshot.ignore_file]261}262263fn test_migrate_refuses_a_tree_that_does_not_match_the_last_save() {264	koh2 := koh2_binary() or { return }265	fixture := make_fixture(koh2)!266	defer {267		os.rmdir_all(fixture.dir) or {}268	}269	os.write_file(os.join_path(fixture.dir, 'big.txt'), 'unsaved change\n')!270	if _ := prepare(fixture.dir, default_lane, []) {271		assert false, 'a dirty tree must be refused'272	} else {273		assert err.msg().contains('save with koh2 first')274	}275	// Nothing was touched: still a Koh 2 repo, no archive, no draft file.276	assert os.is_dir(os.join_path(fixture.dir, '.koh'))277	assert !os.exists(os.join_path(fixture.dir, '.koh-v2'))278	assert !os.exists(os.join_path(fixture.dir, draft_file))279}280281// A damaged manifest naming a path outside the repo must never be read there.282fn test_a_saved_path_is_only_read_inside_the_repo() {283	root := os.join_path(os.temp_dir(), 'koh3_migrate_paths_${os.getpid()}', 'repo')284	os.mkdir_all(root)!285	defer {286		os.rmdir_all(os.dir(root)) or {}287	}288	os.write_file(os.join_path(os.dir(root), 'outside.txt'), 'secret\n')!289	os.write_file(os.join_path(root, 'inside.txt'), 'fine\n')!290	assert disk_hash(root, 'inside.txt') != none291	for path in ['../outside.txt', '/etc/hosts', './inside.txt', 'a//inside.txt'] {292		assert disk_hash(root, path) == none, path293	}294}295296fn test_disk_hash_uses_the_shared_path_rule() {297	root := os.join_path(os.temp_dir(), 'koh3_migrate_rule_${os.getpid()}')298	os.mkdir_all(root)!299	defer {300		os.rmdir_all(root) or {}301	}302	for name in ['back\\slash', 'Icon\r'] {303		// Joined by hand: os.join_path turns a backslash into a separator.304		os.write_file('${root}/${name}', 'odd\n')!305		assert disk_hash(root, name) == none, name306	}307}308309fn test_unsafe_manifest_paths_are_dropped_and_named_once() {310	mut line := [311		Manifest{312			id:    'a'313			files: [ManifestFile{314				path: 'ok.txt'315			}, ManifestFile{316				path: 'Icon\r'317			}]318		},319		Manifest{320			id:    'b'321			files: [ManifestFile{322				path: 'Icon\r'323			}, ManifestFile{324				path: '../x'325			}]326		},327	]328	assert drop_unsafe_paths(mut line) == ['../x', 'Icon\\x0d']329	assert line[0].files.map(it.path) == ['ok.txt']330	assert line[1].files.len == 0331}332333// A version Koh 2 lost is named with the saves it was in; those saves come over334// without it, each flagged with the reason, rather than the repo staying stuck.335fn test_a_version_koh2_lost_is_named_and_its_saves_flagged() {336	koh2 := koh2_binary() or { return }337	fixture := make_fixture(koh2)!338	defer {339		os.rmdir_all(fixture.dir) or {}340	}341	hash := storage.hash_hex('x'.repeat(700).bytes())342	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!343	plan := prepare(fixture.dir, default_lane, [])!344	assert plan.lost.map(it.path) == ['big.txt']345	assert plan.lost[0].saves.len == 2346	run(plan, quiet)!347	store := storage.open(fixture.dir)!348	defer {349		store.close()350	}351	flags := events.latest_flags(store)!352	assert flags[fixture.first].reason.contains('Koh 2 had lost big.txt'), flags.str()353	assert 'big.txt' !in history.files_of(store, fixture.first)!.map(it.path)354}355356// Handed back with --with, found anywhere, the lost version comes home.357fn test_a_lost_version_handed_back_is_imported() {358	koh2 := koh2_binary() or { return }359	fixture := make_fixture(koh2)!360	defer {361		os.rmdir_all(fixture.dir) or {}362	}363	hash := storage.hash_hex('x'.repeat(700).bytes())364	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!365	backup := os.join_path(os.temp_dir(), 'koh3_backup_${os.getpid()}.txt')366	os.write_file(backup, 'x'.repeat(700))!367	defer {368		os.rm(backup) or {}369	}370	plan := prepare(fixture.dir, default_lane, [backup])!371	assert plan.lost.len == 0372	run(plan, quiet)!373	store := storage.open(fixture.dir)!374	defer {375		store.close()376	}377	assert 'big.txt' in history.files_of(store, fixture.first)!.map(it.path)378	assert store.get_object(hash)! == 'x'.repeat(700).bytes()379}380381// A migrate that fails clears its own draft, so the next attempt can start.382fn test_a_failed_migrate_leaves_no_draft() {383	koh2 := koh2_binary() or { return }384	fixture := make_fixture(koh2)!385	defer {386		os.rmdir_all(fixture.dir) or {}387	}388	hash := storage.hash_hex('y'.repeat(700).bytes())389	os.write_file(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]), 'damaged')!390	if _ := run(prepare(fixture.dir, default_lane, [])!, quiet) {391		assert false, 'a damaged object migrated'392	}393	assert !os.exists(os.join_path(fixture.dir, draft_file))394	assert os.is_dir(os.join_path(fixture.dir, '.koh'))395}396397// Koh 2 did not always write a manifest's files in path order (nina has one such save).398fn test_a_manifest_listing_its_files_out_of_order_migrates() {399	koh2 := koh2_binary() or { return }400	fixture := make_fixture(koh2)!401	defer {402		os.rmdir_all(fixture.dir) or {}403	}404	path := os.join_path(fixture.dir, '.koh', 'snapshots', '${fixture.first}.json')405	mut raw := json2.decode[json2.Any](os.read_file(path)!)!.as_map()406	mut files := raw['files']!.as_array()407	files.reverse_in_place()408	raw['files'] = json2.Any(files)409	os.write_file(path, json2.encode(raw))!410	run(prepare(fixture.dir, default_lane, [])!, quiet)!411	store := storage.open(fixture.dir)!412	defer {413		store.close()414	}415	assert history.files_of(store, fixture.first)!.map(it.path) == ['big.txt', 'small.txt']416}