add phase 5 migrate with the koh2 hand-over and migration tests

@@ -0,0 +1,64 @@1+//! `koh migrate` — one-time conversion from Koh 2.2+3+module cli4+5+import json6+import os7+import src.fmt8+import src.machine9+import src.migrate10+import src.snapshot11+12+pub fn cmd_migrate(raw []string) int {13+	args := parse('migrate', raw, Options{14+		values: ['lane']15+		flags:  ['json']16+	}) or { return usage_error(err.msg()) }17+	mut lane := args.value('lane')18+	if lane == '' {19+		lane = migrate.default_lane20+	}21+	if lane !in ['dev', 'main'] {22+		return usage_error('--lane takes dev or main, not "${lane}"')23+	}24+	root := os.getwd()25+	report := migrate.run(root, lane) or { return refuse(args.json(), err.msg()) }26+	if args.json() {27+		emit_json(json.encode(MigrateJson{28+			repo:         report.root29+			lane:         report.lane30+			saves:        report.saves31+			checkpoints:  report.checkpoints32+			flags:        report.flags33+			objects:      report.objects34+			head:         report.head35+			ignore_moved: report.ignore_carried36+		}))37+		return 038+	}39+	ok('migrated ${fmt.plural(report.saves, 'save')} from Koh 2 (lane ${report.lane})')40+	note('${report.checkpoints} checkpoints · ${report.flags} flags · ${report.objects} objects')41+	note('the Koh 2 data is untouched in ${migrate.archive_dir}/ — move it back to go back')42+	if report.ignore_carried {43+		note('carried the ignore rules to ${snapshot.ignore_file}')44+	}45+	if report.log_md_was_tracked {46+		warn('LOG.md was tracked by Koh 2; koh writes it itself now, so it is no longer snapshotted')47+	}48+	note('run "koh status" to see what the migration left, then "koh save"')49+	if machine.has_remote() {50+		note('this machine has a remote — run "koh offer" to publish the migrated history')51+	}52+	return 053+}54+55+struct MigrateJson {56+	repo         string57+	lane         string58+	saves        int59+	checkpoints  int60+	flags        int61+	objects      int62+	head         string63+	ignore_moved bool64+}

@@ -0,0 +1,75 @@1+//! The cut-over hand-off.2+//!3+//! While repos with an unmigrated Koh 2 `.koh/` directory still exist on this4+//! machine, `koh` is Koh 3 but those repos belong to Koh 2. Rather than let a5+//! command fail confusingly, Koh 3 hands it to `koh2` unchanged — same6+//! arguments, same terminal. Agents always type `koh`, and every repo gets the7+//! Koh that owns it.8+//!9+//! `migrate` is the exception: it is how such a repo stops being a Koh 2 repo.10+11+module cli12+13+import os14+import src.fmt15+16+// hand_over reports whether this invocation belongs to Koh 2, and if so runs17+// it. It never returns when the hand-off happens.18+pub fn hand_over(argv []string) {19+	if argv.len < 2 {20+		return21+	}22+	command := argv[1]23+	if command in ['migrate', 'man', 'help', '--help', '-h', '--version', '-v'] {24+		return25+	}26+	if !inside_koh2_repo(os.getwd()) {27+		return28+	}29+	target := koh2_binary()30+	if target == '' {31+		eprintln('  ${fmt.red(fmt.glyph_fail)} this folder holds a Koh 2 repo, and koh2 is not on PATH')32+		eprintln('  install koh2, or run "koh migrate" to convert this repo')33+		exit(1)34+	}35+	os.execvp(target, argv) or {36+		eprintln('  ${fmt.red(fmt.glyph_fail)} could not hand this command to ${target}: ${err.msg()}')37+		exit(1)38+	}39+}40+41+// koh2_binary finds the Koh 2 binary. Before the cutover it is still called42+// `koh`, so that name is a fallback — never this binary itself, which would43+// hand the command straight back to us for ever.44+fn koh2_binary() string {45+	if found := os.find_abs_path_of_executable('koh2') {46+		return found47+	}48+	if legacy := os.find_abs_path_of_executable('koh') {49+		if os.real_path(legacy) != os.real_path(os.executable()) {50+			return legacy51+		}52+	}53+	return ''54+}55+56+// inside_koh2_repo looks upward for a `.koh` that is a directory. A `.koh`57+// file is Koh 3's.58+fn inside_koh2_repo(start string) bool {59+	mut dir := os.real_path(start)60+	for {61+		path := os.join_path(dir, '.koh')62+		if os.is_dir(path) {63+			return true64+		}65+		if os.is_file(path) {66+			return false67+		}68+		parent := os.dir(dir)69+		if parent == dir || parent == '' {70+			return false71+		}72+		dir = parent73+	}74+	return false75+}

@@ -0,0 +1,394 @@1+//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2+//!3+//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4+//! same objects in one SQLite file. The conversion reads the manifests as the5+//! source of truth, imports everything into a fresh `.koh`, verifies it against6+//! those manifests, and only then swaps it into place. The Koh 2 data is not7+//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8+//! undoes the migration entirely.9+10+module migrate11+12+import encoding.base6413+import json14+import os15+import src.events16+import src.history17+import src.snapshot18+import src.storage19+20+pub const koh2_dir = '.koh'21+pub const archive_dir = '.koh-v2'22+pub const staging_file = '.koh-migrate'23+pub const default_lane = 'dev'24+25+// One file in a Koh 2 manifest. Small files were inlined as base64 instead of26+// being written to the object store, and those entries carry `inline` with no27+// `object` key at all — so `inline` decides, and `object` is only the28+// alternative.29+struct ManifestFile {30+	path   string31+	hash   string32+	size   i6433+	object bool34+	inline ?string35+}36+37+struct Manifest {38+	id            string39+	lane          string40+	message       string41+	timestamp     i6442+	parent_id     ?string43+	size_bytes    i6444+	agent         ?string45+	session       ?string46+	is_checkpoint bool47+	flagged       bool48+	flag_reason   ?string49+	files         []ManifestFile50+}51+52+pub struct Report {53+pub mut:54+	root               string55+	lane               string56+	saves              int57+	checkpoints        int58+	flags              int59+	objects            int60+	inlined            int61+	head               string62+	ignore_carried     bool63+	log_md_was_tracked bool64+}65+66+// run converts the repo at `root`. `lane` picks which line to import: `dev`67+// unless told otherwise, because that is where the work is.68+pub fn run(root string, lane string) !Report {69+	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {70+		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')71+	}72+	manifests := read_manifests(root)!73+	if manifests.len == 0 {74+		return error('the Koh 2 repo has no manifests to import')75+	}76+	// `dev` holds everything when it was used at all. A repo that never left77+	// `main` — which is what koh2 gave a new repo — imports from main instead78+	// of failing on an empty lane.79+	mut chosen := lane80+	head := lane_head(root, manifests, chosen) or {81+		if chosen != default_lane {82+			return err83+		}84+		chosen = other_lane(manifests) or { return err }85+		lane_head(root, manifests, chosen)!86+	}87+	line := ancestry(manifests, head)!88+	check_clean_tree(root, head)!89+90+	mut report := Report{91+		root: root92+		lane: chosen93+		head: head.id94+	}95+	// Import into a staging file first: nothing is touched until it verifies.96+	staging := os.join_path(root, staging_file)97+	if os.exists(staging) {98+		return error('${staging} already exists — a previous migrate did not finish; move it away and try again')99+	}100+	mut staged := storage.create_at(staging, root)!101+	// The staged file must be closed before it is moved: SQLite keeps committed102+	// work in its WAL until the last connection goes away, and moving the file103+	// alone would leave that work behind.104+	mut failure := ''105+	if _ := import_and_verify(staged, root, line, mut report) {106+	} else {107+		failure = err.msg()108+	}109+	staged.close()110+	if failure != '' {111+		return error(failure)112+	}113+	swap(root, mut report)!114+	report.saves = line.len115+	for manifest in line {116+		if manifest.is_checkpoint {117+			report.checkpoints++118+		}119+		if manifest.flagged {120+			report.flags++121+		}122+		for file in manifest.files {123+			if file.inline != none {124+				report.inlined++125+			}126+		}127+	}128+	return report129+}130+131+fn read_manifests(root string) ![]Manifest {132+	dir := os.join_path(root, koh2_dir, 'snapshots')133+	mut manifests := []Manifest{}134+	for name in os.ls(dir)! {135+		if !name.ends_with('.json') {136+			continue137+		}138+		body := os.read_file(os.join_path(dir, name))!139+		manifests << json.decode(Manifest, body)!140+	}141+	return manifests142+}143+144+// lane_head finds the newest save on a lane: the state table when it is there,145+// otherwise the manifest nothing else claims as a parent.146+fn lane_head(root string, manifests []Manifest, lane string) !Manifest {147+	if recorded := state_head(root, lane) {148+		for manifest in manifests {149+			if manifest.id == recorded {150+				return manifest151+			}152+		}153+	}154+	mut claimed := map[string]bool{}155+	for manifest in manifests {156+		if parent := manifest.parent_id {157+			claimed[parent] = true158+		}159+	}160+	mut candidates := []Manifest{}161+	for manifest in manifests {162+		if manifest.lane == lane && manifest.id !in claimed {163+			candidates << manifest164+		}165+	}166+	if candidates.len == 0 {167+		return error('no saves on lane "${lane}"')168+	}169+	candidates.sort(a.timestamp < b.timestamp)170+	return candidates[candidates.len - 1]171+}172+173+// other_lane names the lane that does have saves, when the preferred one is174+// empty.175+fn other_lane(manifests []Manifest) ?string {176+	mut lanes := map[string]bool{}177+	for manifest in manifests {178+		lanes[manifest.lane] = true179+	}180+	for lane in ['main', 'dev'] {181+		if lane in lanes {182+			return lane183+		}184+	}185+	for lane in lanes.keys() {186+		return lane187+	}188+	return none189+}190+191+fn state_head(root string, lane string) ?string {192+	db := os.join_path(root, koh2_dir, 'koh.db')193+	if !os.is_file(db) {194+		return none195+	}196+	store := storage.connect(db, 1000) or { return none }197+	defer {198+		store.close()199+	}200+	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }201+	defer {202+		stmt.finalize()203+	}204+	stmt.bind_text(1, 'head_${lane}') or { return none }205+	if stmt.step() or { return none } {206+		value := stmt.text(0)207+		if value != '' {208+			return value209+		}210+	}211+	return none212+}213+214+// ancestry walks back from the lane head to the first save, oldest first, and215+// refuses a manifest whose parent is missing: a gap would silently lose work.216+fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {217+	mut by_id := map[string]Manifest{}218+	for manifest in manifests {219+		by_id[manifest.id] = manifest220+	}221+	mut line := []Manifest{}222+	mut cursor := head223+	for {224+		line << cursor225+		parent := cursor.parent_id or { break }226+		if parent == '' {227+			break228+		}229+		next := by_id[parent] or {230+			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')231+		}232+		cursor = next233+	}234+	line.reverse_in_place()235+	return line236+}237+238+// check_clean_tree refuses unless the working tree matches the lane head, which239+// is the same rule the document states: save with koh2 first.240+fn check_clean_tree(root string, head Manifest) ! {241+	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {242+		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }243+	})244+	entries := snapshot.walk_raw(root, ignor)!245+	mut expected := map[string]string{}246+	for file in head.files {247+		expected[file.path] = bare_hash(file.hash)248+	}249+	mut actual := map[string]string{}250+	for entry in entries {251+		actual[entry.path] = entry.hash252+	}253+	for path, hash in expected {254+		if path !in actual {255+			return error('${path} is missing from the working tree — save with koh2 first')256+		}257+		if actual[path] != hash {258+			return error('${path} does not match the last save — save with koh2 first')259+		}260+	}261+	for path in actual.keys() {262+		if path !in expected {263+			return error('${path} is not in the last save — save with koh2 first')264+		}265+	}266+}267+268+fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report) ! {269+	store.begin()!270+	if _ := import_line(store, root, line, mut report) {271+		store.commit()!272+	} else {273+		store.rollback()274+		return err275+	}276+	verify(store, line)!277+}278+279+fn import_line(store &storage.Store, root string, line []Manifest, mut report Report) !bool {280+	for manifest in line {281+		mut files := []snapshot.FileEntry{}282+		for file in manifest.files {283+			hash := bare_hash(file.hash)284+			if encoded := file.inline {285+				data := base64_decode(encoded)!286+				stored := store.put_object(data)!287+				if stored != hash {288+					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')289+				}290+				report.objects++291+			} else if file.object {292+				blob := read_object(root, hash)!293+				store.put_object_blob(hash, blob)!294+				report.objects++295+			} else {296+				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')297+			}298+			files << snapshot.FileEntry{299+				path: file.path300+				hash: hash301+			}302+		}303+		save := history.Save{304+			id:            manifest.id305+			parent:        manifest.parent_id or { '' }306+			message:       manifest.message307+			created_at:    manifest.timestamp308+			machine:       'koh2'309+			is_checkpoint: manifest.is_checkpoint310+			size_bytes:    manifest.size_bytes311+			agent:         manifest.agent or { '' }312+			session:       manifest.session or { '' }313+		}314+		history.import_save(store, save, files)!315+		if manifest.flagged {316+			events.flag(store, manifest.id, manifest.flag_reason or { '' })!317+		}318+		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')319+	}320+	// The newest save on the imported line becomes head.321+	history.set_head(store, line[line.len - 1].id)!322+	return true323+}324+325+fn has_path(files []snapshot.FileEntry, path string) bool {326+	for file in files {327+		if file.path == path {328+			return true329+		}330+	}331+	return false332+}333+334+fn read_object(root string, hash string) ![]u8 {335+	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])336+	return os.read_bytes(path) or {337+		return error('object ${hash} is missing from ${koh2_dir}/objects')338+	}339+}340+341+fn verify(store &storage.Store, line []Manifest) ! {342+	imported := history.save_count(store)!343+	if imported != line.len {344+		return error('imported ${imported} saves, expected ${line.len}')345+	}346+	for manifest in line {347+		files := history.files_of(store, manifest.id)!348+		if files.len != manifest.files.len {349+			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')350+		}351+		for i, file in manifest.files {352+			hash := bare_hash(file.hash)353+			if files[i].path != file.path || files[i].hash != hash {354+				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')355+			}356+			store.get_object(hash) or {357+				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')358+			}359+		}360+	}361+}362+363+// swap puts the imported database in place and moves the Koh 2 data aside.364+fn swap(root string, mut report Report) ! {365+	old := os.join_path(root, koh2_dir)366+	archive := os.join_path(root, archive_dir)367+	if os.exists(archive) {368+		return error('${archive} already exists — move it away first')369+	}370+	os.mv(old, archive, os.MvParams{})!371+	os.mv(os.join_path(root, staging_file), os.join_path(root, storage.file_name), os.MvParams{})!372+	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is373+	// where Koh 3 reads them from.374+	ignore_source := os.join_path(archive, 'ignore')375+	ignore_target := os.join_path(root, snapshot.ignore_file)376+	if os.is_file(ignore_source) && !os.exists(ignore_target) {377+		body := os.read_file(ignore_source)!378+		os.write_file(ignore_target, body)!379+		report.ignore_carried = true380+	}381+}382+383+// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.384+fn bare_hash(value string) string {385+	if at := value.index(':') {386+		return value[at + 1..]387+	}388+	return value389+}390+391+// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.392+fn base64_decode(encoded string) ![]u8 {393+	return base64.decode(encoded)394+}

@@ -0,0 +1,190 @@1+//! Migration tests, run against a fixture made by the Koh 2 binary itself.2+//!3+//! The fixture is why these are worth having: it is not a guess at Koh 2's4+//! format, it is the format, including the inlined small files and the object5+//! store. When `koh` (Koh 2) is not on the machine these tests skip rather6+//! than pretend.7+8+module migrate9+10+import json11+import os12+import src.events13+import src.history14+import src.snapshot15+import src.storage16+17+struct Fixture {18+	dir     string19+	first   string20+	second  string21+	message string22+}23+24+fn koh2_binary() ?string {25+	path := os.find_abs_path_of_executable('koh') or { return none }26+	return path27+}28+29+fn sh(dir string, command string) !string {30+	result := os.execute('cd ${dir} && ${command}')31+	if result.exit_code != 0 {32+		return error('${command} failed: ${result.output}')33+	}34+	return result.output35+}36+37+fn make_fixture(koh2 string) !Fixture {38+	dir := os.join_path(os.temp_dir(), 'koh3_migrate_${os.getpid()}')39+	os.rmdir_all(dir) or {}40+	os.mkdir_all(dir, os.MkdirParams{})!41+	sh(dir, '${koh2} init')!42+	// One file big enough for the object store, one small enough to be inlined.43+	os.write_file(os.join_path(dir, 'big.txt'), 'x'.repeat(700))!44+	os.write_file(os.join_path(dir, 'small.txt'), 'tiny\n')!45+	sh(dir, '${koh2} save "first save"')!46+	sh(dir, '${koh2} checkpoint "before something"')!47+	os.write_file(os.join_path(dir, 'big.txt'), 'y'.repeat(700))!48+	sh(dir, '${koh2} save "second save"')!49+50+	// The head comes from Koh 2's own state, because save IDs are hashes (their51+	// directory order means nothing) and koh2 timestamps are whole seconds (so52+	// a save and the checkpoint after it can share one).53+	mut first := ''54+	mut first_time := i64(9223372036854775807)55+	for name in os.ls(os.join_path(dir, '.koh', 'snapshots'))! {56+		if !name.ends_with('.json') {57+			continue58+		}59+		record :=60+			json.decode(Manifest, os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!61+		if record.timestamp < first_time {62+			first_time = record.timestamp63+			first = record.id64+		}65+	}66+	return Fixture{67+		dir:     dir68+		first:   first69+		second:  koh2_head(dir)70+		message: 'first save'71+	}72+}73+74+// koh2_head reads the lane head Koh 2 recorded.75+fn koh2_head(dir string) string {76+	db := storage.connect(os.join_path(dir, '.koh', 'koh.db'), 1000) or { return '' }77+	defer {78+		db.close()79+	}80+	mut stmt := db.prepare('SELECT value FROM state WHERE key = ?') or { return '' }81+	defer {82+		stmt.finalize()83+	}84+	stmt.bind_text(1, 'head_main') or { return '' }85+	if stmt.step() or { return '' } {86+		return stmt.text(0)87+	}88+	return ''89+}90+91+fn test_migrating_a_koh2_repo_keeps_every_save_exactly() {92+	koh2 := koh2_binary() or { return }93+	fixture := make_fixture(koh2)!94+	defer {95+		os.rmdir_all(fixture.dir) or {}96+	}97+	// Flag the first save, so a save-level flag has to become an event.98+	sh(fixture.dir, '${koh2} flag ${fixture.first} --reason "sketchy"')!99+100+	// What the manifests say, read independently of the migration.101+	mut manifests := map[string][]string{}102+	for name in os.ls(os.join_path(fixture.dir, '.koh', 'snapshots'))! {103+		if !name.ends_with('.json') {104+			continue105+		}106+		record := json.decode(Manifest,107+			os.read_file(os.join_path(fixture.dir, '.koh', 'snapshots', name))!)!108+		mut files := []string{}109+		for file in record.files {110+			files << '${file.path} ${bare_hash(file.hash)}'111+		}112+		manifests[record.id] = files113+	}114+115+	report := run(fixture.dir, default_lane)!116+	// The fixture lives on main, because that is the lane koh2 starts on.117+	assert report.lane == 'main'118+	assert report.saves == 3119+	assert report.checkpoints == 1120+	assert report.flags == 1121+	assert report.objects >= 3122+123+	// The swap happened: `.koh` is a file now and the old data is archived.124+	assert os.is_file(os.join_path(fixture.dir, '.koh'))125+	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))126+	assert os.is_file(os.join_path(fixture.dir, '.kohignore'))127+128+	store := storage.open(fixture.dir)!129+	defer {130+		store.close()131+	}132+	// Every save kept its ID, its files and their hashes.133+	assert history.save_count(store)! == 3134+	for id, expected in manifests {135+		save := history.by_id(store, id) or {136+			assert false, 'save ${id} did not survive the migration'137+			return138+		}139+		assert save.machine == 'koh2'140+		mut actual := []string{}141+		for file in history.files_of(store, id)! {142+			actual << '${file.path} ${file.hash}'143+			// Every object is present and hashes back to its key.144+			if _ := store.get_object(file.hash) {145+			} else {146+				assert false, 'object ${file.hash} is missing or corrupt'147+			}148+		}149+		assert actual == expected150+	}151+	// The checkpoint mark and the parent chain came across.152+	middle := history.by_id(store, fixture.first) or { panic('the first save did not survive') }153+	assert (history.head(store) or { panic('no head after migrating') }).id == fixture.second154+	assert middle.message == 'first save'155+156+	// The flag became an event, and the working tree is exactly what Koh 3157+	// expects once the carried ignore file is saved.158+	assert events.flag_state(store, fixture.first)!.flagged159+	assert events.flag_state(store, fixture.first)!.reason == 'sketchy'160+	entries := snapshot.walk(store, fixture.dir, snapshot.load(fixture.dir))!161+	mut changed := []string{}162+	head_files := snapshot.index_files(history.files_of(store, fixture.second)!)163+	for entry in entries {164+		if entry.path !in head_files {165+			changed << entry.path166+		} else if head_files[entry.path].hash != entry.hash {167+			changed << entry.path168+		}169+	}170+	// The only difference is the ignore file Koh 3 just carried to the root.171+	assert changed == ['.kohignore']172+}173+174+fn test_migrate_refuses_a_tree_that_does_not_match_the_last_save() {175+	koh2 := koh2_binary() or { return }176+	fixture := make_fixture(koh2)!177+	defer {178+		os.rmdir_all(fixture.dir) or {}179+	}180+	os.write_file(os.join_path(fixture.dir, 'big.txt'), 'unsaved change\n')!181+	if _ := run(fixture.dir, default_lane) {182+		assert false, 'a dirty tree must be refused'183+	} else {184+		assert err.msg().contains('save with koh2 first')185+	}186+	// Nothing was touched: still a Koh 2 repo, no archive, no staging file.187+	assert os.is_dir(os.join_path(fixture.dir, '.koh'))188+	assert !os.exists(os.join_path(fixture.dir, '.koh-v2'))189+	assert !os.exists(os.join_path(fixture.dir, staging_file))190+}