migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.history18import src.snapshot19import src.storage2021pub const koh2_dir = '.koh'22pub const archive_dir = '.koh-v2'23// Koh 2's build file, renamed to `koh.build` (build.config_name) by the24// migration, which is the name Koh 3 reads.25const kepr_build_file = 'kepr.build'26pub const draft_file = '.koh-migrate'27pub const default_lane = 'dev'2829// One file in a Koh 2 manifest. Small files were inlined as base64 instead of30// being written to the object store, and those entries carry `inline` with no31// `object` key at all — so `inline` decides, and `object` is only the32// alternative.33struct ManifestFile {34	path   string35	hash   string36	size   i6437	object bool38	inline ?string39}4041struct Manifest {42	id            string43	lane          string44	message       string45	timestamp     i6446	parent_id     ?string47	size_bytes    i6448	agent         ?string49	session       ?string50	is_checkpoint bool51	flagged       bool52	flag_reason   ?string53mut:54	files []ManifestFile55}5657pub struct Report {58pub mut:59	root               string60	lane               string61	saves              int62	checkpoints        int63	flags              int64	objects            int65	inlined            int66	head               string67	ignore_carried     bool68	log_md_was_tracked bool69	build_renamed      bool70	unsafe_paths       []string71}7273// Plan is what migrate found and will do, shown before anything is written.74pub struct Plan {75	root         string76	line         []Manifest77	unsafe_paths []string78pub:79	lane          string80	saves         int81	checkpoints   int82	objects       int83	last_saved    i6484	renames_build bool85}8687// Progress hears how far a stage ("importing", "checking") has come.88pub type Progress = fn (stage string, done int, total int)8990// prepare reads the repo at `root` and checks it can be converted, writing91// nothing. `lane` picks which line to import: `dev` unless told otherwise,92// because that is where the work is.93pub fn prepare(root string, lane string) !Plan {94	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {95		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')96	}97	manifests := read_manifests(root)!98	if manifests.len == 0 {99		return error('the Koh 2 repo has no manifests to import')100	}101	// `dev` holds everything when it was used at all. A repo that never left102	// `main` — which is what koh2 gave a new repo — imports from main instead103	// of failing on an empty lane.104	mut chosen := lane105	head := lane_head(root, manifests, chosen) or {106		if chosen != default_lane {107			return err108		}109		chosen = other_lane(manifests) or { return err }110		lane_head(root, manifests, chosen)!111	}112	mut line := ancestry(manifests, head)!113	unsafe_paths := drop_unsafe_paths(mut line)114	check_clean_tree(root, line[line.len - 1])!115	check_build_files(root)!116	mut hashes := map[string]bool{}117	for manifest in line {118		for file in manifest.files {119			hashes[file.hash] = true120		}121	}122	return Plan{123		root:          root124		line:          line125		unsafe_paths:  unsafe_paths126		lane:          chosen127		saves:         line.len128		checkpoints:   line.filter(it.is_checkpoint).len129		objects:       hashes.len130		last_saved:    head.timestamp131		renames_build: os.is_file(os.join_path(root, kepr_build_file))132	}133}134135// run converts the repo `prepare` read, telling `progress` as it goes.136pub fn run(plan Plan, progress Progress) !Report {137	root := plan.root138	line := plan.line139	mut report := Report{140		root:         root141		lane:         plan.lane142		head:         line[line.len - 1].id143		unsafe_paths: plan.unsafe_paths144	}145	// Import into a draft file first: nothing is touched until it verifies.146	draft_path := os.join_path(root, draft_file)147	if os.exists(draft_path) {148		return error('${draft_path} already exists — a previous migrate did not finish; move it away and try again')149	}150	mut draft := storage.create_at(draft_path, root)!151	// The draft must be closed before it is moved: SQLite keeps finished152	// work in its WAL until the last connection goes away, and moving the file153	// alone would leave that work behind.154	mut failure := ''155	if _ := import_and_verify(draft, root, line, mut report, progress) {156		report.objects = int(draft.object_count() or { 0 })157	} else {158		failure = err.msg()159	}160	draft.close()161	if failure != '' {162		return error(failure)163	}164	swap(root, mut report)!165	rename_build_file(root, mut report)!166	report.saves = line.len167	report.checkpoints = plan.checkpoints168	for manifest in line {169		if manifest.flagged {170			report.flags++171		}172		for file in manifest.files {173			if file.inline != none {174				report.inlined++175			}176		}177	}178	return report179}180181fn read_manifests(root string) ![]Manifest {182	dir := os.join_path(root, koh2_dir, 'snapshots')183	mut manifests := []Manifest{}184	for name in os.ls(dir)! {185		if !name.ends_with('.json') {186			continue187		}188		body := os.read_file(os.join_path(dir, name))!189		manifests << json2.decode[Manifest](body)!190	}191	return manifests192}193194// lane_head finds the newest save on a lane: the state table when it is there,195// otherwise the manifest nothing else claims as a parent.196fn lane_head(root string, manifests []Manifest, lane string) !Manifest {197	if recorded := state_head(root, lane) {198		for manifest in manifests {199			if manifest.id == recorded {200				return manifest201			}202		}203	}204	mut claimed := map[string]bool{}205	for manifest in manifests {206		if parent := manifest.parent_id {207			claimed[parent] = true208		}209	}210	mut candidates := []Manifest{}211	for manifest in manifests {212		if manifest.lane == lane && manifest.id !in claimed {213			candidates << manifest214		}215	}216	if candidates.len == 0 {217		return error('no saves on lane "${lane}"')218	}219	candidates.sort(a.timestamp < b.timestamp)220	return candidates[candidates.len - 1]221}222223// other_lane names the lane that does have saves, when the preferred one is224// empty.225fn other_lane(manifests []Manifest) ?string {226	mut lanes := map[string]bool{}227	for manifest in manifests {228		lanes[manifest.lane] = true229	}230	for lane in ['main', 'dev'] {231		if lane in lanes {232			return lane233		}234	}235	for lane in lanes.keys() {236		return lane237	}238	return none239}240241fn state_head(root string, lane string) ?string {242	db := os.join_path(root, koh2_dir, 'koh.db')243	if !os.is_file(db) {244		return none245	}246	store := storage.connect(db, 1000) or { return none }247	defer {248		store.close()249	}250	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }251	defer {252		stmt.finalize()253	}254	stmt.bind_text(1, 'head_${lane}') or { return none }255	if stmt.step() or { return none } {256		value := stmt.text(0)257		if value != '' {258			return value259		}260	}261	return none262}263264// ancestry walks back from the lane head to the first save, oldest first, and265// refuses a manifest whose parent is missing: a gap would silently lose work.266fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {267	mut by_id := map[string]Manifest{}268	for manifest in manifests {269		by_id[manifest.id] = manifest270	}271	mut line := []Manifest{}272	mut cursor := head273	for {274		line << cursor275		parent := cursor.parent_id or { break }276		if parent == '' {277			break278		}279		next := by_id[parent] or {280			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')281		}282		cursor = next283	}284	line.reverse_in_place()285	return line286}287288// check_clean_tree refuses unless the working tree matches the lane head, which289// is the same rule the document states: save with koh2 first.290fn check_clean_tree(root string, head Manifest) ! {291	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {292		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }293	})294	entries := snapshot.walk_raw(root, ignor)!295	mut expected := map[string]string{}296	for file in head.files {297		expected[file.path] = bare_hash(file.hash)298	}299	mut actual := map[string]string{}300	for entry in entries {301		actual[entry.path] = entry.hash302	}303	for path, hash in expected {304		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as305		// covering the folder, so a saved path is checked on disk even when306		// the walk skipped it.307		have := actual[path] or {308			disk_hash(root, path) or {309				return error('${path} is missing from the working tree — save with koh2 first')310			}311		}312		if have != hash {313			return error('${path} does not match the last save — save with koh2 first')314		}315	}316	for path in actual.keys() {317		if path !in expected && !koh2_never_saved(path) {318			return error('${path} is not in the last save — save with koh2 first')319		}320	}321}322323// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,324// nor does Koh 3, so a link on the way to it means the saved file is missing.325fn disk_hash(root string, path string) ?string {326	// Only a legal path inside the repo; a damaged manifest cannot point out.327	if !snapshot.legal_path(path) {328		return none329	}330	full := os.join_path(root, path)331	if !os.is_file(full) || os.is_link(full) {332		return none333	}334	snapshot.check_links(root, [snapshot.FileEntry{335		path: path336	}]) or { return none }337	return storage.hash_hex(os.read_bytes(full) or { return none })338}339340// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3341// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder342// with such a name too, so every segment counts. The files stay where they are.343fn koh2_never_saved(path string) bool {344	parts := path.split('/')345	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')346		|| parts.any(it.ends_with('.bundle'))347}348349// drop_unsafe_paths removes, by the shared path rule, every file Koh 3 would350// refuse to restore, and names each one once for the report.351fn drop_unsafe_paths(mut line []Manifest) []string {352	mut dropped := map[string]bool{}353	for mut manifest in line {354		for file in manifest.files {355			if !snapshot.legal_path(file.path) {356				dropped[snapshot.escape_path(file.path)] = true357			}358		}359		manifest.files = manifest.files.filter(snapshot.legal_path(it.path))360	}361	mut names := dropped.keys()362	names.sort()363	return names364}365366fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report, progress Progress) ! {367	store.begin()!368	if _ := import_line(store, root, line, mut report, progress) {369		store.end()!370	} else {371		store.rollback()372		return err373	}374	verify(store, line, progress)!375}376377fn import_line(store &storage.Store, root string, line []Manifest, mut report Report, progress Progress) !bool {378	for i, manifest in line {379		progress('importing', i + 1, line.len)380		mut files := []snapshot.FileEntry{}381		for file in manifest.files {382			hash := bare_hash(file.hash)383			if encoded := file.inline {384				data := base64_decode(encoded)!385				stored := store.put_object(data)!386				if stored != hash {387					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')388				}389			} else if file.object {390				blob := read_object(root, hash)!391				store.put_object_blob(hash, blob)!392			} else {393				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')394			}395			files << snapshot.FileEntry{396				path: file.path397				hash: hash398			}399		}400		save := history.Save{401			id:            manifest.id402			parent:        manifest.parent_id or { '' }403			message:       manifest.message404			created_at:    manifest.timestamp405			machine:       'koh2'406			is_checkpoint: manifest.is_checkpoint407			size_bytes:    manifest.size_bytes408			agent:         manifest.agent or { '' }409			session:       manifest.session or { '' }410		}411		history.import_save(store, save, files)!412		if manifest.flagged {413			events.flag(store, manifest.id, manifest.flag_reason or { '' })!414		}415		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')416	}417	// The newest save on the imported line becomes head.418	history.set_head(store, line[line.len - 1].id)!419	return true420}421422fn has_path(files []snapshot.FileEntry, path string) bool {423	for file in files {424		if file.path == path {425			return true426		}427	}428	return false429}430431fn read_object(root string, hash string) ![]u8 {432	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])433	return os.read_bytes(path) or {434		return error('object ${hash} is missing from ${koh2_dir}/objects')435	}436}437438fn verify(store &storage.Store, line []Manifest, progress Progress) ! {439	imported := history.save_count(store)!440	if imported != line.len {441		return error('imported ${imported} saves, expected ${line.len}')442	}443	for n, manifest in line {444		progress('checking', n + 1, line.len)445		files := history.files_of(store, manifest.id)!446		if files.len != manifest.files.len {447			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')448		}449		for i, file in manifest.files {450			hash := bare_hash(file.hash)451			if files[i].path != file.path || files[i].hash != hash {452				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')453			}454			store.get_object(hash) or {455				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')456			}457		}458	}459}460461// swap puts the imported database in place and moves the Koh 2 data aside.462fn swap(root string, mut report Report) ! {463	old := os.join_path(root, koh2_dir)464	archive := os.join_path(root, archive_dir)465	if os.exists(archive) {466		return error('${archive} already exists — move it away first')467	}468	os.mv(old, archive, os.MvParams{})!469	os.mv(os.join_path(root, draft_file), os.join_path(root, storage.file_name), os.MvParams{})!470	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is471	// where Koh 3 reads them from.472	ignore_source := os.join_path(archive, 'ignore')473	ignore_target := os.join_path(root, snapshot.ignore_file)474	if os.is_file(ignore_source) && !os.exists(ignore_target) {475		body := os.read_file(ignore_source)!476		os.write_file(ignore_target, body)!477		report.ignore_carried = true478	}479}480481// check_build_files runs in the check phase, before the swap: a repo holding482// both build files is refused while nothing has moved, never left half done.483fn check_build_files(root string) ! {484	if os.is_file(os.join_path(root, kepr_build_file))485		&& os.exists(os.join_path(root, build.config_name)) {486		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')487	}488}489490// rename_build_file finishes the move to `koh.build`, and records it as a save:491// the migration must not leave the working tree dirty with a rename the next492// save would otherwise pick up as unreviewed work.493fn rename_build_file(root string, mut report Report) ! {494	old := os.join_path(root, kepr_build_file)495	if !os.is_file(old) {496		return497	}498	fresh := os.join_path(root, build.config_name)499	os.mv(old, fresh, os.MvParams{})!500	mut store := storage.open(root)!501	defer {502		store.close()503	}504	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{505		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'506	})!507	report.build_renamed = true508}509510// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.511fn bare_hash(value string) string {512	if at := value.index(':') {513		return value[at + 1..]514	}515	return value516}517518// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.519fn base64_decode(encoded string) ![]u8 {520	return base64.decode(encoded)521}