migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.events16import src.history17import src.snapshot18import src.storage1920pub const koh2_dir = '.koh'21pub const archive_dir = '.koh-v2'22pub const staging_file = '.koh-migrate'23pub const default_lane = 'dev'2425// One file in a Koh 2 manifest. Small files were inlined as base64 instead of26// being written to the object store, and those entries carry `inline` with no27// `object` key at all — so `inline` decides, and `object` is only the28// alternative.29struct ManifestFile {30	path   string31	hash   string32	size   i6433	object bool34	inline ?string35}3637struct Manifest {38	id            string39	lane          string40	message       string41	timestamp     i6442	parent_id     ?string43	size_bytes    i6444	agent         ?string45	session       ?string46	is_checkpoint bool47	flagged       bool48	flag_reason   ?string49	files         []ManifestFile50}5152pub struct Report {53pub mut:54	root               string55	lane               string56	saves              int57	checkpoints        int58	flags              int59	objects            int60	inlined            int61	head               string62	ignore_carried     bool63	log_md_was_tracked bool64}6566// run converts the repo at `root`. `lane` picks which line to import: `dev`67// unless told otherwise, because that is where the work is.68pub fn run(root string, lane string) !Report {69	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {70		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')71	}72	manifests := read_manifests(root)!73	if manifests.len == 0 {74		return error('the Koh 2 repo has no manifests to import')75	}76	// `dev` holds everything when it was used at all. A repo that never left77	// `main` — which is what koh2 gave a new repo — imports from main instead78	// of failing on an empty lane.79	mut chosen := lane80	head := lane_head(root, manifests, chosen) or {81		if chosen != default_lane {82			return err83		}84		chosen = other_lane(manifests) or { return err }85		lane_head(root, manifests, chosen)!86	}87	line := ancestry(manifests, head)!88	check_clean_tree(root, head)!8990	mut report := Report{91		root: root92		lane: chosen93		head: head.id94	}95	// Import into a staging file first: nothing is touched until it verifies.96	staging := os.join_path(root, staging_file)97	if os.exists(staging) {98		return error('${staging} already exists — a previous migrate did not finish; move it away and try again')99	}100	mut staged := storage.create_at(staging, root)!101	// The staged file must be closed before it is moved: SQLite keeps committed102	// work in its WAL until the last connection goes away, and moving the file103	// alone would leave that work behind.104	mut failure := ''105	if _ := import_and_verify(staged, root, line, mut report) {106	} else {107		failure = err.msg()108	}109	staged.close()110	if failure != '' {111		return error(failure)112	}113	swap(root, mut report)!114	report.saves = line.len115	for manifest in line {116		if manifest.is_checkpoint {117			report.checkpoints++118		}119		if manifest.flagged {120			report.flags++121		}122		for file in manifest.files {123			if file.inline != none {124				report.inlined++125			}126		}127	}128	return report129}130131fn read_manifests(root string) ![]Manifest {132	dir := os.join_path(root, koh2_dir, 'snapshots')133	mut manifests := []Manifest{}134	for name in os.ls(dir)! {135		if !name.ends_with('.json') {136			continue137		}138		body := os.read_file(os.join_path(dir, name))!139		manifests << json2.decode[Manifest](body)!140	}141	return manifests142}143144// lane_head finds the newest save on a lane: the state table when it is there,145// otherwise the manifest nothing else claims as a parent.146fn lane_head(root string, manifests []Manifest, lane string) !Manifest {147	if recorded := state_head(root, lane) {148		for manifest in manifests {149			if manifest.id == recorded {150				return manifest151			}152		}153	}154	mut claimed := map[string]bool{}155	for manifest in manifests {156		if parent := manifest.parent_id {157			claimed[parent] = true158		}159	}160	mut candidates := []Manifest{}161	for manifest in manifests {162		if manifest.lane == lane && manifest.id !in claimed {163			candidates << manifest164		}165	}166	if candidates.len == 0 {167		return error('no saves on lane "${lane}"')168	}169	candidates.sort(a.timestamp < b.timestamp)170	return candidates[candidates.len - 1]171}172173// other_lane names the lane that does have saves, when the preferred one is174// empty.175fn other_lane(manifests []Manifest) ?string {176	mut lanes := map[string]bool{}177	for manifest in manifests {178		lanes[manifest.lane] = true179	}180	for lane in ['main', 'dev'] {181		if lane in lanes {182			return lane183		}184	}185	for lane in lanes.keys() {186		return lane187	}188	return none189}190191fn state_head(root string, lane string) ?string {192	db := os.join_path(root, koh2_dir, 'koh.db')193	if !os.is_file(db) {194		return none195	}196	store := storage.connect(db, 1000) or { return none }197	defer {198		store.close()199	}200	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }201	defer {202		stmt.finalize()203	}204	stmt.bind_text(1, 'head_${lane}') or { return none }205	if stmt.step() or { return none } {206		value := stmt.text(0)207		if value != '' {208			return value209		}210	}211	return none212}213214// ancestry walks back from the lane head to the first save, oldest first, and215// refuses a manifest whose parent is missing: a gap would silently lose work.216fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {217	mut by_id := map[string]Manifest{}218	for manifest in manifests {219		by_id[manifest.id] = manifest220	}221	mut line := []Manifest{}222	mut cursor := head223	for {224		line << cursor225		parent := cursor.parent_id or { break }226		if parent == '' {227			break228		}229		next := by_id[parent] or {230			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')231		}232		cursor = next233	}234	line.reverse_in_place()235	return line236}237238// check_clean_tree refuses unless the working tree matches the lane head, which239// is the same rule the document states: save with koh2 first.240fn check_clean_tree(root string, head Manifest) ! {241	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {242		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }243	})244	entries := snapshot.walk_raw(root, ignor)!245	mut expected := map[string]string{}246	for file in head.files {247		expected[file.path] = bare_hash(file.hash)248	}249	mut actual := map[string]string{}250	for entry in entries {251		actual[entry.path] = entry.hash252	}253	for path, hash in expected {254		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as255		// covering the folder, so a saved path is checked on disk even when256		// the walk skipped it.257		have := actual[path] or {258			disk_hash(root, path) or {259				return error('${path} is missing from the working tree — save with koh2 first')260			}261		}262		if have != hash {263			return error('${path} does not match the last save — save with koh2 first')264		}265	}266	for path in actual.keys() {267		if path !in expected && !koh2_never_saved(path) {268			return error('${path} is not in the last save — save with koh2 first')269		}270	}271}272273// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,274// nor does Koh 3, so a link on the way to it means the saved file is missing.275fn disk_hash(root string, path string) ?string {276	// Only a plain path inside the repo; a damaged manifest cannot point out.277	if path.starts_with('/') || path.split('/').any(it in ['', '.', '..']) {278		return none279	}280	full := os.join_path(root, path)281	if !os.is_file(full) || os.is_link(full) {282		return none283	}284	snapshot.check_links(root, [snapshot.FileEntry{285		path: path286	}]) or { return none }287	return storage.hash_hex(os.read_bytes(full) or { return none })288}289290// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3291// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder292// with such a name too, so every segment counts. The files stay where they are.293fn koh2_never_saved(path string) bool {294	parts := path.split('/')295	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')296		|| parts.any(it.ends_with('.bundle'))297}298299fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report) ! {300	store.begin()!301	if _ := import_line(store, root, line, mut report) {302		store.commit()!303	} else {304		store.rollback()305		return err306	}307	verify(store, line)!308}309310fn import_line(store &storage.Store, root string, line []Manifest, mut report Report) !bool {311	for manifest in line {312		mut files := []snapshot.FileEntry{}313		for file in manifest.files {314			hash := bare_hash(file.hash)315			if encoded := file.inline {316				data := base64_decode(encoded)!317				stored := store.put_object(data)!318				if stored != hash {319					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')320				}321				report.objects++322			} else if file.object {323				blob := read_object(root, hash)!324				store.put_object_blob(hash, blob)!325				report.objects++326			} else {327				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')328			}329			files << snapshot.FileEntry{330				path: file.path331				hash: hash332			}333		}334		save := history.Save{335			id:            manifest.id336			parent:        manifest.parent_id or { '' }337			message:       manifest.message338			created_at:    manifest.timestamp339			machine:       'koh2'340			is_checkpoint: manifest.is_checkpoint341			size_bytes:    manifest.size_bytes342			agent:         manifest.agent or { '' }343			session:       manifest.session or { '' }344		}345		history.import_save(store, save, files)!346		if manifest.flagged {347			events.flag(store, manifest.id, manifest.flag_reason or { '' })!348		}349		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')350	}351	// The newest save on the imported line becomes head.352	history.set_head(store, line[line.len - 1].id)!353	return true354}355356fn has_path(files []snapshot.FileEntry, path string) bool {357	for file in files {358		if file.path == path {359			return true360		}361	}362	return false363}364365fn read_object(root string, hash string) ![]u8 {366	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])367	return os.read_bytes(path) or {368		return error('object ${hash} is missing from ${koh2_dir}/objects')369	}370}371372fn verify(store &storage.Store, line []Manifest) ! {373	imported := history.save_count(store)!374	if imported != line.len {375		return error('imported ${imported} saves, expected ${line.len}')376	}377	for manifest in line {378		files := history.files_of(store, manifest.id)!379		if files.len != manifest.files.len {380			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')381		}382		for i, file in manifest.files {383			hash := bare_hash(file.hash)384			if files[i].path != file.path || files[i].hash != hash {385				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')386			}387			store.get_object(hash) or {388				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')389			}390		}391	}392}393394// swap puts the imported database in place and moves the Koh 2 data aside.395fn swap(root string, mut report Report) ! {396	old := os.join_path(root, koh2_dir)397	archive := os.join_path(root, archive_dir)398	if os.exists(archive) {399		return error('${archive} already exists — move it away first')400	}401	os.mv(old, archive, os.MvParams{})!402	os.mv(os.join_path(root, staging_file), os.join_path(root, storage.file_name), os.MvParams{})!403	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is404	// where Koh 3 reads them from.405	ignore_source := os.join_path(archive, 'ignore')406	ignore_target := os.join_path(root, snapshot.ignore_file)407	if os.is_file(ignore_source) && !os.exists(ignore_target) {408		body := os.read_file(ignore_source)!409		os.write_file(ignore_target, body)!410		report.ignore_carried = true411	}412}413414// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.415fn bare_hash(value string) string {416	if at := value.index(':') {417		return value[at + 1..]418	}419	return value420}421422// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.423fn base64_decode(encoded string) ![]u8 {424	return base64.decode(encoded)425}