migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.history18import src.snapshot19import src.storage2021pub const koh2_dir = '.koh'22pub const archive_dir = '.koh-v2'23// Koh 2's build file, renamed to `koh.build` (build.config_name) by the24// migration, which is the name Koh 3 reads.25const kepr_build_file = 'kepr.build'26pub const draft_file = '.koh-migrate'27pub const default_lane = 'dev'2829// One file in a Koh 2 manifest. Small files were inlined as base64 instead of30// being written to the object store, and those entries carry `inline` with no31// `object` key at all — so `inline` decides, and `object` is only the32// alternative.33struct ManifestFile {34	path   string35	hash   string36	size   i6437	object bool38	inline ?string39}4041struct Manifest {42	id            string43	lane          string44	message       string45	timestamp     i6446	parent_id     ?string47	size_bytes    i6448	agent         ?string49	session       ?string50	is_checkpoint bool51	flagged       bool52	flag_reason   ?string53mut:54	files []ManifestFile55}5657pub struct Report {58pub mut:59	root               string60	lane               string61	saves              int62	checkpoints        int63	flags              int64	objects            int65	inlined            int66	head               string67	ignore_carried     bool68	log_md_was_tracked bool69	build_renamed      bool70	unsafe_paths       []string71}7273// run converts the repo at `root`. `lane` picks which line to import: `dev`74// unless told otherwise, because that is where the work is.75pub fn run(root string, lane string) !Report {76	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {77		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')78	}79	manifests := read_manifests(root)!80	if manifests.len == 0 {81		return error('the Koh 2 repo has no manifests to import')82	}83	// `dev` holds everything when it was used at all. A repo that never left84	// `main` — which is what koh2 gave a new repo — imports from main instead85	// of failing on an empty lane.86	mut chosen := lane87	head := lane_head(root, manifests, chosen) or {88		if chosen != default_lane {89			return err90		}91		chosen = other_lane(manifests) or { return err }92		lane_head(root, manifests, chosen)!93	}94	mut line := ancestry(manifests, head)!95	unsafe_paths := drop_unsafe_paths(mut line)96	check_clean_tree(root, line[line.len - 1])!97	check_build_files(root)!9899	mut report := Report{100		root:         root101		lane:         chosen102		head:         head.id103		unsafe_paths: unsafe_paths104	}105	// Import into a draft file first: nothing is touched until it verifies.106	draft_path := os.join_path(root, draft_file)107	if os.exists(draft_path) {108		return error('${draft_path} already exists — a previous migrate did not finish; move it away and try again')109	}110	mut draft := storage.create_at(draft_path, root)!111	// The draft must be closed before it is moved: SQLite keeps finished112	// work in its WAL until the last connection goes away, and moving the file113	// alone would leave that work behind.114	mut failure := ''115	if _ := import_and_verify(draft, root, line, mut report) {116	} else {117		failure = err.msg()118	}119	draft.close()120	if failure != '' {121		return error(failure)122	}123	swap(root, mut report)!124	rename_build_file(root, mut report)!125	report.saves = line.len126	for manifest in line {127		if manifest.is_checkpoint {128			report.checkpoints++129		}130		if manifest.flagged {131			report.flags++132		}133		for file in manifest.files {134			if file.inline != none {135				report.inlined++136			}137		}138	}139	return report140}141142fn read_manifests(root string) ![]Manifest {143	dir := os.join_path(root, koh2_dir, 'snapshots')144	mut manifests := []Manifest{}145	for name in os.ls(dir)! {146		if !name.ends_with('.json') {147			continue148		}149		body := os.read_file(os.join_path(dir, name))!150		manifests << json2.decode[Manifest](body)!151	}152	return manifests153}154155// lane_head finds the newest save on a lane: the state table when it is there,156// otherwise the manifest nothing else claims as a parent.157fn lane_head(root string, manifests []Manifest, lane string) !Manifest {158	if recorded := state_head(root, lane) {159		for manifest in manifests {160			if manifest.id == recorded {161				return manifest162			}163		}164	}165	mut claimed := map[string]bool{}166	for manifest in manifests {167		if parent := manifest.parent_id {168			claimed[parent] = true169		}170	}171	mut candidates := []Manifest{}172	for manifest in manifests {173		if manifest.lane == lane && manifest.id !in claimed {174			candidates << manifest175		}176	}177	if candidates.len == 0 {178		return error('no saves on lane "${lane}"')179	}180	candidates.sort(a.timestamp < b.timestamp)181	return candidates[candidates.len - 1]182}183184// other_lane names the lane that does have saves, when the preferred one is185// empty.186fn other_lane(manifests []Manifest) ?string {187	mut lanes := map[string]bool{}188	for manifest in manifests {189		lanes[manifest.lane] = true190	}191	for lane in ['main', 'dev'] {192		if lane in lanes {193			return lane194		}195	}196	for lane in lanes.keys() {197		return lane198	}199	return none200}201202fn state_head(root string, lane string) ?string {203	db := os.join_path(root, koh2_dir, 'koh.db')204	if !os.is_file(db) {205		return none206	}207	store := storage.connect(db, 1000) or { return none }208	defer {209		store.close()210	}211	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }212	defer {213		stmt.finalize()214	}215	stmt.bind_text(1, 'head_${lane}') or { return none }216	if stmt.step() or { return none } {217		value := stmt.text(0)218		if value != '' {219			return value220		}221	}222	return none223}224225// ancestry walks back from the lane head to the first save, oldest first, and226// refuses a manifest whose parent is missing: a gap would silently lose work.227fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {228	mut by_id := map[string]Manifest{}229	for manifest in manifests {230		by_id[manifest.id] = manifest231	}232	mut line := []Manifest{}233	mut cursor := head234	for {235		line << cursor236		parent := cursor.parent_id or { break }237		if parent == '' {238			break239		}240		next := by_id[parent] or {241			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')242		}243		cursor = next244	}245	line.reverse_in_place()246	return line247}248249// check_clean_tree refuses unless the working tree matches the lane head, which250// is the same rule the document states: save with koh2 first.251fn check_clean_tree(root string, head Manifest) ! {252	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {253		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }254	})255	entries := snapshot.walk_raw(root, ignor)!256	mut expected := map[string]string{}257	for file in head.files {258		expected[file.path] = bare_hash(file.hash)259	}260	mut actual := map[string]string{}261	for entry in entries {262		actual[entry.path] = entry.hash263	}264	for path, hash in expected {265		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as266		// covering the folder, so a saved path is checked on disk even when267		// the walk skipped it.268		have := actual[path] or {269			disk_hash(root, path) or {270				return error('${path} is missing from the working tree — save with koh2 first')271			}272		}273		if have != hash {274			return error('${path} does not match the last save — save with koh2 first')275		}276	}277	for path in actual.keys() {278		if path !in expected && !koh2_never_saved(path) {279			return error('${path} is not in the last save — save with koh2 first')280		}281	}282}283284// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,285// nor does Koh 3, so a link on the way to it means the saved file is missing.286fn disk_hash(root string, path string) ?string {287	// Only a legal path inside the repo; a damaged manifest cannot point out.288	if !snapshot.legal_path(path) {289		return none290	}291	full := os.join_path(root, path)292	if !os.is_file(full) || os.is_link(full) {293		return none294	}295	snapshot.check_links(root, [snapshot.FileEntry{296		path: path297	}]) or { return none }298	return storage.hash_hex(os.read_bytes(full) or { return none })299}300301// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3302// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder303// with such a name too, so every segment counts. The files stay where they are.304fn koh2_never_saved(path string) bool {305	parts := path.split('/')306	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')307		|| parts.any(it.ends_with('.bundle'))308}309310// drop_unsafe_paths removes, by the shared path rule, every file Koh 3 would311// refuse to restore, and names each one once for the report.312fn drop_unsafe_paths(mut line []Manifest) []string {313	mut dropped := map[string]bool{}314	for mut manifest in line {315		for file in manifest.files {316			if !snapshot.legal_path(file.path) {317				dropped[snapshot.escape_path(file.path)] = true318			}319		}320		manifest.files = manifest.files.filter(snapshot.legal_path(it.path))321	}322	mut names := dropped.keys()323	names.sort()324	return names325}326327fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report) ! {328	store.begin()!329	if _ := import_line(store, root, line, mut report) {330		store.end()!331	} else {332		store.rollback()333		return err334	}335	verify(store, line)!336}337338fn import_line(store &storage.Store, root string, line []Manifest, mut report Report) !bool {339	for manifest in line {340		mut files := []snapshot.FileEntry{}341		for file in manifest.files {342			hash := bare_hash(file.hash)343			if encoded := file.inline {344				data := base64_decode(encoded)!345				stored := store.put_object(data)!346				if stored != hash {347					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')348				}349				report.objects++350			} else if file.object {351				blob := read_object(root, hash)!352				store.put_object_blob(hash, blob)!353				report.objects++354			} else {355				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')356			}357			files << snapshot.FileEntry{358				path: file.path359				hash: hash360			}361		}362		save := history.Save{363			id:            manifest.id364			parent:        manifest.parent_id or { '' }365			message:       manifest.message366			created_at:    manifest.timestamp367			machine:       'koh2'368			is_checkpoint: manifest.is_checkpoint369			size_bytes:    manifest.size_bytes370			agent:         manifest.agent or { '' }371			session:       manifest.session or { '' }372		}373		history.import_save(store, save, files)!374		if manifest.flagged {375			events.flag(store, manifest.id, manifest.flag_reason or { '' })!376		}377		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')378	}379	// The newest save on the imported line becomes head.380	history.set_head(store, line[line.len - 1].id)!381	return true382}383384fn has_path(files []snapshot.FileEntry, path string) bool {385	for file in files {386		if file.path == path {387			return true388		}389	}390	return false391}392393fn read_object(root string, hash string) ![]u8 {394	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])395	return os.read_bytes(path) or {396		return error('object ${hash} is missing from ${koh2_dir}/objects')397	}398}399400fn verify(store &storage.Store, line []Manifest) ! {401	imported := history.save_count(store)!402	if imported != line.len {403		return error('imported ${imported} saves, expected ${line.len}')404	}405	for manifest in line {406		files := history.files_of(store, manifest.id)!407		if files.len != manifest.files.len {408			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')409		}410		for i, file in manifest.files {411			hash := bare_hash(file.hash)412			if files[i].path != file.path || files[i].hash != hash {413				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')414			}415			store.get_object(hash) or {416				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')417			}418		}419	}420}421422// swap puts the imported database in place and moves the Koh 2 data aside.423fn swap(root string, mut report Report) ! {424	old := os.join_path(root, koh2_dir)425	archive := os.join_path(root, archive_dir)426	if os.exists(archive) {427		return error('${archive} already exists — move it away first')428	}429	os.mv(old, archive, os.MvParams{})!430	os.mv(os.join_path(root, draft_file), os.join_path(root, storage.file_name), os.MvParams{})!431	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is432	// where Koh 3 reads them from.433	ignore_source := os.join_path(archive, 'ignore')434	ignore_target := os.join_path(root, snapshot.ignore_file)435	if os.is_file(ignore_source) && !os.exists(ignore_target) {436		body := os.read_file(ignore_source)!437		os.write_file(ignore_target, body)!438		report.ignore_carried = true439	}440}441442// check_build_files runs in the check phase, before the swap: a repo holding443// both build files is refused while nothing has moved, never left half done.444fn check_build_files(root string) ! {445	if os.is_file(os.join_path(root, kepr_build_file))446		&& os.exists(os.join_path(root, build.config_name)) {447		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')448	}449}450451// rename_build_file finishes the move to `koh.build`, and records it as a save:452// the migration must not leave the working tree dirty with a rename the next453// save would otherwise pick up as unreviewed work.454fn rename_build_file(root string, mut report Report) ! {455	old := os.join_path(root, kepr_build_file)456	if !os.is_file(old) {457		return458	}459	fresh := os.join_path(root, build.config_name)460	os.mv(old, fresh, os.MvParams{})!461	mut store := storage.open(root)!462	defer {463		store.close()464	}465	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{466		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'467	})!468	report.build_renamed = true469}470471// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.472fn bare_hash(value string) string {473	if at := value.index(':') {474		return value[at + 1..]475	}476	return value477}478479// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.480fn base64_decode(encoded string) ![]u8 {481	return base64.decode(encoded)482}