migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.history18import src.snapshot19import src.storage2021pub const koh2_dir = '.koh'22pub const archive_dir = '.koh-v2'23// Koh 2's build file, renamed to `koh.build` (build.config_name) by the24// migration, which is the name Koh 3 reads.25const kepr_build_file = 'kepr.build'26pub const draft_file = '.koh-migrate'27pub const default_lane = 'dev'2829// One file in a Koh 2 manifest. Small files were inlined as base64 instead of30// being written to the object store, and those entries carry `inline` with no31// `object` key at all — so `inline` decides, and `object` is only the32// alternative.33struct ManifestFile {34	path   string35	hash   string36	size   i6437	object bool38	inline ?string39}4041struct Manifest {42	id            string43	lane          string44	message       string45	timestamp     i6446	parent_id     ?string47	size_bytes    i6448	agent         ?string49	session       ?string50	is_checkpoint bool51	flagged       bool52	flag_reason   ?string53mut:54	files []ManifestFile55}5657pub struct Report {58pub mut:59	root               string60	lane               string61	saves              int62	checkpoints        int63	flags              int64	objects            int65	inlined            int66	head               string67	ignore_carried     bool68	log_md_was_tracked bool69	build_renamed      bool70	unsafe_paths       []string71}7273// Plan is what migrate found and will do, shown before anything is written.74pub struct Plan {75	root         string76	line         []Manifest77	unsafe_paths []string78	found        map[string][]u879pub:80	lost          []Lost81	lane          string82	saves         int83	checkpoints   int84	objects       int85	last_saved    i6486	renames_build bool87}8889// Lost is one file version Koh 2's saves list but its store no longer holds.90// Those saves come over without it, flagged, unless `--with` hands it back.91pub struct Lost {92pub mut:93	path  string94	hash  string95	size  i6496	saves []string97	first i6498	last  i6499}100101// Progress hears how far a stage ("importing", "checking") has come.102pub type Progress = fn (stage string, done int, total int)103104// prepare reads the repo at `root` and checks it can be converted, writing105// nothing. `lane` picks which line to import: `dev` unless told otherwise,106// because that is where the work is.107pub fn prepare(root string, lane string, with []string) !Plan {108	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {109		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')110	}111	manifests := read_manifests(root)!112	if manifests.len == 0 {113		return error('the Koh 2 repo has no manifests to import')114	}115	// `dev` holds everything when it was used at all. A repo that never left116	// `main` — which is what koh2 gave a new repo — imports from main instead117	// of failing on an empty lane.118	mut chosen := lane119	head := lane_head(root, manifests, chosen) or {120		if chosen != default_lane {121			return err122		}123		chosen = other_lane(manifests) or { return err }124		lane_head(root, manifests, chosen)!125	}126	mut line := ancestry(manifests, head)!127	unsafe_paths := drop_unsafe_paths(mut line)128	found := versions_by_hash(with)!129	lost := drop_lost_objects(root, mut line, found)130	check_clean_tree(root, line[line.len - 1])!131	check_build_files(root)!132	mut hashes := map[string]bool{}133	for manifest in line {134		for file in manifest.files {135			hashes[file.hash] = true136		}137	}138	return Plan{139		root:          root140		line:          line141		unsafe_paths:  unsafe_paths142		found:         found143		lost:          lost144		lane:          chosen145		saves:         line.len146		checkpoints:   line.filter(it.is_checkpoint).len147		objects:       hashes.len148		last_saved:    head.timestamp149		renames_build: os.is_file(os.join_path(root, kepr_build_file))150	}151}152153// run converts the repo `prepare` read, telling `progress` as it goes.154pub fn run(plan Plan, progress Progress) !Report {155	root := plan.root156	line := plan.line157	mut report := Report{158		root:         root159		lane:         plan.lane160		head:         line[line.len - 1].id161		unsafe_paths: plan.unsafe_paths162	}163	// Import into a draft file first: nothing is touched until it verifies.164	draft_path := os.join_path(root, draft_file)165	if os.exists(draft_path) {166		return error('${draft_path} already exists — a previous migrate did not finish; move it away and try again')167	}168	mut draft := storage.create_at(draft_path, root)!169	// The draft must be closed before it is moved: SQLite keeps finished170	// work in its WAL until the last connection goes away, and moving the file171	// alone would leave that work behind.172	mut failure := ''173	if _ := import_and_verify(draft, plan, mut report, progress) {174		report.objects = int(draft.object_count() or { 0 })175	} else {176		failure = err.msg()177	}178	draft.close()179	if failure != '' {180		// The draft is migrate's own; leaving it would refuse the next attempt.181		for suffix in ['', '-wal', '-shm'] {182			os.rm(draft_path + suffix) or {}183		}184		return error(failure)185	}186	swap(root, mut report)!187	rename_build_file(root, mut report)!188	report.saves = line.len189	report.checkpoints = plan.checkpoints190	for manifest in line {191		if manifest.flagged {192			report.flags++193		}194		for file in manifest.files {195			if file.inline != none {196				report.inlined++197			}198		}199	}200	return report201}202203fn read_manifests(root string) ![]Manifest {204	dir := os.join_path(root, koh2_dir, 'snapshots')205	mut manifests := []Manifest{}206	for name in os.ls(dir)! {207		if !name.ends_with('.json') {208			continue209		}210		body := os.read_file(os.join_path(dir, name))!211		manifests << json2.decode[Manifest](body)!212	}213	return manifests214}215216// lane_head finds the newest save on a lane: the state table when it is there,217// otherwise the manifest nothing else claims as a parent.218fn lane_head(root string, manifests []Manifest, lane string) !Manifest {219	if recorded := state_head(root, lane) {220		for manifest in manifests {221			if manifest.id == recorded {222				return manifest223			}224		}225	}226	mut claimed := map[string]bool{}227	for manifest in manifests {228		if parent := manifest.parent_id {229			claimed[parent] = true230		}231	}232	mut candidates := []Manifest{}233	for manifest in manifests {234		if manifest.lane == lane && manifest.id !in claimed {235			candidates << manifest236		}237	}238	if candidates.len == 0 {239		return error('no saves on lane "${lane}"')240	}241	candidates.sort(a.timestamp < b.timestamp)242	return candidates[candidates.len - 1]243}244245// other_lane names the lane that does have saves, when the preferred one is246// empty.247fn other_lane(manifests []Manifest) ?string {248	mut lanes := map[string]bool{}249	for manifest in manifests {250		lanes[manifest.lane] = true251	}252	for lane in ['main', 'dev'] {253		if lane in lanes {254			return lane255		}256	}257	for lane in lanes.keys() {258		return lane259	}260	return none261}262263fn state_head(root string, lane string) ?string {264	db := os.join_path(root, koh2_dir, 'koh.db')265	if !os.is_file(db) {266		return none267	}268	store := storage.connect(db, 1000) or { return none }269	defer {270		store.close()271	}272	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }273	defer {274		stmt.finalize()275	}276	stmt.bind_text(1, 'head_${lane}') or { return none }277	if stmt.step() or { return none } {278		value := stmt.text(0)279		if value != '' {280			return value281		}282	}283	return none284}285286// ancestry walks back from the lane head to the first save, oldest first, and287// refuses a manifest whose parent is missing: a gap would silently lose work.288fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {289	mut by_id := map[string]Manifest{}290	for manifest in manifests {291		by_id[manifest.id] = manifest292	}293	mut line := []Manifest{}294	mut cursor := head295	for {296		line << cursor297		parent := cursor.parent_id or { break }298		if parent == '' {299			break300		}301		next := by_id[parent] or {302			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')303		}304		cursor = next305	}306	line.reverse_in_place()307	return line308}309310// check_clean_tree refuses unless the working tree matches the lane head, which311// is the same rule the document states: save with koh2 first.312fn check_clean_tree(root string, head Manifest) ! {313	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {314		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }315	})316	entries := snapshot.walk_raw(root, ignor)!317	mut expected := map[string]string{}318	for file in head.files {319		expected[file.path] = bare_hash(file.hash)320	}321	mut actual := map[string]string{}322	for entry in entries {323		actual[entry.path] = entry.hash324	}325	for path, hash in expected {326		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as327		// covering the folder, so a saved path is checked on disk even when328		// the walk skipped it.329		have := actual[path] or {330			disk_hash(root, path) or {331				return error('${path} is missing from the working tree — save with koh2 first')332			}333		}334		if have != hash {335			return error('${path} does not match the last save — save with koh2 first')336		}337	}338	for path in actual.keys() {339		if path !in expected && !koh2_never_saved(path) {340			return error('${path} is not in the last save — save with koh2 first')341		}342	}343}344345// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,346// nor does Koh 3, so a link on the way to it means the saved file is missing.347fn disk_hash(root string, path string) ?string {348	// Only a legal path inside the repo; a damaged manifest cannot point out.349	if !snapshot.legal_path(path) {350		return none351	}352	full := os.join_path(root, path)353	if !os.is_file(full) || os.is_link(full) {354		return none355	}356	snapshot.check_links(root, [snapshot.FileEntry{357		path: path358	}]) or { return none }359	return storage.hash_hex(os.read_bytes(full) or { return none })360}361362// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3363// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder364// with such a name too, so every segment counts. The files stay where they are.365fn koh2_never_saved(path string) bool {366	parts := path.split('/')367	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')368		|| parts.any(it.ends_with('.bundle'))369}370371// drop_unsafe_paths removes, by the shared path rule, every file Koh 3 would372// refuse to restore, and names each one once for the report.373fn drop_unsafe_paths(mut line []Manifest) []string {374	mut dropped := map[string]bool{}375	for mut manifest in line {376		for file in manifest.files {377			if !snapshot.legal_path(file.path) {378				dropped[snapshot.escape_path(file.path)] = true379			}380		}381		manifest.files = manifest.files.filter(snapshot.legal_path(it.path))382	}383	mut names := dropped.keys()384	names.sort()385	return names386}387388// versions_by_hash reads the files `--with` names, so a version Koh 2 lost can389// be recognised by its contents wherever it was found.390fn versions_by_hash(paths []string) !map[string][]u8 {391	mut found := map[string][]u8{}392	for path in paths {393		data := os.read_bytes(path) or { return error('cannot read ${path}: ${err.msg()}') }394		found[storage.hash_hex(data)] = data395	}396	return found397}398399// drop_lost_objects takes out of the line every file whose object Koh 2 no400// longer holds and nobody handed back, and says which, where and when.401fn drop_lost_objects(root string, mut line []Manifest, found map[string][]u8) []Lost {402	mut lost := map[string]Lost{}403	for mut manifest in line {404		mut kept := []ManifestFile{}405		for file in manifest.files {406			hash := bare_hash(file.hash)407			if file.inline != none || !file.object || hash in found408				|| os.is_file(os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])) {409				kept << file410				continue411			}412			key := '${file.path}\n${hash}'413			mut seen := lost[key] or {414				Lost{415					path:  file.path416					hash:  hash417					size:  file.size418					first: manifest.timestamp419				}420			}421			seen.saves << manifest.id422			seen.last = manifest.timestamp423			lost[key] = seen424		}425		manifest.files = kept426	}427	mut all := lost.values()428	all.sort(a.path < b.path)429	return all430}431432fn import_and_verify(store &storage.Store, plan Plan, mut report Report, progress Progress) ! {433	store.begin()!434	if _ := import_line(store, plan, mut report, progress) {435		store.end()!436	} else {437		store.rollback()438		return err439	}440	verify(store, plan.line, progress)!441}442443fn import_line(store &storage.Store, plan Plan, mut report Report, progress Progress) !bool {444	line := plan.line445	mut lost_at := map[string][]string{}446	for lost in plan.lost {447		for id in lost.saves {448			lost_at[id] << lost.path449		}450	}451	for i, manifest in line {452		progress('importing', i + 1, line.len)453		mut files := []snapshot.FileEntry{}454		for file in manifest.files {455			hash := bare_hash(file.hash)456			if encoded := file.inline {457				data := base64_decode(encoded)!458				stored := store.put_object(data)!459				if stored != hash {460					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')461				}462			} else if data := plan.found[hash] {463				store.put_object(data)!464			} else if file.object {465				blob := read_object(plan.root, hash)!466				store.put_object_blob(hash, blob)!467			} else {468				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')469			}470			files << snapshot.FileEntry{471				path: file.path472				hash: hash473			}474		}475		save := history.Save{476			id:            manifest.id477			parent:        manifest.parent_id or { '' }478			message:       manifest.message479			created_at:    manifest.timestamp480			machine:       'koh2'481			is_checkpoint: manifest.is_checkpoint482			size_bytes:    manifest.size_bytes483			agent:         manifest.agent or { '' }484			session:       manifest.session or { '' }485		}486		history.import_save(store, save, files)!487		if paths := lost_at[manifest.id] {488			// Said on the save itself, so loading it later explains the gap.489			reason := 'migrate: Koh 2 had lost ${paths.join(', ')} at this save'490			events.flag(store, manifest.id, if manifest.flagged {491				'${manifest.flag_reason or { '' }}; ${reason}'492			} else {493				reason494			})!495		} else if manifest.flagged {496			events.flag(store, manifest.id, manifest.flag_reason or { '' })!497		}498		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')499	}500	// The newest save on the imported line becomes head.501	history.set_head(store, line[line.len - 1].id)!502	return true503}504505fn has_path(files []snapshot.FileEntry, path string) bool {506	for file in files {507		if file.path == path {508			return true509		}510	}511	return false512}513514fn read_object(root string, hash string) ![]u8 {515	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])516	return os.read_bytes(path) or {517		return error('object ${hash} is missing from ${koh2_dir}/objects')518	}519}520521fn verify(store &storage.Store, line []Manifest, progress Progress) ! {522	imported := history.save_count(store)!523	if imported != line.len {524		return error('imported ${imported} saves, expected ${line.len}')525	}526	for n, manifest in line {527		progress('checking', n + 1, line.len)528		files := history.files_of(store, manifest.id)!529		if files.len != manifest.files.len {530			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')531		}532		for i, file in manifest.files {533			hash := bare_hash(file.hash)534			if files[i].path != file.path || files[i].hash != hash {535				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')536			}537			store.get_object(hash) or {538				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')539			}540		}541	}542}543544// swap puts the imported database in place and moves the Koh 2 data aside.545fn swap(root string, mut report Report) ! {546	old := os.join_path(root, koh2_dir)547	archive := os.join_path(root, archive_dir)548	if os.exists(archive) {549		return error('${archive} already exists — move it away first')550	}551	os.mv(old, archive, os.MvParams{})!552	os.mv(os.join_path(root, draft_file), os.join_path(root, storage.file_name), os.MvParams{})!553	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is554	// where Koh 3 reads them from.555	ignore_source := os.join_path(archive, 'ignore')556	ignore_target := os.join_path(root, snapshot.ignore_file)557	if os.is_file(ignore_source) && !os.exists(ignore_target) {558		body := os.read_file(ignore_source)!559		os.write_file(ignore_target, body)!560		report.ignore_carried = true561	}562}563564// check_build_files runs in the check phase, before the swap: a repo holding565// both build files is refused while nothing has moved, never left half done.566fn check_build_files(root string) ! {567	if os.is_file(os.join_path(root, kepr_build_file))568		&& os.exists(os.join_path(root, build.config_name)) {569		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')570	}571}572573// rename_build_file finishes the move to `koh.build`, and records it as a save:574// the migration must not leave the working tree dirty with a rename the next575// save would otherwise pick up as unreviewed work.576fn rename_build_file(root string, mut report Report) ! {577	old := os.join_path(root, kepr_build_file)578	if !os.is_file(old) {579		return580	}581	fresh := os.join_path(root, build.config_name)582	os.mv(old, fresh, os.MvParams{})!583	mut store := storage.open(root)!584	defer {585		store.close()586	}587	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{588		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'589	})!590	report.build_renamed = true591}592593// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.594fn bare_hash(value string) string {595	if at := value.index(':') {596		return value[at + 1..]597	}598	return value599}600601// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.602fn base64_decode(encoded string) ![]u8 {603	return base64.decode(encoded)604}