migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.fmt18import src.history19import src.snapshot20import src.storage2122pub const koh2_dir = '.koh'23pub const archive_dir = '.koh-v2'24// Koh 2's build file, renamed to `koh.build` (build.config_name) by the25// migration, which is the name Koh 3 reads.26const kepr_build_file = 'kepr.build'27pub const draft_file = '.koh-migrate'28pub const default_lane = 'dev'2930// One file in a Koh 2 manifest. Small files were inlined as base64 instead of31// being written to the object store, and those entries carry `inline` with no32// `object` key at all — so `inline` decides, and `object` is only the33// alternative.34struct ManifestFile {35	path   string36	hash   string37	size   i6438	object bool39	inline ?string40}4142struct Manifest {43	id            string44	lane          string45	message       string46	timestamp     i6447	parent_id     ?string48	size_bytes    i6449	agent         ?string50	session       ?string51	is_checkpoint bool52	flagged       bool53	flag_reason   ?string54mut:55	files []ManifestFile56}5758pub struct Report {59pub mut:60	root               string61	lane               string62	saves              int63	checkpoints        int64	flags              int65	objects            int66	inlined            int67	head               string68	ignore_carried     bool69	log_md_was_tracked bool70	build_renamed      bool71	unsafe_paths       []string72}7374// Plan is what migrate found and will do, shown before anything is written.75pub struct Plan {76	root         string77	line         []Manifest78	unsafe_paths []string79	found        map[string][]u880pub:81	lost          []Lost82	lane          string83	saves         int84	checkpoints   int85	objects       int86	last_saved    i6487	renames_build bool88}8990// Lost is one file version Koh 2's saves list but its store no longer holds.91// Those saves come over without it, flagged, unless `--with` hands it back.92pub struct Lost {93pub mut:94	path  string95	hash  string96	size  i6497	saves []string98	first i6499	last  i64100}101102// Progress hears how far a stage ("importing", "checking") has come.103pub type Progress = fn (stage string, done int, total int)104105// prepare reads the repo at `root` and checks it can be converted, writing106// nothing. `lane` picks which line to import: `dev` unless told otherwise,107// because that is where the work is.108pub fn prepare(root string, lane string, with []string) !Plan {109	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {110		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')111	}112	manifests := read_manifests(root)!113	if manifests.len == 0 {114		return error('the Koh 2 repo has no manifests to import')115	}116	// `dev` holds everything when it was used at all. A repo that never left117	// `main` — which is what koh2 gave a new repo — imports from main instead118	// of failing on an empty lane.119	mut chosen := lane120	head := lane_head(root, manifests, chosen) or {121		if chosen != default_lane {122			return err123		}124		chosen = other_lane(manifests) or { return err }125		lane_head(root, manifests, chosen)!126	}127	mut line := ancestry(manifests, head)!128	unsafe_paths := drop_unsafe_paths(mut line)129	found := versions_by_hash(with)!130	lost := drop_lost_objects(root, mut line, found)131	check_clean_tree(root, line[line.len - 1])!132	check_build_files(root)!133	mut hashes := map[string]bool{}134	for manifest in line {135		for file in manifest.files {136			hashes[file.hash] = true137		}138	}139	return Plan{140		root:          root141		line:          line142		unsafe_paths:  unsafe_paths143		found:         found144		lost:          lost145		lane:          chosen146		saves:         line.len147		checkpoints:   line.filter(it.is_checkpoint).len148		objects:       hashes.len149		last_saved:    head.timestamp150		renames_build: os.is_file(os.join_path(root, kepr_build_file))151	}152}153154// run converts the repo `prepare` read, telling `progress` as it goes.155pub fn run(plan Plan, progress Progress) !Report {156	root := plan.root157	line := plan.line158	mut report := Report{159		root:         root160		lane:         plan.lane161		head:         line[line.len - 1].id162		unsafe_paths: plan.unsafe_paths163	}164	// Import into a draft file first: nothing is touched until it verifies.165	draft_path := os.join_path(root, draft_file)166	if os.exists(draft_path) {167		return error('${draft_path} already exists — a previous migrate did not finish; move it away and try again')168	}169	mut draft := storage.create_at(draft_path, root)!170	// The draft must be closed before it is moved: SQLite keeps finished171	// work in its WAL until the last connection goes away, and moving the file172	// alone would leave that work behind.173	mut failure := ''174	if _ := import_and_verify(draft, plan, mut report, progress) {175		report.objects = int(draft.object_count() or { 0 })176	} else {177		failure = err.msg()178	}179	draft.close()180	if failure != '' {181		// The draft is migrate's own; leaving it would refuse the next attempt.182		for suffix in ['', '-wal', '-shm'] {183			os.rm(draft_path + suffix) or {}184		}185		return error(failure)186	}187	swap(root, mut report)!188	rename_build_file(root, mut report)!189	report.saves = line.len190	report.checkpoints = plan.checkpoints191	for manifest in line {192		if manifest.flagged {193			report.flags++194		}195		for file in manifest.files {196			if file.inline != none {197				report.inlined++198			}199		}200	}201	return report202}203204fn read_manifests(root string) ![]Manifest {205	dir := os.join_path(root, koh2_dir, 'snapshots')206	mut manifests := []Manifest{}207	for name in os.ls(dir)! {208		if !name.ends_with('.json') {209			continue210		}211		body := os.read_file(os.join_path(dir, name))!212		manifests << json2.decode[Manifest](body)!213	}214	return manifests215}216217// lane_head finds the newest save on a lane: the state table when it is there,218// otherwise the manifest nothing else claims as a parent.219fn lane_head(root string, manifests []Manifest, lane string) !Manifest {220	if recorded := state_head(root, lane) {221		for manifest in manifests {222			if manifest.id == recorded {223				return manifest224			}225		}226	}227	mut claimed := map[string]bool{}228	for manifest in manifests {229		if parent := manifest.parent_id {230			claimed[parent] = true231		}232	}233	mut candidates := []Manifest{}234	for manifest in manifests {235		if manifest.lane == lane && manifest.id !in claimed {236			candidates << manifest237		}238	}239	if candidates.len == 0 {240		return error('no saves on lane "${lane}"')241	}242	candidates.sort(a.timestamp < b.timestamp)243	return candidates[candidates.len - 1]244}245246// other_lane names the lane that does have saves, when the preferred one is247// empty.248fn other_lane(manifests []Manifest) ?string {249	mut lanes := map[string]bool{}250	for manifest in manifests {251		lanes[manifest.lane] = true252	}253	for lane in ['main', 'dev'] {254		if lane in lanes {255			return lane256		}257	}258	for lane in lanes.keys() {259		return lane260	}261	return none262}263264fn state_head(root string, lane string) ?string {265	db := os.join_path(root, koh2_dir, 'koh.db')266	if !os.is_file(db) {267		return none268	}269	store := storage.connect(db, 1000) or { return none }270	defer {271		store.close()272	}273	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }274	defer {275		stmt.finalize()276	}277	stmt.bind_text(1, 'head_${lane}') or { return none }278	if stmt.step() or { return none } {279		value := stmt.text(0)280		if value != '' {281			return value282		}283	}284	return none285}286287// ancestry walks back from the lane head to the first save, oldest first, and288// refuses a manifest whose parent is missing: a gap would silently lose work.289fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {290	mut by_id := map[string]Manifest{}291	for manifest in manifests {292		by_id[manifest.id] = manifest293	}294	mut line := []Manifest{}295	mut cursor := head296	for {297		line << cursor298		parent := cursor.parent_id or { break }299		if parent == '' {300			break301		}302		next := by_id[parent] or {303			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')304		}305		cursor = next306	}307	line.reverse_in_place()308	return line309}310311// check_clean_tree refuses unless the working tree matches the lane head, which312// is the same rule the document states: save with koh2 first.313fn check_clean_tree(root string, head Manifest) ! {314	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {315		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }316	})317	entries := snapshot.walk_raw(root, ignor)!318	mut expected := map[string]string{}319	for file in head.files {320		expected[file.path] = bare_hash(file.hash)321	}322	mut actual := map[string]string{}323	for entry in entries {324		actual[entry.path] = entry.hash325	}326	for path, hash in expected {327		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as328		// covering the folder, so a saved path is checked on disk even when329		// the walk skipped it.330		have := actual[path] or {331			disk_hash(root, path) or {332				return error('${path} is missing from the working tree — save with koh2 first')333			}334		}335		if have != hash {336			return error('${path} does not match the last save — save with koh2 first')337		}338	}339	for path in actual.keys() {340		if path !in expected && !koh2_never_saved(path) {341			return error('${path} is not in the last save — save with koh2 first')342		}343	}344}345346// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,347// nor does Koh 3, so a link on the way to it means the saved file is missing.348fn disk_hash(root string, path string) ?string {349	// Only a legal path inside the repo; a damaged manifest cannot point out.350	if !snapshot.legal_path(path) {351		return none352	}353	full := os.join_path(root, path)354	if !os.is_file(full) || os.is_link(full) {355		return none356	}357	snapshot.check_links(root, [snapshot.FileEntry{358		path: path359	}]) or { return none }360	return storage.hash_hex(os.read_bytes(full) or { return none })361}362363// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3364// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder365// with such a name too, so every segment counts. The files stay where they are.366fn koh2_never_saved(path string) bool {367	parts := path.split('/')368	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')369		|| parts.any(it.ends_with('.bundle'))370}371372// drop_unsafe_paths removes, by the shared path rule, every file Koh 3 would373// refuse to restore, and names each one once for the report.374fn drop_unsafe_paths(mut line []Manifest) []string {375	mut dropped := map[string]bool{}376	for mut manifest in line {377		for file in manifest.files {378			if !snapshot.legal_path(file.path) {379				dropped[snapshot.escape_path(file.path)] = true380			}381		}382		manifest.files = manifest.files.filter(snapshot.legal_path(it.path))383	}384	mut names := dropped.keys()385	names.sort()386	return names387}388389// versions_by_hash reads the files `--with` names, so a version Koh 2 lost can390// be recognised by its contents wherever it was found.391fn versions_by_hash(paths []string) !map[string][]u8 {392	mut found := map[string][]u8{}393	for path in paths {394		data := os.read_bytes(path) or { return error('cannot read ${path}: ${err.msg()}') }395		found[storage.hash_hex(data)] = data396	}397	return found398}399400// drop_lost_objects takes out of the line every file whose object Koh 2 no401// longer holds and nobody handed back, and says which, where and when.402fn drop_lost_objects(root string, mut line []Manifest, found map[string][]u8) []Lost {403	mut lost := map[string]Lost{}404	for mut manifest in line {405		mut kept := []ManifestFile{}406		for file in manifest.files {407			hash := bare_hash(file.hash)408			if file.inline != none || !file.object || hash in found409				|| os.is_file(os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])) {410				kept << file411				continue412			}413			key := '${file.path}\n${hash}'414			mut seen := lost[key] or {415				Lost{416					path:  file.path417					hash:  hash418					size:  file.size419					first: manifest.timestamp420				}421			}422			seen.saves << manifest.id423			seen.last = manifest.timestamp424			lost[key] = seen425		}426		manifest.files = kept427	}428	mut all := lost.values()429	all.sort(a.path < b.path)430	return all431}432433fn import_and_verify(store &storage.Store, plan Plan, mut report Report, progress Progress) ! {434	store.begin()!435	if _ := import_line(store, plan, mut report, progress) {436		store.end()!437	} else {438		store.rollback()439		return err440	}441	verify(store, plan.line, progress)!442}443444fn import_line(store &storage.Store, plan Plan, mut report Report, progress Progress) !bool {445	line := plan.line446	mut lost_at := map[string][]string{}447	for lost in plan.lost {448		for id in lost.saves {449			lost_at[id] << lost.path450		}451	}452	for i, manifest in line {453		progress('importing', i + 1, line.len)454		mut files := []snapshot.FileEntry{}455		for file in manifest.files {456			hash := bare_hash(file.hash)457			if encoded := file.inline {458				data := base64_decode(encoded)!459				stored := store.put_object(data)!460				if stored != hash {461					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')462				}463			} else if data := plan.found[hash] {464				store.put_object(data)!465			} else if file.object {466				blob := read_object(plan.root, hash)!467				store.put_object_blob(hash, blob)!468			} else {469				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')470			}471			files << snapshot.FileEntry{472				path: file.path473				hash: hash474			}475		}476		save := history.Save{477			id:            manifest.id478			parent:        manifest.parent_id or { '' }479			message:       manifest.message480			created_at:    manifest.timestamp481			machine:       'koh2'482			is_checkpoint: manifest.is_checkpoint483			size_bytes:    manifest.size_bytes484			agent:         manifest.agent or { '' }485			session:       manifest.session or { '' }486		}487		history.import_save(store, save, files)!488		if paths := lost_at[manifest.id] {489			// Said on the save itself, so loading it later explains the gap.490			reason := 'migrate: Koh 2 had lost ${paths.join(', ')} at this save'491			events.flag(store, manifest.id, if manifest.flagged {492				'${manifest.flag_reason or { '' }}; ${reason}'493			} else {494				reason495			})!496		} else if manifest.flagged {497			events.flag(store, manifest.id, manifest.flag_reason or { '' })!498		}499		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')500	}501	// The newest save on the imported line becomes head.502	history.set_head(store, line[line.len - 1].id)!503	return true504}505506fn has_path(files []snapshot.FileEntry, path string) bool {507	for file in files {508		if file.path == path {509			return true510		}511	}512	return false513}514515fn read_object(root string, hash string) ![]u8 {516	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])517	return os.read_bytes(path) or {518		return error('object ${hash} is missing from ${koh2_dir}/objects')519	}520}521522fn verify(store &storage.Store, line []Manifest, progress Progress) ! {523	imported := history.save_count(store)!524	if imported != line.len {525		return error('imported ${fmt.plural(int(imported), 'save')}, expected ${line.len}')526	}527	for n, manifest in line {528		progress('checking', n + 1, line.len)529		files := history.files_of(store, manifest.id)!530		if files.len != manifest.files.len {531			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')532		}533		// By path, not by position: Koh 2 did not always list a manifest's files in order.534		mut stored := map[string]string{}535		for file in files {536			stored[file.path] = file.hash537		}538		for file in manifest.files {539			hash := bare_hash(file.hash)540			if stored[file.path] or { '' } != hash {541				return error('save ${manifest.id}: ${file.path} does not match its manifest')542			}543			store.get_object(hash) or {544				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')545			}546		}547	}548}549550// swap puts the imported database in place and moves the Koh 2 data aside.551fn swap(root string, mut report Report) ! {552	old := os.join_path(root, koh2_dir)553	archive := os.join_path(root, archive_dir)554	if os.exists(archive) {555		return error('${archive} already exists — move it away first')556	}557	os.mv(old, archive, os.MvParams{})!558	os.mv(os.join_path(root, draft_file), os.join_path(root, storage.file_name), os.MvParams{})!559	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is560	// where Koh 3 reads them from.561	ignore_source := os.join_path(archive, 'ignore')562	ignore_target := os.join_path(root, snapshot.ignore_file)563	if os.is_file(ignore_source) && !os.exists(ignore_target) {564		body := os.read_file(ignore_source)!565		os.write_file(ignore_target, body)!566		report.ignore_carried = true567	}568}569570// check_build_files runs in the check phase, before the swap: a repo holding571// both build files is refused while nothing has moved, never left half done.572fn check_build_files(root string) ! {573	if os.is_file(os.join_path(root, kepr_build_file))574		&& os.exists(os.join_path(root, build.config_name)) {575		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')576	}577}578579// rename_build_file finishes the move to `koh.build`, and records it as a save:580// the migration must not leave the working tree dirty with a rename the next581// save would otherwise pick up as unreviewed work.582fn rename_build_file(root string, mut report Report) ! {583	old := os.join_path(root, kepr_build_file)584	if !os.is_file(old) {585		return586	}587	fresh := os.join_path(root, build.config_name)588	os.mv(old, fresh, os.MvParams{})!589	mut store := storage.open(root)!590	defer {591		store.close()592	}593	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{594		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'595	})!596	report.build_renamed = true597}598599// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.600fn bare_hash(value string) string {601	if at := value.index(':') {602		return value[at + 1..]603	}604	return value605}606607// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.608fn base64_decode(encoded string) ![]u8 {609	return base64.decode(encoded)610}