migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.history18import src.snapshot19import src.storage2021pub const koh2_dir = '.koh'22pub const archive_dir = '.koh-v2'23// Koh 2's build file, renamed to `koh.build` (build.config_name) by the24// migration, which is the name Koh 3 reads.25const kepr_build_file = 'kepr.build'26pub const staging_file = '.koh-migrate'27pub const default_lane = 'dev'2829// One file in a Koh 2 manifest. Small files were inlined as base64 instead of30// being written to the object store, and those entries carry `inline` with no31// `object` key at all — so `inline` decides, and `object` is only the32// alternative.33struct ManifestFile {34	path   string35	hash   string36	size   i6437	object bool38	inline ?string39}4041struct Manifest {42	id            string43	lane          string44	message       string45	timestamp     i6446	parent_id     ?string47	size_bytes    i6448	agent         ?string49	session       ?string50	is_checkpoint bool51	flagged       bool52	flag_reason   ?string53	files         []ManifestFile54}5556pub struct Report {57pub mut:58	root               string59	lane               string60	saves              int61	checkpoints        int62	flags              int63	objects            int64	inlined            int65	head               string66	ignore_carried     bool67	log_md_was_tracked bool68	build_renamed      bool69}7071// run converts the repo at `root`. `lane` picks which line to import: `dev`72// unless told otherwise, because that is where the work is.73pub fn run(root string, lane string) !Report {74	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {75		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')76	}77	manifests := read_manifests(root)!78	if manifests.len == 0 {79		return error('the Koh 2 repo has no manifests to import')80	}81	// `dev` holds everything when it was used at all. A repo that never left82	// `main` — which is what koh2 gave a new repo — imports from main instead83	// of failing on an empty lane.84	mut chosen := lane85	head := lane_head(root, manifests, chosen) or {86		if chosen != default_lane {87			return err88		}89		chosen = other_lane(manifests) or { return err }90		lane_head(root, manifests, chosen)!91	}92	line := ancestry(manifests, head)!93	check_clean_tree(root, head)!94	check_build_files(root)!9596	mut report := Report{97		root: root98		lane: chosen99		head: head.id100	}101	// Import into a staging file first: nothing is touched until it verifies.102	staging := os.join_path(root, staging_file)103	if os.exists(staging) {104		return error('${staging} already exists — a previous migrate did not finish; move it away and try again')105	}106	mut staged := storage.create_at(staging, root)!107	// The staged file must be closed before it is moved: SQLite keeps committed108	// work in its WAL until the last connection goes away, and moving the file109	// alone would leave that work behind.110	mut failure := ''111	if _ := import_and_verify(staged, root, line, mut report) {112	} else {113		failure = err.msg()114	}115	staged.close()116	if failure != '' {117		return error(failure)118	}119	swap(root, mut report)!120	rename_build_file(root, mut report)!121	report.saves = line.len122	for manifest in line {123		if manifest.is_checkpoint {124			report.checkpoints++125		}126		if manifest.flagged {127			report.flags++128		}129		for file in manifest.files {130			if file.inline != none {131				report.inlined++132			}133		}134	}135	return report136}137138fn read_manifests(root string) ![]Manifest {139	dir := os.join_path(root, koh2_dir, 'snapshots')140	mut manifests := []Manifest{}141	for name in os.ls(dir)! {142		if !name.ends_with('.json') {143			continue144		}145		body := os.read_file(os.join_path(dir, name))!146		manifests << json2.decode[Manifest](body)!147	}148	return manifests149}150151// lane_head finds the newest save on a lane: the state table when it is there,152// otherwise the manifest nothing else claims as a parent.153fn lane_head(root string, manifests []Manifest, lane string) !Manifest {154	if recorded := state_head(root, lane) {155		for manifest in manifests {156			if manifest.id == recorded {157				return manifest158			}159		}160	}161	mut claimed := map[string]bool{}162	for manifest in manifests {163		if parent := manifest.parent_id {164			claimed[parent] = true165		}166	}167	mut candidates := []Manifest{}168	for manifest in manifests {169		if manifest.lane == lane && manifest.id !in claimed {170			candidates << manifest171		}172	}173	if candidates.len == 0 {174		return error('no saves on lane "${lane}"')175	}176	candidates.sort(a.timestamp < b.timestamp)177	return candidates[candidates.len - 1]178}179180// other_lane names the lane that does have saves, when the preferred one is181// empty.182fn other_lane(manifests []Manifest) ?string {183	mut lanes := map[string]bool{}184	for manifest in manifests {185		lanes[manifest.lane] = true186	}187	for lane in ['main', 'dev'] {188		if lane in lanes {189			return lane190		}191	}192	for lane in lanes.keys() {193		return lane194	}195	return none196}197198fn state_head(root string, lane string) ?string {199	db := os.join_path(root, koh2_dir, 'koh.db')200	if !os.is_file(db) {201		return none202	}203	store := storage.connect(db, 1000) or { return none }204	defer {205		store.close()206	}207	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }208	defer {209		stmt.finalize()210	}211	stmt.bind_text(1, 'head_${lane}') or { return none }212	if stmt.step() or { return none } {213		value := stmt.text(0)214		if value != '' {215			return value216		}217	}218	return none219}220221// ancestry walks back from the lane head to the first save, oldest first, and222// refuses a manifest whose parent is missing: a gap would silently lose work.223fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {224	mut by_id := map[string]Manifest{}225	for manifest in manifests {226		by_id[manifest.id] = manifest227	}228	mut line := []Manifest{}229	mut cursor := head230	for {231		line << cursor232		parent := cursor.parent_id or { break }233		if parent == '' {234			break235		}236		next := by_id[parent] or {237			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')238		}239		cursor = next240	}241	line.reverse_in_place()242	return line243}244245// check_clean_tree refuses unless the working tree matches the lane head, which246// is the same rule the document states: save with koh2 first.247fn check_clean_tree(root string, head Manifest) ! {248	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {249		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }250	})251	entries := snapshot.walk_raw(root, ignor)!252	mut expected := map[string]string{}253	for file in head.files {254		expected[file.path] = bare_hash(file.hash)255	}256	mut actual := map[string]string{}257	for entry in entries {258		actual[entry.path] = entry.hash259	}260	for path, hash in expected {261		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as262		// covering the folder, so a saved path is checked on disk even when263		// the walk skipped it.264		have := actual[path] or {265			disk_hash(root, path) or {266				return error('${path} is missing from the working tree — save with koh2 first')267			}268		}269		if have != hash {270			return error('${path} does not match the last save — save with koh2 first')271		}272	}273	for path in actual.keys() {274		if path !in expected && !koh2_never_saved(path) {275			return error('${path} is not in the last save — save with koh2 first')276		}277	}278}279280// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,281// nor does Koh 3, so a link on the way to it means the saved file is missing.282fn disk_hash(root string, path string) ?string {283	// Only a plain path inside the repo; a damaged manifest cannot point out.284	if path.starts_with('/') || path.split('/').any(it in ['', '.', '..']) {285		return none286	}287	full := os.join_path(root, path)288	if !os.is_file(full) || os.is_link(full) {289		return none290	}291	snapshot.check_links(root, [snapshot.FileEntry{292		path: path293	}]) or { return none }294	return storage.hash_hex(os.read_bytes(full) or { return none })295}296297// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3298// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder299// with such a name too, so every segment counts. The files stay where they are.300fn koh2_never_saved(path string) bool {301	parts := path.split('/')302	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')303		|| parts.any(it.ends_with('.bundle'))304}305306fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report) ! {307	store.begin()!308	if _ := import_line(store, root, line, mut report) {309		store.commit()!310	} else {311		store.rollback()312		return err313	}314	verify(store, line)!315}316317fn import_line(store &storage.Store, root string, line []Manifest, mut report Report) !bool {318	for manifest in line {319		mut files := []snapshot.FileEntry{}320		for file in manifest.files {321			hash := bare_hash(file.hash)322			if encoded := file.inline {323				data := base64_decode(encoded)!324				stored := store.put_object(data)!325				if stored != hash {326					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')327				}328				report.objects++329			} else if file.object {330				blob := read_object(root, hash)!331				store.put_object_blob(hash, blob)!332				report.objects++333			} else {334				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')335			}336			files << snapshot.FileEntry{337				path: file.path338				hash: hash339			}340		}341		save := history.Save{342			id:            manifest.id343			parent:        manifest.parent_id or { '' }344			message:       manifest.message345			created_at:    manifest.timestamp346			machine:       'koh2'347			is_checkpoint: manifest.is_checkpoint348			size_bytes:    manifest.size_bytes349			agent:         manifest.agent or { '' }350			session:       manifest.session or { '' }351		}352		history.import_save(store, save, files)!353		if manifest.flagged {354			events.flag(store, manifest.id, manifest.flag_reason or { '' })!355		}356		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')357	}358	// The newest save on the imported line becomes head.359	history.set_head(store, line[line.len - 1].id)!360	return true361}362363fn has_path(files []snapshot.FileEntry, path string) bool {364	for file in files {365		if file.path == path {366			return true367		}368	}369	return false370}371372fn read_object(root string, hash string) ![]u8 {373	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])374	return os.read_bytes(path) or {375		return error('object ${hash} is missing from ${koh2_dir}/objects')376	}377}378379fn verify(store &storage.Store, line []Manifest) ! {380	imported := history.save_count(store)!381	if imported != line.len {382		return error('imported ${imported} saves, expected ${line.len}')383	}384	for manifest in line {385		files := history.files_of(store, manifest.id)!386		if files.len != manifest.files.len {387			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')388		}389		for i, file in manifest.files {390			hash := bare_hash(file.hash)391			if files[i].path != file.path || files[i].hash != hash {392				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')393			}394			store.get_object(hash) or {395				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')396			}397		}398	}399}400401// swap puts the imported database in place and moves the Koh 2 data aside.402fn swap(root string, mut report Report) ! {403	old := os.join_path(root, koh2_dir)404	archive := os.join_path(root, archive_dir)405	if os.exists(archive) {406		return error('${archive} already exists — move it away first')407	}408	os.mv(old, archive, os.MvParams{})!409	os.mv(os.join_path(root, staging_file), os.join_path(root, storage.file_name), os.MvParams{})!410	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is411	// where Koh 3 reads them from.412	ignore_source := os.join_path(archive, 'ignore')413	ignore_target := os.join_path(root, snapshot.ignore_file)414	if os.is_file(ignore_source) && !os.exists(ignore_target) {415		body := os.read_file(ignore_source)!416		os.write_file(ignore_target, body)!417		report.ignore_carried = true418	}419}420421// check_build_files runs in the check phase, before the swap: a repo holding422// both build files is refused while nothing has moved, never left half done.423fn check_build_files(root string) ! {424	if os.is_file(os.join_path(root, kepr_build_file))425		&& os.exists(os.join_path(root, build.config_name)) {426		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')427	}428}429430// rename_build_file finishes the move to `koh.build`, and records it as a save:431// the migration must not leave the working tree dirty with a rename the next432// save would otherwise pick up as unreviewed work.433fn rename_build_file(root string, mut report Report) ! {434	old := os.join_path(root, kepr_build_file)435	if !os.is_file(old) {436		return437	}438	fresh := os.join_path(root, build.config_name)439	os.mv(old, fresh, os.MvParams{})!440	mut store := storage.open(root)!441	defer {442		store.close()443	}444	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{445		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'446	})!447	report.build_renamed = true448}449450// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.451fn bare_hash(value string) string {452	if at := value.index(':') {453		return value[at + 1..]454	}455	return value456}457458// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.459fn base64_decode(encoded string) ![]u8 {460	return base64.decode(encoded)461}