migrate.v

1//! `koh migrate` — convert one Koh 2 repo into a Koh 3 repo, in place.2//!3//! Koh 2 kept one JSON manifest per save plus an object store; Koh 3 keeps the4//! same objects in one SQLite file. The conversion reads the manifests as the5//! source of truth, imports everything into a fresh `.koh`, verifies it against6//! those manifests, and only then swaps it into place. The Koh 2 data is not7//! deleted — it is moved aside as `.koh-v2/`, so `trash .koh` and moving it back8//! undoes the migration entirely.910module migrate1112import encoding.base6413import json214import os15import src.build16import src.events17import src.history18import src.snapshot19import src.storage2021pub const koh2_dir = '.koh'22pub const archive_dir = '.koh-v2'23// Koh 2's build file, renamed to `koh.build` (build.config_name) by the24// migration, which is the name Koh 3 reads.25const kepr_build_file = 'kepr.build'26pub const staging_file = '.koh-migrate'27pub const default_lane = 'dev'2829// One file in a Koh 2 manifest. Small files were inlined as base64 instead of30// being written to the object store, and those entries carry `inline` with no31// `object` key at all — so `inline` decides, and `object` is only the32// alternative.33struct ManifestFile {34	path   string35	hash   string36	size   i6437	object bool38	inline ?string39}4041struct Manifest {42	id            string43	lane          string44	message       string45	timestamp     i6446	parent_id     ?string47	size_bytes    i6448	agent         ?string49	session       ?string50	is_checkpoint bool51	flagged       bool52	flag_reason   ?string53	files         []ManifestFile54}5556pub struct Report {57pub mut:58	root               string59	lane               string60	saves              int61	checkpoints        int62	flags              int63	objects            int64	inlined            int65	head               string66	ignore_carried     bool67	log_md_was_tracked bool68	build_renamed      bool69}7071// run converts the repo at `root`. `lane` picks which line to import: `dev`72// unless told otherwise, because that is where the work is.73pub fn run(root string, lane string) !Report {74	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {75		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')76	}77	manifests := read_manifests(root)!78	if manifests.len == 0 {79		return error('the Koh 2 repo has no manifests to import')80	}81	// `dev` holds everything when it was used at all. A repo that never left82	// `main` — which is what koh2 gave a new repo — imports from main instead83	// of failing on an empty lane.84	mut chosen := lane85	head := lane_head(root, manifests, chosen) or {86		if chosen != default_lane {87			return err88		}89		chosen = other_lane(manifests) or { return err }90		lane_head(root, manifests, chosen)!91	}92	line := ancestry(manifests, head)!93	check_clean_tree(root, head)!9495	mut report := Report{96		root: root97		lane: chosen98		head: head.id99	}100	// Import into a staging file first: nothing is touched until it verifies.101	staging := os.join_path(root, staging_file)102	if os.exists(staging) {103		return error('${staging} already exists — a previous migrate did not finish; move it away and try again')104	}105	mut staged := storage.create_at(staging, root)!106	// The staged file must be closed before it is moved: SQLite keeps committed107	// work in its WAL until the last connection goes away, and moving the file108	// alone would leave that work behind.109	mut failure := ''110	if _ := import_and_verify(staged, root, line, mut report) {111	} else {112		failure = err.msg()113	}114	staged.close()115	if failure != '' {116		return error(failure)117	}118	swap(root, mut report)!119	rename_build_file(root, mut report)!120	report.saves = line.len121	for manifest in line {122		if manifest.is_checkpoint {123			report.checkpoints++124		}125		if manifest.flagged {126			report.flags++127		}128		for file in manifest.files {129			if file.inline != none {130				report.inlined++131			}132		}133	}134	return report135}136137fn read_manifests(root string) ![]Manifest {138	dir := os.join_path(root, koh2_dir, 'snapshots')139	mut manifests := []Manifest{}140	for name in os.ls(dir)! {141		if !name.ends_with('.json') {142			continue143		}144		body := os.read_file(os.join_path(dir, name))!145		manifests << json2.decode[Manifest](body)!146	}147	return manifests148}149150// lane_head finds the newest save on a lane: the state table when it is there,151// otherwise the manifest nothing else claims as a parent.152fn lane_head(root string, manifests []Manifest, lane string) !Manifest {153	if recorded := state_head(root, lane) {154		for manifest in manifests {155			if manifest.id == recorded {156				return manifest157			}158		}159	}160	mut claimed := map[string]bool{}161	for manifest in manifests {162		if parent := manifest.parent_id {163			claimed[parent] = true164		}165	}166	mut candidates := []Manifest{}167	for manifest in manifests {168		if manifest.lane == lane && manifest.id !in claimed {169			candidates << manifest170		}171	}172	if candidates.len == 0 {173		return error('no saves on lane "${lane}"')174	}175	candidates.sort(a.timestamp < b.timestamp)176	return candidates[candidates.len - 1]177}178179// other_lane names the lane that does have saves, when the preferred one is180// empty.181fn other_lane(manifests []Manifest) ?string {182	mut lanes := map[string]bool{}183	for manifest in manifests {184		lanes[manifest.lane] = true185	}186	for lane in ['main', 'dev'] {187		if lane in lanes {188			return lane189		}190	}191	for lane in lanes.keys() {192		return lane193	}194	return none195}196197fn state_head(root string, lane string) ?string {198	db := os.join_path(root, koh2_dir, 'koh.db')199	if !os.is_file(db) {200		return none201	}202	store := storage.connect(db, 1000) or { return none }203	defer {204		store.close()205	}206	mut stmt := store.prepare('SELECT value FROM state WHERE key = ?') or { return none }207	defer {208		stmt.finalize()209	}210	stmt.bind_text(1, 'head_${lane}') or { return none }211	if stmt.step() or { return none } {212		value := stmt.text(0)213		if value != '' {214			return value215		}216	}217	return none218}219220// ancestry walks back from the lane head to the first save, oldest first, and221// refuses a manifest whose parent is missing: a gap would silently lose work.222fn ancestry(manifests []Manifest, head Manifest) ![]Manifest {223	mut by_id := map[string]Manifest{}224	for manifest in manifests {225		by_id[manifest.id] = manifest226	}227	mut line := []Manifest{}228	mut cursor := head229	for {230		line << cursor231		parent := cursor.parent_id or { break }232		if parent == '' {233			break234		}235		next := by_id[parent] or {236			return error('save ${cursor.id} names parent ${parent}, which is not in ${koh2_dir}/snapshots')237		}238		cursor = next239	}240	line.reverse_in_place()241	return line242}243244// check_clean_tree refuses unless the working tree matches the lane head, which245// is the same rule the document states: save with koh2 first.246fn check_clean_tree(root string, head Manifest) ! {247	ignor := snapshot.parse(os.read_file(os.join_path(root, koh2_dir, 'ignore')) or {248		os.read_file(os.join_path(root, snapshot.ignore_file)) or { '' }249	})250	entries := snapshot.walk_raw(root, ignor)!251	mut expected := map[string]string{}252	for file in head.files {253		expected[file.path] = bare_hash(file.hash)254	}255	mut actual := map[string]string{}256	for entry in entries {257		actual[entry.path] = entry.hash258	}259	for path, hash in expected {260		// Koh 2 kept saving files under a folder pattern that Koh 3 reads as261		// covering the folder, so a saved path is checked on disk even when262		// the walk skipped it.263		have := actual[path] or {264			disk_hash(root, path) or {265				return error('${path} is missing from the working tree — save with koh2 first')266			}267		}268		if have != hash {269			return error('${path} does not match the last save — save with koh2 first')270		}271	}272	for path in actual.keys() {273		if path !in expected && !koh2_never_saved(path) {274			return error('${path} is not in the last save — save with koh2 first')275		}276	}277}278279// disk_hash hashes a regular file inside the repo. Koh 2 never followed links,280// nor does Koh 3, so a link on the way to it means the saved file is missing.281fn disk_hash(root string, path string) ?string {282	// Only a plain path inside the repo; a damaged manifest cannot point out.283	if path.starts_with('/') || path.split('/').any(it in ['', '.', '..']) {284		return none285	}286	full := os.join_path(root, path)287	if !os.is_file(full) || os.is_link(full) {288		return none289	}290	snapshot.check_links(root, [snapshot.FileEntry{291		path: path292	}]) or { return none }293	return storage.hash_hex(os.read_bytes(full) or { return none })294}295296// koh2_never_saved mirrors the hard ignores in Koh 2's snapshot.zig that Koh 3297// does not share, so a clean Koh 2 tree reads as clean. Koh 2 prunes a folder298// with such a name too, so every segment counts. The files stay where they are.299fn koh2_never_saved(path string) bool {300	parts := path.split('/')301	return parts[0] == '.kohconfig' || parts[0].starts_with('.kohignore.')302		|| parts.any(it.ends_with('.bundle'))303}304305fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report) ! {306	store.begin()!307	if _ := import_line(store, root, line, mut report) {308		store.commit()!309	} else {310		store.rollback()311		return err312	}313	verify(store, line)!314}315316fn import_line(store &storage.Store, root string, line []Manifest, mut report Report) !bool {317	for manifest in line {318		mut files := []snapshot.FileEntry{}319		for file in manifest.files {320			hash := bare_hash(file.hash)321			if encoded := file.inline {322				data := base64_decode(encoded)!323				stored := store.put_object(data)!324				if stored != hash {325					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')326				}327				report.objects++328			} else if file.object {329				blob := read_object(root, hash)!330				store.put_object_blob(hash, blob)!331				report.objects++332			} else {333				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')334			}335			files << snapshot.FileEntry{336				path: file.path337				hash: hash338			}339		}340		save := history.Save{341			id:            manifest.id342			parent:        manifest.parent_id or { '' }343			message:       manifest.message344			created_at:    manifest.timestamp345			machine:       'koh2'346			is_checkpoint: manifest.is_checkpoint347			size_bytes:    manifest.size_bytes348			agent:         manifest.agent or { '' }349			session:       manifest.session or { '' }350		}351		history.import_save(store, save, files)!352		if manifest.flagged {353			events.flag(store, manifest.id, manifest.flag_reason or { '' })!354		}355		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')356	}357	// The newest save on the imported line becomes head.358	history.set_head(store, line[line.len - 1].id)!359	return true360}361362fn has_path(files []snapshot.FileEntry, path string) bool {363	for file in files {364		if file.path == path {365			return true366		}367	}368	return false369}370371fn read_object(root string, hash string) ![]u8 {372	path := os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])373	return os.read_bytes(path) or {374		return error('object ${hash} is missing from ${koh2_dir}/objects')375	}376}377378fn verify(store &storage.Store, line []Manifest) ! {379	imported := history.save_count(store)!380	if imported != line.len {381		return error('imported ${imported} saves, expected ${line.len}')382	}383	for manifest in line {384		files := history.files_of(store, manifest.id)!385		if files.len != manifest.files.len {386			return error('save ${manifest.id} has ${files.len} files, its manifest has ${manifest.files.len}')387		}388		for i, file in manifest.files {389			hash := bare_hash(file.hash)390			if files[i].path != file.path || files[i].hash != hash {391				return error('save ${manifest.id}: ${files[i].path} does not match its manifest')392			}393			store.get_object(hash) or {394				return error('save ${manifest.id} needs object ${hash}: ${err.msg()}')395			}396		}397	}398}399400// swap puts the imported database in place and moves the Koh 2 data aside.401fn swap(root string, mut report Report) ! {402	old := os.join_path(root, koh2_dir)403	archive := os.join_path(root, archive_dir)404	if os.exists(archive) {405		return error('${archive} already exists — move it away first')406	}407	os.mv(old, archive, os.MvParams{})!408	os.mv(os.join_path(root, staging_file), os.join_path(root, storage.file_name), os.MvParams{})!409	// Koh 2 kept its ignore rules under .koh/; carry them to the root, which is410	// where Koh 3 reads them from.411	ignore_source := os.join_path(archive, 'ignore')412	ignore_target := os.join_path(root, snapshot.ignore_file)413	if os.is_file(ignore_source) && !os.exists(ignore_target) {414		body := os.read_file(ignore_source)!415		os.write_file(ignore_target, body)!416		report.ignore_carried = true417	}418}419420// rename_build_file finishes the move to `koh.build`, and records it as a save:421// the migration must not leave the working tree dirty with a rename the next422// save would otherwise pick up as unreviewed work.423fn rename_build_file(root string, mut report Report) ! {424	old := os.join_path(root, kepr_build_file)425	if !os.is_file(old) {426		return427	}428	fresh := os.join_path(root, build.config_name)429	if os.exists(fresh) {430		return error('both ${kepr_build_file} and ${build.config_name} exist — keep the one you want and migrate again')431	}432	os.mv(old, fresh, os.MvParams{})!433	mut store := storage.open(root)!434	defer {435		store.close()436	}437	history.save_worktree(store, root, snapshot.load(root), history.SaveInput{438		message: 'migrate: rename ${kepr_build_file} to ${build.config_name}'439	})!440	report.build_renamed = true441}442443// bare_hash drops the "blake3:" prefix Koh 2 wrote in its manifests.444fn bare_hash(value string) string {445	if at := value.index(':') {446		return value[at + 1..]447	}448	return value449}450451// base64_decode reads the inlined form Koh 2 used for files under 512 bytes.452fn base64_decode(encoded string) ![]u8 {453	return base64.decode(encoded)454}