migrate past file versions Koh 2 lost: name them before asking, take them back from a backup with --with, or bring those saves over without them, flagged; a failed migrate removes its own draft (3.2.14)

@@ -1 +1 @@1-3.2.131+3.2.14

@@ -165,7 +165,7 @@165165     steal)           _arguments '--list' '--mine' '--theirs' '--json' ;;166166     strip)           _arguments '--save[one save]:save:_koh3_ids' '*:path:_files' ;;167167     doctor)          _arguments '--backup[copy .koh to]:path:_files' ;;168-    migrate)         _arguments '--lane[Koh 2 lane]:lane:(dev main)' '--yes[skip the question]' '--json' ;;168+    migrate)         _arguments '--lane[Koh 2 lane]:lane:(dev main)' '*--with[a lost version, from a backup]:file:_files' '--yes[skip the question]' '--json' ;;169169     man)             _describe 'command' commands ;;170170   esac171171 }

@@ -12,7 +12,7 @@1212 1313 pub fn cmd_migrate(raw []string) int {1414 	args := parse('migrate', raw, Options{15-		values: ['lane']15+		values: ['lane', 'with']1616 		flags:  ['json', 'yes']1717 	}) or { return usage_error(err.msg()) }1818 	mut lane := args.value('lane')@@ -22,9 +22,10 @@2222 	if lane !in ['dev', 'main'] {2323 		return usage_error('--lane takes dev or main, not "${lane}"')2424 	}25-	plan := migrate.prepare(os.getwd(), lane) or { return refuse(args.json(), err.msg()) }25+	plan := migrate.prepare(os.getwd(), lane, args.values_of('with')) or { return refuse(args.json(), err.msg()) }2626 	if !args.json() {2727 		note('Koh 2 repo · lane ${plan.lane} · ${fmt.plural(plan.saves, 'save')} (${fmt.plural(plan.checkpoints, 'checkpoint')}) · ${fmt.plural(plan.objects, 'object')} · last save ${time.unix(plan.last_saved).ymmdd()}')28+		show_lost(plan.lost)2829 		note('this will:')2930 		note('  · copy the history into one .koh file and check every save')3031 		if plan.renames_build {@@ -50,6 +51,7 @@5051 			head:         report.head5152 			ignore_moved: report.ignore_carried5253 			build_file:   report.build_renamed54+			lost:         plan.lost.map(it.path)5355 			unsafe_paths: report.unsafe_paths5456 		}))5557 		return 0@@ -84,5 +86,43 @@8486 	head         string8587 	ignore_moved bool8688 	build_file   bool89+	lost         []string8790 	unsafe_paths []string91+}92+93+// show_lost names each file Koh 2 lost versions of, where they were, and what94+// can still be done about it before anything is written.95+fn show_lost(lost []migrate.Lost) {96+	if lost.len == 0 {97+		return98+	}99+	mut paths := []string{}100+	for gone in lost {101+		if gone.path !in paths {102+			paths << gone.path103+		}104+	}105+	warn('Koh 2 has lost ${fmt.plural(lost.len, 'file version')} its saves still list:')106+	for path in paths {107+		versions := lost.filter(it.path == path)108+		mut saves := 0109+		mut first := versions[0].first110+		mut last := versions[0].last111+		for gone in versions {112+			saves += gone.saves.len113+			first = if gone.first < first { gone.first } else { first }114+			last = if gone.last > last { gone.last } else { last }115+		}116+		mut when := time.unix(first).ymmdd()117+		if time.unix(last).ymmdd() != when {118+			when += ' to ${time.unix(last).ymmdd()}'119+		}120+		note('    ${path} · ${fmt.plural(versions.len, 'version')} · in ${fmt.plural(saves, 'save')} · ${when}')121+	}122+	one := lost.len == 1123+	them := if one { 'that version' } else { 'any of those versions' }124+	it := if one { 'it' } else { 'them' }125+	note("  if you have ${them} — a backup, Time Machine, another machine's copy —")126+	note('  hand ${it} back and Koh uses ${it} instead: koh migrate --with <file> …')127+	note('  continuing brings those saves over without ${if one { 'it' } else { 'what was lost' }}, each one flagged')88128 }

@@ -75,7 +75,9 @@7575 	root         string7676 	line         []Manifest7777 	unsafe_paths []string78+	found        map[string][]u87879 pub:80+	lost          []Lost7981 	lane          string8082 	saves         int8183 	checkpoints   int@@ -84,13 +86,25 @@8486 	renames_build bool8587 }8688 89+// Lost is one file version Koh 2's saves list but its store no longer holds.90+// Those saves come over without it, flagged, unless `--with` hands it back.91+pub struct Lost {92+pub mut:93+	path  string94+	hash  string95+	size  i6496+	saves []string97+	first i6498+	last  i6499+}100+87101 // Progress hears how far a stage ("importing", "checking") has come.88102 pub type Progress = fn (stage string, done int, total int)89103 90104 // prepare reads the repo at `root` and checks it can be converted, writing91105 // nothing. `lane` picks which line to import: `dev` unless told otherwise,92106 // because that is where the work is.93-pub fn prepare(root string, lane string) !Plan {107+pub fn prepare(root string, lane string, with []string) !Plan {94108 	if !os.is_dir(os.join_path(root, koh2_dir, 'snapshots')) {95109 		return error('no Koh 2 repo here — migrate expects a .koh/ folder with snapshots/')96110 	}@@ -111,6 +125,8 @@111125 	}112126 	mut line := ancestry(manifests, head)!113127 	unsafe_paths := drop_unsafe_paths(mut line)128+	found := versions_by_hash(with)!129+	lost := drop_lost_objects(root, mut line, found)114130 	check_clean_tree(root, line[line.len - 1])!115131 	check_build_files(root)!116132 	mut hashes := map[string]bool{}@@ -123,6 +139,8 @@123139 		root:          root124140 		line:          line125141 		unsafe_paths:  unsafe_paths142+		found:         found143+		lost:          lost126144 		lane:          chosen127145 		saves:         line.len128146 		checkpoints:   line.filter(it.is_checkpoint).len@@ -152,13 +170,17 @@152170 	// work in its WAL until the last connection goes away, and moving the file153171 	// alone would leave that work behind.154172 	mut failure := ''155-	if _ := import_and_verify(draft, root, line, mut report, progress) {173+	if _ := import_and_verify(draft, plan, mut report, progress) {156174 		report.objects = int(draft.object_count() or { 0 })157175 	} else {158176 		failure = err.msg()159177 	}160178 	draft.close()161179 	if failure != '' {180+		// The draft is migrate's own; leaving it would refuse the next attempt.181+		for suffix in ['', '-wal', '-shm'] {182+			os.rm(draft_path + suffix) or {}183+		}162184 		return error(failure)163185 	}164186 	swap(root, mut report)!@@ -363,18 +385,69 @@363385 	return names364386 }365387 366-fn import_and_verify(store &storage.Store, root string, line []Manifest, mut report Report, progress Progress) ! {388+// versions_by_hash reads the files `--with` names, so a version Koh 2 lost can389+// be recognised by its contents wherever it was found.390+fn versions_by_hash(paths []string) !map[string][]u8 {391+	mut found := map[string][]u8{}392+	for path in paths {393+		data := os.read_bytes(path) or { return error('cannot read ${path}: ${err.msg()}') }394+		found[storage.hash_hex(data)] = data395+	}396+	return found397+}398+399+// drop_lost_objects takes out of the line every file whose object Koh 2 no400+// longer holds and nobody handed back, and says which, where and when.401+fn drop_lost_objects(root string, mut line []Manifest, found map[string][]u8) []Lost {402+	mut lost := map[string]Lost{}403+	for mut manifest in line {404+		mut kept := []ManifestFile{}405+		for file in manifest.files {406+			hash := bare_hash(file.hash)407+			if file.inline != none || !file.object || hash in found408+				|| os.is_file(os.join_path(root, koh2_dir, 'objects', hash[..2], hash[2..])) {409+				kept << file410+				continue411+			}412+			key := '${file.path}\n${hash}'413+			mut seen := lost[key] or {414+				Lost{415+					path:  file.path416+					hash:  hash417+					size:  file.size418+					first: manifest.timestamp419+				}420+			}421+			seen.saves << manifest.id422+			seen.last = manifest.timestamp423+			lost[key] = seen424+		}425+		manifest.files = kept426+	}427+	mut all := lost.values()428+	all.sort(a.path < b.path)429+	return all430+}431+432+fn import_and_verify(store &storage.Store, plan Plan, mut report Report, progress Progress) ! {367433 	store.begin()!368-	if _ := import_line(store, root, line, mut report, progress) {434+	if _ := import_line(store, plan, mut report, progress) {369435 		store.end()!370436 	} else {371437 		store.rollback()372438 		return err373439 	}374-	verify(store, line, progress)!440+	verify(store, plan.line, progress)!375441 }376442 377-fn import_line(store &storage.Store, root string, line []Manifest, mut report Report, progress Progress) !bool {443+fn import_line(store &storage.Store, plan Plan, mut report Report, progress Progress) !bool {444+	line := plan.line445+	mut lost_at := map[string][]string{}446+	for lost in plan.lost {447+		for id in lost.saves {448+			lost_at[id] << lost.path449+		}450+	}378451 	for i, manifest in line {379452 		progress('importing', i + 1, line.len)380453 		mut files := []snapshot.FileEntry{}@@ -386,8 +459,10 @@386459 				if stored != hash {387460 					return error('the inlined ${file.path} in save ${manifest.id} does not hash to ${hash}')388461 				}462+			} else if data := plan.found[hash] {463+				store.put_object(data)!389464 			} else if file.object {390-				blob := read_object(root, hash)!465+				blob := read_object(plan.root, hash)!391466 				store.put_object_blob(hash, blob)!392467 			} else {393468 				return error('${file.path} in save ${manifest.id} is neither an object nor inlined')@@ -409,7 +484,15 @@409484 			session:       manifest.session or { '' }410485 		}411486 		history.import_save(store, save, files)!412-		if manifest.flagged {487+		if paths := lost_at[manifest.id] {488+			// Said on the save itself, so loading it later explains the gap.489+			reason := 'migrate: Koh 2 had lost ${paths.join(', ')} at this save'490+			events.flag(store, manifest.id, if manifest.flagged {491+				'${manifest.flag_reason or { '' }}; ${reason}'492+			} else {493+				reason494+			})!495+		} else if manifest.flagged {413496 			events.flag(store, manifest.id, manifest.flag_reason or { '' })!414497 		}415498 		report.log_md_was_tracked = report.log_md_was_tracked || has_path(files, 'LOG.md')

@@ -118,7 +118,7 @@118118 		manifests[record.id] = files119119 	}120120 121-	report := run(prepare(fixture.dir, default_lane)!, quiet)!121+	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!122122 	// The fixture lives on main, because that is the lane koh2 starts on.123123 	assert report.lane == 'main'124124 	assert report.saves == 3@@ -191,7 +191,7 @@191191 	os.write_file(os.join_path(fixture.dir, 'kepr.build'), build_file)!192192 	sh(fixture.dir, '${koh2} save "add the build file"')!193193 194-	report := run(prepare(fixture.dir, default_lane)!, quiet)!194+	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!195195 	assert report.build_renamed196196 	assert !os.exists(os.join_path(fixture.dir, 'kepr.build'))197197 	assert os.read_file(os.join_path(fixture.dir, 'koh.build'))! == build_file@@ -217,7 +217,7 @@217217 	os.write_file(os.join_path(fixture.dir, 'kepr.build'), 'command = true\n')!218218 	os.write_file(os.join_path(fixture.dir, 'koh.build'), 'command = true\n')!219219 	sh(fixture.dir, '${koh2} save "both build files"')!220-	if _ := prepare(fixture.dir, default_lane) {220+	if _ := prepare(fixture.dir, default_lane, []) {221221 		assert false, 'both build files must be refused'222222 	} else {223223 		assert err.msg().contains('both kepr.build and koh.build')@@ -233,7 +233,7 @@233233 	defer {234234 		os.rmdir_all(fixture.dir) or {}235235 	}236-	report := run(prepare(fixture.dir, default_lane)!, quiet)!236+	report := run(prepare(fixture.dir, default_lane, [])!, quiet)!237237 	assert !report.build_renamed238238 	assert !os.exists(os.join_path(fixture.dir, 'koh.build'))239239 	assert os.is_dir(os.join_path(fixture.dir, '.koh-v2'))@@ -262,7 +262,7 @@262262 		os.rmdir_all(fixture.dir) or {}263263 	}264264 	os.write_file(os.join_path(fixture.dir, 'big.txt'), 'unsaved change\n')!265-	if _ := prepare(fixture.dir, default_lane) {265+	if _ := prepare(fixture.dir, default_lane, []) {266266 		assert false, 'a dirty tree must be refused'267267 	} else {268268 		assert err.msg().contains('save with koh2 first')@@ -323,4 +323,68 @@323323 	assert drop_unsafe_paths(mut line) == ['../x', 'Icon\\x0d']324324 	assert line[0].files.map(it.path) == ['ok.txt']325325 	assert line[1].files.len == 0326+}327+328+// A version Koh 2 lost is named with the saves it was in; those saves come over329+// without it, each flagged with the reason, rather than the repo staying stuck.330+fn test_a_version_koh2_lost_is_named_and_its_saves_flagged() {331+	koh2 := koh2_binary() or { return }332+	fixture := make_fixture(koh2)!333+	defer {334+		os.rmdir_all(fixture.dir) or {}335+	}336+	hash := storage.hash_hex('x'.repeat(700).bytes())337+	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!338+	plan := prepare(fixture.dir, default_lane, [])!339+	assert plan.lost.map(it.path) == ['big.txt']340+	assert plan.lost[0].saves.len == 2341+	run(plan, quiet)!342+	store := storage.open(fixture.dir)!343+	defer {344+		store.close()345+	}346+	flags := events.latest_flags(store)!347+	assert flags[fixture.first].reason.contains('Koh 2 had lost big.txt'), flags.str()348+	assert 'big.txt' !in history.files_of(store, fixture.first)!.map(it.path)349+}350+351+// Handed back with --with, found anywhere, the lost version comes home.352+fn test_a_lost_version_handed_back_is_imported() {353+	koh2 := koh2_binary() or { return }354+	fixture := make_fixture(koh2)!355+	defer {356+		os.rmdir_all(fixture.dir) or {}357+	}358+	hash := storage.hash_hex('x'.repeat(700).bytes())359+	os.rm(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]))!360+	backup := os.join_path(os.temp_dir(), 'koh3_backup_${os.getpid()}.txt')361+	os.write_file(backup, 'x'.repeat(700))!362+	defer {363+		os.rm(backup) or {}364+	}365+	plan := prepare(fixture.dir, default_lane, [backup])!366+	assert plan.lost.len == 0367+	run(plan, quiet)!368+	store := storage.open(fixture.dir)!369+	defer {370+		store.close()371+	}372+	assert 'big.txt' in history.files_of(store, fixture.first)!.map(it.path)373+	assert store.get_object(hash)! == 'x'.repeat(700).bytes()374+}375+376+// A migrate that fails clears its own draft, so the next attempt can start.377+fn test_a_failed_migrate_leaves_no_draft() {378+	koh2 := koh2_binary() or { return }379+	fixture := make_fixture(koh2)!380+	defer {381+		os.rmdir_all(fixture.dir) or {}382+	}383+	hash := storage.hash_hex('y'.repeat(700).bytes())384+	os.write_file(os.join_path(fixture.dir, '.koh', 'objects', hash[..2], hash[2..]), 'damaged')!385+	if _ := run(prepare(fixture.dir, default_lane, [])!, quiet) {386+		assert false, 'a damaged object migrated'387+	}388+	assert !os.exists(os.join_path(fixture.dir, draft_file))389+	assert os.is_dir(os.join_path(fixture.dir, '.koh'))326390 }

@@ -1419,6 +1419,9 @@14191419 	original := save_ids(dir)14201420 	assert original.len == 314211421 1422+	// A version handed back must be readable.1423+	unreadable := run_home(dir, home, 'migrate', '--with', os.join_path(dir, 'no-such-backup'), '--yes')1424+	assert unreadable.code == 1 && unreadable.err.contains('cannot read'), unreadable.err14221425 	// Off a terminal it asks nothing and changes nothing without --yes.14231426 	unasked := run_home(dir, home, 'migrate')14241427 	assert unasked.code == 1 && unasked.err.contains('needs --yes'), unasked.err

@@ -1,6 +1,6 @@11 Module {22 	name: 'koh3'3-	version: '3.2.13'3+	version: '3.2.14'44 	description: 'One SQLite file per repo, one line of history.'55 	license: 'MIT'66 }