56ab9cab6ad2

make Koh 3 build and pass on V b9a2d87: json2 for the removed json module, src/exchange for the module that shadowed V's sync, the pin written down and guarded (3.0.54)

time
2026-09-24 20:10 UTC · 2 weeks ago
machine
koh2
agent
deepseek/deepseek-flash
session
pin V
parent
9396ba8479a4
32 files README.md+4 −2src/cli/cmd_back.v+3 −3src/cli/cmd_diff.v+2 −2src/cli/cmd_doctor.v+2 −2src/cli/cmd_flag.v+4 −4src/cli/cmd_init.v+2 −2src/cli/cmd_log.v+2 −2src/cli/cmd_migrate.v+2 −2src/cli/cmd_offer.v+5 −5src/cli/cmd_peek.v+2 −2src/cli/cmd_pluck.v+3 −3src/cli/cmd_remote.v+5 −5src/cli/cmd_save.v+2 −2src/cli/cmd_status.v+2 −2src/cli/cmd_steal.v+12 −12src/cli/cmd_strip.v+9 −9src/cli/output.v+2 −2src/exchange/exchange.v+268 −0src/exchange/exchange_test.v+1236 −0src/exchange/records.v+166 −0src/exchange/steal.v+497 −0src/exchange/strip.v+227 −0src/migrate/migrate.v+2 −2src/migrate/migrate_test.v+3 −3src/remote/remote.v+1 −1src/sync/records.v+0 −166src/sync/steal.v+0 −497src/sync/strip.v+0 −227src/sync/sync.v+0 −268src/sync/sync_test.v+0 −1236tests/library_check_test.v+7 −0v.mod+1 −1
README.md+4 −2
@@ -15,8 +15,10 @@1515 v -o build/koh3 src/koh3.v1616 ```1717 18-Requires V 0.5.x. SQLite is vendored in `vendor/sqlite/`; there are no other19-dependencies.18+Pinned to V master at commit `b9a2d87` (`V 0.5.2 b9a2d87`); install steps are19+in `AGENTS.md`. V master can break Koh, so the pin is the compiler the build and20+the tests are verified against. SQLite is vendored in `vendor/sqlite/`; there21+are no other dependencies.2022 2123 ## Development2224 

view file

src/cli/cmd_back.v+3 −3
@@ -10,7 +10,7 @@1010 1111 module cli1212 13-import json13+import json21414 import src.fmt1515 import src.history1616 import src.snapshot@@ -65,7 +65,7 @@6565 		// The files all landed before whatever stopped the last restore.6666 		snapshot.clear_restore_mark(store) or { return refuse(args.json(), err.msg()) }6767 		if args.json() {68-			emit_json(json.encode(RevertJson{68+			emit_json(json2.encode(RevertJson{6969 				reverted: false7070 				head_id:  head.id7171 			}))@@ -139,7 +139,7 @@139139 	// A revert's restore save sits right on the checkpoint of what it threw away.140140 	kept := if discard_first { save.parent } else { '' }141141 	if as_json {142-		emit_json(json.encode(BackJson{142+		emit_json(json2.encode(BackJson{143143 			id:         save.id144144 			restores:   target.id145145 			written:    report.written

view file

src/cli/cmd_diff.v+2 −2
@@ -3,7 +3,7 @@33 44 module cli55 6-import json6+import json277 import src.fmt88 import src.history99 import src.snapshot@@ -23,7 +23,7 @@2323 		return refuse(args.json(), err.msg())2424 	}2525 	if args.json() {26-		emit_json(json.encode(DiffJson{26+		emit_json(json2.encode(DiffJson{2727 			from:  sides.from_name2828 			to:    sides.to_name2929 			files: deltas.map(delta_json(it))

view file

src/cli/cmd_doctor.v+2 −2
@@ -7,7 +7,7 @@77 88 module cli99 10-import json10+import json21111 import os1212 import time1313 import src.history@@ -45,7 +45,7 @@4545 	}4646 4747 	if args.json() {48-		emit_json(json.encode(report))48+		emit_json(json2.encode(report))4949 	} else {5050 		if report.problems.len == 0 {5151 			ok('${store.root} is sound')

view file

src/cli/cmd_flag.v+4 −4
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import src.events1111 import src.fmt1212 import src.history@@ -32,7 +32,7 @@3232 	if args.flag('clear') {3333 		events.clear_flag(store, save.id) or { return refuse(args.json(), err.msg()) }3434 		if args.json() {35-			emit_json(json.encode(FlagJson{35+			emit_json(json2.encode(FlagJson{3636 				save_id: save.id3737 				flagged: false3838 				reason:  ''@@ -45,7 +45,7 @@4545 	reason := args.value('reason')4646 	events.flag(store, save.id, reason) or { return refuse(args.json(), err.msg()) }4747 	if args.json() {48-		emit_json(json.encode(FlagJson{48+		emit_json(json2.encode(FlagJson{4949 			save_id: save.id5050 			flagged: true5151 			reason:  reason@@ -71,7 +71,7 @@7171 	}7272 	flagged.sort(a.save_id < b.save_id)7373 	if args.json() {74-		emit_json(json.encode(FlagListJson{74+		emit_json(json2.encode(FlagListJson{7575 			flagged: flagged7676 		}))7777 		return 0

view file

src/cli/cmd_init.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import os77 import src.snapshot88 import src.storage@@ -23,7 +23,7 @@2323 		wrote_ignore = true2424 	}2525 	if args.json() {26-		emit_json(json.encode(InitJson{26+		emit_json(json2.encode(InitJson{2727 			repo:           root2828 			ignore_created: wrote_ignore2929 		}))

view file

src/cli/cmd_log.v+2 −2
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import os1111 import time1212 import src.events@@ -68,7 +68,7 @@6868 		shown << save6969 	}7070 	if args.json() {71-		emit_json(json.encode(LogJson{71+		emit_json(json2.encode(LogJson{7272 			head_id: head_id7373 			saves:   shown.map(to_json(it))7474 		}))

view file

src/cli/cmd_migrate.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import os77 import src.fmt88 import src.machine@@ -24,7 +24,7 @@2424 	root := os.getwd()2525 	report := migrate.run(root, lane) or { return refuse(args.json(), err.msg()) }2626 	if args.json() {27-		emit_json(json.encode(MigrateJson{27+		emit_json(json2.encode(MigrateJson{2828 			repo:         report.root2929 			lane:         report.lane3030 			saves:        report.saves

view file

src/cli/cmd_offer.v+5 −5
@@ -2,12 +2,12 @@22 33 module cli44 5-import json5+import json266 import os77 import src.machine88 import src.remote99 import src.storage10-import src.sync10+import src.exchange1111 1212 pub fn cmd_offer(raw []string) int {1313 	args := parse('offer', raw, Options{@@ -18,9 +18,9 @@1818 		store.close()1919 	}2020 	mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }21-	report := sync.offer(store, mut backend) or { return refuse(args.json(), err.msg()) }21+	report := exchange.offer(store, mut backend) or { return refuse(args.json(), err.msg()) }2222 	if args.json() {23-		emit_json(json.encode(OfferedJson{23+		emit_json(json2.encode(OfferedJson{2424 			repo:    report.repo2525 			saves:   report.saves2626 			objects: report.objects@@ -68,7 +68,7 @@6868 		}6969 		return7070 	}71-	report := sync.offer(store, mut backend) or {71+	report := exchange.offer(store, mut backend) or {7272 		if !quiet {7373 			note('not offered: ${err.msg()}')7474 		}

view file

src/cli/cmd_peek.v+2 −2
@@ -5,7 +5,7 @@55 66 module cli77 8-import json8+import json299 import os1010 import src.fmt1111 import src.history@@ -35,7 +35,7 @@3535 		return refuse(args.json(), err.msg())3636 	}3737 	if args.json() {38-		emit_json(json.encode(PeekJson{38+		emit_json(json2.encode(PeekJson{3939 			id:    save.id4040 			path:  path4141 			files: files.len

view file

src/cli/cmd_pluck.v+3 −3
@@ -7,7 +7,7 @@77 88 module cli99 10-import json10+import json21111 import src.apply1212 import src.fmt1313 import src.history@@ -45,7 +45,7 @@4545 4646 	if args.flag('dry-run') || plan.applied.len == 0 {4747 		if args.json() {48-			emit_json(json.encode(PluckJson{48+			emit_json(json2.encode(PluckJson{4949 				source:     source.id5050 				reverse:    reverse5151 				applied:    plan.applied@@ -97,7 +97,7 @@9797 	report := snapshot.restore_files(store, store.root, snapshot.load(store.root), fresh.files,9898 		remove) or { return refuse(args.json(), err.msg()) }9999 	if args.json() {100-		emit_json(json.encode(PluckedJson{100+		emit_json(json2.encode(PluckedJson{101101 			id:      save.id102102 			source:  source.id103103 			reverse: reverse

view file

src/cli/cmd_remote.v+5 −5
@@ -8,7 +8,7 @@88 99 module cli1010 11-import json11+import json21212 import os1313 import src.fmt1414 import src.machine@@ -25,7 +25,7 @@2525 		}2626 		machine.forget_remote() or { return refuse(args.json(), err.msg()) }2727 		if args.json() {28-			emit_json(json.encode(RemoteJson{28+			emit_json(json2.encode(RemoteJson{2929 				configured: false3030 			}))3131 			return 0@@ -36,7 +36,7 @@3636 	if args.positional.len == 0 {3737 		if !machine.has_remote() {3838 			if args.json() {39-				emit_json(json.encode(RemoteJson{39+				emit_json(json2.encode(RemoteJson{4040 					configured: false4141 				}))4242 				return 0@@ -45,7 +45,7 @@4545 		}4646 		settings := machine.load_remote() or { return refuse(args.json(), err.msg()) }4747 		if args.json() {48-			emit_json(json.encode(RemoteJson{48+			emit_json(json2.encode(RemoteJson{4949 				configured: true5050 				kind:       settings.kind5151 				target:     describe(settings)@@ -113,7 +113,7 @@113113 		}) or { return refuse(args.json(), err.msg()) }114114 	}115115 	if args.json() {116-		emit_json(json.encode(RemoteJson{116+		emit_json(json2.encode(RemoteJson{117117 			configured: true118118 			kind:       settings.kind119119 			target:     describe(settings)

view file

src/cli/cmd_save.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import src.fmt77 import src.history88 import src.snapshot@@ -35,7 +35,7 @@3535 3636 	files := history.files_of(store, save.id) or { []snapshot.FileEntry{} }3737 	if args.json() {38-		emit_json(json.encode(SaveOut{38+		emit_json(json2.encode(SaveOut{3939 			id:            save.id4040 			parent:        save.parent4141 			message:       save.message

view file

src/cli/cmd_status.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import src.fmt77 import src.history88 import src.snapshot@@ -31,7 +31,7 @@3131 	}3232 3333 	if args.json() {34-		emit_json(json.encode(StatusJson{34+		emit_json(json2.encode(StatusJson{3535 			head_id:            head.id3636 			head_msg:           head.message3737 			dirty:              changes.dirty

view file

src/cli/cmd_steal.v+12 −12
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import os1111 import src.events1212 import src.fmt@@ -14,7 +14,7 @@1414 import src.remote1515 import src.snapshot1616 import src.storage17-import src.sync17+import src.exchange1818 1919 pub fn cmd_steal(raw []string) int {2020 	args := parse('steal', raw, Options{@@ -38,18 +38,18 @@3838 		return usage_error('steal takes no name inside a repo — run it outside to fetch a whole repo')3939 	}4040 	mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }41-	mut options := sync.StealOptions{41+	mut options := exchange.StealOptions{4242 		mine:   args.flag('mine')4343 		theirs: args.flag('theirs')4444 	}4545 	if !options.mine && !options.theirs && interactive() && os.is_atty(1) != 0 {4646 		options.ask = ask_clash4747 	}48-	report := sync.steal(store, mut backend, options) or { return refuse(args.json(), err.msg()) }48+	report := exchange.steal(store, mut backend, options) or { return refuse(args.json(), err.msg()) }4949 	// The remote should end up with whatever this machine just decided.5050 	offered := offer_after_steal(store, mut backend)5151 	if args.json() {52-		emit_json(json.encode(StolenJson{52+		emit_json(json2.encode(StolenJson{5353 			repo:          report.repo5454 			forward:       report.forward5555 			saves:         report.fetched_saves@@ -92,7 +92,7 @@9292 // offer_after_steal pushes what this machine just decided. A failure here does9393 // not undo the steal; it is reported.9494 fn offer_after_steal(store &storage.Store, mut backend remote.Backend) bool {95-	if _ := sync.offer(store, mut backend) {95+	if _ := exchange.offer(store, mut backend) {9696 		return true9797 	}9898 	return false@@ -100,7 +100,7 @@100100 101101 // ask_clash is the per-file question a terminal gets. Only a lowercase y-style102102 // answer is needed here: `m` for mine, `t` for theirs.103-fn ask_clash(clash sync.Clash) sync.Side {103+fn ask_clash(clash exchange.Clash) exchange.Side {104104 	println('  ${fmt.amber('both changed')} ${clash.path}   ${clash.stat}')105105 	print('    ${fmt.amber('m')}ine or ${fmt.amber('t')}heirs? [t] ')106106 	answer := os.get_line().trim_space()@@ -123,7 +123,7 @@123123 	mut sorted := names.keys()124124 	sorted.sort()125125 	if args.json() {126-		emit_json(json.encode(ReposJson{126+		emit_json(json2.encode(ReposJson{127127 			repos: sorted128128 		}))129129 		return 0@@ -146,7 +146,7 @@146146 		return refuse(args.json(), '${name} already exists here — run steal inside it instead')147147 	}148148 	mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }149-	fetched := sync.fetch_repo(mut backend, name) or { return refuse(args.json(), err.msg()) }149+	fetched := exchange.fetch_repo(mut backend, name) or { return refuse(args.json(), err.msg()) }150150 	// SQLite creates the file, not the folder holding it.151151 	os.mkdir_all(target, os.MkdirParams{}) or { return refuse(args.json(), err.msg()) }152152 	mut store := storage.create(target) or { return refuse(args.json(), err.msg()) }@@ -154,7 +154,7 @@154154 		store.close()155155 	}156156 	store.begin() or { return refuse(args.json(), err.msg()) }157-	sync.write_fetched(store, fetched) or {157+	exchange.write_fetched(store, fetched) or {158158 		store.rollback()159159 		return refuse(args.json(), err.msg())160160 	}@@ -162,7 +162,7 @@162162 		store.rollback()163163 		return refuse(args.json(), err.msg())164164 	}165-	sync.remember_repo(store, name) or {165+	exchange.remember_repo(store, name) or {166166 		store.rollback()167167 		return refuse(args.json(), err.msg())168168 	}@@ -183,7 +183,7 @@183183 	}184184 	flagged.sort(a.save_id < b.save_id)185185 	if args.json() {186-		emit_json(json.encode(StolenJson{186+		emit_json(json2.encode(StolenJson{187187 			repo:    name188188 			forward: true189189 			saves:   fetched.saves.len

view file

src/cli/cmd_strip.v+9 −9
@@ -11,13 +11,13 @@1111 1212 module cli1313 14-import json14+import json21515 import src.fmt1616 import src.history1717 import src.machine1818 import src.snapshot1919 import src.storage20-import src.sync20+import src.exchange2121 2222 pub fn cmd_strip(raw []string) int {2323 	args := parse('strip', raw, Options{@@ -41,12 +41,12 @@4141 		save := history.resolve(store, given) or { return refuse(args.json(), err.msg()) }4242 		only_save = save.id4343 	}44-	scope := sync.strip_scope(store, path, only_save) or { return refuse(args.json(), err.msg()) }45-	mut local := sync.LocalStrip{}44+	scope := exchange.strip_scope(store, path, only_save) or { return refuse(args.json(), err.msg()) }45+	mut local := exchange.LocalStrip{}4646 	if scope.files == 0 {4747 		// Nothing left here, but the remote half of an earlier strip may not4848 		// have finished; re-running is how it finishes.49-		if !sync.strip_pending(store, path) {49+		if !exchange.strip_pending(store, path) {5050 			ok('nothing in history matches ${path}')5151 			return 05252 		}@@ -66,7 +66,7 @@6666 			note('nothing was stripped — the path did not match')6767 			return 16868 		}69-		local = sync.strip_local(store, path, only_save) or {69+		local = exchange.strip_local(store, path, only_save) or {7070 			return refuse(args.json(), err.msg())7171 		}7272 	}@@ -74,7 +74,7 @@7474 	// and running the same strip again finishes the remote.7575 	push := push_strip_to_remote(store, path)7676 	if args.json() {77-		emit_json(json.encode(StripJson{77+		emit_json(json2.encode(StripJson{7878 			path:                   path7979 			save:                   only_save8080 			files_removed:          local.files_removed@@ -103,7 +103,7 @@103103 104104 struct PushOutcome {105105 mut:106-	pushed  sync.PushReport106+	pushed  exchange.PushReport107107 	failed  string108108 	skipped bool109109 }@@ -118,7 +118,7 @@118118 	mut backend := this_machine_backend() or { return PushOutcome{119119 		failed: err.msg()120120 	} }121-	report := sync.push_strip(store, mut backend, path) or {121+	report := exchange.push_strip(store, mut backend, path) or {122122 		return PushOutcome{123123 			failed: err.msg()124124 		}

view file

src/cli/output.v+2 −2
@@ -8,7 +8,7 @@88 99 module cli1010 11-import json11+import json21212 import os1313 import src.fmt1414 @@ -31,7 +31,7 @@3131 }3232 3333 pub fn fail_json(message string) int {34-	eprintln(json.encode({34+	eprintln(json2.encode({3535 		'error': message3636 	}))3737 	return 1

view file

src/exchange/exchange.v+268 −0
@@ -0,0 +1,268 @@1+//! Sync: offer and steal.2+//!3+//! All the cleverness sits here, above the five-operation remote interface, so4+//! the backends stay thin and this logic is written once and tested offline.5+//!6+//! The remote is a line of saves, the same shape as the local one. `offer`7+//! pushes local saves the remote lacks; `steal` pulls remote saves and, when8+//! both sides moved, replays the local work on top of the remote's. Divergence9+//! is resolved per file, never per line.10+11+module exchange12+13+import os14+import src.events15+import src.history16+import src.remote17+import src.snapshot18+import src.storage19+20+// Where the repo's name on the remote is remembered.21+const repo_state_key = 'remote_repo'22+23+// The remote head as of this machine's last successful offer, as the document24+// asks; nothing reads it back yet.25+const remote_head_key = 'remote_head'26+27+pub const no_saves = 'this repo has no saves yet — nothing to offer'28+29+pub struct OfferReport {30+pub mut:31+	repo    string32+	saves   int33+	objects int34+	events  int35+	head    string36+}37+38+// remember_repo binds a repo to its name on the remote. A clone does it at39+// once: its folder may be renamed before it ever offers.40+pub fn remember_repo(store &storage.Store, name string) ! {41+	store.set_state(repo_state_key, name)!42+}43+44+// repo_name is the folder's name at first offer, remembered in the repo so45+// renaming the folder later changes nothing.46+pub fn repo_name(store &storage.Store, mut backend remote.Backend) !string {47+	if remembered := store.state(repo_state_key) {48+		return remembered49+	}50+	name := os.base(store.root)51+	existing := backend.get(head_key(name))!52+	if existing.found {53+		theirs := existing.body.bytestr().trim_space()54+		if history.by_id(store, theirs) == none {55+			return error('the remote already has "${name}" with a history this repo does not share — rename the folder, or use another remote')56+		}57+	}58+	store.set_state(repo_state_key, name)!59+	return name60+}61+62+// offer sends what the remote lacks and then swaps `head`. Nothing is ever63+// queued: the next offer simply sends everything the remote does not have.64+pub fn offer(store &storage.Store, mut backend remote.Backend) !OfferReport {65+	name := repo_name(store, mut backend)!66+	local := history.head(store) or { return error(no_saves) }67+	mut report := OfferReport{68+		repo: name69+		head: local.id70+	}71+	remote_head := backend.get(head_key(name))!72+	// Records offered by a machine that has not applied a strip would put the73+	// stripped path back on the remote, where nothing would rewrite it again.74+	// ponytail: an offer racing a strip on another machine inside the same few75+	// seconds can still slip through; closing that needs a remote lock.76+	if unapplied_strip(store, mut backend, name)! {77+		return error('the remote has a strip this repo has not applied — run koh steal')78+	}79+	if remote_head.found {80+		theirs := remote_head.body.bytestr().trim_space()81+		newer := saves_after(store, local.id, theirs) or {82+			return error('remote has new saves — run koh steal')83+		}84+		for save in newer {85+			upload_save(store, mut backend, name, save, mut report)!86+		}87+	} else {88+		oldest_first := history.line(store)!89+		for index := oldest_first.len - 1; index >= 0; index-- {90+			upload_save(store, mut backend, name, oldest_first[index], mut report)!91+		}92+	}93+	upload_events(store, mut backend, name, mut report)!94+	if !backend.put_if_match(head_key(name), local.id.bytes(), remote_head.tag)! {95+		return error('remote has new saves — run koh steal')96+	}97+	store.set_state(remote_head_key, local.id)!98+	return report99+}100+101+// saves_after lists the local saves newer than `theirs`, oldest first. It fails102+// when `theirs` is not on this line, which is the honest answer: we cannot103+// offer on top of a save we do not have.104+fn saves_after(store &storage.Store, ours string, theirs string) ![]history.Save {105+	mut newer := []history.Save{}106+	mut cursor := ours107+	for cursor != '' && cursor != theirs {108+		save := history.by_id(store, cursor) or { return error('save ${cursor} is missing') }109+		newer << save110+		cursor = save.parent111+	}112+	if cursor != theirs {113+		return error('${theirs} is not on this line')114+	}115+	newer.reverse_in_place()116+	return newer117+}118+119+fn upload_save(store &storage.Store, mut backend remote.Backend, repo string, save history.Save, mut report OfferReport) ! {120+	files := history.files_of(store, save.id)!121+	// Saves go up oldest first and head moves only once all of them landed, so122+	// the parent and its objects are on the remote already: only content new123+	// in this save travels, not a round trip per file in the repo.124+	mut sent := map[string]bool{}125+	if save.parent != '' {126+		for file in history.files_of(store, save.parent)! {127+			sent[file.hash] = true128+		}129+	}130+	for file in files {131+		if file.hash in sent {132+			continue133+		}134+		sent[file.hash] = true135+		if backend.put_if_absent(object_key(repo, file.hash), store.object_blob(file.hash)!)! {136+			report.objects++137+		}138+	}139+	record := save_record(store, save)!140+	if backend.put_if_absent(save_key(repo, save.id), encode_save(record).bytes())! {141+		report.saves++142+	}143+}144+145+// upload_events sends every event the remote does not have. Events are tiny, so146+// listing them costs less than tracking which ones were sent.147+fn upload_events(store &storage.Store, mut backend remote.Backend, repo string, mut report OfferReport) ! {148+	already := backend.list(events_prefix(repo))!149+	mut already_sent := map[string]bool{}150+	for key in already {151+		already_sent[os.base(key)] = true152+	}153+	for event in events.all(store)! {154+		if event.id in already_sent {155+			continue156+		}157+		if backend.put_if_absent(event_key(repo, event.id), encode_event(event).bytes())! {158+			report.events++159+		}160+	}161+}162+163+// fetched holds everything a steal read from the remote, before any of it is164+// written locally. Network work happens first; the database gets one165+// transaction, so a failure leaves nothing half-applied.166+pub struct Fetched {167+pub mut:168+	repo_objects map[string][]u8169+	saves        []SaveRecord // oldest first170+	events_out   []events.Event171+	head         string172+	base         string // the newest save both sides already shared, '' when none173+}174+175+pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched {176+	remote_head := backend.get(head_key(repo))!177+	if !remote_head.found {178+		return error('the remote has no "${repo}" yet — run koh offer here first')179+	}180+	mut fetched := Fetched{181+		head: remote_head.body.bytestr().trim_space()182+	}183+	mut cursor := fetched.head184+	for cursor != '' && history.by_id(store, cursor) == none {185+		record_object := backend.get(save_key(repo, cursor))!186+		if !record_object.found {187+			return error('the remote is missing save ${cursor} — run koh offer there')188+		}189+		record := decode_save(record_object.body)!190+		for file in record.files {191+			if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects {192+				blob := backend.get(object_key(repo, file.hash))!193+				if !blob.found {194+					return error('the remote is missing object ${file.hash} that save ${record.id} needs')195+				}196+				fetched.repo_objects[file.hash] = blob.body197+			}198+		}199+		fetched.saves << record200+		cursor = record.parent201+	}202+	fetched.saves.reverse_in_place()203+	fetched.base = cursor204+	fetched.events_out = fetch_events(store, mut backend, repo)!205+	return fetched206+}207+208+fn fetch_events(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {209+	have := events.all(store)!210+	mut known := map[string]bool{}211+	for event in have {212+		known[event.id] = true213+	}214+	mut incoming := []events.Event{}215+	for key in backend.list(events_prefix(repo))! {216+		id := os.base(key)217+		if id in known {218+			continue219+		}220+		body := backend.get(event_key(repo, id))!221+		if !body.found {222+			continue223+		}224+		incoming << decode_event(body.body)!225+	}226+	return incoming227+}228+229+// write_fetched lands a fetch in one transaction: objects first (verified),230+// then the events, then the saves they belong to. A new strip reaches every231+// save already here, but an arriving save only when `reaches` says so: one232+// made after the strip is new work, not what the strip removed.233+pub fn write_fetched(store &storage.Store, fetched Fetched) ! {234+	for hash, blob in fetched.repo_objects {235+		store.put_object_blob(hash, blob)!236+	}237+	mut strips := []events.Event{}238+	for event in fetched.events_out {239+		if !events.receive(store, event)! {240+			continue241+		}242+		if event.kind == events.kind_strip {243+			// Somebody stripped a path: remove it here too, so the two244+			// histories agree from now on. Objects go once the saves are in.245+			remove_path(store, event.path, event.save_id)!246+			strips << event247+		}248+	}249+	for record in fetched.saves {250+		files := to_files(record).filter(fn [strips, record] (file snapshot.FileEntry) bool {251+			return !strips.any(reaches(it, record.id, record.created_at)252+				&& covers(it.path, file.path))253+		})254+		history.import_save(store, to_save(record), files)!255+	}256+	if strips.len > 0 {257+		store.drop_unreferenced_objects()!258+	}259+}260+261+// reaches reports whether a strip event applies to a save arriving from the262+// remote. A strip of every save covers the saves made up to its own time.263+fn reaches(strip events.Event, save_id string, created_at i64) bool {264+	if strip.save_id != '' {265+		return strip.save_id == save_id266+	}267+	return created_at <= strip.created_at268+}

view file

src/exchange/exchange_test.v+1236 −0
@@ -0,0 +1,1236 @@1+//! Two-machine tests: two repos and one folder remote standing in for the Mac2+//! and the PC. This is where offer, steal and combine are actually proven —3+//! the folder backend exists so these run offline.4+5+module exchange6+7+import os8+import src.events9+import src.history10+import src.remote11+import src.snapshot12+import src.storage13+14+// Machines keep their repo in a folder of the same NAME, because the remote key15+// is the folder's name; that is what makes them the same repo to the remote.16+fn machine(name string) !string {17+	dir := os.join_path(os.temp_dir(), 'koh3_sync_${os.getpid()}_${name}', 'proj')18+	os.rmdir_all(os.dir(dir)) or {}19+	os.mkdir_all(dir, os.MkdirParams{})!20+	return dir21+}22+23+fn scratch(name string) !string {24+	dir := os.join_path(os.temp_dir(), 'koh3_sync_${os.getpid()}_${name}')25+	os.rmdir_all(dir) or {}26+	os.mkdir_all(dir, os.MkdirParams{})!27+	return dir28+}29+30+fn open_repo(dir string) !storage.Store {31+	mut store := storage.create(dir)!32+	os.write_file(os.join_path(dir, '.kohignore'), '*.log\n')!33+	return store34+}35+36+fn put(root string, rel string, text string) ! {37+	full := os.join_path(root, rel)38+	os.mkdir_all(os.dir(full), os.MkdirParams{})!39+	os.write_file(full, text)!40+}41+42+fn commit(store &storage.Store, message string) !history.Save {43+	return history.commit_worktree(store, store.root, snapshot.load(store.root), history.SaveInput{44+		message: message45+	})!46+}47+48+fn offer_now(store &storage.Store, mut backend remote.Backend) !OfferReport {49+	return offer(store, mut backend)!50+}51+52+fn steal_now(store &storage.Store, mut backend remote.Backend) !StealReport {53+	return steal(store, mut backend, StealOptions{54+		theirs: true55+	})!56+}57+58+fn test_first_offer_publishes_the_whole_line() {59+	base := scratch('first')!60+	defer {61+		os.rmdir_all(base) or {}62+	}63+	mut a := open_repo(machine('first_a')!)!64+	defer {65+		a.close()66+	}67+	put(a.root, 'a.txt', 'one\n')!68+	commit(a, 'first')!69+	put(a.root, 'a.txt', 'two\n')!70+	commit(a, 'second')!71+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!72+73+	report := offer_now(a, mut backend)!74+	assert report.repo == 'proj'75+	assert report.saves == 276+	assert report.objects >= 277+	assert backend.get(head_key('proj'))!.found78+	assert backend.list('proj/saves')!.len == 279+80+	// A second offer with nothing new uploads nothing and still succeeds.81+	again := offer_now(a, mut backend)!82+	assert again.saves == 083+}84+85+fn test_the_repo_name_is_remembered_and_head_offered() {86+	base := scratch('remember')!87+	defer {88+		os.rmdir_all(base) or {}89+	}90+	mut a := open_repo(machine('remember_a')!)!91+	defer {92+		a.close()93+	}94+	put(a.root, 'a.txt', 'one\n')!95+	commit(a, 'first')!96+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!97+	offer_now(a, mut backend)!98+	assert a.state('remote_repo') or { '' } == 'proj'99+	assert backend.get(head_key('proj'))!.body.bytestr() == head_of(a).id100+	// Offer's step 3: the new remote head is recorded in state.101+	assert a.state('remote_head') or { '' } == head_of(a).id102+103+	// A second offer, with the remote already holding our head, must not104+	// refuse: the name is remembered, so the remote is ours.105+	put(a.root, 'a.txt', 'two\n')!106+	commit(a, 'second')!107+	offer_now(a, mut backend)!108+	assert backend.get(head_key('proj'))!.body.bytestr() == head_of(a).id109+}110+111+fn test_steal_fast_forwards_a_machine_that_has_nothing_of_its_own() {112+	base := scratch('forward')!113+	defer {114+		os.rmdir_all(base) or {}115+	}116+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!117+	mut a := open_repo(machine('fwd_a')!)!118+	defer {119+		a.close()120+	}121+	put(a.root, 'a.txt', 'from A\n')!122+	commit(a, 'A writes')!123+	offer_now(a, mut backend)!124+125+	// B starts empty and pulls the whole repo.126+	mut b := open_repo(machine('fwd_b')!)!127+	defer {128+		b.close()129+	}130+	fetched := fetch_repo(mut backend, 'proj')!131+	b.begin()!132+	write_fetched(b, fetched)!133+	history.set_head(b, fetched.head)!134+	b.commit()!135+	snapshot.restore_files(b, b.root, snapshot.load(b.root), history.files_of(b, fetched.head)!, [])!136+	assert os.read_file(os.join_path(b.root, 'a.txt'))! == 'from A\n'137+	assert head_of(b).id == fetched.head138+}139+140+// A machine that is only ahead has nothing to combine: stealing must leave its141+// saves, and their IDs, exactly where they are.142+fn test_steal_while_only_ahead_leaves_the_line_alone() {143+	base := scratch('ahead')!144+	defer {145+		os.rmdir_all(base) or {}146+	}147+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!148+	mut a := open_repo(machine('ahead_a')!)!149+	defer {150+		a.close()151+	}152+	put(a.root, 'a.txt', 'one\n')!153+	commit(a, 'offered')!154+	offer_now(a, mut backend)!155+	put(a.root, 'a.txt', 'two\n')!156+	ahead := commit(a, 'not offered yet')!157+	report := steal_now(a, mut backend)!158+	assert report.replayed == 0159+	assert head_of(a).id == ahead.id160+	// Nothing was restored, so nothing is left marked as half-restored.161+	assert !snapshot.restore_pending(a)162+	assert history.all_saves(a)!.len == 2163+	offer_now(a, mut backend)!164+	assert backend.get(head_key('proj'))!.body.bytestr() == ahead.id165+}166+167+// A link is invisible to the walk, so a tree holding one is clean; the files168+// a steal brings in must still never land in the folder it points at.169+fn test_steal_never_writes_through_a_symlinked_folder() {170+	base := scratch('steal_link')!171+	defer {172+		os.rmdir_all(base) or {}173+	}174+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!175+	mut a := open_repo(machine('steal_link_a')!)!176+	defer {177+		a.close()178+	}179+	mut b := open_repo(machine('steal_link_b')!)!180+	defer {181+		b.close()182+	}183+	put(a.root, 'a.txt', 'one\n')!184+	commit(a, 'first')!185+	offer_now(a, mut backend)!186+	join(b, mut backend)!187+	outside := os.join_path(base, 'outside')188+	put(outside, 'a.md', 'precious\n')!189+	os.symlink(outside, os.join_path(b.root, 'docs'))!190+	put(a.root, 'docs/a.md', 'from A\n')!191+	commit(a, 'A adds docs')!192+	offer_now(a, mut backend)!193+	head_before := head_of(b).id194+	if _ := steal_now(b, mut backend) {195+		assert false, 'a steal writing through a symlinked folder must refuse'196+	} else {197+		assert err.msg().contains('docs')198+	}199+	assert os.read_file(os.join_path(outside, 'a.md'))! == 'precious\n'200+	assert head_of(b).id == head_before201+}202+203+// A steal onto a half-restored tree would replay it as somebody's work.204+fn test_steal_refuses_after_a_restore_cut_short() {205+	base := scratch('steal_cut')!206+	defer {207+		os.rmdir_all(base) or {}208+	}209+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!210+	mut a := open_repo(machine('steal_cut_a')!)!211+	defer {212+		a.close()213+	}214+	put(a.root, 'a.txt', 'one\n')!215+	commit(a, 'first')!216+	offer_now(a, mut backend)!217+	a.set_state('restoring', 'yes')!218+	if _ := steal_now(a, mut backend) {219+		assert false, 'steal must refuse while a restore is unfinished'220+	} else {221+		assert err.msg().contains('koh revert')222+	}223+}224+225+// A replayed save keeps its original time but sits on a newer parent, so a226+// clone cannot order saves by time.227+fn test_a_clone_imports_parents_before_children() {228+	base := scratch('clone_order')!229+	defer {230+		os.rmdir_all(base) or {}231+	}232+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!233+	mut a := open_repo(machine('order_a')!)!234+	defer {235+		a.close()236+	}237+	put(a.root, 'a.txt', 'one\n')!238+	parent := commit(a, 'parent')!239+	child := history.commit_files(a, '', history.files_of(a, parent.id)!, history.SaveInput{240+		message:    'replayed, older than its parent'241+		created_at: parent.created_at - 3600242+	})!243+	offer_now(a, mut backend)!244+	mut b := open_repo(machine('order_b')!)!245+	defer {246+		b.close()247+	}248+	join(b, mut backend)!249+	assert head_of(b).id == child.id250+}251+252+// A chmod is unsaved work too: a steal would restore the old mode over it.253+fn test_steal_refuses_an_unsaved_executable_bit_change() {254+	base := scratch('chmod')!255+	defer {256+		os.rmdir_all(base) or {}257+	}258+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!259+	mut a := open_repo(machine('chmod_a')!)!260+	defer {261+		a.close()262+	}263+	put(a.root, 'run.sh', 'echo hi\n')!264+	commit(a, 'script')!265+	offer_now(a, mut backend)!266+	os.chmod(os.join_path(a.root, 'run.sh'), 0o755)!267+	if _ := steal_now(a, mut backend) {268+		assert false, 'a chmod-only change should make the tree dirty'269+	} else {270+		assert err.msg() == dirty_tree271+	}272+}273+274+fn test_divergence_without_clashes_keeps_both_machines_work() {275+	base := scratch('diverge')!276+	defer {277+		os.rmdir_all(base) or {}278+	}279+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!280+	mut a := open_repo(machine('div_a')!)!281+	defer {282+		a.close()283+	}284+	mut b := open_repo(machine('div_b')!)!285+	defer {286+		b.close()287+	}288+	put(a.root, 'shared.txt', 'base\n')!289+	commit(a, 'base')!290+	offer_now(a, mut backend)!291+292+	// B joins, then both machines change a different file.293+	join(b, mut backend)!294+	put(a.root, 'fromA.txt', 'A\n')!295+	commit(a, 'A adds a file')!296+	put(b.root, 'fromB.txt', 'B\n')!297+	commit(b, 'B adds a file')!298+	offer_now(a, mut backend)!299+300+	// B's offer is refused: the remote has moved on.301+	if _ := offer_now(b, mut backend) {302+		assert false, 'offering on top of unknown remote saves should be refused'303+	} else {304+		assert err.msg().contains('run koh steal')305+	}306+307+	report := steal_now(b, mut backend)!308+	assert !report.forward309+	assert report.replayed == 1310+	assert report.clashes.len == 0311+	// Both files are on B's line now, and on disk.312+	assert os.read_file(os.join_path(b.root, 'fromA.txt'))! == 'A\n'313+	assert os.read_file(os.join_path(b.root, 'fromB.txt'))! == 'B\n'314+	assert os.read_file(os.join_path(b.root, 'shared.txt'))! == 'base\n'315+	// B's own save was replayed, not erased.316+	mut saw_replay := false317+	for save in history.all_saves(b)! {318+		if save.replayed_from != '' {319+			saw_replay = true320+		}321+	}322+	assert saw_replay323+	// And the remote now has B's work too.324+	offer_now(b, mut backend)!325+	line_head := backend.get(head_key('proj'))!.body.bytestr().trim_space()326+	assert line_head == head_of(b).id327+}328+329+// An agent that edits a file and then undoes the edit leaves no net change, so330+// the file is not a clash when the other machine changed it. Replaying the331+// edit itself must not stop the steal: the remote's version simply stands.332+// Likewise a file both machines ended up changing the same way.333+fn test_a_local_change_undone_does_not_block_a_combine() {334+	base := scratch('undone')!335+	defer {336+		os.rmdir_all(base) or {}337+	}338+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!339+	mut a := open_repo(machine('undone_a')!)!340+	defer {341+		a.close()342+	}343+	mut b := open_repo(machine('undone_b')!)!344+	defer {345+		b.close()346+	}347+	put(a.root, 'f.txt', 'base\n')!348+	put(a.root, 'g.txt', 'base\n')!349+	commit(a, 'base')!350+	offer_now(a, mut backend)!351+	join(b, mut backend)!352+353+	put(a.root, 'f.txt', 'from A\n')!354+	put(a.root, 'g.txt', 'same\n')!355+	commit(a, 'A edits both')!356+	offer_now(a, mut backend)!357+358+	put(b.root, 'f.txt', 'tried on B\n')!359+	put(b.root, 'g.txt', 'first try\n')!360+	tried := commit(b, 'B tries')!361+	put(b.root, 'f.txt', 'base\n')!362+	put(b.root, 'g.txt', 'same\n')!363+	put(b.root, 'h.txt', 'B\n')!364+	undone := commit(b, 'B undoes f, lands on the same g, adds h')!365+366+	report := steal_now(b, mut backend)!367+	assert report.clashes.len == 0368+	assert os.read_file(os.join_path(b.root, 'f.txt'))! == 'from A\n'369+	assert os.read_file(os.join_path(b.root, 'g.txt'))! == 'same\n'370+	assert os.read_file(os.join_path(b.root, 'h.txt'))! == 'B\n'371+	// "B tries" only touched files A changed, so it has nothing left to372+	// replay; the second save carries h.txt. Both originals stay in history.373+	assert report.replayed == 1374+	assert head_of(b).replayed_from == undone.id375+	ids := history.all_saves(b)!.map(it.id)376+	assert tried.id in ids && undone.id in ids377+	assert history.files_of(b, tried.id)!.any(it.path == 'f.txt')378+}379+380+// A person deciding a clash may take minutes; the question must come before381+// the write lock, or every agent's save on this repo waits and then fails.382+fn test_clash_questions_are_asked_before_the_write_lock() {383+	base := scratch('ask_unlocked')!384+	defer {385+		os.rmdir_all(base) or {}386+	}387+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!388+	mut a := open_repo(machine('ask_a')!)!389+	defer {390+		a.close()391+	}392+	mut b := open_repo(machine('ask_b')!)!393+	defer {394+		b.close()395+	}396+	put(a.root, 'same.txt', 'base\n')!397+	commit(a, 'base')!398+	offer_now(a, mut backend)!399+	join(b, mut backend)!400+	put(a.root, 'same.txt', 'A version\n')!401+	commit(a, 'A edits same')!402+	offer_now(a, mut backend)!403+	put(b.root, 'same.txt', 'B version\n')!404+	commit(b, 'B edits same')!405+	root := b.root406+	// An agent saves while the person is still choosing.407+	if _ := steal(b, mut backend, StealOptions{408+		ask: fn [root] (clash Clash) Side {409+			other := storage.open(root) or { panic(err) }410+			defer {411+				other.close()412+			}413+			os.write_file(os.join_path(root, 'agent.txt'), 'saved meanwhile\n') or { panic(err) }414+			history.commit_worktree(other, root, snapshot.load(root), history.SaveInput{415+				message: 'agent saves during the question'416+			}) or { panic('steal held the write lock while asking: ${err}') }417+			return .mine418+		}419+	})420+	{421+		assert false, 'a steal whose line moved during the question must refuse'422+	} else {423+		assert err.msg().contains('changed while')424+	}425+	// The agent's save is still head, and a second steal combines it too.426+	assert head_of(b).message == 'agent saves during the question'427+	report := steal(b, mut backend, StealOptions{428+		mine: true429+	})!430+	assert report.kept_mine == 1431+	assert os.read_file(os.join_path(b.root, 'agent.txt'))! == 'saved meanwhile\n'432+	assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'B version\n'433+}434+435+fn test_clashing_files_are_chosen_one_at_a_time() {436+	base := scratch('clash')!437+	defer {438+		os.rmdir_all(base) or {}439+	}440+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!441+	mut a := open_repo(machine('clash_a')!)!442+	defer {443+		a.close()444+	}445+	mut b := open_repo(machine('clash_b')!)!446+	defer {447+		b.close()448+	}449+	put(a.root, 'same.txt', 'base\n')!450+	put(a.root, 'other.txt', 'quiet\n')!451+	commit(a, 'base')!452+	offer_now(a, mut backend)!453+	join(b, mut backend)!454+455+	// Both machines edit the same file, differently.456+	put(a.root, 'same.txt', 'A version\n')!457+	commit(a, 'A edits same')!458+	offer_now(a, mut backend)!459+	put(b.root, 'same.txt', 'B version\n')!460+	commit(b, 'B edits same')!461+462+	// Without a decision, steal refuses and says which files clash.463+	if _ := steal(b, mut backend, StealOptions{}) {464+		assert false, 'an undecided clash must be refused'465+	} else {466+		assert err.msg().contains('--mine') || err.msg().contains('--theirs')467+	}468+	assert find_clashes(history.files_of(b, head_of(b).id)!, history.files_of(b, head_of(b).id)!, history.files_of(b,469+		head_of(b).id)!).len == 0470+471+	// Choosing the remote's version ends with A's content.472+	report := steal_now(b, mut backend)!473+	assert report.clashes == ['same.txt']474+	assert report.kept_mine == 0475+	assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'A version\n'476+	assert os.read_file(os.join_path(b.root, 'other.txt'))! == 'quiet\n'477+}478+479+fn test_choosing_mine_keeps_the_local_version() {480+	base := scratch('mine')!481+	defer {482+		os.rmdir_all(base) or {}483+	}484+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!485+	mut a := open_repo(machine('mine_a')!)!486+	defer {487+		a.close()488+	}489+	mut b := open_repo(machine('mine_b')!)!490+	defer {491+		b.close()492+	}493+	put(a.root, 'same.txt', 'base\n')!494+	commit(a, 'base')!495+	offer_now(a, mut backend)!496+	join(b, mut backend)!497+	put(a.root, 'same.txt', 'A version\n')!498+	commit(a, 'A edits')!499+	offer_now(a, mut backend)!500+	put(b.root, 'same.txt', 'B version\n')!501+	commit(b, 'B edits')!502+503+	report := steal(b, mut backend, StealOptions{504+		mine: true505+	})!506+	assert report.kept_mine == 1507+	assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'B version\n'508+	// The final save says what it kept.509+	assert head_of(b).message.contains('combine: kept mine for 1 file')510+	// The pre-combine tree is still reachable in history.511+	assert head_of(b).restores == ''512+	assert history.all_saves(b)!.len >= 4513+}514+515+fn test_flags_travel_with_the_saves() {516+	base := scratch('flags')!517+	defer {518+		os.rmdir_all(base) or {}519+	}520+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!521+	mut a := open_repo(machine('flag_a')!)!522+	defer {523+		a.close()524+	}525+	mut b := open_repo(machine('flag_b')!)!526+	defer {527+		b.close()528+	}529+	put(a.root, 'a.txt', 'one\n')!530+	save := commit(a, 'suspect save')!531+	events.flag(a, save.id, 'look at this')!532+	offer_now(a, mut backend)!533+534+	// B has nothing yet, so a fetch brings the save and its flag across.535+	fetched := fetch_back_to_known(b, mut backend, 'proj')!536+	assert fetched.events_out.len == 1537+	assert fetched.events_out[0].kind == events.kind_flag538+	b.begin()!539+	write_fetched(b, fetched)!540+	history.set_head(b, fetched.head)!541+	b.commit()!542+	state := events.flag_state(b, save.id)!543+	assert state.flagged544+	assert state.reason == 'look at this'545+}546+547+fn test_a_second_offer_racing_head_is_refused_not_forced() {548+	base := scratch('race')!549+	defer {550+		os.rmdir_all(base) or {}551+	}552+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!553+	mut a := open_repo(machine('race_a')!)!554+	defer {555+		a.close()556+	}557+	put(a.root, 'a.txt', 'one\n')!558+	commit(a, 'base')!559+	offer_now(a, mut backend)!560+561+	// Someone else replaces head behind our back.562+	assert backend.put_if_match(head_key('proj'), 'a-save-we-do-not-have'.bytes(),563+		backend.get(head_key('proj'))!.tag)!564+565+	put(a.root, 'a.txt', 'two\n')!566+	commit(a, 'second')!567+	if _ := offer_now(a, mut backend) {568+		assert false, 'offering while the remote head moved must be refused'569+	} else {570+		assert err.msg().contains('run koh steal')571+	}572+}573+574+// join makes a brand new machine share the remote's line.575+fn join(store &storage.Store, mut backend remote.Backend) ! {576+	fetched := fetch_repo(mut backend, 'proj')!577+	store.begin()!578+	write_fetched(store, fetched)!579+	history.set_head(store, fetched.head)!580+	store.commit()!581+	snapshot.restore_files(store, store.root, snapshot.load(store.root), history.files_of(store,582+		fetched.head)!, [])!583+	store.set_state('remote_repo', 'proj')!584+}585+586+struct FaultyBackend {587+mut:588+	inner      remote.Backend589+	writes     int590+	reads      int591+	fail_write int // fail when the write counter reaches this, 0 never592+	fail_read  int593+	// fail every delete while set594+	fail_delete bool595+}596+597+fn (mut f FaultyBackend) get(key string) !remote.Object {598+	f.reads++599+	if f.fail_read > 0 && f.reads >= f.fail_read {600+		return error('injected read failure')601+	}602+	return f.inner.get(key)!603+}604+605+fn (mut f FaultyBackend) list(prefix string) ![]string {606+	return f.inner.list(prefix)!607+}608+609+fn (mut f FaultyBackend) put_if_absent(key string, body []u8) !bool {610+	f.writes++611+	if f.fail_write > 0 && f.writes >= f.fail_write {612+		return error('injected write failure')613+	}614+	return f.inner.put_if_absent(key, body)!615+}616+617+fn (mut f FaultyBackend) put_if_match(key string, body []u8, tag string) !bool {618+	f.writes++619+	if f.fail_write > 0 && f.writes >= f.fail_write {620+		return error('injected write failure')621+	}622+	return f.inner.put_if_match(key, body, tag)!623+}624+625+fn (mut f FaultyBackend) delete(key string) ! {626+	if f.fail_delete {627+		return error('injected delete failure')628+	}629+	f.inner.delete(key)!630+}631+632+// Every save offers after it is made, so a one-file change must cost one633+// object, not one conditional write per file in the repo: on S3 each of those634+// is a round trip.635+fn test_an_offer_sends_only_the_objects_a_save_adds() {636+	base := scratch('offer_diff')!637+	defer {638+		os.rmdir_all(base) or {}639+	}640+	mut a := open_repo(machine('offer_diff_a')!)!641+	defer {642+		a.close()643+	}644+	for i in 0 .. 20 {645+		put(a.root, 'f${i}.txt', 'file ${i}\n')!646+	}647+	commit(a, 'twenty files')!648+	mut backend := FaultyBackend{649+		inner: remote.new_folder(os.join_path(base, 'remote'))!650+	}651+	offer(a, mut backend)!652+	put(a.root, 'f3.txt', 'changed\n')!653+	commit(a, 'one change')!654+	before := backend.writes655+	report := offer(a, mut backend)!656+	assert report.objects == 1657+	// One object, one save record, one head swap.658+	assert backend.writes - before == 3659+}660+661+fn test_a_failed_offer_is_safe_to_re_run() {662+	base := scratch('fault_offer')!663+	defer {664+		os.rmdir_all(base) or {}665+	}666+	remote_dir := os.join_path(base, 'remote')667+	mut a := open_repo(machine('fault_a')!)!668+	defer {669+		a.close()670+	}671+	put(a.root, 'a.txt', 'one\n')!672+	commit(a, 'first')!673+	put(a.root, 'a.txt', 'two\n')!674+	commit(a, 'second')!675+676+	// The first offer dies after the first write.677+	mut flaky := FaultyBackend{678+		inner:      remote.new_folder(remote_dir)!679+		fail_write: 2680+	}681+	if _ := offer(a, mut flaky) {682+		assert false, 'the injected failure should have surfaced'683+	}684+685+	// Re-running with a healthy remote finishes the job: "already exists" is686+	// success, so nothing is lost and nothing is duplicated.687+	mut healthy := remote.new_folder(remote_dir)!688+	report := offer(a, mut healthy)!689+	assert report.saves == 2690+	assert healthy.get(head_key('proj'))!.body.bytestr().trim_space() == head_of(a).id691+	assert healthy.list('proj/saves')!.len == 2692+}693+694+fn test_a_failed_steal_leaves_nothing_behind() {695+	base := scratch('fault_steal')!696+	defer {697+		os.rmdir_all(base) or {}698+	}699+	remote_dir := os.join_path(base, 'remote')700+	mut a := open_repo(machine('fault_s_a')!)!701+	defer {702+		a.close()703+	}704+	mut b := open_repo(machine('fault_s_b')!)!705+	defer {706+		b.close()707+	}708+	put(a.root, 'a.txt', 'from A\n')!709+	commit(a, 'A writes')!710+	mut healthy := remote.new_folder(remote_dir)!711+	offer(a, mut healthy)!712+713+	// B's fetch fails half way through reading the remote.714+	mut flaky := FaultyBackend{715+		inner:     remote.new_folder(remote_dir)!716+		fail_read: 2717+	}718+	if _ := fetch_back_to_known(b, mut flaky, 'proj') {719+		assert false, 'the injected read failure should have surfaced'720+	}721+	// Nothing was written: the store is untouched and still has no saves.722+	assert history.head(b) == none723+	assert b.object_count()! == 0724+725+	// A healthy retry works.726+	fetched := fetch_back_to_known(b, mut healthy, 'proj')!727+	b.begin()!728+	write_fetched(b, fetched)!729+	history.set_head(b, fetched.head)!730+	b.commit()!731+	snapshot.restore_files(b, b.root, snapshot.load(b.root), history.files_of(b, fetched.head)!, [])!732+	assert os.read_file(os.join_path(b.root, 'a.txt'))! == 'from A\n'733+}734+735+fn test_an_interrupted_strip_is_idempotent() {736+	// The local half of a strip is one transaction, so an interrupted strip is737+	// simply a strip that did not happen; re-running it must land the same738+	// result once, not twice.739+	base := scratch('fault_strip')!740+	defer {741+		os.rmdir_all(base) or {}742+	}743+	mut a := open_repo(machine('fault_strip_a')!)!744+	defer {745+		a.close()746+	}747+	put(a.root, 'secret.txt', 'shh\n')!748+	put(a.root, 'keep.txt', 'fine\n')!749+	save := commit(a, 'add both')!750+751+	for _ in 0 .. 2 {752+		strip_local(a, 'secret.txt', '')!753+	}754+	files := history.files_of(a, save.id)!755+	// The repo's own `.kohignore` is tracked content, so it stays; only the756+	// stripped path goes.757+	assert files.map(it.path) == ['.kohignore', 'keep.txt']758+	assert events.strip_events(a)!.len == 1759+}760+761+// SQLite's LIKE ignores ASCII case, so a strip of `docs` must not reach into762+// `Docs/`, and `a_b` must not reach `aXb`: strip destroys, so it matches exactly.763+// The tree is built by hand: on macOS `Docs` and `docs` are one folder on disk,764+// but a save made on Linux can hold both.765+fn test_a_strip_matches_the_path_exactly() {766+	mut a := open_repo(machine('strip_exact')!)!767+	defer {768+		a.close()769+	}770+	hash := a.put_object('x\n'.bytes())!771+	mut files := []snapshot.FileEntry{}772+	for path in ['DOCS.txt', 'Docs/keep.txt', 'a_b/x.txt', 'aXb/y.txt', 'docs/secret.txt'] {773+		files << snapshot.FileEntry{774+			path: path775+			hash: hash776+		}777+	}778+	save := history.commit_files(a, '', files, history.SaveInput{779+		message: 'add all'780+	})!781+	assert strip_scope(a, 'docs', '')!.files == 1782+	strip_local(a, 'docs', '')!783+	strip_local(a, 'a_b', '')!784+	assert history.files_of(a, save.id)!.map(it.path) == ['DOCS.txt', 'Docs/keep.txt', 'aXb/y.txt']785+}786+787+fn test_a_strip_reaches_the_remote_and_other_machines() {788+	base := scratch('strip_sync')!789+	defer {790+		os.rmdir_all(base) or {}791+	}792+	mut a := open_repo(machine('strip_a')!)!793+	defer {794+		a.close()795+	}796+	mut b := open_repo(machine('strip_b')!)!797+	defer {798+		b.close()799+	}800+	put(a.root, 'secrets.env', 'api_key=1\n')!801+	put(a.root, 'keep.txt', 'fine\n')!802+	save := commit(a, 'add both')!803+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!804+	offer_now(a, mut backend)!805+	join(b, mut backend)!806+	assert b.object_count()! > 0807+808+	// A strips the path locally, the way `koh strip` does, then tells the809+	// remote.810+	mut secret_hash := ''811+	for file in history.files_of(a, save.id)! {812+		if file.path == 'secrets.env' {813+			secret_hash = file.hash814+		}815+	}816+	assert secret_hash != ''817+	strip_local(a, 'secrets.env', '')!818+	report := push_strip(a, mut backend, 'secrets.env')!819+	assert report.event_sent820+	assert report.records == 1821+822+	// The remote's copy of the save no longer lists the file, and the object is823+	// gone from the remote.824+	record := decode_save(backend.get(save_key('proj', save.id))!.body)!825+	for file in record.files {826+		assert file.path != 'secrets.env'827+	}828+	// The stripped object is gone from the remote; the ones still referenced829+	// are not.830+	assert !backend.get(object_key('proj', secret_hash))!.found831+	assert backend.list('proj/objects')!.len == 2832+833+	// Another machine hears about it on its next steal and applies it.834+	b.begin()!835+	fetched := fetch_back_to_known(b, mut backend, 'proj')!836+	write_fetched(b, fetched)!837+	b.commit()!838+	mut paths := []string{}839+	for file in history.files_of(b, save.id)! {840+		paths << file.path841+	}842+	assert 'secrets.env' !in paths843+	assert 'keep.txt' in paths844+845+	// The event arrives under its own ID, so it is applied once: B's offer adds846+	// no second copy, and a file B saves again after the strip stays saved847+	// through B's next steal.848+	assert events.strip_events(b)!.map(it.id) == events.strip_events(a)!.map(it.id)849+	put(b.root, 'secrets.env', 'rotated\n')!850+	readded := commit(b, 'add the rotated key back')!851+	offer_now(b, mut backend)!852+	assert backend.list('proj/events')!.len == 1853+	b.begin()!854+	write_fetched(b, fetch_back_to_known(b, mut backend, 'proj')!)!855+	b.commit()!856+	assert 'secrets.env' in history.files_of(b, readded.id)!.map(it.path)857+}858+859+// A strip removes a path from the saves that existed when it was made. A key860+// rotated and saved again afterwards is new work: a machine that clones the861+// repo later must get it, not strip it on arrival.862+fn test_a_clone_keeps_a_file_saved_again_after_a_strip() {863+	base := scratch('strip_clone')!864+	defer {865+		os.rmdir_all(base) or {}866+	}867+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!868+	mut a := open_repo(machine('strip_clone_a')!)!869+	defer {870+		a.close()871+	}872+	put(a.root, 'secrets.env', 'api_key=1\n')!873+	old := commit(a, 'leak the key')!874+	offer_now(a, mut backend)!875+	strip_local(a, 'secrets.env', '')!876+	push_strip(a, mut backend, 'secrets.env')!877+	// Saves after the strip must be later in time, as they would be for real.878+	put(a.root, 'secrets.env', 'api_key=2\n')!879+	rotated := history.commit_files(a, a.root, snapshot.walk(a, a.root, snapshot.load(a.root))!, history.SaveInput{880+		message:    'rotate the key'881+		created_at: events.strip_events(a)!.last().created_at + 60882+	})!883+	offer_now(a, mut backend)!884+885+	mut c := open_repo(machine('strip_clone_c')!)!886+	defer {887+		c.close()888+	}889+	join(c, mut backend)!890+	assert 'secrets.env' in history.files_of(c, rotated.id)!.map(it.path)891+	assert 'secrets.env' !in history.files_of(c, old.id)!.map(it.path)892+	assert os.read_file(os.join_path(c.root, 'secrets.env'))! == 'api_key=2\n'893+}894+895+// The local half commits before the remote hears anything, so a remote failure896+// leaves nothing to find locally; the pending note is what lets a re-run897+// finish. Objects a remote-only save still needs must survive the strip.898+fn test_an_interrupted_remote_strip_finishes_when_re_run() {899+	base := scratch('strip_resume')!900+	defer {901+		os.rmdir_all(base) or {}902+	}903+	remote_dir := os.join_path(base, 'remote')904+	mut backend := remote.new_folder(remote_dir)!905+	mut a := open_repo(machine('resume_a')!)!906+	defer {907+		a.close()908+	}909+	mut b := open_repo(machine('resume_b')!)!910+	defer {911+		b.close()912+	}913+	put(a.root, 'secrets.env', 'api_key=1\n')!914+	put(a.root, 'shared.env', 'same bytes on B\n')!915+	save := commit(a, 'add secrets')!916+	offer_now(a, mut backend)!917+	join(b, mut backend)!918+	put(b.root, 'copy.txt', 'same bytes on B\n')!919+	b_save := commit(b, 'B keeps a copy')!920+	offer_now(b, mut backend)!921+	shared_hash := storage.hash_hex('same bytes on B\n'.bytes())922+	secret_hash := storage.hash_hex('api_key=1\n'.bytes())923+924+	strip_local(a, 'secrets.env', '')!925+	// Write 1 uploads the event and write 2 rewrites the first record; B's save,926+	// made from A's tree, lists the file too, and its rewrite (write 3) fails.927+	mut flaky := FaultyBackend{928+		inner:      remote.new_folder(remote_dir)!929+		fail_write: 3930+	}931+	if _ := push_strip(a, mut flaky, 'secrets.env') {932+		assert false, 'the injected failure should have surfaced'933+	}934+	assert strip_pending(a, 'secrets.env')935+	assert strip_local(a, 'secrets.env', '')!.files_removed == 0936+	report := push_strip(a, mut backend, 'secrets.env')!937+	assert report.records == 1938+	assert !strip_pending(a, 'secrets.env')939+	assert !backend.get(object_key('proj', secret_hash))!.found940+	record := decode_save(backend.get(save_key('proj', save.id))!.body)!941+	assert 'secrets.env' !in record.files.map(it.path)942+	b_record := decode_save(backend.get(save_key('proj', b_save.id))!.body)!943+	assert 'secrets.env' !in b_record.files.map(it.path)944+945+	// B's save still names the shared bytes, so stripping A's copy keeps them.946+	strip_local(a, 'shared.env', '')!947+	push_strip(a, mut backend, 'shared.env')!948+	assert backend.get(object_key('proj', shared_hash))!.found949+}950+951+// A machine that has not heard of a strip must not offer: its records still952+// list the stripped path, and nothing would rewrite them on the remote again.953+fn test_offer_waits_for_an_unapplied_strip() {954+	base := scratch('strip_first')!955+	defer {956+		os.rmdir_all(base) or {}957+	}958+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!959+	mut a := open_repo(machine('strip_first_a')!)!960+	defer {961+		a.close()962+	}963+	mut b := open_repo(machine('strip_first_b')!)!964+	defer {965+		b.close()966+	}967+	put(a.root, 'secrets.env', 'api_key=1\n')!968+	commit(a, 'add secrets')!969+	offer_now(a, mut backend)!970+	join(b, mut backend)!971+	strip_local(a, 'secrets.env', '')!972+	push_strip(a, mut backend, 'secrets.env')!973+	put(b.root, 'more.txt', 'B keeps working\n')!974+	later := commit(b, 'B saves with the secret still in its tree')!975+	if _ := offer_now(b, mut backend) {976+		assert false, 'B offered records that still list the stripped path'977+	}978+	steal_now(b, mut backend)!979+	offer_now(b, mut backend)!980+	record := decode_save(backend.get(save_key('proj', later.id))!.body)!981+	assert 'secrets.env' !in record.files.map(it.path)982+}983+984+// A strip whose object deletions failed, followed by a second strip of the same985+// path, must still delete the first strip's objects when it finishes.986+fn test_a_second_strip_keeps_the_first_ones_unfinished_objects() {987+	base := scratch('strip_twice')!988+	defer {989+		os.rmdir_all(base) or {}990+	}991+	remote_dir := os.join_path(base, 'remote')992+	mut backend := remote.new_folder(remote_dir)!993+	mut a := open_repo(machine('strip_twice_a')!)!994+	defer {995+		a.close()996+	}997+	put(a.root, 'secrets.env', 'first key\n')!998+	commit(a, 'first key')!999+	offer_now(a, mut backend)!1000+	strip_local(a, 'secrets.env', '')!1001+	mut flaky := FaultyBackend{1002+		inner:       remote.new_folder(remote_dir)!1003+		fail_delete: true1004+	}1005+	if _ := push_strip(a, mut flaky, 'secrets.env') {1006+		assert false, 'the injected delete failure should have surfaced'1007+	}1008+	put(a.root, 'secrets.env', 'second key\n')!1009+	commit(a, 'second key')!1010+	offer_now(a, mut backend)!1011+	strip_local(a, 'secrets.env', '')!1012+	push_strip(a, mut backend, 'secrets.env')!1013+	for key in ['first key\n', 'second key\n'] {1014+		assert !backend.get(object_key('proj', storage.hash_hex(key.bytes())))!.found1015+	}1016+}1017+1018+// Unfinished strips of one path in different scopes finish together.1019+fn test_unfinished_strips_of_one_path_in_different_saves_finish_together() {1020+	base := scratch('strip_scopes')!1021+	defer {1022+		os.rmdir_all(base) or {}1023+	}1024+	remote_dir := os.join_path(base, 'remote')1025+	mut backend := remote.new_folder(remote_dir)!1026+	mut a := open_repo(machine('strip_scopes_a')!)!1027+	defer {1028+		a.close()1029+	}1030+	put(a.root, 'secrets.env', 'first key\n')!1031+	first := commit(a, 'first key')!1032+	put(a.root, 'secrets.env', 'second key\n')!1033+	second := commit(a, 'second key')!1034+	offer_now(a, mut backend)!1035+	strip_local(a, 'secrets.env', first.id)!1036+	// Write 1 uploads the event; write 2, the record rewrite, fails.1037+	mut flaky := FaultyBackend{1038+		inner:      remote.new_folder(remote_dir)!1039+		fail_write: 21040+	}1041+	if _ := push_strip(a, mut flaky, 'secrets.env') {1042+		assert false, 'the injected failure should have surfaced'1043+	}1044+	strip_local(a, 'secrets.env', second.id)!1045+	push_strip(a, mut backend, 'secrets.env')!1046+	assert !strip_pending(a, 'secrets.env')1047+	for save in [first, second] {1048+		record := decode_save(backend.get(save_key('proj', save.id))!.body)!1049+		assert 'secrets.env' !in record.files.map(it.path)1050+	}1051+	for key in ['first key\n', 'second key\n'] {1052+		assert !backend.get(object_key('proj', storage.hash_hex(key.bytes())))!.found1053+	}1054+}1055+1056+// Stripping a path, re-adding it, and stripping it again is two strips; a1057+// machine that heard about the first must still apply the second.1058+fn test_a_second_strip_of_the_same_path_reaches_other_machines() {1059+	base := scratch('strip_again')!1060+	defer {1061+		os.rmdir_all(base) or {}1062+	}1063+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!1064+	mut a := open_repo(machine('strip_again_a')!)!1065+	defer {1066+		a.close()1067+	}1068+	mut b := open_repo(machine('strip_again_b')!)!1069+	defer {1070+		b.close()1071+	}1072+	put(a.root, 'secrets.env', 'first key\n')!1073+	commit(a, 'first key')!1074+	offer_now(a, mut backend)!1075+	join(b, mut backend)!1076+	strip_local(a, 'secrets.env', '')!1077+	push_strip(a, mut backend, 'secrets.env')!1078+	steal_now(b, mut backend)!1079+	// Strip leaves the file on disk; deleting it is the user's part.1080+	os.rm(os.join_path(b.root, 'secrets.env'))!1081+	put(a.root, 'secrets.env', 'second key\n')!1082+	second := commit(a, 'second key')!1083+	offer_now(a, mut backend)!1084+	steal_now(b, mut backend)!1085+	assert 'secrets.env' in history.files_of(b, second.id)!.map(it.path)1086+	strip_local(a, 'secrets.env', '')!1087+	push_strip(a, mut backend, 'secrets.env')!1088+	steal_now(b, mut backend)!1089+	assert 'secrets.env' !in history.files_of(b, second.id)!.map(it.path)1090+}1091+1092+// A clash on a path the other machine stripped is no clash: the strip removes1093+// the path from every save, so there is nothing to choose.1094+fn test_a_stripped_path_is_not_asked_about() {1095+	base := scratch('clash_strip')!1096+	defer {1097+		os.rmdir_all(base) or {}1098+	}1099+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!1100+	mut a := open_repo(machine('clash_strip_a')!)!1101+	defer {1102+		a.close()1103+	}1104+	mut b := open_repo(machine('clash_strip_b')!)!1105+	defer {1106+		b.close()1107+	}1108+	put(a.root, 'secrets.env', 'key one\n')!1109+	commit(a, 'base')!1110+	offer_now(a, mut backend)!1111+	join(b, mut backend)!1112+	put(a.root, 'secrets.env', 'key two\n')!1113+	commit(a, 'A rotates')!1114+	offer_now(a, mut backend)!1115+	strip_local(a, 'secrets.env', '')!1116+	push_strip(a, mut backend, 'secrets.env')!1117+	put(b.root, 'secrets.env', 'key three\n')!1118+	mine := commit(b, 'B rotates')!1119+	report := steal(b, mut backend, StealOptions{})!1120+	assert report.clashes.len == 01121+	for save in history.all_saves(b)! {1122+		assert 'secrets.env' !in history.files_of(b, save.id)!.map(it.path)1123+	}1124+	assert history.by_id(b, mine.id) != none1125+}1126+1127+// A strip scoped to the shared base save changes what the replay sees; the1128+// clashes must be found on the same trees, or the replay meets a conflict.1129+fn test_a_strip_of_the_shared_base_is_seen_before_asking() {1130+	base := scratch('clash_base_strip')!1131+	defer {1132+		os.rmdir_all(base) or {}1133+	}1134+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!1135+	mut a := open_repo(machine('base_strip_a')!)!1136+	defer {1137+		a.close()1138+	}1139+	mut b := open_repo(machine('base_strip_b')!)!1140+	defer {1141+		b.close()1142+	}1143+	put(a.root, 'notes.txt', 'shared\n')!1144+	common := commit(a, 'base')!1145+	offer_now(a, mut backend)!1146+	join(b, mut backend)!1147+	put(a.root, 'other.txt', 'A moves on\n')!1148+	commit(a, 'A moves on')!1149+	offer_now(a, mut backend)!1150+	strip_local(a, 'notes.txt', common.id)!1151+	push_strip(a, mut backend, 'notes.txt')!1152+	put(b.root, 'notes.txt', 'B edits\n')!1153+	commit(b, 'B edits notes')!1154+	if _ := steal(b, mut backend, StealOptions{}) {1155+		assert false, 'with the base stripped, notes.txt differs on both sides'1156+	} else {1157+		assert err.msg().contains('changed on both machines')1158+	}1159+	report := steal(b, mut backend, StealOptions{1160+		mine: true1161+	})!1162+	assert report.clashes == ['notes.txt']1163+	assert os.read_file(os.join_path(b.root, 'notes.txt'))! == 'B edits\n'1164+}1165+1166+// A folder with files but no saves has nothing to combine with; a steal that1167+// fast-forwarded into it would overwrite whatever shares a name.1168+fn test_steal_into_a_folder_with_unsaved_files_refuses() {1169+	base := scratch('steal_unsaved')!1170+	defer {1171+		os.rmdir_all(base) or {}1172+	}1173+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!1174+	mut a := open_repo(machine('unsaved_a')!)!1175+	defer {1176+		a.close()1177+	}1178+	put(a.root, 'notes.txt', 'from A\n')!1179+	commit(a, 'A writes')!1180+	offer_now(a, mut backend)!1181+	b_root := machine('unsaved_b')!1182+	mut b := storage.create(b_root)!1183+	defer {1184+		b.close()1185+	}1186+	put(b_root, 'notes.txt', 'never saved\n')!1187+	// Only a repo that already knows its remote name gets this far.1188+	b.set_state('remote_repo', 'proj')!1189+	if _ := steal(b, mut backend, StealOptions{}) {1190+		assert false, 'steal overwrote a folder with unsaved files'1191+	} else {1192+		assert err.msg() == no_saves_here1193+	}1194+	assert os.read_file(os.join_path(b_root, 'notes.txt'))! == 'never saved\n'1195+	// A fresh `koh init` folder, holding only the default ignore list, is fine.1196+	os.rm(os.join_path(b_root, 'notes.txt'))!1197+	put(b_root, snapshot.ignore_file, snapshot.default_ignore)!1198+	steal(b, mut backend, StealOptions{})!1199+	assert os.read_file(os.join_path(b_root, 'notes.txt'))! == 'from A\n'1200+}1201+1202+// The document asks steal to warn about every incoming flagged save, so the1203+// report names them.1204+fn test_steal_reports_incoming_flagged_saves() {1205+	base := scratch('flag_warn')!1206+	defer {1207+		os.rmdir_all(base) or {}1208+	}1209+	mut backend := remote.new_folder(os.join_path(base, 'remote'))!1210+	mut a := open_repo(machine('flag_warn_a')!)!1211+	defer {1212+		a.close()1213+	}1214+	mut b := open_repo(machine('flag_warn_b')!)!1215+	defer {1216+		b.close()1217+	}1218+	put(a.root, 'a.txt', 'one\n')!1219+	commit(a, 'fine')!1220+	offer_now(a, mut backend)!1221+	join(b, mut backend)!1222+	put(a.root, 'a.txt', 'two\n')!1223+	suspect := commit(a, 'suspect')!1224+	put(a.root, 'a.txt', 'three\n')!1225+	commit(a, 'also fine')!1226+	events.flag(a, suspect.id, 'broke the build')!1227+	offer_now(a, mut backend)!1228+	report := steal_now(b, mut backend)!1229+	assert report.flagged.map(it.save_id) == [suspect.id]1230+	assert report.flagged[0].reason == 'broke the build'1231+}1232+1233+// head_of unwraps the head for tests that know the repo has saves.1234+fn head_of(store &storage.Store) history.Save {1235+	return history.head(store) or { panic('this repo has no head') }1236+}

view file

src/exchange/records.v+166 −0
@@ -0,0 +1,166 @@1+//! What a save and an event look like on the remote.2+//!3+//! Records are JSON so a person can read the bucket, and the layout is the4+//! same for every backend:5+//!6+//!   <repo>/objects/<hash>   the object's compressed bytes, as in the table7+//!   <repo>/saves/<id>       the full save record, including its file list8+//!   <repo>/events/<id>      one flag, clear or strip event9+//!   <repo>/head             the ID of the newest save on the line10+//!11+//! Only `head` is ever overwritten, and only with a matching version tag.12+13+module exchange14+15+import json216+import src.events17+import src.history18+import src.snapshot19+import src.storage20+21+pub fn object_key(repo string, hash string) string {22+	return '${repo}/objects/${hash}'23+}24+25+pub fn save_key(repo string, id string) string {26+	return '${repo}/saves/${id}'27+}28+29+pub fn event_key(repo string, id string) string {30+	return '${repo}/events/${id}'31+}32+33+pub fn head_key(repo string) string {34+	return '${repo}/head'35+}36+37+pub fn events_prefix(repo string) string {38+	return '${repo}/events'39+}40+41+pub struct FileRecord {42+pub:43+	path       string44+	hash       string45+	executable bool46+}47+48+pub struct SaveRecord {49+pub:50+	id            string51+	parent        string52+	message       string53+	created_at    i6454+	machine       string55+	is_checkpoint bool56+	size_bytes    i6457+	agent         string58+	session       string59+	restores      string60+	replayed_from string61+	files         []FileRecord62+}63+64+// save_record turns a local save into what travels. The file list is part of65+// the record, which is why a save needs one round trip, not one per file.66+pub fn save_record(store &storage.Store, save history.Save) !SaveRecord {67+	files := history.files_of(store, save.id)!68+	mut records := []FileRecord{}69+	for file in files {70+		records << FileRecord{71+			path:       file.path72+			hash:       file.hash73+			executable: file.executable74+		}75+	}76+	return SaveRecord{77+		id:            save.id78+		parent:        save.parent79+		message:       save.message80+		created_at:    save.created_at81+		machine:       save.machine82+		is_checkpoint: save.is_checkpoint83+		size_bytes:    save.size_bytes84+		agent:         save.agent85+		session:       save.session86+		restores:      save.restores87+		replayed_from: save.replayed_from88+		files:         records89+	}90+}91+92+pub fn encode_save(record SaveRecord) string {93+	return json2.encode(record)94+}95+96+pub fn decode_save(body []u8) !SaveRecord {97+	return json2.decode[SaveRecord](body.bytestr())!98+}99+100+pub fn to_save(record SaveRecord) history.Save {101+	return history.Save{102+		id:            record.id103+		parent:        record.parent104+		message:       record.message105+		created_at:    record.created_at106+		machine:       record.machine107+		is_checkpoint: record.is_checkpoint108+		size_bytes:    record.size_bytes109+		agent:         record.agent110+		session:       record.session111+		restores:      record.restores112+		replayed_from: record.replayed_from113+	}114+}115+116+pub fn to_files(record SaveRecord) []snapshot.FileEntry {117+	mut files := []snapshot.FileEntry{}118+	for file in record.files {119+		files << snapshot.FileEntry{120+			path:       file.path121+			hash:       file.hash122+			executable: file.executable123+		}124+	}125+	return files126+}127+128+pub struct EventRecord {129+pub:130+	id         string131+	kind       string132+	save_id    string133+	path       string134+	reason     string135+	created_at i64136+	machine    string137+}138+139+pub fn event_record(event events.Event) EventRecord {140+	return EventRecord{141+		id:         event.id142+		kind:       event.kind143+		save_id:    event.save_id144+		path:       event.path145+		reason:     event.reason146+		created_at: event.created_at147+		machine:    event.machine148+	}149+}150+151+pub fn encode_event(event events.Event) string {152+	return json2.encode(event_record(event))153+}154+155+pub fn decode_event(body []u8) !events.Event {156+	record := json2.decode[EventRecord](body.bytestr())!157+	return events.Event{158+		id:         record.id159+		kind:       record.kind160+		save_id:    record.save_id161+		path:       record.path162+		reason:     record.reason163+		created_at: record.created_at164+		machine:    record.machine165+	}166+}

view file

src/exchange/steal.v+497 −0
@@ -0,0 +1,497 @@1+//! Stealing: bringing another machine's saves in, and combining when both2+//! machines moved.3+//!4+//! The order is always the same, and it is what makes an interrupted steal5+//! harmless: read everything from the remote first, then land it in one6+//! transaction, then write the working tree. Divergence is resolved per file —7+//! a clash is never merged line by line, it is chosen.8+9+module exchange10+11+import compress.zlib12+import os13+import src.apply14+import src.events15+import src.history16+import src.remote17+import src.snapshot18+import src.storage19+20+pub enum Side {21+	mine22+	theirs23+}24+25+// Clash is one file both machines changed differently. `stat` is the short26+// diff stat the terminal prompt shows, so the choice is informed.27+pub struct Clash {28+pub mut:29+	path   string30+	mine   snapshot.FileEntry31+	theirs snapshot.FileEntry32+	base   snapshot.FileEntry33+	stat   string34+}35+36+pub struct StealOptions {37+pub mut:38+	mine   bool // resolve every clash as mine39+	theirs bool // resolve every clash as theirs40+	// Asked once per clash when neither flag is set. Nil means "nobody to ask".41+	ask fn (clash Clash) Side = unsafe { nil }42+}43+44+pub struct StealReport {45+pub mut:46+	repo            string47+	forward         bool48+	fetched_saves   int49+	fetched_objects int50+	fetched_events  int51+	clashes         []string52+	kept_mine       int53+	replayed        int54+	removed         int55+	head            string56+	flagged         []events.Flag // incoming saves that are flagged57+	// Set when a combine took this machine's saves off the line: loading it58+	// brings back the tree from before the steal, less anything stripped.59+	previous_head string60+}61+62+pub const dirty_tree = 'save or checkpoint first — steal needs a clean tree'63+pub const no_saves_here = 'this folder has files but no saves — steal a repo into an empty folder with "koh steal <name>"'64+65+// steal brings the remote's new saves in. Inside a repo with its own work this66+// may have to combine; a repo with nothing of its own simply fast-forwards.67+pub fn steal(store &storage.Store, mut backend remote.Backend, opts StealOptions) !StealReport {68+	name := repo_name(store, mut backend)!69+	if snapshot.restore_pending(store) {70+		return error(snapshot.restore_unfinished)71+	}72+	head_before := history.head(store)73+	// Combining rewrites the working tree, so unsaved work would be lost.74+	if dirty(store, head_before)! {75+		if head_before == none {76+			return error(no_saves_here)77+		}78+		return error(dirty_tree)79+	}80+	fetched := fetch_back_to_known(store, mut backend, name)!81+	mut report := StealReport{82+		repo:            name83+		fetched_saves:   fetched.saves.len84+		fetched_objects: fetched.repo_objects.len85+		fetched_events:  fetched.events_out.len86+		head:            fetched.head87+	}88+	mut local_only := []history.Save{}89+	if head := head_before {90+		// The remote builds on `base`. If that save is here but no longer on91+		// the line, neither line contains the other; jumping to the remote92+		// would silently leave this line behind.93+		local_only = saves_after(store, head.id, fetched.base) or {94+			return error('the remote builds on save ${fetched.base}, which is not on this line — load the save you want, then offer')95+		}96+	}97+	if fetched.base == '' && local_only.len > 0 {98+		return error('the remote keeps a different history — it shares no save with this repo')99+	}100+101+	strips_before := events.strip_events(store)!.len102+	mut clashes := []Clash{}103+	mut side_of := map[string]Side{}104+	if fetched.saves.len > 0 && local_only.len > 0 {105+		clashes = combine_clashes(store, fetched)!106+		report.clashes = clashes.map(it.path)107+		// Asked before the write lock: a person thinking must not stall every108+		// agent's save on this repo.109+		side_of = choose_sides(clashes, opts)!110+	}111+112+	store.begin() or { return err }113+	// The questions ran without the lock. If a save or an edit landed114+	// meanwhile, the answers were about a line that no longer exists.115+	if moved := changed_since(store, head_before, strips_before) {116+		if moved {117+			store.rollback()118+			return error('this repo changed while steal was asking — run koh steal again')119+		}120+	} else {121+		store.rollback()122+		return err123+	}124+	final := (write_and_combine(store, fetched, local_only, clashes, side_of, mut report) or {125+		store.rollback()126+		return err127+	})128+	files := history.files_of(store, final.id) or {129+		store.rollback()130+		return err131+	}132+	snapshot.check_links(store.root, files) or {133+		store.rollback()134+		return err135+	}136+	// The files follow the commit; the mark lands with it, in one step.137+	// No head before means a fresh clone, whose whole tree is new.138+	moved := if head := head_before { final.id != head.id } else { true }139+	if moved {140+		snapshot.mark_restore(store) or {141+			store.rollback()142+			return err143+		}144+	}145+	store.commit() or { return err }146+	report.head = final.id147+	if local_only.len > 0 && report.fetched_saves > 0 {148+		report.previous_head = local_only.last().id149+	}150+	flags := events.latest_flags(store)!151+	for record in fetched.saves {152+		if flag := flags[record.id] {153+			if flag.flagged {154+				report.flagged << flag155+			}156+		}157+	}158+159+	if head := head_before {160+		if final.id == head.id {161+			// The line did not move, so neither does the working tree.162+			return report163+		}164+		mut wanted := map[string]bool{}165+		for file in files {166+			wanted[file.path] = true167+		}168+		mut remove := []string{}169+		for file in history.files_of(store, head.id)! {170+			if file.path !in wanted {171+				remove << file.path172+			}173+		}174+		result := snapshot.restore_files(store, store.root, snapshot.load(store.root), files,175+			remove)!176+		report.removed = result.removed177+	} else {178+		snapshot.restore_files(store, store.root, snapshot.load(store.root), files, [])!179+	}180+	history.refresh_log_md(store)181+	return report182+}183+184+// write_and_combine lands a fetch and, when the local line has saves the remote185+// lacks, replays them on top of the remote head. It runs inside the caller's186+// transaction and returns the new head.187+fn write_and_combine(store &storage.Store, fetched Fetched, local_only []history.Save, clashes []Clash, side_of map[string]Side, mut report StealReport) !history.Save {188+	write_fetched(store, fetched)!189+	if fetched.saves.len == 0 {190+		// The remote head is already on this line: level, or this machine is191+		// ahead and the offer that follows a steal sends the rest.192+		return history.head(store) or { return error('this repo has no head') }193+	}194+	if local_only.len == 0 {195+		// Nothing of ours is missing from the remote: the line simply moves on.196+		history.set_head(store, fetched.head)!197+		report.forward = true198+		return history.head(store) or { return error('the remote head did not land') }199+	}200+	remote_files := history.files_of(store, fetched.head)!201+	mut mine_files := []snapshot.FileEntry{}202+	if our_head := history.head(store) {203+		mine_files = history.files_of(store, our_head.id)!204+	}205+206+	// A file the remote changed is a clash, decided below, or one this machine207+	// left as it was or changed the same way, so the remote's version stands.208+	// Replaying this machine's steps on it could only conflict: an edit it209+	// later undid would no longer apply.210+	base_files := history.files_of(store, fetched.base)!211+	settled := changed_paths(base_files, remote_files)212+213+	// Take the remote's line first, so the replayed saves chain onto it.214+	history.set_head(store, fetched.head)!215+	mut current := remote_files.clone()216+	for save in local_only {217+		parent_files := if save.parent != '' {218+			history.files_of(store, save.parent)!219+		} else {220+			[]snapshot.FileEntry{}221+		}222+		target_files := history.files_of(store, save.id)!223+		plan := apply.three_way(current, parent_files, target_files, [], settled)224+		if apply.conflicts(plan) {225+			return error('combine could not apply ${plan.conflict} — it is not a clash file')226+		}227+		if plan.applied.len == 0 {228+			// Everything this save touched was a file the remote changed.229+			continue230+		}231+		history.record_save(store, '', plan.files, history.SaveInput{232+			message:       save.message233+			created_at:    save.created_at234+			machine:       save.machine235+			agent:         save.agent236+			session:       save.session237+			is_checkpoint: save.is_checkpoint238+			replayed_from: save.id239+		})!240+		current = plan.files.clone()241+		report.replayed++242+	}243+244+	mut mine_paths := []string{}245+	for clash in clashes {246+		if side_of[clash.path] == .mine {247+			mine_paths << clash.path248+		}249+	}250+	if mine_paths.len > 0 {251+		// One last save puts my versions of the chosen files back.252+		final_plan := apply.three_way(current, current, mine_files, mine_paths, [])253+		if final_plan.applied.len > 0 {254+			history.record_save(store, '', final_plan.files, history.SaveInput{255+				message: 'combine: kept mine for ${mine_paths.len} ${if mine_paths.len == 1 {256+					'file'257+				} else {258+					'files'259+				}}'260+			})!261+			report.kept_mine = mine_paths.len262+		}263+	}264+	return history.head(store) or { return error('the combined line has no head') }265+}266+267+// find_clashes lists the files both sides changed, differently. Files changed268+// identically on both sides are not clashes: there is nothing to decide.269+pub fn find_clashes(base []snapshot.FileEntry, mine []snapshot.FileEntry, theirs []snapshot.FileEntry) []Clash {270+	base_map := snapshot.index_files(base)271+	mine_map := snapshot.index_files(mine)272+	theirs_map := snapshot.index_files(theirs)273+	mut paths := mine_map.keys()274+	for path in theirs_map.keys() {275+		if path !in mine_map {276+			paths << path277+		}278+	}279+	paths.sort()280+	mut clashes := []Clash{}281+	for path in paths {282+		here := mine_map[path] or {283+			snapshot.FileEntry{284+				path: path285+			}286+		}287+		there := theirs_map[path] or {288+			snapshot.FileEntry{289+				path: path290+			}291+		}292+		was := base_map[path] or {293+			snapshot.FileEntry{294+				path: path295+			}296+		}297+		if same(here, there) {298+			continue299+		}300+		if same(here, was) || same(there, was) {301+			continue302+		}303+		clashes << Clash{304+			path:   path305+			mine:   here306+			theirs: there307+			base:   was308+		}309+	}310+	return clashes311+}312+313+// changed_paths lists the paths whose entry differs between two trees.314+fn changed_paths(before []snapshot.FileEntry, after []snapshot.FileEntry) []string {315+	was := snapshot.index_files(before)316+	now := snapshot.index_files(after)317+	mut paths := []string{}318+	for path, entry in now {319+		if !same(entry, was[path] or { snapshot.FileEntry{} }) {320+			paths << path321+		}322+	}323+	for path, _ in was {324+		if path !in now {325+			paths << path326+		}327+	}328+	return paths329+}330+331+// combine_clashes finds the files both lines changed, with the diff stat each332+// prompt shows. It reads the remote head from the fetch, not the store, so it333+// can run before anything is written.334+fn combine_clashes(store &storage.Store, fetched Fetched) ![]Clash {335+	base_files := if fetched.base != '' {336+		history.files_of(store, fetched.base)!337+	} else {338+		[]snapshot.FileEntry{}339+	}340+	head := history.head(store) or { return []Clash{} }341+	// The transaction applies the incoming strips before it replays, so the342+	// clashes are found on the trees as they will be then, or the answers343+	// would be about files that are no longer there.344+	strips := fetched.events_out.filter(it.kind == events.kind_strip)345+	// Base and mine are here already, so every strip reaches them (created 0).346+	remote_head := fetched.saves.last()347+	mut clashes := find_clashes(after_strips(base_files, fetched.base, 0, strips), after_strips(history.files_of(store,348+		head.id)!, head.id, 0, strips), after_strips(to_files(remote_head), remote_head.id,349+		remote_head.created_at, strips))350+	for mut clash in clashes {351+		mine := fetched_bytes(store, fetched, clash.mine)!352+		theirs := fetched_bytes(store, fetched, clash.theirs)!353+		delta := snapshot.diff_bytes(clash.path, 'modified', mine, theirs)!354+		clash.stat = if delta.binary { 'binary' } else { '+${delta.added} -${delta.removed}' }355+	}356+	return clashes357+}358+359+// after_strips is a save's file list once the given strip events are applied,360+// by the same rule write_fetched uses.361+fn after_strips(files []snapshot.FileEntry, save_id string, created_at i64, strips []events.Event) []snapshot.FileEntry {362+	return files.filter(fn [strips, save_id, created_at] (file snapshot.FileEntry) bool {363+		return !strips.any(reaches(it, save_id, created_at) && covers(it.path, file.path))364+	})365+}366+367+// fetched_bytes reads one side of a clash: from the fetch when it has just368+// arrived, else from the store. An absent side is empty.369+fn fetched_bytes(store &storage.Store, fetched Fetched, entry snapshot.FileEntry) ![]u8 {370+	if entry.hash == '' {371+		return []u8{}372+	}373+	if blob := fetched.repo_objects[entry.hash] {374+		return zlib.decompress(blob)!375+	}376+	return store.get_object(entry.hash)!377+}378+379+fn choose_sides(clashes []Clash, opts StealOptions) !map[string]Side {380+	mut sides := map[string]Side{}381+	if clashes.len == 0 {382+		return sides383+	}384+	for clash in clashes {385+		if opts.mine {386+			sides[clash.path] = .mine387+			continue388+		}389+		if opts.theirs {390+			sides[clash.path] = .theirs391+			continue392+		}393+		if opts.ask != unsafe { nil } {394+			sides[clash.path] = opts.ask(clash)395+			continue396+		}397+		return error('${clashes.len} files changed on both machines — run on a terminal to choose, or pass --mine or --theirs')398+	}399+	return sides400+}401+402+fn same(a snapshot.FileEntry, b snapshot.FileEntry) bool {403+	if a.hash == '' && b.hash == '' {404+		return true405+	}406+	return a.hash == b.hash && a.executable == b.executable407+}408+409+fn changed_since(store &storage.Store, before ?history.Save, strips_before int) !bool {410+	if events.strip_events(store)!.len != strips_before {411+		return true412+	}413+	now := history.head(store)414+	if now or { history.Save{} }.id != before or { history.Save{} }.id {415+		return true416+	}417+	return dirty(store, now)!418+}419+420+// dirty reports whether the working tree holds anything a steal would421+// overwrite. With no saves yet, that is every file but the untouched ignore422+// list `koh init` wrote.423+fn dirty(store &storage.Store, head ?history.Save) !bool {424+	files := snapshot.walk(store, store.root, snapshot.load(store.root))!425+	if saved := head {426+		return !snapshot.same_tree(history.files_of(store, saved.id)!, files)427+	}428+	seed := storage.hash_hex(snapshot.default_ignore.bytes())429+	return files.any(it.path != snapshot.ignore_file || it.hash != seed)430+}431+432+// fetch_repo pulls a whole repo — head, saves, objects and events — for433+// `koh steal <name>`, which lands a copy in a new folder.434+pub fn fetch_repo(mut backend remote.Backend, name string) !Fetched {435+	remote_head := backend.get(head_key(name))!436+	if !remote_head.found {437+		return error('the remote has no repo called "${name}"')438+	}439+	save_ids := backend.list('${name}/saves')!440+	object_keys := backend.list('${name}/objects')!441+	event_keys := backend.list('${name}/events')!442+	mut fetched := Fetched{443+		head: remote_head.body.bytestr().trim_space()444+	}445+	for key in object_keys {446+		blob := backend.get(key)!447+		if blob.found {448+			fetched.repo_objects[os.base(key)] = blob.body449+		}450+	}451+	for key in save_ids {452+		body := backend.get(key)!453+		if !body.found {454+			continue455+		}456+		record := decode_save(body.body)!457+		fetched.saves << record458+	}459+	fetched.saves = parent_first(fetched.saves)!460+	for key in event_keys {461+		body := backend.get(key)!462+		if body.found {463+			fetched.events_out << decode_event(body.body)!464+		}465+	}466+	return fetched467+}468+469+// parent_first orders saves so every parent lands before its children, as the470+// foreign key on `parent` demands. Time cannot do it: a replayed save keeps its471+// original, older time but sits on a newer parent.472+fn parent_first(records []SaveRecord) ![]SaveRecord {473+	mut by_id := map[string]SaveRecord{}474+	for record in records {475+		by_id[record.id] = record476+	}477+	mut placed := map[string]bool{}478+	mut ordered := []SaveRecord{}479+	for record in records {480+		mut chain := []SaveRecord{}481+		mut cursor := record.id482+		for cursor != '' && cursor !in placed {483+			if chain.len > records.len {484+				return error('the save records on the remote form a loop')485+			}486+			current := by_id[cursor] or { return error('the remote is missing save ${cursor}') }487+			chain << current488+			cursor = current.parent489+		}490+		chain.reverse_in_place()491+		for current in chain {492+			placed[current.id] = true493+			ordered << current494+		}495+	}496+	return ordered497+}

view file

src/exchange/strip.v+227 −0
@@ -0,0 +1,227 @@1+//! Strip: removing a path from history, here and on the remote.2+//!3+//! The local half is one transaction: the saved files go, the objects nobody4+//! references go, a strip event is recorded, and so is a note of which objects5+//! the remote may now drop. The remote half reads that note, so it can be6+//! re-run after an interruption even though the local rows are long gone.7+8+module exchange9+10+import json211+import os12+import src.events13+import src.remote14+import src.storage15+16+// Scope is what a strip would remove, for the warning shown before it runs.17+pub struct Scope {18+pub:19+	files   int20+	saves   int21+	objects int22+}23+24+pub struct LocalStrip {25+pub:26+	files_removed   int27+	objects_dropped i6428+}29+30+// PushReport says what a strip managed to make visible on the remote.31+pub struct PushReport {32+pub mut:33+	records    int34+	objects    int35+	event_sent bool36+}37+38+// Pending is the note a strip leaves until its remote half has finished: which39+// saves it covered ('' for every save) and which objects it freed. One note per40+// path, so strips of the same path that did not finish are finished together.41+struct Pending {42+mut:43+	saves  []string44+	hashes []string45+}46+47+fn pending_key(path string) string {48+	return 'strip_pending:${path}'49+}50+51+fn read_pending(store &storage.Store, path string) Pending {52+	text := store.state(pending_key(path)) or { return Pending{} }53+	return json2.decode[Pending](text) or { Pending{} }54+}55+56+pub fn strip_pending(store &storage.Store, path string) bool {57+	return store.state(pending_key(path)) != none58+}59+60+// covers is the one rule for what a strip removes: the path itself, or61+// anything beneath it when it names a folder.62+fn covers(path string, file string) bool {63+	return file == path || file.starts_with('${path.trim_right('/')}/')64+}65+66+pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {67+	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)68+		FROM save_files WHERE ${match_clause(only_save)}')!69+	defer {70+		stmt.finalize()71+	}72+	bind_match(stmt, path, only_save)!73+	stmt.step()!74+	return Scope{75+		files:   int(stmt.int(0))76+		saves:   int(stmt.int(1))77+		objects: int(stmt.int(2))78+	}79+}80+81+// strip_local removes the path in one transaction and leaves the note the82+// remote half needs.83+pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {84+	store.begin()!85+	report := strip_rows(store, path, only_save) or {86+		store.rollback()87+		return err88+	}89+	store.commit()!90+	return report91+}92+93+fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {94+	mut hashes := []string{}95+	mut stmt :=96+		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!97+	bind_match(stmt, path, only_save)!98+	for stmt.step()! {99+		hashes << stmt.text(0)100+	}101+	stmt.finalize()102+	if hashes.len == 0 {103+		// Already stripped: keep the pending note an unfinished remote needs.104+		return LocalStrip{}105+	}106+	removed := remove_path(store, path, only_save)!107+	dropped := store.drop_unreferenced_objects()!108+	events.strip(store, only_save, path)!109+	// An earlier strip of the same path may not have finished on the remote,110+	// so the note grows rather than resets.111+	mut note := read_pending(store, path)112+	if only_save !in note.saves {113+		note.saves << only_save114+	}115+	note.hashes << hashes116+	store.set_state(pending_key(path), json2.encode(note))!117+	return LocalStrip{118+		files_removed:   removed119+		objects_dropped: dropped120+	}121+}122+123+// push_strip makes a strip visible on the remote: the event travels like any124+// other, every remote save record listing the path is rewritten, and objects125+// the strip freed are deleted once no remote record and no local save needs126+// them. Every step is safe to repeat, and the pending note is cleared only127+// when all of them have succeeded.128+pub fn push_strip(store &storage.Store, mut backend remote.Backend, path string) !PushReport {129+	name := repo_name(store, mut backend)!130+	mut report := PushReport{}131+	mut upload := OfferReport{132+		repo: name133+	}134+	upload_events(store, mut backend, name, mut upload)!135+	report.event_sent = upload.events > 0136+	note := read_pending(store, path)137+	mut freed := map[string]bool{}138+	for hash in note.hashes {139+		freed[hash] = true140+	}141+	every_save := '' in note.saves142+	mut needed := map[string]bool{}143+	for key in backend.list('${name}/saves')! {144+		remote_object := backend.get(key)!145+		if !remote_object.found {146+			continue147+		}148+		record := decode_save(remote_object.body)!149+		mut kept := []FileRecord{}150+		for file in record.files {151+			if (every_save || record.id in note.saves) && covers(path, file.path) {152+				freed[file.hash] = true153+				continue154+			}155+			kept << file156+			needed[file.hash] = true157+		}158+		if kept.len == record.files.len {159+			continue160+		}161+		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {162+			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')163+		}164+		report.records++165+	}166+	for hash, _ in freed {167+		if hash in needed || store.has_object(hash)! {168+			continue169+		}170+		backend.delete(object_key(name, hash))!171+		report.objects++172+	}173+	store.remove_state(pending_key(path))!174+	return report175+}176+177+// unapplied_strip reports whether the remote holds a strip event this repo178+// has not received yet.179+fn unapplied_strip(store &storage.Store, mut backend remote.Backend, repo string) !bool {180+	mut known := map[string]bool{}181+	for event in events.all(store)! {182+		known[event.id] = true183+	}184+	for key in backend.list(events_prefix(repo))! {185+		if os.base(key) in known {186+			continue187+		}188+		body := backend.get(key)!189+		if body.found && decode_event(body.body)!.kind == events.kind_strip {190+			return true191+		}192+	}193+	return false194+}195+196+// rewrite returns the save record with a new file list.197+fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {198+	return SaveRecord{199+		...record200+		files: files201+	}202+}203+204+fn remove_path(store &storage.Store, path string, only_save string) !int {205+	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!206+	defer {207+		stmt.finalize()208+	}209+	bind_match(stmt, path, only_save)!210+	stmt.step()!211+	return store.db.changes()212+}213+214+// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`215+// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.216+fn match_clause(only_save string) string {217+	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }218+	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'219+}220+221+fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {222+	stmt.bind_text(1, path)!223+	stmt.bind_text(2, '${path.trim_right('/')}/')!224+	if only_save != '' {225+		stmt.bind_text(3, only_save)!226+	}227+}

view file

src/migrate/migrate.v+2 −2
@@ -10,7 +10,7 @@1010 module migrate1111 1212 import encoding.base6413-import json13+import json21414 import os1515 import src.events1616 import src.history@@ -136,7 +136,7 @@136136 			continue137137 		}138138 		body := os.read_file(os.join_path(dir, name))!139-		manifests << json.decode(Manifest, body)!139+		manifests << json2.decode[Manifest](body)!140140 	}141141 	return manifests142142 }

view file

src/migrate/migrate_test.v+3 −3
@@ -7,7 +7,7 @@77 88 module migrate99 10-import json10+import json21111 import os1212 import src.events1313 import src.history@@ -56,7 +56,7 @@5656 			continue5757 		}5858 		record :=59-			json.decode(Manifest, os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!59+			json2.decode[Manifest](os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!6060 		if record.message == 'first save' {6161 			first = record.id6262 		}@@ -102,7 +102,7 @@102102 		if !name.ends_with('.json') {103103 			continue104104 		}105-		record := json.decode(Manifest,105+		record := json2.decode[Manifest](106106 			os.read_file(os.join_path(fixture.dir, '.koh', 'snapshots', name))!)!107107 		mut files := []string{}108108 		for file in record.files {

view file

src/remote/remote.v+1 −1
@@ -1,7 +1,7 @@11 //! The remote interface: five operations, two backends.22 //!33 //! A remote is dumb storage that can do one safe swap. Everything clever —4-//! what to upload, what diverged, how to combine — lives in `src/sync`, above4+//! what to upload, what diverged, how to combine — lives in `src/exchange`, above55 //! this interface, so the folder and S3 backends stay thin and the sync logic66 //! is written once and tested offline.77 

view file

src/sync/records.v+0 −166
@@ -1,166 +0,0 @@1-//! What a save and an event look like on the remote.2-//!3-//! Records are JSON so a person can read the bucket, and the layout is the4-//! same for every backend:5-//!6-//!   <repo>/objects/<hash>   the object's compressed bytes, as in the table7-//!   <repo>/saves/<id>       the full save record, including its file list8-//!   <repo>/events/<id>      one flag, clear or strip event9-//!   <repo>/head             the ID of the newest save on the line10-//!11-//! Only `head` is ever overwritten, and only with a matching version tag.12-13-module sync14-15-import json16-import src.events17-import src.history18-import src.snapshot19-import src.storage20-21-pub fn object_key(repo string, hash string) string {22-	return '${repo}/objects/${hash}'23-}24-25-pub fn save_key(repo string, id string) string {26-	return '${repo}/saves/${id}'27-}28-29-pub fn event_key(repo string, id string) string {30-	return '${repo}/events/${id}'31-}32-33-pub fn head_key(repo string) string {34-	return '${repo}/head'35-}36-37-pub fn events_prefix(repo string) string {38-	return '${repo}/events'39-}40-41-pub struct FileRecord {42-pub:43-	path       string44-	hash       string45-	executable bool46-}47-48-pub struct SaveRecord {49-pub:50-	id            string51-	parent        string52-	message       string53-	created_at    i6454-	machine       string55-	is_checkpoint bool56-	size_bytes    i6457-	agent         string58-	session       string59-	restores      string60-	replayed_from string61-	files         []FileRecord62-}63-64-// save_record turns a local save into what travels. The file list is part of65-// the record, which is why a save needs one round trip, not one per file.66-pub fn save_record(store &storage.Store, save history.Save) !SaveRecord {67-	files := history.files_of(store, save.id)!68-	mut records := []FileRecord{}69-	for file in files {70-		records << FileRecord{71-			path:       file.path72-			hash:       file.hash73-			executable: file.executable74-		}75-	}76-	return SaveRecord{77-		id:            save.id78-		parent:        save.parent79-		message:       save.message80-		created_at:    save.created_at81-		machine:       save.machine82-		is_checkpoint: save.is_checkpoint83-		size_bytes:    save.size_bytes84-		agent:         save.agent85-		session:       save.session86-		restores:      save.restores87-		replayed_from: save.replayed_from88-		files:         records89-	}90-}91-92-pub fn encode_save(record SaveRecord) string {93-	return json.encode(record)94-}95-96-pub fn decode_save(body []u8) !SaveRecord {97-	return json.decode(SaveRecord, body.bytestr())!98-}99-100-pub fn to_save(record SaveRecord) history.Save {101-	return history.Save{102-		id:            record.id103-		parent:        record.parent104-		message:       record.message105-		created_at:    record.created_at106-		machine:       record.machine107-		is_checkpoint: record.is_checkpoint108-		size_bytes:    record.size_bytes109-		agent:         record.agent110-		session:       record.session111-		restores:      record.restores112-		replayed_from: record.replayed_from113-	}114-}115-116-pub fn to_files(record SaveRecord) []snapshot.FileEntry {117-	mut files := []snapshot.FileEntry{}118-	for file in record.files {119-		files << snapshot.FileEntry{120-			path:       file.path121-			hash:       file.hash122-			executable: file.executable123-		}124-	}125-	return files126-}127-128-pub struct EventRecord {129-pub:130-	id         string131-	kind       string132-	save_id    string133-	path       string134-	reason     string135-	created_at i64136-	machine    string137-}138-139-pub fn event_record(event events.Event) EventRecord {140-	return EventRecord{141-		id:         event.id142-		kind:       event.kind143-		save_id:    event.save_id144-		path:       event.path145-		reason:     event.reason146-		created_at: event.created_at147-		machine:    event.machine148-	}149-}150-151-pub fn encode_event(event events.Event) string {152-	return json.encode(event_record(event))153-}154-155-pub fn decode_event(body []u8) !events.Event {156-	record := json.decode(EventRecord, body.bytestr())!157-	return events.Event{158-		id:         record.id159-		kind:       record.kind160-		save_id:    record.save_id161-		path:       record.path162-		reason:     record.reason163-		created_at: record.created_at164-		machine:    record.machine165-	}166-}

view file at the parent

src/sync/steal.v+0 −497
@@ -1,497 +0,0 @@1-//! Stealing: bringing another machine's saves in, and combining when both2-//! machines moved.3-//!4-//! The order is always the same, and it is what makes an interrupted steal5-//! harmless: read everything from the remote first, then land it in one6-//! transaction, then write the working tree. Divergence is resolved per file —7-//! a clash is never merged line by line, it is chosen.8-9-module sync10-11-import compress.zlib12-import os13-import src.apply14-import src.events15-import src.history16-import src.remote17-import src.snapshot18-import src.storage19-20-pub enum Side {21-	mine22-	theirs23-}24-25-// Clash is one file both machines changed differently. `stat` is the short26-// diff stat the terminal prompt shows, so the choice is informed.27-pub struct Clash {28-pub mut:29-	path   string30-	mine   snapshot.FileEntry31-	theirs snapshot.FileEntry32-	base   snapshot.FileEntry33-	stat   string34-}35-36-pub struct StealOptions {37-pub mut:38-	mine   bool // resolve every clash as mine39-	theirs bool // resolve every clash as theirs40-	// Asked once per clash when neither flag is set. Nil means "nobody to ask".41-	ask fn (clash Clash) Side = unsafe { nil }42-}43-44-pub struct StealReport {45-pub mut:46-	repo            string47-	forward         bool48-	fetched_saves   int49-	fetched_objects int50-	fetched_events  int51-	clashes         []string52-	kept_mine       int53-	replayed        int54-	removed         int55-	head            string56-	flagged         []events.Flag // incoming saves that are flagged57-	// Set when a combine took this machine's saves off the line: loading it58-	// brings back the tree from before the steal, less anything stripped.59-	previous_head string60-}61-62-pub const dirty_tree = 'save or checkpoint first — steal needs a clean tree'63-pub const no_saves_here = 'this folder has files but no saves — steal a repo into an empty folder with "koh steal <name>"'64-65-// steal brings the remote's new saves in. Inside a repo with its own work this66-// may have to combine; a repo with nothing of its own simply fast-forwards.67-pub fn steal(store &storage.Store, mut backend remote.Backend, opts StealOptions) !StealReport {68-	name := repo_name(store, mut backend)!69-	if snapshot.restore_pending(store) {70-		return error(snapshot.restore_unfinished)71-	}72-	head_before := history.head(store)73-	// Combining rewrites the working tree, so unsaved work would be lost.74-	if dirty(store, head_before)! {75-		if head_before == none {76-			return error(no_saves_here)77-		}78-		return error(dirty_tree)79-	}80-	fetched := fetch_back_to_known(store, mut backend, name)!81-	mut report := StealReport{82-		repo:            name83-		fetched_saves:   fetched.saves.len84-		fetched_objects: fetched.repo_objects.len85-		fetched_events:  fetched.events_out.len86-		head:            fetched.head87-	}88-	mut local_only := []history.Save{}89-	if head := head_before {90-		// The remote builds on `base`. If that save is here but no longer on91-		// the line, neither line contains the other; jumping to the remote92-		// would silently leave this line behind.93-		local_only = saves_after(store, head.id, fetched.base) or {94-			return error('the remote builds on save ${fetched.base}, which is not on this line — load the save you want, then offer')95-		}96-	}97-	if fetched.base == '' && local_only.len > 0 {98-		return error('the remote keeps a different history — it shares no save with this repo')99-	}100-101-	strips_before := events.strip_events(store)!.len102-	mut clashes := []Clash{}103-	mut side_of := map[string]Side{}104-	if fetched.saves.len > 0 && local_only.len > 0 {105-		clashes = combine_clashes(store, fetched)!106-		report.clashes = clashes.map(it.path)107-		// Asked before the write lock: a person thinking must not stall every108-		// agent's save on this repo.109-		side_of = choose_sides(clashes, opts)!110-	}111-112-	store.begin() or { return err }113-	// The questions ran without the lock. If a save or an edit landed114-	// meanwhile, the answers were about a line that no longer exists.115-	if moved := changed_since(store, head_before, strips_before) {116-		if moved {117-			store.rollback()118-			return error('this repo changed while steal was asking — run koh steal again')119-		}120-	} else {121-		store.rollback()122-		return err123-	}124-	final := (write_and_combine(store, fetched, local_only, clashes, side_of, mut report) or {125-		store.rollback()126-		return err127-	})128-	files := history.files_of(store, final.id) or {129-		store.rollback()130-		return err131-	}132-	snapshot.check_links(store.root, files) or {133-		store.rollback()134-		return err135-	}136-	// The files follow the commit; the mark lands with it, in one step.137-	// No head before means a fresh clone, whose whole tree is new.138-	moved := if head := head_before { final.id != head.id } else { true }139-	if moved {140-		snapshot.mark_restore(store) or {141-			store.rollback()142-			return err143-		}144-	}145-	store.commit() or { return err }146-	report.head = final.id147-	if local_only.len > 0 && report.fetched_saves > 0 {148-		report.previous_head = local_only.last().id149-	}150-	flags := events.latest_flags(store)!151-	for record in fetched.saves {152-		if flag := flags[record.id] {153-			if flag.flagged {154-				report.flagged << flag155-			}156-		}157-	}158-159-	if head := head_before {160-		if final.id == head.id {161-			// The line did not move, so neither does the working tree.162-			return report163-		}164-		mut wanted := map[string]bool{}165-		for file in files {166-			wanted[file.path] = true167-		}168-		mut remove := []string{}169-		for file in history.files_of(store, head.id)! {170-			if file.path !in wanted {171-				remove << file.path172-			}173-		}174-		result := snapshot.restore_files(store, store.root, snapshot.load(store.root), files,175-			remove)!176-		report.removed = result.removed177-	} else {178-		snapshot.restore_files(store, store.root, snapshot.load(store.root), files, [])!179-	}180-	history.refresh_log_md(store)181-	return report182-}183-184-// write_and_combine lands a fetch and, when the local line has saves the remote185-// lacks, replays them on top of the remote head. It runs inside the caller's186-// transaction and returns the new head.187-fn write_and_combine(store &storage.Store, fetched Fetched, local_only []history.Save, clashes []Clash, side_of map[string]Side, mut report StealReport) !history.Save {188-	write_fetched(store, fetched)!189-	if fetched.saves.len == 0 {190-		// The remote head is already on this line: level, or this machine is191-		// ahead and the offer that follows a steal sends the rest.192-		return history.head(store) or { return error('this repo has no head') }193-	}194-	if local_only.len == 0 {195-		// Nothing of ours is missing from the remote: the line simply moves on.196-		history.set_head(store, fetched.head)!197-		report.forward = true198-		return history.head(store) or { return error('the remote head did not land') }199-	}200-	remote_files := history.files_of(store, fetched.head)!201-	mut mine_files := []snapshot.FileEntry{}202-	if our_head := history.head(store) {203-		mine_files = history.files_of(store, our_head.id)!204-	}205-206-	// A file the remote changed is a clash, decided below, or one this machine207-	// left as it was or changed the same way, so the remote's version stands.208-	// Replaying this machine's steps on it could only conflict: an edit it209-	// later undid would no longer apply.210-	base_files := history.files_of(store, fetched.base)!211-	settled := changed_paths(base_files, remote_files)212-213-	// Take the remote's line first, so the replayed saves chain onto it.214-	history.set_head(store, fetched.head)!215-	mut current := remote_files.clone()216-	for save in local_only {217-		parent_files := if save.parent != '' {218-			history.files_of(store, save.parent)!219-		} else {220-			[]snapshot.FileEntry{}221-		}222-		target_files := history.files_of(store, save.id)!223-		plan := apply.three_way(current, parent_files, target_files, [], settled)224-		if apply.conflicts(plan) {225-			return error('combine could not apply ${plan.conflict} — it is not a clash file')226-		}227-		if plan.applied.len == 0 {228-			// Everything this save touched was a file the remote changed.229-			continue230-		}231-		history.record_save(store, '', plan.files, history.SaveInput{232-			message:       save.message233-			created_at:    save.created_at234-			machine:       save.machine235-			agent:         save.agent236-			session:       save.session237-			is_checkpoint: save.is_checkpoint238-			replayed_from: save.id239-		})!240-		current = plan.files.clone()241-		report.replayed++242-	}243-244-	mut mine_paths := []string{}245-	for clash in clashes {246-		if side_of[clash.path] == .mine {247-			mine_paths << clash.path248-		}249-	}250-	if mine_paths.len > 0 {251-		// One last save puts my versions of the chosen files back.252-		final_plan := apply.three_way(current, current, mine_files, mine_paths, [])253-		if final_plan.applied.len > 0 {254-			history.record_save(store, '', final_plan.files, history.SaveInput{255-				message: 'combine: kept mine for ${mine_paths.len} ${if mine_paths.len == 1 {256-					'file'257-				} else {258-					'files'259-				}}'260-			})!261-			report.kept_mine = mine_paths.len262-		}263-	}264-	return history.head(store) or { return error('the combined line has no head') }265-}266-267-// find_clashes lists the files both sides changed, differently. Files changed268-// identically on both sides are not clashes: there is nothing to decide.269-pub fn find_clashes(base []snapshot.FileEntry, mine []snapshot.FileEntry, theirs []snapshot.FileEntry) []Clash {270-	base_map := snapshot.index_files(base)271-	mine_map := snapshot.index_files(mine)272-	theirs_map := snapshot.index_files(theirs)273-	mut paths := mine_map.keys()274-	for path in theirs_map.keys() {275-		if path !in mine_map {276-			paths << path277-		}278-	}279-	paths.sort()280-	mut clashes := []Clash{}281-	for path in paths {282-		here := mine_map[path] or {283-			snapshot.FileEntry{284-				path: path285-			}286-		}287-		there := theirs_map[path] or {288-			snapshot.FileEntry{289-				path: path290-			}291-		}292-		was := base_map[path] or {293-			snapshot.FileEntry{294-				path: path295-			}296-		}297-		if same(here, there) {298-			continue299-		}300-		if same(here, was) || same(there, was) {301-			continue302-		}303-		clashes << Clash{304-			path:   path305-			mine:   here306-			theirs: there307-			base:   was308-		}309-	}310-	return clashes311-}312-313-// changed_paths lists the paths whose entry differs between two trees.314-fn changed_paths(before []snapshot.FileEntry, after []snapshot.FileEntry) []string {315-	was := snapshot.index_files(before)316-	now := snapshot.index_files(after)317-	mut paths := []string{}318-	for path, entry in now {319-		if !same(entry, was[path] or { snapshot.FileEntry{} }) {320-			paths << path321-		}322-	}323-	for path, _ in was {324-		if path !in now {325-			paths << path326-		}327-	}328-	return paths329-}330-331-// combine_clashes finds the files both lines changed, with the diff stat each332-// prompt shows. It reads the remote head from the fetch, not the store, so it333-// can run before anything is written.334-fn combine_clashes(store &storage.Store, fetched Fetched) ![]Clash {335-	base_files := if fetched.base != '' {336-		history.files_of(store, fetched.base)!337-	} else {338-		[]snapshot.FileEntry{}339-	}340-	head := history.head(store) or { return []Clash{} }341-	// The transaction applies the incoming strips before it replays, so the342-	// clashes are found on the trees as they will be then, or the answers343-	// would be about files that are no longer there.344-	strips := fetched.events_out.filter(it.kind == events.kind_strip)345-	// Base and mine are here already, so every strip reaches them (created 0).346-	remote_head := fetched.saves.last()347-	mut clashes := find_clashes(after_strips(base_files, fetched.base, 0, strips), after_strips(history.files_of(store,348-		head.id)!, head.id, 0, strips), after_strips(to_files(remote_head), remote_head.id,349-		remote_head.created_at, strips))350-	for mut clash in clashes {351-		mine := fetched_bytes(store, fetched, clash.mine)!352-		theirs := fetched_bytes(store, fetched, clash.theirs)!353-		delta := snapshot.diff_bytes(clash.path, 'modified', mine, theirs)!354-		clash.stat = if delta.binary { 'binary' } else { '+${delta.added} -${delta.removed}' }355-	}356-	return clashes357-}358-359-// after_strips is a save's file list once the given strip events are applied,360-// by the same rule write_fetched uses.361-fn after_strips(files []snapshot.FileEntry, save_id string, created_at i64, strips []events.Event) []snapshot.FileEntry {362-	return files.filter(fn [strips, save_id, created_at] (file snapshot.FileEntry) bool {363-		return !strips.any(reaches(it, save_id, created_at) && covers(it.path, file.path))364-	})365-}366-367-// fetched_bytes reads one side of a clash: from the fetch when it has just368-// arrived, else from the store. An absent side is empty.369-fn fetched_bytes(store &storage.Store, fetched Fetched, entry snapshot.FileEntry) ![]u8 {370-	if entry.hash == '' {371-		return []u8{}372-	}373-	if blob := fetched.repo_objects[entry.hash] {374-		return zlib.decompress(blob)!375-	}376-	return store.get_object(entry.hash)!377-}378-379-fn choose_sides(clashes []Clash, opts StealOptions) !map[string]Side {380-	mut sides := map[string]Side{}381-	if clashes.len == 0 {382-		return sides383-	}384-	for clash in clashes {385-		if opts.mine {386-			sides[clash.path] = .mine387-			continue388-		}389-		if opts.theirs {390-			sides[clash.path] = .theirs391-			continue392-		}393-		if opts.ask != unsafe { nil } {394-			sides[clash.path] = opts.ask(clash)395-			continue396-		}397-		return error('${clashes.len} files changed on both machines — run on a terminal to choose, or pass --mine or --theirs')398-	}399-	return sides400-}401-402-fn same(a snapshot.FileEntry, b snapshot.FileEntry) bool {403-	if a.hash == '' && b.hash == '' {404-		return true405-	}406-	return a.hash == b.hash && a.executable == b.executable407-}408-409-fn changed_since(store &storage.Store, before ?history.Save, strips_before int) !bool {410-	if events.strip_events(store)!.len != strips_before {411-		return true412-	}413-	now := history.head(store)414-	if now or { history.Save{} }.id != before or { history.Save{} }.id {415-		return true416-	}417-	return dirty(store, now)!418-}419-420-// dirty reports whether the working tree holds anything a steal would421-// overwrite. With no saves yet, that is every file but the untouched ignore422-// list `koh init` wrote.423-fn dirty(store &storage.Store, head ?history.Save) !bool {424-	files := snapshot.walk(store, store.root, snapshot.load(store.root))!425-	if saved := head {426-		return !snapshot.same_tree(history.files_of(store, saved.id)!, files)427-	}428-	seed := storage.hash_hex(snapshot.default_ignore.bytes())429-	return files.any(it.path != snapshot.ignore_file || it.hash != seed)430-}431-432-// fetch_repo pulls a whole repo — head, saves, objects and events — for433-// `koh steal <name>`, which lands a copy in a new folder.434-pub fn fetch_repo(mut backend remote.Backend, name string) !Fetched {435-	remote_head := backend.get(head_key(name))!436-	if !remote_head.found {437-		return error('the remote has no repo called "${name}"')438-	}439-	save_ids := backend.list('${name}/saves')!440-	object_keys := backend.list('${name}/objects')!441-	event_keys := backend.list('${name}/events')!442-	mut fetched := Fetched{443-		head: remote_head.body.bytestr().trim_space()444-	}445-	for key in object_keys {446-		blob := backend.get(key)!447-		if blob.found {448-			fetched.repo_objects[os.base(key)] = blob.body449-		}450-	}451-	for key in save_ids {452-		body := backend.get(key)!453-		if !body.found {454-			continue455-		}456-		record := decode_save(body.body)!457-		fetched.saves << record458-	}459-	fetched.saves = parent_first(fetched.saves)!460-	for key in event_keys {461-		body := backend.get(key)!462-		if body.found {463-			fetched.events_out << decode_event(body.body)!464-		}465-	}466-	return fetched467-}468-469-// parent_first orders saves so every parent lands before its children, as the470-// foreign key on `parent` demands. Time cannot do it: a replayed save keeps its471-// original, older time but sits on a newer parent.472-fn parent_first(records []SaveRecord) ![]SaveRecord {473-	mut by_id := map[string]SaveRecord{}474-	for record in records {475-		by_id[record.id] = record476-	}477-	mut placed := map[string]bool{}478-	mut ordered := []SaveRecord{}479-	for record in records {480-		mut chain := []SaveRecord{}481-		mut cursor := record.id482-		for cursor != '' && cursor !in placed {483-			if chain.len > records.len {484-				return error('the save records on the remote form a loop')485-			}486-			current := by_id[cursor] or { return error('the remote is missing save ${cursor}') }487-			chain << current488-			cursor = current.parent489-		}490-		chain.reverse_in_place()491-		for current in chain {492-			placed[current.id] = true493-			ordered << current494-		}495-	}496-	return ordered497-}

view file at the parent

src/sync/strip.v+0 −227
@@ -1,227 +0,0 @@1-//! Strip: removing a path from history, here and on the remote.2-//!3-//! The local half is one transaction: the saved files go, the objects nobody4-//! references go, a strip event is recorded, and so is a note of which objects5-//! the remote may now drop. The remote half reads that note, so it can be6-//! re-run after an interruption even though the local rows are long gone.7-8-module sync9-10-import json11-import os12-import src.events13-import src.remote14-import src.storage15-16-// Scope is what a strip would remove, for the warning shown before it runs.17-pub struct Scope {18-pub:19-	files   int20-	saves   int21-	objects int22-}23-24-pub struct LocalStrip {25-pub:26-	files_removed   int27-	objects_dropped i6428-}29-30-// PushReport says what a strip managed to make visible on the remote.31-pub struct PushReport {32-pub mut:33-	records    int34-	objects    int35-	event_sent bool36-}37-38-// Pending is the note a strip leaves until its remote half has finished: which39-// saves it covered ('' for every save) and which objects it freed. One note per40-// path, so strips of the same path that did not finish are finished together.41-struct Pending {42-mut:43-	saves  []string44-	hashes []string45-}46-47-fn pending_key(path string) string {48-	return 'strip_pending:${path}'49-}50-51-fn read_pending(store &storage.Store, path string) Pending {52-	text := store.state(pending_key(path)) or { return Pending{} }53-	return json.decode(Pending, text) or { Pending{} }54-}55-56-pub fn strip_pending(store &storage.Store, path string) bool {57-	return store.state(pending_key(path)) != none58-}59-60-// covers is the one rule for what a strip removes: the path itself, or61-// anything beneath it when it names a folder.62-fn covers(path string, file string) bool {63-	return file == path || file.starts_with('${path.trim_right('/')}/')64-}65-66-pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {67-	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)68-		FROM save_files WHERE ${match_clause(only_save)}')!69-	defer {70-		stmt.finalize()71-	}72-	bind_match(stmt, path, only_save)!73-	stmt.step()!74-	return Scope{75-		files:   int(stmt.int(0))76-		saves:   int(stmt.int(1))77-		objects: int(stmt.int(2))78-	}79-}80-81-// strip_local removes the path in one transaction and leaves the note the82-// remote half needs.83-pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {84-	store.begin()!85-	report := strip_rows(store, path, only_save) or {86-		store.rollback()87-		return err88-	}89-	store.commit()!90-	return report91-}92-93-fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {94-	mut hashes := []string{}95-	mut stmt :=96-		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!97-	bind_match(stmt, path, only_save)!98-	for stmt.step()! {99-		hashes << stmt.text(0)100-	}101-	stmt.finalize()102-	if hashes.len == 0 {103-		// Already stripped: keep the pending note an unfinished remote needs.104-		return LocalStrip{}105-	}106-	removed := remove_path(store, path, only_save)!107-	dropped := store.drop_unreferenced_objects()!108-	events.strip(store, only_save, path)!109-	// An earlier strip of the same path may not have finished on the remote,110-	// so the note grows rather than resets.111-	mut note := read_pending(store, path)112-	if only_save !in note.saves {113-		note.saves << only_save114-	}115-	note.hashes << hashes116-	store.set_state(pending_key(path), json.encode(note))!117-	return LocalStrip{118-		files_removed:   removed119-		objects_dropped: dropped120-	}121-}122-123-// push_strip makes a strip visible on the remote: the event travels like any124-// other, every remote save record listing the path is rewritten, and objects125-// the strip freed are deleted once no remote record and no local save needs126-// them. Every step is safe to repeat, and the pending note is cleared only127-// when all of them have succeeded.128-pub fn push_strip(store &storage.Store, mut backend remote.Backend, path string) !PushReport {129-	name := repo_name(store, mut backend)!130-	mut report := PushReport{}131-	mut upload := OfferReport{132-		repo: name133-	}134-	upload_events(store, mut backend, name, mut upload)!135-	report.event_sent = upload.events > 0136-	note := read_pending(store, path)137-	mut freed := map[string]bool{}138-	for hash in note.hashes {139-		freed[hash] = true140-	}141-	every_save := '' in note.saves142-	mut needed := map[string]bool{}143-	for key in backend.list('${name}/saves')! {144-		remote_object := backend.get(key)!145-		if !remote_object.found {146-			continue147-		}148-		record := decode_save(remote_object.body)!149-		mut kept := []FileRecord{}150-		for file in record.files {151-			if (every_save || record.id in note.saves) && covers(path, file.path) {152-				freed[file.hash] = true153-				continue154-			}155-			kept << file156-			needed[file.hash] = true157-		}158-		if kept.len == record.files.len {159-			continue160-		}161-		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {162-			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')163-		}164-		report.records++165-	}166-	for hash, _ in freed {167-		if hash in needed || store.has_object(hash)! {168-			continue169-		}170-		backend.delete(object_key(name, hash))!171-		report.objects++172-	}173-	store.remove_state(pending_key(path))!174-	return report175-}176-177-// unapplied_strip reports whether the remote holds a strip event this repo178-// has not received yet.179-fn unapplied_strip(store &storage.Store, mut backend remote.Backend, repo string) !bool {180-	mut known := map[string]bool{}181-	for event in events.all(store)! {182-		known[event.id] = true183-	}184-	for key in backend.list(events_prefix(repo))! {185-		if os.base(key) in known {186-			continue187-		}188-		body := backend.get(key)!189-		if body.found && decode_event(body.body)!.kind == events.kind_strip {190-			return true191-		}192-	}193-	return false194-}195-196-// rewrite returns the save record with a new file list.197-fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {198-	return SaveRecord{199-		...record200-		files: files201-	}202-}203-204-fn remove_path(store &storage.Store, path string, only_save string) !int {205-	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!206-	defer {207-		stmt.finalize()208-	}209-	bind_match(stmt, path, only_save)!210-	stmt.step()!211-	return store.db.changes()212-}213-214-// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`215-// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.216-fn match_clause(only_save string) string {217-	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }218-	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'219-}220-221-fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {222-	stmt.bind_text(1, path)!223-	stmt.bind_text(2, '${path.trim_right('/')}/')!224-	if only_save != '' {225-		stmt.bind_text(3, only_save)!226-	}227-}

view file at the parent

src/sync/sync.v+0 −268
@@ -1,268 +0,0 @@1-//! Sync: offer and steal.2-//!3-//! All the cleverness sits here, above the five-operation remote interface, so4-//! the backends stay thin and this logic is written once and tested offline.5-//!6-//! The remote is a line of saves, the same shape as the local one. `offer`7-//! pushes local saves the remote lacks; `steal` pulls remote saves and, when8-//! both sides moved, replays the local work on top of the remote's. Divergence9-//! is resolved per file, never per line.10-11-module sync12-13-import os14-import src.events15-import src.history16-import src.remote17-import src.snapshot18-import src.storage19-20-// Where the repo's name on the remote is remembered.21-const repo_state_key = 'remote_repo'22-23-// The remote head as of this machine's last successful offer, as the document24-// asks; nothing reads it back yet.25-const remote_head_key = 'remote_head'26-27-pub const no_saves = 'this repo has no saves yet — nothing to offer'28-29-pub struct OfferReport {30-pub mut:31-	repo    string32-	saves   int33-	objects int34-	events  int35-	head    string36-}37-38-// remember_repo binds a repo to its name on the remote. A clone does it at39-// once: its folder may be renamed before it ever offers.40-pub fn remember_repo(store &storage.Store, name string) ! {41-	store.set_state(repo_state_key, name)!42-}43-44-// repo_name is the folder's name at first offer, remembered in the repo so45-// renaming the folder later changes nothing.46-pub fn repo_name(store &storage.Store, mut backend remote.Backend) !string {47-	if remembered := store.state(repo_state_key) {48-		return remembered49-	}50-	name := os.base(store.root)51-	existing := backend.get(head_key(name))!52-	if existing.found {53-		theirs := existing.body.bytestr().trim_space()54-		if history.by_id(store, theirs) == none {55-			return error('the remote already has "${name}" with a history this repo does not share — rename the folder, or use another remote')56-		}57-	}58-	store.set_state(repo_state_key, name)!59-	return name60-}61-62-// offer sends what the remote lacks and then swaps `head`. Nothing is ever63-// queued: the next offer simply sends everything the remote does not have.64-pub fn offer(store &storage.Store, mut backend remote.Backend) !OfferReport {65-	name := repo_name(store, mut backend)!66-	local := history.head(store) or { return error(no_saves) }67-	mut report := OfferReport{68-		repo: name69-		head: local.id70-	}71-	remote_head := backend.get(head_key(name))!72-	// Records offered by a machine that has not applied a strip would put the73-	// stripped path back on the remote, where nothing would rewrite it again.74-	// ponytail: an offer racing a strip on another machine inside the same few75-	// seconds can still slip through; closing that needs a remote lock.76-	if unapplied_strip(store, mut backend, name)! {77-		return error('the remote has a strip this repo has not applied — run koh steal')78-	}79-	if remote_head.found {80-		theirs := remote_head.body.bytestr().trim_space()81-		newer := saves_after(store, local.id, theirs) or {82-			return error('remote has new saves — run koh steal')83-		}84-		for save in newer {85-			upload_save(store, mut backend, name, save, mut report)!86-		}87-	} else {88-		oldest_first := history.line(store)!89-		for index := oldest_first.len - 1; index >= 0; index-- {90-			upload_save(store, mut backend, name, oldest_first[index], mut report)!91-		}92-	}93-	upload_events(store, mut backend, name, mut report)!94-	if !backend.put_if_match(head_key(name), local.id.bytes(), remote_head.tag)! {95-		return error('remote has new saves — run koh steal')96-	}97-	store.set_state(remote_head_key, local.id)!98-	return report99-}100-101-// saves_after lists the local saves newer than `theirs`, oldest first. It fails102-// when `theirs` is not on this line, which is the honest answer: we cannot103-// offer on top of a save we do not have.104-fn saves_after(store &storage.Store, ours string, theirs string) ![]history.Save {105-	mut newer := []history.Save{}106-	mut cursor := ours107-	for cursor != '' && cursor != theirs {108-		save := history.by_id(store, cursor) or { return error('save ${cursor} is missing') }109-		newer << save110-		cursor = save.parent111-	}112-	if cursor != theirs {113-		return error('${theirs} is not on this line')114-	}115-	newer.reverse_in_place()116-	return newer117-}118-119-fn upload_save(store &storage.Store, mut backend remote.Backend, repo string, save history.Save, mut report OfferReport) ! {120-	files := history.files_of(store, save.id)!121-	// Saves go up oldest first and head moves only once all of them landed, so122-	// the parent and its objects are on the remote already: only content new123-	// in this save travels, not a round trip per file in the repo.124-	mut sent := map[string]bool{}125-	if save.parent != '' {126-		for file in history.files_of(store, save.parent)! {127-			sent[file.hash] = true128-		}129-	}130-	for file in files {131-		if file.hash in sent {132-			continue133-		}134-		sent[file.hash] = true135-		if backend.put_if_absent(object_key(repo, file.hash), store.object_blob(file.hash)!)! {136-			report.objects++137-		}138-	}139-	record := save_record(store, save)!140-	if backend.put_if_absent(save_key(repo, save.id), encode_save(record).bytes())! {141-		report.saves++142-	}143-}144-145-// upload_events sends every event the remote does not have. Events are tiny, so146-// listing them costs less than tracking which ones were sent.147-fn upload_events(store &storage.Store, mut backend remote.Backend, repo string, mut report OfferReport) ! {148-	already := backend.list(events_prefix(repo))!149-	mut already_sent := map[string]bool{}150-	for key in already {151-		already_sent[os.base(key)] = true152-	}153-	for event in events.all(store)! {154-		if event.id in already_sent {155-			continue156-		}157-		if backend.put_if_absent(event_key(repo, event.id), encode_event(event).bytes())! {158-			report.events++159-		}160-	}161-}162-163-// fetched holds everything a steal read from the remote, before any of it is164-// written locally. Network work happens first; the database gets one165-// transaction, so a failure leaves nothing half-applied.166-pub struct Fetched {167-pub mut:168-	repo_objects map[string][]u8169-	saves        []SaveRecord // oldest first170-	events_out   []events.Event171-	head         string172-	base         string // the newest save both sides already shared, '' when none173-}174-175-pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched {176-	remote_head := backend.get(head_key(repo))!177-	if !remote_head.found {178-		return error('the remote has no "${repo}" yet — run koh offer here first')179-	}180-	mut fetched := Fetched{181-		head: remote_head.body.bytestr().trim_space()182-	}183-	mut cursor := fetched.head184-	for cursor != '' && history.by_id(store, cursor) == none {185-		record_object := backend.get(save_key(repo, cursor))!186-		if !record_object.found {187-			return error('the remote is missing save ${cursor} — run koh offer there')188-		}189-		record := decode_save(record_object.body)!190-		for file in record.files {191-			if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects {192-				blob := backend.get(object_key(repo, file.hash))!193-				if !blob.found {194-					return error('the remote is missing object ${file.hash} that save ${record.id} needs')195-				}196-				fetched.repo_objects[file.hash] = blob.body197-			}198-		}199-		fetched.saves << record200-		cursor = record.parent201-	}202-	fetched.saves.reverse_in_place()203-	fetched.base = cursor204-	fetched.events_out = fetch_events(store, mut backend, repo)!205-	return fetched206-}207-208-fn fetch_events(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {209-	have := events.all(store)!210-	mut known := map[string]bool{}211-	for event in have {212-		known[event.id] = true213-	}214-	mut incoming := []events.Event{}215-	for key in backend.list(events_prefix(repo))! {216-		id := os.base(key)217-		if id in known {218-			continue219-		}220-		body := backend.get(event_key(repo, id))!221-		if !body.found {222-			continue223-		}224-		incoming << decode_event(body.body)!225-	}226-	return incoming227-}228-229-// write_fetched lands a fetch in one transaction: objects first (verified),230-// then the events, then the saves they belong to. A new strip reaches every231-// save already here, but an arriving save only when `reaches` says so: one232-// made after the strip is new work, not what the strip removed.233-pub fn write_fetched(store &storage.Store, fetched Fetched) ! {234-	for hash, blob in fetched.repo_objects {235-		store.put_object_blob(hash, blob)!236-	}237-	mut strips := []events.Event{}238-	for event in fetched.events_out {239-		if !events.receive(store, event)! {240-			continue241-		}242-		if event.kind == events.kind_strip {243-			// Somebody stripped a path: remove it here too, so the two244-			// histories agree from now on. Objects go once the saves are in.245-			remove_path(store, event.path, event.save_id)!246-			strips << event247-		}248-	}249-	for record in fetched.saves {250-		files := to_files(record).filter(fn [strips, record] (file snapshot.FileEntry) bool {251-			return !strips.any(reaches(it, record.id, record.created_at)252-				&& covers(it.path, file.path))253-		})254-		history.import_save(store, to_save(record), files)!255-	}256-	if strips.len > 0 {257-		store.drop_unreferenced_objects()!258-	}259-}260-261-// reaches reports whether a strip event applies to a save arriving from the262-// remote. A strip of every save covers the saves made up to its own time.263-fn reaches(strip events.Event, save_id string, created_at i64) bool {264-	if strip.save_id != '' {265-		return strip.save_id == save_id266-	}267-	return created_at <= strip.created_at268-}

view file at the parent

src/sync/sync_test.v+0 −1236

not shown here: too long for one page

view file at the parent

tests/library_check_test.v+7 −0
@@ -16,6 +16,13 @@1616 import net.http1717 import time1818 19+// Koh 3 is built and tested on V master at b9a2d87, and that commit is the pin:20+// V master moves, and a newer V may break Koh at any time. Building with any21+// other compiler fails here rather than somewhere stranger.22+fn test_built_with_the_pinned_v() {23+	assert @VCURRENTHASH == 'b9a2d87'24+}25+1926 // Vectors from the official blake3 bindings, cross-checked against the2027 // naming of real Koh 2 objects: `.koh/objects/<hash[0..2]>/<hash[2..]>` is2128 // blake3(uncompressed bytes) of a zlib stream.

view file

v.mod+1 −1
@@ -1,6 +1,6 @@11 Module {22 	name: 'koh3'3-	version: '3.0.53'3+	version: '3.0.54'44 	description: 'One SQLite file per repo, one line of history.'55 	license: 'MIT'66 }

view file