56ab9cab6ad2
make Koh 3 build and pass on V b9a2d87: json2 for the removed json module, src/exchange for the module that shadowed V's sync, the pin written down and guarded (3.0.54)
- time
- 2026-09-24 20:10 UTC · 2 weeks ago
- machine
- koh2
- agent
- deepseek/deepseek-flash
- session
- pin V
- parent
- 9396ba8479a4
↓ koh-56ab9cab.tar.gzatlas at this save
files changed32 files
README.md+4 −2src/cli/cmd_back.v+3 −3src/cli/cmd_diff.v+2 −2src/cli/cmd_doctor.v+2 −2src/cli/cmd_flag.v+4 −4src/cli/cmd_init.v+2 −2src/cli/cmd_log.v+2 −2src/cli/cmd_migrate.v+2 −2src/cli/cmd_offer.v+5 −5src/cli/cmd_peek.v+2 −2src/cli/cmd_pluck.v+3 −3src/cli/cmd_remote.v+5 −5src/cli/cmd_save.v+2 −2src/cli/cmd_status.v+2 −2src/cli/cmd_steal.v+12 −12src/cli/cmd_strip.v+9 −9src/cli/output.v+2 −2src/exchange/exchange.v+268 −0src/exchange/exchange_test.v+1236 −0src/exchange/records.v+166 −0src/exchange/steal.v+497 −0src/exchange/strip.v+227 −0src/migrate/migrate.v+2 −2src/migrate/migrate_test.v+3 −3src/remote/remote.v+1 −1src/sync/records.v+0 −166src/sync/steal.v+0 −497src/sync/strip.v+0 −227src/sync/sync.v+0 −268src/sync/sync_test.v+0 −1236tests/library_check_test.v+7 −0v.mod+1 −1README.md+4 −2
@@ -15,8 +15,10 @@1515 v -o build/koh3 src/koh3.v1616 ```1717 18-Requires V 0.5.x. SQLite is vendored in `vendor/sqlite/`; there are no other19-dependencies.18+Pinned to V master at commit `b9a2d87` (`V 0.5.2 b9a2d87`); install steps are19+in `AGENTS.md`. V master can break Koh, so the pin is the compiler the build and20+the tests are verified against. SQLite is vendored in `vendor/sqlite/`; there21+are no other dependencies.2022 2123 ## Development2224 src/cli/cmd_back.v+3 −3
@@ -10,7 +10,7 @@1010 1111 module cli1212 13-import json13+import json21414 import src.fmt1515 import src.history1616 import src.snapshot@@ -65,7 +65,7 @@6565 // The files all landed before whatever stopped the last restore.6666 snapshot.clear_restore_mark(store) or { return refuse(args.json(), err.msg()) }6767 if args.json() {68- emit_json(json.encode(RevertJson{68+ emit_json(json2.encode(RevertJson{6969 reverted: false7070 head_id: head.id7171 }))@@ -139,7 +139,7 @@139139 // A revert's restore save sits right on the checkpoint of what it threw away.140140 kept := if discard_first { save.parent } else { '' }141141 if as_json {142- emit_json(json.encode(BackJson{142+ emit_json(json2.encode(BackJson{143143 id: save.id144144 restores: target.id145145 written: report.writtensrc/cli/cmd_diff.v+2 −2
@@ -3,7 +3,7 @@33 44 module cli55 6-import json6+import json277 import src.fmt88 import src.history99 import src.snapshot@@ -23,7 +23,7 @@2323 return refuse(args.json(), err.msg())2424 }2525 if args.json() {26- emit_json(json.encode(DiffJson{26+ emit_json(json2.encode(DiffJson{2727 from: sides.from_name2828 to: sides.to_name2929 files: deltas.map(delta_json(it))src/cli/cmd_doctor.v+2 −2
@@ -7,7 +7,7 @@77 88 module cli99 10-import json10+import json21111 import os1212 import time1313 import src.history@@ -45,7 +45,7 @@4545 }4646 4747 if args.json() {48- emit_json(json.encode(report))48+ emit_json(json2.encode(report))4949 } else {5050 if report.problems.len == 0 {5151 ok('${store.root} is sound')src/cli/cmd_flag.v+4 −4
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import src.events1111 import src.fmt1212 import src.history@@ -32,7 +32,7 @@3232 if args.flag('clear') {3333 events.clear_flag(store, save.id) or { return refuse(args.json(), err.msg()) }3434 if args.json() {35- emit_json(json.encode(FlagJson{35+ emit_json(json2.encode(FlagJson{3636 save_id: save.id3737 flagged: false3838 reason: ''@@ -45,7 +45,7 @@4545 reason := args.value('reason')4646 events.flag(store, save.id, reason) or { return refuse(args.json(), err.msg()) }4747 if args.json() {48- emit_json(json.encode(FlagJson{48+ emit_json(json2.encode(FlagJson{4949 save_id: save.id5050 flagged: true5151 reason: reason@@ -71,7 +71,7 @@7171 }7272 flagged.sort(a.save_id < b.save_id)7373 if args.json() {74- emit_json(json.encode(FlagListJson{74+ emit_json(json2.encode(FlagListJson{7575 flagged: flagged7676 }))7777 return 0src/cli/cmd_init.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import os77 import src.snapshot88 import src.storage@@ -23,7 +23,7 @@2323 wrote_ignore = true2424 }2525 if args.json() {26- emit_json(json.encode(InitJson{26+ emit_json(json2.encode(InitJson{2727 repo: root2828 ignore_created: wrote_ignore2929 }))src/cli/cmd_log.v+2 −2
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import os1111 import time1212 import src.events@@ -68,7 +68,7 @@6868 shown << save6969 }7070 if args.json() {71- emit_json(json.encode(LogJson{71+ emit_json(json2.encode(LogJson{7272 head_id: head_id7373 saves: shown.map(to_json(it))7474 }))src/cli/cmd_migrate.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import os77 import src.fmt88 import src.machine@@ -24,7 +24,7 @@2424 root := os.getwd()2525 report := migrate.run(root, lane) or { return refuse(args.json(), err.msg()) }2626 if args.json() {27- emit_json(json.encode(MigrateJson{27+ emit_json(json2.encode(MigrateJson{2828 repo: report.root2929 lane: report.lane3030 saves: report.savessrc/cli/cmd_offer.v+5 −5
@@ -2,12 +2,12 @@22 33 module cli44 5-import json5+import json266 import os77 import src.machine88 import src.remote99 import src.storage10-import src.sync10+import src.exchange1111 1212 pub fn cmd_offer(raw []string) int {1313 args := parse('offer', raw, Options{@@ -18,9 +18,9 @@1818 store.close()1919 }2020 mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }21- report := sync.offer(store, mut backend) or { return refuse(args.json(), err.msg()) }21+ report := exchange.offer(store, mut backend) or { return refuse(args.json(), err.msg()) }2222 if args.json() {23- emit_json(json.encode(OfferedJson{23+ emit_json(json2.encode(OfferedJson{2424 repo: report.repo2525 saves: report.saves2626 objects: report.objects@@ -68,7 +68,7 @@6868 }6969 return7070 }71- report := sync.offer(store, mut backend) or {71+ report := exchange.offer(store, mut backend) or {7272 if !quiet {7373 note('not offered: ${err.msg()}')7474 }src/cli/cmd_peek.v+2 −2
@@ -5,7 +5,7 @@55 66 module cli77 8-import json8+import json299 import os1010 import src.fmt1111 import src.history@@ -35,7 +35,7 @@3535 return refuse(args.json(), err.msg())3636 }3737 if args.json() {38- emit_json(json.encode(PeekJson{38+ emit_json(json2.encode(PeekJson{3939 id: save.id4040 path: path4141 files: files.lensrc/cli/cmd_pluck.v+3 −3
@@ -7,7 +7,7 @@77 88 module cli99 10-import json10+import json21111 import src.apply1212 import src.fmt1313 import src.history@@ -45,7 +45,7 @@4545 4646 if args.flag('dry-run') || plan.applied.len == 0 {4747 if args.json() {48- emit_json(json.encode(PluckJson{48+ emit_json(json2.encode(PluckJson{4949 source: source.id5050 reverse: reverse5151 applied: plan.applied@@ -97,7 +97,7 @@9797 report := snapshot.restore_files(store, store.root, snapshot.load(store.root), fresh.files,9898 remove) or { return refuse(args.json(), err.msg()) }9999 if args.json() {100- emit_json(json.encode(PluckedJson{100+ emit_json(json2.encode(PluckedJson{101101 id: save.id102102 source: source.id103103 reverse: reversesrc/cli/cmd_remote.v+5 −5
@@ -8,7 +8,7 @@88 99 module cli1010 11-import json11+import json21212 import os1313 import src.fmt1414 import src.machine@@ -25,7 +25,7 @@2525 }2626 machine.forget_remote() or { return refuse(args.json(), err.msg()) }2727 if args.json() {28- emit_json(json.encode(RemoteJson{28+ emit_json(json2.encode(RemoteJson{2929 configured: false3030 }))3131 return 0@@ -36,7 +36,7 @@3636 if args.positional.len == 0 {3737 if !machine.has_remote() {3838 if args.json() {39- emit_json(json.encode(RemoteJson{39+ emit_json(json2.encode(RemoteJson{4040 configured: false4141 }))4242 return 0@@ -45,7 +45,7 @@4545 }4646 settings := machine.load_remote() or { return refuse(args.json(), err.msg()) }4747 if args.json() {48- emit_json(json.encode(RemoteJson{48+ emit_json(json2.encode(RemoteJson{4949 configured: true5050 kind: settings.kind5151 target: describe(settings)@@ -113,7 +113,7 @@113113 }) or { return refuse(args.json(), err.msg()) }114114 }115115 if args.json() {116- emit_json(json.encode(RemoteJson{116+ emit_json(json2.encode(RemoteJson{117117 configured: true118118 kind: settings.kind119119 target: describe(settings)src/cli/cmd_save.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import src.fmt77 import src.history88 import src.snapshot@@ -35,7 +35,7 @@3535 3636 files := history.files_of(store, save.id) or { []snapshot.FileEntry{} }3737 if args.json() {38- emit_json(json.encode(SaveOut{38+ emit_json(json2.encode(SaveOut{3939 id: save.id4040 parent: save.parent4141 message: save.messagesrc/cli/cmd_status.v+2 −2
@@ -2,7 +2,7 @@22 33 module cli44 5-import json5+import json266 import src.fmt77 import src.history88 import src.snapshot@@ -31,7 +31,7 @@3131 }3232 3333 if args.json() {34- emit_json(json.encode(StatusJson{34+ emit_json(json2.encode(StatusJson{3535 head_id: head.id3636 head_msg: head.message3737 dirty: changes.dirtysrc/cli/cmd_steal.v+12 −12
@@ -6,7 +6,7 @@66 77 module cli88 9-import json9+import json21010 import os1111 import src.events1212 import src.fmt@@ -14,7 +14,7 @@1414 import src.remote1515 import src.snapshot1616 import src.storage17-import src.sync17+import src.exchange1818 1919 pub fn cmd_steal(raw []string) int {2020 args := parse('steal', raw, Options{@@ -38,18 +38,18 @@3838 return usage_error('steal takes no name inside a repo — run it outside to fetch a whole repo')3939 }4040 mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }41- mut options := sync.StealOptions{41+ mut options := exchange.StealOptions{4242 mine: args.flag('mine')4343 theirs: args.flag('theirs')4444 }4545 if !options.mine && !options.theirs && interactive() && os.is_atty(1) != 0 {4646 options.ask = ask_clash4747 }48- report := sync.steal(store, mut backend, options) or { return refuse(args.json(), err.msg()) }48+ report := exchange.steal(store, mut backend, options) or { return refuse(args.json(), err.msg()) }4949 // The remote should end up with whatever this machine just decided.5050 offered := offer_after_steal(store, mut backend)5151 if args.json() {52- emit_json(json.encode(StolenJson{52+ emit_json(json2.encode(StolenJson{5353 repo: report.repo5454 forward: report.forward5555 saves: report.fetched_saves@@ -92,7 +92,7 @@9292 // offer_after_steal pushes what this machine just decided. A failure here does9393 // not undo the steal; it is reported.9494 fn offer_after_steal(store &storage.Store, mut backend remote.Backend) bool {95- if _ := sync.offer(store, mut backend) {95+ if _ := exchange.offer(store, mut backend) {9696 return true9797 }9898 return false@@ -100,7 +100,7 @@100100 101101 // ask_clash is the per-file question a terminal gets. Only a lowercase y-style102102 // answer is needed here: `m` for mine, `t` for theirs.103-fn ask_clash(clash sync.Clash) sync.Side {103+fn ask_clash(clash exchange.Clash) exchange.Side {104104 println(' ${fmt.amber('both changed')} ${clash.path} ${clash.stat}')105105 print(' ${fmt.amber('m')}ine or ${fmt.amber('t')}heirs? [t] ')106106 answer := os.get_line().trim_space()@@ -123,7 +123,7 @@123123 mut sorted := names.keys()124124 sorted.sort()125125 if args.json() {126- emit_json(json.encode(ReposJson{126+ emit_json(json2.encode(ReposJson{127127 repos: sorted128128 }))129129 return 0@@ -146,7 +146,7 @@146146 return refuse(args.json(), '${name} already exists here — run steal inside it instead')147147 }148148 mut backend := this_machine_backend() or { return refuse(args.json(), err.msg()) }149- fetched := sync.fetch_repo(mut backend, name) or { return refuse(args.json(), err.msg()) }149+ fetched := exchange.fetch_repo(mut backend, name) or { return refuse(args.json(), err.msg()) }150150 // SQLite creates the file, not the folder holding it.151151 os.mkdir_all(target, os.MkdirParams{}) or { return refuse(args.json(), err.msg()) }152152 mut store := storage.create(target) or { return refuse(args.json(), err.msg()) }@@ -154,7 +154,7 @@154154 store.close()155155 }156156 store.begin() or { return refuse(args.json(), err.msg()) }157- sync.write_fetched(store, fetched) or {157+ exchange.write_fetched(store, fetched) or {158158 store.rollback()159159 return refuse(args.json(), err.msg())160160 }@@ -162,7 +162,7 @@162162 store.rollback()163163 return refuse(args.json(), err.msg())164164 }165- sync.remember_repo(store, name) or {165+ exchange.remember_repo(store, name) or {166166 store.rollback()167167 return refuse(args.json(), err.msg())168168 }@@ -183,7 +183,7 @@183183 }184184 flagged.sort(a.save_id < b.save_id)185185 if args.json() {186- emit_json(json.encode(StolenJson{186+ emit_json(json2.encode(StolenJson{187187 repo: name188188 forward: true189189 saves: fetched.saves.lensrc/cli/cmd_strip.v+9 −9
@@ -11,13 +11,13 @@1111 1212 module cli1313 14-import json14+import json21515 import src.fmt1616 import src.history1717 import src.machine1818 import src.snapshot1919 import src.storage20-import src.sync20+import src.exchange2121 2222 pub fn cmd_strip(raw []string) int {2323 args := parse('strip', raw, Options{@@ -41,12 +41,12 @@4141 save := history.resolve(store, given) or { return refuse(args.json(), err.msg()) }4242 only_save = save.id4343 }44- scope := sync.strip_scope(store, path, only_save) or { return refuse(args.json(), err.msg()) }45- mut local := sync.LocalStrip{}44+ scope := exchange.strip_scope(store, path, only_save) or { return refuse(args.json(), err.msg()) }45+ mut local := exchange.LocalStrip{}4646 if scope.files == 0 {4747 // Nothing left here, but the remote half of an earlier strip may not4848 // have finished; re-running is how it finishes.49- if !sync.strip_pending(store, path) {49+ if !exchange.strip_pending(store, path) {5050 ok('nothing in history matches ${path}')5151 return 05252 }@@ -66,7 +66,7 @@6666 note('nothing was stripped — the path did not match')6767 return 16868 }69- local = sync.strip_local(store, path, only_save) or {69+ local = exchange.strip_local(store, path, only_save) or {7070 return refuse(args.json(), err.msg())7171 }7272 }@@ -74,7 +74,7 @@7474 // and running the same strip again finishes the remote.7575 push := push_strip_to_remote(store, path)7676 if args.json() {77- emit_json(json.encode(StripJson{77+ emit_json(json2.encode(StripJson{7878 path: path7979 save: only_save8080 files_removed: local.files_removed@@ -103,7 +103,7 @@103103 104104 struct PushOutcome {105105 mut:106- pushed sync.PushReport106+ pushed exchange.PushReport107107 failed string108108 skipped bool109109 }@@ -118,7 +118,7 @@118118 mut backend := this_machine_backend() or { return PushOutcome{119119 failed: err.msg()120120 } }121- report := sync.push_strip(store, mut backend, path) or {121+ report := exchange.push_strip(store, mut backend, path) or {122122 return PushOutcome{123123 failed: err.msg()124124 }src/cli/output.v+2 −2
@@ -8,7 +8,7 @@88 99 module cli1010 11-import json11+import json21212 import os1313 import src.fmt1414 @@ -31,7 +31,7 @@3131 }3232 3333 pub fn fail_json(message string) int {34- eprintln(json.encode({34+ eprintln(json2.encode({3535 'error': message3636 }))3737 return 1src/exchange/exchange.v+268 −0
@@ -0,0 +1,268 @@1+//! Sync: offer and steal.2+//!3+//! All the cleverness sits here, above the five-operation remote interface, so4+//! the backends stay thin and this logic is written once and tested offline.5+//!6+//! The remote is a line of saves, the same shape as the local one. `offer`7+//! pushes local saves the remote lacks; `steal` pulls remote saves and, when8+//! both sides moved, replays the local work on top of the remote's. Divergence9+//! is resolved per file, never per line.10+11+module exchange12+13+import os14+import src.events15+import src.history16+import src.remote17+import src.snapshot18+import src.storage19+20+// Where the repo's name on the remote is remembered.21+const repo_state_key = 'remote_repo'22+23+// The remote head as of this machine's last successful offer, as the document24+// asks; nothing reads it back yet.25+const remote_head_key = 'remote_head'26+27+pub const no_saves = 'this repo has no saves yet — nothing to offer'28+29+pub struct OfferReport {30+pub mut:31+ repo string32+ saves int33+ objects int34+ events int35+ head string36+}37+38+// remember_repo binds a repo to its name on the remote. A clone does it at39+// once: its folder may be renamed before it ever offers.40+pub fn remember_repo(store &storage.Store, name string) ! {41+ store.set_state(repo_state_key, name)!42+}43+44+// repo_name is the folder's name at first offer, remembered in the repo so45+// renaming the folder later changes nothing.46+pub fn repo_name(store &storage.Store, mut backend remote.Backend) !string {47+ if remembered := store.state(repo_state_key) {48+ return remembered49+ }50+ name := os.base(store.root)51+ existing := backend.get(head_key(name))!52+ if existing.found {53+ theirs := existing.body.bytestr().trim_space()54+ if history.by_id(store, theirs) == none {55+ return error('the remote already has "${name}" with a history this repo does not share — rename the folder, or use another remote')56+ }57+ }58+ store.set_state(repo_state_key, name)!59+ return name60+}61+62+// offer sends what the remote lacks and then swaps `head`. Nothing is ever63+// queued: the next offer simply sends everything the remote does not have.64+pub fn offer(store &storage.Store, mut backend remote.Backend) !OfferReport {65+ name := repo_name(store, mut backend)!66+ local := history.head(store) or { return error(no_saves) }67+ mut report := OfferReport{68+ repo: name69+ head: local.id70+ }71+ remote_head := backend.get(head_key(name))!72+ // Records offered by a machine that has not applied a strip would put the73+ // stripped path back on the remote, where nothing would rewrite it again.74+ // ponytail: an offer racing a strip on another machine inside the same few75+ // seconds can still slip through; closing that needs a remote lock.76+ if unapplied_strip(store, mut backend, name)! {77+ return error('the remote has a strip this repo has not applied — run koh steal')78+ }79+ if remote_head.found {80+ theirs := remote_head.body.bytestr().trim_space()81+ newer := saves_after(store, local.id, theirs) or {82+ return error('remote has new saves — run koh steal')83+ }84+ for save in newer {85+ upload_save(store, mut backend, name, save, mut report)!86+ }87+ } else {88+ oldest_first := history.line(store)!89+ for index := oldest_first.len - 1; index >= 0; index-- {90+ upload_save(store, mut backend, name, oldest_first[index], mut report)!91+ }92+ }93+ upload_events(store, mut backend, name, mut report)!94+ if !backend.put_if_match(head_key(name), local.id.bytes(), remote_head.tag)! {95+ return error('remote has new saves — run koh steal')96+ }97+ store.set_state(remote_head_key, local.id)!98+ return report99+}100+101+// saves_after lists the local saves newer than `theirs`, oldest first. It fails102+// when `theirs` is not on this line, which is the honest answer: we cannot103+// offer on top of a save we do not have.104+fn saves_after(store &storage.Store, ours string, theirs string) ![]history.Save {105+ mut newer := []history.Save{}106+ mut cursor := ours107+ for cursor != '' && cursor != theirs {108+ save := history.by_id(store, cursor) or { return error('save ${cursor} is missing') }109+ newer << save110+ cursor = save.parent111+ }112+ if cursor != theirs {113+ return error('${theirs} is not on this line')114+ }115+ newer.reverse_in_place()116+ return newer117+}118+119+fn upload_save(store &storage.Store, mut backend remote.Backend, repo string, save history.Save, mut report OfferReport) ! {120+ files := history.files_of(store, save.id)!121+ // Saves go up oldest first and head moves only once all of them landed, so122+ // the parent and its objects are on the remote already: only content new123+ // in this save travels, not a round trip per file in the repo.124+ mut sent := map[string]bool{}125+ if save.parent != '' {126+ for file in history.files_of(store, save.parent)! {127+ sent[file.hash] = true128+ }129+ }130+ for file in files {131+ if file.hash in sent {132+ continue133+ }134+ sent[file.hash] = true135+ if backend.put_if_absent(object_key(repo, file.hash), store.object_blob(file.hash)!)! {136+ report.objects++137+ }138+ }139+ record := save_record(store, save)!140+ if backend.put_if_absent(save_key(repo, save.id), encode_save(record).bytes())! {141+ report.saves++142+ }143+}144+145+// upload_events sends every event the remote does not have. Events are tiny, so146+// listing them costs less than tracking which ones were sent.147+fn upload_events(store &storage.Store, mut backend remote.Backend, repo string, mut report OfferReport) ! {148+ already := backend.list(events_prefix(repo))!149+ mut already_sent := map[string]bool{}150+ for key in already {151+ already_sent[os.base(key)] = true152+ }153+ for event in events.all(store)! {154+ if event.id in already_sent {155+ continue156+ }157+ if backend.put_if_absent(event_key(repo, event.id), encode_event(event).bytes())! {158+ report.events++159+ }160+ }161+}162+163+// fetched holds everything a steal read from the remote, before any of it is164+// written locally. Network work happens first; the database gets one165+// transaction, so a failure leaves nothing half-applied.166+pub struct Fetched {167+pub mut:168+ repo_objects map[string][]u8169+ saves []SaveRecord // oldest first170+ events_out []events.Event171+ head string172+ base string // the newest save both sides already shared, '' when none173+}174+175+pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched {176+ remote_head := backend.get(head_key(repo))!177+ if !remote_head.found {178+ return error('the remote has no "${repo}" yet — run koh offer here first')179+ }180+ mut fetched := Fetched{181+ head: remote_head.body.bytestr().trim_space()182+ }183+ mut cursor := fetched.head184+ for cursor != '' && history.by_id(store, cursor) == none {185+ record_object := backend.get(save_key(repo, cursor))!186+ if !record_object.found {187+ return error('the remote is missing save ${cursor} — run koh offer there')188+ }189+ record := decode_save(record_object.body)!190+ for file in record.files {191+ if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects {192+ blob := backend.get(object_key(repo, file.hash))!193+ if !blob.found {194+ return error('the remote is missing object ${file.hash} that save ${record.id} needs')195+ }196+ fetched.repo_objects[file.hash] = blob.body197+ }198+ }199+ fetched.saves << record200+ cursor = record.parent201+ }202+ fetched.saves.reverse_in_place()203+ fetched.base = cursor204+ fetched.events_out = fetch_events(store, mut backend, repo)!205+ return fetched206+}207+208+fn fetch_events(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {209+ have := events.all(store)!210+ mut known := map[string]bool{}211+ for event in have {212+ known[event.id] = true213+ }214+ mut incoming := []events.Event{}215+ for key in backend.list(events_prefix(repo))! {216+ id := os.base(key)217+ if id in known {218+ continue219+ }220+ body := backend.get(event_key(repo, id))!221+ if !body.found {222+ continue223+ }224+ incoming << decode_event(body.body)!225+ }226+ return incoming227+}228+229+// write_fetched lands a fetch in one transaction: objects first (verified),230+// then the events, then the saves they belong to. A new strip reaches every231+// save already here, but an arriving save only when `reaches` says so: one232+// made after the strip is new work, not what the strip removed.233+pub fn write_fetched(store &storage.Store, fetched Fetched) ! {234+ for hash, blob in fetched.repo_objects {235+ store.put_object_blob(hash, blob)!236+ }237+ mut strips := []events.Event{}238+ for event in fetched.events_out {239+ if !events.receive(store, event)! {240+ continue241+ }242+ if event.kind == events.kind_strip {243+ // Somebody stripped a path: remove it here too, so the two244+ // histories agree from now on. Objects go once the saves are in.245+ remove_path(store, event.path, event.save_id)!246+ strips << event247+ }248+ }249+ for record in fetched.saves {250+ files := to_files(record).filter(fn [strips, record] (file snapshot.FileEntry) bool {251+ return !strips.any(reaches(it, record.id, record.created_at)252+ && covers(it.path, file.path))253+ })254+ history.import_save(store, to_save(record), files)!255+ }256+ if strips.len > 0 {257+ store.drop_unreferenced_objects()!258+ }259+}260+261+// reaches reports whether a strip event applies to a save arriving from the262+// remote. A strip of every save covers the saves made up to its own time.263+fn reaches(strip events.Event, save_id string, created_at i64) bool {264+ if strip.save_id != '' {265+ return strip.save_id == save_id266+ }267+ return created_at <= strip.created_at268+}src/exchange/exchange_test.v+1236 −0
@@ -0,0 +1,1236 @@1+//! Two-machine tests: two repos and one folder remote standing in for the Mac2+//! and the PC. This is where offer, steal and combine are actually proven —3+//! the folder backend exists so these run offline.4+5+module exchange6+7+import os8+import src.events9+import src.history10+import src.remote11+import src.snapshot12+import src.storage13+14+// Machines keep their repo in a folder of the same NAME, because the remote key15+// is the folder's name; that is what makes them the same repo to the remote.16+fn machine(name string) !string {17+ dir := os.join_path(os.temp_dir(), 'koh3_sync_${os.getpid()}_${name}', 'proj')18+ os.rmdir_all(os.dir(dir)) or {}19+ os.mkdir_all(dir, os.MkdirParams{})!20+ return dir21+}22+23+fn scratch(name string) !string {24+ dir := os.join_path(os.temp_dir(), 'koh3_sync_${os.getpid()}_${name}')25+ os.rmdir_all(dir) or {}26+ os.mkdir_all(dir, os.MkdirParams{})!27+ return dir28+}29+30+fn open_repo(dir string) !storage.Store {31+ mut store := storage.create(dir)!32+ os.write_file(os.join_path(dir, '.kohignore'), '*.log\n')!33+ return store34+}35+36+fn put(root string, rel string, text string) ! {37+ full := os.join_path(root, rel)38+ os.mkdir_all(os.dir(full), os.MkdirParams{})!39+ os.write_file(full, text)!40+}41+42+fn commit(store &storage.Store, message string) !history.Save {43+ return history.commit_worktree(store, store.root, snapshot.load(store.root), history.SaveInput{44+ message: message45+ })!46+}47+48+fn offer_now(store &storage.Store, mut backend remote.Backend) !OfferReport {49+ return offer(store, mut backend)!50+}51+52+fn steal_now(store &storage.Store, mut backend remote.Backend) !StealReport {53+ return steal(store, mut backend, StealOptions{54+ theirs: true55+ })!56+}57+58+fn test_first_offer_publishes_the_whole_line() {59+ base := scratch('first')!60+ defer {61+ os.rmdir_all(base) or {}62+ }63+ mut a := open_repo(machine('first_a')!)!64+ defer {65+ a.close()66+ }67+ put(a.root, 'a.txt', 'one\n')!68+ commit(a, 'first')!69+ put(a.root, 'a.txt', 'two\n')!70+ commit(a, 'second')!71+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!72+73+ report := offer_now(a, mut backend)!74+ assert report.repo == 'proj'75+ assert report.saves == 276+ assert report.objects >= 277+ assert backend.get(head_key('proj'))!.found78+ assert backend.list('proj/saves')!.len == 279+80+ // A second offer with nothing new uploads nothing and still succeeds.81+ again := offer_now(a, mut backend)!82+ assert again.saves == 083+}84+85+fn test_the_repo_name_is_remembered_and_head_offered() {86+ base := scratch('remember')!87+ defer {88+ os.rmdir_all(base) or {}89+ }90+ mut a := open_repo(machine('remember_a')!)!91+ defer {92+ a.close()93+ }94+ put(a.root, 'a.txt', 'one\n')!95+ commit(a, 'first')!96+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!97+ offer_now(a, mut backend)!98+ assert a.state('remote_repo') or { '' } == 'proj'99+ assert backend.get(head_key('proj'))!.body.bytestr() == head_of(a).id100+ // Offer's step 3: the new remote head is recorded in state.101+ assert a.state('remote_head') or { '' } == head_of(a).id102+103+ // A second offer, with the remote already holding our head, must not104+ // refuse: the name is remembered, so the remote is ours.105+ put(a.root, 'a.txt', 'two\n')!106+ commit(a, 'second')!107+ offer_now(a, mut backend)!108+ assert backend.get(head_key('proj'))!.body.bytestr() == head_of(a).id109+}110+111+fn test_steal_fast_forwards_a_machine_that_has_nothing_of_its_own() {112+ base := scratch('forward')!113+ defer {114+ os.rmdir_all(base) or {}115+ }116+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!117+ mut a := open_repo(machine('fwd_a')!)!118+ defer {119+ a.close()120+ }121+ put(a.root, 'a.txt', 'from A\n')!122+ commit(a, 'A writes')!123+ offer_now(a, mut backend)!124+125+ // B starts empty and pulls the whole repo.126+ mut b := open_repo(machine('fwd_b')!)!127+ defer {128+ b.close()129+ }130+ fetched := fetch_repo(mut backend, 'proj')!131+ b.begin()!132+ write_fetched(b, fetched)!133+ history.set_head(b, fetched.head)!134+ b.commit()!135+ snapshot.restore_files(b, b.root, snapshot.load(b.root), history.files_of(b, fetched.head)!, [])!136+ assert os.read_file(os.join_path(b.root, 'a.txt'))! == 'from A\n'137+ assert head_of(b).id == fetched.head138+}139+140+// A machine that is only ahead has nothing to combine: stealing must leave its141+// saves, and their IDs, exactly where they are.142+fn test_steal_while_only_ahead_leaves_the_line_alone() {143+ base := scratch('ahead')!144+ defer {145+ os.rmdir_all(base) or {}146+ }147+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!148+ mut a := open_repo(machine('ahead_a')!)!149+ defer {150+ a.close()151+ }152+ put(a.root, 'a.txt', 'one\n')!153+ commit(a, 'offered')!154+ offer_now(a, mut backend)!155+ put(a.root, 'a.txt', 'two\n')!156+ ahead := commit(a, 'not offered yet')!157+ report := steal_now(a, mut backend)!158+ assert report.replayed == 0159+ assert head_of(a).id == ahead.id160+ // Nothing was restored, so nothing is left marked as half-restored.161+ assert !snapshot.restore_pending(a)162+ assert history.all_saves(a)!.len == 2163+ offer_now(a, mut backend)!164+ assert backend.get(head_key('proj'))!.body.bytestr() == ahead.id165+}166+167+// A link is invisible to the walk, so a tree holding one is clean; the files168+// a steal brings in must still never land in the folder it points at.169+fn test_steal_never_writes_through_a_symlinked_folder() {170+ base := scratch('steal_link')!171+ defer {172+ os.rmdir_all(base) or {}173+ }174+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!175+ mut a := open_repo(machine('steal_link_a')!)!176+ defer {177+ a.close()178+ }179+ mut b := open_repo(machine('steal_link_b')!)!180+ defer {181+ b.close()182+ }183+ put(a.root, 'a.txt', 'one\n')!184+ commit(a, 'first')!185+ offer_now(a, mut backend)!186+ join(b, mut backend)!187+ outside := os.join_path(base, 'outside')188+ put(outside, 'a.md', 'precious\n')!189+ os.symlink(outside, os.join_path(b.root, 'docs'))!190+ put(a.root, 'docs/a.md', 'from A\n')!191+ commit(a, 'A adds docs')!192+ offer_now(a, mut backend)!193+ head_before := head_of(b).id194+ if _ := steal_now(b, mut backend) {195+ assert false, 'a steal writing through a symlinked folder must refuse'196+ } else {197+ assert err.msg().contains('docs')198+ }199+ assert os.read_file(os.join_path(outside, 'a.md'))! == 'precious\n'200+ assert head_of(b).id == head_before201+}202+203+// A steal onto a half-restored tree would replay it as somebody's work.204+fn test_steal_refuses_after_a_restore_cut_short() {205+ base := scratch('steal_cut')!206+ defer {207+ os.rmdir_all(base) or {}208+ }209+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!210+ mut a := open_repo(machine('steal_cut_a')!)!211+ defer {212+ a.close()213+ }214+ put(a.root, 'a.txt', 'one\n')!215+ commit(a, 'first')!216+ offer_now(a, mut backend)!217+ a.set_state('restoring', 'yes')!218+ if _ := steal_now(a, mut backend) {219+ assert false, 'steal must refuse while a restore is unfinished'220+ } else {221+ assert err.msg().contains('koh revert')222+ }223+}224+225+// A replayed save keeps its original time but sits on a newer parent, so a226+// clone cannot order saves by time.227+fn test_a_clone_imports_parents_before_children() {228+ base := scratch('clone_order')!229+ defer {230+ os.rmdir_all(base) or {}231+ }232+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!233+ mut a := open_repo(machine('order_a')!)!234+ defer {235+ a.close()236+ }237+ put(a.root, 'a.txt', 'one\n')!238+ parent := commit(a, 'parent')!239+ child := history.commit_files(a, '', history.files_of(a, parent.id)!, history.SaveInput{240+ message: 'replayed, older than its parent'241+ created_at: parent.created_at - 3600242+ })!243+ offer_now(a, mut backend)!244+ mut b := open_repo(machine('order_b')!)!245+ defer {246+ b.close()247+ }248+ join(b, mut backend)!249+ assert head_of(b).id == child.id250+}251+252+// A chmod is unsaved work too: a steal would restore the old mode over it.253+fn test_steal_refuses_an_unsaved_executable_bit_change() {254+ base := scratch('chmod')!255+ defer {256+ os.rmdir_all(base) or {}257+ }258+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!259+ mut a := open_repo(machine('chmod_a')!)!260+ defer {261+ a.close()262+ }263+ put(a.root, 'run.sh', 'echo hi\n')!264+ commit(a, 'script')!265+ offer_now(a, mut backend)!266+ os.chmod(os.join_path(a.root, 'run.sh'), 0o755)!267+ if _ := steal_now(a, mut backend) {268+ assert false, 'a chmod-only change should make the tree dirty'269+ } else {270+ assert err.msg() == dirty_tree271+ }272+}273+274+fn test_divergence_without_clashes_keeps_both_machines_work() {275+ base := scratch('diverge')!276+ defer {277+ os.rmdir_all(base) or {}278+ }279+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!280+ mut a := open_repo(machine('div_a')!)!281+ defer {282+ a.close()283+ }284+ mut b := open_repo(machine('div_b')!)!285+ defer {286+ b.close()287+ }288+ put(a.root, 'shared.txt', 'base\n')!289+ commit(a, 'base')!290+ offer_now(a, mut backend)!291+292+ // B joins, then both machines change a different file.293+ join(b, mut backend)!294+ put(a.root, 'fromA.txt', 'A\n')!295+ commit(a, 'A adds a file')!296+ put(b.root, 'fromB.txt', 'B\n')!297+ commit(b, 'B adds a file')!298+ offer_now(a, mut backend)!299+300+ // B's offer is refused: the remote has moved on.301+ if _ := offer_now(b, mut backend) {302+ assert false, 'offering on top of unknown remote saves should be refused'303+ } else {304+ assert err.msg().contains('run koh steal')305+ }306+307+ report := steal_now(b, mut backend)!308+ assert !report.forward309+ assert report.replayed == 1310+ assert report.clashes.len == 0311+ // Both files are on B's line now, and on disk.312+ assert os.read_file(os.join_path(b.root, 'fromA.txt'))! == 'A\n'313+ assert os.read_file(os.join_path(b.root, 'fromB.txt'))! == 'B\n'314+ assert os.read_file(os.join_path(b.root, 'shared.txt'))! == 'base\n'315+ // B's own save was replayed, not erased.316+ mut saw_replay := false317+ for save in history.all_saves(b)! {318+ if save.replayed_from != '' {319+ saw_replay = true320+ }321+ }322+ assert saw_replay323+ // And the remote now has B's work too.324+ offer_now(b, mut backend)!325+ line_head := backend.get(head_key('proj'))!.body.bytestr().trim_space()326+ assert line_head == head_of(b).id327+}328+329+// An agent that edits a file and then undoes the edit leaves no net change, so330+// the file is not a clash when the other machine changed it. Replaying the331+// edit itself must not stop the steal: the remote's version simply stands.332+// Likewise a file both machines ended up changing the same way.333+fn test_a_local_change_undone_does_not_block_a_combine() {334+ base := scratch('undone')!335+ defer {336+ os.rmdir_all(base) or {}337+ }338+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!339+ mut a := open_repo(machine('undone_a')!)!340+ defer {341+ a.close()342+ }343+ mut b := open_repo(machine('undone_b')!)!344+ defer {345+ b.close()346+ }347+ put(a.root, 'f.txt', 'base\n')!348+ put(a.root, 'g.txt', 'base\n')!349+ commit(a, 'base')!350+ offer_now(a, mut backend)!351+ join(b, mut backend)!352+353+ put(a.root, 'f.txt', 'from A\n')!354+ put(a.root, 'g.txt', 'same\n')!355+ commit(a, 'A edits both')!356+ offer_now(a, mut backend)!357+358+ put(b.root, 'f.txt', 'tried on B\n')!359+ put(b.root, 'g.txt', 'first try\n')!360+ tried := commit(b, 'B tries')!361+ put(b.root, 'f.txt', 'base\n')!362+ put(b.root, 'g.txt', 'same\n')!363+ put(b.root, 'h.txt', 'B\n')!364+ undone := commit(b, 'B undoes f, lands on the same g, adds h')!365+366+ report := steal_now(b, mut backend)!367+ assert report.clashes.len == 0368+ assert os.read_file(os.join_path(b.root, 'f.txt'))! == 'from A\n'369+ assert os.read_file(os.join_path(b.root, 'g.txt'))! == 'same\n'370+ assert os.read_file(os.join_path(b.root, 'h.txt'))! == 'B\n'371+ // "B tries" only touched files A changed, so it has nothing left to372+ // replay; the second save carries h.txt. Both originals stay in history.373+ assert report.replayed == 1374+ assert head_of(b).replayed_from == undone.id375+ ids := history.all_saves(b)!.map(it.id)376+ assert tried.id in ids && undone.id in ids377+ assert history.files_of(b, tried.id)!.any(it.path == 'f.txt')378+}379+380+// A person deciding a clash may take minutes; the question must come before381+// the write lock, or every agent's save on this repo waits and then fails.382+fn test_clash_questions_are_asked_before_the_write_lock() {383+ base := scratch('ask_unlocked')!384+ defer {385+ os.rmdir_all(base) or {}386+ }387+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!388+ mut a := open_repo(machine('ask_a')!)!389+ defer {390+ a.close()391+ }392+ mut b := open_repo(machine('ask_b')!)!393+ defer {394+ b.close()395+ }396+ put(a.root, 'same.txt', 'base\n')!397+ commit(a, 'base')!398+ offer_now(a, mut backend)!399+ join(b, mut backend)!400+ put(a.root, 'same.txt', 'A version\n')!401+ commit(a, 'A edits same')!402+ offer_now(a, mut backend)!403+ put(b.root, 'same.txt', 'B version\n')!404+ commit(b, 'B edits same')!405+ root := b.root406+ // An agent saves while the person is still choosing.407+ if _ := steal(b, mut backend, StealOptions{408+ ask: fn [root] (clash Clash) Side {409+ other := storage.open(root) or { panic(err) }410+ defer {411+ other.close()412+ }413+ os.write_file(os.join_path(root, 'agent.txt'), 'saved meanwhile\n') or { panic(err) }414+ history.commit_worktree(other, root, snapshot.load(root), history.SaveInput{415+ message: 'agent saves during the question'416+ }) or { panic('steal held the write lock while asking: ${err}') }417+ return .mine418+ }419+ })420+ {421+ assert false, 'a steal whose line moved during the question must refuse'422+ } else {423+ assert err.msg().contains('changed while')424+ }425+ // The agent's save is still head, and a second steal combines it too.426+ assert head_of(b).message == 'agent saves during the question'427+ report := steal(b, mut backend, StealOptions{428+ mine: true429+ })!430+ assert report.kept_mine == 1431+ assert os.read_file(os.join_path(b.root, 'agent.txt'))! == 'saved meanwhile\n'432+ assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'B version\n'433+}434+435+fn test_clashing_files_are_chosen_one_at_a_time() {436+ base := scratch('clash')!437+ defer {438+ os.rmdir_all(base) or {}439+ }440+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!441+ mut a := open_repo(machine('clash_a')!)!442+ defer {443+ a.close()444+ }445+ mut b := open_repo(machine('clash_b')!)!446+ defer {447+ b.close()448+ }449+ put(a.root, 'same.txt', 'base\n')!450+ put(a.root, 'other.txt', 'quiet\n')!451+ commit(a, 'base')!452+ offer_now(a, mut backend)!453+ join(b, mut backend)!454+455+ // Both machines edit the same file, differently.456+ put(a.root, 'same.txt', 'A version\n')!457+ commit(a, 'A edits same')!458+ offer_now(a, mut backend)!459+ put(b.root, 'same.txt', 'B version\n')!460+ commit(b, 'B edits same')!461+462+ // Without a decision, steal refuses and says which files clash.463+ if _ := steal(b, mut backend, StealOptions{}) {464+ assert false, 'an undecided clash must be refused'465+ } else {466+ assert err.msg().contains('--mine') || err.msg().contains('--theirs')467+ }468+ assert find_clashes(history.files_of(b, head_of(b).id)!, history.files_of(b, head_of(b).id)!, history.files_of(b,469+ head_of(b).id)!).len == 0470+471+ // Choosing the remote's version ends with A's content.472+ report := steal_now(b, mut backend)!473+ assert report.clashes == ['same.txt']474+ assert report.kept_mine == 0475+ assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'A version\n'476+ assert os.read_file(os.join_path(b.root, 'other.txt'))! == 'quiet\n'477+}478+479+fn test_choosing_mine_keeps_the_local_version() {480+ base := scratch('mine')!481+ defer {482+ os.rmdir_all(base) or {}483+ }484+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!485+ mut a := open_repo(machine('mine_a')!)!486+ defer {487+ a.close()488+ }489+ mut b := open_repo(machine('mine_b')!)!490+ defer {491+ b.close()492+ }493+ put(a.root, 'same.txt', 'base\n')!494+ commit(a, 'base')!495+ offer_now(a, mut backend)!496+ join(b, mut backend)!497+ put(a.root, 'same.txt', 'A version\n')!498+ commit(a, 'A edits')!499+ offer_now(a, mut backend)!500+ put(b.root, 'same.txt', 'B version\n')!501+ commit(b, 'B edits')!502+503+ report := steal(b, mut backend, StealOptions{504+ mine: true505+ })!506+ assert report.kept_mine == 1507+ assert os.read_file(os.join_path(b.root, 'same.txt'))! == 'B version\n'508+ // The final save says what it kept.509+ assert head_of(b).message.contains('combine: kept mine for 1 file')510+ // The pre-combine tree is still reachable in history.511+ assert head_of(b).restores == ''512+ assert history.all_saves(b)!.len >= 4513+}514+515+fn test_flags_travel_with_the_saves() {516+ base := scratch('flags')!517+ defer {518+ os.rmdir_all(base) or {}519+ }520+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!521+ mut a := open_repo(machine('flag_a')!)!522+ defer {523+ a.close()524+ }525+ mut b := open_repo(machine('flag_b')!)!526+ defer {527+ b.close()528+ }529+ put(a.root, 'a.txt', 'one\n')!530+ save := commit(a, 'suspect save')!531+ events.flag(a, save.id, 'look at this')!532+ offer_now(a, mut backend)!533+534+ // B has nothing yet, so a fetch brings the save and its flag across.535+ fetched := fetch_back_to_known(b, mut backend, 'proj')!536+ assert fetched.events_out.len == 1537+ assert fetched.events_out[0].kind == events.kind_flag538+ b.begin()!539+ write_fetched(b, fetched)!540+ history.set_head(b, fetched.head)!541+ b.commit()!542+ state := events.flag_state(b, save.id)!543+ assert state.flagged544+ assert state.reason == 'look at this'545+}546+547+fn test_a_second_offer_racing_head_is_refused_not_forced() {548+ base := scratch('race')!549+ defer {550+ os.rmdir_all(base) or {}551+ }552+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!553+ mut a := open_repo(machine('race_a')!)!554+ defer {555+ a.close()556+ }557+ put(a.root, 'a.txt', 'one\n')!558+ commit(a, 'base')!559+ offer_now(a, mut backend)!560+561+ // Someone else replaces head behind our back.562+ assert backend.put_if_match(head_key('proj'), 'a-save-we-do-not-have'.bytes(),563+ backend.get(head_key('proj'))!.tag)!564+565+ put(a.root, 'a.txt', 'two\n')!566+ commit(a, 'second')!567+ if _ := offer_now(a, mut backend) {568+ assert false, 'offering while the remote head moved must be refused'569+ } else {570+ assert err.msg().contains('run koh steal')571+ }572+}573+574+// join makes a brand new machine share the remote's line.575+fn join(store &storage.Store, mut backend remote.Backend) ! {576+ fetched := fetch_repo(mut backend, 'proj')!577+ store.begin()!578+ write_fetched(store, fetched)!579+ history.set_head(store, fetched.head)!580+ store.commit()!581+ snapshot.restore_files(store, store.root, snapshot.load(store.root), history.files_of(store,582+ fetched.head)!, [])!583+ store.set_state('remote_repo', 'proj')!584+}585+586+struct FaultyBackend {587+mut:588+ inner remote.Backend589+ writes int590+ reads int591+ fail_write int // fail when the write counter reaches this, 0 never592+ fail_read int593+ // fail every delete while set594+ fail_delete bool595+}596+597+fn (mut f FaultyBackend) get(key string) !remote.Object {598+ f.reads++599+ if f.fail_read > 0 && f.reads >= f.fail_read {600+ return error('injected read failure')601+ }602+ return f.inner.get(key)!603+}604+605+fn (mut f FaultyBackend) list(prefix string) ![]string {606+ return f.inner.list(prefix)!607+}608+609+fn (mut f FaultyBackend) put_if_absent(key string, body []u8) !bool {610+ f.writes++611+ if f.fail_write > 0 && f.writes >= f.fail_write {612+ return error('injected write failure')613+ }614+ return f.inner.put_if_absent(key, body)!615+}616+617+fn (mut f FaultyBackend) put_if_match(key string, body []u8, tag string) !bool {618+ f.writes++619+ if f.fail_write > 0 && f.writes >= f.fail_write {620+ return error('injected write failure')621+ }622+ return f.inner.put_if_match(key, body, tag)!623+}624+625+fn (mut f FaultyBackend) delete(key string) ! {626+ if f.fail_delete {627+ return error('injected delete failure')628+ }629+ f.inner.delete(key)!630+}631+632+// Every save offers after it is made, so a one-file change must cost one633+// object, not one conditional write per file in the repo: on S3 each of those634+// is a round trip.635+fn test_an_offer_sends_only_the_objects_a_save_adds() {636+ base := scratch('offer_diff')!637+ defer {638+ os.rmdir_all(base) or {}639+ }640+ mut a := open_repo(machine('offer_diff_a')!)!641+ defer {642+ a.close()643+ }644+ for i in 0 .. 20 {645+ put(a.root, 'f${i}.txt', 'file ${i}\n')!646+ }647+ commit(a, 'twenty files')!648+ mut backend := FaultyBackend{649+ inner: remote.new_folder(os.join_path(base, 'remote'))!650+ }651+ offer(a, mut backend)!652+ put(a.root, 'f3.txt', 'changed\n')!653+ commit(a, 'one change')!654+ before := backend.writes655+ report := offer(a, mut backend)!656+ assert report.objects == 1657+ // One object, one save record, one head swap.658+ assert backend.writes - before == 3659+}660+661+fn test_a_failed_offer_is_safe_to_re_run() {662+ base := scratch('fault_offer')!663+ defer {664+ os.rmdir_all(base) or {}665+ }666+ remote_dir := os.join_path(base, 'remote')667+ mut a := open_repo(machine('fault_a')!)!668+ defer {669+ a.close()670+ }671+ put(a.root, 'a.txt', 'one\n')!672+ commit(a, 'first')!673+ put(a.root, 'a.txt', 'two\n')!674+ commit(a, 'second')!675+676+ // The first offer dies after the first write.677+ mut flaky := FaultyBackend{678+ inner: remote.new_folder(remote_dir)!679+ fail_write: 2680+ }681+ if _ := offer(a, mut flaky) {682+ assert false, 'the injected failure should have surfaced'683+ }684+685+ // Re-running with a healthy remote finishes the job: "already exists" is686+ // success, so nothing is lost and nothing is duplicated.687+ mut healthy := remote.new_folder(remote_dir)!688+ report := offer(a, mut healthy)!689+ assert report.saves == 2690+ assert healthy.get(head_key('proj'))!.body.bytestr().trim_space() == head_of(a).id691+ assert healthy.list('proj/saves')!.len == 2692+}693+694+fn test_a_failed_steal_leaves_nothing_behind() {695+ base := scratch('fault_steal')!696+ defer {697+ os.rmdir_all(base) or {}698+ }699+ remote_dir := os.join_path(base, 'remote')700+ mut a := open_repo(machine('fault_s_a')!)!701+ defer {702+ a.close()703+ }704+ mut b := open_repo(machine('fault_s_b')!)!705+ defer {706+ b.close()707+ }708+ put(a.root, 'a.txt', 'from A\n')!709+ commit(a, 'A writes')!710+ mut healthy := remote.new_folder(remote_dir)!711+ offer(a, mut healthy)!712+713+ // B's fetch fails half way through reading the remote.714+ mut flaky := FaultyBackend{715+ inner: remote.new_folder(remote_dir)!716+ fail_read: 2717+ }718+ if _ := fetch_back_to_known(b, mut flaky, 'proj') {719+ assert false, 'the injected read failure should have surfaced'720+ }721+ // Nothing was written: the store is untouched and still has no saves.722+ assert history.head(b) == none723+ assert b.object_count()! == 0724+725+ // A healthy retry works.726+ fetched := fetch_back_to_known(b, mut healthy, 'proj')!727+ b.begin()!728+ write_fetched(b, fetched)!729+ history.set_head(b, fetched.head)!730+ b.commit()!731+ snapshot.restore_files(b, b.root, snapshot.load(b.root), history.files_of(b, fetched.head)!, [])!732+ assert os.read_file(os.join_path(b.root, 'a.txt'))! == 'from A\n'733+}734+735+fn test_an_interrupted_strip_is_idempotent() {736+ // The local half of a strip is one transaction, so an interrupted strip is737+ // simply a strip that did not happen; re-running it must land the same738+ // result once, not twice.739+ base := scratch('fault_strip')!740+ defer {741+ os.rmdir_all(base) or {}742+ }743+ mut a := open_repo(machine('fault_strip_a')!)!744+ defer {745+ a.close()746+ }747+ put(a.root, 'secret.txt', 'shh\n')!748+ put(a.root, 'keep.txt', 'fine\n')!749+ save := commit(a, 'add both')!750+751+ for _ in 0 .. 2 {752+ strip_local(a, 'secret.txt', '')!753+ }754+ files := history.files_of(a, save.id)!755+ // The repo's own `.kohignore` is tracked content, so it stays; only the756+ // stripped path goes.757+ assert files.map(it.path) == ['.kohignore', 'keep.txt']758+ assert events.strip_events(a)!.len == 1759+}760+761+// SQLite's LIKE ignores ASCII case, so a strip of `docs` must not reach into762+// `Docs/`, and `a_b` must not reach `aXb`: strip destroys, so it matches exactly.763+// The tree is built by hand: on macOS `Docs` and `docs` are one folder on disk,764+// but a save made on Linux can hold both.765+fn test_a_strip_matches_the_path_exactly() {766+ mut a := open_repo(machine('strip_exact')!)!767+ defer {768+ a.close()769+ }770+ hash := a.put_object('x\n'.bytes())!771+ mut files := []snapshot.FileEntry{}772+ for path in ['DOCS.txt', 'Docs/keep.txt', 'a_b/x.txt', 'aXb/y.txt', 'docs/secret.txt'] {773+ files << snapshot.FileEntry{774+ path: path775+ hash: hash776+ }777+ }778+ save := history.commit_files(a, '', files, history.SaveInput{779+ message: 'add all'780+ })!781+ assert strip_scope(a, 'docs', '')!.files == 1782+ strip_local(a, 'docs', '')!783+ strip_local(a, 'a_b', '')!784+ assert history.files_of(a, save.id)!.map(it.path) == ['DOCS.txt', 'Docs/keep.txt', 'aXb/y.txt']785+}786+787+fn test_a_strip_reaches_the_remote_and_other_machines() {788+ base := scratch('strip_sync')!789+ defer {790+ os.rmdir_all(base) or {}791+ }792+ mut a := open_repo(machine('strip_a')!)!793+ defer {794+ a.close()795+ }796+ mut b := open_repo(machine('strip_b')!)!797+ defer {798+ b.close()799+ }800+ put(a.root, 'secrets.env', 'api_key=1\n')!801+ put(a.root, 'keep.txt', 'fine\n')!802+ save := commit(a, 'add both')!803+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!804+ offer_now(a, mut backend)!805+ join(b, mut backend)!806+ assert b.object_count()! > 0807+808+ // A strips the path locally, the way `koh strip` does, then tells the809+ // remote.810+ mut secret_hash := ''811+ for file in history.files_of(a, save.id)! {812+ if file.path == 'secrets.env' {813+ secret_hash = file.hash814+ }815+ }816+ assert secret_hash != ''817+ strip_local(a, 'secrets.env', '')!818+ report := push_strip(a, mut backend, 'secrets.env')!819+ assert report.event_sent820+ assert report.records == 1821+822+ // The remote's copy of the save no longer lists the file, and the object is823+ // gone from the remote.824+ record := decode_save(backend.get(save_key('proj', save.id))!.body)!825+ for file in record.files {826+ assert file.path != 'secrets.env'827+ }828+ // The stripped object is gone from the remote; the ones still referenced829+ // are not.830+ assert !backend.get(object_key('proj', secret_hash))!.found831+ assert backend.list('proj/objects')!.len == 2832+833+ // Another machine hears about it on its next steal and applies it.834+ b.begin()!835+ fetched := fetch_back_to_known(b, mut backend, 'proj')!836+ write_fetched(b, fetched)!837+ b.commit()!838+ mut paths := []string{}839+ for file in history.files_of(b, save.id)! {840+ paths << file.path841+ }842+ assert 'secrets.env' !in paths843+ assert 'keep.txt' in paths844+845+ // The event arrives under its own ID, so it is applied once: B's offer adds846+ // no second copy, and a file B saves again after the strip stays saved847+ // through B's next steal.848+ assert events.strip_events(b)!.map(it.id) == events.strip_events(a)!.map(it.id)849+ put(b.root, 'secrets.env', 'rotated\n')!850+ readded := commit(b, 'add the rotated key back')!851+ offer_now(b, mut backend)!852+ assert backend.list('proj/events')!.len == 1853+ b.begin()!854+ write_fetched(b, fetch_back_to_known(b, mut backend, 'proj')!)!855+ b.commit()!856+ assert 'secrets.env' in history.files_of(b, readded.id)!.map(it.path)857+}858+859+// A strip removes a path from the saves that existed when it was made. A key860+// rotated and saved again afterwards is new work: a machine that clones the861+// repo later must get it, not strip it on arrival.862+fn test_a_clone_keeps_a_file_saved_again_after_a_strip() {863+ base := scratch('strip_clone')!864+ defer {865+ os.rmdir_all(base) or {}866+ }867+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!868+ mut a := open_repo(machine('strip_clone_a')!)!869+ defer {870+ a.close()871+ }872+ put(a.root, 'secrets.env', 'api_key=1\n')!873+ old := commit(a, 'leak the key')!874+ offer_now(a, mut backend)!875+ strip_local(a, 'secrets.env', '')!876+ push_strip(a, mut backend, 'secrets.env')!877+ // Saves after the strip must be later in time, as they would be for real.878+ put(a.root, 'secrets.env', 'api_key=2\n')!879+ rotated := history.commit_files(a, a.root, snapshot.walk(a, a.root, snapshot.load(a.root))!, history.SaveInput{880+ message: 'rotate the key'881+ created_at: events.strip_events(a)!.last().created_at + 60882+ })!883+ offer_now(a, mut backend)!884+885+ mut c := open_repo(machine('strip_clone_c')!)!886+ defer {887+ c.close()888+ }889+ join(c, mut backend)!890+ assert 'secrets.env' in history.files_of(c, rotated.id)!.map(it.path)891+ assert 'secrets.env' !in history.files_of(c, old.id)!.map(it.path)892+ assert os.read_file(os.join_path(c.root, 'secrets.env'))! == 'api_key=2\n'893+}894+895+// The local half commits before the remote hears anything, so a remote failure896+// leaves nothing to find locally; the pending note is what lets a re-run897+// finish. Objects a remote-only save still needs must survive the strip.898+fn test_an_interrupted_remote_strip_finishes_when_re_run() {899+ base := scratch('strip_resume')!900+ defer {901+ os.rmdir_all(base) or {}902+ }903+ remote_dir := os.join_path(base, 'remote')904+ mut backend := remote.new_folder(remote_dir)!905+ mut a := open_repo(machine('resume_a')!)!906+ defer {907+ a.close()908+ }909+ mut b := open_repo(machine('resume_b')!)!910+ defer {911+ b.close()912+ }913+ put(a.root, 'secrets.env', 'api_key=1\n')!914+ put(a.root, 'shared.env', 'same bytes on B\n')!915+ save := commit(a, 'add secrets')!916+ offer_now(a, mut backend)!917+ join(b, mut backend)!918+ put(b.root, 'copy.txt', 'same bytes on B\n')!919+ b_save := commit(b, 'B keeps a copy')!920+ offer_now(b, mut backend)!921+ shared_hash := storage.hash_hex('same bytes on B\n'.bytes())922+ secret_hash := storage.hash_hex('api_key=1\n'.bytes())923+924+ strip_local(a, 'secrets.env', '')!925+ // Write 1 uploads the event and write 2 rewrites the first record; B's save,926+ // made from A's tree, lists the file too, and its rewrite (write 3) fails.927+ mut flaky := FaultyBackend{928+ inner: remote.new_folder(remote_dir)!929+ fail_write: 3930+ }931+ if _ := push_strip(a, mut flaky, 'secrets.env') {932+ assert false, 'the injected failure should have surfaced'933+ }934+ assert strip_pending(a, 'secrets.env')935+ assert strip_local(a, 'secrets.env', '')!.files_removed == 0936+ report := push_strip(a, mut backend, 'secrets.env')!937+ assert report.records == 1938+ assert !strip_pending(a, 'secrets.env')939+ assert !backend.get(object_key('proj', secret_hash))!.found940+ record := decode_save(backend.get(save_key('proj', save.id))!.body)!941+ assert 'secrets.env' !in record.files.map(it.path)942+ b_record := decode_save(backend.get(save_key('proj', b_save.id))!.body)!943+ assert 'secrets.env' !in b_record.files.map(it.path)944+945+ // B's save still names the shared bytes, so stripping A's copy keeps them.946+ strip_local(a, 'shared.env', '')!947+ push_strip(a, mut backend, 'shared.env')!948+ assert backend.get(object_key('proj', shared_hash))!.found949+}950+951+// A machine that has not heard of a strip must not offer: its records still952+// list the stripped path, and nothing would rewrite them on the remote again.953+fn test_offer_waits_for_an_unapplied_strip() {954+ base := scratch('strip_first')!955+ defer {956+ os.rmdir_all(base) or {}957+ }958+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!959+ mut a := open_repo(machine('strip_first_a')!)!960+ defer {961+ a.close()962+ }963+ mut b := open_repo(machine('strip_first_b')!)!964+ defer {965+ b.close()966+ }967+ put(a.root, 'secrets.env', 'api_key=1\n')!968+ commit(a, 'add secrets')!969+ offer_now(a, mut backend)!970+ join(b, mut backend)!971+ strip_local(a, 'secrets.env', '')!972+ push_strip(a, mut backend, 'secrets.env')!973+ put(b.root, 'more.txt', 'B keeps working\n')!974+ later := commit(b, 'B saves with the secret still in its tree')!975+ if _ := offer_now(b, mut backend) {976+ assert false, 'B offered records that still list the stripped path'977+ }978+ steal_now(b, mut backend)!979+ offer_now(b, mut backend)!980+ record := decode_save(backend.get(save_key('proj', later.id))!.body)!981+ assert 'secrets.env' !in record.files.map(it.path)982+}983+984+// A strip whose object deletions failed, followed by a second strip of the same985+// path, must still delete the first strip's objects when it finishes.986+fn test_a_second_strip_keeps_the_first_ones_unfinished_objects() {987+ base := scratch('strip_twice')!988+ defer {989+ os.rmdir_all(base) or {}990+ }991+ remote_dir := os.join_path(base, 'remote')992+ mut backend := remote.new_folder(remote_dir)!993+ mut a := open_repo(machine('strip_twice_a')!)!994+ defer {995+ a.close()996+ }997+ put(a.root, 'secrets.env', 'first key\n')!998+ commit(a, 'first key')!999+ offer_now(a, mut backend)!1000+ strip_local(a, 'secrets.env', '')!1001+ mut flaky := FaultyBackend{1002+ inner: remote.new_folder(remote_dir)!1003+ fail_delete: true1004+ }1005+ if _ := push_strip(a, mut flaky, 'secrets.env') {1006+ assert false, 'the injected delete failure should have surfaced'1007+ }1008+ put(a.root, 'secrets.env', 'second key\n')!1009+ commit(a, 'second key')!1010+ offer_now(a, mut backend)!1011+ strip_local(a, 'secrets.env', '')!1012+ push_strip(a, mut backend, 'secrets.env')!1013+ for key in ['first key\n', 'second key\n'] {1014+ assert !backend.get(object_key('proj', storage.hash_hex(key.bytes())))!.found1015+ }1016+}1017+1018+// Unfinished strips of one path in different scopes finish together.1019+fn test_unfinished_strips_of_one_path_in_different_saves_finish_together() {1020+ base := scratch('strip_scopes')!1021+ defer {1022+ os.rmdir_all(base) or {}1023+ }1024+ remote_dir := os.join_path(base, 'remote')1025+ mut backend := remote.new_folder(remote_dir)!1026+ mut a := open_repo(machine('strip_scopes_a')!)!1027+ defer {1028+ a.close()1029+ }1030+ put(a.root, 'secrets.env', 'first key\n')!1031+ first := commit(a, 'first key')!1032+ put(a.root, 'secrets.env', 'second key\n')!1033+ second := commit(a, 'second key')!1034+ offer_now(a, mut backend)!1035+ strip_local(a, 'secrets.env', first.id)!1036+ // Write 1 uploads the event; write 2, the record rewrite, fails.1037+ mut flaky := FaultyBackend{1038+ inner: remote.new_folder(remote_dir)!1039+ fail_write: 21040+ }1041+ if _ := push_strip(a, mut flaky, 'secrets.env') {1042+ assert false, 'the injected failure should have surfaced'1043+ }1044+ strip_local(a, 'secrets.env', second.id)!1045+ push_strip(a, mut backend, 'secrets.env')!1046+ assert !strip_pending(a, 'secrets.env')1047+ for save in [first, second] {1048+ record := decode_save(backend.get(save_key('proj', save.id))!.body)!1049+ assert 'secrets.env' !in record.files.map(it.path)1050+ }1051+ for key in ['first key\n', 'second key\n'] {1052+ assert !backend.get(object_key('proj', storage.hash_hex(key.bytes())))!.found1053+ }1054+}1055+1056+// Stripping a path, re-adding it, and stripping it again is two strips; a1057+// machine that heard about the first must still apply the second.1058+fn test_a_second_strip_of_the_same_path_reaches_other_machines() {1059+ base := scratch('strip_again')!1060+ defer {1061+ os.rmdir_all(base) or {}1062+ }1063+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!1064+ mut a := open_repo(machine('strip_again_a')!)!1065+ defer {1066+ a.close()1067+ }1068+ mut b := open_repo(machine('strip_again_b')!)!1069+ defer {1070+ b.close()1071+ }1072+ put(a.root, 'secrets.env', 'first key\n')!1073+ commit(a, 'first key')!1074+ offer_now(a, mut backend)!1075+ join(b, mut backend)!1076+ strip_local(a, 'secrets.env', '')!1077+ push_strip(a, mut backend, 'secrets.env')!1078+ steal_now(b, mut backend)!1079+ // Strip leaves the file on disk; deleting it is the user's part.1080+ os.rm(os.join_path(b.root, 'secrets.env'))!1081+ put(a.root, 'secrets.env', 'second key\n')!1082+ second := commit(a, 'second key')!1083+ offer_now(a, mut backend)!1084+ steal_now(b, mut backend)!1085+ assert 'secrets.env' in history.files_of(b, second.id)!.map(it.path)1086+ strip_local(a, 'secrets.env', '')!1087+ push_strip(a, mut backend, 'secrets.env')!1088+ steal_now(b, mut backend)!1089+ assert 'secrets.env' !in history.files_of(b, second.id)!.map(it.path)1090+}1091+1092+// A clash on a path the other machine stripped is no clash: the strip removes1093+// the path from every save, so there is nothing to choose.1094+fn test_a_stripped_path_is_not_asked_about() {1095+ base := scratch('clash_strip')!1096+ defer {1097+ os.rmdir_all(base) or {}1098+ }1099+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!1100+ mut a := open_repo(machine('clash_strip_a')!)!1101+ defer {1102+ a.close()1103+ }1104+ mut b := open_repo(machine('clash_strip_b')!)!1105+ defer {1106+ b.close()1107+ }1108+ put(a.root, 'secrets.env', 'key one\n')!1109+ commit(a, 'base')!1110+ offer_now(a, mut backend)!1111+ join(b, mut backend)!1112+ put(a.root, 'secrets.env', 'key two\n')!1113+ commit(a, 'A rotates')!1114+ offer_now(a, mut backend)!1115+ strip_local(a, 'secrets.env', '')!1116+ push_strip(a, mut backend, 'secrets.env')!1117+ put(b.root, 'secrets.env', 'key three\n')!1118+ mine := commit(b, 'B rotates')!1119+ report := steal(b, mut backend, StealOptions{})!1120+ assert report.clashes.len == 01121+ for save in history.all_saves(b)! {1122+ assert 'secrets.env' !in history.files_of(b, save.id)!.map(it.path)1123+ }1124+ assert history.by_id(b, mine.id) != none1125+}1126+1127+// A strip scoped to the shared base save changes what the replay sees; the1128+// clashes must be found on the same trees, or the replay meets a conflict.1129+fn test_a_strip_of_the_shared_base_is_seen_before_asking() {1130+ base := scratch('clash_base_strip')!1131+ defer {1132+ os.rmdir_all(base) or {}1133+ }1134+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!1135+ mut a := open_repo(machine('base_strip_a')!)!1136+ defer {1137+ a.close()1138+ }1139+ mut b := open_repo(machine('base_strip_b')!)!1140+ defer {1141+ b.close()1142+ }1143+ put(a.root, 'notes.txt', 'shared\n')!1144+ common := commit(a, 'base')!1145+ offer_now(a, mut backend)!1146+ join(b, mut backend)!1147+ put(a.root, 'other.txt', 'A moves on\n')!1148+ commit(a, 'A moves on')!1149+ offer_now(a, mut backend)!1150+ strip_local(a, 'notes.txt', common.id)!1151+ push_strip(a, mut backend, 'notes.txt')!1152+ put(b.root, 'notes.txt', 'B edits\n')!1153+ commit(b, 'B edits notes')!1154+ if _ := steal(b, mut backend, StealOptions{}) {1155+ assert false, 'with the base stripped, notes.txt differs on both sides'1156+ } else {1157+ assert err.msg().contains('changed on both machines')1158+ }1159+ report := steal(b, mut backend, StealOptions{1160+ mine: true1161+ })!1162+ assert report.clashes == ['notes.txt']1163+ assert os.read_file(os.join_path(b.root, 'notes.txt'))! == 'B edits\n'1164+}1165+1166+// A folder with files but no saves has nothing to combine with; a steal that1167+// fast-forwarded into it would overwrite whatever shares a name.1168+fn test_steal_into_a_folder_with_unsaved_files_refuses() {1169+ base := scratch('steal_unsaved')!1170+ defer {1171+ os.rmdir_all(base) or {}1172+ }1173+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!1174+ mut a := open_repo(machine('unsaved_a')!)!1175+ defer {1176+ a.close()1177+ }1178+ put(a.root, 'notes.txt', 'from A\n')!1179+ commit(a, 'A writes')!1180+ offer_now(a, mut backend)!1181+ b_root := machine('unsaved_b')!1182+ mut b := storage.create(b_root)!1183+ defer {1184+ b.close()1185+ }1186+ put(b_root, 'notes.txt', 'never saved\n')!1187+ // Only a repo that already knows its remote name gets this far.1188+ b.set_state('remote_repo', 'proj')!1189+ if _ := steal(b, mut backend, StealOptions{}) {1190+ assert false, 'steal overwrote a folder with unsaved files'1191+ } else {1192+ assert err.msg() == no_saves_here1193+ }1194+ assert os.read_file(os.join_path(b_root, 'notes.txt'))! == 'never saved\n'1195+ // A fresh `koh init` folder, holding only the default ignore list, is fine.1196+ os.rm(os.join_path(b_root, 'notes.txt'))!1197+ put(b_root, snapshot.ignore_file, snapshot.default_ignore)!1198+ steal(b, mut backend, StealOptions{})!1199+ assert os.read_file(os.join_path(b_root, 'notes.txt'))! == 'from A\n'1200+}1201+1202+// The document asks steal to warn about every incoming flagged save, so the1203+// report names them.1204+fn test_steal_reports_incoming_flagged_saves() {1205+ base := scratch('flag_warn')!1206+ defer {1207+ os.rmdir_all(base) or {}1208+ }1209+ mut backend := remote.new_folder(os.join_path(base, 'remote'))!1210+ mut a := open_repo(machine('flag_warn_a')!)!1211+ defer {1212+ a.close()1213+ }1214+ mut b := open_repo(machine('flag_warn_b')!)!1215+ defer {1216+ b.close()1217+ }1218+ put(a.root, 'a.txt', 'one\n')!1219+ commit(a, 'fine')!1220+ offer_now(a, mut backend)!1221+ join(b, mut backend)!1222+ put(a.root, 'a.txt', 'two\n')!1223+ suspect := commit(a, 'suspect')!1224+ put(a.root, 'a.txt', 'three\n')!1225+ commit(a, 'also fine')!1226+ events.flag(a, suspect.id, 'broke the build')!1227+ offer_now(a, mut backend)!1228+ report := steal_now(b, mut backend)!1229+ assert report.flagged.map(it.save_id) == [suspect.id]1230+ assert report.flagged[0].reason == 'broke the build'1231+}1232+1233+// head_of unwraps the head for tests that know the repo has saves.1234+fn head_of(store &storage.Store) history.Save {1235+ return history.head(store) or { panic('this repo has no head') }1236+}src/exchange/records.v+166 −0
@@ -0,0 +1,166 @@1+//! What a save and an event look like on the remote.2+//!3+//! Records are JSON so a person can read the bucket, and the layout is the4+//! same for every backend:5+//!6+//! <repo>/objects/<hash> the object's compressed bytes, as in the table7+//! <repo>/saves/<id> the full save record, including its file list8+//! <repo>/events/<id> one flag, clear or strip event9+//! <repo>/head the ID of the newest save on the line10+//!11+//! Only `head` is ever overwritten, and only with a matching version tag.12+13+module exchange14+15+import json216+import src.events17+import src.history18+import src.snapshot19+import src.storage20+21+pub fn object_key(repo string, hash string) string {22+ return '${repo}/objects/${hash}'23+}24+25+pub fn save_key(repo string, id string) string {26+ return '${repo}/saves/${id}'27+}28+29+pub fn event_key(repo string, id string) string {30+ return '${repo}/events/${id}'31+}32+33+pub fn head_key(repo string) string {34+ return '${repo}/head'35+}36+37+pub fn events_prefix(repo string) string {38+ return '${repo}/events'39+}40+41+pub struct FileRecord {42+pub:43+ path string44+ hash string45+ executable bool46+}47+48+pub struct SaveRecord {49+pub:50+ id string51+ parent string52+ message string53+ created_at i6454+ machine string55+ is_checkpoint bool56+ size_bytes i6457+ agent string58+ session string59+ restores string60+ replayed_from string61+ files []FileRecord62+}63+64+// save_record turns a local save into what travels. The file list is part of65+// the record, which is why a save needs one round trip, not one per file.66+pub fn save_record(store &storage.Store, save history.Save) !SaveRecord {67+ files := history.files_of(store, save.id)!68+ mut records := []FileRecord{}69+ for file in files {70+ records << FileRecord{71+ path: file.path72+ hash: file.hash73+ executable: file.executable74+ }75+ }76+ return SaveRecord{77+ id: save.id78+ parent: save.parent79+ message: save.message80+ created_at: save.created_at81+ machine: save.machine82+ is_checkpoint: save.is_checkpoint83+ size_bytes: save.size_bytes84+ agent: save.agent85+ session: save.session86+ restores: save.restores87+ replayed_from: save.replayed_from88+ files: records89+ }90+}91+92+pub fn encode_save(record SaveRecord) string {93+ return json2.encode(record)94+}95+96+pub fn decode_save(body []u8) !SaveRecord {97+ return json2.decode[SaveRecord](body.bytestr())!98+}99+100+pub fn to_save(record SaveRecord) history.Save {101+ return history.Save{102+ id: record.id103+ parent: record.parent104+ message: record.message105+ created_at: record.created_at106+ machine: record.machine107+ is_checkpoint: record.is_checkpoint108+ size_bytes: record.size_bytes109+ agent: record.agent110+ session: record.session111+ restores: record.restores112+ replayed_from: record.replayed_from113+ }114+}115+116+pub fn to_files(record SaveRecord) []snapshot.FileEntry {117+ mut files := []snapshot.FileEntry{}118+ for file in record.files {119+ files << snapshot.FileEntry{120+ path: file.path121+ hash: file.hash122+ executable: file.executable123+ }124+ }125+ return files126+}127+128+pub struct EventRecord {129+pub:130+ id string131+ kind string132+ save_id string133+ path string134+ reason string135+ created_at i64136+ machine string137+}138+139+pub fn event_record(event events.Event) EventRecord {140+ return EventRecord{141+ id: event.id142+ kind: event.kind143+ save_id: event.save_id144+ path: event.path145+ reason: event.reason146+ created_at: event.created_at147+ machine: event.machine148+ }149+}150+151+pub fn encode_event(event events.Event) string {152+ return json2.encode(event_record(event))153+}154+155+pub fn decode_event(body []u8) !events.Event {156+ record := json2.decode[EventRecord](body.bytestr())!157+ return events.Event{158+ id: record.id159+ kind: record.kind160+ save_id: record.save_id161+ path: record.path162+ reason: record.reason163+ created_at: record.created_at164+ machine: record.machine165+ }166+}src/exchange/steal.v+497 −0
@@ -0,0 +1,497 @@1+//! Stealing: bringing another machine's saves in, and combining when both2+//! machines moved.3+//!4+//! The order is always the same, and it is what makes an interrupted steal5+//! harmless: read everything from the remote first, then land it in one6+//! transaction, then write the working tree. Divergence is resolved per file —7+//! a clash is never merged line by line, it is chosen.8+9+module exchange10+11+import compress.zlib12+import os13+import src.apply14+import src.events15+import src.history16+import src.remote17+import src.snapshot18+import src.storage19+20+pub enum Side {21+ mine22+ theirs23+}24+25+// Clash is one file both machines changed differently. `stat` is the short26+// diff stat the terminal prompt shows, so the choice is informed.27+pub struct Clash {28+pub mut:29+ path string30+ mine snapshot.FileEntry31+ theirs snapshot.FileEntry32+ base snapshot.FileEntry33+ stat string34+}35+36+pub struct StealOptions {37+pub mut:38+ mine bool // resolve every clash as mine39+ theirs bool // resolve every clash as theirs40+ // Asked once per clash when neither flag is set. Nil means "nobody to ask".41+ ask fn (clash Clash) Side = unsafe { nil }42+}43+44+pub struct StealReport {45+pub mut:46+ repo string47+ forward bool48+ fetched_saves int49+ fetched_objects int50+ fetched_events int51+ clashes []string52+ kept_mine int53+ replayed int54+ removed int55+ head string56+ flagged []events.Flag // incoming saves that are flagged57+ // Set when a combine took this machine's saves off the line: loading it58+ // brings back the tree from before the steal, less anything stripped.59+ previous_head string60+}61+62+pub const dirty_tree = 'save or checkpoint first — steal needs a clean tree'63+pub const no_saves_here = 'this folder has files but no saves — steal a repo into an empty folder with "koh steal <name>"'64+65+// steal brings the remote's new saves in. Inside a repo with its own work this66+// may have to combine; a repo with nothing of its own simply fast-forwards.67+pub fn steal(store &storage.Store, mut backend remote.Backend, opts StealOptions) !StealReport {68+ name := repo_name(store, mut backend)!69+ if snapshot.restore_pending(store) {70+ return error(snapshot.restore_unfinished)71+ }72+ head_before := history.head(store)73+ // Combining rewrites the working tree, so unsaved work would be lost.74+ if dirty(store, head_before)! {75+ if head_before == none {76+ return error(no_saves_here)77+ }78+ return error(dirty_tree)79+ }80+ fetched := fetch_back_to_known(store, mut backend, name)!81+ mut report := StealReport{82+ repo: name83+ fetched_saves: fetched.saves.len84+ fetched_objects: fetched.repo_objects.len85+ fetched_events: fetched.events_out.len86+ head: fetched.head87+ }88+ mut local_only := []history.Save{}89+ if head := head_before {90+ // The remote builds on `base`. If that save is here but no longer on91+ // the line, neither line contains the other; jumping to the remote92+ // would silently leave this line behind.93+ local_only = saves_after(store, head.id, fetched.base) or {94+ return error('the remote builds on save ${fetched.base}, which is not on this line — load the save you want, then offer')95+ }96+ }97+ if fetched.base == '' && local_only.len > 0 {98+ return error('the remote keeps a different history — it shares no save with this repo')99+ }100+101+ strips_before := events.strip_events(store)!.len102+ mut clashes := []Clash{}103+ mut side_of := map[string]Side{}104+ if fetched.saves.len > 0 && local_only.len > 0 {105+ clashes = combine_clashes(store, fetched)!106+ report.clashes = clashes.map(it.path)107+ // Asked before the write lock: a person thinking must not stall every108+ // agent's save on this repo.109+ side_of = choose_sides(clashes, opts)!110+ }111+112+ store.begin() or { return err }113+ // The questions ran without the lock. If a save or an edit landed114+ // meanwhile, the answers were about a line that no longer exists.115+ if moved := changed_since(store, head_before, strips_before) {116+ if moved {117+ store.rollback()118+ return error('this repo changed while steal was asking — run koh steal again')119+ }120+ } else {121+ store.rollback()122+ return err123+ }124+ final := (write_and_combine(store, fetched, local_only, clashes, side_of, mut report) or {125+ store.rollback()126+ return err127+ })128+ files := history.files_of(store, final.id) or {129+ store.rollback()130+ return err131+ }132+ snapshot.check_links(store.root, files) or {133+ store.rollback()134+ return err135+ }136+ // The files follow the commit; the mark lands with it, in one step.137+ // No head before means a fresh clone, whose whole tree is new.138+ moved := if head := head_before { final.id != head.id } else { true }139+ if moved {140+ snapshot.mark_restore(store) or {141+ store.rollback()142+ return err143+ }144+ }145+ store.commit() or { return err }146+ report.head = final.id147+ if local_only.len > 0 && report.fetched_saves > 0 {148+ report.previous_head = local_only.last().id149+ }150+ flags := events.latest_flags(store)!151+ for record in fetched.saves {152+ if flag := flags[record.id] {153+ if flag.flagged {154+ report.flagged << flag155+ }156+ }157+ }158+159+ if head := head_before {160+ if final.id == head.id {161+ // The line did not move, so neither does the working tree.162+ return report163+ }164+ mut wanted := map[string]bool{}165+ for file in files {166+ wanted[file.path] = true167+ }168+ mut remove := []string{}169+ for file in history.files_of(store, head.id)! {170+ if file.path !in wanted {171+ remove << file.path172+ }173+ }174+ result := snapshot.restore_files(store, store.root, snapshot.load(store.root), files,175+ remove)!176+ report.removed = result.removed177+ } else {178+ snapshot.restore_files(store, store.root, snapshot.load(store.root), files, [])!179+ }180+ history.refresh_log_md(store)181+ return report182+}183+184+// write_and_combine lands a fetch and, when the local line has saves the remote185+// lacks, replays them on top of the remote head. It runs inside the caller's186+// transaction and returns the new head.187+fn write_and_combine(store &storage.Store, fetched Fetched, local_only []history.Save, clashes []Clash, side_of map[string]Side, mut report StealReport) !history.Save {188+ write_fetched(store, fetched)!189+ if fetched.saves.len == 0 {190+ // The remote head is already on this line: level, or this machine is191+ // ahead and the offer that follows a steal sends the rest.192+ return history.head(store) or { return error('this repo has no head') }193+ }194+ if local_only.len == 0 {195+ // Nothing of ours is missing from the remote: the line simply moves on.196+ history.set_head(store, fetched.head)!197+ report.forward = true198+ return history.head(store) or { return error('the remote head did not land') }199+ }200+ remote_files := history.files_of(store, fetched.head)!201+ mut mine_files := []snapshot.FileEntry{}202+ if our_head := history.head(store) {203+ mine_files = history.files_of(store, our_head.id)!204+ }205+206+ // A file the remote changed is a clash, decided below, or one this machine207+ // left as it was or changed the same way, so the remote's version stands.208+ // Replaying this machine's steps on it could only conflict: an edit it209+ // later undid would no longer apply.210+ base_files := history.files_of(store, fetched.base)!211+ settled := changed_paths(base_files, remote_files)212+213+ // Take the remote's line first, so the replayed saves chain onto it.214+ history.set_head(store, fetched.head)!215+ mut current := remote_files.clone()216+ for save in local_only {217+ parent_files := if save.parent != '' {218+ history.files_of(store, save.parent)!219+ } else {220+ []snapshot.FileEntry{}221+ }222+ target_files := history.files_of(store, save.id)!223+ plan := apply.three_way(current, parent_files, target_files, [], settled)224+ if apply.conflicts(plan) {225+ return error('combine could not apply ${plan.conflict} — it is not a clash file')226+ }227+ if plan.applied.len == 0 {228+ // Everything this save touched was a file the remote changed.229+ continue230+ }231+ history.record_save(store, '', plan.files, history.SaveInput{232+ message: save.message233+ created_at: save.created_at234+ machine: save.machine235+ agent: save.agent236+ session: save.session237+ is_checkpoint: save.is_checkpoint238+ replayed_from: save.id239+ })!240+ current = plan.files.clone()241+ report.replayed++242+ }243+244+ mut mine_paths := []string{}245+ for clash in clashes {246+ if side_of[clash.path] == .mine {247+ mine_paths << clash.path248+ }249+ }250+ if mine_paths.len > 0 {251+ // One last save puts my versions of the chosen files back.252+ final_plan := apply.three_way(current, current, mine_files, mine_paths, [])253+ if final_plan.applied.len > 0 {254+ history.record_save(store, '', final_plan.files, history.SaveInput{255+ message: 'combine: kept mine for ${mine_paths.len} ${if mine_paths.len == 1 {256+ 'file'257+ } else {258+ 'files'259+ }}'260+ })!261+ report.kept_mine = mine_paths.len262+ }263+ }264+ return history.head(store) or { return error('the combined line has no head') }265+}266+267+// find_clashes lists the files both sides changed, differently. Files changed268+// identically on both sides are not clashes: there is nothing to decide.269+pub fn find_clashes(base []snapshot.FileEntry, mine []snapshot.FileEntry, theirs []snapshot.FileEntry) []Clash {270+ base_map := snapshot.index_files(base)271+ mine_map := snapshot.index_files(mine)272+ theirs_map := snapshot.index_files(theirs)273+ mut paths := mine_map.keys()274+ for path in theirs_map.keys() {275+ if path !in mine_map {276+ paths << path277+ }278+ }279+ paths.sort()280+ mut clashes := []Clash{}281+ for path in paths {282+ here := mine_map[path] or {283+ snapshot.FileEntry{284+ path: path285+ }286+ }287+ there := theirs_map[path] or {288+ snapshot.FileEntry{289+ path: path290+ }291+ }292+ was := base_map[path] or {293+ snapshot.FileEntry{294+ path: path295+ }296+ }297+ if same(here, there) {298+ continue299+ }300+ if same(here, was) || same(there, was) {301+ continue302+ }303+ clashes << Clash{304+ path: path305+ mine: here306+ theirs: there307+ base: was308+ }309+ }310+ return clashes311+}312+313+// changed_paths lists the paths whose entry differs between two trees.314+fn changed_paths(before []snapshot.FileEntry, after []snapshot.FileEntry) []string {315+ was := snapshot.index_files(before)316+ now := snapshot.index_files(after)317+ mut paths := []string{}318+ for path, entry in now {319+ if !same(entry, was[path] or { snapshot.FileEntry{} }) {320+ paths << path321+ }322+ }323+ for path, _ in was {324+ if path !in now {325+ paths << path326+ }327+ }328+ return paths329+}330+331+// combine_clashes finds the files both lines changed, with the diff stat each332+// prompt shows. It reads the remote head from the fetch, not the store, so it333+// can run before anything is written.334+fn combine_clashes(store &storage.Store, fetched Fetched) ![]Clash {335+ base_files := if fetched.base != '' {336+ history.files_of(store, fetched.base)!337+ } else {338+ []snapshot.FileEntry{}339+ }340+ head := history.head(store) or { return []Clash{} }341+ // The transaction applies the incoming strips before it replays, so the342+ // clashes are found on the trees as they will be then, or the answers343+ // would be about files that are no longer there.344+ strips := fetched.events_out.filter(it.kind == events.kind_strip)345+ // Base and mine are here already, so every strip reaches them (created 0).346+ remote_head := fetched.saves.last()347+ mut clashes := find_clashes(after_strips(base_files, fetched.base, 0, strips), after_strips(history.files_of(store,348+ head.id)!, head.id, 0, strips), after_strips(to_files(remote_head), remote_head.id,349+ remote_head.created_at, strips))350+ for mut clash in clashes {351+ mine := fetched_bytes(store, fetched, clash.mine)!352+ theirs := fetched_bytes(store, fetched, clash.theirs)!353+ delta := snapshot.diff_bytes(clash.path, 'modified', mine, theirs)!354+ clash.stat = if delta.binary { 'binary' } else { '+${delta.added} -${delta.removed}' }355+ }356+ return clashes357+}358+359+// after_strips is a save's file list once the given strip events are applied,360+// by the same rule write_fetched uses.361+fn after_strips(files []snapshot.FileEntry, save_id string, created_at i64, strips []events.Event) []snapshot.FileEntry {362+ return files.filter(fn [strips, save_id, created_at] (file snapshot.FileEntry) bool {363+ return !strips.any(reaches(it, save_id, created_at) && covers(it.path, file.path))364+ })365+}366+367+// fetched_bytes reads one side of a clash: from the fetch when it has just368+// arrived, else from the store. An absent side is empty.369+fn fetched_bytes(store &storage.Store, fetched Fetched, entry snapshot.FileEntry) ![]u8 {370+ if entry.hash == '' {371+ return []u8{}372+ }373+ if blob := fetched.repo_objects[entry.hash] {374+ return zlib.decompress(blob)!375+ }376+ return store.get_object(entry.hash)!377+}378+379+fn choose_sides(clashes []Clash, opts StealOptions) !map[string]Side {380+ mut sides := map[string]Side{}381+ if clashes.len == 0 {382+ return sides383+ }384+ for clash in clashes {385+ if opts.mine {386+ sides[clash.path] = .mine387+ continue388+ }389+ if opts.theirs {390+ sides[clash.path] = .theirs391+ continue392+ }393+ if opts.ask != unsafe { nil } {394+ sides[clash.path] = opts.ask(clash)395+ continue396+ }397+ return error('${clashes.len} files changed on both machines — run on a terminal to choose, or pass --mine or --theirs')398+ }399+ return sides400+}401+402+fn same(a snapshot.FileEntry, b snapshot.FileEntry) bool {403+ if a.hash == '' && b.hash == '' {404+ return true405+ }406+ return a.hash == b.hash && a.executable == b.executable407+}408+409+fn changed_since(store &storage.Store, before ?history.Save, strips_before int) !bool {410+ if events.strip_events(store)!.len != strips_before {411+ return true412+ }413+ now := history.head(store)414+ if now or { history.Save{} }.id != before or { history.Save{} }.id {415+ return true416+ }417+ return dirty(store, now)!418+}419+420+// dirty reports whether the working tree holds anything a steal would421+// overwrite. With no saves yet, that is every file but the untouched ignore422+// list `koh init` wrote.423+fn dirty(store &storage.Store, head ?history.Save) !bool {424+ files := snapshot.walk(store, store.root, snapshot.load(store.root))!425+ if saved := head {426+ return !snapshot.same_tree(history.files_of(store, saved.id)!, files)427+ }428+ seed := storage.hash_hex(snapshot.default_ignore.bytes())429+ return files.any(it.path != snapshot.ignore_file || it.hash != seed)430+}431+432+// fetch_repo pulls a whole repo — head, saves, objects and events — for433+// `koh steal <name>`, which lands a copy in a new folder.434+pub fn fetch_repo(mut backend remote.Backend, name string) !Fetched {435+ remote_head := backend.get(head_key(name))!436+ if !remote_head.found {437+ return error('the remote has no repo called "${name}"')438+ }439+ save_ids := backend.list('${name}/saves')!440+ object_keys := backend.list('${name}/objects')!441+ event_keys := backend.list('${name}/events')!442+ mut fetched := Fetched{443+ head: remote_head.body.bytestr().trim_space()444+ }445+ for key in object_keys {446+ blob := backend.get(key)!447+ if blob.found {448+ fetched.repo_objects[os.base(key)] = blob.body449+ }450+ }451+ for key in save_ids {452+ body := backend.get(key)!453+ if !body.found {454+ continue455+ }456+ record := decode_save(body.body)!457+ fetched.saves << record458+ }459+ fetched.saves = parent_first(fetched.saves)!460+ for key in event_keys {461+ body := backend.get(key)!462+ if body.found {463+ fetched.events_out << decode_event(body.body)!464+ }465+ }466+ return fetched467+}468+469+// parent_first orders saves so every parent lands before its children, as the470+// foreign key on `parent` demands. Time cannot do it: a replayed save keeps its471+// original, older time but sits on a newer parent.472+fn parent_first(records []SaveRecord) ![]SaveRecord {473+ mut by_id := map[string]SaveRecord{}474+ for record in records {475+ by_id[record.id] = record476+ }477+ mut placed := map[string]bool{}478+ mut ordered := []SaveRecord{}479+ for record in records {480+ mut chain := []SaveRecord{}481+ mut cursor := record.id482+ for cursor != '' && cursor !in placed {483+ if chain.len > records.len {484+ return error('the save records on the remote form a loop')485+ }486+ current := by_id[cursor] or { return error('the remote is missing save ${cursor}') }487+ chain << current488+ cursor = current.parent489+ }490+ chain.reverse_in_place()491+ for current in chain {492+ placed[current.id] = true493+ ordered << current494+ }495+ }496+ return ordered497+}src/exchange/strip.v+227 −0
@@ -0,0 +1,227 @@1+//! Strip: removing a path from history, here and on the remote.2+//!3+//! The local half is one transaction: the saved files go, the objects nobody4+//! references go, a strip event is recorded, and so is a note of which objects5+//! the remote may now drop. The remote half reads that note, so it can be6+//! re-run after an interruption even though the local rows are long gone.7+8+module exchange9+10+import json211+import os12+import src.events13+import src.remote14+import src.storage15+16+// Scope is what a strip would remove, for the warning shown before it runs.17+pub struct Scope {18+pub:19+ files int20+ saves int21+ objects int22+}23+24+pub struct LocalStrip {25+pub:26+ files_removed int27+ objects_dropped i6428+}29+30+// PushReport says what a strip managed to make visible on the remote.31+pub struct PushReport {32+pub mut:33+ records int34+ objects int35+ event_sent bool36+}37+38+// Pending is the note a strip leaves until its remote half has finished: which39+// saves it covered ('' for every save) and which objects it freed. One note per40+// path, so strips of the same path that did not finish are finished together.41+struct Pending {42+mut:43+ saves []string44+ hashes []string45+}46+47+fn pending_key(path string) string {48+ return 'strip_pending:${path}'49+}50+51+fn read_pending(store &storage.Store, path string) Pending {52+ text := store.state(pending_key(path)) or { return Pending{} }53+ return json2.decode[Pending](text) or { Pending{} }54+}55+56+pub fn strip_pending(store &storage.Store, path string) bool {57+ return store.state(pending_key(path)) != none58+}59+60+// covers is the one rule for what a strip removes: the path itself, or61+// anything beneath it when it names a folder.62+fn covers(path string, file string) bool {63+ return file == path || file.starts_with('${path.trim_right('/')}/')64+}65+66+pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {67+ mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)68+ FROM save_files WHERE ${match_clause(only_save)}')!69+ defer {70+ stmt.finalize()71+ }72+ bind_match(stmt, path, only_save)!73+ stmt.step()!74+ return Scope{75+ files: int(stmt.int(0))76+ saves: int(stmt.int(1))77+ objects: int(stmt.int(2))78+ }79+}80+81+// strip_local removes the path in one transaction and leaves the note the82+// remote half needs.83+pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {84+ store.begin()!85+ report := strip_rows(store, path, only_save) or {86+ store.rollback()87+ return err88+ }89+ store.commit()!90+ return report91+}92+93+fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {94+ mut hashes := []string{}95+ mut stmt :=96+ store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!97+ bind_match(stmt, path, only_save)!98+ for stmt.step()! {99+ hashes << stmt.text(0)100+ }101+ stmt.finalize()102+ if hashes.len == 0 {103+ // Already stripped: keep the pending note an unfinished remote needs.104+ return LocalStrip{}105+ }106+ removed := remove_path(store, path, only_save)!107+ dropped := store.drop_unreferenced_objects()!108+ events.strip(store, only_save, path)!109+ // An earlier strip of the same path may not have finished on the remote,110+ // so the note grows rather than resets.111+ mut note := read_pending(store, path)112+ if only_save !in note.saves {113+ note.saves << only_save114+ }115+ note.hashes << hashes116+ store.set_state(pending_key(path), json2.encode(note))!117+ return LocalStrip{118+ files_removed: removed119+ objects_dropped: dropped120+ }121+}122+123+// push_strip makes a strip visible on the remote: the event travels like any124+// other, every remote save record listing the path is rewritten, and objects125+// the strip freed are deleted once no remote record and no local save needs126+// them. Every step is safe to repeat, and the pending note is cleared only127+// when all of them have succeeded.128+pub fn push_strip(store &storage.Store, mut backend remote.Backend, path string) !PushReport {129+ name := repo_name(store, mut backend)!130+ mut report := PushReport{}131+ mut upload := OfferReport{132+ repo: name133+ }134+ upload_events(store, mut backend, name, mut upload)!135+ report.event_sent = upload.events > 0136+ note := read_pending(store, path)137+ mut freed := map[string]bool{}138+ for hash in note.hashes {139+ freed[hash] = true140+ }141+ every_save := '' in note.saves142+ mut needed := map[string]bool{}143+ for key in backend.list('${name}/saves')! {144+ remote_object := backend.get(key)!145+ if !remote_object.found {146+ continue147+ }148+ record := decode_save(remote_object.body)!149+ mut kept := []FileRecord{}150+ for file in record.files {151+ if (every_save || record.id in note.saves) && covers(path, file.path) {152+ freed[file.hash] = true153+ continue154+ }155+ kept << file156+ needed[file.hash] = true157+ }158+ if kept.len == record.files.len {159+ continue160+ }161+ if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {162+ return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')163+ }164+ report.records++165+ }166+ for hash, _ in freed {167+ if hash in needed || store.has_object(hash)! {168+ continue169+ }170+ backend.delete(object_key(name, hash))!171+ report.objects++172+ }173+ store.remove_state(pending_key(path))!174+ return report175+}176+177+// unapplied_strip reports whether the remote holds a strip event this repo178+// has not received yet.179+fn unapplied_strip(store &storage.Store, mut backend remote.Backend, repo string) !bool {180+ mut known := map[string]bool{}181+ for event in events.all(store)! {182+ known[event.id] = true183+ }184+ for key in backend.list(events_prefix(repo))! {185+ if os.base(key) in known {186+ continue187+ }188+ body := backend.get(key)!189+ if body.found && decode_event(body.body)!.kind == events.kind_strip {190+ return true191+ }192+ }193+ return false194+}195+196+// rewrite returns the save record with a new file list.197+fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {198+ return SaveRecord{199+ ...record200+ files: files201+ }202+}203+204+fn remove_path(store &storage.Store, path string, only_save string) !int {205+ mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!206+ defer {207+ stmt.finalize()208+ }209+ bind_match(stmt, path, only_save)!210+ stmt.step()!211+ return store.db.changes()212+}213+214+// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`215+// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.216+fn match_clause(only_save string) string {217+ scope := if only_save == '' { '' } else { ' AND save_id = ?3' }218+ return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'219+}220+221+fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {222+ stmt.bind_text(1, path)!223+ stmt.bind_text(2, '${path.trim_right('/')}/')!224+ if only_save != '' {225+ stmt.bind_text(3, only_save)!226+ }227+}src/migrate/migrate.v+2 −2
@@ -10,7 +10,7 @@1010 module migrate1111 1212 import encoding.base6413-import json13+import json21414 import os1515 import src.events1616 import src.history@@ -136,7 +136,7 @@136136 continue137137 }138138 body := os.read_file(os.join_path(dir, name))!139- manifests << json.decode(Manifest, body)!139+ manifests << json2.decode[Manifest](body)!140140 }141141 return manifests142142 }src/migrate/migrate_test.v+3 −3
@@ -7,7 +7,7 @@77 88 module migrate99 10-import json10+import json21111 import os1212 import src.events1313 import src.history@@ -56,7 +56,7 @@5656 continue5757 }5858 record :=59- json.decode(Manifest, os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!59+ json2.decode[Manifest](os.read_file(os.join_path(dir, '.koh', 'snapshots', name))!)!6060 if record.message == 'first save' {6161 first = record.id6262 }@@ -102,7 +102,7 @@102102 if !name.ends_with('.json') {103103 continue104104 }105- record := json.decode(Manifest,105+ record := json2.decode[Manifest](106106 os.read_file(os.join_path(fixture.dir, '.koh', 'snapshots', name))!)!107107 mut files := []string{}108108 for file in record.files {src/remote/remote.v+1 −1
@@ -1,7 +1,7 @@11 //! The remote interface: five operations, two backends.22 //!33 //! A remote is dumb storage that can do one safe swap. Everything clever —4-//! what to upload, what diverged, how to combine — lives in `src/sync`, above4+//! what to upload, what diverged, how to combine — lives in `src/exchange`, above55 //! this interface, so the folder and S3 backends stay thin and the sync logic66 //! is written once and tested offline.77 src/sync/records.v+0 −166
@@ -1,166 +0,0 @@1-//! What a save and an event look like on the remote.2-//!3-//! Records are JSON so a person can read the bucket, and the layout is the4-//! same for every backend:5-//!6-//! <repo>/objects/<hash> the object's compressed bytes, as in the table7-//! <repo>/saves/<id> the full save record, including its file list8-//! <repo>/events/<id> one flag, clear or strip event9-//! <repo>/head the ID of the newest save on the line10-//!11-//! Only `head` is ever overwritten, and only with a matching version tag.12-13-module sync14-15-import json16-import src.events17-import src.history18-import src.snapshot19-import src.storage20-21-pub fn object_key(repo string, hash string) string {22- return '${repo}/objects/${hash}'23-}24-25-pub fn save_key(repo string, id string) string {26- return '${repo}/saves/${id}'27-}28-29-pub fn event_key(repo string, id string) string {30- return '${repo}/events/${id}'31-}32-33-pub fn head_key(repo string) string {34- return '${repo}/head'35-}36-37-pub fn events_prefix(repo string) string {38- return '${repo}/events'39-}40-41-pub struct FileRecord {42-pub:43- path string44- hash string45- executable bool46-}47-48-pub struct SaveRecord {49-pub:50- id string51- parent string52- message string53- created_at i6454- machine string55- is_checkpoint bool56- size_bytes i6457- agent string58- session string59- restores string60- replayed_from string61- files []FileRecord62-}63-64-// save_record turns a local save into what travels. The file list is part of65-// the record, which is why a save needs one round trip, not one per file.66-pub fn save_record(store &storage.Store, save history.Save) !SaveRecord {67- files := history.files_of(store, save.id)!68- mut records := []FileRecord{}69- for file in files {70- records << FileRecord{71- path: file.path72- hash: file.hash73- executable: file.executable74- }75- }76- return SaveRecord{77- id: save.id78- parent: save.parent79- message: save.message80- created_at: save.created_at81- machine: save.machine82- is_checkpoint: save.is_checkpoint83- size_bytes: save.size_bytes84- agent: save.agent85- session: save.session86- restores: save.restores87- replayed_from: save.replayed_from88- files: records89- }90-}91-92-pub fn encode_save(record SaveRecord) string {93- return json.encode(record)94-}95-96-pub fn decode_save(body []u8) !SaveRecord {97- return json.decode(SaveRecord, body.bytestr())!98-}99-100-pub fn to_save(record SaveRecord) history.Save {101- return history.Save{102- id: record.id103- parent: record.parent104- message: record.message105- created_at: record.created_at106- machine: record.machine107- is_checkpoint: record.is_checkpoint108- size_bytes: record.size_bytes109- agent: record.agent110- session: record.session111- restores: record.restores112- replayed_from: record.replayed_from113- }114-}115-116-pub fn to_files(record SaveRecord) []snapshot.FileEntry {117- mut files := []snapshot.FileEntry{}118- for file in record.files {119- files << snapshot.FileEntry{120- path: file.path121- hash: file.hash122- executable: file.executable123- }124- }125- return files126-}127-128-pub struct EventRecord {129-pub:130- id string131- kind string132- save_id string133- path string134- reason string135- created_at i64136- machine string137-}138-139-pub fn event_record(event events.Event) EventRecord {140- return EventRecord{141- id: event.id142- kind: event.kind143- save_id: event.save_id144- path: event.path145- reason: event.reason146- created_at: event.created_at147- machine: event.machine148- }149-}150-151-pub fn encode_event(event events.Event) string {152- return json.encode(event_record(event))153-}154-155-pub fn decode_event(body []u8) !events.Event {156- record := json.decode(EventRecord, body.bytestr())!157- return events.Event{158- id: record.id159- kind: record.kind160- save_id: record.save_id161- path: record.path162- reason: record.reason163- created_at: record.created_at164- machine: record.machine165- }166-}src/sync/steal.v+0 −497
@@ -1,497 +0,0 @@1-//! Stealing: bringing another machine's saves in, and combining when both2-//! machines moved.3-//!4-//! The order is always the same, and it is what makes an interrupted steal5-//! harmless: read everything from the remote first, then land it in one6-//! transaction, then write the working tree. Divergence is resolved per file —7-//! a clash is never merged line by line, it is chosen.8-9-module sync10-11-import compress.zlib12-import os13-import src.apply14-import src.events15-import src.history16-import src.remote17-import src.snapshot18-import src.storage19-20-pub enum Side {21- mine22- theirs23-}24-25-// Clash is one file both machines changed differently. `stat` is the short26-// diff stat the terminal prompt shows, so the choice is informed.27-pub struct Clash {28-pub mut:29- path string30- mine snapshot.FileEntry31- theirs snapshot.FileEntry32- base snapshot.FileEntry33- stat string34-}35-36-pub struct StealOptions {37-pub mut:38- mine bool // resolve every clash as mine39- theirs bool // resolve every clash as theirs40- // Asked once per clash when neither flag is set. Nil means "nobody to ask".41- ask fn (clash Clash) Side = unsafe { nil }42-}43-44-pub struct StealReport {45-pub mut:46- repo string47- forward bool48- fetched_saves int49- fetched_objects int50- fetched_events int51- clashes []string52- kept_mine int53- replayed int54- removed int55- head string56- flagged []events.Flag // incoming saves that are flagged57- // Set when a combine took this machine's saves off the line: loading it58- // brings back the tree from before the steal, less anything stripped.59- previous_head string60-}61-62-pub const dirty_tree = 'save or checkpoint first — steal needs a clean tree'63-pub const no_saves_here = 'this folder has files but no saves — steal a repo into an empty folder with "koh steal <name>"'64-65-// steal brings the remote's new saves in. Inside a repo with its own work this66-// may have to combine; a repo with nothing of its own simply fast-forwards.67-pub fn steal(store &storage.Store, mut backend remote.Backend, opts StealOptions) !StealReport {68- name := repo_name(store, mut backend)!69- if snapshot.restore_pending(store) {70- return error(snapshot.restore_unfinished)71- }72- head_before := history.head(store)73- // Combining rewrites the working tree, so unsaved work would be lost.74- if dirty(store, head_before)! {75- if head_before == none {76- return error(no_saves_here)77- }78- return error(dirty_tree)79- }80- fetched := fetch_back_to_known(store, mut backend, name)!81- mut report := StealReport{82- repo: name83- fetched_saves: fetched.saves.len84- fetched_objects: fetched.repo_objects.len85- fetched_events: fetched.events_out.len86- head: fetched.head87- }88- mut local_only := []history.Save{}89- if head := head_before {90- // The remote builds on `base`. If that save is here but no longer on91- // the line, neither line contains the other; jumping to the remote92- // would silently leave this line behind.93- local_only = saves_after(store, head.id, fetched.base) or {94- return error('the remote builds on save ${fetched.base}, which is not on this line — load the save you want, then offer')95- }96- }97- if fetched.base == '' && local_only.len > 0 {98- return error('the remote keeps a different history — it shares no save with this repo')99- }100-101- strips_before := events.strip_events(store)!.len102- mut clashes := []Clash{}103- mut side_of := map[string]Side{}104- if fetched.saves.len > 0 && local_only.len > 0 {105- clashes = combine_clashes(store, fetched)!106- report.clashes = clashes.map(it.path)107- // Asked before the write lock: a person thinking must not stall every108- // agent's save on this repo.109- side_of = choose_sides(clashes, opts)!110- }111-112- store.begin() or { return err }113- // The questions ran without the lock. If a save or an edit landed114- // meanwhile, the answers were about a line that no longer exists.115- if moved := changed_since(store, head_before, strips_before) {116- if moved {117- store.rollback()118- return error('this repo changed while steal was asking — run koh steal again')119- }120- } else {121- store.rollback()122- return err123- }124- final := (write_and_combine(store, fetched, local_only, clashes, side_of, mut report) or {125- store.rollback()126- return err127- })128- files := history.files_of(store, final.id) or {129- store.rollback()130- return err131- }132- snapshot.check_links(store.root, files) or {133- store.rollback()134- return err135- }136- // The files follow the commit; the mark lands with it, in one step.137- // No head before means a fresh clone, whose whole tree is new.138- moved := if head := head_before { final.id != head.id } else { true }139- if moved {140- snapshot.mark_restore(store) or {141- store.rollback()142- return err143- }144- }145- store.commit() or { return err }146- report.head = final.id147- if local_only.len > 0 && report.fetched_saves > 0 {148- report.previous_head = local_only.last().id149- }150- flags := events.latest_flags(store)!151- for record in fetched.saves {152- if flag := flags[record.id] {153- if flag.flagged {154- report.flagged << flag155- }156- }157- }158-159- if head := head_before {160- if final.id == head.id {161- // The line did not move, so neither does the working tree.162- return report163- }164- mut wanted := map[string]bool{}165- for file in files {166- wanted[file.path] = true167- }168- mut remove := []string{}169- for file in history.files_of(store, head.id)! {170- if file.path !in wanted {171- remove << file.path172- }173- }174- result := snapshot.restore_files(store, store.root, snapshot.load(store.root), files,175- remove)!176- report.removed = result.removed177- } else {178- snapshot.restore_files(store, store.root, snapshot.load(store.root), files, [])!179- }180- history.refresh_log_md(store)181- return report182-}183-184-// write_and_combine lands a fetch and, when the local line has saves the remote185-// lacks, replays them on top of the remote head. It runs inside the caller's186-// transaction and returns the new head.187-fn write_and_combine(store &storage.Store, fetched Fetched, local_only []history.Save, clashes []Clash, side_of map[string]Side, mut report StealReport) !history.Save {188- write_fetched(store, fetched)!189- if fetched.saves.len == 0 {190- // The remote head is already on this line: level, or this machine is191- // ahead and the offer that follows a steal sends the rest.192- return history.head(store) or { return error('this repo has no head') }193- }194- if local_only.len == 0 {195- // Nothing of ours is missing from the remote: the line simply moves on.196- history.set_head(store, fetched.head)!197- report.forward = true198- return history.head(store) or { return error('the remote head did not land') }199- }200- remote_files := history.files_of(store, fetched.head)!201- mut mine_files := []snapshot.FileEntry{}202- if our_head := history.head(store) {203- mine_files = history.files_of(store, our_head.id)!204- }205-206- // A file the remote changed is a clash, decided below, or one this machine207- // left as it was or changed the same way, so the remote's version stands.208- // Replaying this machine's steps on it could only conflict: an edit it209- // later undid would no longer apply.210- base_files := history.files_of(store, fetched.base)!211- settled := changed_paths(base_files, remote_files)212-213- // Take the remote's line first, so the replayed saves chain onto it.214- history.set_head(store, fetched.head)!215- mut current := remote_files.clone()216- for save in local_only {217- parent_files := if save.parent != '' {218- history.files_of(store, save.parent)!219- } else {220- []snapshot.FileEntry{}221- }222- target_files := history.files_of(store, save.id)!223- plan := apply.three_way(current, parent_files, target_files, [], settled)224- if apply.conflicts(plan) {225- return error('combine could not apply ${plan.conflict} — it is not a clash file')226- }227- if plan.applied.len == 0 {228- // Everything this save touched was a file the remote changed.229- continue230- }231- history.record_save(store, '', plan.files, history.SaveInput{232- message: save.message233- created_at: save.created_at234- machine: save.machine235- agent: save.agent236- session: save.session237- is_checkpoint: save.is_checkpoint238- replayed_from: save.id239- })!240- current = plan.files.clone()241- report.replayed++242- }243-244- mut mine_paths := []string{}245- for clash in clashes {246- if side_of[clash.path] == .mine {247- mine_paths << clash.path248- }249- }250- if mine_paths.len > 0 {251- // One last save puts my versions of the chosen files back.252- final_plan := apply.three_way(current, current, mine_files, mine_paths, [])253- if final_plan.applied.len > 0 {254- history.record_save(store, '', final_plan.files, history.SaveInput{255- message: 'combine: kept mine for ${mine_paths.len} ${if mine_paths.len == 1 {256- 'file'257- } else {258- 'files'259- }}'260- })!261- report.kept_mine = mine_paths.len262- }263- }264- return history.head(store) or { return error('the combined line has no head') }265-}266-267-// find_clashes lists the files both sides changed, differently. Files changed268-// identically on both sides are not clashes: there is nothing to decide.269-pub fn find_clashes(base []snapshot.FileEntry, mine []snapshot.FileEntry, theirs []snapshot.FileEntry) []Clash {270- base_map := snapshot.index_files(base)271- mine_map := snapshot.index_files(mine)272- theirs_map := snapshot.index_files(theirs)273- mut paths := mine_map.keys()274- for path in theirs_map.keys() {275- if path !in mine_map {276- paths << path277- }278- }279- paths.sort()280- mut clashes := []Clash{}281- for path in paths {282- here := mine_map[path] or {283- snapshot.FileEntry{284- path: path285- }286- }287- there := theirs_map[path] or {288- snapshot.FileEntry{289- path: path290- }291- }292- was := base_map[path] or {293- snapshot.FileEntry{294- path: path295- }296- }297- if same(here, there) {298- continue299- }300- if same(here, was) || same(there, was) {301- continue302- }303- clashes << Clash{304- path: path305- mine: here306- theirs: there307- base: was308- }309- }310- return clashes311-}312-313-// changed_paths lists the paths whose entry differs between two trees.314-fn changed_paths(before []snapshot.FileEntry, after []snapshot.FileEntry) []string {315- was := snapshot.index_files(before)316- now := snapshot.index_files(after)317- mut paths := []string{}318- for path, entry in now {319- if !same(entry, was[path] or { snapshot.FileEntry{} }) {320- paths << path321- }322- }323- for path, _ in was {324- if path !in now {325- paths << path326- }327- }328- return paths329-}330-331-// combine_clashes finds the files both lines changed, with the diff stat each332-// prompt shows. It reads the remote head from the fetch, not the store, so it333-// can run before anything is written.334-fn combine_clashes(store &storage.Store, fetched Fetched) ![]Clash {335- base_files := if fetched.base != '' {336- history.files_of(store, fetched.base)!337- } else {338- []snapshot.FileEntry{}339- }340- head := history.head(store) or { return []Clash{} }341- // The transaction applies the incoming strips before it replays, so the342- // clashes are found on the trees as they will be then, or the answers343- // would be about files that are no longer there.344- strips := fetched.events_out.filter(it.kind == events.kind_strip)345- // Base and mine are here already, so every strip reaches them (created 0).346- remote_head := fetched.saves.last()347- mut clashes := find_clashes(after_strips(base_files, fetched.base, 0, strips), after_strips(history.files_of(store,348- head.id)!, head.id, 0, strips), after_strips(to_files(remote_head), remote_head.id,349- remote_head.created_at, strips))350- for mut clash in clashes {351- mine := fetched_bytes(store, fetched, clash.mine)!352- theirs := fetched_bytes(store, fetched, clash.theirs)!353- delta := snapshot.diff_bytes(clash.path, 'modified', mine, theirs)!354- clash.stat = if delta.binary { 'binary' } else { '+${delta.added} -${delta.removed}' }355- }356- return clashes357-}358-359-// after_strips is a save's file list once the given strip events are applied,360-// by the same rule write_fetched uses.361-fn after_strips(files []snapshot.FileEntry, save_id string, created_at i64, strips []events.Event) []snapshot.FileEntry {362- return files.filter(fn [strips, save_id, created_at] (file snapshot.FileEntry) bool {363- return !strips.any(reaches(it, save_id, created_at) && covers(it.path, file.path))364- })365-}366-367-// fetched_bytes reads one side of a clash: from the fetch when it has just368-// arrived, else from the store. An absent side is empty.369-fn fetched_bytes(store &storage.Store, fetched Fetched, entry snapshot.FileEntry) ![]u8 {370- if entry.hash == '' {371- return []u8{}372- }373- if blob := fetched.repo_objects[entry.hash] {374- return zlib.decompress(blob)!375- }376- return store.get_object(entry.hash)!377-}378-379-fn choose_sides(clashes []Clash, opts StealOptions) !map[string]Side {380- mut sides := map[string]Side{}381- if clashes.len == 0 {382- return sides383- }384- for clash in clashes {385- if opts.mine {386- sides[clash.path] = .mine387- continue388- }389- if opts.theirs {390- sides[clash.path] = .theirs391- continue392- }393- if opts.ask != unsafe { nil } {394- sides[clash.path] = opts.ask(clash)395- continue396- }397- return error('${clashes.len} files changed on both machines — run on a terminal to choose, or pass --mine or --theirs')398- }399- return sides400-}401-402-fn same(a snapshot.FileEntry, b snapshot.FileEntry) bool {403- if a.hash == '' && b.hash == '' {404- return true405- }406- return a.hash == b.hash && a.executable == b.executable407-}408-409-fn changed_since(store &storage.Store, before ?history.Save, strips_before int) !bool {410- if events.strip_events(store)!.len != strips_before {411- return true412- }413- now := history.head(store)414- if now or { history.Save{} }.id != before or { history.Save{} }.id {415- return true416- }417- return dirty(store, now)!418-}419-420-// dirty reports whether the working tree holds anything a steal would421-// overwrite. With no saves yet, that is every file but the untouched ignore422-// list `koh init` wrote.423-fn dirty(store &storage.Store, head ?history.Save) !bool {424- files := snapshot.walk(store, store.root, snapshot.load(store.root))!425- if saved := head {426- return !snapshot.same_tree(history.files_of(store, saved.id)!, files)427- }428- seed := storage.hash_hex(snapshot.default_ignore.bytes())429- return files.any(it.path != snapshot.ignore_file || it.hash != seed)430-}431-432-// fetch_repo pulls a whole repo — head, saves, objects and events — for433-// `koh steal <name>`, which lands a copy in a new folder.434-pub fn fetch_repo(mut backend remote.Backend, name string) !Fetched {435- remote_head := backend.get(head_key(name))!436- if !remote_head.found {437- return error('the remote has no repo called "${name}"')438- }439- save_ids := backend.list('${name}/saves')!440- object_keys := backend.list('${name}/objects')!441- event_keys := backend.list('${name}/events')!442- mut fetched := Fetched{443- head: remote_head.body.bytestr().trim_space()444- }445- for key in object_keys {446- blob := backend.get(key)!447- if blob.found {448- fetched.repo_objects[os.base(key)] = blob.body449- }450- }451- for key in save_ids {452- body := backend.get(key)!453- if !body.found {454- continue455- }456- record := decode_save(body.body)!457- fetched.saves << record458- }459- fetched.saves = parent_first(fetched.saves)!460- for key in event_keys {461- body := backend.get(key)!462- if body.found {463- fetched.events_out << decode_event(body.body)!464- }465- }466- return fetched467-}468-469-// parent_first orders saves so every parent lands before its children, as the470-// foreign key on `parent` demands. Time cannot do it: a replayed save keeps its471-// original, older time but sits on a newer parent.472-fn parent_first(records []SaveRecord) ![]SaveRecord {473- mut by_id := map[string]SaveRecord{}474- for record in records {475- by_id[record.id] = record476- }477- mut placed := map[string]bool{}478- mut ordered := []SaveRecord{}479- for record in records {480- mut chain := []SaveRecord{}481- mut cursor := record.id482- for cursor != '' && cursor !in placed {483- if chain.len > records.len {484- return error('the save records on the remote form a loop')485- }486- current := by_id[cursor] or { return error('the remote is missing save ${cursor}') }487- chain << current488- cursor = current.parent489- }490- chain.reverse_in_place()491- for current in chain {492- placed[current.id] = true493- ordered << current494- }495- }496- return ordered497-}src/sync/strip.v+0 −227
@@ -1,227 +0,0 @@1-//! Strip: removing a path from history, here and on the remote.2-//!3-//! The local half is one transaction: the saved files go, the objects nobody4-//! references go, a strip event is recorded, and so is a note of which objects5-//! the remote may now drop. The remote half reads that note, so it can be6-//! re-run after an interruption even though the local rows are long gone.7-8-module sync9-10-import json11-import os12-import src.events13-import src.remote14-import src.storage15-16-// Scope is what a strip would remove, for the warning shown before it runs.17-pub struct Scope {18-pub:19- files int20- saves int21- objects int22-}23-24-pub struct LocalStrip {25-pub:26- files_removed int27- objects_dropped i6428-}29-30-// PushReport says what a strip managed to make visible on the remote.31-pub struct PushReport {32-pub mut:33- records int34- objects int35- event_sent bool36-}37-38-// Pending is the note a strip leaves until its remote half has finished: which39-// saves it covered ('' for every save) and which objects it freed. One note per40-// path, so strips of the same path that did not finish are finished together.41-struct Pending {42-mut:43- saves []string44- hashes []string45-}46-47-fn pending_key(path string) string {48- return 'strip_pending:${path}'49-}50-51-fn read_pending(store &storage.Store, path string) Pending {52- text := store.state(pending_key(path)) or { return Pending{} }53- return json.decode(Pending, text) or { Pending{} }54-}55-56-pub fn strip_pending(store &storage.Store, path string) bool {57- return store.state(pending_key(path)) != none58-}59-60-// covers is the one rule for what a strip removes: the path itself, or61-// anything beneath it when it names a folder.62-fn covers(path string, file string) bool {63- return file == path || file.starts_with('${path.trim_right('/')}/')64-}65-66-pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {67- mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)68- FROM save_files WHERE ${match_clause(only_save)}')!69- defer {70- stmt.finalize()71- }72- bind_match(stmt, path, only_save)!73- stmt.step()!74- return Scope{75- files: int(stmt.int(0))76- saves: int(stmt.int(1))77- objects: int(stmt.int(2))78- }79-}80-81-// strip_local removes the path in one transaction and leaves the note the82-// remote half needs.83-pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {84- store.begin()!85- report := strip_rows(store, path, only_save) or {86- store.rollback()87- return err88- }89- store.commit()!90- return report91-}92-93-fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {94- mut hashes := []string{}95- mut stmt :=96- store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!97- bind_match(stmt, path, only_save)!98- for stmt.step()! {99- hashes << stmt.text(0)100- }101- stmt.finalize()102- if hashes.len == 0 {103- // Already stripped: keep the pending note an unfinished remote needs.104- return LocalStrip{}105- }106- removed := remove_path(store, path, only_save)!107- dropped := store.drop_unreferenced_objects()!108- events.strip(store, only_save, path)!109- // An earlier strip of the same path may not have finished on the remote,110- // so the note grows rather than resets.111- mut note := read_pending(store, path)112- if only_save !in note.saves {113- note.saves << only_save114- }115- note.hashes << hashes116- store.set_state(pending_key(path), json.encode(note))!117- return LocalStrip{118- files_removed: removed119- objects_dropped: dropped120- }121-}122-123-// push_strip makes a strip visible on the remote: the event travels like any124-// other, every remote save record listing the path is rewritten, and objects125-// the strip freed are deleted once no remote record and no local save needs126-// them. Every step is safe to repeat, and the pending note is cleared only127-// when all of them have succeeded.128-pub fn push_strip(store &storage.Store, mut backend remote.Backend, path string) !PushReport {129- name := repo_name(store, mut backend)!130- mut report := PushReport{}131- mut upload := OfferReport{132- repo: name133- }134- upload_events(store, mut backend, name, mut upload)!135- report.event_sent = upload.events > 0136- note := read_pending(store, path)137- mut freed := map[string]bool{}138- for hash in note.hashes {139- freed[hash] = true140- }141- every_save := '' in note.saves142- mut needed := map[string]bool{}143- for key in backend.list('${name}/saves')! {144- remote_object := backend.get(key)!145- if !remote_object.found {146- continue147- }148- record := decode_save(remote_object.body)!149- mut kept := []FileRecord{}150- for file in record.files {151- if (every_save || record.id in note.saves) && covers(path, file.path) {152- freed[file.hash] = true153- continue154- }155- kept << file156- needed[file.hash] = true157- }158- if kept.len == record.files.len {159- continue160- }161- if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {162- return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')163- }164- report.records++165- }166- for hash, _ in freed {167- if hash in needed || store.has_object(hash)! {168- continue169- }170- backend.delete(object_key(name, hash))!171- report.objects++172- }173- store.remove_state(pending_key(path))!174- return report175-}176-177-// unapplied_strip reports whether the remote holds a strip event this repo178-// has not received yet.179-fn unapplied_strip(store &storage.Store, mut backend remote.Backend, repo string) !bool {180- mut known := map[string]bool{}181- for event in events.all(store)! {182- known[event.id] = true183- }184- for key in backend.list(events_prefix(repo))! {185- if os.base(key) in known {186- continue187- }188- body := backend.get(key)!189- if body.found && decode_event(body.body)!.kind == events.kind_strip {190- return true191- }192- }193- return false194-}195-196-// rewrite returns the save record with a new file list.197-fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {198- return SaveRecord{199- ...record200- files: files201- }202-}203-204-fn remove_path(store &storage.Store, path string, only_save string) !int {205- mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!206- defer {207- stmt.finalize()208- }209- bind_match(stmt, path, only_save)!210- stmt.step()!211- return store.db.changes()212-}213-214-// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`215-// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.216-fn match_clause(only_save string) string {217- scope := if only_save == '' { '' } else { ' AND save_id = ?3' }218- return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'219-}220-221-fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {222- stmt.bind_text(1, path)!223- stmt.bind_text(2, '${path.trim_right('/')}/')!224- if only_save != '' {225- stmt.bind_text(3, only_save)!226- }227-}src/sync/sync.v+0 −268
@@ -1,268 +0,0 @@1-//! Sync: offer and steal.2-//!3-//! All the cleverness sits here, above the five-operation remote interface, so4-//! the backends stay thin and this logic is written once and tested offline.5-//!6-//! The remote is a line of saves, the same shape as the local one. `offer`7-//! pushes local saves the remote lacks; `steal` pulls remote saves and, when8-//! both sides moved, replays the local work on top of the remote's. Divergence9-//! is resolved per file, never per line.10-11-module sync12-13-import os14-import src.events15-import src.history16-import src.remote17-import src.snapshot18-import src.storage19-20-// Where the repo's name on the remote is remembered.21-const repo_state_key = 'remote_repo'22-23-// The remote head as of this machine's last successful offer, as the document24-// asks; nothing reads it back yet.25-const remote_head_key = 'remote_head'26-27-pub const no_saves = 'this repo has no saves yet — nothing to offer'28-29-pub struct OfferReport {30-pub mut:31- repo string32- saves int33- objects int34- events int35- head string36-}37-38-// remember_repo binds a repo to its name on the remote. A clone does it at39-// once: its folder may be renamed before it ever offers.40-pub fn remember_repo(store &storage.Store, name string) ! {41- store.set_state(repo_state_key, name)!42-}43-44-// repo_name is the folder's name at first offer, remembered in the repo so45-// renaming the folder later changes nothing.46-pub fn repo_name(store &storage.Store, mut backend remote.Backend) !string {47- if remembered := store.state(repo_state_key) {48- return remembered49- }50- name := os.base(store.root)51- existing := backend.get(head_key(name))!52- if existing.found {53- theirs := existing.body.bytestr().trim_space()54- if history.by_id(store, theirs) == none {55- return error('the remote already has "${name}" with a history this repo does not share — rename the folder, or use another remote')56- }57- }58- store.set_state(repo_state_key, name)!59- return name60-}61-62-// offer sends what the remote lacks and then swaps `head`. Nothing is ever63-// queued: the next offer simply sends everything the remote does not have.64-pub fn offer(store &storage.Store, mut backend remote.Backend) !OfferReport {65- name := repo_name(store, mut backend)!66- local := history.head(store) or { return error(no_saves) }67- mut report := OfferReport{68- repo: name69- head: local.id70- }71- remote_head := backend.get(head_key(name))!72- // Records offered by a machine that has not applied a strip would put the73- // stripped path back on the remote, where nothing would rewrite it again.74- // ponytail: an offer racing a strip on another machine inside the same few75- // seconds can still slip through; closing that needs a remote lock.76- if unapplied_strip(store, mut backend, name)! {77- return error('the remote has a strip this repo has not applied — run koh steal')78- }79- if remote_head.found {80- theirs := remote_head.body.bytestr().trim_space()81- newer := saves_after(store, local.id, theirs) or {82- return error('remote has new saves — run koh steal')83- }84- for save in newer {85- upload_save(store, mut backend, name, save, mut report)!86- }87- } else {88- oldest_first := history.line(store)!89- for index := oldest_first.len - 1; index >= 0; index-- {90- upload_save(store, mut backend, name, oldest_first[index], mut report)!91- }92- }93- upload_events(store, mut backend, name, mut report)!94- if !backend.put_if_match(head_key(name), local.id.bytes(), remote_head.tag)! {95- return error('remote has new saves — run koh steal')96- }97- store.set_state(remote_head_key, local.id)!98- return report99-}100-101-// saves_after lists the local saves newer than `theirs`, oldest first. It fails102-// when `theirs` is not on this line, which is the honest answer: we cannot103-// offer on top of a save we do not have.104-fn saves_after(store &storage.Store, ours string, theirs string) ![]history.Save {105- mut newer := []history.Save{}106- mut cursor := ours107- for cursor != '' && cursor != theirs {108- save := history.by_id(store, cursor) or { return error('save ${cursor} is missing') }109- newer << save110- cursor = save.parent111- }112- if cursor != theirs {113- return error('${theirs} is not on this line')114- }115- newer.reverse_in_place()116- return newer117-}118-119-fn upload_save(store &storage.Store, mut backend remote.Backend, repo string, save history.Save, mut report OfferReport) ! {120- files := history.files_of(store, save.id)!121- // Saves go up oldest first and head moves only once all of them landed, so122- // the parent and its objects are on the remote already: only content new123- // in this save travels, not a round trip per file in the repo.124- mut sent := map[string]bool{}125- if save.parent != '' {126- for file in history.files_of(store, save.parent)! {127- sent[file.hash] = true128- }129- }130- for file in files {131- if file.hash in sent {132- continue133- }134- sent[file.hash] = true135- if backend.put_if_absent(object_key(repo, file.hash), store.object_blob(file.hash)!)! {136- report.objects++137- }138- }139- record := save_record(store, save)!140- if backend.put_if_absent(save_key(repo, save.id), encode_save(record).bytes())! {141- report.saves++142- }143-}144-145-// upload_events sends every event the remote does not have. Events are tiny, so146-// listing them costs less than tracking which ones were sent.147-fn upload_events(store &storage.Store, mut backend remote.Backend, repo string, mut report OfferReport) ! {148- already := backend.list(events_prefix(repo))!149- mut already_sent := map[string]bool{}150- for key in already {151- already_sent[os.base(key)] = true152- }153- for event in events.all(store)! {154- if event.id in already_sent {155- continue156- }157- if backend.put_if_absent(event_key(repo, event.id), encode_event(event).bytes())! {158- report.events++159- }160- }161-}162-163-// fetched holds everything a steal read from the remote, before any of it is164-// written locally. Network work happens first; the database gets one165-// transaction, so a failure leaves nothing half-applied.166-pub struct Fetched {167-pub mut:168- repo_objects map[string][]u8169- saves []SaveRecord // oldest first170- events_out []events.Event171- head string172- base string // the newest save both sides already shared, '' when none173-}174-175-pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched {176- remote_head := backend.get(head_key(repo))!177- if !remote_head.found {178- return error('the remote has no "${repo}" yet — run koh offer here first')179- }180- mut fetched := Fetched{181- head: remote_head.body.bytestr().trim_space()182- }183- mut cursor := fetched.head184- for cursor != '' && history.by_id(store, cursor) == none {185- record_object := backend.get(save_key(repo, cursor))!186- if !record_object.found {187- return error('the remote is missing save ${cursor} — run koh offer there')188- }189- record := decode_save(record_object.body)!190- for file in record.files {191- if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects {192- blob := backend.get(object_key(repo, file.hash))!193- if !blob.found {194- return error('the remote is missing object ${file.hash} that save ${record.id} needs')195- }196- fetched.repo_objects[file.hash] = blob.body197- }198- }199- fetched.saves << record200- cursor = record.parent201- }202- fetched.saves.reverse_in_place()203- fetched.base = cursor204- fetched.events_out = fetch_events(store, mut backend, repo)!205- return fetched206-}207-208-fn fetch_events(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {209- have := events.all(store)!210- mut known := map[string]bool{}211- for event in have {212- known[event.id] = true213- }214- mut incoming := []events.Event{}215- for key in backend.list(events_prefix(repo))! {216- id := os.base(key)217- if id in known {218- continue219- }220- body := backend.get(event_key(repo, id))!221- if !body.found {222- continue223- }224- incoming << decode_event(body.body)!225- }226- return incoming227-}228-229-// write_fetched lands a fetch in one transaction: objects first (verified),230-// then the events, then the saves they belong to. A new strip reaches every231-// save already here, but an arriving save only when `reaches` says so: one232-// made after the strip is new work, not what the strip removed.233-pub fn write_fetched(store &storage.Store, fetched Fetched) ! {234- for hash, blob in fetched.repo_objects {235- store.put_object_blob(hash, blob)!236- }237- mut strips := []events.Event{}238- for event in fetched.events_out {239- if !events.receive(store, event)! {240- continue241- }242- if event.kind == events.kind_strip {243- // Somebody stripped a path: remove it here too, so the two244- // histories agree from now on. Objects go once the saves are in.245- remove_path(store, event.path, event.save_id)!246- strips << event247- }248- }249- for record in fetched.saves {250- files := to_files(record).filter(fn [strips, record] (file snapshot.FileEntry) bool {251- return !strips.any(reaches(it, record.id, record.created_at)252- && covers(it.path, file.path))253- })254- history.import_save(store, to_save(record), files)!255- }256- if strips.len > 0 {257- store.drop_unreferenced_objects()!258- }259-}260-261-// reaches reports whether a strip event applies to a save arriving from the262-// remote. A strip of every save covers the saves made up to its own time.263-fn reaches(strip events.Event, save_id string, created_at i64) bool {264- if strip.save_id != '' {265- return strip.save_id == save_id266- }267- return created_at <= strip.created_at268-}tests/library_check_test.v+7 −0
@@ -16,6 +16,13 @@1616 import net.http1717 import time1818 19+// Koh 3 is built and tested on V master at b9a2d87, and that commit is the pin:20+// V master moves, and a newer V may break Koh at any time. Building with any21+// other compiler fails here rather than somewhere stranger.22+fn test_built_with_the_pinned_v() {23+ assert @VCURRENTHASH == 'b9a2d87'24+}25+1926 // Vectors from the official blake3 bindings, cross-checked against the2027 // naming of real Koh 2 objects: `.koh/objects/<hash[0..2]>/<hash[2..]>` is2128 // blake3(uncompressed bytes) of a zlib stream.v.mod+1 −1
@@ -1,6 +1,6 @@11 Module {22 name: 'koh3'3- version: '3.0.53'3+ version: '3.0.54'44 description: 'One SQLite file per repo, one line of history.'55 license: 'MIT'66 }