//! Sync: offer and steal. //! //! All the cleverness sits here, above the five-operation remote interface, so //! the backends stay thin and this logic is written once and tested offline. //! //! The remote is a line of saves, the same shape as the local one. `offer` //! pushes local saves the remote lacks; `steal` pulls remote saves and, when //! both sides moved, replays the local work on top of the remote's. Divergence //! is resolved per file, never per line. module sync import os import src.events import src.history import src.remote import src.snapshot import src.storage // Where the repo's name on the remote and the last offered head are remembered. const repo_state_key = 'remote_repo' const offered_head_key = 'remote_head' pub const no_saves = 'this repo has no saves yet — nothing to offer' pub struct OfferReport { pub mut: repo string saves int objects int events int head string } // repo_name is the folder's name at first offer, remembered in the repo so // renaming the folder later changes nothing. pub fn repo_name(store &storage.Store, mut backend remote.Backend) !string { if remembered := store.state(repo_state_key) { return remembered } name := os.base(store.root) existing := backend.get(head_key(name))! if existing.found { theirs := existing.body.bytestr().trim_space() if history.by_id(store, theirs) == none { return error('the remote already has "${name}" with a history this repo does not share — rename the folder, or use another remote') } } store.set_state(repo_state_key, name)! return name } // offer sends what the remote lacks and then swaps `head`. Nothing is ever // queued: the next offer simply sends everything the remote does not have. pub fn offer(store &storage.Store, mut backend remote.Backend) !OfferReport { name := repo_name(store, mut backend)! local := history.head(store) or { return error(no_saves) } mut report := OfferReport{ repo: name head: local.id } remote_head := backend.get(head_key(name))! if remote_head.found { theirs := remote_head.body.bytestr().trim_space() newer := saves_after(store, local.id, theirs) or { return error('remote has new saves — run koh steal') } for save in newer { upload_save(store, mut backend, name, save, mut report)! } } else { oldest_first := history.line(store)! for index := oldest_first.len - 1; index >= 0; index-- { upload_save(store, mut backend, name, oldest_first[index], mut report)! } } upload_events(store, mut backend, name, mut report)! if !backend.put_if_match(head_key(name), local.id.bytes(), remote_head.tag)! { return error('remote has new saves — run koh steal') } store.set_state(offered_head_key, local.id)! return report } // saves_after lists the local saves newer than `theirs`, oldest first. It fails // when `theirs` is not on this line, which is the honest answer: we cannot // offer on top of a save we do not have. fn saves_after(store &storage.Store, ours string, theirs string) ![]history.Save { mut newer := []history.Save{} mut cursor := ours for cursor != '' && cursor != theirs { save := history.by_id(store, cursor) or { return error('save ${cursor} is missing') } newer << save cursor = save.parent } if cursor != theirs { return error('${theirs} is not on this line') } newer.reverse_in_place() return newer } fn upload_save(store &storage.Store, mut backend remote.Backend, repo string, save history.Save, mut report OfferReport) ! { files := history.files_of(store, save.id)! for file in files { if backend.put_if_absent(object_key(repo, file.hash), store.object_blob(file.hash)!)! { report.objects++ } } record := save_record(store, save)! if backend.put_if_absent(save_key(repo, save.id), encode_save(record).bytes())! { report.saves++ } } // upload_events sends every event the remote does not have. Events are tiny, so // listing them costs less than tracking which ones were sent. fn upload_events(store &storage.Store, mut backend remote.Backend, repo string, mut report OfferReport) ! { already := backend.list(events_prefix(repo))! mut already_sent := map[string]bool{} for key in already { already_sent[os.base(key)] = true } for event in events.all(store)! { if event.id in already_sent { continue } if backend.put_if_absent(event_key(repo, event.id), encode_event(event).bytes())! { report.events++ } } } // fetched holds everything a steal read from the remote, before any of it is // written locally. Network work happens first; the database gets one // transaction, so a failure leaves nothing half-applied. pub struct Fetched { pub mut: repo_objects map[string][]u8 saves []SaveRecord // oldest first events_out []events.Event head string base string // the newest save both sides already shared, '' when none remote_save_ids map[string]bool } pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched { remote_head := backend.get(head_key(repo))! if !remote_head.found { return error('the remote has no "${repo}" yet — run koh offer here first') } mut fetched := Fetched{ head: remote_head.body.bytestr().trim_space() remote_save_ids: map[string]bool{} } mut cursor := fetched.head for cursor != '' && history.by_id(store, cursor) == none { record_object := backend.get(save_key(repo, cursor))! if !record_object.found { return error('the remote is missing save ${cursor} — run koh offer there') } record := decode_save(record_object.body)! for file in record.files { if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects { blob := backend.get(object_key(repo, file.hash))! if !blob.found { return error('the remote is missing object ${file.hash} that save ${record.id} needs') } fetched.repo_objects[file.hash] = blob.body } } fetched.remote_save_ids[record.id] = true fetched.saves << record cursor = record.parent } fetched.saves.reverse_in_place() fetched.base = cursor fetched.events_out = fetch_events(store, mut backend, repo)! return fetched } fn fetch_events(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event { have := events.all(store)! mut known := map[string]bool{} for event in have { known[event.id] = true } mut incoming := []events.Event{} for key in backend.list(events_prefix(repo))! { id := os.base(key) if id in known { continue } body := backend.get(event_key(repo, id))! if !body.found { continue } incoming << decode_event(body.body)! } return incoming } // write_fetched lands a fetch in one transaction: objects first (verified), then // the saves they belong to, then the events. pub fn write_fetched(store &storage.Store, fetched Fetched) ! { for hash, blob in fetched.repo_objects { store.put_object_blob(hash, blob)! } for record in fetched.saves { history.import_save(store, to_save(record), to_files(record))! } for event in fetched.events_out { events.write(store, event)! if event.kind == events.kind_strip { // Somebody stripped a path: remove it here too, so the two // histories agree from now on. apply_strip(store, event)! } } } // can_fast_forward reports whether our line sits behind the remote's, which is // the case when every local save is an ancestor of the remote head. pub fn can_fast_forward(store &storage.Store, remote_head string, remote_ids map[string]bool) bool { mut cursor := remote_head for cursor != '' { if history.by_id(store, cursor) == none { return false } save := history.by_id(store, cursor) or { return false } if save.parent == '' { // Reached the first save without leaving the line we have. return true } cursor = save.parent } return false } // PushReport says what a strip managed to make visible on the remote. pub struct PushReport { pub mut: records int objects int event_sent bool } // push_strip makes a strip visible on the remote. The event itself travels like // any other event; on top of that, every remote save record that listed the // path is rewritten in place, and — when the strip covered the whole local // history — the objects nobody references any more are deleted. // // The caller passes what it stripped, because the local rows are already gone // by the time the remote hears about it. Every step is safe to repeat, so an // interrupted strip finishes on the next run. pub fn push_strip(store &storage.Store, mut backend remote.Backend, path string, save_ids []string, object_hashes []string, drop_objects bool) !PushReport { name := repo_name(store, mut backend)! mut report := PushReport{} mut upload := OfferReport{ repo: name } upload_events(store, mut backend, name, mut upload)! report.event_sent = upload.events > 0 for save_id in save_ids { remote_object := backend.get(save_key(name, save_id))! if !remote_object.found { continue } record := decode_save(remote_object.body)! mut kept := []FileRecord{} for file in record.files { if file.path == path || file.path.starts_with('${path.trim_right('/')}/') { continue } kept << file } if kept.len == record.files.len { continue } if !backend.put_if_match(save_key(name, save_id), encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! { return error('the remote copy of save ${save_id} moved while stripping — run koh steal, then strip again') } report.records++ } if drop_objects { for hash in object_hashes { // Only objects no local save refers to any more: with `--save`, the // other saves still need them. if !store.has_object(hash)! { backend.delete(object_key(name, hash))! report.objects++ } } } return report } // rewrite returns the save record with a new file list. fn rewrite(record SaveRecord, files []FileRecord) SaveRecord { return SaveRecord{ id: record.id parent: record.parent message: record.message created_at: record.created_at machine: record.machine is_checkpoint: record.is_checkpoint size_bytes: record.size_bytes agent: record.agent session: record.session restores: record.restores replayed_from: record.replayed_from files: files } } // apply_strip is what a machine does when it hears about somebody else's strip: // the same removal, so both histories agree from then on. It returns how many // saved files it removed. pub fn apply_strip(store &storage.Store, event events.Event) !int { mut query := "DELETE FROM save_files WHERE (path = ? OR path LIKE ? ESCAPE '\\')" if event.save_id != '' { query += ' AND save_id = ?' } mut stmt := store.db.prepare(query)! stmt.bind_text(1, event.path)! stmt.bind_text(2, like_under(event.path))! if event.save_id != '' { stmt.bind_text(3, event.save_id)! } stmt.step()! removed := store.db.changes() stmt.finalize() if removed > 0 { store.drop_unreferenced_objects()! } return removed } // like_under escapes LIKE's own wildcards so a path containing `%` or `_` // matches itself and nothing else. fn like_under(folder string) string { escaped := folder.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') return '${escaped}/%' }