strip.v

1//! Strip: removing a path from history, here and on the remote.2//!3//! The local half is one transaction: the saved files go, the objects nobody4//! references go, a strip event is recorded, and so is a note of which objects5//! the remote may now drop. The remote half reads that note, so it can be6//! re-run after an interruption even though the local rows are long gone.78module exchange910import json211import os12import src.events13import src.fmt14import src.remote15import src.storage1617// Scope is what a strip would remove, for the warning shown before it runs.18pub struct Scope {19pub:20	files   int21	saves   int22	objects int23}2425pub struct LocalStrip {26pub:27	files_removed   int28	objects_dropped i6429}3031// SendReport says what a strip managed to make visible on the remote.32pub struct SendReport {33pub mut:34	records    int35	objects    int36	event_sent bool37}3839// Pending is the note a strip leaves until its remote half has finished: which40// saves it covered ('' for every save) and which objects it freed. One note per41// path, so strips of the same path that did not finish are finished together.42pub struct Pending {43mut:44	saves  []string45	hashes []string46}4748fn pending_key(path string) string {49	return 'strip_pending:${path}'50}5152// read_pending refuses a note it cannot parse, one naming something other53// than an object, or one claiming a scope no recorded strip of the path has:54// sending it would lose the retry, delete outside the repo's objects, or55// remove the path where no strip asked. The note stays for recovery.56pub fn read_pending(store &storage.Store, path string) !Pending {57	text := store.state(pending_key(path)) or { return Pending{} }58	note := json2.decode[Pending](text) or {59		return error('the unfinished strip note for ${path} is unreadable; it is kept for recovery')60	}61	if !note.hashes.all(storage.valid_hash(it)) {62		return error('the unfinished strip note for ${path} names something other than an object; it is kept for recovery')63	}64	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)65	if note.saves.len == 0 || !note.saves.all(fn [recorded] (scope string) bool {66		return recorded.any(it.save_id == scope)67	}) {68		return error('no recorded strip matches the unfinished strip note for ${path}; it is kept for recovery')69	}70	return note71}7273// strip_pending is true while the remote half of a strip has not finished.74pub fn strip_pending(store &storage.Store, path string) bool {75	return store.state(pending_key(path)) != none76}7778// covers is the one rule for what a strip removes: the path itself, or79// anything beneath it when it names a folder.80fn covers(path string, file string) bool {81	return file == path || file.starts_with('${path.trim_right('/')}/')82}8384// strip_scope counts the rows strip_rows will delete, using the same match. `objects` is the distinct85// content they hold; content another path still uses is kept, so fewer may be deleted.86pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {87	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)88		FROM save_files WHERE ${match_clause(only_save)}')!89	defer {90		stmt.finalize()91	}92	bind_match(stmt, path, only_save)!93	stmt.step()!94	return Scope{95		files:   int(stmt.int(0))96		saves:   int(stmt.int(1))97		objects: int(stmt.int(2))98	}99}100101// strip_local removes the path in one transaction and leaves the note the102// remote half needs.103pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {104	store.begin()!105	report := strip_rows(store, path, only_save) or {106		store.rollback()107		return err108	}109	store.end()!110	return report111}112113/// Strip the batch in one transaction; failures leave history unchanged.114pub fn strip_local_batch(store &storage.Store, paths []string, only_save string) ![]LocalStrip {115	store.begin()!116	mut reports := []LocalStrip{}117	for path in paths {118		reports << strip_rows(store, path, only_save) or {119			store.rollback()120			return err121		}122	}123	store.end() or {124		store.rollback()125		return err126	}127	return reports128}129130// strip_rows is the body of strip_local's transaction. Its event has no save id for a whole-history131// strip (spec §10.2), and that empty id is what bans the path from every later walk.132fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {133	hashes := stripped_hashes(store, path, only_save)!134	if hashes.len == 0 {135		// Already stripped: keep the pending note an unfinished remote needs.136		return LocalStrip{}137	}138	removed := remove_path(store, path, only_save)!139	dropped := store.drop_unused(hashes)!140	events.strip(store, only_save, path)!141	// An earlier strip of the same path may not have finished on the remote,142	// so the note grows rather than resets.143	mut note := read_pending(store, path)!144	if only_save !in note.saves {145		note.saves << only_save146	}147	note.hashes << hashes148	store.set_state(pending_key(path), json2.encode(note))!149	return LocalStrip{150		files_removed:   removed151		objects_dropped: dropped152	}153}154155// send_strip makes a strip visible on the remote: the event travels like any156// other, every remote save record listing the path is rewritten, and objects157// the strip freed are deleted once no remote record and no local save needs158// them. Every step is safe to repeat, and the pending note is cleared only159// when all of them have succeeded.160pub fn send_strip(store &storage.Store, mut backend remote.Backend, path string) !SendReport {161	mut report := SendReport{}162	send_steps(store, mut backend, path, mut report) or {163		mut done := []string{}164		if report.records + report.objects > 0 {165			done << 'rewrote ${fmt.plural(report.records, 'remote record')} and deleted ${fmt.plural(report.objects, 'remote object')} for ${path}'166		}167		return partway(done, err)168	}169	return report170}171172fn send_steps(store &storage.Store, mut backend remote.Backend, path string, mut report SendReport) ! {173	mut note := read_pending(store, path)!174	name := repo_name(store, mut backend)!175	mut upload := OfferReport{176		repo: name177	}178	upload_events(store, mut backend, name, mut upload)!179	report.event_sent = upload.events > 0180	mut freed := map[string]bool{}181	for hash in note.hashes {182		freed[hash] = true183	}184	every_save := '' in note.saves185	mut needed := map[string]bool{}186	for key in backend.list('${name}/saves')! {187		remote_object := backend.get(key)!188		if !remote_object.found {189			continue190		}191		record := decode_save(remote_object.body)!192		mut kept := []FileRecord{}193		mut dropped := []string{}194		for file in record.files {195			if (every_save || record.id in note.saves) && covers(path, file.path) {196				freed[file.hash] = true197				dropped << file.hash198				continue199			}200			kept << file201			needed[file.hash] = true202		}203		if kept.len == record.files.len {204			continue205		}206		// Once rewritten, the record no longer names these objects; the note207		// keeps them so a retry can still delete them.208		unnoted := dropped.filter(it !in note.hashes)209		if unnoted.len > 0 {210			note.hashes << unnoted211			store.set_state(pending_key(path), json2.encode(note))!212		}213		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {214			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')215		}216		report.records++217	}218	for hash, _ in freed {219		if hash in needed || store.has_object(hash)! {220			continue221		}222		backend.delete(object_key(name, hash))!223		report.objects++224	}225	store.remove_state(pending_key(path))!226}227228// requeue_strip notes every valid recorded strip of `path` again, so that229// send_strip removes the path from remote records that still hold it.230pub fn requeue_strip(store &storage.Store, path string) ! {231	mut note := read_pending(store, path)!232	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)233	if recorded.len == 0 {234		return error('this repo has no recorded strip of ${path}; run koh steal, then koh doctor')235	}236	for event in recorded {237		if event.save_id !in note.saves {238			note.saves << event.save_id239		}240	}241	store.set_state(pending_key(path), json2.encode(note))!242}243244// unapplied_strips are the strip events on the remote this repo has not245// received yet. offer asks before uploading and again after, because a strip that lands246// in between would otherwise leave the stripped path in the bucket.247fn unapplied_strips(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {248	mut known := map[string]bool{}249	for event in events.all(store)! {250		known[event.id] = true251	}252	mut strips := []events.Event{}253	for key in backend.list(events_prefix(repo))! {254		if os.base(key) in known {255			continue256		}257		body := backend.get(key)!258		if !body.found {259			continue260		}261		event := decode_event(body.body)!262		if events.valid_strip(event) && !ignored_strip(event) {263			strips << event264		}265	}266	return strips267}268269// rewrite returns the save record with a new file list.270fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {271	return SaveRecord{272		...record273		files: files274	}275}276277// stripped_hashes is the content of the rows remove_path will delete.278fn stripped_hashes(store &storage.Store, path string, only_save string) ![]string {279	mut stmt :=280		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!281	defer {282		stmt.finalize()283	}284	bind_match(stmt, path, only_save)!285	mut hashes := []string{}286	for stmt.step()! {287		hashes << stmt.text(0)288	}289	return hashes290}291292// remove_path deletes the rows and counts them; the objects go afterwards, once nothing references them.293fn remove_path(store &storage.Store, path string, only_save string) !int {294	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!295	defer {296		stmt.finalize()297	}298	bind_match(stmt, path, only_save)!299	stmt.step()!300	return store.db.changes()301}302303// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`304// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.305fn match_clause(only_save string) string {306	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }307	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'308}309310// bind_match fills what match_clause names: ?1 the path, ?2 its folder prefix, ?3 the save.311fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {312	stmt.bind_text(1, path)!313	stmt.bind_text(2, '${path.trim_right('/')}/')!314	if only_save != '' {315		stmt.bind_text(3, only_save)!316	}317}