strip.v

1//! Strip: removing a path from history, here and on the remote.2//!3//! The local half is one transaction: the saved files go, the objects nobody4//! references go, a strip event is recorded, and so is a note of which objects5//! the remote may now drop. The remote half reads that note, so it can be6//! re-run after an interruption even though the local rows are long gone.78module exchange910import json211import os12import src.events13import src.fmt14import src.remote15import src.storage1617// Scope is what a strip would remove, for the warning shown before it runs.18pub struct Scope {19pub:20	files   int21	saves   int22	objects int23}2425pub struct LocalStrip {26pub:27	files_removed   int28	objects_dropped i6429}3031// SendReport says what a strip managed to make visible on the remote.32pub struct SendReport {33pub mut:34	records    int35	objects    int36	event_sent bool37}3839// Pending is the note a strip leaves until its remote half has finished: which40// saves it covered ('' for every save) and which objects it freed. One note per41// path, so strips of the same path that did not finish are finished together.42pub struct Pending {43mut:44	saves  []string45	hashes []string46}4748fn pending_key(path string) string {49	return 'strip_pending:${path}'50}5152// read_pending refuses a note it cannot parse, one naming something other53// than an object, or one claiming a scope no recorded strip of the path has:54// sending it would lose the retry, delete outside the repo's objects, or55// remove the path where no strip asked. The note stays for recovery.56pub fn read_pending(store &storage.Store, path string) !Pending {57	text := store.state(pending_key(path)) or { return Pending{} }58	note := json2.decode[Pending](text) or {59		return error('the unfinished strip note for ${path} is unreadable; it is kept for recovery')60	}61	if !note.hashes.all(storage.valid_hash(it)) {62		return error('the unfinished strip note for ${path} names something other than an object; it is kept for recovery')63	}64	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)65	if note.saves.len == 0 || !note.saves.all(fn [recorded] (scope string) bool {66		return recorded.any(it.save_id == scope)67	}) {68		return error('no recorded strip matches the unfinished strip note for ${path}; it is kept for recovery')69	}70	return note71}7273// strip_pending is true while the remote half of a strip has not finished.74pub fn strip_pending(store &storage.Store, path string) bool {75	return store.state(pending_key(path)) != none76}7778// covers is the one rule for what a strip removes: the path itself, or79// anything beneath it when it names a folder.80fn covers(path string, file string) bool {81	return file == path || file.starts_with('${path.trim_right('/')}/')82}8384// strip_scope counts the rows strip_rows will delete, using the same match. `objects` is the distinct85// content they hold; content another path still uses is kept, so fewer may be deleted.86pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {87	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)88		FROM save_files WHERE ${match_clause(only_save)}')!89	defer {90		stmt.finalize()91	}92	bind_match(stmt, path, only_save)!93	stmt.step()!94	return Scope{95		files:   int(stmt.int(0))96		saves:   int(stmt.int(1))97		objects: int(stmt.int(2))98	}99}100101// strip_local removes the path in one transaction and leaves the note the102// remote half needs.103pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {104	store.begin()!105	report := strip_rows(store, path, only_save) or {106		store.rollback()107		return err108	}109	store.end()!110	return report111}112113/// Strip the batch in one transaction; failures leave history unchanged.114pub fn strip_local_batch(store &storage.Store, paths []string, only_save string) ![]LocalStrip {115	store.begin()!116	mut reports := []LocalStrip{}117	for path in paths {118		reports << strip_rows(store, path, only_save) or {119			store.rollback()120			return err121		}122	}123	store.end() or {124		store.rollback()125		return err126	}127	return reports128}129130// strip_rows is the body of strip_local's transaction. Its event has no save id for a whole-history131// strip (spec §10.2), and that empty id is what bans the path from every later walk.132fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {133	hashes := stripped_hashes(store, path, only_save)!134	if hashes.len == 0 {135		// Already stripped: keep the pending note an unfinished remote needs.136		return LocalStrip{}137	}138	removed := remove_path(store, path, only_save)!139	dropped := store.drop_unused(hashes)!140	events.strip(store, only_save, path)!141	// An earlier strip of the same path may not have finished on the remote,142	// so the note grows rather than resets.143	mut note := read_pending(store, path)!144	if only_save !in note.saves {145		note.saves << only_save146	}147	note.hashes << hashes148	store.set_state(pending_key(path), json2.encode(note))!149	return LocalStrip{150		files_removed:   removed151		objects_dropped: dropped152	}153}154155// send_strip makes a strip visible on the remote: the event travels like any156// other, every remote save record listing the path is rewritten, and objects157// the strip freed are deleted once no remote record and no local save needs158// them. Every step is safe to repeat, and the pending note is cleared only159// when all of them have succeeded.160pub fn send_strip(store &storage.Store, mut backend remote.Backend, path string) !SendReport {161	mut report := SendReport{}162	send_steps(store, mut backend, path, mut report) or {163		mut done := []string{}164		if report.records + report.objects > 0 {165			done << 'rewrote ${fmt.plural(report.records, 'remote record')} and deleted ${fmt.plural(report.objects, 'remote object')} for ${path}'166		}167		return partway(done, err)168	}169	return report170}171172fn send_steps(store &storage.Store, mut backend remote.Backend, path string, mut report SendReport) ! {173	note := read_pending(store, path)!174	name := repo_name(store, mut backend)!175	mut upload := OfferReport{176		repo: name177	}178	upload_events(store, mut backend, name, mut upload)!179	report.event_sent = upload.events > 0180	mut freed := map[string]bool{}181	for hash in note.hashes {182		freed[hash] = true183	}184	every_save := '' in note.saves185	mut needed := map[string]bool{}186	for key in backend.list('${name}/saves')! {187		remote_object := backend.get(key)!188		if !remote_object.found {189			continue190		}191		record := decode_save(remote_object.body)!192		mut kept := []FileRecord{}193		for file in record.files {194			if (every_save || record.id in note.saves) && covers(path, file.path) {195				freed[file.hash] = true196				continue197			}198			kept << file199			needed[file.hash] = true200		}201		if kept.len == record.files.len {202			continue203		}204		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {205			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')206		}207		report.records++208	}209	for hash, _ in freed {210		if hash in needed || store.has_object(hash)! {211			continue212		}213		backend.delete(object_key(name, hash))!214		report.objects++215	}216	store.remove_state(pending_key(path))!217}218219// requeue_strip notes every valid recorded strip of `path` again, so that220// send_strip removes the path from remote records that still hold it.221pub fn requeue_strip(store &storage.Store, path string) ! {222	mut note := read_pending(store, path)!223	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)224	if recorded.len == 0 {225		return error('this repo has no recorded strip of ${path}; run koh steal, then koh doctor')226	}227	for event in recorded {228		if event.save_id !in note.saves {229			note.saves << event.save_id230		}231	}232	store.set_state(pending_key(path), json2.encode(note))!233}234235// unapplied_strips are the strip events on the remote this repo has not236// received yet. offer asks before uploading and again after, because a strip that lands237// in between would otherwise leave the stripped path in the bucket.238fn unapplied_strips(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {239	mut known := map[string]bool{}240	for event in events.all(store)! {241		known[event.id] = true242	}243	mut strips := []events.Event{}244	for key in backend.list(events_prefix(repo))! {245		if os.base(key) in known {246			continue247		}248		body := backend.get(key)!249		if !body.found {250			continue251		}252		event := decode_event(body.body)!253		if events.valid_strip(event) && !ignored_strip(event) {254			strips << event255		}256	}257	return strips258}259260// rewrite returns the save record with a new file list.261fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {262	return SaveRecord{263		...record264		files: files265	}266}267268// stripped_hashes is the content of the rows remove_path will delete.269fn stripped_hashes(store &storage.Store, path string, only_save string) ![]string {270	mut stmt :=271		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!272	defer {273		stmt.finalize()274	}275	bind_match(stmt, path, only_save)!276	mut hashes := []string{}277	for stmt.step()! {278		hashes << stmt.text(0)279	}280	return hashes281}282283// remove_path deletes the rows and counts them; the objects go afterwards, once nothing references them.284fn remove_path(store &storage.Store, path string, only_save string) !int {285	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!286	defer {287		stmt.finalize()288	}289	bind_match(stmt, path, only_save)!290	stmt.step()!291	return store.db.changes()292}293294// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`295// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.296fn match_clause(only_save string) string {297	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }298	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'299}300301// bind_match fills what match_clause names: ?1 the path, ?2 its folder prefix, ?3 the save.302fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {303	stmt.bind_text(1, path)!304	stmt.bind_text(2, '${path.trim_right('/')}/')!305	if only_save != '' {306		stmt.bind_text(3, only_save)!307	}308}