strip.v

1//! Strip: removing a path from history, here and on the remote.2//!3//! The local half is one transaction: the saved files go, the objects nobody4//! references go, a strip event is recorded, and so is a note of which objects5//! the remote may now drop. The remote half reads that note, so it can be6//! re-run after an interruption even though the local rows are long gone.78module exchange910import json211import os12import src.events13import src.fmt14import src.remote15import src.storage1617// Scope is what a strip would remove, for the warning shown before it runs.18pub struct Scope {19pub:20	files   int21	saves   int22	objects int23}2425pub struct LocalStrip {26pub:27	files_removed   int28	objects_dropped i6429}3031// SendReport says what a strip managed to make visible on the remote.32pub struct SendReport {33pub mut:34	records    int35	objects    int36	event_sent bool37}3839// Pending is the note a strip leaves until its remote half has finished: which40// saves it covered ('' for every save) and which objects it freed. One note per41// path, so strips of the same path that did not finish are finished together.42pub struct Pending {43mut:44	saves  []string45	hashes []string46}4748fn pending_key(path string) string {49	return 'strip_pending:${path}'50}5152// read_pending refuses a note it cannot parse, one naming something other53// than an object, or one claiming a scope no recorded strip of the path has:54// sending it would lose the retry, delete outside the repo's objects, or55// remove the path where no strip asked. The note stays for recovery.56pub fn read_pending(store &storage.Store, path string) !Pending {57	text := store.state(pending_key(path)) or { return Pending{} }58	note := json2.decode[Pending](text) or {59		return error('the unfinished strip note for ${path} is unreadable; it is kept for recovery')60	}61	if !note.hashes.all(storage.valid_hash(it)) {62		return error('the unfinished strip note for ${path} names something other than an object; it is kept for recovery')63	}64	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)65	if note.saves.len == 0 || !note.saves.all(fn [recorded] (scope string) bool {66		return recorded.any(it.save_id == scope)67	}) {68		return error('no recorded strip matches the unfinished strip note for ${path}; it is kept for recovery, and koh steal brings a strip made elsewhere')69	}70	return note71}7273// strip_pending is true while the remote half of a strip has not finished.74pub fn strip_pending(store &storage.Store, path string) bool {75	return store.state(pending_key(path)) != none76}7778// covers is the one rule for what a strip removes: the path itself, or79// anything beneath it when it names a folder.80fn covers(path string, file string) bool {81	return file == path || file.starts_with('${path.trim_right('/')}/')82}8384// strip_scope counts the rows strip_rows will delete, using the same match. `objects` is the distinct85// content they hold; content another path still uses is kept, so fewer may be deleted.86pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {87	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)88		FROM save_files WHERE ${match_clause(only_save)}')!89	defer {90		stmt.finalize()91	}92	bind_match(stmt, path, only_save)!93	stmt.step()!94	return Scope{95		files:   int(stmt.int(0))96		saves:   int(stmt.int(1))97		objects: int(stmt.int(2))98	}99}100101// strip_local removes the path in one transaction and leaves the note the102// remote half needs.103pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {104	store.begin()!105	report := strip_rows(store, path, only_save) or {106		store.rollback()107		return err108	}109	store.end()!110	return report111}112113/// Strip the batch in one transaction; failures leave history unchanged.114pub fn strip_local_batch(store &storage.Store, paths []string, only_save string) ![]LocalStrip {115	store.begin()!116	mut reports := []LocalStrip{}117	for path in paths {118		reports << strip_rows(store, path, only_save) or {119			store.rollback()120			return err121		}122	}123	store.end() or {124		store.rollback()125		return err126	}127	return reports128}129130// strip_rows is the body of strip_local's transaction. Its event has no save id for a whole-history131// strip (spec §10.2), and that empty id is what bans the path from every later walk.132fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {133	hashes := stripped_hashes(store, path, only_save)!134	if hashes.len == 0 {135		// Already stripped: keep the pending note an unfinished remote needs.136		return LocalStrip{}137	}138	removed := remove_path(store, path, only_save)!139	dropped := store.drop_unused(hashes)!140	events.strip(store, only_save, path)!141	// An earlier strip of the same path may not have finished on the remote,142	// so the note grows rather than resets.143	queue_strip(store, path, [only_save], hashes)!144	return LocalStrip{145		files_removed:   removed146		objects_dropped: dropped147	}148}149150// send_strip makes a strip visible on the remote: the event travels like any151// other, every remote save record listing the path is rewritten, and objects152// the strip freed are deleted once no remote record and no local save needs153// them. Every step is safe to repeat, and the pending note is cleared only154// when all of them have succeeded.155pub fn send_strip(store &storage.Store, mut backend remote.Backend, path string) !SendReport {156	mut report := SendReport{}157	send_steps(store, mut backend, path, mut report) or {158		mut done := []string{}159		if report.event_sent || report.records + report.objects > 0 {160			done << '${if report.event_sent { 'sent its event, ' } else { '' }}rewrote ${fmt.plural(report.records, 'remote record')} and deleted ${fmt.plural(report.objects, 'remote object')} for ${path}'161		}162		return partway(done, err)163	}164	return report165}166167fn send_steps(store &storage.Store, mut backend remote.Backend, path string, mut report SendReport) ! {168	note := read_pending(store, path)!169	name := repo_name(store, mut backend)!170	mut upload := OfferReport{171		repo: name172	}173	defer {174		report.event_sent = upload.events > 0175	}176	upload_events(store, mut backend, name, mut upload)!177	mut freed := map[string]bool{}178	for hash in note.hashes {179		freed[hash] = true180	}181	every_save := '' in note.saves182	mut needed := map[string]bool{}183	for key in backend.list('${name}/saves')! {184		remote_object := backend.get(key)!185		if !remote_object.found {186			continue187		}188		record := decode_save(remote_object.body)!189		mut kept := []FileRecord{}190		mut dropped := []string{}191		for file in record.files {192			if (every_save || record.id in note.saves) && covers(path, file.path) {193				freed[file.hash] = true194				dropped << file.hash195				continue196			}197			kept << file198			needed[file.hash] = true199		}200		if kept.len == record.files.len {201			continue202		}203		// Once rewritten, the record no longer names these objects; the note204		// keeps them so a retry can still delete them.205		queue_strip(store, path, []string{}, dropped)!206		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {207			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')208		}209		report.records++210	}211	for hash, _ in freed {212		if hash in needed || !storage.valid_hash(hash) || store.has_object(hash)!213			|| store.paths_of_object(hash)!.len > 0 {214			continue215		}216		backend.delete(object_key(name, hash))!217		report.objects++218	}219	store.remove_state(pending_key(path))!220}221222// requeue_strip notes every valid recorded strip of `path` again, so that223// send_strip removes the path from remote records that still hold it.224pub fn requeue_strip(store &storage.Store, path string) ! {225	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)226	if recorded.len == 0 {227		return error('this repo has no recorded strip of ${path}; run koh steal, then koh doctor')228	}229	queue_strip(store, path, recorded.map(it.save_id), []string{})!230}231232// queue_strip adds scopes and objects to the note send_strip works from,233// writing it only when something is new. Only object hashes are noted.234fn queue_strip(store &storage.Store, path string, scopes []string, hashes []string) ! {235	mut note := read_pending(store, path)!236	count := note.saves.len + note.hashes.len237	note.saves << scopes.filter(it !in note.saves)238	note.hashes << hashes.filter(it !in note.hashes && storage.valid_hash(it))239	if note.saves.len + note.hashes.len > count {240		store.set_state(pending_key(path), json2.encode(note))!241	}242}243244// unapplied_strips are the strip events on the remote this repo has not245// received yet. offer asks before uploading and again after, because a strip that lands246// in between would otherwise leave the stripped path in the bucket.247fn unapplied_strips(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {248	mut known := map[string]bool{}249	for event in events.all(store)! {250		known[event.id] = true251	}252	mut strips := []events.Event{}253	for key in backend.list(events_prefix(repo))! {254		if os.base(key) in known {255			continue256		}257		body := backend.get(key)!258		if !body.found {259			continue260		}261		event := decode_event(body.body)!262		if events.valid_strip(event) && !ignored_strip(event) {263			strips << event264		}265	}266	return strips267}268269// rewrite returns the save record with a new file list.270fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {271	return SaveRecord{272		...record273		files: files274	}275}276277// stripped_hashes is the content of the rows remove_path will delete.278fn stripped_hashes(store &storage.Store, path string, only_save string) ![]string {279	mut stmt :=280		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!281	defer {282		stmt.finalize()283	}284	bind_match(stmt, path, only_save)!285	mut hashes := []string{}286	for stmt.step()! {287		hashes << stmt.text(0)288	}289	return hashes290}291292// remove_path deletes the rows and counts them; the objects go afterwards, once nothing references them.293fn remove_path(store &storage.Store, path string, only_save string) !int {294	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!295	defer {296		stmt.finalize()297	}298	bind_match(stmt, path, only_save)!299	stmt.step()!300	return store.db.changes()301}302303// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`304// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.305fn match_clause(only_save string) string {306	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }307	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'308}309310// bind_match fills what match_clause names: ?1 the path, ?2 its folder prefix, ?3 the save.311fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {312	stmt.bind_text(1, path)!313	stmt.bind_text(2, '${path.trim_right('/')}/')!314	if only_save != '' {315		stmt.bind_text(3, only_save)!316	}317}