strip.v

1//! Strip: removing a path from history, here and on the remote.2//!3//! The local half is one transaction: the saved files go, the objects nobody4//! references go, a strip event is recorded, and so is a note of which objects5//! the remote may now drop. The remote half reads that note, so it can be6//! re-run after an interruption even though the local rows are long gone.78module exchange910import json211import os12import src.events13import src.fmt14import src.remote15import src.storage1617// Scope is what a strip would remove, for the warning shown before it runs.18pub struct Scope {19pub:20	files   int21	saves   int22	objects int23}2425pub struct LocalStrip {26pub:27	files_removed   int28	objects_dropped i6429}3031// SendReport says what a strip managed to make visible on the remote.32pub struct SendReport {33pub mut:34	records    int35	objects    int36	event_sent bool37}3839// Pending is the note a strip leaves until its remote half has finished: which40// saves it covered ('' for every save) and which objects it freed. One note per41// path, so strips of the same path that did not finish are finished together.42pub struct Pending {43mut:44	saves  []string45	hashes []string46}4748fn pending_key(path string) string {49	return 'strip_pending:${path}'50}5152// read_pending refuses a note it cannot parse, one naming something other53// than an object, or one claiming a scope no recorded strip of the path has:54// sending it would lose the retry, delete outside the repo's objects, or55// remove the path where no strip asked. The note stays for recovery.56pub fn read_pending(store &storage.Store, path string) !Pending {57	text := store.state(pending_key(path)) or { return Pending{} }58	note := json2.decode[Pending](text) or {59		return error('the unfinished strip note for ${path} is unreadable; it is kept for recovery')60	}61	if !note.hashes.all(storage.valid_hash(it)) {62		return error('the unfinished strip note for ${path} names something other than an object; it is kept for recovery')63	}64	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)65	if note.saves.len == 0 || !note.saves.all(fn [recorded] (scope string) bool {66		return recorded.any(it.save_id == scope)67	}) {68		return error('no recorded strip matches the unfinished strip note for ${path}; it is kept for recovery, and koh steal brings a strip made elsewhere')69	}70	return note71}7273// strip_pending is true while the remote half of a strip has not finished.74pub fn strip_pending(store &storage.Store, path string) bool {75	return store.state(pending_key(path)) != none76}7778// covers is the one rule for what a strip removes: the path itself, or79// anything beneath it when it names a folder.80fn covers(path string, file string) bool {81	return file == path || file.starts_with('${path.trim_right('/')}/')82}8384// strip_scope counts the rows strip_rows will delete, using the same match. `objects` is the distinct85// content they hold; content another path still uses is kept, so fewer may be deleted.86pub fn strip_scope(store &storage.Store, path string, only_save string) !Scope {87	mut stmt := store.db.prepare('SELECT COUNT(*), COUNT(DISTINCT save_id), COUNT(DISTINCT hash)88		FROM save_files WHERE ${match_clause(only_save)}')!89	defer {90		stmt.finalize()91	}92	bind_match(stmt, path, only_save)!93	stmt.step()!94	return Scope{95		files:   int(stmt.int(0))96		saves:   int(stmt.int(1))97		objects: int(stmt.int(2))98	}99}100101// strip_local removes the path in one transaction and leaves the note the102// remote half needs.103pub fn strip_local(store &storage.Store, path string, only_save string) !LocalStrip {104	store.begin()!105	report := strip_rows(store, path, only_save) or {106		store.rollback()107		return err108	}109	store.end()!110	return report111}112113/// Strip the batch in one transaction; failures leave history unchanged.114pub fn strip_local_batch(store &storage.Store, paths []string, only_save string) ![]LocalStrip {115	store.begin()!116	mut reports := []LocalStrip{}117	for path in paths {118		reports << strip_rows(store, path, only_save) or {119			store.rollback()120			return err121		}122	}123	store.end() or {124		store.rollback()125		return err126	}127	return reports128}129130// strip_rows is the body of strip_local's transaction. Its event has no save id for a whole-history131// strip (spec §10.2), and that empty id is what bans the path from every later walk.132fn strip_rows(store &storage.Store, path string, only_save string) !LocalStrip {133	hashes := stripped_hashes(store, path, only_save)!134	if hashes.len == 0 {135		// Already stripped: keep the pending note an unfinished remote needs.136		return LocalStrip{}137	}138	removed := remove_path(store, path, only_save)!139	dropped := store.drop_unused(hashes)!140	events.strip(store, only_save, path)!141	// An earlier strip of the same path may not have finished on the remote,142	// so the note grows rather than resets.143	queue_strip(store, path, [only_save], hashes)!144	return LocalStrip{145		files_removed:   removed146		objects_dropped: dropped147	}148}149150// send_strip makes a strip visible on the remote: the event travels like any151// other, every remote save record listing the path is rewritten, and objects152// the strip freed are deleted once no remote record and no local save needs153// them. Every step is safe to repeat, and the pending note is cleared only154// when all of them have succeeded.155pub fn send_strip(store &storage.Store, mut backend remote.Backend, path string) !SendReport {156	mut report := SendReport{}157	send_steps(store, mut backend, path, mut report) or {158		mut done := []string{}159		if report.records + report.objects > 0 {160			done << 'rewrote ${fmt.plural(report.records, 'remote record')} and deleted ${fmt.plural(report.objects, 'remote object')} for ${path}'161		}162		return partway(done, err)163	}164	return report165}166167fn send_steps(store &storage.Store, mut backend remote.Backend, path string, mut report SendReport) ! {168	note := read_pending(store, path)!169	name := repo_name(store, mut backend)!170	mut upload := OfferReport{171		repo: name172	}173	upload_events(store, mut backend, name, mut upload)!174	report.event_sent = upload.events > 0175	mut freed := map[string]bool{}176	for hash in note.hashes {177		freed[hash] = true178	}179	every_save := '' in note.saves180	mut needed := map[string]bool{}181	for key in backend.list('${name}/saves')! {182		remote_object := backend.get(key)!183		if !remote_object.found {184			continue185		}186		record := decode_save(remote_object.body)!187		mut kept := []FileRecord{}188		mut dropped := []string{}189		for file in record.files {190			if (every_save || record.id in note.saves) && covers(path, file.path) {191				freed[file.hash] = true192				dropped << file.hash193				continue194			}195			kept << file196			needed[file.hash] = true197		}198		if kept.len == record.files.len {199			continue200		}201		// Once rewritten, the record no longer names these objects; the note202		// keeps them so a retry can still delete them.203		queue_strip(store, path, []string{}, dropped)!204		if !backend.put_if_match(key, encode_save(rewrite(record, kept)).bytes(), remote_object.tag)! {205			return error('the remote copy of save ${record.id} moved while stripping — run koh steal, then strip again')206		}207		report.records++208	}209	for hash, _ in freed {210		if hash in needed || !storage.valid_hash(hash) || store.has_object(hash)! {211			continue212		}213		backend.delete(object_key(name, hash))!214		report.objects++215	}216	store.remove_state(pending_key(path))!217}218219// requeue_strip notes every valid recorded strip of `path` again, so that220// send_strip removes the path from remote records that still hold it.221pub fn requeue_strip(store &storage.Store, path string) ! {222	recorded := events.strip_events(store)!.filter(events.valid_strip(it) && it.path == path)223	if recorded.len == 0 {224		return error('this repo has no recorded strip of ${path}; run koh steal, then koh doctor')225	}226	queue_strip(store, path, recorded.map(it.save_id), []string{})!227}228229// queue_strip adds scopes and objects to the note send_strip works from,230// writing it only when something is new. Only object hashes are noted.231fn queue_strip(store &storage.Store, path string, scopes []string, hashes []string) ! {232	mut note := read_pending(store, path)!233	count := note.saves.len + note.hashes.len234	note.saves << scopes.filter(it !in note.saves)235	note.hashes << hashes.filter(it !in note.hashes && storage.valid_hash(it))236	if note.saves.len + note.hashes.len > count {237		store.set_state(pending_key(path), json2.encode(note))!238	}239}240241// unapplied_strips are the strip events on the remote this repo has not242// received yet. offer asks before uploading and again after, because a strip that lands243// in between would otherwise leave the stripped path in the bucket.244fn unapplied_strips(store &storage.Store, mut backend remote.Backend, repo string) ![]events.Event {245	mut known := map[string]bool{}246	for event in events.all(store)! {247		known[event.id] = true248	}249	mut strips := []events.Event{}250	for key in backend.list(events_prefix(repo))! {251		if os.base(key) in known {252			continue253		}254		body := backend.get(key)!255		if !body.found {256			continue257		}258		event := decode_event(body.body)!259		if events.valid_strip(event) && !ignored_strip(event) {260			strips << event261		}262	}263	return strips264}265266// rewrite returns the save record with a new file list.267fn rewrite(record SaveRecord, files []FileRecord) SaveRecord {268	return SaveRecord{269		...record270		files: files271	}272}273274// stripped_hashes is the content of the rows remove_path will delete.275fn stripped_hashes(store &storage.Store, path string, only_save string) ![]string {276	mut stmt :=277		store.db.prepare('SELECT DISTINCT hash FROM save_files WHERE ${match_clause(only_save)}')!278	defer {279		stmt.finalize()280	}281	bind_match(stmt, path, only_save)!282	mut hashes := []string{}283	for stmt.step()! {284		hashes << stmt.text(0)285	}286	return hashes287}288289// remove_path deletes the rows and counts them; the objects go afterwards, once nothing references them.290fn remove_path(store &storage.Store, path string, only_save string) !int {291	mut stmt := store.db.prepare('DELETE FROM save_files WHERE ${match_clause(only_save)}')!292	defer {293		stmt.finalize()294	}295	bind_match(stmt, path, only_save)!296	stmt.step()!297	return store.db.changes()298}299300// match_clause is `covers` in SQL. LIKE would ignore ASCII case and treat `_`301// and `%` as wildcards; a strip destroys, so the prefix is compared exactly.302fn match_clause(only_save string) string {303	scope := if only_save == '' { '' } else { ' AND save_id = ?3' }304	return '(path = ?1 OR substr(path, 1, length(?2)) = ?2)${scope}'305}306307// bind_match fills what match_clause names: ?1 the path, ?2 its folder prefix, ?3 the save.308fn bind_match(stmt &storage.Stmt, path string, only_save string) ! {309	stmt.bind_text(1, path)!310	stmt.bind_text(2, '${path.trim_right('/')}/')!311	if only_save != '' {312		stmt.bind_text(3, only_save)!313	}314}