objects.v

1//! The object table: content-addressed file bytes.2//!3//! Objects are Blake3-addressed and stored zlib-compressed, exactly as Koh 24//! stored them on disk, so the same bytes migrate without a re-hash and a5//! future reader can decompress them with any zlib. Objects are shared between6//! saves: identical content is stored once.78module storage910import compress.zlib11import crypto.blake31213pub const max_file_size = 512 * 1024 * 10241415pub fn hash_hex(data []u8) string {16	return blake3.sum256(data).hex()17}1819// put_object stores `data` and returns its hash. Writing an object that is20// already present is a no-op, which is what makes repeated saves cheap.21pub fn (s &Store) put_object(data []u8) !string {22	hash := hash_hex(data)23	if s.has_object(hash)! {24		return hash25	}26	packed := zlib.compress(data)!27	mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')!28	defer {29		stmt.finalize()30	}31	stmt.bind_text(1, hash)!32	stmt.bind_int(2, i64(data.len))!33	stmt.bind_blob(3, packed)!34	stmt.step()!35	return hash36}3738// object_blob returns the bytes exactly as they sit in the table — the39// zlib-compressed form a remote receives.40pub fn (s &Store) object_blob(hash string) ![]u8 {41	mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')!42	defer {43		stmt.finalize()44	}45	stmt.bind_text(1, hash)!46	if !stmt.step()! {47		return error('object ${hash} is missing')48	}49	return stmt.blob(0)50}5152// put_object_blob stores a blob that came from a remote. The content is53// decompressed to check it hashes back to its key, and the blob itself is kept54// as it arrived.55pub fn (s &Store) put_object_blob(hash string, blob []u8) ! {56	if s.has_object(hash)! {57		return58	}59	data := zlib.decompress(blob) or { return error('object ${hash} is damaged: ${err.msg()}') }60	if hash_hex(data) != hash {61		return error('object ${hash} is damaged: it does not match its hash')62	}63	mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')!64	defer {65		stmt.finalize()66	}67	stmt.bind_text(1, hash)!68	stmt.bind_int(2, i64(data.len))!69	stmt.bind_blob(3, blob)!70	stmt.step()!71}7273pub fn (s &Store) has_object(hash string) !bool {74	mut stmt := s.db.prepare('SELECT 1 FROM objects WHERE hash = ?')!75	defer {76		stmt.finalize()77	}78	stmt.bind_text(1, hash)!79	return stmt.step()!80}8182// get_object returns the uncompressed bytes and refuses anything that does not83// hash back to the key it was asked for.84pub fn (s &Store) get_object(hash string) ![]u8 {85	mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')!86	defer {87		stmt.finalize()88	}89	stmt.bind_text(1, hash)!90	if !stmt.step()! {91		return error('object ${hash} is missing')92	}93	packed := stmt.blob(0)94	data := zlib.decompress(packed)!95	if hash_hex(data) != hash {96		return error('object ${hash} does not match its hash')97	}98	return data99}100101pub fn (s &Store) object_size(hash string) !i64 {102	mut stmt := s.db.prepare('SELECT size FROM objects WHERE hash = ?')!103	defer {104		stmt.finalize()105	}106	stmt.bind_text(1, hash)!107	if !stmt.step()! {108		return error('object ${hash} is missing')109	}110	return stmt.int(0)111}112113pub fn (s &Store) object_count() !i64 {114	mut stmt := s.db.prepare('SELECT COUNT(*) FROM objects')!115	defer {116		stmt.finalize()117	}118	if !stmt.step()! {119		return 0120	}121	return stmt.int(0)122}123124// drop_unreferenced_objects removes objects no save points at any more. Only125// `strip` leaves objects behind, so this is what makes a strip permanent.126pub fn (s &Store) drop_unreferenced_objects() !i64 {127	before := s.object_count()!128	mut stmt := s.db.prepare('DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)')!129	defer {130		stmt.finalize()131	}132	stmt.step()!133	after := s.object_count()!134	return before - after135}136137// valid_hash is the shape of an object key: 64 lowercase hex characters, so it138// can name nothing but an object.139pub fn valid_hash(hash string) bool {140	return hash.len == 64 && hash.bytes().all((it >= `0` && it <= `9`) || (it >= `a` && it <= `f`))141}142143// drop_unused removes those of `hashes` no save points at any more. Other144// unused objects stay: with damaged rows one may be the only copy of saved145// content, and Doctor offers their removal separately.146pub fn (s &Store) drop_unused(hashes []string) !i64 {147	mut stmt := s.db.prepare('DELETE FROM objects WHERE hash = ?1148		AND NOT EXISTS (SELECT 1 FROM save_files WHERE hash = ?1)')!149	defer {150		stmt.finalize()151	}152	mut dropped := i64(0)153	for hash in hashes {154		stmt.reset()155		stmt.bind_text(1, hash)!156		stmt.step()!157		dropped += s.db.changes()158	}159	return dropped160}161162/// An object Doctor found damaged or missing.163pub struct ObjectProblem {164pub:165	hash   string166	detail string167}168169/// Verify every stored object's content hash and declared size.170pub fn (s &Store) check_objects() ![]ObjectProblem {171	mut problems := []ObjectProblem{}172	mut stmt := s.db.prepare('SELECT hash FROM objects ORDER BY hash')!173	defer {174		stmt.finalize()175	}176	mut hashes := []string{}177	for stmt.step()! {178		hashes << stmt.text(0)179	}180	for hash in hashes {181		data := s.get_object(hash) or {182			problems << ObjectProblem{hash, 'object ${hash}: ${err.msg()}'}183			continue184		}185		if s.object_size(hash)! != data.len {186			problems << ObjectProblem{hash, 'object ${hash}: stored size does not match ${data.len} bytes'}187		}188	}189	return problems190}191192/// Find missing objects, including damage from writes with foreign keys disabled.193pub fn (s &Store) check_save_files() ![]ObjectProblem {194	mut problems := []ObjectProblem{}195	mut stmt := s.db.prepare('SELECT DISTINCT f.save_id, f.hash FROM save_files f196		LEFT JOIN objects o ON o.hash = f.hash WHERE o.hash IS NULL')!197	defer {198		stmt.finalize()199	}200	for stmt.step()! {201		problems << ObjectProblem{stmt.text(1), 'save ${stmt.text(0)} needs missing object ${stmt.text(1)}'}202	}203	return problems204}205206/// Store verified bytes for `hash`, replacing a damaged row in place so the207/// saves that refer to it keep their rows.208pub fn (s &Store) repair_object(hash string, data []u8) ! {209	if hash_hex(data) != hash {210		return error('the copy of object ${hash} does not match its hash')211	}212	packed := zlib.compress(data)!213	mut stmt := s.db.prepare('INSERT INTO objects (hash, size, data) VALUES (?, ?, ?)214		ON CONFLICT(hash) DO UPDATE SET size = excluded.size, data = excluded.data')!215	defer {216		stmt.finalize()217	}218	stmt.bind_text(1, hash)!219	stmt.bind_int(2, i64(data.len))!220	stmt.bind_blob(3, packed)!221	stmt.step()!222}223224/// Saved paths whose content is `hash`, where a working copy may still exist.225pub fn (s &Store) paths_of_object(hash string) ![]string {226	mut stmt := s.db.prepare('SELECT DISTINCT path FROM save_files WHERE hash = ?')!227	defer {228		stmt.finalize()229	}230	stmt.bind_text(1, hash)!231	mut paths := []string{}232	for stmt.step()! {233		paths << stmt.text(0)234	}235	return paths236}