objects.v

1//! The object table: content-addressed file bytes.2//!3//! Objects are Blake3-addressed and stored zlib-compressed, exactly as Koh 24//! stored them on disk, so the same bytes migrate without a re-hash and a5//! future reader can decompress them with any zlib. Objects are shared between6//! saves: identical content is stored once.78module storage910import compress.zlib11import crypto.blake31213pub const max_file_size = 512 * 1024 * 10241415pub fn hash_hex(data []u8) string {16	return blake3.sum256(data).hex()17}1819// put_object stores `data` and returns its hash. Writing an object that is20// already present is a no-op, which is what makes repeated saves cheap.21pub fn (s &Store) put_object(data []u8) !string {22	hash := hash_hex(data)23	if s.has_object(hash)! {24		return hash25	}26	packed := zlib.compress(data)!27	mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')!28	defer {29		stmt.finalize()30	}31	stmt.bind_text(1, hash)!32	stmt.bind_int(2, i64(data.len))!33	stmt.bind_blob(3, packed)!34	stmt.step()!35	return hash36}3738// object_blob returns the bytes exactly as they sit in the table — the39// zlib-compressed form a remote receives.40pub fn (s &Store) object_blob(hash string) ![]u8 {41	mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')!42	defer {43		stmt.finalize()44	}45	stmt.bind_text(1, hash)!46	if !stmt.step()! {47		return error('object ${hash} is missing')48	}49	return stmt.blob(0)50}5152// put_object_blob stores a blob that came from a remote. The content is53// decompressed to check it hashes back to its key, and the blob itself is kept54// as it arrived.55pub fn (s &Store) put_object_blob(hash string, blob []u8) ! {56	if s.has_object(hash)! {57		return58	}59	data := zlib.decompress(blob) or { return error('object ${hash} is damaged: ${err.msg()}') }60	if hash_hex(data) != hash {61		return error('object ${hash} is damaged: it does not match its hash')62	}63	mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')!64	defer {65		stmt.finalize()66	}67	stmt.bind_text(1, hash)!68	stmt.bind_int(2, i64(data.len))!69	stmt.bind_blob(3, blob)!70	stmt.step()!71}7273pub fn (s &Store) has_object(hash string) !bool {74	mut stmt := s.db.prepare('SELECT 1 FROM objects WHERE hash = ?')!75	defer {76		stmt.finalize()77	}78	stmt.bind_text(1, hash)!79	return stmt.step()!80}8182// get_object returns the uncompressed bytes and refuses anything that does not83// hash back to the key it was asked for.84pub fn (s &Store) get_object(hash string) ![]u8 {85	mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')!86	defer {87		stmt.finalize()88	}89	stmt.bind_text(1, hash)!90	if !stmt.step()! {91		return error('object ${hash} is missing')92	}93	packed := stmt.blob(0)94	data := zlib.decompress(packed)!95	if hash_hex(data) != hash {96		return error('object ${hash} does not match its hash')97	}98	return data99}100101pub fn (s &Store) object_size(hash string) !i64 {102	mut stmt := s.db.prepare('SELECT size FROM objects WHERE hash = ?')!103	defer {104		stmt.finalize()105	}106	stmt.bind_text(1, hash)!107	if !stmt.step()! {108		return error('object ${hash} is missing')109	}110	return stmt.int(0)111}112113pub fn (s &Store) object_count() !i64 {114	mut stmt := s.db.prepare('SELECT COUNT(*) FROM objects')!115	defer {116		stmt.finalize()117	}118	if !stmt.step()! {119		return 0120	}121	return stmt.int(0)122}123124// drop_unreferenced_objects removes objects no save points at any more. Only125// `strip` leaves objects behind, so this is what makes a strip permanent.126pub fn (s &Store) drop_unreferenced_objects() !i64 {127	before := s.object_count()!128	mut stmt := s.db.prepare('DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)')!129	defer {130		stmt.finalize()131	}132	stmt.step()!133	after := s.object_count()!134	return before - after135}136137// drop_unused removes those of `hashes` no save points at any more. Other138// unused objects stay: with damaged rows one may be the only copy of saved139// content, and Doctor offers their removal separately.140pub fn (s &Store) drop_unused(hashes []string) !i64 {141	mut stmt := s.db.prepare('DELETE FROM objects WHERE hash = ?1142		AND NOT EXISTS (SELECT 1 FROM save_files WHERE hash = ?1)')!143	defer {144		stmt.finalize()145	}146	mut dropped := i64(0)147	for hash in hashes {148		stmt.reset()149		stmt.bind_text(1, hash)!150		stmt.step()!151		dropped += s.db.changes()152	}153	return dropped154}155156/// An object Doctor found damaged or missing.157pub struct ObjectProblem {158pub:159	hash   string160	detail string161}162163/// Verify every stored object's content hash and declared size.164pub fn (s &Store) check_objects() ![]ObjectProblem {165	mut problems := []ObjectProblem{}166	mut stmt := s.db.prepare('SELECT hash FROM objects ORDER BY hash')!167	defer {168		stmt.finalize()169	}170	mut hashes := []string{}171	for stmt.step()! {172		hashes << stmt.text(0)173	}174	for hash in hashes {175		data := s.get_object(hash) or {176			problems << ObjectProblem{hash, 'object ${hash}: ${err.msg()}'}177			continue178		}179		if s.object_size(hash)! != data.len {180			problems << ObjectProblem{hash, 'object ${hash}: stored size does not match ${data.len} bytes'}181		}182	}183	return problems184}185186/// Find missing objects, including damage from writes with foreign keys disabled.187pub fn (s &Store) check_save_files() ![]ObjectProblem {188	mut problems := []ObjectProblem{}189	mut stmt := s.db.prepare('SELECT DISTINCT f.save_id, f.hash FROM save_files f190		LEFT JOIN objects o ON o.hash = f.hash WHERE o.hash IS NULL')!191	defer {192		stmt.finalize()193	}194	for stmt.step()! {195		problems << ObjectProblem{stmt.text(1), 'save ${stmt.text(0)} needs missing object ${stmt.text(1)}'}196	}197	return problems198}199200/// Store verified bytes for `hash`, replacing a damaged row in place so the201/// saves that refer to it keep their rows.202pub fn (s &Store) repair_object(hash string, data []u8) ! {203	if hash_hex(data) != hash {204		return error('the copy of object ${hash} does not match its hash')205	}206	packed := zlib.compress(data)!207	mut stmt := s.db.prepare('INSERT INTO objects (hash, size, data) VALUES (?, ?, ?)208		ON CONFLICT(hash) DO UPDATE SET size = excluded.size, data = excluded.data')!209	defer {210		stmt.finalize()211	}212	stmt.bind_text(1, hash)!213	stmt.bind_int(2, i64(data.len))!214	stmt.bind_blob(3, packed)!215	stmt.step()!216}217218/// Saved paths whose content is `hash`, where a working copy may still exist.219pub fn (s &Store) paths_of_object(hash string) ![]string {220	mut stmt := s.db.prepare('SELECT DISTINCT path FROM save_files WHERE hash = ?')!221	defer {222		stmt.finalize()223	}224	stmt.bind_text(1, hash)!225	mut paths := []string{}226	for stmt.step()! {227		paths << stmt.text(0)228	}229	return paths230}