//! The object table: content-addressed file bytes. //! //! Objects are Blake3-addressed and stored zlib-compressed, exactly as Koh 2 //! stored them on disk, so the same bytes migrate without a re-hash and a //! future reader can decompress them with any zlib. Objects are shared between //! saves: identical content is stored once. module storage import compress.zlib import crypto.blake3 pub const max_file_size = 512 * 1024 * 1024 pub fn hash_hex(data []u8) string { return blake3.sum256(data).hex() } // put_object stores `data` and returns its hash. Writing an object that is // already present is a no-op, which is what makes repeated saves cheap. pub fn (s &Store) put_object(data []u8) !string { hash := hash_hex(data) if s.has_object(hash)! { return hash } packed := zlib.compress(data)! mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')! defer { stmt.finalize() } stmt.bind_text(1, hash)! stmt.bind_int(2, i64(data.len))! stmt.bind_blob(3, packed)! stmt.step()! return hash } // object_blob returns the bytes exactly as they sit in the table — the // zlib-compressed form a remote receives. pub fn (s &Store) object_blob(hash string) ![]u8 { mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')! defer { stmt.finalize() } stmt.bind_text(1, hash)! if !stmt.step()! { return error('object ${hash} is missing') } return stmt.blob(0) } // put_object_blob stores a blob that came from a remote. The content is // decompressed to check it hashes back to its key, and the blob itself is kept // as it arrived. pub fn (s &Store) put_object_blob(hash string, blob []u8) ! { if s.has_object(hash)! { return } data := zlib.decompress(blob) or { return error('object ${hash} is damaged: ${err.msg()}') } if hash_hex(data) != hash { return error('object ${hash} is damaged: it does not match its hash') } mut stmt := s.db.prepare('INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)')! defer { stmt.finalize() } stmt.bind_text(1, hash)! stmt.bind_int(2, i64(data.len))! stmt.bind_blob(3, blob)! stmt.step()! } pub fn (s &Store) has_object(hash string) !bool { mut stmt := s.db.prepare('SELECT 1 FROM objects WHERE hash = ?')! defer { stmt.finalize() } stmt.bind_text(1, hash)! return stmt.step()! } // get_object returns the uncompressed bytes and refuses anything that does not // hash back to the key it was asked for. pub fn (s &Store) get_object(hash string) ![]u8 { mut stmt := s.db.prepare('SELECT data FROM objects WHERE hash = ?')! defer { stmt.finalize() } stmt.bind_text(1, hash)! if !stmt.step()! { return error('object ${hash} is missing') } packed := stmt.blob(0) data := zlib.decompress(packed)! if hash_hex(data) != hash { return error('object ${hash} does not match its hash') } return data } pub fn (s &Store) object_size(hash string) !i64 { mut stmt := s.db.prepare('SELECT size FROM objects WHERE hash = ?')! defer { stmt.finalize() } stmt.bind_text(1, hash)! if !stmt.step()! { return error('object ${hash} is missing') } return stmt.int(0) } pub fn (s &Store) object_count() !i64 { mut stmt := s.db.prepare('SELECT COUNT(*) FROM objects')! defer { stmt.finalize() } if !stmt.step()! { return 0 } return stmt.int(0) } // drop_unreferenced_objects removes objects no save points at any more. Only // `strip` leaves objects behind, so this is what makes a strip permanent. pub fn (s &Store) drop_unreferenced_objects() !i64 { before := s.object_count()! mut stmt := s.db.prepare('DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)')! defer { stmt.finalize() } stmt.step()! after := s.object_count()! return before - after } // drop_unused removes those of `hashes` no save points at any more. Other // unused objects stay: with damaged rows one may be the only copy of saved // content, and Doctor offers their removal separately. pub fn (s &Store) drop_unused(hashes []string) !i64 { mut stmt := s.db.prepare('DELETE FROM objects WHERE hash = ?1 AND NOT EXISTS (SELECT 1 FROM save_files WHERE hash = ?1)')! defer { stmt.finalize() } mut dropped := i64(0) for hash in hashes { stmt.reset() stmt.bind_text(1, hash)! stmt.step()! dropped += s.db.changes() } return dropped } /// An object Doctor found damaged or missing. pub struct ObjectProblem { pub: hash string detail string } /// Verify every stored object's content hash and declared size. pub fn (s &Store) check_objects() ![]ObjectProblem { mut problems := []ObjectProblem{} mut stmt := s.db.prepare('SELECT hash FROM objects ORDER BY hash')! defer { stmt.finalize() } mut hashes := []string{} for stmt.step()! { hashes << stmt.text(0) } for hash in hashes { data := s.get_object(hash) or { problems << ObjectProblem{hash, 'object ${hash}: ${err.msg()}'} continue } if s.object_size(hash)! != data.len { problems << ObjectProblem{hash, 'object ${hash}: stored size does not match ${data.len} bytes'} } } return problems } /// Find missing objects, including damage from writes with foreign keys disabled. pub fn (s &Store) check_save_files() ![]ObjectProblem { mut problems := []ObjectProblem{} mut stmt := s.db.prepare('SELECT DISTINCT f.save_id, f.hash FROM save_files f LEFT JOIN objects o ON o.hash = f.hash WHERE o.hash IS NULL')! defer { stmt.finalize() } for stmt.step()! { problems << ObjectProblem{stmt.text(1), 'save ${stmt.text(0)} needs missing object ${stmt.text(1)}'} } return problems } /// Store verified bytes for `hash`, replacing a damaged row in place so the /// saves that refer to it keep their rows. pub fn (s &Store) repair_object(hash string, data []u8) ! { if hash_hex(data) != hash { return error('the copy of object ${hash} does not match its hash') } packed := zlib.compress(data)! mut stmt := s.db.prepare('INSERT INTO objects (hash, size, data) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET size = excluded.size, data = excluded.data')! defer { stmt.finalize() } stmt.bind_text(1, hash)! stmt.bind_int(2, i64(data.len))! stmt.bind_blob(3, packed)! stmt.step()! } /// Saved paths whose content is `hash`, where a working copy may still exist. pub fn (s &Store) paths_of_object(hash string) ![]string { mut stmt := s.db.prepare('SELECT DISTINCT path FROM save_files WHERE hash = ?')! defer { stmt.finalize() } stmt.bind_text(1, hash)! mut paths := []string{} for stmt.step()! { paths << stmt.text(0) } return paths }