store.ts

1import { Database } from "bun:sqlite";2import { mkdirSync } from "node:fs";3import { join } from "node:path";4import { inflateSync } from "node:zlib";5import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";6import { isTestPath } from "../describe/links.ts";78// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs,9// plus `releases`: each version's release.json (§11.2) as read from the bucket, since that is bucket10// data too and pages and feeds need it on every request.11const SCHEMA_VERSION = 2;12const SCHEMA = `13CREATE TABLE objects (14	hash TEXT PRIMARY KEY,15	size INTEGER NOT NULL,16	data BLOB NOT NULL17);18CREATE TABLE saves (19	id            TEXT PRIMARY KEY,20	parent        TEXT REFERENCES saves(id),21	message       TEXT NOT NULL,22	created_at    INTEGER NOT NULL,23	machine       TEXT NOT NULL,24	is_checkpoint INTEGER NOT NULL DEFAULT 0,25	size_bytes    INTEGER NOT NULL DEFAULT 0,26	agent         TEXT,27	session       TEXT,28	restores      TEXT,29	replayed_from TEXT30);31CREATE TABLE save_files (32	save_id    TEXT NOT NULL REFERENCES saves(id),33	path       TEXT NOT NULL,34	hash       TEXT NOT NULL REFERENCES objects(hash),35	executable INTEGER NOT NULL DEFAULT 0,36	PRIMARY KEY (save_id, path)37);38CREATE TABLE events (39	id         TEXT PRIMARY KEY,40	kind       TEXT NOT NULL,41	save_id    TEXT,42	path       TEXT,43	reason     TEXT,44	created_at INTEGER NOT NULL,45	machine    TEXT NOT NULL46);47CREATE TABLE state (48	key   TEXT PRIMARY KEY,49	value TEXT NOT NULL50);51CREATE TABLE releases (52	version TEXT PRIMARY KEY,53	record  TEXT NOT NULL54);55CREATE INDEX saves_parent ON saves(parent);56CREATE INDEX save_files_hash ON save_files(hash);57`;5859/** An object as the bucket stores it: zlib bytes, already checked against its hash. */60export type StoredObject = { compressed: Uint8Array; size: number };6162/** One batch of a sync, landed in one transaction. Saves are oldest first. */63export type Landing = {64	head: string;65	releasesLatest: string;66	saves: SaveRecord[];67	objects: Map<string, StoredObject>;68	events: EventRecord[];69	releases: ReleaseRecord[];70	/** Escaped names of the paths these saves arrived without (§5.5, A3). */71	quarantined: string[];72	/**73	 * The operator's hidden paths (June, 2026-09-30), removed from every save like a strip, and the74	 * names ignored in every repo (June, 2026-10-01), each as `ANY_DEPTH` + name.75	 */76	hidden: string[];77};7879/** Marks a hidden entry as a name matched at any depth rather than a path from the root. */80export const ANY_DEPTH = "**/";8182// A recount inflates every file at the newest save; a file this large is data, not lines of code.83const MAX_COUNTED = 8 * 1024 * 1024;8485// Display names for the header's language, by extension; data and docs formats never count.86const LANGUAGES: Record<string, string> = {87	ts: "TypeScript",88	tsx: "TypeScript",89	js: "JavaScript",90	mjs: "JavaScript",91	zig: "Zig",92	v: "V",93	rs: "Rust",94	go: "Go",95	py: "Python",96	c: "C",97	h: "C",98	cpp: "C++",99	cc: "C++",100	hpp: "C++",101	swift: "Swift",102	kt: "Kotlin",103	java: "Java",104	rb: "Ruby",105	lua: "Lua",106	sh: "Shell",107	nix: "Nix",108	css: "CSS",109	html: "HTML",110};111112// Vendored code is someone else's: counting Koh 3's bundled SQLite made it read as 290k lines of C.113const VENDORED = /(^|\/)(vendor|third_party|node_modules)\//;114// Bumped when what counts as a line changes, so every mirror recounts once when it opens.115const STATS_RULE = "source-only";116117const open = new Map<string, Database>();118119/** Closes a repo's mirror, so a rename can move its file. */120export function closeMirror(dir: string, repo: string): void {121	const path = join(dir, `${repo}.db`);122	open.get(path)?.close();123	open.delete(path);124}125126/** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */127export function openMirror(dir: string, repo: string): Database {128	const path = join(dir, `${repo}.db`);129	const cached = open.get(path);130	if (cached) return cached;131	mkdirSync(dir, { recursive: true });132	const db = new Database(path, { strict: true });133	db.run("PRAGMA journal_mode = WAL");134	db.run("PRAGMA foreign_keys = ON");135	db.run("PRAGMA busy_timeout = 5000");136	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {137		user_version: 0,138	};139	if (user_version !== 0 && user_version !== SCHEMA_VERSION) {140		db.close();141		throw new Error(142			`${path} has mirror schema version ${user_version}; this Kepr reads version ${SCHEMA_VERSION}. ` +143				"The mirror is disposable: delete it and the next sync rebuilds it from the bucket.",144		);145	}146	if (user_version === 0) {147		db.transaction(() => {148			db.run(SCHEMA);149			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);150		})();151	}152	// Mirrors counted under an older rule are recounted now, not at the next save.153	if (mirrorState(db, "head") && mirrorState(db, "stats_rule") !== STATS_RULE) refreshStats(db);154	open.set(path, db);155	return db;156}157158export function mirrorState(db: Database, key: string): string | null {159	const row = db160		.query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")161		.get(key);162	return row?.value ?? null;163}164165export function knownSave(db: Database, id: string): boolean {166	return db.query("SELECT 1 FROM saves WHERE id = ?").get(id) !== null;167}168169export function knownObject(db: Database, hash: string): boolean {170	return db.query("SELECT 1 FROM objects WHERE hash = ?").get(hash) !== null;171}172173export function knownEventIds(db: Database): Set<string> {174	const rows = db.query<{ id: string }, []>("SELECT id FROM events").all();175	return new Set(rows.map((row) => row.id));176}177178/** Drops a version deleted from the bucket; a sync only ever adds releases. */179export function forgetRelease(db: Database, version: string): void {180	db.query("DELETE FROM releases WHERE version = ?").run(version);181}182183/** Replaces a version's record after Kepr rewrote it in the bucket. */184export function replaceRelease(db: Database, record: ReleaseRecord): void {185	db.query("UPDATE releases SET record = ? WHERE version = ?").run(186		JSON.stringify(record),187		record.version,188	);189}190191export function knownReleases(db: Database): Set<string> {192	const rows = db.query<{ version: string }, []>("SELECT version FROM releases").all();193	return new Set(rows.map((row) => row.version));194}195196/** Lands a sync like Koh's steal: objects, events, saves, then every strip over every save. */197/** Stores objects on their own, ahead of the save that refers to them. */198export function storeObjects(db: Database, objects: Map<string, StoredObject>): void {199	const put = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");200	db.transaction(() => {201		for (const [hash, object] of objects) put.run(hash, object.size, object.compressed);202	})();203}204205export function land(db: Database, landing: Landing): void {206	const putEvent = db.query(207		"INSERT OR IGNORE INTO events (id, kind, save_id, path, reason, created_at, machine) VALUES " +208			"(?, ?, ?, ?, ?, ?, ?)",209	);210	const putSave = db.query(211		`INSERT OR IGNORE INTO saves (id, parent, message, created_at, machine, is_checkpoint, size_bytes,212			agent, session, restores, replayed_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,213	);214	const putFile = db.query(215		"INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",216	);217	const putRelease = db.query("INSERT OR IGNORE INTO releases (version, record) VALUES (?, ?)");218	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");219	db.transaction(() => {220		storeObjects(db, landing.objects);221		for (const e of landing.events) {222			putEvent.run(e.id, e.kind, e.save_id, e.path, e.reason, e.created_at, e.machine);223		}224		for (const s of landing.saves) {225			putSave.run(226				s.id,227				s.parent === "" ? null : s.parent,228				s.message,229				s.created_at,230				s.machine,231				s.is_checkpoint ? 1 : 0,232				s.size_bytes,233				s.agent,234				s.session,235				s.restores,236				s.replayed_from,237			);238			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);239		}240		for (const r of landing.releases) putRelease.run(r.version, JSON.stringify(r));241		applyStrips(db);242		let hidden = 0;243		for (const path of landing.hidden) hidden += removePath(db, path, "");244		if (hidden > 0) db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");245		// In the same transaction as the removal, so the tarball key never names a list the rows246		// don't reflect yet.247		setState.run("hidden", landing.hidden.join("\n"));248		setState.run("head", landing.head);249		setState.run("releases_latest", landing.releasesLatest);250		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);251		setState.run("quarantined", JSON.stringify([...quarantined].sort()));252	})();253	// Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.254	refreshStats(db);255}256257/** Escaped names of every path this mirror's saves arrived without. */258export function quarantinedPaths(db: Database): string[] {259	return JSON.parse(mirrorState(db, "quarantined") ?? "[]");260}261262// §10.2: a strip of one save reaches that save; a strip of every save bans its path from all of263// history, whenever a save was made or arrives. Re-applying every strip is harmless, which is how a264// save that arrives after its strip event is covered too.265function applyStrips(db: Database): void {266	const strips = db267		.query<{ save_id: string; path: string }, []>(268			"SELECT save_id, path FROM events WHERE kind = 'strip'",269		)270		.all();271	if (strips.length === 0) return;272	for (const strip of strips) removePath(db, strip.path, strip.save_id);273	db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");274}275276/** Removes a path, file or folder, from one save or ('') every save; returns the rows removed. */277function removePath(db: Database, path: string, save: string): number {278	if (path.startsWith(ANY_DEPTH)) {279		// A name ignored in every repo: any path with a segment of exactly that name.280		return db281			.query(282				`DELETE FROM save_files283				WHERE instr('/' || path || '/', $segment) > 0284				AND ($save = '' OR save_id = $save)`,285			)286			.run({ segment: `/${path.slice(ANY_DEPTH.length)}/`, save }).changes;287	}288	// Koh compares the prefix exactly, never with LIKE, so `_` and `%` in a path match only themselves.289	const dir = `${path.replace(/\/+$/, "")}/`;290	return db291		.query(292			`DELETE FROM save_files293			WHERE (path = $path OR substr(path, 1, length($dir)) = $dir)294			AND ($save = '' OR save_id = $save)`,295		)296		.run({ path, dir, save }).changes;297}298299/**300 * Empties a mirror in place so the next sync rebuilds it from the bucket, as a change to the301 * hidden paths needs. The file and its open handles stay, so a request holding one reads an empty302 * repo for a moment rather than failing. A new epoch keeps tarballs kept before it from matching.303 */304export function clearMirror(db: Database): void {305	db.transaction(() => {306		// Children before parents, for the foreign keys; every name is a table of SCHEMA's.307		for (const table of ["save_files", "events", "saves", "objects", "releases", "state"]) {308			db.run(`DELETE FROM ${table}`);309		}310		db.query("INSERT INTO state (key, value) VALUES ('epoch', ?)").run(crypto.randomUUID());311	})();312}313314// Lines and languages never change between syncs, so they are counted once here, not per view. They315// come from the newest real save, not `head`: a checkpoint's files are not public history (§21.4).316// Lines are source lines: files in a known language, tests left out; docs and data never count.317function refreshStats(db: Database): void {318	const save =319		db320			.query<{ id: string }, []>(321				"SELECT id FROM saves WHERE is_checkpoint = 0 ORDER BY rowid DESC LIMIT 1",322			)323			.get()?.id ?? "";324	const files = db325		.query<{ path: string; size: number; data: Uint8Array }, [string]>(326			"SELECT f.path, o.size, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",327		)328		.all(save);329	let lines = 0;330	const byLanguage = new Map<string, number>();331	for (const file of files) {332		const language = LANGUAGES[file.path.split(".").pop() ?? ""];333		// Tests are not the program: counted, Koh 3's read as 18k lines of V where it has 11k.334		if (!language || VENDORED.test(file.path) || isTestPath(file.path)) continue;335		if (file.size > MAX_COUNTED) continue;336		const bytes = inflateSync(file.data);337		if (bytes.includes(0)) continue;338		let count = 0;339		for (const byte of bytes) if (byte === 10) count++;340		if (bytes.length > 0 && bytes[bytes.length - 1] !== 10) count++;341		lines += count;342		byLanguage.set(language, (byLanguage.get(language) ?? 0) + count);343	}344	const languages = [...byLanguage].sort((a, b) => b[1] - a[1]);345	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");346	setState.run("lines", String(lines));347	setState.run("language", languages[0]?.[0] ?? "");348	setState.run("languages", JSON.stringify(languages));349	setState.run("stats_save", save);350	setState.run("stats_rule", STATS_RULE);351}