store.ts

1import { Database } from "bun:sqlite";2import { mkdirSync } from "node:fs";3import { join } from "node:path";4import { inflateSync } from "node:zlib";5import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";6import { isTestPath } from "../describe/links.ts";78// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs,9// plus `releases`: each version's release.json (§11.2) as read from the bucket, since that is bucket10// data too and pages and feeds need it on every request, and `hidden_files`.11const SCHEMA_VERSION = 3;12// The operator's hidden rows, moved out of `save_files` rather than deleted, so a changed hidden13// list is applied in place, with nothing fetched again (2026-10-02).14const HIDDEN_FILES = `15CREATE TABLE hidden_files (16	save_id    TEXT NOT NULL REFERENCES saves(id),17	path       TEXT NOT NULL,18	hash       TEXT NOT NULL REFERENCES objects(hash),19	executable INTEGER NOT NULL DEFAULT 0,20	PRIMARY KEY (save_id, path)21);22`;23const SCHEMA = `24CREATE TABLE objects (25	hash TEXT PRIMARY KEY,26	size INTEGER NOT NULL,27	data BLOB NOT NULL28);29CREATE TABLE saves (30	id            TEXT PRIMARY KEY,31	parent        TEXT REFERENCES saves(id),32	message       TEXT NOT NULL,33	created_at    INTEGER NOT NULL,34	machine       TEXT NOT NULL,35	is_checkpoint INTEGER NOT NULL DEFAULT 0,36	size_bytes    INTEGER NOT NULL DEFAULT 0,37	agent         TEXT,38	session       TEXT,39	restores      TEXT,40	replayed_from TEXT41);42CREATE TABLE save_files (43	save_id    TEXT NOT NULL REFERENCES saves(id),44	path       TEXT NOT NULL,45	hash       TEXT NOT NULL REFERENCES objects(hash),46	executable INTEGER NOT NULL DEFAULT 0,47	PRIMARY KEY (save_id, path)48);49CREATE TABLE events (50	id         TEXT PRIMARY KEY,51	kind       TEXT NOT NULL,52	save_id    TEXT,53	path       TEXT,54	reason     TEXT,55	created_at INTEGER NOT NULL,56	machine    TEXT NOT NULL57);58CREATE TABLE state (59	key   TEXT PRIMARY KEY,60	value TEXT NOT NULL61);62CREATE TABLE releases (63	version TEXT PRIMARY KEY,64	record  TEXT NOT NULL65);66CREATE INDEX saves_parent ON saves(parent);67CREATE INDEX save_files_hash ON save_files(hash);68${HIDDEN_FILES}`;6970/** An object as the bucket stores it: zlib bytes, already checked against its hash. */71export type StoredObject = { compressed: Uint8Array; size: number };7273/** One batch of a sync, landed in one transaction. Saves are oldest first. */74export type Landing = {75	head: string;76	releasesLatest: string;77	saves: SaveRecord[];78	objects: Map<string, StoredObject>;79	events: EventRecord[];80	releases: ReleaseRecord[];81	/** Escaped names of the paths these saves arrived without (§5.5, A3). */82	quarantined: string[];83};8485/** Marks a hidden entry as a name matched at any depth rather than a path from the root. */86export const ANY_DEPTH = "**/";8788// A recount inflates every file at the newest save; a file this large is data, not lines of code.89const MAX_COUNTED = 8 * 1024 * 1024;9091// Display names for the header's language, by extension; data and docs formats never count.92const LANGUAGES: Record<string, string> = {93	ts: "TypeScript",94	tsx: "TypeScript",95	js: "JavaScript",96	mjs: "JavaScript",97	zig: "Zig",98	v: "V",99	rs: "Rust",100	go: "Go",101	py: "Python",102	c: "C",103	h: "C",104	cpp: "C++",105	cc: "C++",106	hpp: "C++",107	swift: "Swift",108	kt: "Kotlin",109	java: "Java",110	rb: "Ruby",111	lua: "Lua",112	sh: "Shell",113	nix: "Nix",114	css: "CSS",115	html: "HTML",116};117118// Vendored code is someone else's: counting Koh 3's bundled SQLite made it read as 290k lines of C.119const VENDORED = /(^|\/)(vendor|third_party|node_modules)\//;120// Bumped when what counts as a line changes, so every mirror recounts once when it opens.121const STATS_RULE = "source-only";122123const open = new Map<string, Database>();124125/** Closes a repo's mirror, so a rename can move its file. */126export function closeMirror(dir: string, repo: string): void {127	const path = join(dir, `${repo}.db`);128	open.get(path)?.close();129	open.delete(path);130}131132/** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */133export function openMirror(dir: string, repo: string): Database {134	const path = join(dir, `${repo}.db`);135	const cached = open.get(path);136	if (cached) return cached;137	mkdirSync(dir, { recursive: true });138	const db = new Database(path, { strict: true });139	db.run("PRAGMA journal_mode = WAL");140	db.run("PRAGMA foreign_keys = ON");141	db.run("PRAGMA busy_timeout = 5000");142	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {143		user_version: 0,144	};145	if (user_version !== 0 && user_version !== 2 && user_version !== SCHEMA_VERSION) {146		db.close();147		throw new Error(148			`${path} has mirror schema version ${user_version}; this Kepr reads version ${SCHEMA_VERSION}. ` +149				"The mirror is disposable: delete it and the next sync rebuilds it from the bucket.",150		);151	}152	if (user_version === 0) {153		db.transaction(() => {154			db.run(SCHEMA);155			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);156		})();157	}158	if (user_version === 2) {159		db.transaction(() => {160			db.run(HIDDEN_FILES);161			// Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.162			if (mirrorState(db, "hidden")) clearMirror(db);163			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);164		})();165	}166	// Mirrors counted under an older rule are recounted now, not at the next save.167	if (mirrorState(db, "head") && mirrorState(db, "stats_rule") !== STATS_RULE) refreshStats(db);168	open.set(path, db);169	return db;170}171172export function mirrorState(db: Database, key: string): string | null {173	const row = db174		.query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")175		.get(key);176	return row?.value ?? null;177}178179export function knownSave(db: Database, id: string): boolean {180	return db.query("SELECT 1 FROM saves WHERE id = ?").get(id) !== null;181}182183export function knownObject(db: Database, hash: string): boolean {184	return db.query("SELECT 1 FROM objects WHERE hash = ?").get(hash) !== null;185}186187export function knownEventIds(db: Database): Set<string> {188	const rows = db.query<{ id: string }, []>("SELECT id FROM events").all();189	return new Set(rows.map((row) => row.id));190}191192/** Drops a version deleted from the bucket; a sync only ever adds releases. */193export function forgetRelease(db: Database, version: string): void {194	db.query("DELETE FROM releases WHERE version = ?").run(version);195}196197/** Replaces a version's record after Kepr rewrote it in the bucket. */198export function replaceRelease(db: Database, record: ReleaseRecord): void {199	db.query("UPDATE releases SET record = ? WHERE version = ?").run(200		JSON.stringify(record),201		record.version,202	);203}204205export function knownReleases(db: Database): Set<string> {206	const rows = db.query<{ version: string }, []>("SELECT version FROM releases").all();207	return new Set(rows.map((row) => row.version));208}209210/** Stores objects on their own, ahead of the save that refers to them. */211export function storeObjects(db: Database, objects: Map<string, StoredObject>): void {212	const put = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");213	db.transaction(() => {214		for (const [hash, object] of objects) put.run(hash, object.size, object.compressed);215	})();216}217218/**219 * Lands a sync like Koh's steal: objects, events, saves, then every strip over every save. The new220 * saves are hidden by the list the mirror holds; `applyHidden` changes that list.221 */222export function land(db: Database, landing: Landing): void {223	const putEvent = db.query(224		"INSERT OR IGNORE INTO events (id, kind, save_id, path, reason, created_at, machine) VALUES " +225			"(?, ?, ?, ?, ?, ?, ?)",226	);227	const putSave = db.query(228		`INSERT OR IGNORE INTO saves (id, parent, message, created_at, machine, is_checkpoint, size_bytes,229			agent, session, restores, replayed_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,230	);231	const putFile = db.query(232		"INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",233	);234	const putRelease = db.query("INSERT OR IGNORE INTO releases (version, record) VALUES (?, ?)");235	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");236	db.transaction(() => {237		storeObjects(db, landing.objects);238		for (const e of landing.events) {239			putEvent.run(e.id, e.kind, e.save_id, e.path, e.reason, e.created_at, e.machine);240		}241		for (const s of landing.saves) {242			putSave.run(243				s.id,244				s.parent === "" ? null : s.parent,245				s.message,246				s.created_at,247				s.machine,248				s.is_checkpoint ? 1 : 0,249				s.size_bytes,250				s.agent,251				s.session,252				s.restores,253				s.replayed_from,254			);255			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);256		}257		for (const r of landing.releases) putRelease.run(r.version, JSON.stringify(r));258		applyStrips(db);259		hideRows(260			db,261			hiddenList(db),262			landing.saves.map((save) => save.id),263		);264		setState.run("head", landing.head);265		setState.run("releases_latest", landing.releasesLatest);266		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);267		setState.run("quarantined", JSON.stringify([...quarantined].sort()));268	})();269	// Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.270	refreshStats(db);271}272273/**274 * Makes `hidden` the mirror's hidden list: the operator's hidden paths (June, 2026-09-30) and the275 * names ignored in every repo (June, 2026-10-01), each as `ANY_DEPTH` + name. Rows move between276 * `save_files` and `hidden_files`, so it takes milliseconds and fetches nothing.277 */278export function applyHidden(db: Database, hidden: string[]): void {279	if (hiddenList(db).join("\n") === hidden.join("\n")) return;280	db.transaction(() => {281		db.run(282			`INSERT OR IGNORE INTO save_files (save_id, path, hash, executable)283			SELECT save_id, path, hash, executable FROM hidden_files`,284		);285		db.run("DELETE FROM hidden_files");286		hideRows(db, hidden, null);287	})();288	refreshStats(db);289}290291function hiddenList(db: Database): string[] {292	return mirrorState(db, "hidden")?.split("\n").filter(Boolean) ?? [];293}294295// Moves the rows `hidden` matches, in the `saves` given or (null) every save, into `hidden_files`.296// The list is recorded in the same transaction, so the tarball key never names a list the rows297// don't reflect yet.298function hideRows(db: Database, hidden: string[], saves: string[] | null): void {299	for (const entry of hidden) {300		const { where, params } = pathMatch(entry);301		const scope = { ...params, all: saves === null ? 1 : 0, saves: JSON.stringify(saves ?? []) };302		const chosen = `${where} AND ($all = 1 OR save_id IN (SELECT value FROM json_each($saves)))`;303		db.query(304			`INSERT OR IGNORE INTO hidden_files (save_id, path, hash, executable)305			SELECT save_id, path, hash, executable FROM save_files306			WHERE ${chosen}`,307		).run(scope);308		db.query(`DELETE FROM save_files WHERE ${chosen}`).run(scope);309	}310	db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('hidden', ?)").run(hidden.join("\n"));311}312313/** Escaped names of every path this mirror's saves arrived without. */314export function quarantinedPaths(db: Database): string[] {315	return JSON.parse(mirrorState(db, "quarantined") ?? "[]");316}317318// §10.2: a strip of one save reaches that save; a strip of every save bans its path from all of319// history, whenever a save was made or arrives. Re-applying every strip is harmless, which is how a320// save that arrives after its strip event is covered too.321function applyStrips(db: Database): void {322	const strips = db323		.query<{ save_id: string; path: string }, []>(324			"SELECT save_id, path FROM events WHERE kind = 'strip'",325		)326		.all();327	if (strips.length === 0) return;328	for (const strip of strips) {329		const { where, params } = pathMatch(strip.path);330		// Hidden rows too, so unhiding never brings back what a strip removed.331		for (const table of ["save_files", "hidden_files"]) {332			db.query(`DELETE FROM ${table} WHERE ${where} AND ($save = '' OR save_id = $save)`).run({333				...params,334				save: strip.save_id,335			});336		}337	}338	db.run(339		`DELETE FROM objects340		WHERE hash NOT IN (SELECT hash FROM save_files)341		AND hash NOT IN (SELECT hash FROM hidden_files)`,342	);343}344345/** The SQL condition matching a path, file or folder, or an `ANY_DEPTH` name at any depth. */346function pathMatch(path: string): { where: string; params: Record<string, string> } {347	if (path.startsWith(ANY_DEPTH)) {348		return {349			where: "instr('/' || path || '/', $segment) > 0",350			params: { segment: `/${path.slice(ANY_DEPTH.length)}/` },351		};352	}353	// Koh compares the prefix exactly, never with LIKE, so `_` and `%` in a path match only themselves.354	return {355		where: "(path = $path OR substr(path, 1, length($dir)) = $dir)",356		params: { path, dir: `${path.replace(/\/+$/, "")}/` },357	};358}359360/**361 * Empties a mirror in place so the next sync rebuilds it from the bucket, as the move to schema362 * version 3 needs when hidden rows were deleted. The file and its open handles stay, so a request holding one reads an empty363 * repo for a moment rather than failing. A new epoch keeps tarballs kept before it from matching.364 */365export function clearMirror(db: Database): void {366	db.transaction(() => {367		// Children before parents, for the foreign keys; every name is a table of SCHEMA's.368		const tables = [369			"save_files",370			"hidden_files",371			"events",372			"saves",373			"objects",374			"releases",375			"state",376		];377		for (const table of tables) {378			db.run(`DELETE FROM ${table}`);379		}380		db.query("INSERT INTO state (key, value) VALUES ('epoch', ?)").run(crypto.randomUUID());381	})();382}383384// Lines and languages never change between syncs, so they are counted once here, not per view. They385// come from the newest real save, not `head`: a checkpoint's files are not public history (§21.4).386// Lines are source lines: files in a known language, tests left out; docs and data never count.387function refreshStats(db: Database): void {388	const save =389		db390			.query<{ id: string }, []>(391				"SELECT id FROM saves WHERE is_checkpoint = 0 ORDER BY rowid DESC LIMIT 1",392			)393			.get()?.id ?? "";394	const files = db395		.query<{ path: string; size: number; data: Uint8Array }, [string]>(396			"SELECT f.path, o.size, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",397		)398		.all(save);399	let lines = 0;400	const byLanguage = new Map<string, number>();401	for (const file of files) {402		const language = LANGUAGES[file.path.split(".").pop() ?? ""];403		// Tests are not the program: counted, Koh 3's read as 18k lines of V where it has 11k.404		if (!language || VENDORED.test(file.path) || isTestPath(file.path)) continue;405		if (file.size > MAX_COUNTED) continue;406		const bytes = inflateSync(file.data);407		if (bytes.includes(0)) continue;408		let count = 0;409		for (const byte of bytes) if (byte === 10) count++;410		if (bytes.length > 0 && bytes[bytes.length - 1] !== 10) count++;411		lines += count;412		byLanguage.set(language, (byLanguage.get(language) ?? 0) + count);413	}414	const languages = [...byLanguage].sort((a, b) => b[1] - a[1]);415	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");416	setState.run("lines", String(lines));417	setState.run("language", languages[0]?.[0] ?? "");418	setState.run("languages", JSON.stringify(languages));419	setState.run("stats_save", save);420	setState.run("stats_rule", STATS_RULE);421}