store.ts

1import { Database } from "bun:sqlite";2import { mkdirSync } from "node:fs";3import { join } from "node:path";4import { inflateSync } from "node:zlib";5import type { EventRecord, SaveRecord } from "../bucket/repo.ts";67// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs.8const SCHEMA = `9CREATE TABLE objects (10	hash TEXT PRIMARY KEY,11	size INTEGER NOT NULL,12	data BLOB NOT NULL13);14CREATE TABLE saves (15	id            TEXT PRIMARY KEY,16	parent        TEXT REFERENCES saves(id),17	message       TEXT NOT NULL,18	created_at    INTEGER NOT NULL,19	machine       TEXT NOT NULL,20	is_checkpoint INTEGER NOT NULL DEFAULT 0,21	size_bytes    INTEGER NOT NULL DEFAULT 0,22	agent         TEXT,23	session       TEXT,24	restores      TEXT,25	replayed_from TEXT26);27CREATE TABLE save_files (28	save_id    TEXT NOT NULL REFERENCES saves(id),29	path       TEXT NOT NULL,30	hash       TEXT NOT NULL REFERENCES objects(hash),31	executable INTEGER NOT NULL DEFAULT 0,32	PRIMARY KEY (save_id, path)33);34CREATE TABLE events (35	id         TEXT PRIMARY KEY,36	kind       TEXT NOT NULL,37	save_id    TEXT,38	path       TEXT,39	reason     TEXT,40	created_at INTEGER NOT NULL,41	machine    TEXT NOT NULL42);43CREATE TABLE state (44	key   TEXT PRIMARY KEY,45	value TEXT NOT NULL46);47CREATE INDEX saves_parent ON saves(parent);48CREATE INDEX save_files_hash ON save_files(hash);49`;5051/** An object as the bucket stores it: zlib bytes, already checked against its hash. */52export type StoredObject = { compressed: Uint8Array; size: number };5354/** What one sync brings in, landed in one transaction. Saves are oldest first. */55export type Landing = {56	head: string;57	saves: SaveRecord[];58	objects: Map<string, StoredObject>;59	events: EventRecord[];60};6162// Display names for the header's language, by extension; data and docs formats never count.63const LANGUAGES: Record<string, string> = {64	ts: "TypeScript",65	tsx: "TypeScript",66	js: "JavaScript",67	mjs: "JavaScript",68	zig: "Zig",69	v: "V",70	rs: "Rust",71	go: "Go",72	py: "Python",73	c: "C",74	h: "C",75	cpp: "C++",76	cc: "C++",77	hpp: "C++",78	swift: "Swift",79	kt: "Kotlin",80	java: "Java",81	rb: "Ruby",82	lua: "Lua",83	sh: "Shell",84	nix: "Nix",85	css: "CSS",86	html: "HTML",87};8889// Vendored code is someone else's: counting Koh 3's bundled SQLite made it read as 290k lines of C.90const VENDORED = /(^|\/)(vendor|third_party|node_modules)\//;9192const open = new Map<string, Database>();9394/** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */95export function openMirror(dir: string, repo: string): Database {96	const path = join(dir, `${repo}.db`);97	const cached = open.get(path);98	if (cached) return cached;99	mkdirSync(dir, { recursive: true });100	const db = new Database(path, { strict: true });101	db.run("PRAGMA journal_mode = WAL");102	db.run("PRAGMA foreign_keys = ON");103	db.run("PRAGMA busy_timeout = 5000");104	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {105		user_version: 0,106	};107	if (user_version !== 0 && user_version !== 1) {108		db.close();109		throw new Error(110			`${path} has mirror schema version ${user_version}; this Kepr reads version 1. The mirror is disposable: delete it and the next sync rebuilds it from the bucket.`,111		);112	}113	if (user_version === 0) {114		db.transaction(() => {115			db.run(SCHEMA);116			db.run("PRAGMA user_version = 1");117		})();118	}119	open.set(path, db);120	return db;121}122123export function mirrorState(db: Database, key: string): string | null {124	const row = db125		.query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")126		.get(key);127	return row?.value ?? null;128}129130export function knownSave(db: Database, id: string): boolean {131	return db.query("SELECT 1 FROM saves WHERE id = ?").get(id) !== null;132}133134export function knownObject(db: Database, hash: string): boolean {135	return db.query("SELECT 1 FROM objects WHERE hash = ?").get(hash) !== null;136}137138export function knownEventIds(db: Database): Set<string> {139	const rows = db.query<{ id: string }, []>("SELECT id FROM events").all();140	return new Set(rows.map((row) => row.id));141}142143/** Lands a sync like Koh's steal: objects, then events, then saves, then strips, all or nothing. */144export function land(db: Database, landing: Landing): void {145	const putObject = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");146	const putEvent = db.query(147		"INSERT OR IGNORE INTO events (id, kind, save_id, path, reason, created_at, machine) VALUES (?, ?, ?, ?, ?, ?, ?)",148	);149	const putSave = db.query(150		`INSERT OR IGNORE INTO saves (id, parent, message, created_at, machine, is_checkpoint, size_bytes,151			agent, session, restores, replayed_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,152	);153	const putFile = db.query(154		"INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",155	);156	db.transaction(() => {157		for (const [hash, object] of landing.objects)158			putObject.run(hash, object.size, object.compressed);159		for (const e of landing.events) {160			putEvent.run(e.id, e.kind, e.save_id, e.path, e.reason, e.created_at, e.machine);161		}162		for (const s of landing.saves) {163			putSave.run(164				s.id,165				s.parent === "" ? null : s.parent,166				s.message,167				s.created_at,168				s.machine,169				s.is_checkpoint ? 1 : 0,170				s.size_bytes,171				s.agent,172				s.session,173				s.restores,174				s.replayed_from,175			);176			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);177		}178		applyStrips(db);179		db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('head', ?)").run(landing.head);180		refreshStats(db, landing.head);181	})();182}183184// §10.2: a strip of one save reaches that save; a strip of every save reaches the saves made up to185// and including its own time, so a file saved again after the strip survives. Re-applying every186// strip is harmless, which lets a save that arrives after its strip event be covered too.187function applyStrips(db: Database): void {188	const strips = db189		.query<{ save_id: string; path: string; created_at: number }, []>(190			"SELECT save_id, path, created_at FROM events WHERE kind = 'strip'",191		)192		.all();193	if (strips.length === 0) return;194	const remove = db.query(195		`DELETE FROM save_files196		WHERE (path = $path OR substr(path, 1, length($dir)) = $dir)197		AND save_id IN (SELECT id FROM saves WHERE198			CASE WHEN $save = '' THEN created_at <= $at ELSE id = $save END)`,199	);200	for (const strip of strips) {201		// Koh compares the prefix exactly, never with LIKE, so `_` and `%` in a path match only themselves.202		const dir = `${strip.path.replace(/\/+$/, "")}/`;203		remove.run({ path: strip.path, dir, save: strip.save_id, at: strip.created_at });204	}205	db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");206}207208// Lines and language at `head` never change between syncs, so they are counted once here, not per view.209function refreshStats(db: Database, head: string): void {210	const files = db211		.query<{ path: string; data: Uint8Array }, [string]>(212			"SELECT f.path, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",213		)214		.all(head);215	let lines = 0;216	const byLanguage = new Map<string, number>();217	for (const file of files) {218		if (VENDORED.test(file.path)) continue;219		const bytes = inflateSync(file.data);220		if (bytes.includes(0)) continue;221		let count = 0;222		for (const byte of bytes) if (byte === 10) count++;223		if (bytes.length > 0 && bytes[bytes.length - 1] !== 10) count++;224		lines += count;225		const language = LANGUAGES[file.path.split(".").pop() ?? ""];226		if (language) byLanguage.set(language, (byLanguage.get(language) ?? 0) + count);227	}228	const language = [...byLanguage].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";229	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");230	setState.run("lines", String(lines));231	setState.run("language", language);232}