aab0488febf0

add kepr.db, the per-repo koh 3 mirror with its queries, and the two-minute sync loop that applies strip events and records the last sync (v3.0.6)

time
2026-09-26 16:23 UTC · 2 weeks ago
machine
Junes-MacBook-Pro.local
agent
claude-opus-5-5
session
kepr 3 implementation
parent
1972104e46af
10 files package.json+1 −1src/bucket/repo.ts+57 −3src/bucket/sync.ts+94 −0src/main.ts+10 −0src/mirror/queries.ts+101 −0src/mirror/store.ts+222 −0src/server/db.ts+78 −0tests/bucket.test.ts+28 −2tests/fixture.ts+71 −9tests/sync.test.ts+123 −0
package.json+1 −1
@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.5",3+	"version": "3.0.6",44 	"private": true,55 	"type": "module",66 	"scripts": {

view file

src/bucket/repo.ts+57 −3
@@ -21,6 +21,20 @@2121 	files: SaveFile[];2222 };2323 24+/** A flag, clear or strip event, as Koh 3 writes it to `<repo>/events/<id>` (§9.2, §10). */25+export type EventRecord = {26+	id: string;27+	kind: "flag" | "clear" | "strip";28+	save_id: string;29+	path: string;30+	reason: string;31+	created_at: number;32+	machine: string;33+};34+35+/** An object's stored zlib bytes and the inflated bytes they were verified from. */36+export type VerifiedObject = { compressed: Uint8Array; bytes: Uint8Array };37+2438 export class BucketError extends Error {}2539 2640 const SAVE_ID = /^[0-9a-f]{12}$/;@@ -90,7 +104,7 @@90104 	bucket: BucketReader,91105 	repo: string,92106 	hash: string,93-): Promise<Uint8Array> {107+): Promise<VerifiedObject> {94108 	requireRepo(repo);95109 	if (!OBJECT_HASH.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);96110 	const compressed = await bucket.get(`${repo}/objects/${hash}`);@@ -99,9 +113,32 @@99113 	if (bytesToHex(blake3(bytes)) !== hash) {100114 		throw new BucketError(`${repo}/objects/${hash} does not match its hash`);101115 	}102-	return bytes;116+	return { compressed, bytes };103117 }104118 119+export async function listEventIds(bucket: BucketReader, repo: string): Promise<string[]> {120+	requireRepo(repo);121+	const prefix = `${repo}/events/`;122+	// A folder remote's half-written temp files can show up in a listing; only IDs are events.123+	return (await bucket.list(prefix))124+		.map((key) => key.slice(prefix.length))125+		.filter((id) => SAVE_ID.test(id));126+}127+128+export async function readEvent(129+	bucket: BucketReader,130+	repo: string,131+	id: string,132+): Promise<EventRecord> {133+	requireRepo(repo);134+	requireSaveId(id);135+	const bytes = await bucket.get(`${repo}/events/${id}`);136+	if (bytes === null) throw new BucketError(`${repo}/events/${id} is missing`);137+	const record = parseEventRecord(JSON.parse(decoder.decode(bytes)));138+	if (record.id !== id) throw new BucketError(`${repo}/events/${id} holds event ${record.id}`);139+	return record;140+}141+105142 function parseSaveRecord(value: unknown): SaveRecord {106143 	const r = value as Record<string, unknown>;107144 	const valid =@@ -121,6 +158,23 @@121158 	return value as SaveRecord;122159 }123160 161+function parseEventRecord(value: unknown): EventRecord {162+	const r = value as Record<string, unknown>;163+	const valid =164+		typeof value === "object" &&165+		value !== null &&166+		typeof r.id === "string" &&167+		SAVE_ID.test(r.id) &&168+		["flag", "clear", "strip"].includes(r.kind as string) &&169+		isOptionalSaveId(r.save_id) &&170+		(r.path === "" || (typeof r.path === "string" && isSafePath(r.path))) &&171+		typeof r.reason === "string" &&172+		typeof r.machine === "string" &&173+		isCount(r.created_at);174+	if (!valid) throw new BucketError("event record does not match the Koh 3 schema");175+	return value as EventRecord;176+}177+124178 function isOptionalSaveId(value: unknown): boolean {125179 	return value === "" || (typeof value === "string" && SAVE_ID.test(value));126180 }@@ -130,7 +184,7 @@130184 }131185 132186 /** A repo-relative path that can't escape or alias: no empty, `.` or `..` segments, no backslashes or control characters. */133-function isSafePath(path: string): boolean {187+export function isSafePath(path: string): boolean {134188 	return (135189 		!path.includes("\\") &&136190 		!CONTROL_CHAR.test(path) &&

view file

src/bucket/sync.ts+94 −0
@@ -0,0 +1,94 @@1+import type { Database } from "bun:sqlite";2+import {3+	knownEventIds,4+	knownObject,5+	knownSave,6+	land,7+	mirrorState,8+	openMirror,9+	type StoredObject,10+} from "../mirror/store.ts";11+import { recordSync, registerRepo } from "../server/db.ts";12+import type { BucketReader } from "./reader.ts";13+import {14+	BucketError,15+	checkFormat,16+	type EventRecord,17+	listEventIds,18+	listRepos,19+	readEvent,20+	readHead,21+	readObject,22+	readSave,23+	type SaveRecord,24+} from "./repo.ts";25+26+const INTERVAL_MS = 2 * 60 * 1000;27+28+export type SyncTarget = { bucket: BucketReader; kepr: Database; mirrorDir: string };29+30+let running: Promise<void> | null = null;31+32+/** Syncs every repo in the bucket; the admin console's "sync now" calls this too. Never overlaps itself. */33+export function syncAll(target: SyncTarget): Promise<void> {34+	running ??= syncEach(target).finally(() => {35+		running = null;36+	});37+	return running;38+}39+40+export function startSyncLoop(target: SyncTarget): void {41+	const tick = () => syncAll(target).catch((error) => console.error(`kepr: sync failed: ${error}`));42+	tick();43+	setInterval(tick, INTERVAL_MS);44+}45+46+async function syncEach(target: SyncTarget): Promise<void> {47+	for (const repo of await listRepos(target.bucket)) {48+		registerRepo(target.kepr, repo);49+		let error: string | null = null;50+		try {51+			await syncRepo(target.bucket, openMirror(target.mirrorDir, repo), repo);52+		} catch (cause) {53+			error = cause instanceof Error ? cause.message : String(cause);54+		}55+		recordSync(target.kepr, repo, Math.floor(Date.now() / 1000), error);56+	}57+}58+59+/**60+ * Koh's steal flow (§9.4) against a mirror: walk back from the remote head to a save the mirror has,61+ * fetch what is missing, and land it in one transaction. Events are listed every time, since a62+ * flag or a strip arrives without moving `head`.63+ */64+export async function syncRepo(65+	bucket: BucketReader,66+	mirror: Database,67+	repo: string,68+): Promise<void> {69+	await checkFormat(bucket, repo);70+	const head = await readHead(bucket, repo);71+	if (head === null) return;72+	const saves: SaveRecord[] = [];73+	const objects = new Map<string, StoredObject>();74+	const seen = new Set<string>();75+	for (let cursor = head; cursor !== "" && !knownSave(mirror, cursor); ) {76+		if (seen.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`);77+		seen.add(cursor);78+		const save = await readSave(bucket, repo, cursor);79+		for (const file of save.files) {80+			if (objects.has(file.hash) || knownObject(mirror, file.hash)) continue;81+			const { compressed, bytes } = await readObject(bucket, repo, file.hash);82+			objects.set(file.hash, { compressed, size: bytes.length });83+		}84+		saves.push(save);85+		cursor = save.parent;86+	}87+	const known = knownEventIds(mirror);88+	const events: EventRecord[] = [];89+	for (const id of await listEventIds(bucket, repo)) {90+		if (!known.has(id)) events.push(await readEvent(bucket, repo, id));91+	}92+	if (saves.length === 0 && events.length === 0 && mirrorState(mirror, "head") === head) return;93+	land(mirror, { head, saves: saves.reverse(), objects, events });94+}

view file

src/main.ts+10 −0
@@ -1,8 +1,18 @@1+import { join } from "node:path";2+import { openBucket } from "./bucket/reader.ts";3+import { startSyncLoop } from "./bucket/sync.ts";14 import { createApp } from "./server/app.ts";25 import { ConfigError, loadConfig } from "./server/config.ts";6+import { openKeprDb } from "./server/db.ts";37 48 try {59 	const config = await loadConfig("kepr.toml");10+	const kepr = openKeprDb(config.dataDir);11+	startSyncLoop({12+		bucket: openBucket(config.bucket),13+		kepr,14+		mirrorDir: join(config.dataDir, "mirror"),15+	});616 	// Bun.serve reads the port from $PORT (default 3000).717 	const server = Bun.serve({ fetch: createApp(config).fetch });818 	console.log(`kepr listening on ${server.url}`);

view file

src/mirror/queries.ts+101 −0
@@ -0,0 +1,101 @@1+import type { Database } from "bun:sqlite";2+import { inflateSync } from "node:zlib";3+import { mirrorState } from "./store.ts";4+5+export type Save = {6+	id: string;7+	parent: string;8+	message: string;9+	created_at: number;10+	machine: string;11+	agent: string;12+	session: string;13+	is_checkpoint: boolean;14+};15+16+export type FileEntry = { path: string; hash: string; executable: boolean; size: number };17+18+export type RepoStats = { saves: number; lines: number; language: string; updated: number | null };19+20+const SAVE_COLUMNS = `id, COALESCE(parent, '') AS parent, message, created_at, machine,21+	COALESCE(agent, '') AS agent, COALESCE(session, '') AS session, is_checkpoint = 1 AS is_checkpoint`;22+const ID_PREFIX = /^[0-9a-f]{6,12}$/;23+24+// Checkpoints are working marks, not history (§21.4): every list and count below leaves them out.25+const NOT_CHECKPOINT = "is_checkpoint = 0";26+27+export function headId(db: Database): string | null {28+	return mirrorState(db, "head");29+}30+31+/** A page of saves, newest first. Rows land oldest first, so rowid is the order of the line. */32+export function history(db: Database, limit: number, offset: number): Save[] {33+	return db34+		.query<Save, [number, number]>(35+			`SELECT ${SAVE_COLUMNS} FROM saves WHERE ${NOT_CHECKPOINT} ORDER BY rowid DESC LIMIT ? OFFSET ?`,36+		)37+		.all(limit, offset)38+		.map(asSave);39+}40+41+/** Every save, oldest first, for the timeline. */42+export function timeline(db: Database): Save[] {43+	return db44+		.query<Save, []>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE ${NOT_CHECKPOINT} ORDER BY rowid`)45+		.all()46+		.map(asSave);47+}48+49+/** The save a unique prefix of 6–12 hex characters names (§6.1), or null when none or several do. */50+export function saveByPrefix(db: Database, prefix: string): Save | null {51+	if (!ID_PREFIX.test(prefix)) return null;52+	const rows = db53+		.query<Save, [string]>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE id LIKE ? || '%' LIMIT 2`)54+		.all(prefix);55+	return rows.length === 1 ? asSave(rows[0] as Save) : null;56+}57+58+export function filesAt(db: Database, saveId: string): FileEntry[] {59+	return db60+		.query<FileEntry, [string]>(61+			`SELECT f.path, f.hash, f.executable = 1 AS executable, o.size FROM save_files f62+			JOIN objects o ON o.hash = f.hash WHERE f.save_id = ? ORDER BY f.path`,63+		)64+		.all(saveId)65+		.map((file) => ({ ...file, executable: Boolean(file.executable) }));66+}67+68+export function fileAt(db: Database, saveId: string, path: string): FileEntry | null {69+	const file = db70+		.query<FileEntry, [string, string]>(71+			`SELECT f.path, f.hash, f.executable = 1 AS executable, o.size FROM save_files f72+			JOIN objects o ON o.hash = f.hash WHERE f.save_id = ? AND f.path = ?`,73+		)74+		.get(saveId, path);75+	return file ? { ...file, executable: Boolean(file.executable) } : null;76+}77+78+export function objectBytes(db: Database, hash: string): Uint8Array | null {79+	const row = db80+		.query<{ data: Uint8Array }, [string]>("SELECT data FROM objects WHERE hash = ?")81+		.get(hash);82+	return row ? new Uint8Array(inflateSync(row.data)) : null;83+}84+85+export function repoStats(db: Database): RepoStats {86+	const row = db87+		.query<{ saves: number; updated: number | null }, []>(88+			`SELECT COUNT(*) AS saves, MAX(created_at) AS updated FROM saves WHERE ${NOT_CHECKPOINT}`,89+		)90+		.get() ?? { saves: 0, updated: null };91+	return {92+		saves: row.saves,93+		updated: row.updated,94+		lines: Number(mirrorState(db, "lines") ?? 0),95+		language: mirrorState(db, "language") ?? "",96+	};97+}98+99+function asSave(row: Save): Save {100+	return { ...row, is_checkpoint: Boolean(row.is_checkpoint) };101+}

view file

src/mirror/store.ts+222 −0
@@ -0,0 +1,222 @@1+import { Database } from "bun:sqlite";2+import { mkdirSync } from "node:fs";3+import { join } from "node:path";4+import { inflateSync } from "node:zlib";5+import type { EventRecord, SaveRecord } from "../bucket/repo.ts";6+7+// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs.8+const SCHEMA = `9+CREATE TABLE objects (10+	hash TEXT PRIMARY KEY,11+	size INTEGER NOT NULL,12+	data BLOB NOT NULL13+);14+CREATE TABLE saves (15+	id            TEXT PRIMARY KEY,16+	parent        TEXT REFERENCES saves(id),17+	message       TEXT NOT NULL,18+	created_at    INTEGER NOT NULL,19+	machine       TEXT NOT NULL,20+	is_checkpoint INTEGER NOT NULL DEFAULT 0,21+	size_bytes    INTEGER NOT NULL DEFAULT 0,22+	agent         TEXT,23+	session       TEXT,24+	restores      TEXT,25+	replayed_from TEXT26+);27+CREATE TABLE save_files (28+	save_id    TEXT NOT NULL REFERENCES saves(id),29+	path       TEXT NOT NULL,30+	hash       TEXT NOT NULL REFERENCES objects(hash),31+	executable INTEGER NOT NULL DEFAULT 0,32+	PRIMARY KEY (save_id, path)33+);34+CREATE TABLE events (35+	id         TEXT PRIMARY KEY,36+	kind       TEXT NOT NULL,37+	save_id    TEXT,38+	path       TEXT,39+	reason     TEXT,40+	created_at INTEGER NOT NULL,41+	machine    TEXT NOT NULL42+);43+CREATE TABLE state (44+	key   TEXT PRIMARY KEY,45+	value TEXT NOT NULL46+);47+CREATE INDEX saves_parent ON saves(parent);48+CREATE INDEX save_files_hash ON save_files(hash);49+`;50+51+/** An object as the bucket stores it: zlib bytes, already checked against its hash. */52+export type StoredObject = { compressed: Uint8Array; size: number };53+54+/** What one sync brings in, landed in one transaction. Saves are oldest first. */55+export type Landing = {56+	head: string;57+	saves: SaveRecord[];58+	objects: Map<string, StoredObject>;59+	events: EventRecord[];60+};61+62+// Display names for the header's language, by extension; data and docs formats never count.63+const LANGUAGES: Record<string, string> = {64+	ts: "TypeScript",65+	tsx: "TypeScript",66+	js: "JavaScript",67+	mjs: "JavaScript",68+	zig: "Zig",69+	v: "V",70+	rs: "Rust",71+	go: "Go",72+	py: "Python",73+	c: "C",74+	h: "C",75+	cpp: "C++",76+	cc: "C++",77+	hpp: "C++",78+	swift: "Swift",79+	kt: "Kotlin",80+	java: "Java",81+	rb: "Ruby",82+	lua: "Lua",83+	sh: "Shell",84+	nix: "Nix",85+	css: "CSS",86+	html: "HTML",87+};88+89+const open = new Map<string, Database>();90+91+/** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */92+export function openMirror(dir: string, repo: string): Database {93+	const path = join(dir, `${repo}.db`);94+	const cached = open.get(path);95+	if (cached) return cached;96+	mkdirSync(dir, { recursive: true });97+	const db = new Database(path, { strict: true });98+	db.run("PRAGMA journal_mode = WAL");99+	db.run("PRAGMA foreign_keys = ON");100+	db.run("PRAGMA busy_timeout = 5000");101+	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {102+		user_version: 0,103+	};104+	if (user_version === 0) {105+		db.transaction(() => {106+			db.run(SCHEMA);107+			db.run("PRAGMA user_version = 1");108+		})();109+	}110+	open.set(path, db);111+	return db;112+}113+114+export function mirrorState(db: Database, key: string): string | null {115+	const row = db116+		.query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")117+		.get(key);118+	return row?.value ?? null;119+}120+121+export function knownSave(db: Database, id: string): boolean {122+	return db.query("SELECT 1 FROM saves WHERE id = ?").get(id) !== null;123+}124+125+export function knownObject(db: Database, hash: string): boolean {126+	return db.query("SELECT 1 FROM objects WHERE hash = ?").get(hash) !== null;127+}128+129+export function knownEventIds(db: Database): Set<string> {130+	const rows = db.query<{ id: string }, []>("SELECT id FROM events").all();131+	return new Set(rows.map((row) => row.id));132+}133+134+/** Lands a sync like Koh's steal: objects, then events, then saves, then strips, all or nothing. */135+export function land(db: Database, landing: Landing): void {136+	const putObject = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");137+	const putEvent = db.query(138+		"INSERT OR IGNORE INTO events (id, kind, save_id, path, reason, created_at, machine) VALUES (?, ?, ?, ?, ?, ?, ?)",139+	);140+	const putSave = db.query(141+		`INSERT OR IGNORE INTO saves (id, parent, message, created_at, machine, is_checkpoint, size_bytes,142+			agent, session, restores, replayed_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,143+	);144+	const putFile = db.query(145+		"INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",146+	);147+	db.transaction(() => {148+		for (const [hash, object] of landing.objects)149+			putObject.run(hash, object.size, object.compressed);150+		for (const e of landing.events) {151+			putEvent.run(e.id, e.kind, e.save_id, e.path, e.reason, e.created_at, e.machine);152+		}153+		for (const s of landing.saves) {154+			putSave.run(155+				s.id,156+				s.parent === "" ? null : s.parent,157+				s.message,158+				s.created_at,159+				s.machine,160+				s.is_checkpoint ? 1 : 0,161+				s.size_bytes,162+				s.agent,163+				s.session,164+				s.restores,165+				s.replayed_from,166+			);167+			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);168+		}169+		applyStrips(db);170+		db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('head', ?)").run(landing.head);171+		refreshStats(db, landing.head);172+	})();173+}174+175+// §10.2: a strip of one save reaches that save; a strip of every save reaches the saves made up to176+// and including its own time, so a file saved again after the strip survives. Re-applying every177+// strip is harmless, which lets a save that arrives after its strip event be covered too.178+function applyStrips(db: Database): void {179+	const strips = db180+		.query<{ save_id: string; path: string; created_at: number }, []>(181+			"SELECT save_id, path, created_at FROM events WHERE kind = 'strip'",182+		)183+		.all();184+	if (strips.length === 0) return;185+	const remove = db.query(186+		`DELETE FROM save_files187+		WHERE (path = $path OR substr(path, 1, length($dir)) = $dir)188+		AND save_id IN (SELECT id FROM saves WHERE189+			CASE WHEN $save = '' THEN created_at <= $at ELSE id = $save END)`,190+	);191+	for (const strip of strips) {192+		// Koh compares the prefix exactly, never with LIKE, so `_` and `%` in a path match only themselves.193+		const dir = `${strip.path.replace(/\/+$/, "")}/`;194+		remove.run({ path: strip.path, dir, save: strip.save_id, at: strip.created_at });195+	}196+	db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");197+}198+199+// Lines and language at `head` never change between syncs, so they are counted once here, not per view.200+function refreshStats(db: Database, head: string): void {201+	const files = db202+		.query<{ path: string; data: Uint8Array }, [string]>(203+			"SELECT f.path, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",204+		)205+		.all(head);206+	let lines = 0;207+	const byLanguage = new Map<string, number>();208+	for (const file of files) {209+		const bytes = inflateSync(file.data);210+		if (bytes.includes(0)) continue;211+		let count = 0;212+		for (const byte of bytes) if (byte === 10) count++;213+		if (bytes.length > 0 && bytes[bytes.length - 1] !== 10) count++;214+		lines += count;215+		const language = LANGUAGES[file.path.split(".").pop() ?? ""];216+		if (language) byLanguage.set(language, (byLanguage.get(language) ?? 0) + count);217+	}218+	const language = [...byLanguage].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";219+	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");220+	setState.run("lines", String(lines));221+	setState.run("language", language);222+}

view file

src/server/db.ts+78 −0
@@ -0,0 +1,78 @@1+import { Database } from "bun:sqlite";2+import { mkdirSync } from "node:fs";3+import { join } from "node:path";4+5+// §19: only what the pages and the sync loop read so far; later phases add their tables.6+const SCHEMA = `7+CREATE TABLE IF NOT EXISTS settings (8+	key   TEXT PRIMARY KEY,9+	value TEXT NOT NULL10+);11+CREATE TABLE IF NOT EXISTS repos (12+	name            TEXT PRIMARY KEY,13+	public          INTEGER NOT NULL DEFAULT 0,14+	category        TEXT NOT NULL DEFAULT '',15+	homepage        TEXT NOT NULL DEFAULT '',16+	sort_order      INTEGER NOT NULL DEFAULT 0,17+	install1        TEXT NOT NULL DEFAULT '',18+	install2        TEXT NOT NULL DEFAULT '',19+	install3        TEXT NOT NULL DEFAULT '',20+	last_sync_at    INTEGER,21+	last_sync_error TEXT22+);23+`;24+25+export type RepoSettings = {26+	name: string;27+	homepage: string;28+	installLines: string[];29+};30+31+type RepoRow = {32+	name: string;33+	homepage: string;34+	install1: string;35+	install2: string;36+	install3: string;37+};38+39+export function openKeprDb(dataDir: string): Database {40+	mkdirSync(dataDir, { recursive: true });41+	const db = new Database(join(dataDir, "kepr.db"), { strict: true });42+	db.run("PRAGMA journal_mode = WAL");43+	db.run("PRAGMA busy_timeout = 5000");44+	db.run(SCHEMA);45+	return db;46+}47+48+export function setting(db: Database, key: string): string | null {49+	const row = db50+		.query<{ value: string }, [string]>("SELECT value FROM settings WHERE key = ?")51+		.get(key);52+	return row?.value ?? null;53+}54+55+/** Every repo starts private (§20); registering a known repo changes nothing. */56+export function registerRepo(db: Database, name: string): void {57+	db.query("INSERT OR IGNORE INTO repos (name) VALUES (?)").run(name);58+}59+60+export function recordSync(db: Database, name: string, at: number, error: string | null): void {61+	db.query("UPDATE repos SET last_sync_at = ?, last_sync_error = ? WHERE name = ?").run(62+		at,63+		error,64+		name,65+	);66+}67+68+/** The settings of a public repo, or null when it is private or unknown: both must look missing. */69+export function publicRepo(db: Database, name: string): RepoSettings | null {70+	const row = db71+		.query<RepoRow, [string]>(72+			"SELECT name, homepage, install1, install2, install3 FROM repos WHERE name = ? AND public = 1",73+		)74+		.get(name);75+	if (!row) return null;76+	const installLines = [row.install1, row.install2, row.install3].filter((line) => line !== "");77+	return { name: row.name, homepage: row.homepage, installLines };78+}

view file

tests/bucket.test.ts+28 −2
@@ -2,7 +2,15 @@22 import { writeFile } from "node:fs/promises";33 import { join } from "node:path";44 import { openBucket } from "../src/bucket/reader.ts";5-import { checkFormat, listRepos, readHead, readObject, readSave } from "../src/bucket/repo.ts";5+import {6+	checkFormat,7+	listEventIds,8+	listRepos,9+	readEvent,10+	readHead,11+	readObject,12+	readSave,13+} from "../src/bucket/repo.ts";614 import { type Fixture, makeKohBucket } from "./fixture.ts";715 816 let fx: Fixture;@@ -32,7 +40,7 @@3240 		expect([first.agent, first.session]).toEqual(["test-agent", "fixture"]);3341 3442 		const main = save.files.find((file) => file.path === "src/main.ts");35-		const bytes = await readObject(b, fx.repo, main?.hash as string);43+		const { bytes } = await readObject(b, fx.repo, main?.hash as string);3644 		expect(new TextDecoder().decode(bytes)).toBe("// Entry point.\nconsole.log('hello');\n");3745 	});3846 @@ -175,6 +183,24 @@175183 		}176184 		for (const id of ["../head", "abc", "ABCDEF123456", `${fx.saveIds[0]}0`]) {177185 			await expect(readSave(b, fx.repo, id)).rejects.toThrow("not a save ID");186+		}187+	});188+});189+190+describe("event records", () => {191+	test("reads a genuine koh3 flag event and refuses a tampered one", async () => {192+		const b = bucket();193+		const [id] = await listEventIds(b, fx.repo);194+		const event = await readEvent(b, fx.repo, id as string);195+		expect([event.kind, event.save_id, event.reason]).toEqual([196+			"flag",197+			fx.saveIds[3] as string,198+			"holds a secret",199+		]);200+		for (const change of [{ kind: "delete" }, { path: "../x" }, { save_id: "nope" }]) {201+			const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));202+			const tampered = { get: async () => bytes, list: async () => [] };203+			await expect(readEvent(tampered, fx.repo, event.id)).rejects.toThrow("Koh 3 schema");178204 		}179205 	});180206 });

view file

tests/fixture.ts+71 −9
@@ -2,21 +2,62 @@22 import { tmpdir } from "node:os";33 import { join } from "node:path";44 5-export type Fixture = { bucketDir: string; repo: string; repoDir: string; saveIds: string[] };5+export type Fixture = {6+	bucketDir: string;7+	repo: string;8+	repoDir: string;9+	/** Every save, newest first, checkpoint included. */10+	saveIds: string[];11+	/** Strips a path from every save with koh3, so tests can sync before and after. */12+	strip(path: string): Promise<void>;13+};614 7-/** Builds a genuine Koh 3 folder remote with koh3 itself, under a throwaway HOME. */15+// The smallest valid PNG (1×1), so the fixture has a real binary file.16+const PNG = Uint8Array.from(17+	atob(18+		"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",19+	),20+	(c) => c.charCodeAt(0),21+);22+23+const README = `# Demo24+25+A small repo for **Kepr's** tests.26+27+See [the guide](docs/guide.md), [the site](https://example.com) and [a trap](javascript:alert(1)).28+29+![logo](docs/logo.png)30+31+<script>alert("raw html")</script>32+`;33+34+/**35+ * Builds a genuine Koh 3 folder remote with koh3 itself, under a throwaway HOME. The line is:36+ * "add docs, a secret and a deep helper" (flagged) → a second later "add readme and entry point"37+ * (agent and session) → a checkpoint → "say hello" (head).38+ */839 export async function makeKohBucket(repo = "demo"): Promise<Fixture> {940 	const root = await mkdtemp(join(tmpdir(), "kepr-fixture-"));1041 	const home = join(root, "home");1142 	const bucketDir = join(root, "bucket");1243 	const repoDir = join(root, "work", repo);1344 	await mkdir(home);14-	await mkdir(join(repoDir, "src"), { recursive: true });45+	await mkdir(join(repoDir, "src/lib/deep"), { recursive: true });46+	await mkdir(join(repoDir, "docs"));1547 	const koh = (...args: string[]) => run(repoDir, home, args);1648 1749 	koh("init");1850 	koh("remote", "folder", bucketDir);19-	await writeFile(join(repoDir, "README.md"), "# Demo\n\nA small repo for Kepr's tests.\n");51+	await writeFile(join(repoDir, "docs/guide.md"), "# Guide\n\nRead me.\n");52+	await writeFile(join(repoDir, "docs/logo.png"), PNG);53+	await writeFile(join(repoDir, "secret.txt"), "hunter2\n");54+	await writeFile(join(repoDir, "src/lib/deep/util.ts"), "export const one = 1;\n");55+	koh("save", "add docs, a secret and a deep helper");56+	const first = JSON.parse(koh("log", "--json")).head_id as string;57+	koh("flag", first, "--reason", "holds a secret");58+	// Saves a second apart, so the timeline has time to spread over.59+	Bun.sleepSync(1100);60+	await writeFile(join(repoDir, "README.md"), README);2061 	await writeFile(join(repoDir, "src/main.ts"), "// Entry point.\nconsole.log('hi');\n");2162 	koh("save", "add readme and entry point", "--agent", "test-agent", "--session", "fixture");2263 	koh("checkpoint", "before the second change");@@ -25,18 +66,39 @@2566 	koh("offer");2667 2768 	const log = JSON.parse(koh("log", "--json")) as { saves: { id: string }[] };28-	return { bucketDir, repo, repoDir, saveIds: log.saves.map((save) => save.id) };69+	return {70+		bucketDir,71+		repo,72+		repoDir,73+		saveIds: log.saves.map((save) => save.id),74+		strip: (path) => strip(repoDir, home, path),75+	};2976 }3077 3178 function run(cwd: string, home: string, args: string[]): string {32-	const result = Bun.spawnSync(["koh3", ...args], {33-		cwd,34-		env: { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" },35-	});79+	const result = Bun.spawnSync(["koh3", ...args], { cwd, env: env(home) });3680 	if (result.exitCode !== 0) {3781 		throw new Error(3882 			`koh3 ${args.join(" ")} failed: ${result.stderr.toString()}${result.stdout.toString()}`,3983 		);4084 	}4185 	return result.stdout.toString();86+}87+88+function env(home: string) {89+	return { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };90+}91+92+// `koh strip` insists on a terminal and a typed confirmation, so it runs under a Bun PTY.93+async function strip(cwd: string, home: string, path: string): Promise<void> {94+	let output = "";95+	const proc = Bun.spawn(["koh3", "strip", path], {96+		cwd,97+		env: env(home),98+		terminal: { cols: 120, rows: 24, data: (_term, data) => (output += data.toString()) },99+	});100+	proc.terminal?.write(`y\n${path}\n`);101+	const code = await proc.exited;102+	proc.terminal?.close();103+	if (code !== 0) throw new Error(`koh3 strip ${path} failed: ${output}`);42104 }

view file

tests/sync.test.ts+123 −0
@@ -0,0 +1,123 @@1+import { beforeAll, describe, expect, test } from "bun:test";2+import { mkdtemp, writeFile } from "node:fs/promises";3+import { tmpdir } from "node:os";4+import { join } from "node:path";5+import { openBucket } from "../src/bucket/reader.ts";6+import { syncAll } from "../src/bucket/sync.ts";7+import {8+	fileAt,9+	filesAt,10+	headId,11+	history,12+	objectBytes,13+	repoStats,14+	saveByPrefix,15+	timeline,16+} from "../src/mirror/queries.ts";17+import { openMirror } from "../src/mirror/store.ts";18+import { openKeprDb } from "../src/server/db.ts";19+import { type Fixture, makeKohBucket } from "./fixture.ts";20+21+const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());22+23+async function syncTarget(fx: Fixture) {24+	const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));25+	const target = {26+		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),27+		kepr: openKeprDb(dataDir),28+		mirrorDir: join(dataDir, "mirror"),29+	};30+	return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };31+}32+33+describe("a first sync of a koh3 bucket", () => {34+	let fx: Fixture;35+	let t: Awaited<ReturnType<typeof syncTarget>>;36+	beforeAll(async () => {37+		fx = await makeKohBucket();38+		t = await syncTarget(fx);39+		await syncAll(t);40+	});41+	const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);42+43+	test("registers the repo private, with a clean last sync", () => {44+		const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {45+			public: number;46+			last_sync_at: number;47+			last_sync_error: string | null;48+		};49+		expect(row.public).toBe(0);50+		expect(row.last_sync_error).toBeNull();51+		expect(row.last_sync_at).toBeGreaterThan(0);52+	});53+54+	test("history is newest first and leaves out the checkpoint", () => {55+		expect(headId(t.mirror())).toBe(head());56+		expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);57+		expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);58+		expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);59+		expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);60+	});61+62+	test("finds a save by a unique prefix of six or more characters", () => {63+		const save = saveByPrefix(t.mirror(), second().slice(0, 6));64+		expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);65+		expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();66+		expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();67+	});68+69+	test("reads files and their bytes at any save", () => {70+		const paths = filesAt(t.mirror(), head()).map((f) => f.path);71+		expect(paths).toContain("src/lib/deep/util.ts");72+		const now = fileAt(t.mirror(), head(), "src/main.ts");73+		const then = fileAt(t.mirror(), second(), "src/main.ts");74+		expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");75+		expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");76+		expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();77+		expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);78+	});79+80+	test("counts saves without checkpoints, and lines and language at head", () => {81+		const stats = repoStats(t.mirror());82+		expect(stats.saves).toBe(3);83+		expect(stats.language).toBe("TypeScript");84+		// README 9 + guide 3 + secret 1 + util 1 + main 2 + .kohignore85+		expect(stats.lines).toBeGreaterThan(16);86+		expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);87+	});88+89+	test("stores the flag event", () => {90+		const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();91+		expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });92+	});93+94+	test("a strip reaches the mirror and drops the object", async () => {95+		const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;96+		expect(objectBytes(t.mirror(), secret)).not.toBeNull();97+		await fx.strip("secret.txt");98+		await syncAll(t);99+		for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();100+		expect(objectBytes(t.mirror(), secret)).toBeNull();101+		expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();102+	});103+});104+105+test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {106+	const fx = await makeKohBucket("growing");107+	const t = await syncTarget(fx);108+	await syncAll(t);109+	await writeFile(join(fx.repoDir, "new.txt"), "new\n");110+	Bun.spawnSync(["koh3", "save", "add new"], {111+		cwd: fx.repoDir,112+		env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },113+	});114+	await syncAll(t);115+	expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");116+117+	await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");118+	await syncAll(t);119+	const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {120+		last_sync_error: string;121+	};122+	expect(row.last_sync_error).toContain('remote format "koh-remote 2"');123+});

view file