aab0488febf0
add kepr.db, the per-repo koh 3 mirror with its queries, and the two-minute sync loop that applies strip events and records the last sync (v3.0.6)
- time
- 2026-09-26 16:23 UTC · 2 weeks ago
- machine
- Junes-MacBook-Pro.local
- agent
- claude-opus-5-5
- session
- kepr 3 implementation
- parent
- 1972104e46af
↓ kepr-aab0488f.tar.gzatlas at this save
files changed10 files
package.json+1 −1src/bucket/repo.ts+57 −3src/bucket/sync.ts+94 −0src/main.ts+10 −0src/mirror/queries.ts+101 −0src/mirror/store.ts+222 −0src/server/db.ts+78 −0tests/bucket.test.ts+28 −2tests/fixture.ts+71 −9tests/sync.test.ts+123 −0package.json+1 −1
@@ -1,6 +1,6 @@11 {22 "name": "kepr",3- "version": "3.0.5",3+ "version": "3.0.6",44 "private": true,55 "type": "module",66 "scripts": {src/bucket/repo.ts+57 −3
@@ -21,6 +21,20 @@2121 files: SaveFile[];2222 };2323 24+/** A flag, clear or strip event, as Koh 3 writes it to `<repo>/events/<id>` (§9.2, §10). */25+export type EventRecord = {26+ id: string;27+ kind: "flag" | "clear" | "strip";28+ save_id: string;29+ path: string;30+ reason: string;31+ created_at: number;32+ machine: string;33+};34+35+/** An object's stored zlib bytes and the inflated bytes they were verified from. */36+export type VerifiedObject = { compressed: Uint8Array; bytes: Uint8Array };37+2438 export class BucketError extends Error {}2539 2640 const SAVE_ID = /^[0-9a-f]{12}$/;@@ -90,7 +104,7 @@90104 bucket: BucketReader,91105 repo: string,92106 hash: string,93-): Promise<Uint8Array> {107+): Promise<VerifiedObject> {94108 requireRepo(repo);95109 if (!OBJECT_HASH.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);96110 const compressed = await bucket.get(`${repo}/objects/${hash}`);@@ -99,9 +113,32 @@99113 if (bytesToHex(blake3(bytes)) !== hash) {100114 throw new BucketError(`${repo}/objects/${hash} does not match its hash`);101115 }102- return bytes;116+ return { compressed, bytes };103117 }104118 119+export async function listEventIds(bucket: BucketReader, repo: string): Promise<string[]> {120+ requireRepo(repo);121+ const prefix = `${repo}/events/`;122+ // A folder remote's half-written temp files can show up in a listing; only IDs are events.123+ return (await bucket.list(prefix))124+ .map((key) => key.slice(prefix.length))125+ .filter((id) => SAVE_ID.test(id));126+}127+128+export async function readEvent(129+ bucket: BucketReader,130+ repo: string,131+ id: string,132+): Promise<EventRecord> {133+ requireRepo(repo);134+ requireSaveId(id);135+ const bytes = await bucket.get(`${repo}/events/${id}`);136+ if (bytes === null) throw new BucketError(`${repo}/events/${id} is missing`);137+ const record = parseEventRecord(JSON.parse(decoder.decode(bytes)));138+ if (record.id !== id) throw new BucketError(`${repo}/events/${id} holds event ${record.id}`);139+ return record;140+}141+105142 function parseSaveRecord(value: unknown): SaveRecord {106143 const r = value as Record<string, unknown>;107144 const valid =@@ -121,6 +158,23 @@121158 return value as SaveRecord;122159 }123160 161+function parseEventRecord(value: unknown): EventRecord {162+ const r = value as Record<string, unknown>;163+ const valid =164+ typeof value === "object" &&165+ value !== null &&166+ typeof r.id === "string" &&167+ SAVE_ID.test(r.id) &&168+ ["flag", "clear", "strip"].includes(r.kind as string) &&169+ isOptionalSaveId(r.save_id) &&170+ (r.path === "" || (typeof r.path === "string" && isSafePath(r.path))) &&171+ typeof r.reason === "string" &&172+ typeof r.machine === "string" &&173+ isCount(r.created_at);174+ if (!valid) throw new BucketError("event record does not match the Koh 3 schema");175+ return value as EventRecord;176+}177+124178 function isOptionalSaveId(value: unknown): boolean {125179 return value === "" || (typeof value === "string" && SAVE_ID.test(value));126180 }@@ -130,7 +184,7 @@130184 }131185 132186 /** A repo-relative path that can't escape or alias: no empty, `.` or `..` segments, no backslashes or control characters. */133-function isSafePath(path: string): boolean {187+export function isSafePath(path: string): boolean {134188 return (135189 !path.includes("\\") &&136190 !CONTROL_CHAR.test(path) &&src/bucket/sync.ts+94 −0
@@ -0,0 +1,94 @@1+import type { Database } from "bun:sqlite";2+import {3+ knownEventIds,4+ knownObject,5+ knownSave,6+ land,7+ mirrorState,8+ openMirror,9+ type StoredObject,10+} from "../mirror/store.ts";11+import { recordSync, registerRepo } from "../server/db.ts";12+import type { BucketReader } from "./reader.ts";13+import {14+ BucketError,15+ checkFormat,16+ type EventRecord,17+ listEventIds,18+ listRepos,19+ readEvent,20+ readHead,21+ readObject,22+ readSave,23+ type SaveRecord,24+} from "./repo.ts";25+26+const INTERVAL_MS = 2 * 60 * 1000;27+28+export type SyncTarget = { bucket: BucketReader; kepr: Database; mirrorDir: string };29+30+let running: Promise<void> | null = null;31+32+/** Syncs every repo in the bucket; the admin console's "sync now" calls this too. Never overlaps itself. */33+export function syncAll(target: SyncTarget): Promise<void> {34+ running ??= syncEach(target).finally(() => {35+ running = null;36+ });37+ return running;38+}39+40+export function startSyncLoop(target: SyncTarget): void {41+ const tick = () => syncAll(target).catch((error) => console.error(`kepr: sync failed: ${error}`));42+ tick();43+ setInterval(tick, INTERVAL_MS);44+}45+46+async function syncEach(target: SyncTarget): Promise<void> {47+ for (const repo of await listRepos(target.bucket)) {48+ registerRepo(target.kepr, repo);49+ let error: string | null = null;50+ try {51+ await syncRepo(target.bucket, openMirror(target.mirrorDir, repo), repo);52+ } catch (cause) {53+ error = cause instanceof Error ? cause.message : String(cause);54+ }55+ recordSync(target.kepr, repo, Math.floor(Date.now() / 1000), error);56+ }57+}58+59+/**60+ * Koh's steal flow (§9.4) against a mirror: walk back from the remote head to a save the mirror has,61+ * fetch what is missing, and land it in one transaction. Events are listed every time, since a62+ * flag or a strip arrives without moving `head`.63+ */64+export async function syncRepo(65+ bucket: BucketReader,66+ mirror: Database,67+ repo: string,68+): Promise<void> {69+ await checkFormat(bucket, repo);70+ const head = await readHead(bucket, repo);71+ if (head === null) return;72+ const saves: SaveRecord[] = [];73+ const objects = new Map<string, StoredObject>();74+ const seen = new Set<string>();75+ for (let cursor = head; cursor !== "" && !knownSave(mirror, cursor); ) {76+ if (seen.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`);77+ seen.add(cursor);78+ const save = await readSave(bucket, repo, cursor);79+ for (const file of save.files) {80+ if (objects.has(file.hash) || knownObject(mirror, file.hash)) continue;81+ const { compressed, bytes } = await readObject(bucket, repo, file.hash);82+ objects.set(file.hash, { compressed, size: bytes.length });83+ }84+ saves.push(save);85+ cursor = save.parent;86+ }87+ const known = knownEventIds(mirror);88+ const events: EventRecord[] = [];89+ for (const id of await listEventIds(bucket, repo)) {90+ if (!known.has(id)) events.push(await readEvent(bucket, repo, id));91+ }92+ if (saves.length === 0 && events.length === 0 && mirrorState(mirror, "head") === head) return;93+ land(mirror, { head, saves: saves.reverse(), objects, events });94+}src/main.ts+10 −0
@@ -1,8 +1,18 @@1+import { join } from "node:path";2+import { openBucket } from "./bucket/reader.ts";3+import { startSyncLoop } from "./bucket/sync.ts";14 import { createApp } from "./server/app.ts";25 import { ConfigError, loadConfig } from "./server/config.ts";6+import { openKeprDb } from "./server/db.ts";37 48 try {59 const config = await loadConfig("kepr.toml");10+ const kepr = openKeprDb(config.dataDir);11+ startSyncLoop({12+ bucket: openBucket(config.bucket),13+ kepr,14+ mirrorDir: join(config.dataDir, "mirror"),15+ });616 // Bun.serve reads the port from $PORT (default 3000).717 const server = Bun.serve({ fetch: createApp(config).fetch });818 console.log(`kepr listening on ${server.url}`);src/mirror/queries.ts+101 −0
@@ -0,0 +1,101 @@1+import type { Database } from "bun:sqlite";2+import { inflateSync } from "node:zlib";3+import { mirrorState } from "./store.ts";4+5+export type Save = {6+ id: string;7+ parent: string;8+ message: string;9+ created_at: number;10+ machine: string;11+ agent: string;12+ session: string;13+ is_checkpoint: boolean;14+};15+16+export type FileEntry = { path: string; hash: string; executable: boolean; size: number };17+18+export type RepoStats = { saves: number; lines: number; language: string; updated: number | null };19+20+const SAVE_COLUMNS = `id, COALESCE(parent, '') AS parent, message, created_at, machine,21+ COALESCE(agent, '') AS agent, COALESCE(session, '') AS session, is_checkpoint = 1 AS is_checkpoint`;22+const ID_PREFIX = /^[0-9a-f]{6,12}$/;23+24+// Checkpoints are working marks, not history (§21.4): every list and count below leaves them out.25+const NOT_CHECKPOINT = "is_checkpoint = 0";26+27+export function headId(db: Database): string | null {28+ return mirrorState(db, "head");29+}30+31+/** A page of saves, newest first. Rows land oldest first, so rowid is the order of the line. */32+export function history(db: Database, limit: number, offset: number): Save[] {33+ return db34+ .query<Save, [number, number]>(35+ `SELECT ${SAVE_COLUMNS} FROM saves WHERE ${NOT_CHECKPOINT} ORDER BY rowid DESC LIMIT ? OFFSET ?`,36+ )37+ .all(limit, offset)38+ .map(asSave);39+}40+41+/** Every save, oldest first, for the timeline. */42+export function timeline(db: Database): Save[] {43+ return db44+ .query<Save, []>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE ${NOT_CHECKPOINT} ORDER BY rowid`)45+ .all()46+ .map(asSave);47+}48+49+/** The save a unique prefix of 6–12 hex characters names (§6.1), or null when none or several do. */50+export function saveByPrefix(db: Database, prefix: string): Save | null {51+ if (!ID_PREFIX.test(prefix)) return null;52+ const rows = db53+ .query<Save, [string]>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE id LIKE ? || '%' LIMIT 2`)54+ .all(prefix);55+ return rows.length === 1 ? asSave(rows[0] as Save) : null;56+}57+58+export function filesAt(db: Database, saveId: string): FileEntry[] {59+ return db60+ .query<FileEntry, [string]>(61+ `SELECT f.path, f.hash, f.executable = 1 AS executable, o.size FROM save_files f62+ JOIN objects o ON o.hash = f.hash WHERE f.save_id = ? ORDER BY f.path`,63+ )64+ .all(saveId)65+ .map((file) => ({ ...file, executable: Boolean(file.executable) }));66+}67+68+export function fileAt(db: Database, saveId: string, path: string): FileEntry | null {69+ const file = db70+ .query<FileEntry, [string, string]>(71+ `SELECT f.path, f.hash, f.executable = 1 AS executable, o.size FROM save_files f72+ JOIN objects o ON o.hash = f.hash WHERE f.save_id = ? AND f.path = ?`,73+ )74+ .get(saveId, path);75+ return file ? { ...file, executable: Boolean(file.executable) } : null;76+}77+78+export function objectBytes(db: Database, hash: string): Uint8Array | null {79+ const row = db80+ .query<{ data: Uint8Array }, [string]>("SELECT data FROM objects WHERE hash = ?")81+ .get(hash);82+ return row ? new Uint8Array(inflateSync(row.data)) : null;83+}84+85+export function repoStats(db: Database): RepoStats {86+ const row = db87+ .query<{ saves: number; updated: number | null }, []>(88+ `SELECT COUNT(*) AS saves, MAX(created_at) AS updated FROM saves WHERE ${NOT_CHECKPOINT}`,89+ )90+ .get() ?? { saves: 0, updated: null };91+ return {92+ saves: row.saves,93+ updated: row.updated,94+ lines: Number(mirrorState(db, "lines") ?? 0),95+ language: mirrorState(db, "language") ?? "",96+ };97+}98+99+function asSave(row: Save): Save {100+ return { ...row, is_checkpoint: Boolean(row.is_checkpoint) };101+}src/mirror/store.ts+222 −0
@@ -0,0 +1,222 @@1+import { Database } from "bun:sqlite";2+import { mkdirSync } from "node:fs";3+import { join } from "node:path";4+import { inflateSync } from "node:zlib";5+import type { EventRecord, SaveRecord } from "../bucket/repo.ts";6+7+// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs.8+const SCHEMA = `9+CREATE TABLE objects (10+ hash TEXT PRIMARY KEY,11+ size INTEGER NOT NULL,12+ data BLOB NOT NULL13+);14+CREATE TABLE saves (15+ id TEXT PRIMARY KEY,16+ parent TEXT REFERENCES saves(id),17+ message TEXT NOT NULL,18+ created_at INTEGER NOT NULL,19+ machine TEXT NOT NULL,20+ is_checkpoint INTEGER NOT NULL DEFAULT 0,21+ size_bytes INTEGER NOT NULL DEFAULT 0,22+ agent TEXT,23+ session TEXT,24+ restores TEXT,25+ replayed_from TEXT26+);27+CREATE TABLE save_files (28+ save_id TEXT NOT NULL REFERENCES saves(id),29+ path TEXT NOT NULL,30+ hash TEXT NOT NULL REFERENCES objects(hash),31+ executable INTEGER NOT NULL DEFAULT 0,32+ PRIMARY KEY (save_id, path)33+);34+CREATE TABLE events (35+ id TEXT PRIMARY KEY,36+ kind TEXT NOT NULL,37+ save_id TEXT,38+ path TEXT,39+ reason TEXT,40+ created_at INTEGER NOT NULL,41+ machine TEXT NOT NULL42+);43+CREATE TABLE state (44+ key TEXT PRIMARY KEY,45+ value TEXT NOT NULL46+);47+CREATE INDEX saves_parent ON saves(parent);48+CREATE INDEX save_files_hash ON save_files(hash);49+`;50+51+/** An object as the bucket stores it: zlib bytes, already checked against its hash. */52+export type StoredObject = { compressed: Uint8Array; size: number };53+54+/** What one sync brings in, landed in one transaction. Saves are oldest first. */55+export type Landing = {56+ head: string;57+ saves: SaveRecord[];58+ objects: Map<string, StoredObject>;59+ events: EventRecord[];60+};61+62+// Display names for the header's language, by extension; data and docs formats never count.63+const LANGUAGES: Record<string, string> = {64+ ts: "TypeScript",65+ tsx: "TypeScript",66+ js: "JavaScript",67+ mjs: "JavaScript",68+ zig: "Zig",69+ v: "V",70+ rs: "Rust",71+ go: "Go",72+ py: "Python",73+ c: "C",74+ h: "C",75+ cpp: "C++",76+ cc: "C++",77+ hpp: "C++",78+ swift: "Swift",79+ kt: "Kotlin",80+ java: "Java",81+ rb: "Ruby",82+ lua: "Lua",83+ sh: "Shell",84+ nix: "Nix",85+ css: "CSS",86+ html: "HTML",87+};88+89+const open = new Map<string, Database>();90+91+/** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */92+export function openMirror(dir: string, repo: string): Database {93+ const path = join(dir, `${repo}.db`);94+ const cached = open.get(path);95+ if (cached) return cached;96+ mkdirSync(dir, { recursive: true });97+ const db = new Database(path, { strict: true });98+ db.run("PRAGMA journal_mode = WAL");99+ db.run("PRAGMA foreign_keys = ON");100+ db.run("PRAGMA busy_timeout = 5000");101+ const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {102+ user_version: 0,103+ };104+ if (user_version === 0) {105+ db.transaction(() => {106+ db.run(SCHEMA);107+ db.run("PRAGMA user_version = 1");108+ })();109+ }110+ open.set(path, db);111+ return db;112+}113+114+export function mirrorState(db: Database, key: string): string | null {115+ const row = db116+ .query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")117+ .get(key);118+ return row?.value ?? null;119+}120+121+export function knownSave(db: Database, id: string): boolean {122+ return db.query("SELECT 1 FROM saves WHERE id = ?").get(id) !== null;123+}124+125+export function knownObject(db: Database, hash: string): boolean {126+ return db.query("SELECT 1 FROM objects WHERE hash = ?").get(hash) !== null;127+}128+129+export function knownEventIds(db: Database): Set<string> {130+ const rows = db.query<{ id: string }, []>("SELECT id FROM events").all();131+ return new Set(rows.map((row) => row.id));132+}133+134+/** Lands a sync like Koh's steal: objects, then events, then saves, then strips, all or nothing. */135+export function land(db: Database, landing: Landing): void {136+ const putObject = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");137+ const putEvent = db.query(138+ "INSERT OR IGNORE INTO events (id, kind, save_id, path, reason, created_at, machine) VALUES (?, ?, ?, ?, ?, ?, ?)",139+ );140+ const putSave = db.query(141+ `INSERT OR IGNORE INTO saves (id, parent, message, created_at, machine, is_checkpoint, size_bytes,142+ agent, session, restores, replayed_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,143+ );144+ const putFile = db.query(145+ "INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",146+ );147+ db.transaction(() => {148+ for (const [hash, object] of landing.objects)149+ putObject.run(hash, object.size, object.compressed);150+ for (const e of landing.events) {151+ putEvent.run(e.id, e.kind, e.save_id, e.path, e.reason, e.created_at, e.machine);152+ }153+ for (const s of landing.saves) {154+ putSave.run(155+ s.id,156+ s.parent === "" ? null : s.parent,157+ s.message,158+ s.created_at,159+ s.machine,160+ s.is_checkpoint ? 1 : 0,161+ s.size_bytes,162+ s.agent,163+ s.session,164+ s.restores,165+ s.replayed_from,166+ );167+ for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);168+ }169+ applyStrips(db);170+ db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('head', ?)").run(landing.head);171+ refreshStats(db, landing.head);172+ })();173+}174+175+// §10.2: a strip of one save reaches that save; a strip of every save reaches the saves made up to176+// and including its own time, so a file saved again after the strip survives. Re-applying every177+// strip is harmless, which lets a save that arrives after its strip event be covered too.178+function applyStrips(db: Database): void {179+ const strips = db180+ .query<{ save_id: string; path: string; created_at: number }, []>(181+ "SELECT save_id, path, created_at FROM events WHERE kind = 'strip'",182+ )183+ .all();184+ if (strips.length === 0) return;185+ const remove = db.query(186+ `DELETE FROM save_files187+ WHERE (path = $path OR substr(path, 1, length($dir)) = $dir)188+ AND save_id IN (SELECT id FROM saves WHERE189+ CASE WHEN $save = '' THEN created_at <= $at ELSE id = $save END)`,190+ );191+ for (const strip of strips) {192+ // Koh compares the prefix exactly, never with LIKE, so `_` and `%` in a path match only themselves.193+ const dir = `${strip.path.replace(/\/+$/, "")}/`;194+ remove.run({ path: strip.path, dir, save: strip.save_id, at: strip.created_at });195+ }196+ db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");197+}198+199+// Lines and language at `head` never change between syncs, so they are counted once here, not per view.200+function refreshStats(db: Database, head: string): void {201+ const files = db202+ .query<{ path: string; data: Uint8Array }, [string]>(203+ "SELECT f.path, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",204+ )205+ .all(head);206+ let lines = 0;207+ const byLanguage = new Map<string, number>();208+ for (const file of files) {209+ const bytes = inflateSync(file.data);210+ if (bytes.includes(0)) continue;211+ let count = 0;212+ for (const byte of bytes) if (byte === 10) count++;213+ if (bytes.length > 0 && bytes[bytes.length - 1] !== 10) count++;214+ lines += count;215+ const language = LANGUAGES[file.path.split(".").pop() ?? ""];216+ if (language) byLanguage.set(language, (byLanguage.get(language) ?? 0) + count);217+ }218+ const language = [...byLanguage].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";219+ const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");220+ setState.run("lines", String(lines));221+ setState.run("language", language);222+}src/server/db.ts+78 −0
@@ -0,0 +1,78 @@1+import { Database } from "bun:sqlite";2+import { mkdirSync } from "node:fs";3+import { join } from "node:path";4+5+// §19: only what the pages and the sync loop read so far; later phases add their tables.6+const SCHEMA = `7+CREATE TABLE IF NOT EXISTS settings (8+ key TEXT PRIMARY KEY,9+ value TEXT NOT NULL10+);11+CREATE TABLE IF NOT EXISTS repos (12+ name TEXT PRIMARY KEY,13+ public INTEGER NOT NULL DEFAULT 0,14+ category TEXT NOT NULL DEFAULT '',15+ homepage TEXT NOT NULL DEFAULT '',16+ sort_order INTEGER NOT NULL DEFAULT 0,17+ install1 TEXT NOT NULL DEFAULT '',18+ install2 TEXT NOT NULL DEFAULT '',19+ install3 TEXT NOT NULL DEFAULT '',20+ last_sync_at INTEGER,21+ last_sync_error TEXT22+);23+`;24+25+export type RepoSettings = {26+ name: string;27+ homepage: string;28+ installLines: string[];29+};30+31+type RepoRow = {32+ name: string;33+ homepage: string;34+ install1: string;35+ install2: string;36+ install3: string;37+};38+39+export function openKeprDb(dataDir: string): Database {40+ mkdirSync(dataDir, { recursive: true });41+ const db = new Database(join(dataDir, "kepr.db"), { strict: true });42+ db.run("PRAGMA journal_mode = WAL");43+ db.run("PRAGMA busy_timeout = 5000");44+ db.run(SCHEMA);45+ return db;46+}47+48+export function setting(db: Database, key: string): string | null {49+ const row = db50+ .query<{ value: string }, [string]>("SELECT value FROM settings WHERE key = ?")51+ .get(key);52+ return row?.value ?? null;53+}54+55+/** Every repo starts private (§20); registering a known repo changes nothing. */56+export function registerRepo(db: Database, name: string): void {57+ db.query("INSERT OR IGNORE INTO repos (name) VALUES (?)").run(name);58+}59+60+export function recordSync(db: Database, name: string, at: number, error: string | null): void {61+ db.query("UPDATE repos SET last_sync_at = ?, last_sync_error = ? WHERE name = ?").run(62+ at,63+ error,64+ name,65+ );66+}67+68+/** The settings of a public repo, or null when it is private or unknown: both must look missing. */69+export function publicRepo(db: Database, name: string): RepoSettings | null {70+ const row = db71+ .query<RepoRow, [string]>(72+ "SELECT name, homepage, install1, install2, install3 FROM repos WHERE name = ? AND public = 1",73+ )74+ .get(name);75+ if (!row) return null;76+ const installLines = [row.install1, row.install2, row.install3].filter((line) => line !== "");77+ return { name: row.name, homepage: row.homepage, installLines };78+}tests/bucket.test.ts+28 −2
@@ -2,7 +2,15 @@22 import { writeFile } from "node:fs/promises";33 import { join } from "node:path";44 import { openBucket } from "../src/bucket/reader.ts";5-import { checkFormat, listRepos, readHead, readObject, readSave } from "../src/bucket/repo.ts";5+import {6+ checkFormat,7+ listEventIds,8+ listRepos,9+ readEvent,10+ readHead,11+ readObject,12+ readSave,13+} from "../src/bucket/repo.ts";614 import { type Fixture, makeKohBucket } from "./fixture.ts";715 816 let fx: Fixture;@@ -32,7 +40,7 @@3240 expect([first.agent, first.session]).toEqual(["test-agent", "fixture"]);3341 3442 const main = save.files.find((file) => file.path === "src/main.ts");35- const bytes = await readObject(b, fx.repo, main?.hash as string);43+ const { bytes } = await readObject(b, fx.repo, main?.hash as string);3644 expect(new TextDecoder().decode(bytes)).toBe("// Entry point.\nconsole.log('hello');\n");3745 });3846 @@ -175,6 +183,24 @@175183 }176184 for (const id of ["../head", "abc", "ABCDEF123456", `${fx.saveIds[0]}0`]) {177185 await expect(readSave(b, fx.repo, id)).rejects.toThrow("not a save ID");186+ }187+ });188+});189+190+describe("event records", () => {191+ test("reads a genuine koh3 flag event and refuses a tampered one", async () => {192+ const b = bucket();193+ const [id] = await listEventIds(b, fx.repo);194+ const event = await readEvent(b, fx.repo, id as string);195+ expect([event.kind, event.save_id, event.reason]).toEqual([196+ "flag",197+ fx.saveIds[3] as string,198+ "holds a secret",199+ ]);200+ for (const change of [{ kind: "delete" }, { path: "../x" }, { save_id: "nope" }]) {201+ const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));202+ const tampered = { get: async () => bytes, list: async () => [] };203+ await expect(readEvent(tampered, fx.repo, event.id)).rejects.toThrow("Koh 3 schema");178204 }179205 });180206 });tests/fixture.ts+71 −9
@@ -2,21 +2,62 @@22 import { tmpdir } from "node:os";33 import { join } from "node:path";44 5-export type Fixture = { bucketDir: string; repo: string; repoDir: string; saveIds: string[] };5+export type Fixture = {6+ bucketDir: string;7+ repo: string;8+ repoDir: string;9+ /** Every save, newest first, checkpoint included. */10+ saveIds: string[];11+ /** Strips a path from every save with koh3, so tests can sync before and after. */12+ strip(path: string): Promise<void>;13+};614 7-/** Builds a genuine Koh 3 folder remote with koh3 itself, under a throwaway HOME. */15+// The smallest valid PNG (1×1), so the fixture has a real binary file.16+const PNG = Uint8Array.from(17+ atob(18+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",19+ ),20+ (c) => c.charCodeAt(0),21+);22+23+const README = `# Demo24+25+A small repo for **Kepr's** tests.26+27+See [the guide](docs/guide.md), [the site](https://example.com) and [a trap](javascript:alert(1)).28+29+30+31+<script>alert("raw html")</script>32+`;33+34+/**35+ * Builds a genuine Koh 3 folder remote with koh3 itself, under a throwaway HOME. The line is:36+ * "add docs, a secret and a deep helper" (flagged) → a second later "add readme and entry point"37+ * (agent and session) → a checkpoint → "say hello" (head).38+ */839 export async function makeKohBucket(repo = "demo"): Promise<Fixture> {940 const root = await mkdtemp(join(tmpdir(), "kepr-fixture-"));1041 const home = join(root, "home");1142 const bucketDir = join(root, "bucket");1243 const repoDir = join(root, "work", repo);1344 await mkdir(home);14- await mkdir(join(repoDir, "src"), { recursive: true });45+ await mkdir(join(repoDir, "src/lib/deep"), { recursive: true });46+ await mkdir(join(repoDir, "docs"));1547 const koh = (...args: string[]) => run(repoDir, home, args);1648 1749 koh("init");1850 koh("remote", "folder", bucketDir);19- await writeFile(join(repoDir, "README.md"), "# Demo\n\nA small repo for Kepr's tests.\n");51+ await writeFile(join(repoDir, "docs/guide.md"), "# Guide\n\nRead me.\n");52+ await writeFile(join(repoDir, "docs/logo.png"), PNG);53+ await writeFile(join(repoDir, "secret.txt"), "hunter2\n");54+ await writeFile(join(repoDir, "src/lib/deep/util.ts"), "export const one = 1;\n");55+ koh("save", "add docs, a secret and a deep helper");56+ const first = JSON.parse(koh("log", "--json")).head_id as string;57+ koh("flag", first, "--reason", "holds a secret");58+ // Saves a second apart, so the timeline has time to spread over.59+ Bun.sleepSync(1100);60+ await writeFile(join(repoDir, "README.md"), README);2061 await writeFile(join(repoDir, "src/main.ts"), "// Entry point.\nconsole.log('hi');\n");2162 koh("save", "add readme and entry point", "--agent", "test-agent", "--session", "fixture");2263 koh("checkpoint", "before the second change");@@ -25,18 +66,39 @@2566 koh("offer");2667 2768 const log = JSON.parse(koh("log", "--json")) as { saves: { id: string }[] };28- return { bucketDir, repo, repoDir, saveIds: log.saves.map((save) => save.id) };69+ return {70+ bucketDir,71+ repo,72+ repoDir,73+ saveIds: log.saves.map((save) => save.id),74+ strip: (path) => strip(repoDir, home, path),75+ };2976 }3077 3178 function run(cwd: string, home: string, args: string[]): string {32- const result = Bun.spawnSync(["koh3", ...args], {33- cwd,34- env: { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" },35- });79+ const result = Bun.spawnSync(["koh3", ...args], { cwd, env: env(home) });3680 if (result.exitCode !== 0) {3781 throw new Error(3882 `koh3 ${args.join(" ")} failed: ${result.stderr.toString()}${result.stdout.toString()}`,3983 );4084 }4185 return result.stdout.toString();86+}87+88+function env(home: string) {89+ return { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };90+}91+92+// `koh strip` insists on a terminal and a typed confirmation, so it runs under a Bun PTY.93+async function strip(cwd: string, home: string, path: string): Promise<void> {94+ let output = "";95+ const proc = Bun.spawn(["koh3", "strip", path], {96+ cwd,97+ env: env(home),98+ terminal: { cols: 120, rows: 24, data: (_term, data) => (output += data.toString()) },99+ });100+ proc.terminal?.write(`y\n${path}\n`);101+ const code = await proc.exited;102+ proc.terminal?.close();103+ if (code !== 0) throw new Error(`koh3 strip ${path} failed: ${output}`);42104 }tests/sync.test.ts+123 −0
@@ -0,0 +1,123 @@1+import { beforeAll, describe, expect, test } from "bun:test";2+import { mkdtemp, writeFile } from "node:fs/promises";3+import { tmpdir } from "node:os";4+import { join } from "node:path";5+import { openBucket } from "../src/bucket/reader.ts";6+import { syncAll } from "../src/bucket/sync.ts";7+import {8+ fileAt,9+ filesAt,10+ headId,11+ history,12+ objectBytes,13+ repoStats,14+ saveByPrefix,15+ timeline,16+} from "../src/mirror/queries.ts";17+import { openMirror } from "../src/mirror/store.ts";18+import { openKeprDb } from "../src/server/db.ts";19+import { type Fixture, makeKohBucket } from "./fixture.ts";20+21+const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());22+23+async function syncTarget(fx: Fixture) {24+ const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));25+ const target = {26+ bucket: openBucket({ kind: "folder", path: fx.bucketDir }),27+ kepr: openKeprDb(dataDir),28+ mirrorDir: join(dataDir, "mirror"),29+ };30+ return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };31+}32+33+describe("a first sync of a koh3 bucket", () => {34+ let fx: Fixture;35+ let t: Awaited<ReturnType<typeof syncTarget>>;36+ beforeAll(async () => {37+ fx = await makeKohBucket();38+ t = await syncTarget(fx);39+ await syncAll(t);40+ });41+ const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);42+43+ test("registers the repo private, with a clean last sync", () => {44+ const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {45+ public: number;46+ last_sync_at: number;47+ last_sync_error: string | null;48+ };49+ expect(row.public).toBe(0);50+ expect(row.last_sync_error).toBeNull();51+ expect(row.last_sync_at).toBeGreaterThan(0);52+ });53+54+ test("history is newest first and leaves out the checkpoint", () => {55+ expect(headId(t.mirror())).toBe(head());56+ expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);57+ expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);58+ expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);59+ expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);60+ });61+62+ test("finds a save by a unique prefix of six or more characters", () => {63+ const save = saveByPrefix(t.mirror(), second().slice(0, 6));64+ expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);65+ expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();66+ expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();67+ });68+69+ test("reads files and their bytes at any save", () => {70+ const paths = filesAt(t.mirror(), head()).map((f) => f.path);71+ expect(paths).toContain("src/lib/deep/util.ts");72+ const now = fileAt(t.mirror(), head(), "src/main.ts");73+ const then = fileAt(t.mirror(), second(), "src/main.ts");74+ expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");75+ expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");76+ expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();77+ expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);78+ });79+80+ test("counts saves without checkpoints, and lines and language at head", () => {81+ const stats = repoStats(t.mirror());82+ expect(stats.saves).toBe(3);83+ expect(stats.language).toBe("TypeScript");84+ // README 9 + guide 3 + secret 1 + util 1 + main 2 + .kohignore85+ expect(stats.lines).toBeGreaterThan(16);86+ expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);87+ });88+89+ test("stores the flag event", () => {90+ const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();91+ expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });92+ });93+94+ test("a strip reaches the mirror and drops the object", async () => {95+ const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;96+ expect(objectBytes(t.mirror(), secret)).not.toBeNull();97+ await fx.strip("secret.txt");98+ await syncAll(t);99+ for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();100+ expect(objectBytes(t.mirror(), secret)).toBeNull();101+ expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();102+ });103+});104+105+test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {106+ const fx = await makeKohBucket("growing");107+ const t = await syncTarget(fx);108+ await syncAll(t);109+ await writeFile(join(fx.repoDir, "new.txt"), "new\n");110+ Bun.spawnSync(["koh3", "save", "add new"], {111+ cwd: fx.repoDir,112+ env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },113+ });114+ await syncAll(t);115+ expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");116+117+ await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");118+ await syncAll(t);119+ const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {120+ last_sync_error: string;121+ };122+ expect(row.last_sync_error).toContain('remote format "koh-remote 2"');123+});