import { Database } from "bun:sqlite"; import { beforeAll, describe, expect, test } from "bun:test"; import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { deflateSync } from "node:zlib"; import { openBucket } from "../src/bucket/reader.ts"; import { syncAll } from "../src/bucket/sync.ts"; import { fileAt, filesAt, flagReason, headId, history, objectBytes, repoStats, saveByPrefix, timeline, } from "../src/mirror/queries.ts"; import { mirrorState, openMirror } from "../src/mirror/store.ts"; import { openKeprDb } from "../src/server/db.ts"; import { Browser, instance } from "./browser.ts"; import { type Fixture, makeKohBucket } from "./fixture.ts"; const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array()); async function syncTarget(fx: Fixture) { const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-")); const target = { bucket: openBucket({ kind: "folder", path: fx.bucketDir }), kepr: openKeprDb(dataDir), mirrorDir: join(dataDir, "mirror"), tarballDir: join(dataDir, "tarballs"), }; return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) }; } describe("a first sync of a koh3 bucket", () => { let fx: Fixture; let t: Awaited>; beforeAll(async () => { fx = await makeKohBucket(); t = await syncTarget(fx); await syncAll(t); }); const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string); test("registers the repo private, with a clean last sync", () => { const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as { public: number; last_sync_at: number; last_sync_error: string | null; }; expect(row.public).toBe(0); expect(row.last_sync_error).toBeNull(); expect(row.last_sync_at).toBeGreaterThan(0); }); test("history is newest first and leaves out the checkpoint", () => { expect(headId(t.mirror())).toBe(head()); expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]); expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]); expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]); expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true); }); test("finds a save by a unique prefix of six or more characters", () => { const save = saveByPrefix(t.mirror(), second().slice(0, 6)); expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]); expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull(); expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull(); }); test("reads files and their bytes at any save", () => { const paths = filesAt(t.mirror(), head()).map((f) => f.path); expect(paths).toContain("src/lib/deep/util.ts"); const now = fileAt(t.mirror(), head(), "src/main.ts"); const then = fileAt(t.mirror(), second(), "src/main.ts"); expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello"); expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'"); expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull(); expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70); }); test("counts saves without checkpoints, and lines and language at head", () => { const stats = repoStats(t.mirror()); expect(stats.saves).toBe(3); expect(stats.language).toBe("TypeScript"); // README 9 + guide 3 + secret 1 + util 1 + main 2 + .kohignore expect(stats.lines).toBeGreaterThan(16); // vendor/big.c's 100 lines of C count toward neither. expect(stats.lines).toBeLessThan(100); expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1); }); test("stores the flag event", () => { const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get(); expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" }); }); test("a save's flag state is its newest flag or clear event, by time and then ID", async () => { expect(flagReason(t.mirror(), first())).toBe("holds a secret"); expect(flagReason(t.mirror(), head())).toBeNull(); const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-flags-")), "flags"); const event = mirror.query( "INSERT INTO events (id, kind, save_id, reason, created_at, machine) VALUES (?, ?, 'aaaaaaaaaaaa', ?, ?, 'm')", ); event.run("000000000001", "flag", "first", 10); expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("first"); event.run("000000000002", "clear", null, 20); expect(flagReason(mirror, "aaaaaaaaaaaa")).toBeNull(); // Same second: the larger ID is the newer. event.run("000000000003", "flag", "again", 20); expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again"); event.run("000000000000", "clear", null, 20); expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again"); }); test("a strip reaches the mirror and drops the object", async () => { const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string; expect(objectBytes(t.mirror(), secret)).not.toBeNull(); await fx.strip("secret.txt"); await syncAll(t); for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull(); expect(objectBytes(t.mirror(), secret)).toBeNull(); expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull(); }); }); test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => { const fx = await makeKohBucket("growing"); const t = await syncTarget(fx); await syncAll(t); await writeFile(join(fx.repoDir, "new.txt"), "new\n"); Bun.spawnSync(["koh3", "save", "add new"], { cwd: fx.repoDir, env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" }, }); await syncAll(t); expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new"); await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2"); await syncAll(t); const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as { last_sync_error: string; }; expect(row.last_sync_error).toContain('remote format "koh-remote 2"'); }); test("a whole-history strip reaches every save, those arriving after it too, whatever their time (§10.2)", async () => { const fx = await makeKohBucket("late"); const view = await mkdtemp(join(tmpdir(), "kepr-view-")); const viewRepo = join(view, fx.repo); await cp(join(fx.bucketDir, fx.repo), viewRepo, { recursive: true }); // A strip older than every save here: a strip bans its path from all of history, so no save // keeps it, however much later it was made or arrives. await mkdir(join(viewRepo, "events"), { recursive: true }); await writeFile( join(viewRepo, "events", "0000000000bb"), JSON.stringify({ id: "0000000000bb", kind: "strip", save_id: "", path: "secret.txt", reason: "a secret got saved", created_at: 1, machine: "elsewhere", }), ); const oldest = fx.saveIds.at(-1) as string; const newest = fx.saveIds[0] as string; // First sync: the strip and the oldest save only. await writeFile(join(viewRepo, "head"), oldest); const t = await syncTarget({ ...fx, bucketDir: view }); await syncAll(t); expect(fileAt(t.mirror(), oldest, "secret.txt")).toBeNull(); // Then the rest arrive, each made long after the strip. await writeFile(join(viewRepo, "head"), newest); await syncAll(t); expect(headId(t.mirror())).toBe(newest); for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull(); expect(fileAt(t.mirror(), newest, "src/main.ts")).not.toBeNull(); }); describe("the mirror's schema version", () => { test("a version 2 mirror reopens and any other version is refused as disposable", async () => { const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-")); const created = openMirror(dir, "one"); created.run("PRAGMA wal_checkpoint(TRUNCATE)"); await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db"))); expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({ user_version: 2, }); for (const version of [1, 3]) { const other = new Database(join(dir, `v${version}.db`)); other.run(`PRAGMA user_version = ${version}`); other.close(); expect(() => openMirror(dir, `v${version}`)).toThrow("disposable"); } }); }); test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => { const fx = await makeKohBucket("older-strip"); const t = await syncTarget(fx); await syncAll(t); // Every save in the mirror holds secret.txt … for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).not.toBeNull(); // … and now a strip arrives whose own time is older than all of them, which is what a // strip made on another machine and offered late looks like. Every save is reached, whatever // its time. const events = join(fx.bucketDir, fx.repo, "events"); await mkdir(events, { recursive: true }); await writeFile( join(events, "0000000000aa"), JSON.stringify({ id: "0000000000aa", kind: "strip", save_id: "", path: "secret.txt", reason: "a secret got saved", created_at: 1, machine: "elsewhere", }), ); await syncAll(t); for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull(); expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull(); }); test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => { const fx = await makeKohBucket("quarantine"); const head = fx.saveIds[0] as string; const key = join(fx.bucketDir, fx.repo, "saves", head); const record = JSON.parse(await readFile(key, "utf8")); record.files.push({ ...record.files[0], path: "Icon\r" }); await writeFile(key, JSON.stringify(record)); const inst = await instance(fx.bucketDir); const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo); expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({ last_sync_error: null, }); expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md"); expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r"); await writeFile(join(fx.repoDir, "later.txt"), "later\n"); const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" }; Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env }); Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env }); await syncAll({ bucket: openBucket({ kind: "folder", path: fx.bucketDir }), kepr: inst.kepr, mirrorDir: join(inst.dataDir, "mirror"), tarballDir: join(inst.dataDir, "tarballs"), }); expect(history(mirror, 1, 0)[0]?.message).toBe("a later save"); const browser = new Browser(inst.app); await browser.signIn(); const page = await (await browser.get("/admin/repos")).text(); expect(page).toContain("1 path quarantined: Icon\\x0d"); }); test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => { const fx = await makeKohBucket("illegal-strip"); for (const [id, path] of [ ["aaaaaaaaaaaa", "src//"], ["bbbbbbbbbbbb", ""], ["cccccccccccc", "src\ud800"], ]) { const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" }; await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event)); } const t = await syncTarget(fx); await syncAll(t); expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({ last_sync_error: null, }); expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain( "src/main.ts", ); }); test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => { const fx = await makeKohBucket("oversized"); const head = fx.saveIds[0] as string; const key = join(fx.bucketDir, fx.repo, "saves", head); const record = JSON.parse(await readFile(key, "utf8")); const hash = "ab".repeat(32); record.files.push({ path: "huge.bin", hash, executable: false }); await writeFile(key, JSON.stringify(record)); await writeFile( join(fx.bucketDir, fx.repo, "objects", hash), deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)), ); const t = await syncTarget(fx); await syncAll(t); expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({ last_sync_error: null, }); expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin"); expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]); }); test("the line count skips a file over 8 MiB (A3)", async () => { const fx = await makeKohBucket("large-text"); const t = await syncTarget(fx); await syncAll(t); const before = repoStats(t.mirror()).lines; await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024)); const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" }; Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env }); Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env }); await syncAll(t); expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file"); expect(repoStats(t.mirror()).lines).toBe(before); });