1import { Database } from "bun:sqlite";2import { beforeAll, describe, expect, test } from "bun:test";3import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { deflateSync } from "node:zlib";7import { openBucket } from "../src/bucket/reader.ts";8import { syncAll } from "../src/bucket/sync.ts";9import {10 fileAt,11 filesAt,12 flagReason,13 headId,14 history,15 objectBytes,16 repoStats,17 saveByPrefix,18 timeline,19} from "../src/mirror/queries.ts";20import { mirrorState, openMirror } from "../src/mirror/store.ts";21import { openKeprDb } from "../src/server/db.ts";22import { Browser, instance } from "./browser.ts";23import { type Fixture, makeKohBucket } from "./fixture.ts";2425const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());2627async function syncTarget(fx: Fixture) {28 const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));29 const target = {30 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),31 kepr: openKeprDb(dataDir),32 mirrorDir: join(dataDir, "mirror"),33 tarballDir: join(dataDir, "tarballs"),34 };35 return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };36}3738describe("a first sync of a koh3 bucket", () => {39 let fx: Fixture;40 let t: Awaited<ReturnType<typeof syncTarget>>;41 beforeAll(async () => {42 fx = await makeKohBucket();43 t = await syncTarget(fx);44 await syncAll(t);45 });46 const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);4748 test("registers the repo private, with a clean last sync", () => {49 const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {50 public: number;51 last_sync_at: number;52 last_sync_error: string | null;53 };54 expect(row.public).toBe(0);55 expect(row.last_sync_error).toBeNull();56 expect(row.last_sync_at).toBeGreaterThan(0);57 });5859 test("history is newest first and leaves out the checkpoint", () => {60 expect(headId(t.mirror())).toBe(head());61 expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);62 expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);63 expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);64 expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);65 });6667 test("finds a save by a unique prefix of six or more characters", () => {68 const save = saveByPrefix(t.mirror(), second().slice(0, 6));69 expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);70 expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();71 expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();72 });7374 test("reads files and their bytes at any save", () => {75 const paths = filesAt(t.mirror(), head()).map((f) => f.path);76 expect(paths).toContain("src/lib/deep/util.ts");77 const now = fileAt(t.mirror(), head(), "src/main.ts");78 const then = fileAt(t.mirror(), second(), "src/main.ts");79 expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");80 expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");81 expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();82 expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);83 });8485 test("counts saves without checkpoints, and lines and language at head", () => {86 const stats = repoStats(t.mirror());87 expect(stats.saves).toBe(3);88 expect(stats.language).toBe("TypeScript");89 90 expect(stats.lines).toBeGreaterThan(16);91 92 expect(stats.lines).toBeLessThan(100);93 expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);94 });9596 test("stores the flag event", () => {97 const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();98 expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });99 });100101 test("a save's flag state is its newest flag or clear event, by time and then ID", async () => {102 expect(flagReason(t.mirror(), first())).toBe("holds a secret");103 expect(flagReason(t.mirror(), head())).toBeNull();104 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-flags-")), "flags");105 const event = mirror.query(106 "INSERT INTO events (id, kind, save_id, reason, created_at, machine) VALUES (?, ?, 'aaaaaaaaaaaa', ?, ?, 'm')",107 );108 event.run("000000000001", "flag", "first", 10);109 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("first");110 event.run("000000000002", "clear", null, 20);111 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBeNull();112 113 event.run("000000000003", "flag", "again", 20);114 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");115 event.run("000000000000", "clear", null, 20);116 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");117 });118119 test("a strip reaches the mirror and drops the object", async () => {120 const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;121 expect(objectBytes(t.mirror(), secret)).not.toBeNull();122 await fx.strip("secret.txt");123 await syncAll(t);124 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();125 expect(objectBytes(t.mirror(), secret)).toBeNull();126 expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();127 });128});129130test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {131 const fx = await makeKohBucket("growing");132 const t = await syncTarget(fx);133 await syncAll(t);134 await writeFile(join(fx.repoDir, "new.txt"), "new\n");135 Bun.spawnSync(["koh3", "save", "add new"], {136 cwd: fx.repoDir,137 env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },138 });139 await syncAll(t);140 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");141142 await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");143 await syncAll(t);144 const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {145 last_sync_error: string;146 };147 expect(row.last_sync_error).toContain('remote format "koh-remote 2"');148});149150test("a whole-history strip reaches every save, those arriving after it too, whatever their time (§10.2)", async () => {151 const fx = await makeKohBucket("late");152 const view = await mkdtemp(join(tmpdir(), "kepr-view-"));153 const viewRepo = join(view, fx.repo);154 await cp(join(fx.bucketDir, fx.repo), viewRepo, { recursive: true });155 156 157 await mkdir(join(viewRepo, "events"), { recursive: true });158 await writeFile(159 join(viewRepo, "events", "0000000000bb"),160 JSON.stringify({161 id: "0000000000bb",162 kind: "strip",163 save_id: "",164 path: "secret.txt",165 reason: "a secret got saved",166 created_at: 1,167 machine: "elsewhere",168 }),169 );170 const oldest = fx.saveIds.at(-1) as string;171 const newest = fx.saveIds[0] as string;172 173 await writeFile(join(viewRepo, "head"), oldest);174 const t = await syncTarget({ ...fx, bucketDir: view });175 await syncAll(t);176 expect(fileAt(t.mirror(), oldest, "secret.txt")).toBeNull();177 178 await writeFile(join(viewRepo, "head"), newest);179 await syncAll(t);180 expect(headId(t.mirror())).toBe(newest);181 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();182 expect(fileAt(t.mirror(), newest, "src/main.ts")).not.toBeNull();183});184185describe("the mirror's schema version", () => {186 test("a version 2 mirror reopens and any other version is refused as disposable", async () => {187 const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));188 const created = openMirror(dir, "one");189 created.run("PRAGMA wal_checkpoint(TRUNCATE)");190 await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));191 expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({192 user_version: 2,193 });194195 for (const version of [1, 3]) {196 const other = new Database(join(dir, `v${version}.db`));197 other.run(`PRAGMA user_version = ${version}`);198 other.close();199 expect(() => openMirror(dir, `v${version}`)).toThrow("disposable");200 }201 });202});203204test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => {205 const fx = await makeKohBucket("older-strip");206 const t = await syncTarget(fx);207 await syncAll(t);208 209 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).not.toBeNull();210211 212 213 214 const events = join(fx.bucketDir, fx.repo, "events");215 await mkdir(events, { recursive: true });216 await writeFile(217 join(events, "0000000000aa"),218 JSON.stringify({219 id: "0000000000aa",220 kind: "strip",221 save_id: "",222 path: "secret.txt",223 reason: "a secret got saved",224 created_at: 1,225 machine: "elsewhere",226 }),227 );228 await syncAll(t);229230 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();231 expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();232});233234test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => {235 const fx = await makeKohBucket("quarantine");236 const head = fx.saveIds[0] as string;237 const key = join(fx.bucketDir, fx.repo, "saves", head);238 const record = JSON.parse(await readFile(key, "utf8"));239 record.files.push({ ...record.files[0], path: "Icon\r" });240 await writeFile(key, JSON.stringify(record));241 const inst = await instance(fx.bucketDir);242 const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo);243 expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({244 last_sync_error: null,245 });246 expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md");247 expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r");248249 await writeFile(join(fx.repoDir, "later.txt"), "later\n");250 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };251 Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env });252 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });253 await syncAll({254 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),255 kepr: inst.kepr,256 mirrorDir: join(inst.dataDir, "mirror"),257 tarballDir: join(inst.dataDir, "tarballs"),258 });259 expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");260261 const browser = new Browser(inst.app);262 await browser.signIn();263 const page = await (await browser.get("/admin/repos")).text();264 expect(page).toContain("1 path quarantined: Icon\\x0d");265});266267test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => {268 const fx = await makeKohBucket("illegal-strip");269 for (const [id, path] of [270 ["aaaaaaaaaaaa", "src//"],271 ["bbbbbbbbbbbb", ""],272 ["cccccccccccc", "src\ud800"],273 ]) {274 const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };275 await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event));276 }277 const t = await syncTarget(fx);278 await syncAll(t);279 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({280 last_sync_error: null,281 });282 expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain(283 "src/main.ts",284 );285});286287test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {288 const fx = await makeKohBucket("oversized");289 const head = fx.saveIds[0] as string;290 const key = join(fx.bucketDir, fx.repo, "saves", head);291 const record = JSON.parse(await readFile(key, "utf8"));292 const hash = "ab".repeat(32);293 record.files.push({ path: "huge.bin", hash, executable: false });294 await writeFile(key, JSON.stringify(record));295 await writeFile(296 join(fx.bucketDir, fx.repo, "objects", hash),297 deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)),298 );299 const t = await syncTarget(fx);300 await syncAll(t);301 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({302 last_sync_error: null,303 });304 expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin");305 expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);306});307308test("the line count skips a file over 8 MiB (A3)", async () => {309 const fx = await makeKohBucket("large-text");310 const t = await syncTarget(fx);311 await syncAll(t);312 const before = repoStats(t.mirror()).lines;313 await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024));314 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };315 Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env });316 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });317 await syncAll(t);318 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");319 expect(repoStats(t.mirror()).lines).toBe(before);320});