1import { Database } from "bun:sqlite";2import { beforeAll, describe, expect, test } from "bun:test";3import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { deflateSync } from "node:zlib";7import { type Bucket, openBucket } from "../src/bucket/reader.ts";8import { readSave } from "../src/bucket/repo.ts";9import { syncAll, syncRepo } from "../src/bucket/sync.ts";10import {11 fileAt,12 filesAt,13 flagReason,14 headId,15 history,16 objectBytes,17 repoStats,18 saveByPrefix,19 timeline,20} from "../src/mirror/queries.ts";21import { applyHidden, closeMirror, mirrorState, openMirror } from "../src/mirror/store.ts";22import { openKeprDb } from "../src/server/db.ts";23import { Browser, instance } from "./browser.ts";24import { type Fixture, makeKohBucket } from "./fixture.ts";2526const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());2728async function syncTarget(fx: Fixture) {29 const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));30 const target = {31 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),32 kepr: openKeprDb(dataDir),33 mirrorDir: join(dataDir, "mirror"),34 };35 return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };36}3738describe("a first sync of a koh3 bucket", () => {39 let fx: Fixture;40 let t: Awaited<ReturnType<typeof syncTarget>>;41 beforeAll(async () => {42 fx = await makeKohBucket();43 t = await syncTarget(fx);44 await syncAll(t);45 });46 const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);4748 test("registers the repo private, with a clean last sync", () => {49 const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {50 public: number;51 last_sync_at: number;52 last_sync_error: string | null;53 };54 expect(row.public).toBe(0);55 expect(row.last_sync_error).toBeNull();56 expect(row.last_sync_at).toBeGreaterThan(0);57 });5859 test("history is newest first and leaves out the checkpoint", () => {60 expect(headId(t.mirror())).toBe(head());61 expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);62 expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);63 expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);64 expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);65 });6667 test("finds a save by a unique prefix of six or more characters", () => {68 const save = saveByPrefix(t.mirror(), second().slice(0, 6));69 expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);70 expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();71 expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();72 });7374 test("reads files and their bytes at any save", () => {75 const paths = filesAt(t.mirror(), head()).map((f) => f.path);76 expect(paths).toContain("src/lib/deep/util.ts");77 const now = fileAt(t.mirror(), head(), "src/main.ts");78 const then = fileAt(t.mirror(), second(), "src/main.ts");79 expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");80 expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");81 expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();82 expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);83 });8485 test("counts saves without checkpoints, and lines and language at head", () => {86 const stats = repoStats(t.mirror());87 expect(stats.saves).toBe(3);88 expect(stats.language).toBe("TypeScript");89 90 91 expect(stats.lines).toBe(4);92 expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);93 });9495 test("stores the flag event", () => {96 const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();97 expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });98 });99100 test("a save's flag state is its newest flag or clear event, by time and then ID", async () => {101 expect(flagReason(t.mirror(), first())).toBe("holds a secret");102 expect(flagReason(t.mirror(), head())).toBeNull();103 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-flags-")), "flags");104 const event = mirror.query(105 "INSERT INTO events (id, kind, save_id, reason, created_at, machine) VALUES (?, ?, 'aaaaaaaaaaaa', ?, ?, 'm')",106 );107 event.run("000000000001", "flag", "first", 10);108 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("first");109 event.run("000000000002", "clear", null, 20);110 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBeNull();111 112 event.run("000000000003", "flag", "again", 20);113 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");114 event.run("000000000000", "clear", null, 20);115 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");116 });117118 test("a strip reaches the mirror and drops the object", async () => {119 const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;120 expect(objectBytes(t.mirror(), secret)).not.toBeNull();121 await fx.strip("secret.txt");122 await syncAll(t);123 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();124 expect(objectBytes(t.mirror(), secret)).toBeNull();125 expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();126 });127});128129test("a hidden path that is then stripped stays gone when it is unhidden", async () => {130 const fx = await makeKohBucket("hidden-strip");131 const t = await syncTarget(fx);132 await syncAll(t);133 const secret = fileAt(t.mirror(), fx.saveIds[0] as string, "secret.txt")?.hash as string;134 t.kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = ?").run(fx.repo);135 await syncAll(t);136 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();137 expect(objectBytes(t.mirror(), secret)).not.toBeNull();138139 await fx.strip("secret.txt");140 await syncAll(t);141 applyHidden(t.mirror(), []);142 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();143 expect(objectBytes(t.mirror(), secret)).toBeNull();144 expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();145});146147test("a whole six-character ID finds its own save, not a longer one it begins", async () => {148 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), "ids");149 const add = mirror.query(150 "INSERT INTO saves (id, parent, message, created_at, machine) VALUES (?, NULL, ?, 1, 'm')",151 );152 add.run("af96c8", "migrated from Koh 2");153 add.run("af96c8123456", "made by Koh 3");154 expect(saveByPrefix(mirror, "af96c8")?.message).toBe("migrated from Koh 2");155 expect(saveByPrefix(mirror, "af96c81")?.message).toBe("made by Koh 3");156});157158test("tells its target about each repo that synced cleanly, and not one that failed", async () => {159 const fx = await makeKohBucket("told");160 const t = await syncTarget(fx);161 const synced: string[] = [];162 await syncAll({ ...t, synced: (repo) => synced.push(repo) });163 expect(synced).toEqual(["told"]);164 await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");165 await syncAll({ ...t, synced: (repo) => synced.push(repo) });166 expect(synced).toEqual(["told"]);167});168169test("a sync lands history in batches, so one cut short resumes where it stopped", async () => {170 const fx = await makeKohBucket("batched");171 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });172 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);173 174 const failing: Bucket = {175 ...bucket,176 get: (key) =>177 key.includes("/objects/") && mirrorState(mirror, "head") !== null178 ? Promise.reject(new Error("connection reset"))179 : bucket.get(key),180 };181 await expect(syncRepo(failing, mirror, fx.repo, [], 1)).rejects.toThrow("connection reset");182 const first = fx.saveIds[3] as string;183 expect(mirrorState(mirror, "head")).toBe(first);184 expect(history(mirror, 10, 0).map((s) => s.id)).toEqual([first]);185186 await syncRepo(bucket, mirror, fx.repo, [], 1);187 expect(headId(mirror)).toBe(fx.saveIds[0] as string);188 expect(history(mirror, 10, 0)).toHaveLength(3);189 expect(flagReason(mirror, first)).toBe("holds a secret");190});191192test("a save larger than a batch is stored ahead of itself, and a sync cut short within it resumes", async () => {193 const fx = await makeKohBucket("ahead");194 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });195 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);196 197 const first = await readSave(bucket, fx.repo, fx.saveIds.at(-1) as string);198 const firstObjects = new Set(first.files.map((file) => file.hash)).size;199 let objectGets = 0;200 const failing: Bucket = {201 ...bucket,202 get: (key) =>203 key.includes("/objects/") && ++objectGets > firstObjects204 ? Promise.reject(new Error("connection reset"))205 : bucket.get(key),206 };207 await expect(syncRepo(failing, mirror, fx.repo, [], 1)).rejects.toThrow("connection reset");208 const stored = () =>209 mirror.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM objects").get()?.n;210 expect(mirrorState(mirror, "head")).toBeNull();211 expect(stored()).toBe(firstObjects);212213 await syncRepo(bucket, mirror, fx.repo, [], 1);214 expect(headId(mirror)).toBe(fx.saveIds[0] as string);215 expect(history(mirror, 10, 0)).toHaveLength(3);216 const guide = fileAt(mirror, fx.saveIds[0] as string, "docs/guide.md");217 expect(text(objectBytes(mirror, guide?.hash as string))).toBe(218 "# Guide\n\nRead me. It is short.\n",219 );220});221222test("object reads overlap, but never past the bytes allowed in flight", async () => {223 const fx = await makeKohBucket("in-flight");224 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });225 const peak = async (readBytes?: number) => {226 let reading = 0;227 let most = 0;228 const counting: Bucket = {229 ...bucket,230 get: async (key) => {231 if (!key.includes("/objects/")) return bucket.get(key);232 most = Math.max(most, ++reading);233 await Bun.sleep(5);234 reading--;235 return bucket.get(key);236 },237 };238 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);239 await syncRepo(counting, mirror, fx.repo, [], undefined, readBytes);240 expect(headId(mirror)).toBe(fx.saveIds[0] as string);241 return most;242 };243 expect(await peak()).toBeGreaterThan(1);244 245 expect(await peak(1)).toBe(1);246});247248test("a read the gate admits after the batch filled waits for the next batch", async () => {249 const fx = await makeKohBucket("deferred");250 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });251 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);252 const stored = () =>253 mirror.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM objects").get()?.n ?? 0;254 let read = 0;255 const unstored: number[] = [];256 const counting: Bucket = {257 ...bucket,258 get: async (key) => {259 260 if (key.includes("/objects/")) unstored.push(read++ - stored());261 return bucket.get(key);262 },263 };264 await syncRepo(counting, mirror, fx.repo, [], 1, 1);265 expect(read).toBeGreaterThan(2);266 expect(new Set(unstored)).toEqual(new Set([0]));267});268269test("a failed read reports only once every other read has settled", async () => {270 const fx = await makeKohBucket("settled");271 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });272 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);273 let reading = 0;274 let reads = 0;275 const failing: Bucket = {276 ...bucket,277 get: async (key) => {278 if (!key.includes("/objects/")) return bucket.get(key);279 if (reads++ === 0) throw new Error("connection reset");280 reading++;281 await Bun.sleep(30);282 reading--;283 return bucket.get(key);284 },285 };286 await expect(syncRepo(failing, mirror, fx.repo)).rejects.toThrow("connection reset");287 expect(reading).toBe(0);288});289290test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {291 const fx = await makeKohBucket("growing");292 const t = await syncTarget(fx);293 await syncAll(t);294 await writeFile(join(fx.repoDir, "new.txt"), "new\n");295 Bun.spawnSync(["koh3", "save", "add new"], {296 cwd: fx.repoDir,297 env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },298 });299 await syncAll(t);300 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");301302 await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");303 await syncAll(t);304 const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {305 last_sync_error: string;306 };307 expect(row.last_sync_error).toContain('remote format "koh-remote 2"');308});309310test("a whole-history strip reaches every save, those arriving after it too, whatever their time (§10.2)", async () => {311 const fx = await makeKohBucket("late");312 const view = await mkdtemp(join(tmpdir(), "kepr-view-"));313 const viewRepo = join(view, fx.repo);314 await cp(join(fx.bucketDir, fx.repo), viewRepo, { recursive: true });315 316 317 await mkdir(join(viewRepo, "events"), { recursive: true });318 await writeFile(319 join(viewRepo, "events", "0000000000bb"),320 JSON.stringify({321 id: "0000000000bb",322 kind: "strip",323 save_id: "",324 path: "secret.txt",325 reason: "a secret got saved",326 created_at: 1,327 machine: "elsewhere",328 }),329 );330 const oldest = fx.saveIds.at(-1) as string;331 const newest = fx.saveIds[0] as string;332 333 await writeFile(join(viewRepo, "head"), oldest);334 const t = await syncTarget({ ...fx, bucketDir: view });335 await syncAll(t);336 expect(fileAt(t.mirror(), oldest, "secret.txt")).toBeNull();337 338 await writeFile(join(viewRepo, "head"), newest);339 await syncAll(t);340 expect(headId(t.mirror())).toBe(newest);341 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();342 expect(fileAt(t.mirror(), newest, "src/main.ts")).not.toBeNull();343});344345describe("the mirror's schema version", () => {346 test("a version 3 mirror reopens and any other version is refused as disposable", async () => {347 const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));348 const created = openMirror(dir, "one");349 created.run("PRAGMA wal_checkpoint(TRUNCATE)");350 await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));351 expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({352 user_version: 3,353 });354355 for (const version of [1, 4]) {356 const other = new Database(join(dir, `v${version}.db`));357 other.run(`PRAGMA user_version = ${version}`);358 other.close();359 expect(() => openMirror(dir, `v${version}`)).toThrow("disposable");360 }361 });362363 test("a version 2 mirror moves to 3 in place, emptied only when it deleted hidden rows", async () => {364 const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));365 for (const [name, hidden] of [366 ["plain", ""],367 ["hiding", "secret.txt"],368 ] as const) {369 const db = openMirror(dir, name);370 db.run("DROP TABLE hidden_files");371 db.run("PRAGMA user_version = 2");372 db.query("INSERT INTO state (key, value) VALUES ('head', 'abcdef123456'), ('hidden', ?)").run(373 hidden,374 );375 closeMirror(dir, name);376 }377 const head = (name: string) => mirrorState(openMirror(dir, name), "head");378 expect(head("plain")).toBe("abcdef123456");379 expect(head("hiding")).toBeNull();380 for (const name of ["plain", "hiding"]) {381 const db = openMirror(dir, name);382 expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: 3 });383 expect(db.query("SELECT COUNT(*) AS n FROM hidden_files").get()).toEqual({ n: 0 });384 }385 });386});387388test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => {389 const fx = await makeKohBucket("older-strip");390 const t = await syncTarget(fx);391 await syncAll(t);392 393 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).not.toBeNull();394395 396 397 398 const events = join(fx.bucketDir, fx.repo, "events");399 await mkdir(events, { recursive: true });400 await writeFile(401 join(events, "0000000000aa"),402 JSON.stringify({403 id: "0000000000aa",404 kind: "strip",405 save_id: "",406 path: "secret.txt",407 reason: "a secret got saved",408 created_at: 1,409 machine: "elsewhere",410 }),411 );412 await syncAll(t);413414 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();415 expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();416});417418test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => {419 const fx = await makeKohBucket("quarantine");420 const head = fx.saveIds[0] as string;421 const key = join(fx.bucketDir, fx.repo, "saves", head);422 const record = JSON.parse(await readFile(key, "utf8"));423 record.files.push({ ...record.files[0], path: "Icon\r" });424 await writeFile(key, JSON.stringify(record));425 const inst = await instance(fx.bucketDir);426 const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo);427 expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({428 last_sync_error: null,429 });430 expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md");431 expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r");432433 await writeFile(join(fx.repoDir, "later.txt"), "later\n");434 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };435 Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env });436 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });437 await syncAll({438 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),439 kepr: inst.kepr,440 mirrorDir: join(inst.dataDir, "mirror"),441 });442 expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");443444 const browser = new Browser(inst.app);445 await browser.signIn();446 const page = await (await browser.get("/admin/repos")).text();447 expect(page).toContain("1 path quarantined: Icon\\x0d");448});449450test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => {451 const fx = await makeKohBucket("illegal-strip");452 for (const [id, path] of [453 ["aaaaaaaaaaaa", "src//"],454 ["bbbbbbbbbbbb", ""],455 ["cccccccccccc", "src\ud800"],456 ]) {457 const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };458 await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event));459 }460 const t = await syncTarget(fx);461 await syncAll(t);462 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({463 last_sync_error: null,464 });465 expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain(466 "src/main.ts",467 );468});469470test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {471 const fx = await makeKohBucket("oversized");472 const head = fx.saveIds[0] as string;473 const key = join(fx.bucketDir, fx.repo, "saves", head);474 const record = JSON.parse(await readFile(key, "utf8"));475 const hash = "ab".repeat(32);476 record.files.push({ path: "huge.bin", hash, executable: false });477 await writeFile(key, JSON.stringify(record));478 await writeFile(479 join(fx.bucketDir, fx.repo, "objects", hash),480 deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)),481 );482 const t = await syncTarget(fx);483 await syncAll(t);484 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({485 last_sync_error: null,486 });487 expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin");488 expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);489});490491test("the line count skips a file over 8 MiB (A3)", async () => {492 const fx = await makeKohBucket("large-text");493 const t = await syncTarget(fx);494 await syncAll(t);495 const before = repoStats(t.mirror()).lines;496 await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024));497 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };498 Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env });499 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });500 await syncAll(t);501 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");502 expect(repoStats(t.mirror()).lines).toBe(before);503});