1import { Database } from "bun:sqlite";2import { beforeAll, describe, expect, test } from "bun:test";3import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { deflateSync } from "node:zlib";7import { type Bucket, openBucket } from "../src/bucket/reader.ts";8import { syncAll, syncRepo } from "../src/bucket/sync.ts";9import {10 fileAt,11 filesAt,12 flagReason,13 headId,14 history,15 objectBytes,16 repoStats,17 saveByPrefix,18 timeline,19} from "../src/mirror/queries.ts";20import { applyHidden, closeMirror, mirrorState, openMirror } from "../src/mirror/store.ts";21import { openKeprDb } from "../src/server/db.ts";22import { Browser, instance } from "./browser.ts";23import { type Fixture, makeKohBucket } from "./fixture.ts";2425const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());2627async function syncTarget(fx: Fixture) {28 const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));29 const target = {30 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),31 kepr: openKeprDb(dataDir),32 mirrorDir: join(dataDir, "mirror"),33 };34 return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };35}3637describe("a first sync of a koh3 bucket", () => {38 let fx: Fixture;39 let t: Awaited<ReturnType<typeof syncTarget>>;40 beforeAll(async () => {41 fx = await makeKohBucket();42 t = await syncTarget(fx);43 await syncAll(t);44 });45 const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);4647 test("registers the repo private, with a clean last sync", () => {48 const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {49 public: number;50 last_sync_at: number;51 last_sync_error: string | null;52 };53 expect(row.public).toBe(0);54 expect(row.last_sync_error).toBeNull();55 expect(row.last_sync_at).toBeGreaterThan(0);56 });5758 test("history is newest first and leaves out the checkpoint", () => {59 expect(headId(t.mirror())).toBe(head());60 expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);61 expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);62 expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);63 expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);64 });6566 test("finds a save by a unique prefix of six or more characters", () => {67 const save = saveByPrefix(t.mirror(), second().slice(0, 6));68 expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);69 expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();70 expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();71 });7273 test("reads files and their bytes at any save", () => {74 const paths = filesAt(t.mirror(), head()).map((f) => f.path);75 expect(paths).toContain("src/lib/deep/util.ts");76 const now = fileAt(t.mirror(), head(), "src/main.ts");77 const then = fileAt(t.mirror(), second(), "src/main.ts");78 expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");79 expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");80 expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();81 expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);82 });8384 test("counts saves without checkpoints, and lines and language at head", () => {85 const stats = repoStats(t.mirror());86 expect(stats.saves).toBe(3);87 expect(stats.language).toBe("TypeScript");88 89 90 expect(stats.lines).toBe(4);91 expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);92 });9394 test("stores the flag event", () => {95 const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();96 expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });97 });9899 test("a save's flag state is its newest flag or clear event, by time and then ID", async () => {100 expect(flagReason(t.mirror(), first())).toBe("holds a secret");101 expect(flagReason(t.mirror(), head())).toBeNull();102 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-flags-")), "flags");103 const event = mirror.query(104 "INSERT INTO events (id, kind, save_id, reason, created_at, machine) VALUES (?, ?, 'aaaaaaaaaaaa', ?, ?, 'm')",105 );106 event.run("000000000001", "flag", "first", 10);107 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("first");108 event.run("000000000002", "clear", null, 20);109 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBeNull();110 111 event.run("000000000003", "flag", "again", 20);112 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");113 event.run("000000000000", "clear", null, 20);114 expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");115 });116117 test("a strip reaches the mirror and drops the object", async () => {118 const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;119 expect(objectBytes(t.mirror(), secret)).not.toBeNull();120 await fx.strip("secret.txt");121 await syncAll(t);122 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();123 expect(objectBytes(t.mirror(), secret)).toBeNull();124 expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();125 });126});127128test("a hidden path that is then stripped stays gone when it is unhidden", async () => {129 const fx = await makeKohBucket("hidden-strip");130 const t = await syncTarget(fx);131 await syncAll(t);132 const secret = fileAt(t.mirror(), fx.saveIds[0] as string, "secret.txt")?.hash as string;133 t.kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = ?").run(fx.repo);134 await syncAll(t);135 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();136 expect(objectBytes(t.mirror(), secret)).not.toBeNull();137138 await fx.strip("secret.txt");139 await syncAll(t);140 applyHidden(t.mirror(), []);141 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();142 expect(objectBytes(t.mirror(), secret)).toBeNull();143 expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();144});145146test("a whole six-character ID finds its own save, not a longer one it begins", async () => {147 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), "ids");148 const add = mirror.query(149 "INSERT INTO saves (id, parent, message, created_at, machine) VALUES (?, NULL, ?, 1, 'm')",150 );151 add.run("af96c8", "migrated from Koh 2");152 add.run("af96c8123456", "made by Koh 3");153 expect(saveByPrefix(mirror, "af96c8")?.message).toBe("migrated from Koh 2");154 expect(saveByPrefix(mirror, "af96c81")?.message).toBe("made by Koh 3");155});156157test("tells its target about each repo that synced cleanly, and not one that failed", async () => {158 const fx = await makeKohBucket("told");159 const t = await syncTarget(fx);160 const synced: string[] = [];161 await syncAll({ ...t, synced: (repo) => synced.push(repo) });162 expect(synced).toEqual(["told"]);163 await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");164 await syncAll({ ...t, synced: (repo) => synced.push(repo) });165 expect(synced).toEqual(["told"]);166});167168test("a sync lands history in batches, so one cut short resumes where it stopped", async () => {169 const fx = await makeKohBucket("batched");170 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });171 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);172 173 const failing: Bucket = {174 ...bucket,175 get: (key) =>176 key.includes("/objects/") && mirrorState(mirror, "head") !== null177 ? Promise.reject(new Error("connection reset"))178 : bucket.get(key),179 };180 await expect(syncRepo(failing, mirror, fx.repo, [], 1)).rejects.toThrow("connection reset");181 const first = fx.saveIds[3] as string;182 expect(mirrorState(mirror, "head")).toBe(first);183 expect(history(mirror, 10, 0).map((s) => s.id)).toEqual([first]);184185 await syncRepo(bucket, mirror, fx.repo, [], 1);186 expect(headId(mirror)).toBe(fx.saveIds[0] as string);187 expect(history(mirror, 10, 0)).toHaveLength(3);188 expect(flagReason(mirror, first)).toBe("holds a secret");189});190191test("a save larger than a batch is stored ahead of itself, and a sync cut short within it resumes", async () => {192 const fx = await makeKohBucket("ahead");193 const bucket = openBucket({ kind: "folder", path: fx.bucketDir });194 const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-mirror-")), fx.repo);195 let objectGets = 0;196 const failing: Bucket = {197 ...bucket,198 get: (key) =>199 key.includes("/objects/") && ++objectGets > 2200 ? Promise.reject(new Error("connection reset"))201 : bucket.get(key),202 };203 await expect(syncRepo(failing, mirror, fx.repo, [], 1)).rejects.toThrow("connection reset");204 const stored = () =>205 mirror.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM objects").get()?.n;206 expect(mirrorState(mirror, "head")).toBeNull();207 expect(stored()).toBe(2);208209 await syncRepo(bucket, mirror, fx.repo, [], 1);210 expect(headId(mirror)).toBe(fx.saveIds[0] as string);211 expect(history(mirror, 10, 0)).toHaveLength(3);212 const guide = fileAt(mirror, fx.saveIds[0] as string, "docs/guide.md");213 expect(text(objectBytes(mirror, guide?.hash as string))).toBe(214 "# Guide\n\nRead me. It is short.\n",215 );216});217218test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {219 const fx = await makeKohBucket("growing");220 const t = await syncTarget(fx);221 await syncAll(t);222 await writeFile(join(fx.repoDir, "new.txt"), "new\n");223 Bun.spawnSync(["koh3", "save", "add new"], {224 cwd: fx.repoDir,225 env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },226 });227 await syncAll(t);228 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");229230 await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");231 await syncAll(t);232 const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {233 last_sync_error: string;234 };235 expect(row.last_sync_error).toContain('remote format "koh-remote 2"');236});237238test("a whole-history strip reaches every save, those arriving after it too, whatever their time (§10.2)", async () => {239 const fx = await makeKohBucket("late");240 const view = await mkdtemp(join(tmpdir(), "kepr-view-"));241 const viewRepo = join(view, fx.repo);242 await cp(join(fx.bucketDir, fx.repo), viewRepo, { recursive: true });243 244 245 await mkdir(join(viewRepo, "events"), { recursive: true });246 await writeFile(247 join(viewRepo, "events", "0000000000bb"),248 JSON.stringify({249 id: "0000000000bb",250 kind: "strip",251 save_id: "",252 path: "secret.txt",253 reason: "a secret got saved",254 created_at: 1,255 machine: "elsewhere",256 }),257 );258 const oldest = fx.saveIds.at(-1) as string;259 const newest = fx.saveIds[0] as string;260 261 await writeFile(join(viewRepo, "head"), oldest);262 const t = await syncTarget({ ...fx, bucketDir: view });263 await syncAll(t);264 expect(fileAt(t.mirror(), oldest, "secret.txt")).toBeNull();265 266 await writeFile(join(viewRepo, "head"), newest);267 await syncAll(t);268 expect(headId(t.mirror())).toBe(newest);269 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();270 expect(fileAt(t.mirror(), newest, "src/main.ts")).not.toBeNull();271});272273describe("the mirror's schema version", () => {274 test("a version 3 mirror reopens and any other version is refused as disposable", async () => {275 const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));276 const created = openMirror(dir, "one");277 created.run("PRAGMA wal_checkpoint(TRUNCATE)");278 await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));279 expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({280 user_version: 3,281 });282283 for (const version of [1, 4]) {284 const other = new Database(join(dir, `v${version}.db`));285 other.run(`PRAGMA user_version = ${version}`);286 other.close();287 expect(() => openMirror(dir, `v${version}`)).toThrow("disposable");288 }289 });290291 test("a version 2 mirror moves to 3 in place, emptied only when it deleted hidden rows", async () => {292 const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));293 for (const [name, hidden] of [294 ["plain", ""],295 ["hiding", "secret.txt"],296 ] as const) {297 const db = openMirror(dir, name);298 db.run("DROP TABLE hidden_files");299 db.run("PRAGMA user_version = 2");300 db.query("INSERT INTO state (key, value) VALUES ('head', 'abcdef123456'), ('hidden', ?)").run(301 hidden,302 );303 closeMirror(dir, name);304 }305 const head = (name: string) => mirrorState(openMirror(dir, name), "head");306 expect(head("plain")).toBe("abcdef123456");307 expect(head("hiding")).toBeNull();308 for (const name of ["plain", "hiding"]) {309 const db = openMirror(dir, name);310 expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: 3 });311 expect(db.query("SELECT COUNT(*) AS n FROM hidden_files").get()).toEqual({ n: 0 });312 }313 });314});315316test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => {317 const fx = await makeKohBucket("older-strip");318 const t = await syncTarget(fx);319 await syncAll(t);320 321 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).not.toBeNull();322323 324 325 326 const events = join(fx.bucketDir, fx.repo, "events");327 await mkdir(events, { recursive: true });328 await writeFile(329 join(events, "0000000000aa"),330 JSON.stringify({331 id: "0000000000aa",332 kind: "strip",333 save_id: "",334 path: "secret.txt",335 reason: "a secret got saved",336 created_at: 1,337 machine: "elsewhere",338 }),339 );340 await syncAll(t);341342 for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();343 expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();344});345346test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => {347 const fx = await makeKohBucket("quarantine");348 const head = fx.saveIds[0] as string;349 const key = join(fx.bucketDir, fx.repo, "saves", head);350 const record = JSON.parse(await readFile(key, "utf8"));351 record.files.push({ ...record.files[0], path: "Icon\r" });352 await writeFile(key, JSON.stringify(record));353 const inst = await instance(fx.bucketDir);354 const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo);355 expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({356 last_sync_error: null,357 });358 expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md");359 expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r");360361 await writeFile(join(fx.repoDir, "later.txt"), "later\n");362 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };363 Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env });364 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });365 await syncAll({366 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),367 kepr: inst.kepr,368 mirrorDir: join(inst.dataDir, "mirror"),369 });370 expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");371372 const browser = new Browser(inst.app);373 await browser.signIn();374 const page = await (await browser.get("/admin/repos")).text();375 expect(page).toContain("1 path quarantined: Icon\\x0d");376});377378test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => {379 const fx = await makeKohBucket("illegal-strip");380 for (const [id, path] of [381 ["aaaaaaaaaaaa", "src//"],382 ["bbbbbbbbbbbb", ""],383 ["cccccccccccc", "src\ud800"],384 ]) {385 const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };386 await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event));387 }388 const t = await syncTarget(fx);389 await syncAll(t);390 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({391 last_sync_error: null,392 });393 expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain(394 "src/main.ts",395 );396});397398test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {399 const fx = await makeKohBucket("oversized");400 const head = fx.saveIds[0] as string;401 const key = join(fx.bucketDir, fx.repo, "saves", head);402 const record = JSON.parse(await readFile(key, "utf8"));403 const hash = "ab".repeat(32);404 record.files.push({ path: "huge.bin", hash, executable: false });405 await writeFile(key, JSON.stringify(record));406 await writeFile(407 join(fx.bucketDir, fx.repo, "objects", hash),408 deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)),409 );410 const t = await syncTarget(fx);411 await syncAll(t);412 expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({413 last_sync_error: null,414 });415 expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin");416 expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);417});418419test("the line count skips a file over 8 MiB (A3)", async () => {420 const fx = await makeKohBucket("large-text");421 const t = await syncTarget(fx);422 await syncAll(t);423 const before = repoStats(t.mirror()).lines;424 await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024));425 const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };426 Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env });427 Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });428 await syncAll(t);429 expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");430 expect(repoStats(t.mirror()).lines).toBe(before);431});