import { beforeAll, describe, expect, test } from "bun:test"; import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { gunzipSync } from "node:zlib"; import type { Hono } from "hono"; import { type Bucket, openBucket } from "../src/bucket/reader.ts"; import { type SyncTarget, syncAll, syncRepo } from "../src/bucket/sync.ts"; import { openMirror } from "../src/mirror/store.ts"; import { createApp } from "../src/server/app.ts"; import { openKeprDb } from "../src/server/db.ts"; import { cachedTarball } from "../src/tarball/cache.ts"; import { buildTarball, isStripped } from "../src/tarball/tar.ts"; import { type Fixture, makeKohBucket } from "./fixture.ts"; let fx: Fixture; let app: Hono; let target: SyncTarget; let head: string; let dataDir: string; const get = (path: string) => app.request(path); const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer()); // Whether a tarball of HEAD is kept in the folder bucket under this many strips. const kept = (strips: number) => [ ...new Bun.Glob(`${strips}-*/${head}.tar.gz`).scanSync( join(fx.bucketDir, ".kepr/tarballs/tarry"), ), ].length > 0; // Every root-level file on Kepr 2's strip list, the .agent folder, and the look-alikes that must stay. const EXTRA: Record = { "AGENTS.md": "agents\n", "CONTINUITY.md": "notes\n", "Cargo.lock": "lock\n", "koh.build": "build\n", ".gitignore": "x\n", ".agent/CONTINUITY.md": "state\n", ".agentry/keep.txt": "kept\n", "flake.nix": "{ outputs = _: { }; }\n", "flake.lock": "{}\n", "build.zig.zon": ".{}\n", "nested/go.sum": "kept\n", "run.sh": "#!/bin/sh\necho run\n", }; beforeAll(async () => { fx = await makeKohBucket("tarry"); for (const [path, text] of Object.entries(EXTRA)) { await mkdir(join(fx.repoDir, path, ".."), { recursive: true }); await writeFile(join(fx.repoDir, path), text); } await chmod(join(fx.repoDir, "run.sh"), 0o755); const home = join(fx.repoDir, "../../home"); const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" }; const save = Bun.spawnSync(["koh3", "save", "add build files"], { cwd: fx.repoDir, env }); expect(save.exitCode).toBe(0); const log = Bun.spawnSync(["koh3", "view", "--json"], { cwd: fx.repoDir, env }); head = JSON.parse(log.stdout.toString()).head_id; dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-")); const kepr = openKeprDb(dataDir); target = { bucket: openBucket({ kind: "folder", path: fx.bucketDir }), kepr, mirrorDir: join(dataDir, "mirror"), }; await syncAll(target); kepr.run("UPDATE repos SET public = 1"); app = createApp( { domain: "kepr.example", dataDir, adminPasswordHash: "$argon2id$x", bucket: { kind: "folder", path: fx.bucketDir }, }, kepr, ); }); function tar(args: string[], input: Uint8Array): string { const result = Bun.spawnSync(["tar", ...args], { stdin: input }); expect(result.exitCode).toBe(0); return result.stdout.toString(); } describe("the tar format", () => { const header = (tarBytes: Uint8Array, index: number) => tarBytes.subarray(index * 512, index * 512 + 512); const field = (h: Uint8Array, from: number, to: number) => new TextDecoder().decode(h.subarray(from, to)); test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => { const gz = buildTarball("r-12345678", [ { path: "b.txt", bytes: new TextEncoder().encode("hello source") }, { path: "a.sh", bytes: new Uint8Array() }, ]); expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]); const t = gunzipSync(gz); const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)]; expect(field(root, 0, 12)).toBe("r-12345678/\0"); expect(field(root, 100, 156)).toBe( `0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`, ); expect(field(root, 156, 157)).toBe("5"); expect(field(root, 257, 265)).toBe("ustar\u000000"); expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0"); // Every file is 0644, executable or not, exactly as Kepr 2 wrote them (§22.3). expect(field(a, 100, 108)).toBe("0000644\0"); expect(field(b, 100, 108)).toBe("0000644\0"); expect(field(b, 124, 136)).toBe("00000000014\0"); expect(field(b, 156, 157)).toBe("0"); for (const h of [root, a, b]) { let sum = 0; for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte; expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `); } expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source"); expect(t.length).toBe(6 * 512); }); test("writes executable archives at 0755 and plain archives at 0644", () => { const executableTar = gunzipSync( buildTarball( "r", [{ path: "run.sh", bytes: new TextEncoder().encode("#!/bin/sh\n") }], 0o755, ), ); const plainTar = gunzipSync( buildTarball("r", [{ path: "README.md", bytes: new TextEncoder().encode("read me\n") }]), ); expect(field(header(executableTar, 1), 100, 108)).toBe("0000755\0"); expect(field(header(plainTar, 1), 100, 108)).toBe("0000644\0"); }); test("splits a long path into the ustar prefix, and refuses one that can't be split", () => { const long = `${"d".repeat(40)}/${"x".repeat(85)}`; const t = gunzipSync(buildTarball("r", [{ path: long, bytes: new Uint8Array() }])); const h = header(t, 1); expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`); expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85)); expect( tar(["-tzf", "-"], buildTarball("r", [{ path: long, bytes: new Uint8Array() }])), ).toContain(`r/${long}`); expect(() => buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array() }])).toThrow( "path too long", ); }); test("the strip list matches the repo root only", () => { for (const path of [ "koh.build", "kepr.build", "go.sum", "LOG.md", ".agent", ".agent/a/b.md", ".gitignore", ]) expect(isStripped(path)).toBe(true); for (const path of [ "flake.lock", "build.zig.zon", "nested/go.sum", "docs/AGENTS.md", ".agentry/x", "log.md", ]) expect(isStripped(path)).toBe(false); }); }); describe("the archive routes", () => { test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => { const response = await get("/tarry/archive/HEAD.tar.gz"); expect(response.status).toBe(302); expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`); expect(response.headers.get("cache-control")).toBe("no-cache"); expect(response.headers.get("link")).toBe( `; rel="immutable"`, ); }); test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => { const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`); expect(short.status).toBe(302); expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`); expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404); expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404); target.kepr.run("UPDATE repos SET public = 0"); const hidden = [ await get("/tarry/archive/HEAD.tar.gz"), await get(`/tarry/archive/${head}.tar.gz`), ]; target.kepr.run("UPDATE repos SET public = 1"); expect(hidden.map((r) => r.status)).toEqual([404, 404]); }); test("serves the save's files under -/ minus the strip list", async () => { const response = await get(`/tarry/archive/${head}.tar.gz`); expect(response.status).toBe(200); expect(response.headers.get("content-type")).toBe("application/gzip"); expect(response.headers.get("cache-control")).toBe("private, max-age=31536000"); expect(response.headers.get("content-disposition")).toBe( `attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`, ); const gz = new Uint8Array(await response.arrayBuffer()); const root = `tarry-${head.slice(0, 8)}`; const listed = tar(["-tzf", "-"], gz).trim().split("\n"); expect(listed).toEqual([ `${root}/`, ...[ ".agentry/keep.txt", "README.md", "build.zig.zon", "docs/guide.md", "docs/logo.png", "flake.lock", "flake.nix", "nested/go.sum", "run.sh", "secret.txt", "src/lib/deep/util.ts", "src/main.ts", "vendor/big.c", ].map((path) => `${root}/${path}`), ]); const out = await mkdtemp(join(tmpdir(), "kepr-untar-")); Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz }); expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello"); expect(await readFile(join(out, root, "docs/logo.png"))).toEqual( await readFile(join(fx.repoDir, "docs/logo.png")), ); expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/); }); test("is the same bytes when rebuilt, and in another process", async () => { const first = await bytesOf(`/tarry/archive/${head}.tar.gz`); const { dropTarballs } = await import("../src/tarball/cache.ts"); await dropTarballs(target.bucket, "tarry"); expect(kept(0)).toBe(false); expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first); // Every path the other process touches is a fresh temp dir, its working directory included, // so nothing it writes can land in the repo. const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}"; import { openBucket } from "${join(import.meta.dir, "../src/bucket/reader.ts")}"; import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}"; const { BUCKET_DIR, MIRROR_DIR, SAVE_ID } = process.env; const bucket = openBucket({ kind: "folder", path: BUCKET_DIR }); process.stdout.write(await cachedTarball(bucket, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`; const scratch = await mkdtemp(join(tmpdir(), "kepr-tb-")); const other = Bun.spawnSync(["bun", "-e", script], { cwd: scratch, env: { PATH: process.env.PATH, BUCKET_DIR: join(scratch, "bucket"), MIRROR_DIR: target.mirrorDir, SAVE_ID: head, }, }); expect(other.stderr.toString()).toBe(""); expect(new Uint8Array(other.stdout)).toEqual(first); }); test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => { await get(`/tarry/archive/${head}.tar.gz`); expect(kept(0)).toBe(true); await fx.strip("secret.txt"); // syncRepo lands the strip without syncAll's cleanup: a crash between the two. await syncRepo(target.bucket, openMirror(target.mirrorDir, "tarry"), "tarry"); expect(kept(0)).toBe(true); const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`)); expect(listed).not.toContain("secret.txt"); expect(listed).toContain("src/main.ts"); expect(kept(1)).toBe(true); }); test("a sync that lands a strip clears the repo's old cached tarballs", async () => { await get(`/tarry/archive/${head}.tar.gz`); await fx.strip("vendor/big.c"); await syncAll(target); expect(kept(1)).toBe(false); expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain( "big.c", ); }); test("a strip landing while a kept tarball is fetched is not served", async () => { const mirror = openMirror(target.mirrorDir, "tarry"); await get(`/tarry/archive/${head}.tar.gz`); let struck = false; const racing: Bucket = { ...target.bucket, async get(key) { const bytes = await target.bucket.get(key); if (!struck) { struck = true; await fx.strip("docs/guide.md"); await syncRepo(target.bucket, mirror, "tarry"); } return bytes; }, }; const tarball = await cachedTarball(racing, mirror, "tarry", head); const listed = tar(["-tzf", "-"], tarball ?? new Uint8Array()); expect(listed).not.toContain("guide.md"); expect(listed).toContain("src/main.ts"); }); }); describe("the archives panel", () => { test("lists HEAD and recent saves with tarball links, and nix lines where there is a flake", async () => { const response = await get("/tarry/archives"); expect(response.status).toBe(200); expect(response.headers.get("cache-control")).toBe("public, max-age=60"); const page = await response.text(); const ids = [...page.matchAll(/class="history-id"[^>]*>([0-9a-f]{12}) m[1]); expect(ids).toEqual([head, fx.saveIds[0], fx.saveIds[2], fx.saveIds[3]]); expect(page).toContain('HEAD'); expect(page).toContain('class="archive-row archive-row--head"'); expect(page).toContain(`data-copy="${head}"`); expect(page).toContain('data-copy="koh steal kepr.example/tarry"'); expect(page).toContain( 'data-copy="curl -fL https://kepr.example/tarry/archive/HEAD.tar.gz | tar xz"', ); expect(page).toContain('href="/tarry/archive/HEAD.tar.gz">↓ tarry-HEAD.tar.gz'); expect(page).toContain( 'data-copy="nix profile add https://kepr.example/tarry/archive/HEAD.tar.gz"', ); // Only HEAD has flake.nix; older saves get a tarball link but no nix line. expect(page.match(/nix profile add/g)?.length).toBe(2); const older = fx.saveIds[0] as string; expect(page).toContain( `href="/tarry/archive/${older}.tar.gz">↓ tarry-${older.slice(0, 8)}.tar.gz`, ); expect(page).not.toContain(fx.saveIds[1] as string); }); test("a private repo's panel is a 404", async () => { target.kepr.run("UPDATE repos SET public = 0"); const status = (await get("/tarry/archives")).status; target.kepr.run("UPDATE repos SET public = 1"); expect(status).toBe(404); }); });