import { beforeAll, describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { gunzipSync } from "node:zlib"; import type { Hono } from "hono"; import { openBucket } from "../src/bucket/reader.ts"; import { type SyncTarget, syncAll } from "../src/bucket/sync.ts"; import { createApp } from "../src/server/app.ts"; import { openKeprDb } from "../src/server/db.ts"; import { buildTarball, isStripped } from "../src/tarball/tar.ts"; import { type Fixture, makeKohBucket } from "./fixture.ts"; let fx: Fixture; let app: Hono; let target: SyncTarget; let head: string; let dataDir: string; const get = (path: string) => app.request(path); const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer()); // Every root-level file on Kepr 2's strip list, the .agent folder, and the look-alikes that must stay. const EXTRA: Record = { "AGENTS.md": "agents\n", "CONTINUITY.md": "notes\n", "Cargo.lock": "lock\n", "koh.build": "build\n", ".gitignore": "x\n", ".agent/CONTINUITY.md": "state\n", ".agentry/keep.txt": "kept\n", "flake.nix": "{ outputs = _: { }; }\n", "flake.lock": "{}\n", "build.zig.zon": ".{}\n", "nested/go.sum": "kept\n", "run.sh": "#!/bin/sh\necho run\n", }; beforeAll(async () => { fx = await makeKohBucket("tarry"); for (const [path, text] of Object.entries(EXTRA)) { await mkdir(join(fx.repoDir, path, ".."), { recursive: true }); await writeFile(join(fx.repoDir, path), text); } await chmod(join(fx.repoDir, "run.sh"), 0o755); const home = join(fx.repoDir, "../../home"); const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" }; const save = Bun.spawnSync(["koh3", "save", "add build files"], { cwd: fx.repoDir, env }); expect(save.exitCode).toBe(0); const log = Bun.spawnSync(["koh3", "log", "--json"], { cwd: fx.repoDir, env }); head = JSON.parse(log.stdout.toString()).head_id; dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-")); const kepr = openKeprDb(dataDir); target = { bucket: openBucket({ kind: "folder", path: fx.bucketDir }), kepr, mirrorDir: join(dataDir, "mirror"), tarballDir: join(dataDir, "tarballs"), }; await syncAll(target); kepr.run("UPDATE repos SET public = 1"); app = createApp( { domain: "kepr.example", dataDir, adminPasswordHash: "$argon2id$x", bucket: { kind: "folder", path: fx.bucketDir }, }, kepr, ); }); function tar(args: string[], input: Uint8Array): string { const result = Bun.spawnSync(["tar", ...args], { stdin: input }); expect(result.exitCode).toBe(0); return result.stdout.toString(); } describe("the tar format", () => { const header = (tarBytes: Uint8Array, index: number) => tarBytes.subarray(index * 512, index * 512 + 512); const field = (h: Uint8Array, from: number, to: number) => new TextDecoder().decode(h.subarray(from, to)); test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => { const gz = buildTarball("r-12345678", [ { path: "b.txt", bytes: new TextEncoder().encode("hello source"), executable: false }, { path: "a.sh", bytes: new Uint8Array(), executable: true }, ]); expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]); const t = gunzipSync(gz); const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)]; expect(field(root, 0, 12)).toBe("r-12345678/\0"); expect(field(root, 100, 156)).toBe( `0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`, ); expect(field(root, 156, 157)).toBe("5"); expect(field(root, 257, 265)).toBe("ustar\u000000"); expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0"); expect(field(a, 100, 108)).toBe("0000755\0"); expect(field(b, 100, 108)).toBe("0000644\0"); expect(field(b, 124, 136)).toBe("00000000014\0"); expect(field(b, 156, 157)).toBe("0"); for (const h of [root, a, b]) { let sum = 0; for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte; expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `); } expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source"); expect(t.length).toBe(6 * 512); }); test("splits a long path into the ustar prefix, and refuses one that can't be split", () => { const long = `${"d".repeat(40)}/${"x".repeat(85)}`; const t = gunzipSync( buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]), ); const h = header(t, 1); expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`); expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85)); expect( tar( ["-tzf", "-"], buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]), ), ).toContain(`r/${long}`); expect(() => buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array(), executable: false }]), ).toThrow("path too long"); }); test("the strip list matches the repo root only", () => { for (const path of [ "koh.build", "kepr.build", "go.sum", "LOG.md", ".agent", ".agent/a/b.md", ".gitignore", ]) expect(isStripped(path)).toBe(true); for (const path of [ "flake.lock", "build.zig.zon", "nested/go.sum", "docs/AGENTS.md", ".agentry/x", "log.md", ]) expect(isStripped(path)).toBe(false); }); }); describe("the archive routes", () => { test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => { const response = await get("/tarry/archive/HEAD.tar.gz"); expect(response.status).toBe(302); expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`); expect(response.headers.get("cache-control")).toBe("no-cache"); expect(response.headers.get("link")).toBe( `; rel="immutable"`, ); }); test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => { const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`); expect(short.status).toBe(302); expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`); expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404); expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404); target.kepr.run("UPDATE repos SET public = 0"); const hidden = [ await get("/tarry/archive/HEAD.tar.gz"), await get(`/tarry/archive/${head}.tar.gz`), ]; target.kepr.run("UPDATE repos SET public = 1"); expect(hidden.map((r) => r.status)).toEqual([404, 404]); }); test("serves the save's files under -/ minus the strip list, executable bit kept", async () => { const response = await get(`/tarry/archive/${head}.tar.gz`); expect(response.status).toBe(200); expect(response.headers.get("content-type")).toBe("application/gzip"); expect(response.headers.get("cache-control")).toBe("public, max-age=31536000"); expect(response.headers.get("content-disposition")).toBe( `attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`, ); const gz = new Uint8Array(await response.arrayBuffer()); const root = `tarry-${head.slice(0, 8)}`; const listed = tar(["-tzf", "-"], gz).trim().split("\n"); expect(listed).toEqual([ `${root}/`, ...[ ".agentry/keep.txt", "README.md", "build.zig.zon", "docs/guide.md", "docs/logo.png", "flake.lock", "flake.nix", "nested/go.sum", "run.sh", "secret.txt", "src/lib/deep/util.ts", "src/main.ts", "vendor/big.c", ].map((path) => `${root}/${path}`), ]); const out = await mkdtemp(join(tmpdir(), "kepr-untar-")); Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz }); expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello"); expect(await readFile(join(out, root, "docs/logo.png"))).toEqual( await readFile(join(fx.repoDir, "docs/logo.png")), ); expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/); }); test("is the same bytes when rebuilt, and in another process", async () => { const first = await bytesOf(`/tarry/archive/${head}.tar.gz`); const { dropTarballs } = await import("../src/tarball/cache.ts"); dropTarballs(target.tarballDir, "tarry", head); expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first); const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}"; import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}"; process.stdout.write(cachedTarball(process.argv[1], openMirror(process.argv[2], "tarry"), "tarry", process.argv[3]));`; const other = Bun.spawnSync([ "bun", "-e", script, await mkdtemp(join(tmpdir(), "kepr-tb-")), target.mirrorDir, head, ]); expect(new Uint8Array(other.stdout)).toEqual(first); }); test("a strip event drops the cached tarball, so the next one leaves the file out", async () => { await get(`/tarry/archive/${head}.tar.gz`); const cached = join(target.tarballDir, "tarry", `${head}.tar.gz`); expect(existsSync(cached)).toBe(true); await fx.strip("secret.txt"); await syncAll(target); expect(existsSync(cached)).toBe(false); const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`)); expect(listed).not.toContain("secret.txt"); expect(listed).toContain("src/main.ts"); }); });