1import { beforeAll, describe, expect, test } from "bun:test";2import { existsSync } from "node:fs";3import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { gunzipSync } from "node:zlib";7import type { Hono } from "hono";8import { openBucket } from "../src/bucket/reader.ts";9import { type SyncTarget, syncAll } from "../src/bucket/sync.ts";10import { createApp } from "../src/server/app.ts";11import { openKeprDb } from "../src/server/db.ts";12import { buildTarball, isStripped } from "../src/tarball/tar.ts";13import { type Fixture, makeKohBucket } from "./fixture.ts";1415let fx: Fixture;16let app: Hono;17let target: SyncTarget;18let head: string;19let dataDir: string;20const get = (path: string) => app.request(path);21const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());222324const EXTRA: Record<string, string> = {25 "AGENTS.md": "agents\n",26 "CONTINUITY.md": "notes\n",27 "Cargo.lock": "lock\n",28 "koh.build": "build\n",29 ".gitignore": "x\n",30 ".agent/CONTINUITY.md": "state\n",31 ".agentry/keep.txt": "kept\n",32 "flake.nix": "{ outputs = _: { }; }\n",33 "flake.lock": "{}\n",34 "build.zig.zon": ".{}\n",35 "nested/go.sum": "kept\n",36 "run.sh": "#!/bin/sh\necho run\n",37};3839beforeAll(async () => {40 fx = await makeKohBucket("tarry");41 for (const [path, text] of Object.entries(EXTRA)) {42 await mkdir(join(fx.repoDir, path, ".."), { recursive: true });43 await writeFile(join(fx.repoDir, path), text);44 }45 await chmod(join(fx.repoDir, "run.sh"), 0o755);46 const home = join(fx.repoDir, "../../home");47 const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };48 const save = Bun.spawnSync(["koh3", "save", "add build files"], { cwd: fx.repoDir, env });49 expect(save.exitCode).toBe(0);50 const log = Bun.spawnSync(["koh3", "log", "--json"], { cwd: fx.repoDir, env });51 head = JSON.parse(log.stdout.toString()).head_id;52 dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-"));53 const kepr = openKeprDb(dataDir);54 target = {55 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),56 kepr,57 mirrorDir: join(dataDir, "mirror"),58 tarballDir: join(dataDir, "tarballs"),59 };60 await syncAll(target);61 kepr.run("UPDATE repos SET public = 1");62 app = createApp(63 {64 domain: "kepr.example",65 dataDir,66 adminPasswordHash: "$argon2id$x",67 bucket: { kind: "folder", path: fx.bucketDir },68 },69 kepr,70 );71});7273function tar(args: string[], input: Uint8Array): string {74 const result = Bun.spawnSync(["tar", ...args], { stdin: input });75 expect(result.exitCode).toBe(0);76 return result.stdout.toString();77}7879describe("the tar format", () => {80 const header = (tarBytes: Uint8Array, index: number) =>81 tarBytes.subarray(index * 512, index * 512 + 512);82 const field = (h: Uint8Array, from: number, to: number) =>83 new TextDecoder().decode(h.subarray(from, to));8485 test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => {86 const gz = buildTarball("r-12345678", [87 { path: "b.txt", bytes: new TextEncoder().encode("hello source"), executable: false },88 { path: "a.sh", bytes: new Uint8Array(), executable: true },89 ]);90 expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]);91 const t = gunzipSync(gz);92 const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)];93 expect(field(root, 0, 12)).toBe("r-12345678/\0");94 expect(field(root, 100, 156)).toBe(95 `0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`,96 );97 expect(field(root, 156, 157)).toBe("5");98 expect(field(root, 257, 265)).toBe("ustar\u000000");99 expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0");100 expect(field(a, 100, 108)).toBe("0000755\0");101 expect(field(b, 100, 108)).toBe("0000644\0");102 expect(field(b, 124, 136)).toBe("00000000014\0");103 expect(field(b, 156, 157)).toBe("0");104 for (const h of [root, a, b]) {105 let sum = 0;106 for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte;107 expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `);108 }109 expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source");110 expect(t.length).toBe(6 * 512);111 });112113 test("splits a long path into the ustar prefix, and refuses one that can't be split", () => {114 const long = `${"d".repeat(40)}/${"x".repeat(85)}`;115 const t = gunzipSync(116 buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),117 );118 const h = header(t, 1);119 expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`);120 expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85));121 expect(122 tar(123 ["-tzf", "-"],124 buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),125 ),126 ).toContain(`r/${long}`);127 expect(() =>128 buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array(), executable: false }]),129 ).toThrow("path too long");130 });131132 test("the strip list matches the repo root only", () => {133 for (const path of [134 "koh.build",135 "kepr.build",136 "go.sum",137 "LOG.md",138 ".agent",139 ".agent/a/b.md",140 ".gitignore",141 ])142 expect(isStripped(path)).toBe(true);143 for (const path of [144 "flake.lock",145 "build.zig.zon",146 "nested/go.sum",147 "docs/AGENTS.md",148 ".agentry/x",149 "log.md",150 ])151 expect(isStripped(path)).toBe(false);152 });153});154155describe("the archive routes", () => {156 test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => {157 const response = await get("/tarry/archive/HEAD.tar.gz");158 expect(response.status).toBe(302);159 expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);160 expect(response.headers.get("cache-control")).toBe("no-cache");161 expect(response.headers.get("link")).toBe(162 `<https://kepr.example/tarry/archive/${head}.tar.gz>; rel="immutable"`,163 );164 });165166 test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => {167 const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`);168 expect(short.status).toBe(302);169 expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);170 expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404);171 expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404);172 target.kepr.run("UPDATE repos SET public = 0");173 const hidden = [174 await get("/tarry/archive/HEAD.tar.gz"),175 await get(`/tarry/archive/${head}.tar.gz`),176 ];177 target.kepr.run("UPDATE repos SET public = 1");178 expect(hidden.map((r) => r.status)).toEqual([404, 404]);179 });180181 test("serves the save's files under <repo>-<id8>/ minus the strip list, executable bit kept", async () => {182 const response = await get(`/tarry/archive/${head}.tar.gz`);183 expect(response.status).toBe(200);184 expect(response.headers.get("content-type")).toBe("application/gzip");185 expect(response.headers.get("cache-control")).toBe("public, max-age=31536000");186 expect(response.headers.get("content-disposition")).toBe(187 `attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`,188 );189 const gz = new Uint8Array(await response.arrayBuffer());190 const root = `tarry-${head.slice(0, 8)}`;191 const listed = tar(["-tzf", "-"], gz).trim().split("\n");192 expect(listed).toEqual([193 `${root}/`,194 ...[195 ".agentry/keep.txt",196 "README.md",197 "build.zig.zon",198 "docs/guide.md",199 "docs/logo.png",200 "flake.lock",201 "flake.nix",202 "nested/go.sum",203 "run.sh",204 "secret.txt",205 "src/lib/deep/util.ts",206 "src/main.ts",207 "vendor/big.c",208 ].map((path) => `${root}/${path}`),209 ]);210 const out = await mkdtemp(join(tmpdir(), "kepr-untar-"));211 Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz });212 expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello");213 expect(await readFile(join(out, root, "docs/logo.png"))).toEqual(214 await readFile(join(fx.repoDir, "docs/logo.png")),215 );216 expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/);217 });218219 test("is the same bytes when rebuilt, and in another process", async () => {220 const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);221 const { dropTarballs } = await import("../src/tarball/cache.ts");222 dropTarballs(target.tarballDir, "tarry", head);223 expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);224 const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";225import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";226process.stdout.write(cachedTarball(process.argv[1], openMirror(process.argv[2], "tarry"), "tarry", process.argv[3]));`;227 const other = Bun.spawnSync([228 "bun",229 "-e",230 script,231 await mkdtemp(join(tmpdir(), "kepr-tb-")),232 target.mirrorDir,233 head,234 ]);235 expect(new Uint8Array(other.stdout)).toEqual(first);236 });237238 test("a strip event drops the cached tarball, so the next one leaves the file out", async () => {239 await get(`/tarry/archive/${head}.tar.gz`);240 const cached = join(target.tarballDir, "tarry", `${head}.tar.gz`);241 expect(existsSync(cached)).toBe(true);242 await fx.strip("secret.txt");243 await syncAll(target);244 expect(existsSync(cached)).toBe(false);245 const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));246 expect(listed).not.toContain("secret.txt");247 expect(listed).toContain("src/main.ts");248 });249});