1import { beforeAll, describe, expect, test } from "bun:test";2import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";3import { tmpdir } from "node:os";4import { join } from "node:path";5import { gunzipSync } from "node:zlib";6import type { Hono } from "hono";7import { type Bucket, openBucket } from "../src/bucket/reader.ts";8import { type SyncTarget, syncAll, syncRepo } from "../src/bucket/sync.ts";9import { openMirror } from "../src/mirror/store.ts";10import { createApp } from "../src/server/app.ts";11import { openKeprDb } from "../src/server/db.ts";12import { cachedTarball } from "../src/tarball/cache.ts";13import { buildTarball, isStripped } from "../src/tarball/tar.ts";14import { type Fixture, makeKohBucket } from "./fixture.ts";1516let fx: Fixture;17let app: Hono;18let target: SyncTarget;19let head: string;20let dataDir: string;21const get = (path: string) => app.request(path);22const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());2324const kept = (strips: number) =>25 [26 ...new Bun.Glob(`${strips}-*/${head}.tar.gz`).scanSync(27 join(fx.bucketDir, ".kepr/tarballs/tarry"),28 ),29 ].length > 0;303132const EXTRA: Record<string, string> = {33 "AGENTS.md": "agents\n",34 "CONTINUITY.md": "notes\n",35 "Cargo.lock": "lock\n",36 "koh.build": "build\n",37 ".gitignore": "x\n",38 ".agent/CONTINUITY.md": "state\n",39 ".agentry/keep.txt": "kept\n",40 "flake.nix": "{ outputs = _: { }; }\n",41 "flake.lock": "{}\n",42 "build.zig.zon": ".{}\n",43 "nested/go.sum": "kept\n",44 "run.sh": "#!/bin/sh\necho run\n",45};4647beforeAll(async () => {48 fx = await makeKohBucket("tarry");49 for (const [path, text] of Object.entries(EXTRA)) {50 await mkdir(join(fx.repoDir, path, ".."), { recursive: true });51 await writeFile(join(fx.repoDir, path), text);52 }53 await chmod(join(fx.repoDir, "run.sh"), 0o755);54 const home = join(fx.repoDir, "../../home");55 const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };56 const save = Bun.spawnSync(["koh", "save", "add build files"], { cwd: fx.repoDir, env });57 expect(save.exitCode).toBe(0);58 const log = Bun.spawnSync(["koh", "view", "--json"], { cwd: fx.repoDir, env });59 head = JSON.parse(log.stdout.toString()).head_id;60 dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-"));61 const kepr = openKeprDb(dataDir);62 target = {63 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),64 kepr,65 mirrorDir: join(dataDir, "mirror"),66 };67 await syncAll(target);68 kepr.run("UPDATE repos SET public = 1");69 app = createApp(70 {71 domain: "kepr.example",72 dataDir,73 adminPasswordHash: "$argon2id$x",74 bucket: { kind: "folder", path: fx.bucketDir },75 },76 kepr,77 );78});7980function tar(args: string[], input: Uint8Array): string {81 const result = Bun.spawnSync(["tar", ...args], { stdin: input });82 expect(result.exitCode).toBe(0);83 return result.stdout.toString();84}8586describe("the tar format", () => {87 const header = (tarBytes: Uint8Array, index: number) =>88 tarBytes.subarray(index * 512, index * 512 + 512);89 const field = (h: Uint8Array, from: number, to: number) =>90 new TextDecoder().decode(h.subarray(from, to));9192 test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => {93 const gz = buildTarball("r-12345678", [94 { path: "b.txt", bytes: new TextEncoder().encode("hello source") },95 { path: "a.sh", bytes: new Uint8Array() },96 ]);97 expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]);98 const t = gunzipSync(gz);99 const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)];100 expect(field(root, 0, 12)).toBe("r-12345678/\0");101 expect(field(root, 100, 156)).toBe(102 `0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`,103 );104 expect(field(root, 156, 157)).toBe("5");105 expect(field(root, 257, 265)).toBe("ustar\u000000");106 expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0");107 108 expect(field(a, 100, 108)).toBe("0000644\0");109 expect(field(b, 100, 108)).toBe("0000644\0");110 expect(field(b, 124, 136)).toBe("00000000014\0");111 expect(field(b, 156, 157)).toBe("0");112 for (const h of [root, a, b]) {113 let sum = 0;114 for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte;115 expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `);116 }117 expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source");118 expect(t.length).toBe(6 * 512);119 });120121 test("writes executable archives at 0755 and plain archives at 0644", () => {122 const executableTar = gunzipSync(123 buildTarball(124 "r",125 [{ path: "run.sh", bytes: new TextEncoder().encode("#!/bin/sh\n") }],126 0o755,127 ),128 );129 const plainTar = gunzipSync(130 buildTarball("r", [{ path: "README.md", bytes: new TextEncoder().encode("read me\n") }]),131 );132 expect(field(header(executableTar, 1), 100, 108)).toBe("0000755\0");133 expect(field(header(plainTar, 1), 100, 108)).toBe("0000644\0");134 });135136 test("splits a long path into the ustar prefix, and refuses one that can't be split", () => {137 const long = `${"d".repeat(40)}/${"x".repeat(85)}`;138 const t = gunzipSync(buildTarball("r", [{ path: long, bytes: new Uint8Array() }]));139 const h = header(t, 1);140 expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`);141 expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85));142 expect(143 tar(["-tzf", "-"], buildTarball("r", [{ path: long, bytes: new Uint8Array() }])),144 ).toContain(`r/${long}`);145 expect(() => buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array() }])).toThrow(146 "path too long",147 );148 });149150 test("the strip list matches the repo root only", () => {151 for (const path of [152 "koh.build",153 "kepr.build",154 "go.sum",155 "LOG.md",156 ".agent",157 ".agent/a/b.md",158 ".gitignore",159 ])160 expect(isStripped(path)).toBe(true);161 for (const path of [162 "flake.lock",163 "build.zig.zon",164 "nested/go.sum",165 "docs/AGENTS.md",166 ".agentry/x",167 "log.md",168 ])169 expect(isStripped(path)).toBe(false);170 });171});172173describe("the archive routes", () => {174 test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => {175 const response = await get("/tarry/archive/HEAD.tar.gz");176 expect(response.status).toBe(302);177 expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);178 expect(response.headers.get("cache-control")).toBe("no-cache");179 expect(response.headers.get("link")).toBe(180 `<https://kepr.example/tarry/archive/${head}.tar.gz>; rel="immutable"`,181 );182 });183184 test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => {185 const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`);186 expect(short.status).toBe(302);187 expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);188 expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404);189 expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404);190 target.kepr.run("UPDATE repos SET public = 0");191 const hidden = [192 await get("/tarry/archive/HEAD.tar.gz"),193 await get(`/tarry/archive/${head}.tar.gz`),194 ];195 target.kepr.run("UPDATE repos SET public = 1");196 expect(hidden.map((r) => r.status)).toEqual([404, 404]);197 });198199 test("serves the save's files under <repo>-<id8>/ minus the strip list", async () => {200 const response = await get(`/tarry/archive/${head}.tar.gz`);201 expect(response.status).toBe(200);202 expect(response.headers.get("content-type")).toBe("application/gzip");203 expect(response.headers.get("cache-control")).toBe("private, max-age=31536000");204 expect(response.headers.get("content-disposition")).toBe(205 `attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`,206 );207 const gz = new Uint8Array(await response.arrayBuffer());208 const root = `tarry-${head.slice(0, 8)}`;209 const listed = tar(["-tzf", "-"], gz).trim().split("\n");210 expect(listed).toEqual([211 `${root}/`,212 ...[213 ".agentry/keep.txt",214 "README.md",215 "build.zig.zon",216 "docs/guide.md",217 "docs/logo.png",218 "flake.lock",219 "flake.nix",220 "nested/go.sum",221 "run.sh",222 "secret.txt",223 "src/lib/deep/util.ts",224 "src/main.ts",225 "vendor/big.c",226 ].map((path) => `${root}/${path}`),227 ]);228 const out = await mkdtemp(join(tmpdir(), "kepr-untar-"));229 Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz });230 expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello");231 expect(await readFile(join(out, root, "docs/logo.png"))).toEqual(232 await readFile(join(fx.repoDir, "docs/logo.png")),233 );234 expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/);235 });236237 test("is the same bytes when rebuilt, and in another process", async () => {238 const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);239 const { dropTarballs } = await import("../src/tarball/cache.ts");240 await dropTarballs(target.bucket, "tarry");241 expect(kept(0)).toBe(false);242 expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);243 244 245 const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";246import { openBucket } from "${join(import.meta.dir, "../src/bucket/reader.ts")}";247import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";248const { BUCKET_DIR, MIRROR_DIR, SAVE_ID } = process.env;249const bucket = openBucket({ kind: "folder", path: BUCKET_DIR });250process.stdout.write(await cachedTarball(bucket, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`;251 const scratch = await mkdtemp(join(tmpdir(), "kepr-tb-"));252 const other = Bun.spawnSync(["bun", "-e", script], {253 cwd: scratch,254 env: {255 PATH: process.env.PATH,256 BUCKET_DIR: join(scratch, "bucket"),257 MIRROR_DIR: target.mirrorDir,258 SAVE_ID: head,259 },260 });261 expect(other.stderr.toString()).toBe("");262 expect(new Uint8Array(other.stdout)).toEqual(first);263 });264265 test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => {266 await get(`/tarry/archive/${head}.tar.gz`);267 expect(kept(0)).toBe(true);268 await fx.strip("secret.txt");269 270 await syncRepo(target.bucket, openMirror(target.mirrorDir, "tarry"), "tarry");271 expect(kept(0)).toBe(true);272 const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));273 expect(listed).not.toContain("secret.txt");274 expect(listed).toContain("src/main.ts");275 expect(kept(1)).toBe(true);276 });277278 test("a sync that lands a strip clears the repo's old cached tarballs", async () => {279 await get(`/tarry/archive/${head}.tar.gz`);280 await fx.strip("vendor/big.c");281 await syncAll(target);282 expect(kept(1)).toBe(false);283 expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain(284 "big.c",285 );286 });287288 test("a strip landing while a kept tarball is fetched is not served", async () => {289 const mirror = openMirror(target.mirrorDir, "tarry");290 await get(`/tarry/archive/${head}.tar.gz`);291 let struck = false;292 const racing: Bucket = {293 ...target.bucket,294 async get(key) {295 const bytes = await target.bucket.get(key);296 if (!struck) {297 struck = true;298 await fx.strip("docs/guide.md");299 await syncRepo(target.bucket, mirror, "tarry");300 }301 return bytes;302 },303 };304 const tarball = await cachedTarball(racing, mirror, "tarry", head);305 const listed = tar(["-tzf", "-"], tarball ?? new Uint8Array());306 expect(listed).not.toContain("guide.md");307 expect(listed).toContain("src/main.ts");308 });309});310311describe("the archives panel", () => {312 test("lists HEAD and recent saves with tarball links, and nix lines where there is a flake", async () => {313 const response = await get("/tarry/archives");314 expect(response.status).toBe(200);315 expect(response.headers.get("cache-control")).toBe("public, max-age=60");316 const page = await response.text();317 const ids = [...page.matchAll(/class="history-id"[^>]*>([0-9a-f]{12})</g)].map((m) => m[1]);318 expect(ids).toEqual([head, fx.saveIds[0], fx.saveIds[2], fx.saveIds[3]]);319 expect(page).toContain('<span class="archive-head mono">HEAD</span>');320 expect(page).toContain('class="archive-row archive-row--head"');321 expect(page).toContain(`data-copy="${head}"`);322 expect(page).toContain('data-copy="koh steal kepr.example/tarry"');323 expect(page).toContain(324 'data-copy="curl -fL https://kepr.example/tarry/archive/HEAD.tar.gz | tar xz"',325 );326 expect(page).toContain('href="/tarry/archive/HEAD.tar.gz">↓ tarry-HEAD.tar.gz</a>');327 expect(page).toContain(328 'data-copy="nix profile add https://kepr.example/tarry/archive/HEAD.tar.gz"',329 );330 331 expect(page.match(/nix profile add/g)?.length).toBe(2);332 const older = fx.saveIds[0] as string;333 expect(page).toContain(334 `href="/tarry/archive/${older}.tar.gz">↓ tarry-${older.slice(0, 8)}.tar.gz</a>`,335 );336 expect(page).not.toContain(fx.saveIds[1] as string);337 });338339 test("a private repo's panel is a 404", async () => {340 target.kepr.run("UPDATE repos SET public = 0");341 const status = (await get("/tarry/archives")).status;342 target.kepr.run("UPDATE repos SET public = 1");343 expect(status).toBe(404);344 });345});