1import { beforeAll, describe, expect, test } from "bun:test";2import { existsSync } from "node:fs";3import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { gunzipSync } from "node:zlib";7import type { Hono } from "hono";8import { openBucket } from "../src/bucket/reader.ts";9import { type SyncTarget, syncAll, syncRepo } from "../src/bucket/sync.ts";10import { openMirror } from "../src/mirror/store.ts";11import { createApp } from "../src/server/app.ts";12import { openKeprDb } from "../src/server/db.ts";13import { buildTarball, isStripped } from "../src/tarball/tar.ts";14import { type Fixture, makeKohBucket } from "./fixture.ts";1516let fx: Fixture;17let app: Hono;18let target: SyncTarget;19let head: string;20let dataDir: string;21const get = (path: string) => app.request(path);22const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());2324const stored = (generation: string) =>25 join(fx.bucketDir, ".kepr", "tarballs", "tarry", generation, `${head}.tar.gz`);262728const EXTRA: Record<string, string> = {29 "AGENTS.md": "agents\n",30 "CONTINUITY.md": "notes\n",31 "Cargo.lock": "lock\n",32 "koh.build": "build\n",33 ".gitignore": "x\n",34 ".agent/CONTINUITY.md": "state\n",35 ".agentry/keep.txt": "kept\n",36 "flake.nix": "{ outputs = _: { }; }\n",37 "flake.lock": "{}\n",38 "build.zig.zon": ".{}\n",39 "nested/go.sum": "kept\n",40 "run.sh": "#!/bin/sh\necho run\n",41};4243beforeAll(async () => {44 fx = await makeKohBucket("tarry");45 for (const [path, text] of Object.entries(EXTRA)) {46 await mkdir(join(fx.repoDir, path, ".."), { recursive: true });47 await writeFile(join(fx.repoDir, path), text);48 }49 await chmod(join(fx.repoDir, "run.sh"), 0o755);50 const home = join(fx.repoDir, "../../home");51 const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };52 const save = Bun.spawnSync(["koh3", "save", "add build files"], { cwd: fx.repoDir, env });53 expect(save.exitCode).toBe(0);54 const log = Bun.spawnSync(["koh3", "view", "--json"], { cwd: fx.repoDir, env });55 head = JSON.parse(log.stdout.toString()).head_id;56 dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-"));57 const kepr = openKeprDb(dataDir);58 target = {59 bucket: openBucket({ kind: "folder", path: fx.bucketDir }),60 kepr,61 mirrorDir: join(dataDir, "mirror"),62 };63 await syncAll(target);64 kepr.run("UPDATE repos SET public = 1");65 app = createApp(66 {67 domain: "kepr.example",68 dataDir,69 adminPasswordHash: "$argon2id$x",70 bucket: { kind: "folder", path: fx.bucketDir },71 },72 kepr,73 );74});7576function tar(args: string[], input: Uint8Array): string {77 const result = Bun.spawnSync(["tar", ...args], { stdin: input });78 expect(result.exitCode).toBe(0);79 return result.stdout.toString();80}8182describe("the tar format", () => {83 const header = (tarBytes: Uint8Array, index: number) =>84 tarBytes.subarray(index * 512, index * 512 + 512);85 const field = (h: Uint8Array, from: number, to: number) =>86 new TextDecoder().decode(h.subarray(from, to));8788 test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => {89 const gz = buildTarball("r-12345678", [90 { path: "b.txt", bytes: new TextEncoder().encode("hello source") },91 { path: "a.sh", bytes: new Uint8Array() },92 ]);93 expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]);94 const t = gunzipSync(gz);95 const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)];96 expect(field(root, 0, 12)).toBe("r-12345678/\0");97 expect(field(root, 100, 156)).toBe(98 `0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`,99 );100 expect(field(root, 156, 157)).toBe("5");101 expect(field(root, 257, 265)).toBe("ustar\u000000");102 expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0");103 104 expect(field(a, 100, 108)).toBe("0000644\0");105 expect(field(b, 100, 108)).toBe("0000644\0");106 expect(field(b, 124, 136)).toBe("00000000014\0");107 expect(field(b, 156, 157)).toBe("0");108 for (const h of [root, a, b]) {109 let sum = 0;110 for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte;111 expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `);112 }113 expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source");114 expect(t.length).toBe(6 * 512);115 });116117 test("writes executable archives at 0755 and plain archives at 0644", () => {118 const executableTar = gunzipSync(119 buildTarball(120 "r",121 [{ path: "run.sh", bytes: new TextEncoder().encode("#!/bin/sh\n") }],122 0o755,123 ),124 );125 const plainTar = gunzipSync(126 buildTarball("r", [{ path: "README.md", bytes: new TextEncoder().encode("read me\n") }]),127 );128 expect(field(header(executableTar, 1), 100, 108)).toBe("0000755\0");129 expect(field(header(plainTar, 1), 100, 108)).toBe("0000644\0");130 });131132 test("splits a long path into the ustar prefix, and refuses one that can't be split", () => {133 const long = `${"d".repeat(40)}/${"x".repeat(85)}`;134 const t = gunzipSync(buildTarball("r", [{ path: long, bytes: new Uint8Array() }]));135 const h = header(t, 1);136 expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`);137 expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85));138 expect(139 tar(["-tzf", "-"], buildTarball("r", [{ path: long, bytes: new Uint8Array() }])),140 ).toContain(`r/${long}`);141 expect(() => buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array() }])).toThrow(142 "path too long",143 );144 });145146 test("the strip list matches the repo root only", () => {147 for (const path of [148 "koh.build",149 "kepr.build",150 "go.sum",151 "LOG.md",152 ".agent",153 ".agent/a/b.md",154 ".gitignore",155 ])156 expect(isStripped(path)).toBe(true);157 for (const path of [158 "flake.lock",159 "build.zig.zon",160 "nested/go.sum",161 "docs/AGENTS.md",162 ".agentry/x",163 "log.md",164 ])165 expect(isStripped(path)).toBe(false);166 });167});168169describe("the archive routes", () => {170 test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => {171 const response = await get("/tarry/archive/HEAD.tar.gz");172 expect(response.status).toBe(302);173 expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);174 expect(response.headers.get("cache-control")).toBe("no-cache");175 expect(response.headers.get("link")).toBe(176 `<https://kepr.example/tarry/archive/${head}.tar.gz>; rel="immutable"`,177 );178 });179180 test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => {181 const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`);182 expect(short.status).toBe(302);183 expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);184 expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404);185 expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404);186 target.kepr.run("UPDATE repos SET public = 0");187 const hidden = [188 await get("/tarry/archive/HEAD.tar.gz"),189 await get(`/tarry/archive/${head}.tar.gz`),190 ];191 target.kepr.run("UPDATE repos SET public = 1");192 expect(hidden.map((r) => r.status)).toEqual([404, 404]);193 });194195 test("serves the save's files under <repo>-<id8>/ minus the strip list", async () => {196 const response = await get(`/tarry/archive/${head}.tar.gz`);197 expect(response.status).toBe(200);198 expect(response.headers.get("content-type")).toBe("application/gzip");199 expect(response.headers.get("cache-control")).toBe("private, max-age=31536000");200 expect(response.headers.get("content-disposition")).toBe(201 `attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`,202 );203 const gz = new Uint8Array(await response.arrayBuffer());204 const root = `tarry-${head.slice(0, 8)}`;205 const listed = tar(["-tzf", "-"], gz).trim().split("\n");206 expect(listed).toEqual([207 `${root}/`,208 ...[209 ".agentry/keep.txt",210 "README.md",211 "build.zig.zon",212 "docs/guide.md",213 "docs/logo.png",214 "flake.lock",215 "flake.nix",216 "nested/go.sum",217 "run.sh",218 "secret.txt",219 "src/lib/deep/util.ts",220 "src/main.ts",221 "vendor/big.c",222 ].map((path) => `${root}/${path}`),223 ]);224 const out = await mkdtemp(join(tmpdir(), "kepr-untar-"));225 Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz });226 expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello");227 expect(await readFile(join(out, root, "docs/logo.png"))).toEqual(228 await readFile(join(fx.repoDir, "docs/logo.png")),229 );230 expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/);231 });232233 test("is the same bytes when rebuilt, and in another process", async () => {234 const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);235 const { dropTarballs } = await import("../src/tarball/cache.ts");236 await dropTarballs(target.bucket, "tarry");237 expect(existsSync(stored("0"))).toBe(false);238 expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);239 240 241 const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";242import { openBucket } from "${join(import.meta.dir, "../src/bucket/reader.ts")}";243import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";244const { BUCKET_DIR, MIRROR_DIR, SAVE_ID } = process.env;245const bucket = openBucket({ kind: "folder", path: BUCKET_DIR });246process.stdout.write(await cachedTarball(bucket, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`;247 const scratch = await mkdtemp(join(tmpdir(), "kepr-tb-"));248 const other = Bun.spawnSync(["bun", "-e", script], {249 cwd: scratch,250 env: {251 PATH: process.env.PATH,252 BUCKET_DIR: join(scratch, "bucket"),253 MIRROR_DIR: target.mirrorDir,254 SAVE_ID: head,255 },256 });257 expect(other.stderr.toString()).toBe("");258 expect(new Uint8Array(other.stdout)).toEqual(first);259 });260261 test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => {262 await get(`/tarry/archive/${head}.tar.gz`);263 const before = stored("0");264 expect(existsSync(before)).toBe(true);265 await fx.strip("secret.txt");266 267 await syncRepo(target.bucket, openMirror(target.mirrorDir, "tarry"), "tarry");268 expect(existsSync(before)).toBe(true);269 const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));270 expect(listed).not.toContain("secret.txt");271 expect(listed).toContain("src/main.ts");272 expect(existsSync(stored("1"))).toBe(true);273 });274275 test("a sync that lands a strip clears the repo's old cached tarballs", async () => {276 await get(`/tarry/archive/${head}.tar.gz`);277 await fx.strip("vendor/big.c");278 await syncAll(target);279 expect(existsSync(stored("1"))).toBe(false);280 expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain(281 "big.c",282 );283 });284});285286describe("the archives panel", () => {287 test("lists HEAD and recent saves with tarball links, and nix lines where there is a flake", async () => {288 const response = await get("/tarry/archives");289 expect(response.status).toBe(200);290 expect(response.headers.get("cache-control")).toBe("public, max-age=60");291 const page = await response.text();292 const ids = [...page.matchAll(/class="history-id"[^>]*>([0-9a-f]{12})</g)].map((m) => m[1]);293 expect(ids).toEqual([head, fx.saveIds[0], fx.saveIds[2], fx.saveIds[3]]);294 expect(page).toContain('<span class="archive-head mono">HEAD</span>');295 expect(page).toContain('class="archive-row archive-row--head"');296 expect(page).toContain(`data-copy="${head}"`);297 expect(page).toContain('data-copy="koh steal kepr.example/tarry"');298 expect(page).toContain(299 'data-copy="curl -fL https://kepr.example/tarry/archive/HEAD.tar.gz | tar xz"',300 );301 expect(page).toContain('href="/tarry/archive/HEAD.tar.gz">↓ tarry-HEAD.tar.gz</a>');302 expect(page).toContain(303 'data-copy="nix profile add https://kepr.example/tarry/archive/HEAD.tar.gz"',304 );305 306 expect(page.match(/nix profile add/g)?.length).toBe(2);307 const older = fx.saveIds[0] as string;308 expect(page).toContain(309 `href="/tarry/archive/${older}.tar.gz">↓ tarry-${older.slice(0, 8)}.tar.gz</a>`,310 );311 expect(page).not.toContain(fx.saveIds[1] as string);312 });313314 test("a private repo's panel is a 404", async () => {315 target.kepr.run("UPDATE repos SET public = 0");316 const status = (await get("/tarry/archives")).status;317 target.kepr.run("UPDATE repos SET public = 1");318 expect(status).toBe(404);319 });320});