add deterministic source tarballs byte-identical to kepr 2's tar, cached on disk and dropped by strip events, with the HEAD redirect and its immutable link (v3.0.15)

@@ -0,0 +1,37 @@1+import type { Hono } from "hono";2+import { headId, saveByPrefix } from "../mirror/queries.ts";3+import { notFound, openPublicRepo, type PageContext } from "../pages/layout.ts";4+import { cachedTarball } from "./cache.ts";5+6+// Long-lived but never `immutable` (§18): a strip can change what a save's tarball holds.7+const TARBALL_CACHE = "public, max-age=31536000";8+9+export function registerTarballRoutes(app: Hono, ctx: PageContext, tarballDir: string): void {10+	app.get("/:repo/archive/HEAD.tar.gz", (c) => {11+		const repo = openPublicRepo(ctx, c.req.param("repo"));12+		const head = repo && headId(repo.mirror);13+		if (!repo || !head) return notFound(c, ctx);14+		const path = `/${repo.settings.name}/archive/${head}.tar.gz`;15+		// Nix pins the immutable link in its lock file, so it must be absolute.16+		c.header("Link", `<https://${ctx.domain}${path}>; rel="immutable"`);17+		c.header("Cache-Control", "no-cache");18+		return c.redirect(path, 302);19+	});20+21+	app.get("/:repo/archive/:id{[0-9a-f]+\\.tar\\.gz}", (c) => {22+		const repo = openPublicRepo(ctx, c.req.param("repo"));23+		const id = c.req.param("id").replace(/\.tar\.gz$/, "");24+		const save = repo && saveByPrefix(repo.mirror, id);25+		if (!repo || !save) return notFound(c, ctx);26+		const name = repo.settings.name;27+		if (save.id !== id) {28+			c.header("Cache-Control", "public, max-age=60");29+			return c.redirect(`/${name}/archive/${save.id}.tar.gz`, 302);30+		}31+		const bytes = cachedTarball(tarballDir, repo.mirror, name, save.id);32+		c.header("Content-Type", "application/gzip");33+		c.header("Content-Disposition", `attachment; filename="${name}-${save.id.slice(0, 8)}.tar.gz"`);34+		c.header("Cache-Control", TARBALL_CACHE);35+		return c.body(bytes);36+	});37+}

@@ -0,0 +1,40 @@1+import type { Database } from "bun:sqlite";2+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";3+import { join } from "node:path";4+import { filesAt, objectBytes } from "../mirror/queries.ts";5+import { buildTarball } from "./tar.ts";6+7+/**8+ * A save's tarball from `<tarballDir>/<repo>/<id>.tar.gz`, built there on first request. Built and9+ * written in one synchronous run, so a strip landing mid-build can't leave a stale file behind.10+ */11+export function cachedTarball(12+	tarballDir: string,13+	db: Database,14+	repo: string,15+	saveId: string,16+): Uint8Array<ArrayBuffer> {17+	const path = join(tarballDir, repo, `${saveId}.tar.gz`);18+	try {19+		return readFileSync(path);20+	} catch (error) {21+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;22+	}23+	const files = filesAt(db, saveId).map((file) => {24+		const bytes = objectBytes(db, file.hash);25+		if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);26+		return { path: file.path, executable: file.executable, bytes };27+	});28+	const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);29+	mkdirSync(join(tarballDir, repo), { recursive: true });30+	writeFileSync(`${path}.tmp`, tarball);31+	renameSync(`${path}.tmp`, path);32+	return tarball;33+}34+35+/** Drops cached tarballs a strip event reaches: one save's, or every one of the repo's for a whole-history strip. */36+export function dropTarballs(tarballDir: string, repo: string, saveId: string): void {37+	const target =38+		saveId === "" ? join(tarballDir, repo) : join(tarballDir, repo, `${saveId}.tar.gz`);39+	rmSync(target, { recursive: true, force: true });40+}

@@ -0,0 +1,91 @@1+import { gzipSync } from "node:zlib";2+3+export type TarFile = { path: string; bytes: Uint8Array; executable: boolean };4+5+// Kepr 2's strip list (§22.3, `source_archive.zig`): build metadata, Koh bookkeeping and lockfiles6+// that pin a consumer's environment. Matched at the repo root only. `flake.lock` and `build.zig.zon`7+// stay, because Nix needs both.8+const STRIPPED = new Set([9+	"koh.build",10+	"kepr.build",11+	"Cargo.lock",12+	"package-lock.json",13+	"yarn.lock",14+	"pnpm-lock.yaml",15+	"bun.lockb",16+	"go.sum",17+	"LOG.md",18+	"CONTINUITY.md",19+	"AGENTS.md",20+	".kohignore",21+	".gitignore",22+]);23+24+export function isStripped(path: string): boolean {25+	return STRIPPED.has(path) || path === ".agent" || path.startsWith(".agent/");26+}27+28+const encoder = new TextEncoder();29+const BLOCK = 512;30+31+/**32+ * A deterministic `.tar.gz` of `files` under `root/`, byte for byte what Kepr 2 wrote: one directory33+ * entry for the root, then the files in byte order of their paths, with zero owners and times.34+ * The one difference is the executable bit, which Kepr 3 keeps (0755) where Kepr 2 wrote 0644.35+ */36+export function buildTarball(root: string, files: TarFile[]): Uint8Array<ArrayBuffer> {37+	const entries = files38+		.filter((file) => file.path !== "" && !isStripped(file.path))39+		.map((file) => ({ ...file, name: encoder.encode(`${root}/${file.path}`) }))40+		.sort((a, b) => Buffer.compare(a.name, b.name));41+	const blocks: Uint8Array[] = [header(encoder.encode(root), 0, "dir")];42+	for (const entry of entries) {43+		blocks.push(header(entry.name, entry.bytes.length, entry.executable ? "exec" : "file"));44+		blocks.push(entry.bytes, new Uint8Array((BLOCK - (entry.bytes.length % BLOCK)) % BLOCK));45+	}46+	blocks.push(new Uint8Array(2 * BLOCK));47+	const gzip = gzipSync(Buffer.concat(blocks), { level: 6 });48+	// zlib stamps the host's OS in the gzip header (19 on macOS); Kepr 2 wrote 3 (Unix) everywhere.49+	gzip[9] = 3;50+	return gzip;51+}52+53+// A POSIX ustar header, as Kepr 2's `tarWriteHeader`: names over 99 bytes (98 for a directory, which54+// needs its trailing slash) split at a '/' into the 155-byte prefix field, never truncated.55+function header(name: Uint8Array, size: number, kind: "dir" | "file" | "exec"): Uint8Array {56+	const h = new Uint8Array(BLOCK);57+	const isDir = kind === "dir";58+	const maxName = isDir ? 98 : 99;59+	let base = name;60+	if (name.length > maxName) {61+		const split = splitAt(name, maxName);62+		if (split === null)63+			throw new Error(`path too long for a tarball: ${new TextDecoder().decode(name)}`);64+		h.set(name.subarray(0, split), 345);65+		base = name.subarray(split + 1);66+	}67+	h.set(base);68+	if (isDir) h[base.length] = 0x2f;69+	const ascii = (text: string, at: number) => h.set(encoder.encode(text), at);70+	ascii(kind === "file" ? "0000644\0" : "0000755\0", 100);71+	ascii("0000000\0", 108);72+	ascii("0000000\0", 116);73+	if (!isDir) ascii(`${size.toString(8).padStart(11, "0")}\0`, 124);74+	ascii("00000000000\0", 136);75+	ascii("        ", 148);76+	ascii(isDir ? "5" : "0", 156);77+	ascii("ustar\0", 257);78+	ascii("00", 263);79+	const sum = h.reduce((total, byte) => total + byte, 0);80+	ascii(`${sum.toString(8).padStart(6, "0")}\0 `, 148);81+	return h;82+}83+84+// The rightmost '/' leaving a prefix of at most 154 bytes and a name of at most `maxName`.85+function splitAt(path: Uint8Array, maxName: number): number | null {86+	const min = path.length - maxName - 1;87+	for (let i = Math.min(path.length - 1, 154); i >= Math.max(min, 0); i--) {88+		if (path[i] === 0x2f) return i;89+	}90+	return null;91+}

@@ -0,0 +1,249 @@1+import { beforeAll, describe, expect, test } from "bun:test";2+import { existsSync } from "node:fs";3+import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4+import { tmpdir } from "node:os";5+import { join } from "node:path";6+import { gunzipSync } from "node:zlib";7+import type { Hono } from "hono";8+import { openBucket } from "../src/bucket/reader.ts";9+import { type SyncTarget, syncAll } from "../src/bucket/sync.ts";10+import { createApp } from "../src/server/app.ts";11+import { openKeprDb } from "../src/server/db.ts";12+import { buildTarball, isStripped } from "../src/tarball/tar.ts";13+import { type Fixture, makeKohBucket } from "./fixture.ts";14+15+let fx: Fixture;16+let app: Hono;17+let target: SyncTarget;18+let head: string;19+let dataDir: string;20+const get = (path: string) => app.request(path);21+const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());22+23+// Every root-level file on Kepr 2's strip list, the .agent folder, and the look-alikes that must stay.24+const EXTRA: Record<string, string> = {25+	"AGENTS.md": "agents\n",26+	"CONTINUITY.md": "notes\n",27+	"Cargo.lock": "lock\n",28+	"koh.build": "build\n",29+	".gitignore": "x\n",30+	".agent/CONTINUITY.md": "state\n",31+	".agentry/keep.txt": "kept\n",32+	"flake.nix": "{ outputs = _: { }; }\n",33+	"flake.lock": "{}\n",34+	"build.zig.zon": ".{}\n",35+	"nested/go.sum": "kept\n",36+	"run.sh": "#!/bin/sh\necho run\n",37+};38+39+beforeAll(async () => {40+	fx = await makeKohBucket("tarry");41+	for (const [path, text] of Object.entries(EXTRA)) {42+		await mkdir(join(fx.repoDir, path, ".."), { recursive: true });43+		await writeFile(join(fx.repoDir, path), text);44+	}45+	await chmod(join(fx.repoDir, "run.sh"), 0o755);46+	const home = join(fx.repoDir, "../../home");47+	const env = { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" };48+	const save = Bun.spawnSync(["koh3", "save", "add build files"], { cwd: fx.repoDir, env });49+	expect(save.exitCode).toBe(0);50+	const log = Bun.spawnSync(["koh3", "log", "--json"], { cwd: fx.repoDir, env });51+	head = JSON.parse(log.stdout.toString()).head_id;52+	dataDir = await mkdtemp(join(tmpdir(), "kepr-tarball-"));53+	const kepr = openKeprDb(dataDir);54+	target = {55+		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),56+		kepr,57+		mirrorDir: join(dataDir, "mirror"),58+		tarballDir: join(dataDir, "tarballs"),59+	};60+	await syncAll(target);61+	kepr.run("UPDATE repos SET public = 1");62+	app = createApp(63+		{64+			domain: "kepr.example",65+			dataDir,66+			adminPasswordHash: "$argon2id$x",67+			bucket: { kind: "folder", path: fx.bucketDir },68+		},69+		kepr,70+	);71+});72+73+function tar(args: string[], input: Uint8Array): string {74+	const result = Bun.spawnSync(["tar", ...args], { stdin: input });75+	expect(result.exitCode).toBe(0);76+	return result.stdout.toString();77+}78+79+describe("the tar format", () => {80+	const header = (tarBytes: Uint8Array, index: number) =>81+		tarBytes.subarray(index * 512, index * 512 + 512);82+	const field = (h: Uint8Array, from: number, to: number) =>83+		new TextDecoder().decode(h.subarray(from, to));84+85+	test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => {86+		const gz = buildTarball("r-12345678", [87+			{ path: "b.txt", bytes: new TextEncoder().encode("hello source"), executable: false },88+			{ path: "a.sh", bytes: new Uint8Array(), executable: true },89+		]);90+		expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]);91+		const t = gunzipSync(gz);92+		const [root, a, b] = [header(t, 0), header(t, 1), header(t, 2)];93+		expect(field(root, 0, 12)).toBe("r-12345678/\0");94+		expect(field(root, 100, 156)).toBe(95+			`0000755\0${"0000000\0".repeat(2)}${"\0".repeat(12)}00000000000\0${field(root, 148, 156)}`,96+		);97+		expect(field(root, 156, 157)).toBe("5");98+		expect(field(root, 257, 265)).toBe("ustar\u000000");99+		expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0");100+		expect(field(a, 100, 108)).toBe("0000755\0");101+		expect(field(b, 100, 108)).toBe("0000644\0");102+		expect(field(b, 124, 136)).toBe("00000000014\0");103+		expect(field(b, 156, 157)).toBe("0");104+		for (const h of [root, a, b]) {105+			let sum = 0;106+			for (const [i, byte] of h.entries()) sum += i >= 148 && i < 156 ? 32 : byte;107+			expect(field(h, 148, 156)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `);108+		}109+		expect(field(t, 3 * 512, 3 * 512 + 12)).toBe("hello source");110+		expect(t.length).toBe(6 * 512);111+	});112+113+	test("splits a long path into the ustar prefix, and refuses one that can't be split", () => {114+		const long = `${"d".repeat(40)}/${"x".repeat(85)}`;115+		const t = gunzipSync(116+			buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),117+		);118+		const h = header(t, 1);119+		expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`);120+		expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85));121+		expect(122+			tar(123+				["-tzf", "-"],124+				buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),125+			),126+		).toContain(`r/${long}`);127+		expect(() =>128+			buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array(), executable: false }]),129+		).toThrow("path too long");130+	});131+132+	test("the strip list matches the repo root only", () => {133+		for (const path of [134+			"koh.build",135+			"kepr.build",136+			"go.sum",137+			"LOG.md",138+			".agent",139+			".agent/a/b.md",140+			".gitignore",141+		])142+			expect(isStripped(path)).toBe(true);143+		for (const path of [144+			"flake.lock",145+			"build.zig.zon",146+			"nested/go.sum",147+			"docs/AGENTS.md",148+			".agentry/x",149+			"log.md",150+		])151+			expect(isStripped(path)).toBe(false);152+	});153+});154+155+describe("the archive routes", () => {156+	test("HEAD.tar.gz redirects uncached to the current save, naming it as the immutable link", async () => {157+		const response = await get("/tarry/archive/HEAD.tar.gz");158+		expect(response.status).toBe(302);159+		expect(response.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);160+		expect(response.headers.get("cache-control")).toBe("no-cache");161+		expect(response.headers.get("link")).toBe(162+			`<https://kepr.example/tarry/archive/${head}.tar.gz>; rel="immutable"`,163+		);164+	});165+166+	test("a prefix redirects to the full ID; unknown, malformed or private is a 404", async () => {167+		const short = await get(`/tarry/archive/${head.slice(0, 7)}.tar.gz`);168+		expect(short.status).toBe(302);169+		expect(short.headers.get("location")).toBe(`/tarry/archive/${head}.tar.gz`);170+		expect((await get("/tarry/archive/abcdef.tar.gz")).status).toBe(404);171+		expect((await get("/tarry/archive/abc.tar.gz")).status).toBe(404);172+		target.kepr.run("UPDATE repos SET public = 0");173+		const hidden = [174+			await get("/tarry/archive/HEAD.tar.gz"),175+			await get(`/tarry/archive/${head}.tar.gz`),176+		];177+		target.kepr.run("UPDATE repos SET public = 1");178+		expect(hidden.map((r) => r.status)).toEqual([404, 404]);179+	});180+181+	test("serves the save's files under <repo>-<id8>/ minus the strip list, executable bit kept", async () => {182+		const response = await get(`/tarry/archive/${head}.tar.gz`);183+		expect(response.status).toBe(200);184+		expect(response.headers.get("content-type")).toBe("application/gzip");185+		expect(response.headers.get("cache-control")).toBe("public, max-age=31536000");186+		expect(response.headers.get("content-disposition")).toBe(187+			`attachment; filename="tarry-${head.slice(0, 8)}.tar.gz"`,188+		);189+		const gz = new Uint8Array(await response.arrayBuffer());190+		const root = `tarry-${head.slice(0, 8)}`;191+		const listed = tar(["-tzf", "-"], gz).trim().split("\n");192+		expect(listed).toEqual([193+			`${root}/`,194+			...[195+				".agentry/keep.txt",196+				"README.md",197+				"build.zig.zon",198+				"docs/guide.md",199+				"docs/logo.png",200+				"flake.lock",201+				"flake.nix",202+				"nested/go.sum",203+				"run.sh",204+				"secret.txt",205+				"src/lib/deep/util.ts",206+				"src/main.ts",207+				"vendor/big.c",208+			].map((path) => `${root}/${path}`),209+		]);210+		const out = await mkdtemp(join(tmpdir(), "kepr-untar-"));211+		Bun.spawnSync(["tar", "-xzf", "-", "-C", out], { stdin: gz });212+		expect(await readFile(join(out, root, "src/main.ts"), "utf8")).toContain("hello");213+		expect(await readFile(join(out, root, "docs/logo.png"))).toEqual(214+			await readFile(join(fx.repoDir, "docs/logo.png")),215+		);216+		expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/);217+	});218+219+	test("is the same bytes when rebuilt, and in another process", async () => {220+		const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);221+		const { dropTarballs } = await import("../src/tarball/cache.ts");222+		dropTarballs(target.tarballDir, "tarry", head);223+		expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);224+		const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";225+import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";226+process.stdout.write(cachedTarball(process.argv[1], openMirror(process.argv[2], "tarry"), "tarry", process.argv[3]));`;227+		const other = Bun.spawnSync([228+			"bun",229+			"-e",230+			script,231+			await mkdtemp(join(tmpdir(), "kepr-tb-")),232+			target.mirrorDir,233+			head,234+		]);235+		expect(new Uint8Array(other.stdout)).toEqual(first);236+	});237+238+	test("a strip event drops the cached tarball, so the next one leaves the file out", async () => {239+		await get(`/tarry/archive/${head}.tar.gz`);240+		const cached = join(target.tarballDir, "tarry", `${head}.tar.gz`);241+		expect(existsSync(cached)).toBe(true);242+		await fx.strip("secret.txt");243+		await syncAll(target);244+		expect(existsSync(cached)).toBe(false);245+		const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));246+		expect(listed).not.toContain("secret.txt");247+		expect(listed).toContain("src/main.ts");248+	});249+});