key cached tarballs by the mirror's strip count so a strip never serves stale bytes even without cleanup, and write every file 0644 as kepr 2 does (v3.0.17)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.16",3+	"version": "3.0.17",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -55,10 +55,7 @@5555 		let error: string | null = null;5656 		try {5757 			const events = await syncRepo(target.bucket, openMirror(target.mirrorDir, repo), repo);58-			// §22.3: after the strip has landed in the mirror, so no rebuild can bring the old bytes back.59-			for (const event of events) {60-				if (event.kind === "strip") dropTarballs(target.tarballDir, repo, event.save_id);61-			}58+			if (events.some((event) => event.kind === "strip")) dropTarballs(target.tarballDir, repo);6259 		} catch (cause) {6360 			error = cause instanceof Error ? cause.message : String(cause);6461 		}

@@ -81,6 +81,12 @@8181 	return row?.kind === "flag" ? row.reason : null;8282 }8383 84+/** How many strip events have landed: events are append-only, so this only ever grows. */85+export function stripCount(db: Database): number {86+	return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM events WHERE kind = 'strip'").get()87+		?.n as number;88+}89+8490 export function filesAt(db: Database, saveId: string): FileEntry[] {8591 	return db8692 		.query<FileEntry, [string]>(

@@ -1,12 +1,14 @@11 import type { Database } from "bun:sqlite";22 import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";33 import { join } from "node:path";4-import { filesAt, objectBytes } from "../mirror/queries.ts";4+import { filesAt, objectBytes, stripCount } from "../mirror/queries.ts";55 import { buildTarball } from "./tar.ts";66 77 /**8- * A save's tarball from `<tarballDir>/<repo>/<id>.tar.gz`, built there on first request. Built and9- * written in one synchronous run, so a strip landing mid-build can't leave a stale file behind.8+ * A save's tarball, built on first request into `<tarballDir>/<repo>/<generation>/<id>.tar.gz`. The9+ * generation is how many strip events the mirror holds, so once a strip lands no older file is ever10+ * looked up again, even if Kepr stops before the sync's cleanup runs. Built and written in one11+ * synchronous run, so a strip can't land mid-build either.1012  */1113 export function cachedTarball(1214 	tarballDir: string,@@ -14,7 +16,8 @@1416 	repo: string,1517 	saveId: string,1618 ): Uint8Array<ArrayBuffer> {17-	const path = join(tarballDir, repo, `${saveId}.tar.gz`);19+	const dir = join(tarballDir, repo, String(stripCount(db)));20+	const path = join(dir, `${saveId}.tar.gz`);1821 	try {1922 		return readFileSync(path);2023 	} catch (error) {@@ -23,18 +26,16 @@2326 	const files = filesAt(db, saveId).map((file) => {2427 		const bytes = objectBytes(db, file.hash);2528 		if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);26-		return { path: file.path, executable: file.executable, bytes };29+		return { path: file.path, bytes };2730 	});2831 	const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);29-	mkdirSync(join(tarballDir, repo), { recursive: true });32+	mkdirSync(dir, { recursive: true });3033 	writeFileSync(`${path}.tmp`, tarball);3134 	renameSync(`${path}.tmp`, path);3235 	return tarball;3336 }3437 35-/** Drops cached tarballs a strip event reaches: one save's, or every one of the repo's for a whole-history strip. */36-export function dropTarballs(tarballDir: string, repo: string, saveId: string): void {37-	const target =38-		saveId === "" ? join(tarballDir, repo) : join(tarballDir, repo, `${saveId}.tar.gz`);39-	rmSync(target, { recursive: true, force: true });38+/** Clears a repo's cached tarballs after a strip; cleanup only, since the new generation never reads them. */39+export function dropTarballs(tarballDir: string, repo: string): void {40+	rmSync(join(tarballDir, repo), { recursive: true, force: true });4041 }

@@ -1,6 +1,6 @@11 import { gzipSync } from "node:zlib";22 3-export type TarFile = { path: string; bytes: Uint8Array; executable: boolean };3+export type TarFile = { path: string; bytes: Uint8Array };44 55 // Kepr 2's strip list (§22.3, `source_archive.zig`): build metadata, Koh bookkeeping and lockfiles66 // that pin a consumer's environment. Matched at the repo root only. `flake.lock` and `build.zig.zon`@@ -30,17 +30,17 @@3030 3131 /**3232  * A deterministic `.tar.gz` of `files` under `root/`, byte for byte what Kepr 2 wrote: one directory33- * entry for the root, then the files in byte order of their paths, with zero owners and times.34- * The one difference is the executable bit, which Kepr 3 keeps (0755) where Kepr 2 wrote 0644.33+ * entry for the root, then the files in byte order of their paths, all 0644, with zero owners and34+ * times (§22.3: the headers match Kepr 2's, so an executable bit is not kept).3535  */3636 export function buildTarball(root: string, files: TarFile[]): Uint8Array<ArrayBuffer> {3737 	const entries = files3838 		.filter((file) => file.path !== "" && !isStripped(file.path))3939 		.map((file) => ({ ...file, name: encoder.encode(`${root}/${file.path}`) }))4040 		.sort((a, b) => Buffer.compare(a.name, b.name));41-	const blocks: Uint8Array[] = [header(encoder.encode(root), 0, "dir")];41+	const blocks: Uint8Array[] = [header(encoder.encode(root), 0, true)];4242 	for (const entry of entries) {43-		blocks.push(header(entry.name, entry.bytes.length, entry.executable ? "exec" : "file"));43+		blocks.push(header(entry.name, entry.bytes.length, false));4444 		blocks.push(entry.bytes, new Uint8Array((BLOCK - (entry.bytes.length % BLOCK)) % BLOCK));4545 	}4646 	blocks.push(new Uint8Array(2 * BLOCK));@@ -52,9 +52,8 @@5252 5353 // A POSIX ustar header, as Kepr 2's `tarWriteHeader`: names over 99 bytes (98 for a directory, which5454 // needs its trailing slash) split at a '/' into the 155-byte prefix field, never truncated.55-function header(name: Uint8Array, size: number, kind: "dir" | "file" | "exec"): Uint8Array {55+function header(name: Uint8Array, size: number, isDir: boolean): Uint8Array {5656 	const h = new Uint8Array(BLOCK);57-	const isDir = kind === "dir";5857 	const maxName = isDir ? 98 : 99;5958 	let base = name;6059 	if (name.length > maxName) {@@ -67,7 +66,7 @@6766 	h.set(base);6867 	if (isDir) h[base.length] = 0x2f;6968 	const ascii = (text: string, at: number) => h.set(encoder.encode(text), at);70-	ascii(kind === "file" ? "0000644\0" : "0000755\0", 100);69+	ascii(isDir ? "0000755\0" : "0000644\0", 100);7170 	ascii("0000000\0", 108);7271 	ascii("0000000\0", 116);7372 	if (!isDir) ascii(`${size.toString(8).padStart(11, "0")}\0`, 124);

@@ -6,7 +6,8 @@66 import { gunzipSync } from "node:zlib";77 import type { Hono } from "hono";88 import { openBucket } from "../src/bucket/reader.ts";9-import { type SyncTarget, syncAll } from "../src/bucket/sync.ts";9+import { type SyncTarget, syncAll, syncRepo } from "../src/bucket/sync.ts";10+import { openMirror } from "../src/mirror/store.ts";1011 import { createApp } from "../src/server/app.ts";1112 import { openKeprDb } from "../src/server/db.ts";1213 import { buildTarball, isStripped } from "../src/tarball/tar.ts";@@ -84,8 +85,8 @@8485 8586 	test("writes Kepr 2's headers: root directory, then files with zero owners and times", () => {8687 		const gz = buildTarball("r-12345678", [87-			{ path: "b.txt", bytes: new TextEncoder().encode("hello source"), executable: false },88-			{ path: "a.sh", bytes: new Uint8Array(), executable: true },88+			{ path: "b.txt", bytes: new TextEncoder().encode("hello source") },89+			{ path: "a.sh", bytes: new Uint8Array() },8990 		]);9091 		expect([...gz.subarray(0, 10)]).toEqual([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3]);9192 		const t = gunzipSync(gz);@@ -97,7 +98,8 @@9798 		expect(field(root, 156, 157)).toBe("5");9899 		expect(field(root, 257, 265)).toBe("ustar\u000000");99100 		expect(field(a, 0, 16)).toBe("r-12345678/a.sh\0");100-		expect(field(a, 100, 108)).toBe("0000755\0");101+		// Every file is 0644, executable or not, exactly as Kepr 2 wrote them (§22.3).102+		expect(field(a, 100, 108)).toBe("0000644\0");101103 		expect(field(b, 100, 108)).toBe("0000644\0");102104 		expect(field(b, 124, 136)).toBe("00000000014\0");103105 		expect(field(b, 156, 157)).toBe("0");@@ -112,21 +114,16 @@112114 113115 	test("splits a long path into the ustar prefix, and refuses one that can't be split", () => {114116 		const long = `${"d".repeat(40)}/${"x".repeat(85)}`;115-		const t = gunzipSync(116-			buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),117-		);117+		const t = gunzipSync(buildTarball("r", [{ path: long, bytes: new Uint8Array() }]));118118 		const h = header(t, 1);119119 		expect(field(h, 345, 500).replace(/\0+$/, "")).toBe(`r/${"d".repeat(40)}`);120120 		expect(field(h, 0, 100).replace(/\0+$/, "")).toBe("x".repeat(85));121121 		expect(122-			tar(123-				["-tzf", "-"],124-				buildTarball("r", [{ path: long, bytes: new Uint8Array(), executable: false }]),125-			),122+			tar(["-tzf", "-"], buildTarball("r", [{ path: long, bytes: new Uint8Array() }])),126123 		).toContain(`r/${long}`);127-		expect(() =>128-			buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array(), executable: false }]),129-		).toThrow("path too long");124+		expect(() => buildTarball("r", [{ path: "y".repeat(120), bytes: new Uint8Array() }])).toThrow(125+			"path too long",126+		);130127 	});131128 132129 	test("the strip list matches the repo root only", () => {@@ -178,7 +175,7 @@178175 		expect(hidden.map((r) => r.status)).toEqual([404, 404]);179176 	});180177 181-	test("serves the save's files under <repo>-<id8>/ minus the strip list, executable bit kept", async () => {178+	test("serves the save's files under <repo>-<id8>/ minus the strip list, every file 0644", async () => {182179 		const response = await get(`/tarry/archive/${head}.tar.gz`);183180 		expect(response.status).toBe(200);184181 		expect(response.headers.get("content-type")).toBe("application/gzip");@@ -213,38 +210,56 @@213210 		expect(await readFile(join(out, root, "docs/logo.png"))).toEqual(214211 			await readFile(join(fx.repoDir, "docs/logo.png")),215212 		);216-		expect(tar(["-tvzf", "-"], gz)).toMatch(/-rwxr-xr-x .* tarry-[0-9a-f]{8}\/run\.sh/);213+		expect(tar(["-tvzf", "-"], gz)).toMatch(/-rw-r--r-- .* tarry-[0-9a-f]{8}\/run\.sh/);217214 	});218215 219216 	test("is the same bytes when rebuilt, and in another process", async () => {220217 		const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);221218 		const { dropTarballs } = await import("../src/tarball/cache.ts");222-		dropTarballs(target.tarballDir, "tarry", head);219+		dropTarballs(target.tarballDir, "tarry");223220 		expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);221+		// Every path the other process touches is a fresh temp dir, its working directory included,222+		// so nothing it writes can land in the repo.224223 		const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";225224 import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";226-process.stdout.write(cachedTarball(process.argv[1], openMirror(process.argv[2], "tarry"), "tarry", process.argv[3]));`;227-		const other = Bun.spawnSync([228-			"bun",229-			"-e",230-			script,231-			await mkdtemp(join(tmpdir(), "kepr-tb-")),232-			target.mirrorDir,233-			head,234-		]);225+const { TARBALL_DIR, MIRROR_DIR, SAVE_ID } = process.env;226+process.stdout.write(cachedTarball(TARBALL_DIR, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`;227+		const scratch = await mkdtemp(join(tmpdir(), "kepr-tb-"));228+		const other = Bun.spawnSync(["bun", "-e", script], {229+			cwd: scratch,230+			env: {231+				PATH: process.env.PATH,232+				TARBALL_DIR: join(scratch, "tarballs"),233+				MIRROR_DIR: target.mirrorDir,234+				SAVE_ID: head,235+			},236+		});237+		expect(other.stderr.toString()).toBe("");235238 		expect(new Uint8Array(other.stdout)).toEqual(first);236239 	});237240 238-	test("a strip event drops the cached tarball, so the next one leaves the file out", async () => {241+	test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => {239242 		await get(`/tarry/archive/${head}.tar.gz`);240-		const cached = join(target.tarballDir, "tarry", `${head}.tar.gz`);241-		expect(existsSync(cached)).toBe(true);243+		const before = join(target.tarballDir, "tarry", "0", `${head}.tar.gz`);244+		expect(existsSync(before)).toBe(true);242245 		await fx.strip("secret.txt");243-		await syncAll(target);244-		expect(existsSync(cached)).toBe(false);246+		// syncRepo lands the strip without syncAll's cleanup: a crash between the two.247+		await syncRepo(target.bucket, openMirror(target.mirrorDir, "tarry"), "tarry");248+		expect(existsSync(before)).toBe(true);245249 		const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));246250 		expect(listed).not.toContain("secret.txt");247251 		expect(listed).toContain("src/main.ts");252+		expect(existsSync(join(target.tarballDir, "tarry", "1", `${head}.tar.gz`))).toBe(true);253+	});254+255+	test("a sync that lands a strip clears the repo's old cached tarballs", async () => {256+		await get(`/tarry/archive/${head}.tar.gz`);257+		await fx.strip("vendor/big.c");258+		await syncAll(target);259+		expect(existsSync(join(target.tarballDir, "tarry"))).toBe(false);260+		expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain(261+			"big.c",262+		);248263 	});249264 });250265