key kept tarballs by strips, hidden paths and mirror epoch, recheck after every await, and rebuild mirrors in place so held handles stay usable (v3.0.151)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.150",3+	"version": "3.0.151",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -1,7 +1,6 @@11 import type { Database } from "bun:sqlite";22 import {3-	discardMirror,4-	hidePaths,3+	clearMirror,54 	knownEventIds,65 	knownObject,76 	knownReleases,@@ -56,7 +55,7 @@5655 /** Rebuilds one repo's mirror from the bucket, once any sync under way has finished. */5756 export async function rebuildRepo(target: SyncTarget, repo: string): Promise<void> {5857 	await syncAll(target);59-	discardMirror(target.mirrorDir, repo);58+	clearMirror(openMirror(target.mirrorDir, repo));6059 	await syncAll(target);6160 }6261 @@ -75,8 +74,7 @@7574 		} else {7675 			try {7776 				const mirror = openMirror(target.mirrorDir, repo);78-				const events = await syncRepo(target.bucket, mirror, repo);79-				hidePaths(mirror, hiddenPaths(target.kepr, repo));77+				const events = await syncRepo(target.bucket, mirror, repo, hiddenPaths(target.kepr, repo));8078 				if (events.some((event) => event.kind === "strip")) await dropTarballs(target.bucket, repo);8179 			} catch (cause) {8280 				error = cause instanceof Error ? cause.message : String(cause);@@ -98,6 +96,7 @@9896 	bucket: Bucket,9997 	mirror: Database,10098 	repo: string,99+	hidden: string[] = [],101100 ): Promise<EventRecord[]> {102101 	await checkFormat(bucket, repo);103102 	const head = await readHead(bucket, repo);@@ -153,6 +152,7 @@153152 		events,154153 		releases,155154 		quarantined,155+		hidden,156156 	});157157 	return events;158158 }

@@ -1,4 +1,5 @@11 import type { Database } from "bun:sqlite";2+import { createHash } from "node:crypto";23 import { inflateSync } from "node:zlib";34 import type { ReleaseRecord } from "../bucket/repo.ts";45 import { mirrorState } from "./store.ts";@@ -85,7 +86,17 @@8586 }8687 8788 /** How many strip events have landed: events are append-only, so this only ever grows. */88-export function stripCount(db: Database): number {89+/**90+ * What a save's tarball depends on besides the save: the strips landed, and the hidden paths and91+ * epoch the mirror was built with. A change to any of them names a new key, so an old tarball is92+ * never looked up again, whatever a late write or a failed cleanup leaves behind.93+ */94+export function tarballGeneration(db: Database): string {95+	const built = `${mirrorState(db, "hidden") ?? ""}\n${mirrorState(db, "epoch") ?? ""}`;96+	return `${stripCount(db)}-${createHash("sha256").update(built).digest("hex").slice(0, 12)}`;97+}98+99+function stripCount(db: Database): number {89100 	return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM events WHERE kind = 'strip'").get()90101 		?.n as number;91102 }

@@ -1,5 +1,5 @@11 import { Database } from "bun:sqlite";2-import { mkdirSync, rmSync } from "node:fs";2+import { mkdirSync } from "node:fs";33 import { join } from "node:path";44 import { inflateSync } from "node:zlib";55 import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";@@ -69,6 +69,8 @@6969 	releases: ReleaseRecord[];7070 	/** Escaped names of the paths these saves arrived without (§5.5, A3). */7171 	quarantined: string[];72+	/** The operator's hidden paths (June, 2026-09-30), removed from every save like a strip. */73+	hidden: string[];7274 };7375 7476 // A recount inflates every file at the newest save; a file this large is data, not lines of code.@@ -210,6 +212,12 @@210212 		}211213 		for (const r of landing.releases) putRelease.run(r.version, JSON.stringify(r));212214 		applyStrips(db);215+		let hidden = 0;216+		for (const path of landing.hidden) hidden += removePath(db, path, "");217+		if (hidden > 0) db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");218+		// In the same transaction as the removal, so the tarball key never names a list the rows219+		// don't reflect yet.220+		setState.run("hidden", landing.hidden.join("\n"));213221 		setState.run("head", landing.head);214222 		setState.run("releases_latest", landing.releasesLatest);215223 		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);@@ -252,24 +260,18 @@252260 }253261 254262 /**255- * The operator's hidden paths, removed from every save as a strip would be (June, 2026-09-30).256- * Applied after each sync, so saves arriving later lose them too; unhiding rebuilds the mirror.263+ * Empties a mirror in place so the next sync rebuilds it from the bucket, as a change to the264+ * hidden paths needs. The file and its open handles stay, so a request holding one reads an empty265+ * repo for a moment rather than failing. A new epoch keeps tarballs kept before it from matching.257266  */258-export function hidePaths(db: Database, paths: string[]): void {259-	let removed = 0;267+export function clearMirror(db: Database): void {260268 	db.transaction(() => {261-		for (const path of paths) removed += removePath(db, path, "");262-		if (removed > 0) db.run("DELETE FROM objects WHERE hash NOT IN (SELECT hash FROM save_files)");269+		// Children before parents, for the foreign keys; every name is a table of SCHEMA's.270+		for (const table of ["save_files", "events", "saves", "objects", "releases", "state"]) {271+			db.run(`DELETE FROM ${table}`);272+		}273+		db.query("INSERT INTO state (key, value) VALUES ('epoch', ?)").run(crypto.randomUUID());263274 	})();264-	if (removed > 0) refreshStats(db);265-}266-267-/** Closes and deletes a repo's mirror, so the next sync rebuilds it from the bucket. */268-export function discardMirror(dir: string, repo: string): void {269-	const path = join(dir, `${repo}.db`);270-	open.get(path)?.close();271-	open.delete(path);272-	for (const suffix of ["", "-wal", "-shm"]) rmSync(`${path}${suffix}`, { force: true });273275 }274276 275277 // Lines and languages never change between syncs, so they are counted once here, not per view. They

@@ -32,6 +32,7 @@3232 			return c.redirect(`/${name}/archive/${save.id}.tar.gz`, 302);3333 		}3434 		const bytes = await cachedTarball(bucket, repo.mirror, name, save.id);35+		if (!bytes) return notFound(c, ctx);3536 		// §22.2: a Nix fetch is an install. Only the tarball itself counts, not the redirects to it.3637 		if (c.req.header("User-Agent")?.includes("nix/")) countInstall(ctx.kepr, name);3738 		c.header("Content-Type", "application/gzip");

@@ -1,43 +1,49 @@11 import type { Database } from "bun:sqlite";22 import { type Bucket, BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";3-import { filesAt, objectBytes, stripCount } from "../mirror/queries.ts";3+import { filesAt, objectBytes, tarballGeneration } from "../mirror/queries.ts";4+import { knownSave } from "../mirror/store.ts";45 import { buildTarball } from "./tar.ts";56 67 const tarballsOf = (repo: string) => `${KEPR_PREFIX}tarballs/${repo}/`;78 89 /**910  * A save's tarball, kept in the bucket under Kepr's own folder (June, 2026-09-30) as10- * `.kepr/tarballs/<repo>/<generation>/<id>.tar.gz` once first requested. The generation is how11- * many strip events the mirror holds, so once a strip lands no older file is ever looked up again,12- * even if Kepr stops before the sync's cleanup runs. Without a write key nothing is kept, and each13- * request builds it again.11+ * `.kepr/tarballs/<repo>/<generation>/<id>.tar.gz` once first requested (`tarballGeneration`).12+ * Without a write key nothing is kept, and each request builds it again. Null when the save is13+ * gone from the mirror, as it is for a moment while a rebuild runs.1414  */1515 export async function cachedTarball(1616 	bucket: Bucket,1717 	db: Database,1818 	repo: string,1919 	saveId: string,20-): Promise<Uint8Array> {21-	const stored = await bucket.get(`${tarballsOf(repo)}${stripCount(db)}/${saveId}.tar.gz`);22-	if (stored) return stored;23-	// Read again after the await: the generation and the build below run in one synchronous go, so24-	// a strip landing meanwhile can't file new contents under the old generation or the reverse.25-	const key = `${tarballsOf(repo)}${stripCount(db)}/${saveId}.tar.gz`;26-	const files = filesAt(db, saveId).map((file) => {27-		const bytes = objectBytes(db, file.hash);28-		if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);29-		return { path: file.path, bytes, executable: file.executable };30-	});31-	const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);32-	try {33-		await bucket.put(key, tarball, "application/gzip");34-	} catch (error) {35-		if (!(error instanceof BucketWriteError)) throw error;20+): Promise<Uint8Array | null> {21+	for (;;) {22+		const generation = tarballGeneration(db);23+		const key = `${tarballsOf(repo)}${generation}/${saveId}.tar.gz`;24+		const stored = await bucket.get(key);25+		// A strip, a hidden-path change or a rebuild may have landed during any await: bytes are26+		// served only for the generation still current, and the build below runs in one synchronous27+		// go, so what it reads matches the key it files under.28+		if (tarballGeneration(db) !== generation) continue;29+		if (stored) return stored;30+		if (!knownSave(db, saveId)) return null;31+		const files = filesAt(db, saveId).map((file) => {32+			const bytes = objectBytes(db, file.hash);33+			if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);34+			return { path: file.path, bytes, executable: file.executable };35+		});36+		const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);37+		try {38+			await bucket.put(key, tarball, "application/gzip");39+		} catch (error) {40+			if (!(error instanceof BucketWriteError)) throw error;41+		}42+		if (tarballGeneration(db) === generation) return tarball;3643 	}37-	return tarball;3844 }3945 40-/** Deletes a repo's kept tarballs: after a strip, or from the console. They rebuild on request. */46+/** Deletes a repo's kept tarballs to free space; stale ones are never looked up anyway. */4147 export async function dropTarballs(bucket: Bucket, repo: string): Promise<void> {4248 	for (const key of await bucket.list(tarballsOf(repo))) await bucket.remove(key);4349 }

@@ -6,6 +6,7 @@66 import { join } from "node:path";77 import { openBucket } from "../src/bucket/reader.ts";88 import { syncAll } from "../src/bucket/sync.ts";9+import { openMirror } from "../src/mirror/store.ts";910 import { Browser, type Instance, instance } from "./browser.ts";1011 import { makeRepos } from "./fixture.ts";1112 @@ -86,5 +87,35 @@8687 		expect(await text("/app/raw/tool.py")).toBe("print(2)\n");8788 		expect(await status("/app/raw/private/notes.md")).toBe(200);8889 		expect(await Bun.file(join(repos.bucketDir, ".kepr/hidden/app")).text()).toBe("");90+	});91+92+	test("an old kept tarball is never served once a path is hidden, even if cleanup failed", async () => {93+		const head = (await inst.app.request("/app/archive/HEAD.tar.gz")).headers.get("location") ?? "";94+		await inst.app.request(head);95+		const keptDir = join(repos.bucketDir, ".kepr/tarballs/app");96+		const old = new Map<string, Uint8Array>();97+		for (const path of new Bun.Glob("**/*.tar.gz").scanSync(keptDir)) {98+			old.set(path, await Bun.file(join(keptDir, path)).bytes());99+		}100+		expect(old.size).toBeGreaterThan(0);101+		await hide("tool.py");102+		for (const [path, bytes] of old) await Bun.write(join(keptDir, path), bytes);103+		const tarball = new Uint8Array(await (await inst.app.request(head)).arrayBuffer());104+		const listed = Bun.spawnSync(["tar", "-tzf", "-"], { stdin: tarball }).stdout.toString();105+		expect(listed).toContain("main.ts");106+		expect(listed).not.toContain("tool.py");107+		await hide("");108+	});109+110+	test("rebuilding a mirror keeps handles that requests already hold usable", async () => {111+		const mirror = openMirror(join(inst.dataDir, "mirror"), "app");112+		const count = () =>113+			mirror114+				.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM save_files WHERE path = 'tool.py'")115+				.get()?.n;116+		await hide("tool.py");117+		expect(count()).toBe(0);118+		await hide("");119+		expect(count()).toBeGreaterThan(0);89120 	});90121 });

@@ -1,15 +1,15 @@11 import { beforeAll, describe, expect, test } from "bun:test";2-import { existsSync } from "node:fs";32 import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";43 import { tmpdir } from "node:os";54 import { join } from "node:path";65 import { gunzipSync } from "node:zlib";76 import type { Hono } from "hono";8-import { openBucket } from "../src/bucket/reader.ts";7+import { type Bucket, openBucket } from "../src/bucket/reader.ts";98 import { type SyncTarget, syncAll, syncRepo } from "../src/bucket/sync.ts";109 import { openMirror } from "../src/mirror/store.ts";1110 import { createApp } from "../src/server/app.ts";1211 import { openKeprDb } from "../src/server/db.ts";12+import { cachedTarball } from "../src/tarball/cache.ts";1313 import { buildTarball, isStripped } from "../src/tarball/tar.ts";1414 import { type Fixture, makeKohBucket } from "./fixture.ts";1515 @@ -20,9 +20,13 @@2020 let dataDir: string;2121 const get = (path: string) => app.request(path);2222 const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());23-// Where a tarball of HEAD is kept in the folder bucket, for a strip generation.24-const stored = (generation: string) =>25-	join(fx.bucketDir, ".kepr", "tarballs", "tarry", generation, `${head}.tar.gz`);23+// Whether a tarball of HEAD is kept in the folder bucket under this many strips.24+const kept = (strips: number) =>25+	[26+		...new Bun.Glob(`${strips}-*/${head}.tar.gz`).scanSync(27+			join(fx.bucketDir, ".kepr/tarballs/tarry"),28+		),29+	].length > 0;2630 2731 // Every root-level file on Kepr 2's strip list, the .agent folder, and the look-alikes that must stay.2832 const EXTRA: Record<string, string> = {@@ -234,7 +238,7 @@234238 		const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);235239 		const { dropTarballs } = await import("../src/tarball/cache.ts");236240 		await dropTarballs(target.bucket, "tarry");237-		expect(existsSync(stored("0"))).toBe(false);241+		expect(kept(0)).toBe(false);238242 		expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);239243 		// Every path the other process touches is a fresh temp dir, its working directory included,240244 		// so nothing it writes can land in the repo.@@ -260,26 +264,47 @@260264 261265 	test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => {262266 		await get(`/tarry/archive/${head}.tar.gz`);263-		const before = stored("0");264-		expect(existsSync(before)).toBe(true);267+		expect(kept(0)).toBe(true);265268 		await fx.strip("secret.txt");266269 		// syncRepo lands the strip without syncAll's cleanup: a crash between the two.267270 		await syncRepo(target.bucket, openMirror(target.mirrorDir, "tarry"), "tarry");268-		expect(existsSync(before)).toBe(true);271+		expect(kept(0)).toBe(true);269272 		const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));270273 		expect(listed).not.toContain("secret.txt");271274 		expect(listed).toContain("src/main.ts");272-		expect(existsSync(stored("1"))).toBe(true);275+		expect(kept(1)).toBe(true);273276 	});274277 275278 	test("a sync that lands a strip clears the repo's old cached tarballs", async () => {276279 		await get(`/tarry/archive/${head}.tar.gz`);277280 		await fx.strip("vendor/big.c");278281 		await syncAll(target);279-		expect(existsSync(stored("1"))).toBe(false);282+		expect(kept(1)).toBe(false);280283 		expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain(281284 			"big.c",282285 		);286+	});287+288+	test("a strip landing while a kept tarball is fetched is not served", async () => {289+		const mirror = openMirror(target.mirrorDir, "tarry");290+		await get(`/tarry/archive/${head}.tar.gz`);291+		let struck = false;292+		const racing: Bucket = {293+			...target.bucket,294+			async get(key) {295+				const bytes = await target.bucket.get(key);296+				if (!struck) {297+					struck = true;298+					await fx.strip("docs/guide.md");299+					await syncRepo(target.bucket, mirror, "tarry");300+				}301+				return bytes;302+			},303+		};304+		const tarball = await cachedTarball(racing, mirror, "tarry", head);305+		const listed = tar(["-tzf", "-"], tarball ?? new Uint8Array());306+		expect(listed).not.toContain("guide.md");307+		expect(listed).toContain("src/main.ts");283308 	});284309 });285310