keep built tarballs in the bucket under .kepr/tarballs instead of the data directory (v3.0.145)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.144",3+	"version": "3.0.145",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -39,7 +39,6 @@3939 	bucket: Bucket;4040 	kepr: Database;4141 	mirrorDir: string;42-	tarballDir: string;4342 };4443 4544 let running: Promise<void> | null = null;@@ -67,7 +66,7 @@6766 		} else {6867 			try {6968 				const events = await syncRepo(target.bucket, openMirror(target.mirrorDir, repo), repo);70-				if (events.some((event) => event.kind === "strip")) dropTarballs(target.tarballDir, repo);69+				if (events.some((event) => event.kind === "strip")) await dropTarballs(target.bucket, repo);7170 			} catch (cause) {7271 				error = cause instanceof Error ? cause.message : String(cause);7372 			}

@@ -18,7 +18,6 @@1818 		bucket,1919 		kepr,2020 		mirrorDir: join(config.dataDir, "mirror"),21-		tarballDir: join(config.dataDir, "tarballs"),2221 	});2322 	// Bun.serve reads the port from $PORT (default 3000).2423 	const server = Bun.serve({ fetch: createApp(config, kepr, bucket).fetch });

@@ -119,7 +119,7 @@119119 		config,120120 		configPath,121121 		dataDir: config.dataDir,122-		sync: { bucket, kepr, mirrorDir: ctx.mirrorDir, tarballDir: join(config.dataDir, "tarballs") },122+		sync: { bucket, kepr, mirrorDir: ctx.mirrorDir },123123 	};124124 	registerStatus(app, admin);125125 	registerRepos(app, admin);@@ -142,7 +142,7 @@142142 	registerArchivesPage(app, ctx);143143 	registerReleasesPage(app, ctx);144144 	registerIssuePages(app, ctx);145-	registerTarballRoutes(app, ctx, join(config.dataDir, "tarballs"));145+	registerTarballRoutes(app, ctx, bucket);146146 	registerReleaseRoutes(app, ctx, bucket);147147 	return app;148148 }

@@ -1,4 +1,5 @@11 import type { Hono } from "hono";2+import type { Bucket } from "../bucket/reader.ts";23 import { headId, saveByPrefix } from "../mirror/queries.ts";34 import { notFound, openPublicRepo, type PageContext } from "../pages/layout.ts";45 import { countInstall } from "../releases/releases.ts";@@ -8,7 +9,7 @@89 // so no shared cache keeps it after the repo turns private (§20).910 const TARBALL_CACHE = "private, max-age=31536000";1011 11-export function registerTarballRoutes(app: Hono, ctx: PageContext, tarballDir: string): void {12+export function registerTarballRoutes(app: Hono, ctx: PageContext, bucket: Bucket): void {1213 	app.get("/:repo/archive/HEAD.tar.gz", (c) => {1314 		const repo = openPublicRepo(ctx, c.req.param("repo"));1415 		const head = repo && headId(repo.mirror);@@ -20,7 +21,7 @@2021 		return c.redirect(path, 302);2122 	});2223 23-	app.get("/:repo/archive/:id{[0-9a-f]+\\.tar\\.gz}", (c) => {24+	app.get("/:repo/archive/:id{[0-9a-f]+\\.tar\\.gz}", async (c) => {2425 		const repo = openPublicRepo(ctx, c.req.param("repo"));2526 		const id = c.req.param("id").replace(/\.tar\.gz$/, "");2627 		const save = repo && saveByPrefix(repo.mirror, id);@@ -30,12 +31,12 @@3031 			c.header("Cache-Control", "public, max-age=60");3132 			return c.redirect(`/${name}/archive/${save.id}.tar.gz`, 302);3233 		}33-		const bytes = cachedTarball(tarballDir, repo.mirror, name, save.id);34+		const bytes = await cachedTarball(bucket, repo.mirror, name, save.id);3435 		// §22.2: a Nix fetch is an install. Only the tarball itself counts, not the redirects to it.3536 		if (c.req.header("User-Agent")?.includes("nix/")) countInstall(ctx.kepr, name);3637 		c.header("Content-Type", "application/gzip");3738 		c.header("Content-Disposition", `attachment; filename="${name}-${save.id.slice(0, 8)}.tar.gz"`);3839 		c.header("Cache-Control", TARBALL_CACHE);39-		return c.body(bytes);40+		return c.body(bytes.slice());4041 	});4142 }

@@ -1,41 +1,43 @@11 import type { Database } from "bun:sqlite";2-import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";3-import { join } from "node:path";2+import { type Bucket, BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";43 import { filesAt, objectBytes, stripCount } from "../mirror/queries.ts";54 import { buildTarball } from "./tar.ts";65 6+const tarballsOf = (repo: string) => `${KEPR_PREFIX}tarballs/${repo}/`;7+78 /**8- * A save's tarball, built on first request into `<tarballDir>/<repo>/<generation>/<id>.tar.gz`. The9- * generation is how many strip events the mirror holds, so once a strip lands no older file is ever10- * looked up again, even if Kepr stops before the sync's cleanup runs. Built and written in one11- * synchronous run, so a strip can't land mid-build either.9+ * A save's tarball, kept in the bucket under Kepr's own folder (June, 2026-09-30) as10+ * `.kepr/tarballs/<repo>/<generation>/<id>.tar.gz` once first requested. The generation is how11+ * many strip events the mirror holds, so once a strip lands no older file is ever looked up again,12+ * even if Kepr stops before the sync's cleanup runs. Without a write key nothing is kept, and each13+ * request builds it again.1214  */13-export function cachedTarball(14-	tarballDir: string,15+export async function cachedTarball(16+	bucket: Bucket,1517 	db: Database,1618 	repo: string,1719 	saveId: string,18-): Uint8Array<ArrayBuffer> {19-	const dir = join(tarballDir, repo, String(stripCount(db)));20-	const path = join(dir, `${saveId}.tar.gz`);21-	try {22-		return readFileSync(path);23-	} catch (error) {24-		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;25-	}20+): Promise<Uint8Array> {21+	const stored = await bucket.get(`${tarballsOf(repo)}${stripCount(db)}/${saveId}.tar.gz`);22+	if (stored) return stored;23+	// Read again after the await: the generation and the build below run in one synchronous go, so24+	// a strip landing meanwhile can't file new contents under the old generation or the reverse.25+	const key = `${tarballsOf(repo)}${stripCount(db)}/${saveId}.tar.gz`;2626 	const files = filesAt(db, saveId).map((file) => {2727 		const bytes = objectBytes(db, file.hash);2828 		if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);2929 		return { path: file.path, bytes, executable: file.executable };3030 	});3131 	const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);32-	mkdirSync(dir, { recursive: true });33-	writeFileSync(`${path}.tmp`, tarball);34-	renameSync(`${path}.tmp`, path);32+	try {33+		await bucket.put(key, tarball, "application/gzip");34+	} catch (error) {35+		if (!(error instanceof BucketWriteError)) throw error;36+	}3537 	return tarball;3638 }3739 38-/** Clears a repo's cached tarballs after a strip; cleanup only, since the new generation never reads them. */39-export function dropTarballs(tarballDir: string, repo: string): void {40-	rmSync(join(tarballDir, repo), { recursive: true, force: true });40+/** Deletes a repo's kept tarballs: after a strip, or from the console. They rebuild on request. */41+export async function dropTarballs(bucket: Bucket, repo: string): Promise<void> {42+	for (const key of await bucket.list(tarballsOf(repo))) await bucket.remove(key);4143 }

@@ -69,7 +69,6 @@6969 		bucket: openBucket(bucket),7070 		kepr,7171 		mirrorDir: join(dir, "mirror"),72-		tarballDir: join(dir, "tarballs"),7372 	});7473 	const config: Config = {7574 		domain: "kepr.example",

@@ -94,7 +94,6 @@9494 		bucket: openBucket({ kind: "folder", path: bucketDir }),9595 		kepr,9696 		mirrorDir: join(dataDir, "mirror"),97-		tarballDir: join(dataDir, "tarballs"),9897 	});9998 	setup(kepr);10099 	const config = {

@@ -45,7 +45,6 @@4545 		bucket,4646 		kepr,4747 		mirrorDir: join(dataDir, "mirror"),48-		tarballDir: join(dataDir, "tarballs"),4948 	});5049 	kepr.run("UPDATE repos SET public = 1, category = 'tools' WHERE name = 'alpha'");5150 	kepr.run("UPDATE repos SET category = 'secretcat' WHERE name = 'vault'");

@@ -45,7 +45,6 @@4545 		bucket: openBucket({ kind: "folder", path: bucketDir }),4646 		kepr: db,4747 		mirrorDir: join(dataDir, "mirror"),48-		tarballDir: join(dataDir, "tarballs"),4948 	});5049 	db.run("UPDATE repos SET public = 1");5150 	kepr = Bun.serve({ port: 0, fetch: () => new Response("starting") });

@@ -140,7 +140,6 @@140140 			bucket: openBucket(bucket),141141 			kepr,142142 			mirrorDir: join(dataDir, "mirror"),143-			tarballDir: join(dataDir, "tarballs"),144143 		});145144 		kepr.run("UPDATE repos SET public = 1 WHERE name = 'app'");146145 		app = createApp(

@@ -63,7 +63,6 @@6363 		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),6464 		kepr,6565 		mirrorDir: join(dataDir, "mirror"),66-		tarballDir: join(dataDir, "tarballs"),6766 	});6867 	expect(kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({ last_sync_error: null });6968 

@@ -77,7 +77,6 @@7777 			bucket: openBucket({ kind: "folder", path: bucketDir }),7878 			kepr,7979 			mirrorDir: join(dataDir, "mirror"),80-			tarballDir: join(dataDir, "tarballs"),8180 		});8281 		kepr.run(`UPDATE repos SET public = 1 WHERE name = '${REPO}'`);8382 		app = createApp(

@@ -26,7 +26,6 @@2626 		bucket: openBucket(inst.config.bucket),2727 		kepr: inst.kepr,2828 		mirrorDir: join(inst.dataDir, "mirror"),29-		tarballDir: join(inst.dataDir, "tarballs"),3029 	});3130 }3231 

@@ -129,7 +129,6 @@129129 						bucket: openBucket(bucket),130130 						kepr,131131 						mirrorDir: join(dataDir, "mirror"),132-						tarballDir: join(dataDir, "tarballs"),133132 					});134133 				await sync();135134 				kepr.query("UPDATE repos SET public = 1 WHERE name = ?").run(REPO);

@@ -103,7 +103,6 @@103103 			bucket: openBucket({ kind: "folder", path: space.bucketDir }),104104 			kepr,105105 			mirrorDir: join(dataDir, "mirror"),106-			tarballDir: join(dataDir, "tarballs"),107106 		});108107 		kepr.run("UPDATE repos SET public = 1 WHERE name = 'koh'");109108 

@@ -62,7 +62,7 @@6262 	mirrorDir = join(dataDir, "mirror");6363 	for (const bucketDir of [fx.bucketDir, secretRepo.bucketDir, large.bucketDir, tail.bucketDir]) {6464 		const bucket = openBucket({ kind: "folder", path: bucketDir });65-		await syncAll({ bucket, kepr, mirrorDir, tarballDir: join(dataDir, "tarballs") });65+		await syncAll({ bucket, kepr, mirrorDir });6666 	}6767 	kepr.run("UPDATE repos SET public = 1, homepage = 'https://demo.example' WHERE name = 'demo'");6868 	kepr.run("UPDATE repos SET public = 1 WHERE name IN ('large', 'tail')");

@@ -36,7 +36,6 @@3636 			bucket: openBucket({ kind: "folder", path: bucketDir }),3737 			kepr: openKeprDb(dataDir),3838 			mirrorDir: join(dataDir, "mirror"),39-			tarballDir: join(dataDir, "tarballs"),4039 		};4140 		for (const version of ["1.9.0", "1.10.0", "1.2.3"]) {4241 			const files = [{ platform: "linux-x86_64", name: "app", bytes: bytes(version) }];@@ -426,7 +425,6 @@426425 		bucket,427426 		kepr,428427 		mirrorDir: join(dataDir, "mirror"),429-		tarballDir: join(dataDir, "tarballs"),430428 	});431429 	setup(kepr);432430 	const app = createApp(

@@ -30,7 +30,6 @@3030 		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),3131 		kepr: openKeprDb(dataDir),3232 		mirrorDir: join(dataDir, "mirror"),33-		tarballDir: join(dataDir, "tarballs"),3433 	};3534 	return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };3635 }@@ -253,7 +252,6 @@253252 		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),254253 		kepr: inst.kepr,255254 		mirrorDir: join(inst.dataDir, "mirror"),256-		tarballDir: join(inst.dataDir, "tarballs"),257255 	});258256 	expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");259257 

@@ -20,6 +20,9 @@2020 let dataDir: string;2121 const get = (path: string) => app.request(path);2222 const bytesOf = async (path: string) => new Uint8Array(await (await get(path)).arrayBuffer());23+// Where a tarball of HEAD is kept in the folder bucket, for a strip generation.24+const stored = (generation: string) =>25+	join(fx.bucketDir, ".kepr", "tarballs", "tarry", generation, `${head}.tar.gz`);2326 2427 // Every root-level file on Kepr 2's strip list, the .agent folder, and the look-alikes that must stay.2528 const EXTRA: Record<string, string> = {@@ -56,7 +59,6 @@5659 		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),5760 		kepr,5861 		mirrorDir: join(dataDir, "mirror"),59-		tarballDir: join(dataDir, "tarballs"),6062 	};6163 	await syncAll(target);6264 	kepr.run("UPDATE repos SET public = 1");@@ -231,20 +233,23 @@231233 	test("is the same bytes when rebuilt, and in another process", async () => {232234 		const first = await bytesOf(`/tarry/archive/${head}.tar.gz`);233235 		const { dropTarballs } = await import("../src/tarball/cache.ts");234-		dropTarballs(target.tarballDir, "tarry");236+		await dropTarballs(target.bucket, "tarry");237+		expect(existsSync(stored("0"))).toBe(false);235238 		expect(await bytesOf(`/tarry/archive/${head}.tar.gz`)).toEqual(first);236239 		// Every path the other process touches is a fresh temp dir, its working directory included,237240 		// so nothing it writes can land in the repo.238241 		const script = `import { cachedTarball } from "${join(import.meta.dir, "../src/tarball/cache.ts")}";242+import { openBucket } from "${join(import.meta.dir, "../src/bucket/reader.ts")}";239243 import { openMirror } from "${join(import.meta.dir, "../src/mirror/store.ts")}";240-const { TARBALL_DIR, MIRROR_DIR, SAVE_ID } = process.env;241-process.stdout.write(cachedTarball(TARBALL_DIR, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`;244+const { BUCKET_DIR, MIRROR_DIR, SAVE_ID } = process.env;245+const bucket = openBucket({ kind: "folder", path: BUCKET_DIR });246+process.stdout.write(await cachedTarball(bucket, openMirror(MIRROR_DIR, "tarry"), "tarry", SAVE_ID));`;242247 		const scratch = await mkdtemp(join(tmpdir(), "kepr-tb-"));243248 		const other = Bun.spawnSync(["bun", "-e", script], {244249 			cwd: scratch,245250 			env: {246251 				PATH: process.env.PATH,247-				TARBALL_DIR: join(scratch, "tarballs"),252+				BUCKET_DIR: join(scratch, "bucket"),248253 				MIRROR_DIR: target.mirrorDir,249254 				SAVE_ID: head,250255 			},@@ -255,7 +260,7 @@255260 256261 	test("after a strip, an old cached tarball is never served, even if its cleanup never ran", async () => {257262 		await get(`/tarry/archive/${head}.tar.gz`);258-		const before = join(target.tarballDir, "tarry", "0", `${head}.tar.gz`);263+		const before = stored("0");259264 		expect(existsSync(before)).toBe(true);260265 		await fx.strip("secret.txt");261266 		// syncRepo lands the strip without syncAll's cleanup: a crash between the two.@@ -264,14 +269,14 @@264269 		const listed = tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`));265270 		expect(listed).not.toContain("secret.txt");266271 		expect(listed).toContain("src/main.ts");267-		expect(existsSync(join(target.tarballDir, "tarry", "1", `${head}.tar.gz`))).toBe(true);272+		expect(existsSync(stored("1"))).toBe(true);268273 	});269274 270275 	test("a sync that lands a strip clears the repo's old cached tarballs", async () => {271276 		await get(`/tarry/archive/${head}.tar.gz`);272277 		await fx.strip("vendor/big.c");273278 		await syncAll(target);274-		expect(existsSync(join(target.tarballDir, "tarry"))).toBe(false);279+		expect(existsSync(stored("1"))).toBe(false);275280 		expect(tar(["-tzf", "-"], await bytesOf(`/tarry/archive/${head}.tar.gz`))).not.toContain(276281 			"big.c",277282 		);