show bucket storage by repo at /admin/storage, with per-tarball delete and cleanup of Kepr's orphaned files (v3.0.162)

@@ -0,0 +1,120 @@1+import type { Context, Hono } from "hono";2+import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";3+import { listRepos } from "../bucket/repo.ts";4+import { tarballGeneration } from "../mirror/queries.ts";5+import { openMirror } from "../mirror/store.ts";6+import { html } from "../pages/html.ts";7+import { formatSize, plural } from "../pages/layout.ts";8+import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";9+10+const TARBALL = /^\.kepr\/tarballs\/([^/]+)\/([^/]+)\/([0-9a-f]{12})\.tar\.gz$/;11+const HIDDEN = /^\.kepr\/hidden\/([^/]+)$/;12+13+type RepoUse = { history: number; builds: number; tarballs: number };14+type Tarball = { key: string; repo: string; save: string; size: number };15+16+/**17+ * The bucket sorted into what §24's Storage section shows. Orphans are only Kepr's own files that18+ * nothing reads again; Koh's never are, since a build's files land before its record (§11.2).19+ */20+async function survey(ctx: AdminContext) {21+	const sizes = await ctx.sync.bucket.sizes("");22+	const repos = await listRepos(ctx.sync.bucket);23+	const generations = new Map(24+		repos.map((repo) => [repo, tarballGeneration(openMirror(ctx.mirrorDir, repo))]),25+	);26+	const use = new Map<string, RepoUse>(27+		repos.map((repo) => [repo, { history: 0, builds: 0, tarballs: 0 }]),28+	);29+	const kept: Tarball[] = [];30+	const orphans: string[] = [];31+	let kepr = 0;32+	let total = 0;33+	for (const [key, size] of sizes) {34+		total += size;35+		const tarball = TARBALL.exec(key);36+		const hidden = HIDDEN.exec(key);37+		if (tarball) {38+			const [, repo = "", generation, save = ""] = tarball;39+			const owner = use.get(repo);40+			if (owner && generations.get(repo) === generation) {41+				owner.tarballs += size;42+				kept.push({ key, repo, save, size });43+			} else orphans.push(key);44+		} else if (hidden && !use.has(hidden[1] ?? "")) orphans.push(key);45+		else if (key.startsWith(KEPR_PREFIX)) kepr += size;46+		else {47+			const [repo = "", kind] = key.split("/");48+			const owner = use.get(repo);49+			if (owner && kind === "releases") owner.builds += size;50+			else if (owner) owner.history += size;51+		}52+	}53+	const orphanBytes = orphans.reduce((sum, key) => sum + (sizes.get(key) ?? 0), 0);54+	return { use, kept, orphans, orphanBytes, kepr, total };55+}56+57+async function storagePage(c: Context, ctx: AdminContext, error: string) {58+	const { use, kept, orphans, orphanBytes, kepr, total } = await survey(ctx);59+	const body = html`${notice(error)}60+<p class="admin-intro">The bucket holds ${formatSize(total)}; Kepr's own folder ${formatSize(kepr)}61+besides its kept tarballs.</p>62+<section class="repo-section"><h2 class="section-label mono">by repository</h2>63+<table class="admin-table mono">64+<thead><tr><th>repo</th><th>history</th><th>builds</th><th>kept tarballs</th></tr></thead>65+<tbody>66+${[...use].map(67+	([repo, u]) => html`<tr>68+<td>${repo}</td>69+<td>${formatSize(u.history)}</td>70+<td>${formatSize(u.builds)}</td>71+<td>${formatSize(u.tarballs)}</td>72+</tr>`,73+)}74+</tbody></table></section>75+<section class="repo-section"><h2 class="section-label mono">orphans</h2>76+${77+	orphans.length === 078+		? html`<p class="muted mono">no orphans</p>`79+		: html`<form class="issues-form" method="post" action="/admin/storage">80+${formField(c)}<input type="hidden" name="action" value="delete_orphans">81+<p class="issue-meta mono">${plural(orphans.length, "orphan")}, ${formatSize(orphanBytes)}: tarballs82+of an older generation and files of repos no longer in the bucket.</p>83+<button class="issues-submit" type="submit">delete orphans</button></form>`84+}85+</section>86+<section class="repo-section"><h2 class="section-label mono">kept tarballs</h2>87+${kept.length === 0 && html`<p class="muted mono">none kept</p>`}88+${kept.map(89+	(t) => html`<form class="issue-row" method="post" action="/admin/storage">90+${formField(c)}<input type="hidden" name="action" value="delete_tarball">91+<input type="hidden" name="key" value="${t.key}">92+<span class="mono">${t.repo} ${t.save}.tar.gz</span>93+<span class="issue-meta mono">${formatSize(t.size)}</span>94+<button class="issues-submit" type="submit">delete</button></form>`,95+)}96+</section>`;97+	return adminPage(c, ctx, "storage", body);98+}99+100+/** §24's Storage section: sizes, kept tarballs and orphan cleanup. */101+export function registerStorage(app: Hono, ctx: AdminContext): void {102+	app.get("/admin/storage", async (c) => c.html(await storagePage(c, ctx, "")));103+104+	app.post("/admin/storage", async (c) => {105+		const fields = await textFields(c);106+		const { kept, orphans } = await survey(ctx);107+		const doomed =108+			fields.action === "delete_orphans"109+				? orphans110+				: kept.filter((t) => t.key === fields.key).map((t) => t.key);111+		if (doomed.length === 0 && fields.action !== "delete_orphans") return c.notFound();112+		try {113+			for (const key of doomed) await ctx.sync.bucket.remove(key);114+		} catch (error) {115+			if (!(error instanceof BucketWriteError)) throw error;116+			return c.html(await storagePage(c, ctx, error.message), 503);117+		}118+		return c.redirect("/admin/storage?saved=1", 303);119+	});120+}

@@ -0,0 +1,77 @@1+// Covers §24's Storage section: sizes by repo and kind, kept tarballs with a delete each, and2+// orphan cleanup that touches only Kepr's own files.3+4+import { beforeAll, describe, expect, test } from "bun:test";5+import { mkdir, writeFile } from "node:fs/promises";6+import { dirname, join } from "node:path";7+import { Browser, type Instance, instance } from "./browser.ts";8+import { makeRepos, writeRelease } from "./fixture.ts";9+10+describe("storage view (§24)", () => {11+	let inst: Instance;12+	let operator: Browser;13+	let bucketDir: string;14+	let save: string;15+	const exists = (key: string) => Bun.file(join(bucketDir, key)).exists();16+	const put = async (key: string) => {17+		await mkdir(dirname(join(bucketDir, key)), { recursive: true });18+		await writeFile(join(bucketDir, key), "x");19+	};20+	let generation = "";21+	const kept = () => `.kepr/tarballs/app/${generation}/${save}.tar.gz`;22+	const orphans = [23+		".kepr/tarballs/app/0-000000000000/aaaaaaaaaaaa.tar.gz",24+		".kepr/tarballs/gone/0-000000000000/aaaaaaaaaaaa.tar.gz",25+		".kepr/hidden/gone",26+	];27+	const unlisted = "app/releases/9.9.9/linux-x86_64/app";28+29+	beforeAll(async () => {30+		const repos = await makeRepos(["app"]);31+		bucketDir = repos.bucketDir;32+		save = repos.saves.app as string;33+		await writeRelease(bucketDir, "app", {34+			version: "1.0.0",35+			save,36+			files: [{ platform: "linux-x86_64", name: "app", bytes: new Uint8Array(2048) }],37+		});38+		inst = await instance(bucketDir);39+		operator = new Browser(inst.app);40+		await operator.signIn();41+		await operator.post("/admin/repos/app", { public: "1" });42+		expect((await inst.app.request(`/app/archive/${save}.tar.gz`)).status).toBe(200);43+		const listed = new Bun.Glob(`.kepr/tarballs/app/*/${save}.tar.gz`).scanSync({44+			cwd: bucketDir,45+			dot: true,46+		});47+		generation = (Array.from(listed)[0] as string).split("/")[3] as string;48+		for (const key of [...orphans, unlisted]) await put(key);49+	});50+51+	test("shows each repo's history, builds and kept tarballs, and the orphans", async () => {52+		const page = await (await operator.get("/admin/storage")).text();53+		expect(page).toMatch(/<td>app<\/td>\s*<td>[\d.]+ K?B<\/td>\s*<td>2\.\d KB<\/td>/);54+		expect(page).toContain(`${save}.tar.gz`);55+		expect(page).toContain("3 orphans");56+	});57+58+	test("deleting orphans removes only Kepr's stale files", async () => {59+		expect((await operator.post("/admin/storage", { action: "delete_orphans" })).status).toBe(303);60+		for (const key of orphans) expect(await exists(key)).toBe(false);61+		expect(await exists(kept())).toBe(true);62+		expect(await exists(unlisted)).toBe(true);63+		expect(await (await operator.get("/admin/storage")).text()).toContain("no orphans");64+	});65+66+	test("one kept tarball can be deleted; any other key is a 404", async () => {67+		const other = { action: "delete_tarball", key: "app/releases/1.0.0/linux-x86_64/app" };68+		expect((await operator.post("/admin/storage", other)).status).toBe(404);69+		expect(await exists("app/releases/1.0.0/linux-x86_64/app")).toBe(true);70+		const response = await operator.post("/admin/storage", {71+			action: "delete_tarball",72+			key: kept(),73+		});74+		expect(response.status).toBe(303);75+		expect(await exists(kept())).toBe(false);76+	});77+});