storage.ts

1import type { Context, Hono } from "hono";2import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";3import { listRepos } from "../bucket/repo.ts";4import type { SyncTarget } from "../bucket/sync.ts";5import { tarballGeneration } from "../mirror/queries.ts";6import { openMirror } from "../mirror/store.ts";7import { html } from "../pages/html.ts";8import { formatSize, plural, relativeTime } from "../pages/layout.ts";9import { keepState, setting } from "../server/db.ts";10import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";11import { allRepos } from "./store.ts";1213const TARBALL = /^\.kepr\/tarballs\/([^/]+)\/([^/]+)\/([0-9a-f]{6}|[0-9a-f]{12})\.tar\.gz$/;14const HIDDEN = /^\.kepr\/hidden\/([^/]+)$/;1516type RepoUse = { history: number; builds: number; tarballs: number };17type Tarball = { key: string; repo: string; save: string; size: number };18type Survey = {19	/** When it was measured, in Unix seconds. */20	at: number;21	use: [repo: string, use: RepoUse][];22	kept: Tarball[];23	orphans: [key: string, size: number][];24	kepr: number;25	total: number;26};2728// A full listing of the bucket takes seconds (3.2 s for 22k keys on Tylee, 2026-10-02), so it is29// measured in the background and the page shows the last measurement.30const MEASURE_MS = 60 * 60 * 1000;3132// Measuring and deleting take turns, so neither stores a survey the other has made stale.33let turn: Promise<unknown> = Promise.resolve();34function inStorageTurn<T>(work: () => Promise<T>): Promise<T> {35	const mine = turn.then(work);36	turn = mine.catch(() => {});37	return mine;38}3940export function startStorageLoop(target: SyncTarget): void {41	const tick = () =>42		measureStorage(target).catch((error) =>43			console.error(`kepr: measuring storage failed: ${error}`),44		);45	tick();46	setInterval(tick, MEASURE_MS);47}4849/**50 * Sorts the bucket into what §24's Storage section shows, and keeps it for the page. Orphans are51 * only Kepr's own files that nothing reads again; Koh's never are, since a build's files land52 * before its record (§11.2).53 */54export function measureStorage(target: SyncTarget): Promise<void> {55	return inStorageTurn(() => measure(target));56}5758async function measure(target: SyncTarget): Promise<void> {59	const sizes = await target.bucket.sizes("");60	const repos = await listRepos(target.bucket);61	const isOrphan = orphanTest(target, repos);62	const use = new Map<string, RepoUse>(63		repos.map((repo) => [repo, { history: 0, builds: 0, tarballs: 0 }]),64	);65	const survey: Survey = {66		at: Math.floor(Date.now() / 1000),67		use: [],68		kept: [],69		orphans: [],70		kepr: 0,71		total: 0,72	};73	for (const [key, size] of sizes) {74		survey.total += size;75		const tarball = TARBALL.exec(key);76		if (isOrphan(key)) survey.orphans.push([key, size]);77		else if (tarball) {78			const [, repo = "", , save = ""] = tarball;79			// Not an orphan, so its repo is in the bucket and in `use`.80			(use.get(repo) as RepoUse).tarballs += size;81			survey.kept.push({ key, repo, save, size });82		} else if (key.startsWith(KEPR_PREFIX)) survey.kepr += size;83		else {84			const [repo = "", kind] = key.split("/");85			const owner = use.get(repo);86			if (owner && kind === "releases") owner.builds += size;87			else if (owner) owner.history += size;88		}89	}90	survey.use = [...use];91	keepState(target.kepr, "storage", JSON.stringify(survey));92}9394/**95 * Whether a key is an orphan given the repos now in the bucket: a tarball of an older generation,96 * or Kepr's file of a repo no longer there. Only mirrors of repos Kepr has registered are opened, as97 * pages do: a rename's new name is registered only once its mirror has moved into place.98 */99function orphanTest(target: SyncTarget, repos: string[]): (key: string) => boolean {100	const registered = new Set(allRepos(target.kepr).map((repo) => repo.name));101	const generations = new Map(102		repos103			.filter((repo) => registered.has(repo))104			.map((repo) => [repo, tarballGeneration(openMirror(target.mirrorDir, repo))]),105	);106	return (key) => {107		const tarball = TARBALL.exec(key);108		const [, repo = "", generation] = tarball ?? HIDDEN.exec(key) ?? [];109		if (!repos.includes(repo)) return tarball !== null || HIDDEN.test(key);110		// A repo not registered yet has no tarballs of its own making; its keys are left alone.111		return tarball !== null && generations.has(repo) && generations.get(repo) !== generation;112	};113}114115function storedSurvey(ctx: AdminContext): Survey | null {116	const stored = setting(ctx.kepr, "storage");117	return stored ? (JSON.parse(stored) as Survey) : null;118}119120/** The survey without keys just deleted, so the page never lists what is gone. */121function withoutKeys(survey: Survey, gone: Set<string>): Survey {122	const sizeOf = new Map([...survey.orphans, ...survey.kept.map((t) => [t.key, t.size] as const)]);123	const freed = [...gone].reduce((sum, key) => sum + (sizeOf.get(key) ?? 0), 0);124	const use = survey.use.map(([repo, u]): [string, RepoUse] => {125		const dropped = survey.kept.filter((t) => t.repo === repo && gone.has(t.key));126		return [repo, { ...u, tarballs: u.tarballs - dropped.reduce((sum, t) => sum + t.size, 0) }];127	});128	return {129		...survey,130		use,131		kept: survey.kept.filter((t) => !gone.has(t.key)),132		orphans: survey.orphans.filter(([key]) => !gone.has(key)),133		total: survey.total - freed,134	};135}136137/**138 * Deletes the orphans, or one kept tarball, from the stored survey; returns "unknown" for a key it139 * doesn't list, the bucket's refusal, or "" when done.140 */141async function deleteKeys(ctx: AdminContext, fields: Record<string, string>): Promise<string> {142	const survey = storedSurvey(ctx);143	if (!survey) return "unknown";144	let doomed: string[];145	if (fields.action === "delete_orphans") {146		// Checked again now: an older generation can become current again, as unhiding does.147		const isOrphan = orphanTest(ctx.sync, await listRepos(ctx.sync.bucket));148		doomed = survey.orphans.map(([key]) => key).filter(isOrphan);149	} else {150		doomed = survey.kept.filter((t) => t.key === fields.key).map((t) => t.key);151		if (doomed.length === 0) return "unknown";152	}153	const gone = new Set<string>();154	try {155		for (const key of doomed) {156			await ctx.sync.bucket.remove(key);157			gone.add(key);158		}159	} catch (error) {160		if (!(error instanceof BucketWriteError)) throw error;161		return error.message;162	} finally {163		keepState(ctx.kepr, "storage", JSON.stringify(withoutKeys(survey, gone)));164	}165	return "";166}167168function storagePage(c: Context, ctx: AdminContext, error: string) {169	const survey = storedSurvey(ctx);170	if (!survey) {171		const body = html`${notice(error)}172<p class="admin-intro">Not measured yet: Kepr measures the bucket when it starts and every hour.</p>`;173		return adminPage(c, ctx, "storage", body);174	}175	const { use, kept, orphans, kepr, total } = survey;176	const orphanBytes = orphans.reduce((sum, [, size]) => sum + size, 0);177	const body = html`${notice(error)}178<p class="admin-intro">The bucket holds ${formatSize(total)}; Kepr's own folder ${formatSize(kepr)}179besides its kept tarballs. Measured ${relativeTime(survey.at)}; Kepr measures it every hour.</p>180<section class="repo-section"><h2 class="section-label mono">by repository</h2>181<table class="admin-table mono">182<thead><tr><th>repo</th><th>history</th><th>builds</th><th>kept tarballs</th></tr></thead>183<tbody>184${use.map(185	([repo, u]) => html`<tr>186<td>${repo}</td>187<td>${formatSize(u.history)}</td>188<td>${formatSize(u.builds)}</td>189<td>${formatSize(u.tarballs)}</td>190</tr>`,191)}192</tbody></table></section>193<section class="repo-section"><h2 class="section-label mono">orphans</h2>194${195	orphans.length === 0196		? html`<p class="muted mono">no orphans</p>`197		: html`<form class="issues-form" method="post" action="/admin/storage">198${formField(c)}<input type="hidden" name="action" value="delete_orphans">199<p class="issue-meta mono">${plural(orphans.length, "orphan")}, ${formatSize(orphanBytes)}: tarballs200of an older generation and files of repos no longer in the bucket.</p>201<button class="issues-submit" type="submit">delete orphans</button></form>`202}203</section>204<section class="repo-section"><h2 class="section-label mono">kept tarballs</h2>205${kept.length === 0 && html`<p class="muted mono">none kept</p>`}206${kept.map(207	(t) => html`<form class="issue-row" method="post" action="/admin/storage">208${formField(c)}<input type="hidden" name="action" value="delete_tarball">209<input type="hidden" name="key" value="${t.key}">210<span class="mono">${t.repo} ${t.save}.tar.gz</span>211<span class="issue-meta mono">${formatSize(t.size)}</span>212<button class="issues-submit" type="submit">delete</button></form>`,213)}214</section>`;215	return adminPage(c, ctx, "storage", body);216}217218/** §24's Storage section: sizes, kept tarballs and orphan cleanup. */219export function registerStorage(app: Hono, ctx: AdminContext): void {220	app.get("/admin/storage", (c) => c.html(storagePage(c, ctx, "")));221222	app.post("/admin/storage", async (c) => {223		const fields = await textFields(c);224		// While no sync or rename runs: a rename can make a listed orphan live again mid-delete.225		let outcome = "";226		await ctx.syncer.hold(() =>227			inStorageTurn(async () => {228				outcome = await deleteKeys(ctx, fields);229			}),230		);231		if (outcome === "unknown") return c.notFound();232		if (outcome) return c.html(storagePage(c, ctx, outcome), 503);233		return c.redirect("/admin/storage?saved=1", 303);234	});235}