import type { Context, Hono } from "hono"; import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts"; import { listRepos } from "../bucket/repo.ts"; import type { SyncTarget } from "../bucket/sync.ts"; import { tarballGeneration } from "../mirror/queries.ts"; import { openMirror } from "../mirror/store.ts"; import { html } from "../pages/html.ts"; import { formatSize, plural, relativeTime } from "../pages/layout.ts"; import { keepState, setting } from "../server/db.ts"; import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts"; import { allRepos } from "./store.ts"; const TARBALL = /^\.kepr\/tarballs\/([^/]+)\/([^/]+)\/([0-9a-f]{6}|[0-9a-f]{12})\.tar\.gz$/; const HIDDEN = /^\.kepr\/hidden\/([^/]+)$/; type RepoUse = { history: number; builds: number; tarballs: number }; type Tarball = { key: string; repo: string; save: string; size: number }; type Survey = { /** When it was measured, in Unix seconds. */ at: number; use: [repo: string, use: RepoUse][]; kept: Tarball[]; orphans: [key: string, size: number][]; kepr: number; total: number; }; // A full listing of the bucket takes seconds (3.2 s for 22k keys on Tylee, 2026-10-02), so it is // measured in the background and the page shows the last measurement. const MEASURE_MS = 60 * 60 * 1000; // Measuring and deleting take turns, so neither stores a survey the other has made stale. let turn: Promise = Promise.resolve(); function inStorageTurn(work: () => Promise): Promise { const mine = turn.then(work); turn = mine.catch(() => {}); return mine; } export function startStorageLoop(target: SyncTarget): void { const tick = () => measureStorage(target).catch((error) => console.error(`kepr: measuring storage failed: ${error}`), ); tick(); setInterval(tick, MEASURE_MS); } /** * Sorts the bucket into what §24's Storage section shows, and keeps it for the page. Orphans are * only Kepr's own files that nothing reads again; Koh's never are, since a build's files land * before its record (§11.2). */ export function measureStorage(target: SyncTarget): Promise { return inStorageTurn(() => measure(target)); } async function measure(target: SyncTarget): Promise { const sizes = await target.bucket.sizes(""); const repos = await listRepos(target.bucket); const isOrphan = orphanTest(target, repos); const use = new Map( repos.map((repo) => [repo, { history: 0, builds: 0, tarballs: 0 }]), ); const survey: Survey = { at: Math.floor(Date.now() / 1000), use: [], kept: [], orphans: [], kepr: 0, total: 0, }; for (const [key, size] of sizes) { survey.total += size; const tarball = TARBALL.exec(key); if (isOrphan(key)) survey.orphans.push([key, size]); else if (tarball) { const [, repo = "", , save = ""] = tarball; // Not an orphan, so its repo is in the bucket and in `use`. (use.get(repo) as RepoUse).tarballs += size; survey.kept.push({ key, repo, save, size }); } else if (key.startsWith(KEPR_PREFIX)) survey.kepr += size; else { const [repo = "", kind] = key.split("/"); const owner = use.get(repo); if (owner && kind === "releases") owner.builds += size; else if (owner) owner.history += size; } } survey.use = [...use]; keepState(target.kepr, "storage", JSON.stringify(survey)); } /** * Whether a key is an orphan given the repos now in the bucket: a tarball of an older generation, * or Kepr's file of a repo no longer there. Only mirrors of repos Kepr has registered are opened, as * pages do: a rename's new name is registered only once its mirror has moved into place. */ function orphanTest(target: SyncTarget, repos: string[]): (key: string) => boolean { const registered = new Set(allRepos(target.kepr).map((repo) => repo.name)); const generations = new Map( repos .filter((repo) => registered.has(repo)) .map((repo) => [repo, tarballGeneration(openMirror(target.mirrorDir, repo))]), ); return (key) => { const tarball = TARBALL.exec(key); const [, repo = "", generation] = tarball ?? HIDDEN.exec(key) ?? []; if (!repos.includes(repo)) return tarball !== null || HIDDEN.test(key); // A repo not registered yet has no tarballs of its own making; its keys are left alone. return tarball !== null && generations.has(repo) && generations.get(repo) !== generation; }; } function storedSurvey(ctx: AdminContext): Survey | null { const stored = setting(ctx.kepr, "storage"); return stored ? (JSON.parse(stored) as Survey) : null; } /** The survey without keys just deleted, so the page never lists what is gone. */ function withoutKeys(survey: Survey, gone: Set): Survey { const sizeOf = new Map([...survey.orphans, ...survey.kept.map((t) => [t.key, t.size] as const)]); const freed = [...gone].reduce((sum, key) => sum + (sizeOf.get(key) ?? 0), 0); const use = survey.use.map(([repo, u]): [string, RepoUse] => { const dropped = survey.kept.filter((t) => t.repo === repo && gone.has(t.key)); return [repo, { ...u, tarballs: u.tarballs - dropped.reduce((sum, t) => sum + t.size, 0) }]; }); return { ...survey, use, kept: survey.kept.filter((t) => !gone.has(t.key)), orphans: survey.orphans.filter(([key]) => !gone.has(key)), total: survey.total - freed, }; } /** * Deletes the orphans, or one kept tarball, from the stored survey; returns "unknown" for a key it * doesn't list, the bucket's refusal, or "" when done. */ async function deleteKeys(ctx: AdminContext, fields: Record): Promise { const survey = storedSurvey(ctx); if (!survey) return "unknown"; let doomed: string[]; if (fields.action === "delete_orphans") { // Checked again now: an older generation can become current again, as unhiding does. const isOrphan = orphanTest(ctx.sync, await listRepos(ctx.sync.bucket)); doomed = survey.orphans.map(([key]) => key).filter(isOrphan); } else { doomed = survey.kept.filter((t) => t.key === fields.key).map((t) => t.key); if (doomed.length === 0) return "unknown"; } const gone = new Set(); try { for (const key of doomed) { await ctx.sync.bucket.remove(key); gone.add(key); } } catch (error) { if (!(error instanceof BucketWriteError)) throw error; return error.message; } finally { keepState(ctx.kepr, "storage", JSON.stringify(withoutKeys(survey, gone))); } return ""; } function storagePage(c: Context, ctx: AdminContext, error: string) { const survey = storedSurvey(ctx); if (!survey) { const body = html`${notice(error)}

Not measured yet: Kepr measures the bucket when it starts and every hour.

`; return adminPage(c, ctx, "storage", body); } const { use, kept, orphans, kepr, total } = survey; const orphanBytes = orphans.reduce((sum, [, size]) => sum + size, 0); const body = html`${notice(error)}

The bucket holds ${formatSize(total)}; Kepr's own folder ${formatSize(kepr)} besides its kept tarballs. Measured ${relativeTime(survey.at)}; Kepr measures it every hour.

${use.map( ([repo, u]) => html``, )}
repohistorybuildskept tarballs
${repo} ${formatSize(u.history)} ${formatSize(u.builds)} ${formatSize(u.tarballs)}
${ orphans.length === 0 ? html`

no orphans

` : html`
${formField(c)}

${plural(orphans.length, "orphan")}, ${formatSize(orphanBytes)}: tarballs of an older generation and files of repos no longer in the bucket.

` }
${kept.length === 0 && html`

none kept

`} ${kept.map( (t) => html`
${formField(c)} ${t.repo} ${t.save}.tar.gz ${formatSize(t.size)}
`, )}
`; return adminPage(c, ctx, "storage", body); } /** §24's Storage section: sizes, kept tarballs and orphan cleanup. */ export function registerStorage(app: Hono, ctx: AdminContext): void { app.get("/admin/storage", (c) => c.html(storagePage(c, ctx, ""))); app.post("/admin/storage", async (c) => { const fields = await textFields(c); // While no sync or rename runs: a rename can make a listed orphan live again mid-delete. let outcome = ""; await ctx.syncer.hold(() => inStorageTurn(async () => { outcome = await deleteKeys(ctx, fields); }), ); if (outcome === "unknown") return c.notFound(); if (outcome) return c.html(storagePage(c, ctx, outcome), 503); return c.redirect("/admin/storage?saved=1", 303); }); }