copy each backup to an optional backup bucket set in the console, keeping the newest 14 there (v3.0.155)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.154",3+	"version": "3.0.155",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -3,26 +3,46 @@33 import { formatSize } from "../pages/layout.ts";44 import {55 	attemptBackup,6+	attemptUpload,67 	backupBytes,78 	backups,89 	backupTarball,10+	checkBackupBucket,911 	makeBackup,12+	offsiteBackups,1013 } from "../server/backup.ts";14+import { type BackupBucket, saveConfig } from "../server/config.ts";1115 import { setting } from "../server/db.ts";12-import { type AdminContext, adminPage, formField, notice } from "./console.ts";16+import { s3FieldsProblem } from "./bucket.ts";17+import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";1318 19+// The backup bucket's fields shown back to the operator; its secret is never among them.20+const SHOWN: [key: string, label: string][] = [21+	["endpoint", "endpoint"],22+	["name", "bucket name"],23+	["region", "region"],24+	["access_key_id", "access key ID"],25+];26+const REQUIRED = new Set(SHOWN.map(([key]) => key));27+1428 /** `20260926T031500Z` as `2026-09-26 03:15:00 UTC`. */1529 function backupTime(name: string): string {1630 	return name.replace(/^(\d{4})(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)Z$/, "$1-$2-$3 $4:$5:$6 UTC");1731 }1832 19-/** §24's Backups section: every kept backup, back up now, the last failure, and how to restore. */20-export function backupsPage(c: Context, ctx: AdminContext, error = ""): string {33+type Shown = { error?: string; status?: string; fields?: Record<string, string> };34+35+/**36+ * §24's Backups section: every kept backup, back up now, the last failure, how to restore, and the37+ * optional backup bucket (June, 2026-09-30) with what it holds.38+ */39+export async function backupsPage(c: Context, ctx: AdminContext, shown: Shown = {}) {2140 	const kept = backups(ctx.dataDir).reverse();2241 	const newest = kept[0];2342 	const failure = setting(ctx.kepr, "backup_error");24-	const body = html`${notice(error)}25-${failure && !error && html`<p class="issues-error mono" role="alert">The last backup failed: ${failure}</p>`}43+	const body = html`${notice(shown.error ?? "")}44+${shown.status && html`<p class="admin-notice" role="status">${shown.status}</p>`}45+${failure && !shown.error && html`<p class="issues-error mono" role="alert">The last backup failed: ${failure}</p>`}2646 ${2747 	newest2848 		? html`<p>Last backup: ${backupTime(newest)}. <a href="/admin/backup">download backup</a></p>`@@ -35,18 +55,86 @@3555 		(name) => html`<li><a href="/admin/backup?name=${name}">${backupTime(name)}</a>3656 <span class="muted">${formatSize(backupBytes(ctx.dataDir, name))}</span></li>`,3757 	)}</ul>58+${await bucketSection(c, ctx, shown.fields)}3859 <section class="repo-section"><h2 class="section-label mono">restoring one</h2>3960 <div class="markdown"><ol>4061 <li>Stop Kepr.</li>4162 <li>Unpack the download: it holds a folder with kepr.db and assets/.</li>42-<li>Put kepr.db and assets/ in the data directory named in kepr.toml, over the old ones.</li>63+<li>Put kepr.db and assets/ in Kepr's data directory, over the old ones.</li>4364 <li>Start Kepr. Its first sync rebuilds the mirrors, and everyone signs in again.</li>4465 </ol>45-<p class="muted">Backups stay on this server. Keep a copy elsewhere, along with kepr.toml.</p>46-<p class="muted">Sessions and bucket credentials are not included.</p></div></section>`;66+<p class="muted">Sessions and bucket credentials are not included: after restoring onto a new67+server, enter the bucket settings again here.</p></div></section>`;4768 	return adminPage(c, ctx, "backups", body);4869 }4970 71+/** The backup bucket's form, its last upload and what it holds. */72+async function bucketSection(c: Context, ctx: AdminContext, fields?: Record<string, string>) {73+	const current = ctx.config.backup;74+	const stored: Record<string, string> = {75+		endpoint: current?.endpoint ?? "",76+		name: current?.name ?? "",77+		region: current?.region ?? "",78+		access_key_id: current?.accessKeyId ?? "",79+	};80+	const value = (key: string) => fields?.[key] ?? stored[key] ?? "";81+	const uploadError = setting(ctx.kepr, "backup_upload_error");82+	let held: Awaited<ReturnType<typeof offsiteBackups>> | string = [];83+	if (current) {84+		try {85+			held = (await offsiteBackups(current)).reverse();86+		} catch (error) {87+			held = `Could not list the backup bucket: ${String(error)}`;88+		}89+	}90+	return html`<section class="repo-section"><h2 class="section-label mono">backup bucket</h2>91+<p class="admin-intro">An optional second S3 bucket, apart from the one Koh offers to. Each backup is92+copied there too, and the newest 14 kept. Its key needs to read, write and delete.</p>93+${uploadError && html`<p class="issues-error mono" role="alert">The last upload to the backup bucket failed: ${uploadError}</p>`}94+${typeof held === "string" && html`<p class="issues-error mono">${held}</p>`}95+${96+	Array.isArray(held) &&97+	held.length > 0 &&98+	html`<p class="muted">${String(held.length)} in the backup bucket:</p>99+<ul class="admin-releases mono">${held.map(100+		(backup) =>101+			html`<li>${backupTime(backup.name)} <span class="muted">${formatSize(backup.size)}</span></li>`,102+	)}</ul>`103+}104+<form class="issues-form" method="post" action="/admin/backup-bucket">105+${formField(c)}106+${SHOWN.map(107+	([key, label]) => html`<label class="issues-label" for="backup-${key}">${label}</label>108+<input class="issues-input mono" id="backup-${key}" name="${key}" value="${value(key)}"109+autocomplete="off" spellcheck="false">`,110+)}111+<label class="issues-label" for="backup-secret_access_key">secret key</label>112+<input class="issues-input mono" id="backup-secret_access_key" name="secret_access_key"113+type="password" autocomplete="new-password" placeholder="${current ? "unchanged" : ""}">114+<div class="issues-actions">115+<button class="issues-submit" type="submit" name="action" value="test">test connection</button>116+<button class="issues-submit" type="submit" name="action" value="save">save</button>117+${current && html`<button class="issues-submit" type="submit" name="action" value="remove">remove</button>`}118+</div>119+</form></section>`;120+}121+122+/** The backup bucket a form describes, a blank secret keeping the stored one; or what is wrong. */123+function candidate(fields: Record<string, string>, current?: BackupBucket): BackupBucket | string {124+	const problem = s3FieldsProblem(fields, SHOWN, REQUIRED);125+	if (problem) return problem;126+	const value = (key: string) => (fields[key] ?? "").trim();127+	const secretAccessKey = value("secret_access_key") || current?.secretAccessKey;128+	if (!secretAccessKey) return "The secret key is required.";129+	return {130+		endpoint: value("endpoint"),131+		name: value("name"),132+		region: value("region"),133+		accessKeyId: value("access_key_id"),134+		secretAccessKey,135+	};136+}137+50138 export function registerBackups(app: Hono, ctx: AdminContext): void {51139 	app.get("/admin/backup", (c) => {52140 		const backup = backupTarball(ctx.dataDir, c.req.query("name"));@@ -57,9 +145,43 @@57145 		});58146 	});59147 60-	app.post("/admin/backup", (c) => {148+	app.post("/admin/backup", async (c) => {61149 		const failure = attemptBackup(ctx.kepr, () => makeBackup(ctx.kepr, ctx.dataDir));62-		if (failure) return c.html(backupsPage(c, ctx, `The backup failed: ${failure}`), 500);150+		if (failure)151+			return c.html(await backupsPage(c, ctx, { error: `The backup failed: ${failure}` }), 500);152+		const upload = await attemptUpload(ctx.kepr, ctx.dataDir, ctx.config.backup);153+		if (upload) {154+			const error = `The backup was made here, but not copied to the backup bucket: ${upload}`;155+			return c.html(await backupsPage(c, ctx, { error }), 502);156+		}157+		return c.redirect("/admin?section=backups&saved=1", 303);158+	});159+160+	app.post("/admin/backup-bucket", async (c) => {161+		const fields = await textFields(c);162+		const refuse = async (error: string, status: 400 | 500) =>163+			c.html(await backupsPage(c, ctx, { error, fields }), status);164+		let backup: BackupBucket | undefined;165+		if (fields.action !== "remove") {166+			const next = candidate(fields, ctx.config.backup);167+			if (typeof next === "string") return refuse(next, 400);168+			const failure = await checkBackupBucket(next);169+			if (failure) return refuse(failure, 400);170+			if (fields.action !== "save") {171+				const status = "The backup bucket works. Nothing was saved.";172+				return c.html(await backupsPage(c, ctx, { status, fields }));173+			}174+			backup = next;175+		}176+		const { backup: _, ...rest } = ctx.config;177+		try {178+			await saveConfig(ctx.configPath, backup ? { ...rest, backup } : rest);179+		} catch (error) {180+			return refuse(`The settings could not be saved: ${String(error)}`, 500);181+		}182+		// In place, so the backup loop sees it at its next check.183+		if (backup) ctx.config.backup = backup;184+		else delete ctx.config.backup;63185 		return c.redirect("/admin?section=backups&saved=1", 303);64186 	});65187 }

@@ -18,21 +18,37 @@1818 // Keys, names and URLs have neither; a newline would also end the TOML line it is written to.1919 const UNPRINTABLE = /[\s\p{C}]/u;2020 21-/** The S3 settings a form describes, a blank secret keeping the stored one; or what is wrong. */22-function candidate(fields: Record<string, string>, current: S3Source | null): S3Source | string {21+/**22+ * What is wrong with a form's S3 fields, or "": the required ones filled, none of them nor any23+ * secret holding spaces or control characters, and an http or https endpoint.24+ */25+export function s3FieldsProblem(26+	fields: Record<string, string>,27+	shown: [key: string, label: string][],28+	required: Set<string>,29+): string {2330 	const value = (key: string) => (fields[key] ?? "").trim();24-	for (const [key, label] of SHOWN) {25-		if (REQUIRED.has(key) && value(key) === "") return `The ${label} is required.`;31+	for (const [key, label] of shown) {32+		if (required.has(key) && value(key) === "") return `The ${label} is required.`;2633 		if (UNPRINTABLE.test(fields[key] ?? ""))2734 			return `The ${label} cannot contain spaces or control characters.`;2835 	}29-	for (const key of ["secret_access_key", "write_secret_access_key"]) {36+	for (const key of Object.keys(fields).filter((name) => name.endsWith("secret_access_key"))) {3037 		if (UNPRINTABLE.test(value(key)))3138 			return "A secret cannot contain spaces or control characters.";3239 	}3340 	const endpoint = value("endpoint");3441 	if (!URL.canParse(endpoint) || !/^https?:$/.test(new URL(endpoint).protocol))3542 		return "The endpoint must be an http or https URL.";43+	return "";44+}45+46+/** The S3 settings a form describes, a blank secret keeping the stored one; or what is wrong. */47+function candidate(fields: Record<string, string>, current: S3Source | null): S3Source | string {48+	const value = (key: string) => (fields[key] ?? "").trim();49+	const problem = s3FieldsProblem(fields, SHOWN, REQUIRED);50+	if (problem) return problem;51+	const endpoint = value("endpoint");3652 	const secretAccessKey = value("secret_access_key") || current?.secretAccessKey;3753 	if (!secretAccessKey) return "The read secret key is required.";3854 	const source: S3Source = {

@@ -132,7 +132,7 @@132132 /** §24's Status section at `/admin`: the bucket, each repo's format, backups, the running version. */133133 export function registerStatus(app: Hono, ctx: AdminContext): void {134134 	app.get("/admin", async (c) => {135-		if (c.req.query("section") === "backups") return c.html(backupsPage(c, ctx));135+		if (c.req.query("section") === "backups") return c.html(await backupsPage(c, ctx));136136 		let bucket: Html;137137 		try {138138 			const repos = await listRepos(ctx.sync.bucket);

@@ -12,7 +12,7 @@1212 	const kepr = openKeprDb(config.dataDir);1313 	// One bucket for the sync loop and the app, so settings saved in the console reach both.1414 	const bucket = openBucket(config.bucket);15-	startBackupLoop(kepr, config.dataDir);15+	startBackupLoop(kepr, config);1616 	startMailLoop(kepr, config);1717 	startSyncLoop({1818 		bucket,

@@ -11,6 +11,7 @@1111 } from "node:fs";1212 import { join, relative } from "node:path";1313 import { buildTarball } from "../tarball/tar.ts";14+import type { BackupBucket, Config } from "./config.ts";1415 import { notify, setting } from "./db.ts";1516 1617 const KEEP = 14;@@ -68,9 +69,17 @@6869 	return newest && newest > stamp(now - DAY_MS) ? null : makeBackup(kepr, dataDir, now);6970 }7071 71-/** Daily backups: checked at startup and every hour, so a restart never skips a day. */72-export function startBackupLoop(kepr: Database, dataDir: string): void {73-	const tick = () => attemptBackup(kepr, () => backupIfDue(kepr, dataDir));72+/**73+ * Daily backups: checked at startup and every hour, so a restart never skips a day, and an upload74+ * to the backup bucket that failed is retried. `config` is the live one the console edits.75+ */76+export function startBackupLoop(kepr: Database, config: Config): void {77+	const tick = () => {78+		attemptBackup(kepr, () => backupIfDue(kepr, config.dataDir));79+		attemptUpload(kepr, config.dataDir, config.backup).catch((error) =>80+			console.error(`kepr: backup upload failed: ${error}`),81+		);82+	};7483 	tick();7584 	setInterval(tick, CHECK_MS);7685 }@@ -85,15 +94,91 @@8594 		kepr.query("DELETE FROM settings WHERE key = 'backup_error'").run();8695 		return null;8796 	} catch (error) {88-		const message = String(error);89-		console.error(`kepr: backup failed: ${message}`);90-		if (!setting(kepr, "backup_error")) {91-			notify(kepr, "backup_failed", `Backup failed: ${message}`, "/admin?section=backups");97+		return keepFailure(kepr, "backup_error", "Backup failed", error);98+	}99+}100+101+/** Keeps a failure under `key` for the console, telling the operator only when it starts. */102+function keepFailure(kepr: Database, key: string, what: string, error: unknown): string {103+	const message = String(error);104+	console.error(`kepr: ${what.toLowerCase()}: ${message}`);105+	if (!setting(kepr, key)) {106+		notify(kepr, "backup_failed", `${what}: ${message}`, "/admin?section=backups");107+	}108+	kepr.query("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)").run(key, message);109+	return message;110+}111+112+const OFFSITE = /^kepr-backup-(\d{8}T\d{6}Z)\.tar\.gz$/;113+const offsiteKey = (name: string) => `kepr-backup-${name}.tar.gz`;114+115+function backupClient(bucket: BackupBucket): Bun.S3Client {116+	return new Bun.S3Client({117+		endpoint: bucket.endpoint,118+		bucket: bucket.name,119+		region: bucket.region,120+		accessKeyId: bucket.accessKeyId,121+		secretAccessKey: bucket.secretAccessKey,122+	});123+}124+125+/** The backups in the backup bucket, oldest first, with their sizes. */126+export async function offsiteBackups(127+	bucket: BackupBucket,128+): Promise<{ name: string; size: number }[]> {129+	const found: { name: string; size: number }[] = [];130+	const client = backupClient(bucket);131+	let continuationToken: string | undefined;132+	do {133+		const page = await client.list({ prefix: "kepr-backup-", continuationToken });134+		for (const object of page.contents ?? []) {135+			const name = OFFSITE.exec(object.key)?.[1];136+			if (name) found.push({ name, size: object.size ?? 0 });92137 		}138+		continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;139+	} while (continuationToken);140+	return found.sort((a, b) => a.name.localeCompare(b.name));141+}142+143+/** Tried before a backup bucket is saved: a test file must write, list and delete. Returns what failed. */144+export async function checkBackupBucket(bucket: BackupBucket): Promise<string | null> {145+	const client = backupClient(bucket);146+	try {147+		await client.write("kepr-backup-test", "test", { type: "text/plain" });148+		await offsiteBackups(bucket);149+		await client.delete("kepr-backup-test");150+		return null;151+	} catch (error) {152+		return `The backup bucket did not take a test file: ${error instanceof Error ? error.message : String(error)}`;153+	}154+}155+156+/**157+ * Copies the newest local backup to the backup bucket unless it is there already, then keeps the158+ * newest 14 there. A failure is kept and told once, as a local one is; null when all is well.159+ */160+export async function attemptUpload(161+	kepr: Database,162+	dataDir: string,163+	bucket: BackupBucket | undefined,164+): Promise<string | null> {165+	const newest = backups(dataDir).at(-1);166+	if (!bucket || !newest || setting(kepr, "backup_uploaded") === newest) return null;167+	try {168+		const tarball = backupTarball(dataDir, newest);169+		if (!tarball) return null;170+		const client = backupClient(bucket);171+		await client.write(offsiteKey(newest), tarball.bytes, { type: "application/gzip" });172+		for (const old of (await offsiteBackups(bucket)).slice(0, -KEEP)) {173+			await client.delete(offsiteKey(old.name));174+		}93175 		kepr94-			.query("INSERT OR REPLACE INTO settings (key, value) VALUES ('backup_error', ?)")95-			.run(message);96-		return message;176+			.query("INSERT OR REPLACE INTO settings (key, value) VALUES ('backup_uploaded', ?)")177+			.run(newest);178+		kepr.query("DELETE FROM settings WHERE key = 'backup_upload_error'").run();179+		return null;180+	} catch (error) {181+		return keepFailure(kepr, "backup_upload_error", "Backup upload failed", error);97182 	}98183 }99184 

@@ -27,12 +27,25 @@2727 	to: string;2828 };2929 30+/**31+ * A second S3 bucket that takes a copy of every backup (June, 2026-09-30). One key reads, writes32+ * and deletes there; like every key it lives here and nowhere else (§25).33+ */34+export type BackupBucket = {35+	endpoint: string;36+	name: string;37+	region: string;38+	accessKeyId: string;39+	secretAccessKey: string;40+};41+3042 export type Config = {3143 	domain: string;3244 	dataDir: string;3345 	adminPasswordHash: string;3446 	bucket: BucketSource;3547 	smtp?: SmtpSettings;48+	backup?: BackupBucket;3649 };3750 3851 export class ConfigError extends Error {}@@ -66,9 +79,22 @@6679 		adminPasswordHash,6780 		bucket: parseBucket(bucket),6881 		...(top.smtp === undefined ? {} : { smtp: parseSmtp(asTable(top.smtp, "[smtp]")) }),82+		...(top.backup === undefined ? {} : { backup: parseBackup(asTable(top.backup, "[backup]")) }),6983 	};7084 }7185 86+function parseBackup(backup: Record<string, unknown>): BackupBucket {87+	const endpoint = requireString(backup, "endpoint", "[backup] ");88+	if (!URL.canParse(endpoint)) throw new ConfigError("[backup] endpoint must be a URL");89+	return {90+		endpoint,91+		name: requireString(backup, "name", "[backup] "),92+		region: requireString(backup, "region", "[backup] "),93+		accessKeyId: requireString(backup, "access_key_id", "[backup] "),94+		secretAccessKey: requireString(backup, "secret_access_key", "[backup] "),95+	};96+}97+7298 function parseSmtp(smtp: Record<string, unknown>): SmtpSettings {7399 	const { port, security } = smtp;74100 	if (typeof port !== "number" || !Number.isInteger(port) || port < 1 || port > 65535) {@@ -136,6 +162,18 @@136162 			`password = ${str(smtp.password)}`,137163 			`from = ${str(smtp.from)}`,138164 			`to = ${str(smtp.to)}`,165+		);166+	}167+	const { backup } = config;168+	if (backup) {169+		lines.push(170+			"",171+			"[backup]",172+			`endpoint = ${str(backup.endpoint)}`,173+			`name = ${str(backup.name)}`,174+			`region = ${str(backup.region)}`,175+			`access_key_id = ${str(backup.accessKeyId)}`,176+			`secret_access_key = ${str(backup.secretAccessKey)}`,139177 		);140178 	}141179 	const text = `${lines.join("\n")}\n`;

@@ -74,7 +74,7 @@7474 		expect(text).toContain('href="/admin?section=backups"');7575 		const backupPage = await (await browser.get("/admin?section=backups")).text();7676 		expect(backupPage).toContain("<h1>backups</h1>");77-		expect(backupPage).toContain("Backups stay on this server.");77+		expect(backupPage).toContain("Sessions and bucket credentials are not included");7878 		expect(backupPage).not.toContain("Bucket reachable:");7979 	});8080 

@@ -63,7 +63,7 @@6363 		const saved = `${inst.dataDir}/backups-kept`;6464 		Bun.spawnSync(["mv", `${inst.dataDir}/backups`, saved]);6565 		writeFileSync(`${inst.dataDir}/backups`, "");66-		startBackupLoop(inst.kepr, inst.dataDir);66+		startBackupLoop(inst.kepr, inst.config);6767 		expect(setting(inst.kepr, "backup_error")).toContain("ENOTDIR");6868 		const failed = await operator.post("/admin/backup", {});6969 		expect(failed.status).toBe(500);

@@ -101,7 +101,7 @@101101 		clear();102102 		// A file where the backups folder belongs makes every backup fail.103103 		writeFileSync(`${inst.dataDir}/backups`, "");104-		startBackupLoop(inst.kepr, inst.dataDir);104+		startBackupLoop(inst.kepr, inst.config);105105 		const row = inst.kepr106106 			.query<{ text: string; href: string }, []>("SELECT text, href FROM notifications")107107 			.get();