45b702df7e26

run the sync on its own thread so landings, recounts and migrations never stall a request; console sync, holds, hidden lists and bucket settings go through it; mirror schema checks lock only to migrate, at start; snapshot reads for kept tarballs and the Atlas; bucket budget 1,170, ceiling 12,870 (v3.0.195)

time
2026-10-03 01:21 UTC · 1 week ago
machine
Junes-MacBook-Pro.local
agent
claude-opus-5-5
session
lightning quick kepr
parent
5b76ba01ba36
18 files package.json+1 −1scripts/budget.ts+3 −3src/admin/bucket.ts+2 −1src/admin/console.ts+4 −1src/admin/rename.ts+4 −3src/admin/repos.ts+5 −8src/admin/site.ts+2 −5src/admin/storage.ts+2 −2src/bucket/sync.ts+96 −1src/bucket/worker.ts+64 −0src/issues/store.ts+26 −23src/main.ts+8 −5src/mirror/store.ts+47 −21src/pages/atlas.ts+6 −0src/server/app.ts+3 −0src/server/db.ts+6 −1src/tarball/cache.ts+18 −12tests/worker.test.ts+77 −0
package.json+1 −1
@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.194",3+	"version": "3.0.195",44 	"private": true,55 	"type": "module",66 	"scripts": {

view file

scripts/budget.ts+3 −3
@@ -1,10 +1,10 @@11 // §26: fails when any area passes its budget or the TypeScript total passes the2-// 12,710-line ceiling.3-const CEILING = 12_710;2+// 12,870-line ceiling.3+const CEILING = 12_870;44 55 // Prefixes are matched in order, so the specific files come before the directories.66 const TS_AREAS: [string, string[], number][] = [7-	["bucket", ["src/bucket/"], 1010],7+	["bucket", ["src/bucket/"], 1170],88 	["mirror", ["src/mirror/"], 1100],99 	["describe", ["src/describe/"], 500],1010 	["pages", ["src/pages/"], 2600],

view file

src/admin/bucket.ts+2 −1
@@ -114,9 +114,10 @@114114 			const failed = `The settings work, but kepr.toml could not be written: ${reason}`;115115 			return c.html(bucketPage(c, ctx, { fields, error: failed }), 500);116116 		}117-		// In place, so the sync loop, downloads, the image policy and /connect all see it now.117+		// In place, so the sync, downloads, the image policy and /connect all see it now.118118 		ctx.config.bucket = next;119119 		Object.assign(ctx.sync.bucket, openBucket(next));120+		ctx.syncer.setBucket(next);120121 		return c.redirect("/admin/bucket?saved=1", 303);121122 	});122123 }

view file

src/admin/console.ts+4 −1
@@ -1,5 +1,5 @@11 import type { Context, Hono } from "hono";2-import type { SyncTarget } from "../bucket/sync.ts";2+import type { Syncer, SyncTarget } from "../bucket/sync.ts";33 import { operatorIssues } from "../issues/store.ts";44 import { type Html, html } from "../pages/html.ts";55 import { type PageContext, page, plural, relativeTime } from "../pages/layout.ts";@@ -12,7 +12,10 @@1212 import { allRepos } from "./store.ts";1313 1414 export type AdminContext = PageContext & {15+	/** The bucket, kepr.db and mirrors the console reads and writes itself. */1516 	sync: SyncTarget;17+	/** The sync, which may run on its own thread. */18+	syncer: Syncer;1619 	dataDir: string;1720 	config: Config;1821 	configPath: string;

view file

src/admin/rename.ts+4 −3
@@ -1,7 +1,7 @@11 import { existsSync, renameSync } from "node:fs";22 import { join } from "node:path";33 import { type Bucket, inTurn, KEPR_PREFIX } from "../bucket/reader.ts";4-import { holdSync, type SyncTarget } from "../bucket/sync.ts";4+import type { SyncTarget } from "../bucket/sync.ts";55 import { closeMirror } from "../mirror/store.ts";66 import { RESERVED_NAMES } from "../server/reserved.ts";77 import { dropTarballs } from "../tarball/cache.ts";@@ -44,10 +44,11 @@4444  * old keys go. A machine offering to the old name meanwhile loses nothing: its saves are its own,4545  * and its next offer follows the marker.4646  */47-export function renameRepo(sync: SyncTarget, from: string, to: string): void {47+export function renameRepo(ctx: AdminContext, from: string, to: string): void {4848 	renames.set(from, to);4949 	renameErrors.delete(from);50-	holdSync(() => move(sync, from, to))50+	ctx.syncer51+		.hold(() => move(ctx.sync, from, to))5152 		.catch((error) => renameErrors.set(from, error instanceof Error ? error.message : `${error}`))5253 		.finally(() => renames.delete(from));5354 }

view file

src/admin/repos.ts+5 −8
@@ -1,11 +1,10 @@11 import type { Context, Hono } from "hono";22 import { BucketWriteError } from "../bucket/reader.ts";33 import { isSafePath } from "../bucket/repo.ts";4-import { syncAll } from "../bucket/sync.ts";54 import { MAX_CHARS } from "../issues/spam.ts";65 import { operatorIssues } from "../issues/store.ts";76 import { languageLines, repoStats } from "../mirror/queries.ts";8-import { applyHidden, openMirror, quarantinedPaths } from "../mirror/store.ts";7+import { openMirror, quarantinedPaths } from "../mirror/store.ts";98 import { html } from "../pages/html.ts";109 import { formatDate, plural } from "../pages/layout.ts";1110 import {@@ -17,7 +16,7 @@1716 	setRelease,1817 	visibleReleases,1918 } from "../releases/releases.ts";20-import { hiddenPaths, type RepoText, repoText, setRepoText } from "../server/db.ts";19+import { type RepoText, repoText, setRepoText } from "../server/db.ts";2120 import { dropTarballs } from "../tarball/cache.ts";2221 import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";2322 import { renameErrors, renameProblem, renameRepo, renames } from "./rename.ts";@@ -48,7 +47,7 @@4847 	});4948 5049 	app.post("/admin/sync", async (c) => {51-		await syncAll(ctx.sync);50+		await ctx.syncer.syncAll();5251 		return c.redirect("/admin/repos", 303);5352 	});5453 @@ -101,9 +100,7 @@101100 		updateRepo(ctx.kepr, repo.name, edit);102101 		for (const [field, value] of texts) setRepoText(ctx.kepr, repo.name, field, value);103102 		// The sync hands the list to Koh (`handHidden`); the mirror has it before the response.104-		if (edit.hidden_paths !== repo.hidden_paths) {105-			applyHidden(openMirror(ctx.mirrorDir, repo.name), hiddenPaths(ctx.kepr, repo.name));106-		}103+		if (edit.hidden_paths !== repo.hidden_paths) await ctx.syncer.applyHidden([repo.name]);107104 		return c.redirect(`/admin/repos?repo=${repo.name}&tab=settings&saved=1`, 303);108105 	});109106 @@ -136,7 +133,7 @@136133 		if (problem) {137134 			return c.html(reposPage(c, ctx, `${repo.name}: ${problem}`, repo.name, "settings"), 400);138135 		}139-		renameRepo(ctx.sync, repo.name, to);136+		renameRepo(ctx, repo.name, to);140137 		return c.redirect(`/admin/repos?repo=${repo.name}&tab=settings`, 303);141138 	});142139 

view file

src/admin/site.ts+2 −5
@@ -3,10 +3,9 @@33 import { join } from "node:path";44 import type { Context, Hono } from "hono";55 import { isSafePath } from "../bucket/repo.ts";6-import { applyHidden, openMirror } from "../mirror/store.ts";76 import { supportText } from "../pages/docs.ts";87 import { html } from "../pages/html.ts";9-import { hiddenPaths, ignoredNames, setting } from "../server/db.ts";8+import { ignoredNames, setting } from "../server/db.ts";109 import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";1110 import { allRepos, saveSettings } from "./store.ts";1211 @@ -117,9 +116,7 @@117116 			rmSync(join(assets, previous), { force: true });118117 		}119118 		if (values.ignored !== undefined && values.ignored !== previousIgnored.join("\n")) {120-			for (const { name } of allRepos(ctx.kepr)) {121-				applyHidden(openMirror(ctx.mirrorDir, name), hiddenPaths(ctx.kepr, name));122-			}119+			await ctx.syncer.applyHidden(allRepos(ctx.kepr).map((repo) => repo.name));123120 		}124121 		return c.redirect(`/admin/site?${section === "profile" ? "section=profile&" : ""}saved=1`, 303);125122 	});

view file

src/admin/storage.ts+2 −2
@@ -1,7 +1,7 @@11 import type { Context, Hono } from "hono";22 import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";33 import { listRepos } from "../bucket/repo.ts";4-import { holdSync, type SyncTarget } from "../bucket/sync.ts";4+import type { SyncTarget } from "../bucket/sync.ts";55 import { tarballGeneration } from "../mirror/queries.ts";66 import { openMirror } from "../mirror/store.ts";77 import { html } from "../pages/html.ts";@@ -223,7 +223,7 @@223223 		const fields = await textFields(c);224224 		// While no sync or rename runs: a rename can make a listed orphan live again mid-delete.225225 		let outcome = "";226-		await holdSync(() =>226+		await ctx.syncer.hold(() =>227227 			inStorageTurn(async () => {228228 				outcome = await deleteKeys(ctx, fields);229229 			}),

view file

src/bucket/sync.ts+96 −1
@@ -11,17 +11,20 @@1111 	type StoredObject,1212 	storeObjects,1313 } from "../mirror/store.ts";14+import type { BucketSource } from "../server/config.ts";1415 import {1516 	hiddenPaths,1617 	hiddenToHand,18+	isRegistered,1719 	keepState,1820 	recordHiddenHanded,1921 	recordSync,2022 	registerRepo,23+	renamedTo,2124 } from "../server/db.ts";2225 import { RESERVED_NAMES } from "../server/reserved.ts";2326 import { dropTarballs } from "../tarball/cache.ts";24-import { type Bucket, BucketWriteError, inTurn, KEPR_PREFIX } from "./reader.ts";27+import { type Bucket, BucketWriteError, inTurn, KEPR_PREFIX, openBucket } from "./reader.ts";2528 import {2629 	BucketError,2730 	checkFormat,@@ -80,6 +83,98 @@8083 	const slot = mine.catch(() => {});8184 	running = slot;8285 	return mine;86+}87+88+/**89+ * What the app asks of the sync: one now, syncs held off while some work runs, a changed hidden90+ * list applied, new bucket settings followed. Mirrors have one writer, the sync, so hidden lists go91+ * through it too.92+ */93+export type Syncer = {94+	syncAll(): Promise<void>;95+	hold(work: () => Promise<void>): Promise<void>;96+	applyHidden(repos: string[]): Promise<void>;97+	setBucket(source: BucketSource): void;98+};99+100+/** The sync in this thread, as the worker and the tests run it. */101+export function localSyncer(target: SyncTarget): Syncer {102+	return {103+		syncAll: () => syncAll(target),104+		hold: holdSync,105+		applyHidden: async (repos) => {106+			for (const asked of repos) {107+				// A rename may have moved the repo while this waited on its hold.108+				const repo = renamedTo(target.kepr, asked) ?? asked;109+				if (!isRegistered(target.kepr, repo)) continue;110+				applyHidden(openMirror(target.mirrorDir, repo), hiddenPaths(target.kepr, repo));111+			}112+		},113+		setBucket: (source) => Object.assign(target.bucket, openBucket(source)),114+	};115+}116+117+/** A request to the sync's worker (`worker.ts`), and its reply. */118+export type SyncRequest =119+	| { kind: "start"; dataDir: string; source: BucketSource }120+	| { kind: "sync" | "hold" | "release"; id: number }121+	| { kind: "hidden"; id: number; repos: string[] }122+	| { kind: "bucket"; source: BucketSource };123+export type SyncReply =124+	| { kind: "done"; id: number; error?: string }125+	| { kind: "synced"; repo: string };126+127+/**128+ * The sync on its own thread (2026-10-02), so a landing, a recount or a migration never stalls a129+ * request. `synced` hears of each repo that synced cleanly. A worker that dies takes Kepr down with130+ * it, for the service manager to restart, rather than leaving requests waiting on it.131+ */132+export function startSyncWorker(133+	dataDir: string,134+	source: BucketSource,135+	synced: (repo: string) => void,136+): Syncer {137+	const worker = new Worker(new URL("./worker.ts", import.meta.url));138+	// The server keeps Kepr running; the worker alone never should.139+	worker.unref();140+	const waiting = new Map<number, { resolve: () => void; reject: (error: Error) => void }>();141+	let next = 0;142+	worker.addEventListener("error", (event) => {143+		console.error(`kepr: the sync worker failed: ${event.message}`);144+		process.exit(1);145+	});146+	worker.addEventListener("message", ({ data }: MessageEvent<SyncReply>) => {147+		if (data.kind === "synced") return synced(data.repo);148+		const reply = waiting.get(data.id);149+		waiting.delete(data.id);150+		if (data.error === undefined) reply?.resolve();151+		else reply?.reject(new Error(data.error));152+	});153+	const send = (request: SyncRequest) => worker.postMessage(request);154+	const ask = (request: (id: number) => SyncRequest) =>155+		new Promise<number>((resolve, reject) => {156+			const id = next++;157+			waiting.set(id, { resolve: () => resolve(id), reject });158+			send(request(id));159+		});160+	send({ kind: "start", dataDir, source });161+	return {162+		syncAll: async () => {163+			await ask((id) => ({ kind: "sync", id }));164+		},165+		async hold(work) {166+			const id = await ask((id) => ({ kind: "hold", id }));167+			try {168+				await work();169+			} finally {170+				send({ kind: "release", id });171+			}172+		},173+		applyHidden: async (repos) => {174+			await ask((id) => ({ kind: "hidden", id, repos }));175+		},176+		setBucket: (source) => send({ kind: "bucket", source }),177+	};83178 }84179 85180 export function startSyncLoop(target: SyncTarget): void {

view file

src/bucket/worker.ts+64 −0
@@ -0,0 +1,64 @@1+import { join } from "node:path";2+import { closeMirrors } from "../mirror/store.ts";3+import { openKeprDb } from "../server/db.ts";4+import { openBucket } from "./reader.ts";5+import {6+	localSyncer,7+	type Syncer,8+	type SyncReply,9+	type SyncRequest,10+	startSyncLoop,11+} from "./sync.ts";12+13+// The sync's own thread: started by `startSyncWorker`, it runs the loop and answers its requests.14+15+declare const self: Worker;16+17+const reply = (message: SyncReply) => self.postMessage(message);18+const settle = (id: number, work: Promise<void>) =>19+	work.then(20+		() => reply({ kind: "done", id }),21+		(error) => reply({ kind: "done", id, error: String(error) }),22+	);23+const released = new Map<number, () => void>();24+// Settles when the hold in force, if any, is released: a rename moves mirror files under it.25+let unheld = Promise.resolve();26+let syncer: Syncer;27+28+self.addEventListener("message", ({ data }: MessageEvent<SyncRequest>) => {29+	switch (data.kind) {30+		case "start": {31+			const target = {32+				bucket: openBucket(data.source),33+				kepr: openKeprDb(data.dataDir),34+				mirrorDir: join(data.dataDir, "mirror"),35+			};36+			syncer = localSyncer(target);37+			startSyncLoop({ ...target, synced: (repo) => reply({ kind: "synced", repo }) });38+			return;39+		}40+		case "sync":41+			return void settle(data.id, syncer.syncAll());42+		case "hold":43+			// Answered once no sync runs; mirrors are closed so a rename can move their files.44+			return void syncer.hold(() => {45+				const held = new Promise<void>((release) => released.set(data.id, release));46+				unheld = held;47+				closeMirrors();48+				reply({ kind: "done", id: data.id });49+				return held;50+			});51+		case "release":52+			released.get(data.id)?.();53+			released.delete(data.id);54+			return;55+		case "hidden":56+			// Not behind a sync under way, which can take minutes; only behind a rename's hold.57+			return void settle(58+				data.id,59+				unheld.then(() => syncer.applyHidden(data.repos)),60+			);61+		case "bucket":62+			return syncer.setBucket(data.source);63+	}64+});

view file

src/issues/store.ts+26 −23
@@ -49,34 +49,37 @@4949 /** Files an issue under the repo's next number and returns it with its follow-up token (§23). */5050 export function fileIssue(db: Database, issue: NewIssue): { number: number; token: string } {5151 	const token = randomBytes(32).toString("hex");52-	const number = db.transaction(() => {53-		const row = db54-			.query<{ next: number }, [string]>(55-				`SELECT COALESCE(MAX(number), 0) + 1 AS next52+	// Immediate: it reads the next number before writing, and the sync's thread writes kepr.db too.53+	const number = db54+		.transaction(() => {55+			const row = db56+				.query<{ next: number }, [string]>(57+					`SELECT COALESCE(MAX(number), 0) + 1 AS next5658 FROM issues5759 WHERE repo = ?`,58-			)59-			.get(issue.repo) as { next: number };60-		db.query(61-			`INSERT INTO issues (60+				)61+				.get(issue.repo) as { next: number };62+			db.query(63+				`INSERT INTO issues (6264 repo, number, title, body, type, version, reporter_name, contact_email,6365 filed_at, ip_hash, token_hash6466 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,65-		).run(66-			issue.repo,67-			row.next,68-			issue.title,69-			issue.body,70-			issue.type,71-			issue.version,72-			issue.reporterName,73-			issue.contactEmail,74-			issue.filedAt,75-			issue.ipHash,76-			hashToken(token),77-		);78-		return row.next;79-	})();67+			).run(68+				issue.repo,69+				row.next,70+				issue.title,71+				issue.body,72+				issue.type,73+				issue.version,74+				issue.reporterName,75+				issue.contactEmail,76+				issue.filedAt,77+				issue.ipHash,78+				hashToken(token),79+			);80+			return row.next;81+		})82+		.immediate();8083 	return { number, token };8184 }8285 

view file

src/main.ts+8 −5
@@ -3,7 +3,8 @@33 import { createSetupApp } from "./admin/setup.ts";44 import { startStorageLoop } from "./admin/storage.ts";55 import { openBucket } from "./bucket/reader.ts";6-import { startSyncLoop } from "./bucket/sync.ts";6+import { startSyncWorker } from "./bucket/sync.ts";7+import { openMirrors } from "./mirror/store.ts";78 import { warmAtlas } from "./pages/atlas.ts";89 import { createApp } from "./server/app.ts";910 import { startBackupLoop } from "./server/backup.ts";@@ -23,11 +24,13 @@2324 	startBackupLoop(kepr, config);2425 	startMailLoop(kepr, config);2526 	const mirrorDir = join(config.dataDir, "mirror");27+	openMirrors(mirrorDir);2628 	// The first visitor after a new save would otherwise wait a second or two on a big repo.27-	const sync = { bucket, kepr, mirrorDir };28-	startSyncLoop({ ...sync, synced: (repo) => warmAtlas({ kepr, mirrorDir }, repo) });29-	startStorageLoop(sync);30-	return createApp(config, kepr, bucket).fetch;29+	const syncer = startSyncWorker(config.dataDir, config.bucket, (repo) =>30+		warmAtlas({ kepr, mirrorDir }, repo),31+	);32+	startStorageLoop({ bucket, kepr, mirrorDir });33+	return createApp(config, kepr, bucket, CONFIG_FILE, syncer).fetch;3134 }3235 3336 try {

view file

src/mirror/store.ts+47 −21
@@ -1,5 +1,5 @@11 import { Database } from "bun:sqlite";2-import { mkdirSync } from "node:fs";2+import { existsSync, mkdirSync, readdirSync } from "node:fs";33 import { join } from "node:path";44 import { inflateSync } from "node:zlib";55 import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";@@ -129,6 +129,21 @@129129 	open.delete(path);130130 }131131 132+/**133+ * Opens every mirror in `dir`, bringing each to this Kepr's schema: run at start, before Kepr134+ * listens, so no request waits on a migration.135+ */136+export function openMirrors(dir: string): void {137+	if (!existsSync(dir)) return;138+	for (const name of readdirSync(dir)) if (name.endsWith(".db")) openMirror(dir, name.slice(0, -3));139+}140+141+/** Closes every mirror this thread holds open; the next use of each opens it again. */142+export function closeMirrors(): void {143+	for (const db of open.values()) db.close();144+	open.clear();145+}146+132147 /** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */133148 export function openMirror(dir: string, repo: string): Database {134149 	const path = join(dir, `${repo}.db`);@@ -139,36 +154,45 @@139154 	db.run("PRAGMA journal_mode = WAL");140155 	db.run("PRAGMA foreign_keys = ON");141156 	db.run("PRAGMA busy_timeout = 5000");142-	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {143-		user_version: 0,144-	};145-	if (user_version !== 0 && user_version !== 2 && user_version !== SCHEMA_VERSION) {157+	// A current mirror needs no lock; only creating or migrating one takes the write lock.158+	const found = schemaVersion(db) === SCHEMA_VERSION ? SCHEMA_VERSION : upgrade(db);159+	if (found !== 0 && found !== 2 && found !== SCHEMA_VERSION) {146160 		db.close();147161 		throw new Error(148-			`${path} has mirror schema version ${user_version}; this Kepr reads version ${SCHEMA_VERSION}. ` +162+			`${path} has mirror schema version ${found}; this Kepr reads version ${SCHEMA_VERSION}. ` +149163 				"The mirror is disposable: delete it and the next sync rebuilds it from the bucket.",150164 		);151165 	}152-	if (user_version === 0) {153-		db.transaction(() => {154-			db.run(SCHEMA);155-			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);156-		})();157-	}158-	if (user_version === 2) {159-		db.transaction(() => {160-			db.run(HIDDEN_FILES);161-			// Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.162-			if (mirrorState(db, "hidden")) clearMirror(db);163-			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);164-		})();165-	}166166 	// Mirrors counted under an older rule are recounted now, not at the next save.167167 	if (mirrorState(db, "head") && mirrorState(db, "stats_rule") !== STATS_RULE) refreshStats(db);168168 	open.set(path, db);169169 	return db;170170 }171171 172+function schemaVersion(db: Database): number {173+	return db.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version ?? 0;174+}175+176+/**177+ * Creates or migrates a mirror and returns the version it found, read again under the write lock:178+ * the request thread and the sync's both open mirrors, and must not both create or migrate one.179+ */180+function upgrade(db: Database): number {181+	return db182+		.transaction(() => {183+			const found = schemaVersion(db);184+			if (found === 0) db.run(SCHEMA);185+			if (found === 2) {186+				db.run(HIDDEN_FILES);187+				// Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.188+				if (mirrorState(db, "hidden")) clearMirror(db);189+			}190+			if (found === 0 || found === 2) db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);191+			return found;192+		})193+		.immediate();194+}195+172196 export function mirrorState(db: Database, key: string): string | null {173197 	const row = db174198 		.query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")@@ -233,6 +257,8 @@233257 	);234258 	const putRelease = db.query("INSERT OR IGNORE INTO releases (version, record) VALUES (?, ?)");235259 	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");260+	// Immediate: a landing with nothing new reads before it writes, and the request thread writes261+	// mirrors too (a deleted release).236262 	db.transaction(() => {237263 		storeObjects(db, landing.objects);238264 		for (const e of landing.events) {@@ -265,7 +291,7 @@265291 		setState.run("releases_latest", landing.releasesLatest);266292 		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);267293 		setState.run("quarantined", JSON.stringify([...quarantined].sort()));268-	})();294+	}).immediate();269295 	// Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.270296 	refreshStats(db);271297 }

view file

src/pages/atlas.ts+6 −0
@@ -88,7 +88,13 @@8888 const MODELS = 16;8989 const models = new Map<string, AtlasModel>();9090 91+// In one read snapshot: the sync's thread can apply a hidden list between any two statements, and92+// a model kept under the old generation must never hold a file the new one hides.9193 function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {94+	return repo.mirror.transaction(() => buildAtlasModel(repo, saveId))();95+}96+97+function buildAtlasModel(repo: PublicRepo, saveId: string): AtlasModel {9298 	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;9399 	const cached = models.get(key);94100 	if (cached) return cached;

view file

src/server/app.ts+3 −0
@@ -17,6 +17,7 @@1717 import { registerRepoText } from "../admin/text.ts";1818 import { type Bucket, openBucket } from "../bucket/reader.ts";1919 import { isSafePath } from "../bucket/repo.ts";20+import { localSyncer, type Syncer } from "../bucket/sync.ts";2021 import { registerIssuePages } from "../issues/pages.ts";2122 import { registerKohRemote } from "../mirror/remote.ts";2223 import { registerArchivesPage } from "../pages/archives.ts";@@ -72,6 +73,7 @@7273 	kepr: Database,7374 	bucket: Bucket = openBucket(config.bucket),7475 	configPath = CONFIG_FILE,76+	syncer?: Syncer,7577 ): Hono {7678 	const app = new Hono();7779 	// §25 allows images from the presigned host, which is the bucket endpoint's origin; uploaded@@ -152,6 +154,7 @@152154 		configPath,153155 		dataDir: config.dataDir,154156 		sync: { bucket, kepr, mirrorDir },157+		syncer: syncer ?? localSyncer({ bucket, kepr, mirrorDir }),155158 	};156159 	registerStatus(app, admin);157160 	registerRepos(app, admin);

view file

src/server/db.ts+6 −1
@@ -225,6 +225,10 @@225225 	return to;226226 }227227 228+export function isRegistered(db: Database, name: string): boolean {229+	return db.query("SELECT 1 FROM repos WHERE name = ?").get(name) !== null;230+}231+228232 /** Every repo starts private (§20); registering a known repo changes nothing. */229233 export function registerRepo(db: Database, name: string): void {230234 	db.query("INSERT OR IGNORE INTO repos (name) VALUES (?)").run(name);@@ -236,6 +240,7 @@236240  */237241 export function notify(db: Database, kind: NotificationKind, text: string, href: string): void {238242 	const now = Math.floor(Date.now() / 1000);243+	// Immediate: it reads before writing, and the sync's thread writes kepr.db too.239244 	db.transaction(() => {240245 		if (!setting(db, `notify_off_${kind}`)) {241246 			db.query("INSERT INTO notifications (kind, text, href, created_at) VALUES (?, ?, ?, ?)").run(@@ -257,7 +262,7 @@257262 				WHERE id NOT IN (SELECT id FROM ${table} ORDER BY id DESC LIMIT ?)`,258263 			).run(NOTIFICATIONS_KEPT);259264 		}260-	})();265+	}).immediate();261266 }262267 263268 /** Records a sync's outcome; a repo that starts failing tells the operator, once. */

view file

src/tarball/cache.ts+18 −12
@@ -22,18 +22,24 @@2222 		const generation = tarballGeneration(db);2323 		const key = `${tarballsOf(repo)}${generation}/${saveId}.tar.gz`;2424 		const stored = await bucket.get(key);25-		// A strip, a hidden-path change or a rebuild may have landed during any await: bytes are26-		// served only for the generation still current, and the build below runs in one synchronous27-		// go, so what it reads matches the key it files under.28-		if (tarballGeneration(db) !== generation) continue;29-		if (stored) return stored;30-		if (!knownSave(db, saveId)) return null;31-		const files = filesAt(db, saveId).map((file) => {32-			const bytes = objectBytes(db, file.hash);33-			if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);34-			return { path: file.path, bytes, executable: file.executable };35-		});36-		const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);25+		// A strip, a hidden-path change or a rebuild may land during any await, and the sync's thread26+		// can land one between any two statements: the generation and the files are read in one27+		// snapshot, so what is built matches the key it files under, and bytes are served only for28+		// the generation still current.29+		const read = db.transaction(() => {30+			if (tarballGeneration(db) !== generation) return "moved";31+			if (stored) return "stored";32+			if (!knownSave(db, saveId)) return "unknown";33+			return filesAt(db, saveId).map((file) => {34+				const bytes = objectBytes(db, file.hash);35+				if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);36+				return { path: file.path, bytes, executable: file.executable };37+			});38+		})();39+		if (read === "moved") continue;40+		if (read === "stored") return stored;41+		if (read === "unknown") return null;42+		const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, read);3743 		try {3844 			await bucket.put(key, tarball, "application/gzip");3945 		} catch (error) {

view file

tests/worker.test.ts+77 −0
@@ -0,0 +1,77 @@1+// Covers the sync on its own thread (2026-10-02): a real worker syncs a koh3-made bucket, tells the2+// app which repos synced, holds syncs off while the app works, and applies hidden lists.3+4+import { expect, test } from "bun:test";5+import { existsSync, renameSync } from "node:fs";6+import { mkdtemp } from "node:fs/promises";7+import { tmpdir } from "node:os";8+import { join } from "node:path";9+import { startSyncWorker } from "../src/bucket/sync.ts";10+import { fileAt, headId } from "../src/mirror/queries.ts";11+import { closeMirror, openMirror } from "../src/mirror/store.ts";12+import { openKeprDb } from "../src/server/db.ts";13+import { makeKohBucket } from "./fixture.ts";14+15+test("the sync worker syncs, reports, holds and hides", async () => {16+	const fx = await makeKohBucket("worked");17+	const dataDir = await mkdtemp(join(tmpdir(), "kepr-worker-"));18+	const kepr = openKeprDb(dataDir);19+	const synced: string[] = [];20+	const syncer = startSyncWorker(dataDir, { kind: "folder", path: fx.bucketDir }, (repo) =>21+		synced.push(repo),22+	);23+	await syncer.syncAll();24+	const mirror = () => openMirror(join(dataDir, "mirror"), fx.repo);25+	const head = fx.saveIds[0] as string;26+	expect(headId(mirror())).toBe(head);27+	for (let i = 0; i < 100 && !synced.includes(fx.repo); i++) await Bun.sleep(10);28+	expect(synced).toContain(fx.repo);29+30+	const order: string[] = [];31+	let release = () => {};32+	const held = syncer.hold(async () => {33+		order.push("held");34+		await new Promise<void>((resolve) => {35+			release = resolve;36+		});37+		order.push("released");38+	});39+	for (let i = 0; i < 100 && order.length === 0; i++) await Bun.sleep(10);40+	const synced2 = syncer.syncAll().then(() => order.push("synced"));41+	await Bun.sleep(50);42+	expect(order).toEqual(["held"]);43+	release();44+	await Promise.all([held, synced2]);45+	expect(order).toEqual(["held", "released", "synced"]);46+47+	kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = ?").run(fx.repo);48+	await syncer.applyHidden([fx.repo]);49+	expect(fileAt(mirror(), head, "secret.txt")).toBeNull();50+	expect(fileAt(mirror(), head, "src/main.ts")).not.toBeNull();51+});52+53+test("a hidden list edited during a rename reaches the repo under its new name", async () => {54+	const fx = await makeKohBucket("before");55+	const dataDir = await mkdtemp(join(tmpdir(), "kepr-worker-"));56+	const mirrorDir = join(dataDir, "mirror");57+	const kepr = openKeprDb(dataDir);58+	const syncer = startSyncWorker(dataDir, { kind: "folder", path: fx.bucketDir }, () => {});59+	await syncer.syncAll();60+	const head = fx.saveIds[0] as string;61+	let edited: Promise<void> = Promise.resolve();62+	await syncer.hold(async () => {63+		kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = 'before'").run();64+		edited = syncer.applyHidden(["before"]);65+		// What a rename does under its hold: rows and mirror move to the new name.66+		kepr.query("UPDATE repos SET name = 'after' WHERE name = 'before'").run();67+		kepr.query("INSERT INTO renamed (old, new) VALUES ('before', 'after')").run();68+		closeMirror(mirrorDir, "before");69+		for (const suffix of ["", "-wal", "-shm"]) {70+			const path = join(mirrorDir, `before.db${suffix}`);71+			if (existsSync(path)) renameSync(path, join(mirrorDir, `after.db${suffix}`));72+		}73+	});74+	await edited;75+	expect(fileAt(openMirror(mirrorDir, "after"), head, "secret.txt")).toBeNull();76+	expect(existsSync(join(mirrorDir, "before.db"))).toBe(false);77+});

view file