45b702df7e26
run the sync on its own thread so landings, recounts and migrations never stall a request; console sync, holds, hidden lists and bucket settings go through it; mirror schema checks lock only to migrate, at start; snapshot reads for kept tarballs and the Atlas; bucket budget 1,170, ceiling 12,870 (v3.0.195)
- time
- 2026-10-03 01:21 UTC · 1 week ago
- machine
- Junes-MacBook-Pro.local
- agent
- claude-opus-5-5
- session
- lightning quick kepr
- parent
- 5b76ba01ba36
↓ kepr-45b702df.tar.gzatlas at this save
files changed18 files
package.json+1 −1scripts/budget.ts+3 −3src/admin/bucket.ts+2 −1src/admin/console.ts+4 −1src/admin/rename.ts+4 −3src/admin/repos.ts+5 −8src/admin/site.ts+2 −5src/admin/storage.ts+2 −2src/bucket/sync.ts+96 −1src/bucket/worker.ts+64 −0src/issues/store.ts+26 −23src/main.ts+8 −5src/mirror/store.ts+47 −21src/pages/atlas.ts+6 −0src/server/app.ts+3 −0src/server/db.ts+6 −1src/tarball/cache.ts+18 −12tests/worker.test.ts+77 −0package.json+1 −1
@@ -1,6 +1,6 @@11 {22 "name": "kepr",3- "version": "3.0.194",3+ "version": "3.0.195",44 "private": true,55 "type": "module",66 "scripts": {scripts/budget.ts+3 −3
@@ -1,10 +1,10 @@11 // §26: fails when any area passes its budget or the TypeScript total passes the2-// 12,710-line ceiling.3-const CEILING = 12_710;2+// 12,870-line ceiling.3+const CEILING = 12_870;44 55 // Prefixes are matched in order, so the specific files come before the directories.66 const TS_AREAS: [string, string[], number][] = [7- ["bucket", ["src/bucket/"], 1010],7+ ["bucket", ["src/bucket/"], 1170],88 ["mirror", ["src/mirror/"], 1100],99 ["describe", ["src/describe/"], 500],1010 ["pages", ["src/pages/"], 2600],src/admin/bucket.ts+2 −1
@@ -114,9 +114,10 @@114114 const failed = `The settings work, but kepr.toml could not be written: ${reason}`;115115 return c.html(bucketPage(c, ctx, { fields, error: failed }), 500);116116 }117- // In place, so the sync loop, downloads, the image policy and /connect all see it now.117+ // In place, so the sync, downloads, the image policy and /connect all see it now.118118 ctx.config.bucket = next;119119 Object.assign(ctx.sync.bucket, openBucket(next));120+ ctx.syncer.setBucket(next);120121 return c.redirect("/admin/bucket?saved=1", 303);121122 });122123 }src/admin/console.ts+4 −1
@@ -1,5 +1,5 @@11 import type { Context, Hono } from "hono";2-import type { SyncTarget } from "../bucket/sync.ts";2+import type { Syncer, SyncTarget } from "../bucket/sync.ts";33 import { operatorIssues } from "../issues/store.ts";44 import { type Html, html } from "../pages/html.ts";55 import { type PageContext, page, plural, relativeTime } from "../pages/layout.ts";@@ -12,7 +12,10 @@1212 import { allRepos } from "./store.ts";1313 1414 export type AdminContext = PageContext & {15+ /** The bucket, kepr.db and mirrors the console reads and writes itself. */1516 sync: SyncTarget;17+ /** The sync, which may run on its own thread. */18+ syncer: Syncer;1619 dataDir: string;1720 config: Config;1821 configPath: string;src/admin/rename.ts+4 −3
@@ -1,7 +1,7 @@11 import { existsSync, renameSync } from "node:fs";22 import { join } from "node:path";33 import { type Bucket, inTurn, KEPR_PREFIX } from "../bucket/reader.ts";4-import { holdSync, type SyncTarget } from "../bucket/sync.ts";4+import type { SyncTarget } from "../bucket/sync.ts";55 import { closeMirror } from "../mirror/store.ts";66 import { RESERVED_NAMES } from "../server/reserved.ts";77 import { dropTarballs } from "../tarball/cache.ts";@@ -44,10 +44,11 @@4444 * old keys go. A machine offering to the old name meanwhile loses nothing: its saves are its own,4545 * and its next offer follows the marker.4646 */47-export function renameRepo(sync: SyncTarget, from: string, to: string): void {47+export function renameRepo(ctx: AdminContext, from: string, to: string): void {4848 renames.set(from, to);4949 renameErrors.delete(from);50- holdSync(() => move(sync, from, to))50+ ctx.syncer51+ .hold(() => move(ctx.sync, from, to))5152 .catch((error) => renameErrors.set(from, error instanceof Error ? error.message : `${error}`))5253 .finally(() => renames.delete(from));5354 }src/admin/repos.ts+5 −8
@@ -1,11 +1,10 @@11 import type { Context, Hono } from "hono";22 import { BucketWriteError } from "../bucket/reader.ts";33 import { isSafePath } from "../bucket/repo.ts";4-import { syncAll } from "../bucket/sync.ts";54 import { MAX_CHARS } from "../issues/spam.ts";65 import { operatorIssues } from "../issues/store.ts";76 import { languageLines, repoStats } from "../mirror/queries.ts";8-import { applyHidden, openMirror, quarantinedPaths } from "../mirror/store.ts";7+import { openMirror, quarantinedPaths } from "../mirror/store.ts";98 import { html } from "../pages/html.ts";109 import { formatDate, plural } from "../pages/layout.ts";1110 import {@@ -17,7 +16,7 @@1716 setRelease,1817 visibleReleases,1918 } from "../releases/releases.ts";20-import { hiddenPaths, type RepoText, repoText, setRepoText } from "../server/db.ts";19+import { type RepoText, repoText, setRepoText } from "../server/db.ts";2120 import { dropTarballs } from "../tarball/cache.ts";2221 import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";2322 import { renameErrors, renameProblem, renameRepo, renames } from "./rename.ts";@@ -48,7 +47,7 @@4847 });4948 5049 app.post("/admin/sync", async (c) => {51- await syncAll(ctx.sync);50+ await ctx.syncer.syncAll();5251 return c.redirect("/admin/repos", 303);5352 });5453 @@ -101,9 +100,7 @@101100 updateRepo(ctx.kepr, repo.name, edit);102101 for (const [field, value] of texts) setRepoText(ctx.kepr, repo.name, field, value);103102 // The sync hands the list to Koh (`handHidden`); the mirror has it before the response.104- if (edit.hidden_paths !== repo.hidden_paths) {105- applyHidden(openMirror(ctx.mirrorDir, repo.name), hiddenPaths(ctx.kepr, repo.name));106- }103+ if (edit.hidden_paths !== repo.hidden_paths) await ctx.syncer.applyHidden([repo.name]);107104 return c.redirect(`/admin/repos?repo=${repo.name}&tab=settings&saved=1`, 303);108105 });109106 @@ -136,7 +133,7 @@136133 if (problem) {137134 return c.html(reposPage(c, ctx, `${repo.name}: ${problem}`, repo.name, "settings"), 400);138135 }139- renameRepo(ctx.sync, repo.name, to);136+ renameRepo(ctx, repo.name, to);140137 return c.redirect(`/admin/repos?repo=${repo.name}&tab=settings`, 303);141138 });142139 src/admin/site.ts+2 −5
@@ -3,10 +3,9 @@33 import { join } from "node:path";44 import type { Context, Hono } from "hono";55 import { isSafePath } from "../bucket/repo.ts";6-import { applyHidden, openMirror } from "../mirror/store.ts";76 import { supportText } from "../pages/docs.ts";87 import { html } from "../pages/html.ts";9-import { hiddenPaths, ignoredNames, setting } from "../server/db.ts";8+import { ignoredNames, setting } from "../server/db.ts";109 import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";1110 import { allRepos, saveSettings } from "./store.ts";1211 @@ -117,9 +116,7 @@117116 rmSync(join(assets, previous), { force: true });118117 }119118 if (values.ignored !== undefined && values.ignored !== previousIgnored.join("\n")) {120- for (const { name } of allRepos(ctx.kepr)) {121- applyHidden(openMirror(ctx.mirrorDir, name), hiddenPaths(ctx.kepr, name));122- }119+ await ctx.syncer.applyHidden(allRepos(ctx.kepr).map((repo) => repo.name));123120 }124121 return c.redirect(`/admin/site?${section === "profile" ? "section=profile&" : ""}saved=1`, 303);125122 });src/admin/storage.ts+2 −2
@@ -1,7 +1,7 @@11 import type { Context, Hono } from "hono";22 import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";33 import { listRepos } from "../bucket/repo.ts";4-import { holdSync, type SyncTarget } from "../bucket/sync.ts";4+import type { SyncTarget } from "../bucket/sync.ts";55 import { tarballGeneration } from "../mirror/queries.ts";66 import { openMirror } from "../mirror/store.ts";77 import { html } from "../pages/html.ts";@@ -223,7 +223,7 @@223223 const fields = await textFields(c);224224 // While no sync or rename runs: a rename can make a listed orphan live again mid-delete.225225 let outcome = "";226- await holdSync(() =>226+ await ctx.syncer.hold(() =>227227 inStorageTurn(async () => {228228 outcome = await deleteKeys(ctx, fields);229229 }),src/bucket/sync.ts+96 −1
@@ -11,17 +11,20 @@1111 type StoredObject,1212 storeObjects,1313 } from "../mirror/store.ts";14+import type { BucketSource } from "../server/config.ts";1415 import {1516 hiddenPaths,1617 hiddenToHand,18+ isRegistered,1719 keepState,1820 recordHiddenHanded,1921 recordSync,2022 registerRepo,23+ renamedTo,2124 } from "../server/db.ts";2225 import { RESERVED_NAMES } from "../server/reserved.ts";2326 import { dropTarballs } from "../tarball/cache.ts";24-import { type Bucket, BucketWriteError, inTurn, KEPR_PREFIX } from "./reader.ts";27+import { type Bucket, BucketWriteError, inTurn, KEPR_PREFIX, openBucket } from "./reader.ts";2528 import {2629 BucketError,2730 checkFormat,@@ -80,6 +83,98 @@8083 const slot = mine.catch(() => {});8184 running = slot;8285 return mine;86+}87+88+/**89+ * What the app asks of the sync: one now, syncs held off while some work runs, a changed hidden90+ * list applied, new bucket settings followed. Mirrors have one writer, the sync, so hidden lists go91+ * through it too.92+ */93+export type Syncer = {94+ syncAll(): Promise<void>;95+ hold(work: () => Promise<void>): Promise<void>;96+ applyHidden(repos: string[]): Promise<void>;97+ setBucket(source: BucketSource): void;98+};99+100+/** The sync in this thread, as the worker and the tests run it. */101+export function localSyncer(target: SyncTarget): Syncer {102+ return {103+ syncAll: () => syncAll(target),104+ hold: holdSync,105+ applyHidden: async (repos) => {106+ for (const asked of repos) {107+ // A rename may have moved the repo while this waited on its hold.108+ const repo = renamedTo(target.kepr, asked) ?? asked;109+ if (!isRegistered(target.kepr, repo)) continue;110+ applyHidden(openMirror(target.mirrorDir, repo), hiddenPaths(target.kepr, repo));111+ }112+ },113+ setBucket: (source) => Object.assign(target.bucket, openBucket(source)),114+ };115+}116+117+/** A request to the sync's worker (`worker.ts`), and its reply. */118+export type SyncRequest =119+ | { kind: "start"; dataDir: string; source: BucketSource }120+ | { kind: "sync" | "hold" | "release"; id: number }121+ | { kind: "hidden"; id: number; repos: string[] }122+ | { kind: "bucket"; source: BucketSource };123+export type SyncReply =124+ | { kind: "done"; id: number; error?: string }125+ | { kind: "synced"; repo: string };126+127+/**128+ * The sync on its own thread (2026-10-02), so a landing, a recount or a migration never stalls a129+ * request. `synced` hears of each repo that synced cleanly. A worker that dies takes Kepr down with130+ * it, for the service manager to restart, rather than leaving requests waiting on it.131+ */132+export function startSyncWorker(133+ dataDir: string,134+ source: BucketSource,135+ synced: (repo: string) => void,136+): Syncer {137+ const worker = new Worker(new URL("./worker.ts", import.meta.url));138+ // The server keeps Kepr running; the worker alone never should.139+ worker.unref();140+ const waiting = new Map<number, { resolve: () => void; reject: (error: Error) => void }>();141+ let next = 0;142+ worker.addEventListener("error", (event) => {143+ console.error(`kepr: the sync worker failed: ${event.message}`);144+ process.exit(1);145+ });146+ worker.addEventListener("message", ({ data }: MessageEvent<SyncReply>) => {147+ if (data.kind === "synced") return synced(data.repo);148+ const reply = waiting.get(data.id);149+ waiting.delete(data.id);150+ if (data.error === undefined) reply?.resolve();151+ else reply?.reject(new Error(data.error));152+ });153+ const send = (request: SyncRequest) => worker.postMessage(request);154+ const ask = (request: (id: number) => SyncRequest) =>155+ new Promise<number>((resolve, reject) => {156+ const id = next++;157+ waiting.set(id, { resolve: () => resolve(id), reject });158+ send(request(id));159+ });160+ send({ kind: "start", dataDir, source });161+ return {162+ syncAll: async () => {163+ await ask((id) => ({ kind: "sync", id }));164+ },165+ async hold(work) {166+ const id = await ask((id) => ({ kind: "hold", id }));167+ try {168+ await work();169+ } finally {170+ send({ kind: "release", id });171+ }172+ },173+ applyHidden: async (repos) => {174+ await ask((id) => ({ kind: "hidden", id, repos }));175+ },176+ setBucket: (source) => send({ kind: "bucket", source }),177+ };83178 }84179 85180 export function startSyncLoop(target: SyncTarget): void {src/bucket/worker.ts+64 −0
@@ -0,0 +1,64 @@1+import { join } from "node:path";2+import { closeMirrors } from "../mirror/store.ts";3+import { openKeprDb } from "../server/db.ts";4+import { openBucket } from "./reader.ts";5+import {6+ localSyncer,7+ type Syncer,8+ type SyncReply,9+ type SyncRequest,10+ startSyncLoop,11+} from "./sync.ts";12+13+// The sync's own thread: started by `startSyncWorker`, it runs the loop and answers its requests.14+15+declare const self: Worker;16+17+const reply = (message: SyncReply) => self.postMessage(message);18+const settle = (id: number, work: Promise<void>) =>19+ work.then(20+ () => reply({ kind: "done", id }),21+ (error) => reply({ kind: "done", id, error: String(error) }),22+ );23+const released = new Map<number, () => void>();24+// Settles when the hold in force, if any, is released: a rename moves mirror files under it.25+let unheld = Promise.resolve();26+let syncer: Syncer;27+28+self.addEventListener("message", ({ data }: MessageEvent<SyncRequest>) => {29+ switch (data.kind) {30+ case "start": {31+ const target = {32+ bucket: openBucket(data.source),33+ kepr: openKeprDb(data.dataDir),34+ mirrorDir: join(data.dataDir, "mirror"),35+ };36+ syncer = localSyncer(target);37+ startSyncLoop({ ...target, synced: (repo) => reply({ kind: "synced", repo }) });38+ return;39+ }40+ case "sync":41+ return void settle(data.id, syncer.syncAll());42+ case "hold":43+ // Answered once no sync runs; mirrors are closed so a rename can move their files.44+ return void syncer.hold(() => {45+ const held = new Promise<void>((release) => released.set(data.id, release));46+ unheld = held;47+ closeMirrors();48+ reply({ kind: "done", id: data.id });49+ return held;50+ });51+ case "release":52+ released.get(data.id)?.();53+ released.delete(data.id);54+ return;55+ case "hidden":56+ // Not behind a sync under way, which can take minutes; only behind a rename's hold.57+ return void settle(58+ data.id,59+ unheld.then(() => syncer.applyHidden(data.repos)),60+ );61+ case "bucket":62+ return syncer.setBucket(data.source);63+ }64+});src/issues/store.ts+26 −23
@@ -49,34 +49,37 @@4949 /** Files an issue under the repo's next number and returns it with its follow-up token (§23). */5050 export function fileIssue(db: Database, issue: NewIssue): { number: number; token: string } {5151 const token = randomBytes(32).toString("hex");52- const number = db.transaction(() => {53- const row = db54- .query<{ next: number }, [string]>(55- `SELECT COALESCE(MAX(number), 0) + 1 AS next52+ // Immediate: it reads the next number before writing, and the sync's thread writes kepr.db too.53+ const number = db54+ .transaction(() => {55+ const row = db56+ .query<{ next: number }, [string]>(57+ `SELECT COALESCE(MAX(number), 0) + 1 AS next5658 FROM issues5759 WHERE repo = ?`,58- )59- .get(issue.repo) as { next: number };60- db.query(61- `INSERT INTO issues (60+ )61+ .get(issue.repo) as { next: number };62+ db.query(63+ `INSERT INTO issues (6264 repo, number, title, body, type, version, reporter_name, contact_email,6365 filed_at, ip_hash, token_hash6466 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,65- ).run(66- issue.repo,67- row.next,68- issue.title,69- issue.body,70- issue.type,71- issue.version,72- issue.reporterName,73- issue.contactEmail,74- issue.filedAt,75- issue.ipHash,76- hashToken(token),77- );78- return row.next;79- })();67+ ).run(68+ issue.repo,69+ row.next,70+ issue.title,71+ issue.body,72+ issue.type,73+ issue.version,74+ issue.reporterName,75+ issue.contactEmail,76+ issue.filedAt,77+ issue.ipHash,78+ hashToken(token),79+ );80+ return row.next;81+ })82+ .immediate();8083 return { number, token };8184 }8285 src/main.ts+8 −5
@@ -3,7 +3,8 @@33 import { createSetupApp } from "./admin/setup.ts";44 import { startStorageLoop } from "./admin/storage.ts";55 import { openBucket } from "./bucket/reader.ts";6-import { startSyncLoop } from "./bucket/sync.ts";6+import { startSyncWorker } from "./bucket/sync.ts";7+import { openMirrors } from "./mirror/store.ts";78 import { warmAtlas } from "./pages/atlas.ts";89 import { createApp } from "./server/app.ts";910 import { startBackupLoop } from "./server/backup.ts";@@ -23,11 +24,13 @@2324 startBackupLoop(kepr, config);2425 startMailLoop(kepr, config);2526 const mirrorDir = join(config.dataDir, "mirror");27+ openMirrors(mirrorDir);2628 // The first visitor after a new save would otherwise wait a second or two on a big repo.27- const sync = { bucket, kepr, mirrorDir };28- startSyncLoop({ ...sync, synced: (repo) => warmAtlas({ kepr, mirrorDir }, repo) });29- startStorageLoop(sync);30- return createApp(config, kepr, bucket).fetch;29+ const syncer = startSyncWorker(config.dataDir, config.bucket, (repo) =>30+ warmAtlas({ kepr, mirrorDir }, repo),31+ );32+ startStorageLoop({ bucket, kepr, mirrorDir });33+ return createApp(config, kepr, bucket, CONFIG_FILE, syncer).fetch;3134 }3235 3336 try {src/mirror/store.ts+47 −21
@@ -1,5 +1,5 @@11 import { Database } from "bun:sqlite";2-import { mkdirSync } from "node:fs";2+import { existsSync, mkdirSync, readdirSync } from "node:fs";33 import { join } from "node:path";44 import { inflateSync } from "node:zlib";55 import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";@@ -129,6 +129,21 @@129129 open.delete(path);130130 }131131 132+/**133+ * Opens every mirror in `dir`, bringing each to this Kepr's schema: run at start, before Kepr134+ * listens, so no request waits on a migration.135+ */136+export function openMirrors(dir: string): void {137+ if (!existsSync(dir)) return;138+ for (const name of readdirSync(dir)) if (name.endsWith(".db")) openMirror(dir, name.slice(0, -3));139+}140+141+/** Closes every mirror this thread holds open; the next use of each opens it again. */142+export function closeMirrors(): void {143+ for (const db of open.values()) db.close();144+ open.clear();145+}146+132147 /** The mirror of one repo at `<dir>/<repo>.db`, created on first use and kept open. */133148 export function openMirror(dir: string, repo: string): Database {134149 const path = join(dir, `${repo}.db`);@@ -139,36 +154,45 @@139154 db.run("PRAGMA journal_mode = WAL");140155 db.run("PRAGMA foreign_keys = ON");141156 db.run("PRAGMA busy_timeout = 5000");142- const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {143- user_version: 0,144- };145- if (user_version !== 0 && user_version !== 2 && user_version !== SCHEMA_VERSION) {157+ // A current mirror needs no lock; only creating or migrating one takes the write lock.158+ const found = schemaVersion(db) === SCHEMA_VERSION ? SCHEMA_VERSION : upgrade(db);159+ if (found !== 0 && found !== 2 && found !== SCHEMA_VERSION) {146160 db.close();147161 throw new Error(148- `${path} has mirror schema version ${user_version}; this Kepr reads version ${SCHEMA_VERSION}. ` +162+ `${path} has mirror schema version ${found}; this Kepr reads version ${SCHEMA_VERSION}. ` +149163 "The mirror is disposable: delete it and the next sync rebuilds it from the bucket.",150164 );151165 }152- if (user_version === 0) {153- db.transaction(() => {154- db.run(SCHEMA);155- db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);156- })();157- }158- if (user_version === 2) {159- db.transaction(() => {160- db.run(HIDDEN_FILES);161- // Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.162- if (mirrorState(db, "hidden")) clearMirror(db);163- db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);164- })();165- }166166 // Mirrors counted under an older rule are recounted now, not at the next save.167167 if (mirrorState(db, "head") && mirrorState(db, "stats_rule") !== STATS_RULE) refreshStats(db);168168 open.set(path, db);169169 return db;170170 }171171 172+function schemaVersion(db: Database): number {173+ return db.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version ?? 0;174+}175+176+/**177+ * Creates or migrates a mirror and returns the version it found, read again under the write lock:178+ * the request thread and the sync's both open mirrors, and must not both create or migrate one.179+ */180+function upgrade(db: Database): number {181+ return db182+ .transaction(() => {183+ const found = schemaVersion(db);184+ if (found === 0) db.run(SCHEMA);185+ if (found === 2) {186+ db.run(HIDDEN_FILES);187+ // Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.188+ if (mirrorState(db, "hidden")) clearMirror(db);189+ }190+ if (found === 0 || found === 2) db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);191+ return found;192+ })193+ .immediate();194+}195+172196 export function mirrorState(db: Database, key: string): string | null {173197 const row = db174198 .query<{ value: string }, [string]>("SELECT value FROM state WHERE key = ?")@@ -233,6 +257,8 @@233257 );234258 const putRelease = db.query("INSERT OR IGNORE INTO releases (version, record) VALUES (?, ?)");235259 const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");260+ // Immediate: a landing with nothing new reads before it writes, and the request thread writes261+ // mirrors too (a deleted release).236262 db.transaction(() => {237263 storeObjects(db, landing.objects);238264 for (const e of landing.events) {@@ -265,7 +291,7 @@265291 setState.run("releases_latest", landing.releasesLatest);266292 const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);267293 setState.run("quarantined", JSON.stringify([...quarantined].sort()));268- })();294+ }).immediate();269295 // Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.270296 refreshStats(db);271297 }src/pages/atlas.ts+6 −0
@@ -88,7 +88,13 @@8888 const MODELS = 16;8989 const models = new Map<string, AtlasModel>();9090 91+// In one read snapshot: the sync's thread can apply a hidden list between any two statements, and92+// a model kept under the old generation must never hold a file the new one hides.9193 function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {94+ return repo.mirror.transaction(() => buildAtlasModel(repo, saveId))();95+}96+97+function buildAtlasModel(repo: PublicRepo, saveId: string): AtlasModel {9298 const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;9399 const cached = models.get(key);94100 if (cached) return cached;src/server/app.ts+3 −0
@@ -17,6 +17,7 @@1717 import { registerRepoText } from "../admin/text.ts";1818 import { type Bucket, openBucket } from "../bucket/reader.ts";1919 import { isSafePath } from "../bucket/repo.ts";20+import { localSyncer, type Syncer } from "../bucket/sync.ts";2021 import { registerIssuePages } from "../issues/pages.ts";2122 import { registerKohRemote } from "../mirror/remote.ts";2223 import { registerArchivesPage } from "../pages/archives.ts";@@ -72,6 +73,7 @@7273 kepr: Database,7374 bucket: Bucket = openBucket(config.bucket),7475 configPath = CONFIG_FILE,76+ syncer?: Syncer,7577 ): Hono {7678 const app = new Hono();7779 // §25 allows images from the presigned host, which is the bucket endpoint's origin; uploaded@@ -152,6 +154,7 @@152154 configPath,153155 dataDir: config.dataDir,154156 sync: { bucket, kepr, mirrorDir },157+ syncer: syncer ?? localSyncer({ bucket, kepr, mirrorDir }),155158 };156159 registerStatus(app, admin);157160 registerRepos(app, admin);src/server/db.ts+6 −1
@@ -225,6 +225,10 @@225225 return to;226226 }227227 228+export function isRegistered(db: Database, name: string): boolean {229+ return db.query("SELECT 1 FROM repos WHERE name = ?").get(name) !== null;230+}231+228232 /** Every repo starts private (§20); registering a known repo changes nothing. */229233 export function registerRepo(db: Database, name: string): void {230234 db.query("INSERT OR IGNORE INTO repos (name) VALUES (?)").run(name);@@ -236,6 +240,7 @@236240 */237241 export function notify(db: Database, kind: NotificationKind, text: string, href: string): void {238242 const now = Math.floor(Date.now() / 1000);243+ // Immediate: it reads before writing, and the sync's thread writes kepr.db too.239244 db.transaction(() => {240245 if (!setting(db, `notify_off_${kind}`)) {241246 db.query("INSERT INTO notifications (kind, text, href, created_at) VALUES (?, ?, ?, ?)").run(@@ -257,7 +262,7 @@257262 WHERE id NOT IN (SELECT id FROM ${table} ORDER BY id DESC LIMIT ?)`,258263 ).run(NOTIFICATIONS_KEPT);259264 }260- })();265+ }).immediate();261266 }262267 263268 /** Records a sync's outcome; a repo that starts failing tells the operator, once. */src/tarball/cache.ts+18 −12
@@ -22,18 +22,24 @@2222 const generation = tarballGeneration(db);2323 const key = `${tarballsOf(repo)}${generation}/${saveId}.tar.gz`;2424 const stored = await bucket.get(key);25- // A strip, a hidden-path change or a rebuild may have landed during any await: bytes are26- // served only for the generation still current, and the build below runs in one synchronous27- // go, so what it reads matches the key it files under.28- if (tarballGeneration(db) !== generation) continue;29- if (stored) return stored;30- if (!knownSave(db, saveId)) return null;31- const files = filesAt(db, saveId).map((file) => {32- const bytes = objectBytes(db, file.hash);33- if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);34- return { path: file.path, bytes, executable: file.executable };35- });36- const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, files);25+ // A strip, a hidden-path change or a rebuild may land during any await, and the sync's thread26+ // can land one between any two statements: the generation and the files are read in one27+ // snapshot, so what is built matches the key it files under, and bytes are served only for28+ // the generation still current.29+ const read = db.transaction(() => {30+ if (tarballGeneration(db) !== generation) return "moved";31+ if (stored) return "stored";32+ if (!knownSave(db, saveId)) return "unknown";33+ return filesAt(db, saveId).map((file) => {34+ const bytes = objectBytes(db, file.hash);35+ if (!bytes) throw new Error(`${repo}: object ${file.hash} is missing from the mirror`);36+ return { path: file.path, bytes, executable: file.executable };37+ });38+ })();39+ if (read === "moved") continue;40+ if (read === "stored") return stored;41+ if (read === "unknown") return null;42+ const tarball = buildTarball(`${repo}-${saveId.slice(0, 8)}`, read);3743 try {3844 await bucket.put(key, tarball, "application/gzip");3945 } catch (error) {tests/worker.test.ts+77 −0
@@ -0,0 +1,77 @@1+// Covers the sync on its own thread (2026-10-02): a real worker syncs a koh3-made bucket, tells the2+// app which repos synced, holds syncs off while the app works, and applies hidden lists.3+4+import { expect, test } from "bun:test";5+import { existsSync, renameSync } from "node:fs";6+import { mkdtemp } from "node:fs/promises";7+import { tmpdir } from "node:os";8+import { join } from "node:path";9+import { startSyncWorker } from "../src/bucket/sync.ts";10+import { fileAt, headId } from "../src/mirror/queries.ts";11+import { closeMirror, openMirror } from "../src/mirror/store.ts";12+import { openKeprDb } from "../src/server/db.ts";13+import { makeKohBucket } from "./fixture.ts";14+15+test("the sync worker syncs, reports, holds and hides", async () => {16+ const fx = await makeKohBucket("worked");17+ const dataDir = await mkdtemp(join(tmpdir(), "kepr-worker-"));18+ const kepr = openKeprDb(dataDir);19+ const synced: string[] = [];20+ const syncer = startSyncWorker(dataDir, { kind: "folder", path: fx.bucketDir }, (repo) =>21+ synced.push(repo),22+ );23+ await syncer.syncAll();24+ const mirror = () => openMirror(join(dataDir, "mirror"), fx.repo);25+ const head = fx.saveIds[0] as string;26+ expect(headId(mirror())).toBe(head);27+ for (let i = 0; i < 100 && !synced.includes(fx.repo); i++) await Bun.sleep(10);28+ expect(synced).toContain(fx.repo);29+30+ const order: string[] = [];31+ let release = () => {};32+ const held = syncer.hold(async () => {33+ order.push("held");34+ await new Promise<void>((resolve) => {35+ release = resolve;36+ });37+ order.push("released");38+ });39+ for (let i = 0; i < 100 && order.length === 0; i++) await Bun.sleep(10);40+ const synced2 = syncer.syncAll().then(() => order.push("synced"));41+ await Bun.sleep(50);42+ expect(order).toEqual(["held"]);43+ release();44+ await Promise.all([held, synced2]);45+ expect(order).toEqual(["held", "released", "synced"]);46+47+ kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = ?").run(fx.repo);48+ await syncer.applyHidden([fx.repo]);49+ expect(fileAt(mirror(), head, "secret.txt")).toBeNull();50+ expect(fileAt(mirror(), head, "src/main.ts")).not.toBeNull();51+});52+53+test("a hidden list edited during a rename reaches the repo under its new name", async () => {54+ const fx = await makeKohBucket("before");55+ const dataDir = await mkdtemp(join(tmpdir(), "kepr-worker-"));56+ const mirrorDir = join(dataDir, "mirror");57+ const kepr = openKeprDb(dataDir);58+ const syncer = startSyncWorker(dataDir, { kind: "folder", path: fx.bucketDir }, () => {});59+ await syncer.syncAll();60+ const head = fx.saveIds[0] as string;61+ let edited: Promise<void> = Promise.resolve();62+ await syncer.hold(async () => {63+ kepr.query("UPDATE repos SET hidden_paths = 'secret.txt' WHERE name = 'before'").run();64+ edited = syncer.applyHidden(["before"]);65+ // What a rename does under its hold: rows and mirror move to the new name.66+ kepr.query("UPDATE repos SET name = 'after' WHERE name = 'before'").run();67+ kepr.query("INSERT INTO renamed (old, new) VALUES ('before', 'after')").run();68+ closeMirror(mirrorDir, "before");69+ for (const suffix of ["", "-wal", "-shm"]) {70+ const path = join(mirrorDir, `before.db${suffix}`);71+ if (existsSync(path)) renameSync(path, join(mirrorDir, `after.db${suffix}`));72+ }73+ });74+ await edited;75+ expect(fileAt(openMirror(mirrorDir, "after"), head, "secret.txt")).toBeNull();76+ expect(existsSync(join(mirrorDir, "before.db"))).toBe(false);77+});