import type { Database } from "bun:sqlite"; import { knownEventIds, knownObject, knownSave, land, mirrorState, openMirror, type StoredObject, } from "../mirror/store.ts"; import { recordSync, registerRepo } from "../server/db.ts"; import { dropTarballs } from "../tarball/cache.ts"; import type { BucketReader } from "./reader.ts"; import { BucketError, checkFormat, type EventRecord, listEventIds, listRepos, readEvent, readHead, readObject, readSave, type SaveRecord, } from "./repo.ts"; const INTERVAL_MS = 2 * 60 * 1000; export type SyncTarget = { bucket: BucketReader; kepr: Database; mirrorDir: string; tarballDir: string; }; let running: Promise | null = null; /** Syncs every repo in the bucket; the admin console's "sync now" calls this too. Never overlaps itself. */ export function syncAll(target: SyncTarget): Promise { running ??= syncEach(target).finally(() => { running = null; }); return running; } export function startSyncLoop(target: SyncTarget): void { const tick = () => syncAll(target).catch((error) => console.error(`kepr: sync failed: ${error}`)); tick(); setInterval(tick, INTERVAL_MS); } async function syncEach(target: SyncTarget): Promise { for (const repo of await listRepos(target.bucket)) { registerRepo(target.kepr, repo); let error: string | null = null; try { const events = await syncRepo(target.bucket, openMirror(target.mirrorDir, repo), repo); // §22.3: after the strip has landed in the mirror, so no rebuild can bring the old bytes back. for (const event of events) { if (event.kind === "strip") dropTarballs(target.tarballDir, repo, event.save_id); } } catch (cause) { error = cause instanceof Error ? cause.message : String(cause); } recordSync(target.kepr, repo, Math.floor(Date.now() / 1000), error); } } /** * Koh's steal flow (§9.4) against a mirror: walk back from the remote head to a save the mirror has, * fetch what is missing, and land it in one transaction. Events are listed every time, since a * flag or a strip arrives without moving `head`. Returns the events it landed. */ export async function syncRepo( bucket: BucketReader, mirror: Database, repo: string, ): Promise { await checkFormat(bucket, repo); const head = await readHead(bucket, repo); if (head === null) return []; const saves: SaveRecord[] = []; const objects = new Map(); const seen = new Set(); for (let cursor = head; cursor !== "" && !knownSave(mirror, cursor); ) { if (seen.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`); seen.add(cursor); const save = await readSave(bucket, repo, cursor); for (const file of save.files) { if (objects.has(file.hash) || knownObject(mirror, file.hash)) continue; const { compressed, bytes } = await readObject(bucket, repo, file.hash); objects.set(file.hash, { compressed, size: bytes.length }); } saves.push(save); cursor = save.parent; } const known = knownEventIds(mirror); const events: EventRecord[] = []; for (const id of await listEventIds(bucket, repo)) { if (!known.has(id)) events.push(await readEvent(bucket, repo, id)); } if (saves.length === 0 && events.length === 0 && mirrorState(mirror, "head") === head) return []; land(mirror, { head, saves: saves.reverse(), objects, events }); return events; }