work out the Atlas's touches and file facts and each repo's known objects as saves land (mirror schema 4), so pages read instead of compute and no Atlas or facts cache, warm-up or synced message remains (v3.0.196)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.195",3+	"version": "3.0.196",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -60,8 +60,6 @@6060 	bucket: Bucket;6161 	kepr: Database;6262 	mirrorDir: string;63-	/** Called after each repo that synced without error. */64-	synced?: (repo: string) => void;6563 };6664 6765 let running: Promise<void> | null = null;@@ -120,20 +118,14 @@120118 	| { kind: "sync" | "hold" | "release"; id: number }121119 	| { kind: "hidden"; id: number; repos: string[] }122120 	| { kind: "bucket"; source: BucketSource };123-export type SyncReply =124-	| { kind: "done"; id: number; error?: string }125-	| { kind: "synced"; repo: string };121+export type SyncReply = { kind: "done"; id: number; error?: string };126122 127123 /**128124  * The sync on its own thread (2026-10-02), so a landing, a recount or a migration never stalls a129- * request. `synced` hears of each repo that synced cleanly. A worker that dies takes Kepr down with125+ * request. A worker that dies takes Kepr down with130126  * it, for the service manager to restart, rather than leaving requests waiting on it.131127  */132-export function startSyncWorker(133-	dataDir: string,134-	source: BucketSource,135-	synced: (repo: string) => void,136-): Syncer {128+export function startSyncWorker(dataDir: string, source: BucketSource): Syncer {137129 	const worker = new Worker(new URL("./worker.ts", import.meta.url));138130 	// The server keeps Kepr running; the worker alone never should.139131 	worker.unref();@@ -144,7 +136,6 @@144136 		process.exit(1);145137 	});146138 	worker.addEventListener("message", ({ data }: MessageEvent<SyncReply>) => {147-		if (data.kind === "synced") return synced(data.repo);148139 		const reply = waiting.get(data.id);149140 		waiting.delete(data.id);150141 		if (data.error === undefined) reply?.resolve();@@ -210,7 +201,6 @@210201 			}211202 		}212203 		recordSync(target.kepr, repo, Math.floor(Date.now() / 1000), error);213-		if (error === null) target.synced?.(repo);214204 		await handHidden(target, repo);215205 	}216206 }

@@ -34,7 +34,7 @@3434 				mirrorDir: join(data.dataDir, "mirror"),3535 			};3636 			syncer = localSyncer(target);37-			startSyncLoop({ ...target, synced: (repo) => reply({ kind: "synced", repo }) });37+			startSyncLoop(target);3838 			return;3939 		}4040 		case "sync":

@@ -2,6 +2,53 @@22 // Every function returns Markdown source (hard-wrapped lines kept) or null.33 44 import MarkdownIt from "markdown-it";5+import { type Outline, outline } from "./links.ts";6+7+/** A file page shows a file's first megabyte, and that megabyte decides whether it is text. */8+export const SHOWN_BYTES = 1024 * 1024;9+10+// What a file says about itself is worked out only up to this size; a larger file is data.11+const MAX_DESCRIBED = 8 * 1024 * 1024;12+13+/** What a file says about itself, as the Atlas shows it; all null for binary or very large files. */14+export type FileFacts = {15+	lines: number | null;16+	description: string | null;17+	outline: Outline | null;18+};19+20+/** `read` is called only for a file small enough to describe, so a large one is never inflated. */21+export function fileFacts(path: string, size: number, read: () => Uint8Array): FileFacts {22+	const bytes = size > MAX_DESCRIBED ? null : read();23+	if (!bytes || !isText(bytes)) return { lines: null, description: null, outline: null };24+	const text = new TextDecoder().decode(bytes);25+	return {26+		lines: countLines(text),27+		description: describeFile(path, text),28+		outline: outline(path, text),29+	};30+}31+32+export function isText(bytes: Uint8Array): boolean {33+	if (bytes.subarray(0, 8000).includes(0)) return false;34+	try {35+		// Streaming mode tolerates only a character cut at the 1 MB mark; invalid bytes still throw.36+		const cut = bytes.length > SHOWN_BYTES;37+		new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, SHOWN_BYTES), {38+			stream: cut,39+		});40+		return true;41+	} catch {42+		return false;43+	}44+}45+46+function countLines(text: string): number {47+	if (text === "") return 0;48+	let count = 0;49+	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;50+	return text.endsWith("\n") ? count : count + 1;51+}552 653 type Style = "bang" | "slash" | "jvm" | "hash" | "python" | "markdown";754 

@@ -5,7 +5,6 @@55 import { openBucket } from "./bucket/reader.ts";66 import { startSyncWorker } from "./bucket/sync.ts";77 import { openMirrors } from "./mirror/store.ts";8-import { warmAtlas } from "./pages/atlas.ts";98 import { createApp } from "./server/app.ts";109 import { startBackupLoop } from "./server/backup.ts";1110 import { CONFIG_FILE, type Config, ConfigError, loadConfig } from "./server/config.ts";@@ -26,9 +25,7 @@2625 	const mirrorDir = join(config.dataDir, "mirror");2726 	openMirrors(mirrorDir);2827 	// The first visitor after a new save would otherwise wait a second or two on a big repo.29-	const syncer = startSyncWorker(config.dataDir, config.bucket, (repo) =>30-		warmAtlas({ kepr, mirrorDir }, repo),31-	);28+	const syncer = startSyncWorker(config.dataDir, config.bucket);3229 	startStorageLoop({ bucket, kepr, mirrorDir });3330 	return createApp(config, kepr, bucket, CONFIG_FILE, syncer).fetch;3431 }

@@ -2,6 +2,8 @@22 import { createHash } from "node:crypto";33 import { inflateSync } from "node:zlib";44 import type { ReleaseRecord } from "../bucket/repo.ts";5+import type { FileFacts } from "../describe/describe.ts";6+import type { Outline } from "../describe/links.ts";57 import { mirrorState } from "./store.ts";68 79 export type Save = {@@ -25,8 +27,6 @@2527 2628 const SAVE_COLUMNS = `id, COALESCE(parent, '') AS parent, message, created_at, machine,2729 	COALESCE(agent, '') AS agent, COALESCE(session, '') AS session, is_checkpoint = 1 AS is_checkpoint`;28-const LINE_COLUMNS =29-	"line.id, line.parent, line.message, line.created_at, line.machine, line.agent, line.session, line.is_checkpoint, line.n";3030 const ID_PREFIX = /^[0-9a-f]{6,12}$/;3131 3232 // Checkpoints are working marks, not history (§21.4): every list and count below leaves them out.@@ -148,61 +148,87 @@148148 }149149 150150 /**151- * Which saves touched each path on the line up to `saveId` (§21.5's detail panel). A save touched a path when152- * the path was added, changed or removed against the save before it; checkpoints are skipped, so a153- * change first recorded by a checkpoint counts at the next real save.151+ * Which saves touched each path on the line up to `saveId` (§21.5's detail panel), as recorded when152+ * each save landed (`touches` in the mirror).154153  */155154 export function fileTouches(db: Database, saveId: string): Touches {156-	const rows = db157-		.query<Save & { path: string; n: number }, { at: string }>(158-			`WITH line AS (159-				SELECT ${SAVE_COLUMNS}, LAG(id) OVER (ORDER BY rowid) AS prev, rowid AS n FROM saves160-				WHERE ${NOT_CHECKPOINT} AND rowid <= (SELECT rowid FROM saves WHERE id = $at)155+	const at = { at: saveId };156+	// The saves once, the touches as bare pairs: every touch row carrying its save's columns was157+	// most of the Atlas's cost.158+	const byRowid = new Map(159+		db160+			.query<Save & { n: number }, { at: string }>(161+				`SELECT rowid AS n, ${SAVE_COLUMNS}162+				FROM saves163+				WHERE ${NOT_CHECKPOINT}164+				AND rowid <= (SELECT rowid FROM saves WHERE id = $at)`,161165 			)162-			SELECT ${LINE_COLUMNS}, f.path FROM line JOIN save_files f ON f.save_id = line.id163-			LEFT JOIN save_files p ON p.save_id = line.prev AND p.path = f.path164-			WHERE p.hash IS NOT f.hash OR p.executable IS NOT f.executable165-			UNION ALL166-			SELECT ${LINE_COLUMNS}, p.path FROM line JOIN save_files p ON p.save_id = line.prev167-			WHERE NOT EXISTS (SELECT 1 FROM save_files f WHERE f.save_id = line.id AND f.path = p.path)166+			.all(at)167+			.map(({ n, ...row }) => [n, asSave(row)]),168+	);169+	const touched = db170+		.query<{ path: string; n: number }, { at: string }>(171+			`SELECT path, n172+			FROM touches173+			WHERE n <= (SELECT rowid FROM saves WHERE id = $at)168174 			ORDER BY n`,169175 		)170-		.all({ at: saveId });171-	const saves = new Map<string, Save>();176+		.all(at);177+	const line = new Set<Save>();172178 	const byPath = new Map<string, Save[]>();173-	for (const { path, n: _, ...row } of rows) {174-		const save = saves.get(row.id) ?? asSave(row);175-		saves.set(save.id, save);176-		const touched = byPath.get(path);177-		if (touched) touched.push(save);179+	for (const { path, n } of touched) {180+		const save = byRowid.get(n) as Save;181+		line.add(save);182+		const saves = byPath.get(path);183+		if (saves) saves.push(save);178184 		else byPath.set(path, [save]);179185 	}180-	return { line: [...saves.values()], byPath };186+	return { line: [...line], byPath };181187 }182188 183-/** Lines per language at the newest real save, most first, vendored code left out as in the header's language. */184-export function languageLines(db: Database): [string, number][] {185-	return JSON.parse(mirrorState(db, "languages") ?? "[]") as [string, number][];189+/** The files at a save with what each says about itself, as recorded when it landed. */190+export function filesWithFacts(db: Database, saveId: string): (FileEntry & FileFacts)[] {191+	return db192+		.query<193+			FileEntry & { lines: number | null; description: string | null; outline: string | null },194+			[string]195+		>(196+			`SELECT f.path, f.hash, f.executable = 1 AS executable, o.size, x.lines, x.description, x.outline197+			FROM save_files f198+			JOIN objects o ON o.hash = f.hash199+			LEFT JOIN facts x ON x.hash = f.hash AND x.path = f.path200+			WHERE f.save_id = ?201+			ORDER BY f.path`,202+		)203+		.all(saveId)204+		.map((row) => ({205+			...row,206+			executable: Boolean(row.executable),207+			outline: row.outline === null ? null : (JSON.parse(row.outline) as Outline),208+		}));186209 }187210 188-/** Every distinct object (unique file contents) a real save refers to, at any point in history. */189-export function objectHashes(db: Database): string[] {190-	// From `objects`, probing save_files_hash: every save lists its whole tree, so a DISTINCT over191-	// save_files reads saves × files rows (1.4 s for 1,083 saves on Tylee).211+/** The file contents this repo's real saves refer to, as recounted when its saves land. */212+export function knownObjects(db: Database): string[] {192213 	return db193-		.query<{ hash: string }, []>(194-			`SELECT o.hash195-			 FROM objects o196-			 WHERE EXISTS (197-				SELECT 1198-				FROM save_files f199-				JOIN saves s ON s.id = f.save_id200-				WHERE f.hash = o.hash201-				AND s.${NOT_CHECKPOINT}202-			 )`,203-		)214+		.query<{ hash: string }, []>("SELECT hash FROM known_objects")204215 		.all()205216 		.map((row) => row.hash);217+}218+219+/** The newest real saves, newest first. */220+export function newestSaves(db: Database, limit: number): Save[] {221+	return db222+		.query<Save, [number]>(223+			`SELECT ${SAVE_COLUMNS} FROM saves WHERE ${NOT_CHECKPOINT} ORDER BY rowid DESC LIMIT ?`,224+		)225+		.all(limit)226+		.map(asSave);227+}228+229+/** Lines per language at the newest real save, most first, vendored code left out as in the header's language. */230+export function languageLines(db: Database): [string, number][] {231+	return JSON.parse(mirrorState(db, "languages") ?? "[]") as [string, number][];206232 }207233 208234 /** Every release record the bucket holds, in no particular order. */

@@ -3,12 +3,13 @@33 import { join } from "node:path";44 import { inflateSync } from "node:zlib";55 import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";6+import { fileFacts } from "../describe/describe.ts";67 import { isTestPath } from "../describe/links.ts";78 89 // Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs,910 // plus `releases`: each version's release.json (§11.2) as read from the bucket, since that is bucket10-// data too and pages and feeds need it on every request, and `hidden_files`.11-const SCHEMA_VERSION = 3;11+// data too and pages and feeds need it on every request, `hidden_files`, and what the Atlas reads.12+const SCHEMA_VERSION = 4;1213 // The operator's hidden rows, moved out of `save_files` rather than deleted, so a changed hidden1314 // list is applied in place, with nothing fetched again (2026-10-02).1415 const HIDDEN_FILES = `@@ -20,6 +21,25 @@2021 	PRIMARY KEY (save_id, path)2122 );2223 `;24+// What pages read, worked out as each save lands rather than on every view (2026-10-02): which25+// real saves touched each path (added, changed or removed against the real save before; `n` is the26+// touching save's rowid), what each file says about itself, and the objects real saves refer to.27+const DERIVED = `28+CREATE TABLE known_objects (hash TEXT PRIMARY KEY) WITHOUT ROWID;29+CREATE TABLE touches (30+	path TEXT NOT NULL,31+	n    INTEGER NOT NULL32+);33+CREATE INDEX touches_n ON touches(n);34+CREATE TABLE facts (35+	hash        TEXT NOT NULL,36+	path        TEXT NOT NULL,37+	lines       INTEGER,38+	description TEXT,39+	outline     TEXT,40+	PRIMARY KEY (hash, path)41+);42+`;2343 const SCHEMA = `2444 CREATE TABLE objects (2545 	hash TEXT PRIMARY KEY,@@ -65,7 +85,8 @@6585 );6686 CREATE INDEX saves_parent ON saves(parent);6787 CREATE INDEX save_files_hash ON save_files(hash);68-${HIDDEN_FILES}`;88+${HIDDEN_FILES}89+${DERIVED}`;6990 7091 /** An object as the bucket stores it: zlib bytes, already checked against its hash. */7192 export type StoredObject = { compressed: Uint8Array; size: number };@@ -117,8 +138,8 @@117138 118139 // Vendored code is someone else's: counting Koh 3's bundled SQLite made it read as 290k lines of C.119140 const VENDORED = /(^|\/)(vendor|third_party|node_modules)\//;120-// Bumped when what counts as a line changes, so every mirror recounts once when it opens.121-const STATS_RULE = "source-only";141+// Bumped when what the recount records changes, so every mirror recounts once when it opens.142+const STATS_RULE = "source-only, known objects";122143 123144 const open = new Map<string, Database>();124145 @@ -156,7 +177,7 @@156177 	db.run("PRAGMA busy_timeout = 5000");157178 	// A current mirror needs no lock; only creating or migrating one takes the write lock.158179 	const found = schemaVersion(db) === SCHEMA_VERSION ? SCHEMA_VERSION : upgrade(db);159-	if (found !== 0 && found !== 2 && found !== SCHEMA_VERSION) {180+	if (found !== 0 && found !== 2 && found !== 3 && found !== SCHEMA_VERSION) {160181 		db.close();161182 		throw new Error(162183 			`${path} has mirror schema version ${found}; this Kepr reads version ${SCHEMA_VERSION}. ` +@@ -181,13 +202,17 @@181202 	return db182203 		.transaction(() => {183204 			const found = schemaVersion(db);205+			if (found === SCHEMA_VERSION || ![0, 2, 3].includes(found)) return found;184206 			if (found === 0) db.run(SCHEMA);185-			if (found === 2) {186-				db.run(HIDDEN_FILES);187-				// Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.188-				if (mirrorState(db, "hidden")) clearMirror(db);207+			if (found === 2) db.run(HIDDEN_FILES);208+			if (found === 2 || found === 3) db.run(DERIVED);209+			// Version 2 deleted hidden rows, so only the bucket can bring them back on unhide.210+			if (found === 2 && mirrorState(db, "hidden")) clearMirror(db);211+			else if (found !== 0) {212+				recordTouches(db, 0);213+				recordFacts(db, null);189214 			}190-			if (found === 0 || found === 2) db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);215+			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);191216 			return found;192217 		})193218 		.immediate();@@ -281,16 +306,17 @@281306 			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);282307 		}283308 		for (const r of landing.releases) putRelease.run(r.version, JSON.stringify(r));284-		applyStrips(db);285-		hideRows(286-			db,287-			hiddenList(db),288-			landing.saves.map((save) => save.id),289-		);309+		const stripped = applyStrips(db);310+		const landed = landing.saves.map((save) => save.id);311+		hideRows(db, hiddenList(db), landed);312+		// A strip can reach any save, so every save's touches are worked out again after one.313+		recordTouches(db, stripped ? 0 : Number(mirrorState(db, "touches_to") ?? 0));314+		recordFacts(db, landed);290315 		setState.run("head", landing.head);291316 		setState.run("releases_latest", landing.releasesLatest);292317 		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);293318 		setState.run("quarantined", JSON.stringify([...quarantined].sort()));319+		staleStats(db);294320 	}).immediate();295321 	// Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.296322 	refreshStats(db);@@ -310,10 +336,85 @@310336 		);311337 		db.run("DELETE FROM hidden_files");312338 		hideRows(db, hidden, null);339+		recordTouches(db, 0);340+		recordFacts(db, null);341+		staleStats(db);313342 	})();314343 	refreshStats(db);315344 }316345 346+// Marks the counts out of date in the change's own transaction: if Kepr stops before the recount347+// that follows, the mirror's next open does it.348+function staleStats(db: Database): void {349+	db.run("DELETE FROM state WHERE key = 'stats_rule'");350+}351+352+/**353+ * Records which real saves touched each path, for the saves after rowid `from`; 0 starts again.354+ * A save touched a path when it was added, changed or removed against the real save before it;355+ * checkpoints are skipped, so a change first recorded by a checkpoint counts at the next real save.356+ */357+function recordTouches(db: Database, from: number): void {358+	if (from === 0) db.run("DELETE FROM touches");359+	db.query(360+		`WITH line AS (361+			SELECT id, rowid AS n, LAG(id) OVER (ORDER BY rowid) AS prev362+			FROM saves363+			WHERE is_checkpoint = 0364+		)365+		INSERT INTO touches (path, n)366+		SELECT f.path, line.n367+		FROM line368+		JOIN save_files f ON f.save_id = line.id369+		LEFT JOIN save_files p ON p.save_id = line.prev AND p.path = f.path370+		WHERE line.n > $from371+		AND (p.hash IS NOT f.hash OR p.executable IS NOT f.executable)372+		UNION ALL373+		SELECT p.path, line.n374+		FROM line375+		JOIN save_files p ON p.save_id = line.prev376+		WHERE line.n > $from377+		AND NOT EXISTS (SELECT 1 FROM save_files f WHERE f.save_id = line.id AND f.path = p.path)`,378+	).run({ from });379+	const last = db.query<{ n: number | null }, []>("SELECT MAX(rowid) AS n FROM saves").get()?.n;380+	db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('touches_to', ?)").run(381+		String(last ?? 0),382+	);383+}384+385+/** Records what each file of the given saves, or (null) every save, says about itself, once per version and path. */386+function recordFacts(db: Database, saves: string[] | null): void {387+	const wanted = db388+		.query<{ hash: string; path: string }, { all: number; saves: string }>(389+			`SELECT DISTINCT f.hash, f.path390+			FROM save_files f391+			WHERE ($all = 1 OR f.save_id IN (SELECT value FROM json_each($saves)))392+			AND NOT EXISTS (SELECT 1 FROM facts x WHERE x.hash = f.hash AND x.path = f.path)`,393+		)394+		.all({ all: saves === null ? 1 : 0, saves: JSON.stringify(saves ?? []) });395+	const sizeOf = db.query<{ size: number }, [string]>("SELECT size FROM objects WHERE hash = ?");396+	const dataOf = db.query<{ data: Uint8Array }, [string]>(397+		"SELECT data FROM objects WHERE hash = ?",398+	);399+	const put = db.query(400+		"INSERT OR IGNORE INTO facts (hash, path, lines, description, outline) VALUES (?, ?, ?, ?, ?)",401+	);402+	for (const { hash, path } of wanted) {403+		const size = sizeOf.get(hash)?.size;404+		if (size === undefined) continue;405+		// The stored bytes are read only for a file small enough to describe.406+		const read = () => inflateSync((dataOf.get(hash) as { data: Uint8Array }).data);407+		const facts = fileFacts(path, size, read);408+		put.run(409+			hash,410+			path,411+			facts.lines,412+			facts.description,413+			facts.outline && JSON.stringify(facts.outline),414+		);415+	}416+}417+317418 function hiddenList(db: Database): string[] {318419 	return mirrorState(db, "hidden")?.split("\n").filter(Boolean) ?? [];319420 }@@ -344,21 +445,22 @@344445 // §10.2: a strip of one save reaches that save; a strip of every save bans its path from all of345446 // history, whenever a save was made or arrives. Re-applying every strip is harmless, which is how a346447 // save that arrives after its strip event is covered too.347-function applyStrips(db: Database): void {448+// Returns whether any save lost a file.449+function applyStrips(db: Database): boolean {348450 	const strips = db349451 		.query<{ save_id: string; path: string }, []>(350452 			"SELECT save_id, path FROM events WHERE kind = 'strip'",351453 		)352454 		.all();353-	if (strips.length === 0) return;455+	if (strips.length === 0) return false;456+	let removed = 0;354457 	for (const strip of strips) {355458 		const { where, params } = pathMatch(strip.path);356459 		// Hidden rows too, so unhiding never brings back what a strip removed.357460 		for (const table of ["save_files", "hidden_files"]) {358-			db.query(`DELETE FROM ${table} WHERE ${where} AND ($save = '' OR save_id = $save)`).run({359-				...params,360-				save: strip.save_id,361-			});461+			removed += db462+				.query(`DELETE FROM ${table} WHERE ${where} AND ($save = '' OR save_id = $save)`)463+				.run({ ...params, save: strip.save_id }).changes;362464 		}363465 	}364466 	db.run(@@ -366,6 +468,8 @@366468 		WHERE hash NOT IN (SELECT hash FROM save_files)367469 		AND hash NOT IN (SELECT hash FROM hidden_files)`,368470 	);471+	db.run("DELETE FROM facts WHERE hash NOT IN (SELECT hash FROM objects)");472+	return removed > 0;369473 }370474 371475 /** The SQL condition matching a path, file or folder, or an `ANY_DEPTH` name at any depth. */@@ -394,6 +498,9 @@394498 		const tables = [395499 			"save_files",396500 			"hidden_files",501+			"touches",502+			"facts",503+			"known_objects",397504 			"events",398505 			"saves",399506 			"objects",@@ -410,7 +517,13 @@410517 // Lines and languages never change between syncs, so they are counted once here, not per view. They411518 // come from the newest real save, not `head`: a checkpoint's files are not public history (§21.4).412519 // Lines are source lines: files in a known language, tests left out; docs and data never count.520+// Read and written under one write lock: both threads recount, and a recount must never publish521+// what it read before another thread's newer one.413522 function refreshStats(db: Database): void {523+	db.transaction(() => recount(db)).immediate();524+}525+526+function recount(db: Database): void {414527 	const save =415528 		db416529 			.query<{ id: string }, []>(@@ -443,5 +556,22 @@443556 	setState.run("language", languages[0]?.[0] ?? "");444557 	setState.run("languages", JSON.stringify(languages));445558 	setState.run("stats_save", save);559+	// §21.1's "the library knows N things": the file contents any real save refers to, which the560+	// front page unites across public repos. From `objects`, probing save_files_hash: a DISTINCT561+	// over save_files reads saves × files rows.562+	db.run("DELETE FROM known_objects");563+	db.run(564+		`INSERT INTO known_objects (hash)565+		SELECT o.hash566+		FROM objects o567+		WHERE EXISTS (568+			SELECT 1569+			FROM save_files f570+			JOIN saves s ON s.id = f.save_id571+			WHERE f.hash = o.hash572+			AND s.is_checkpoint = 0573+		)`,574+	);575+	// Last, with the recount it names, so one cut short is done again at the next open.446576 	setState.run("stats_rule", STATS_RULE);447577 }

@@ -1,21 +1,10 @@1-import { describeFile, describeFolder, firstSentence } from "../describe/describe.ts";2-import { isTestPath, type Outline, outline, repoIndex, resolveImports } from "../describe/links.ts";3-import {4-	filesAt,5-	fileTouches,6-	headId,7-	objectBytes,8-	type Save,9-	type Touches,10-	tarballGeneration,11-} from "../mirror/queries.ts";12-import { isText } from "./files.ts";1+import { describeFolder, firstSentence } from "../describe/describe.ts";2+import { isTestPath, type Outline, repoIndex, resolveImports } from "../describe/links.ts";3+import { filesWithFacts, fileTouches, type Save, type Touches } from "../mirror/queries.ts";134 import { type Html, html } from "./html.ts";145 import {156 	formatDate,167 	formatSize,17-	openPublicRepo,18-	type PageContext,198 	type PublicRepo,209 	pathUrl,2110 	plural,@@ -48,61 +37,27 @@4837 4938 type View = { repo: string; at: string | null; links: Links };5039 51-type Facts = { lines: number | null; description: string | null; outline: Outline | null };52-5340 // A panel lists this many names or links, then says how many more.5441 const LISTED = 12;5542 56-// Measured: Koh 3's vendored 9 MB sqlite3.c cost every Atlas view ~25 ms to inflate and scan. An57-// object never changes, so what it says about itself is kept for the life of the process, keyed by58-// hash and file name (the name picks the comment syntax).59-// ponytail: unbounded, one small entry per object viewed; an LRU if a bucket reaches millions.60-const known = new Map<string, Facts>();61-62-function factsOf(db: PublicRepo["mirror"], hash: string, name: string): Facts {63-	const key = `${hash} ${name}`;64-	const cached = known.get(key);65-	if (cached) return cached;66-	const bytes = objectBytes(db, hash);67-	const text = bytes && isText(bytes) ? new TextDecoder().decode(bytes) : null;68-	const facts =69-		text === null70-			? { lines: null, description: null, outline: null }71-			: {72-					lines: countLines(text),73-					description: describeFile(name, text),74-					outline: outline(name, text),75-				};76-	known.set(key, facts);77-	return facts;78-}79-8043 /** The tree at one save with each file's facts and the links between files. */8144 type AtlasModel = { root: AtlasFolder; links: Links; count: number };8245 83-// Building a model reads every file's facts and every save that touched each path: about 165 ms84-// warm for a 550-file repo on Tylee. Panels are fetched one at a time on hover, so the model is85-// kept, keyed by everything that changes what it holds: the save, and the strips and hidden paths86-// (`tarballGeneration`).87-// ponytail: the newest MODELS models in insertion order, not an LRU; enough for a few repos at once.88-const MODELS = 16;89-const models = new Map<string, AtlasModel>();90-91-// In one read snapshot: the sync's thread can apply a hidden list between any two statements, and92-// a model kept under the old generation must never hold a file the new one hides.46+// Built per request from what each save recorded as it landed (touches and facts), so nothing is47+// kept between requests. In one read snapshot: the sync's thread can change the mirror between any48+// two statements.9349 function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {9450 	return repo.mirror.transaction(() => buildAtlasModel(repo, saveId))();9551 }9652 9753 function buildAtlasModel(repo: PublicRepo, saveId: string): AtlasModel {98-	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;99-	const cached = models.get(key);100-	if (cached) return cached;10154 	const touches = fileTouches(repo.mirror, saveId);10255 	const foldersTouches = folderTouches(touches);10356 	const root: AtlasFolder = { name: "", path: "", folders: [], files: [], saves: [] };10457 	// Koh's own rules file is bookkeeping, not part of the project; Koh never saves what it names.105-	const entries = filesAt(repo.mirror, saveId).filter((entry) => entry.path !== ".kohignore");58+	const entries = filesWithFacts(repo.mirror, saveId).filter(59+		(entry) => entry.path !== ".kohignore",60+	);10661 	for (const entry of entries) {10762 		const parts = entry.path.split("/");10863 		let folder = root;@@ -115,28 +70,19 @@11570 			}11671 			folder = child;11772 		}118-		const name = parts[parts.length - 1] as string;11973 		folder.files.push({120-			name,74+			name: parts[parts.length - 1] as string,12175 			path: entry.path,12276 			size: entry.size,123-			...factsOf(repo.mirror, entry.hash, name),77+			lines: entry.lines,78+			description: entry.description,79+			outline: entry.outline,12480 			saves: touches.byPath.get(entry.path) ?? [],12581 		});12682 	}127-	const model = { root, links: linksOf(allFiles(root)), count: entries.length };128-	if (models.size >= MODELS) models.delete(models.keys().next().value as string);129-	models.set(key, model);130-	return model;83+	return { root, links: linksOf(allFiles(root)), count: entries.length };13184 }13285 133-/** Builds a public repo's Atlas at its head ahead of the first visitor, as the sync calls it. */134-export function warmAtlas(ctx: Pick<PageContext, "kepr" | "mirrorDir">, name: string): void {135-	const repo = openPublicRepo(ctx, name);136-	const head = repo && headId(repo.mirror);137-	if (repo && head) atlasModel(repo, head);138-}139-14086 /**14187  * §21.5's Atlas at one save: the folder tree as `<details>`, so it opens and links without14288  * JavaScript. A row's detail panel is fetched from `atlasPanel` on hover or select (kepr.js), so@@ -398,19 +344,19 @@398344 399345 // Each folder's saves are the line's saves that touched any path under it, in line order.400346 function folderTouches(touches: Touches): Map<string, Save[]> {401-	const ids = new Map<string, Set<string>>();402-	for (const [path, saves] of touches.byPath) {347+	const order = new Map(touches.line.map((save, i) => [save, i]));348+	const saves = new Map<string, Set<Save>>();349+	for (const [path, touched] of touches.byPath) {403350 		const parts = path.split("/");404351 		for (let depth = 1; depth < parts.length; depth++) {405352 			const folder = parts.slice(0, depth).join("/");406-			const set = ids.get(folder) ?? new Set<string>();407-			for (const save of saves) set.add(save.id);408-			ids.set(folder, set);353+			const set = saves.get(folder) ?? new Set<Save>();354+			for (const save of touched) set.add(save);355+			saves.set(folder, set);409356 		}410357 	}411-	return new Map(412-		[...ids].map(([folder, set]) => [folder, touches.line.filter((save) => set.has(save.id))]),413-	);358+	const byLine = (a: Save, b: Save) => (order.get(a) ?? 0) - (order.get(b) ?? 0);359+	return new Map([...saves].map(([folder, set]) => [folder, [...set].sort(byLine)]));414360 }415361 416362 function allFiles(folder: AtlasFolder): AtlasFile[] {@@ -420,11 +366,4 @@420366 // Relative links in a description resolve from the file's own folder, as in a README.421367 function markdownBase(view: View, path: string): MarkdownBase {422368 	return { repo: view.repo, dir: path.split("/").slice(0, -1).join("/"), at: view.at };423-}424-425-function countLines(text: string): number {426-	if (text === "") return 0;427-	let count = 0;428-	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;429-	return text.endsWith("\n") ? count : count + 1;430369 }

@@ -1,5 +1,6 @@11 import type { Context, Hono } from "hono";22 import { isSafePath } from "../bucket/repo.ts";3+import { isText, SHOWN_BYTES } from "../describe/describe.ts";34 import { type FileEntry, fileAt, headId, objectBytes, saveByPrefix } from "../mirror/queries.ts";45 import { highlightLines } from "./highlight.ts";56 import { type Html, html } from "./html.ts";@@ -14,7 +15,6 @@1415 } from "./layout.ts";1516 import { renderMarkdown } from "./markdown.ts";1617 17-const SHOWN_BYTES = 1024 * 1024;1818 const IMAGE = /\.(png|jpe?g|gif|webp|avif|ico|svg)$/i;1919 const MARKDOWN = /\.(md|markdown)$/i;2020 // Binary types a browser can show or save as they are; anything else textual stays@@ -69,20 +69,6 @@6969 	const file = saveId ? fileAt(repo.mirror, saveId, path) : null;7070 	const bytes = file && objectBytes(repo.mirror, file.hash);7171 	return file && bytes ? { repo: name, path, at: at && saveId, file, bytes } : null;72-}73-74-export function isText(bytes: Uint8Array): boolean {75-	if (bytes.subarray(0, 8000).includes(0)) return false;76-	try {77-		// Streaming mode tolerates only a character cut at the 1 MB mark; invalid bytes still throw.78-		const cut = bytes.length > SHOWN_BYTES;79-		new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, SHOWN_BYTES), {80-			stream: cut,81-		});82-		return true;83-	} catch {84-		return false;85-	}8672 }8773 8874 function contentType(path: string, bytes: Uint8Array): string {

@@ -1,5 +1,5 @@11 import type { Hono } from "hono";2-import { languageLines, objectHashes } from "../mirror/queries.ts";2+import { knownObjects, languageLines } from "../mirror/queries.ts";33 import { setting } from "../server/db.ts";44 import { type Html, html } from "./html.ts";55 import { cacheFor, type PageContext, page, plural } from "./layout.ts";@@ -14,8 +14,8 @@1414 	app.get("/", (c) => {1515 		const library = publicLibrary(ctx);1616 		const tagline = setting(ctx.kepr, "tagline") ?? "";17-		// §21.1: distinct objects across every public repo's saves, as Kepr 2 counted object refs.18-		const known = new Set(library.flatMap((repo) => objectHashes(repo.mirror))).size;17+		// §21.1: distinct objects across every public repo's real saves.18+		const known = new Set(library.flatMap((repo) => knownObjects(repo.mirror))).size;1919 		const latest = library2020 			.filter((repo) => repo.stats.updated !== null)2121 			.sort((a, b) => (b.stats.updated ?? 0) - (a.stats.updated ?? 0))

@@ -1,11 +1,11 @@1-import { type RepoStats, repoStats, type Save, timeline } from "../mirror/queries.ts";1+import { newestSaves, type RepoStats, repoStats, type Save } from "../mirror/queries.ts";22 import { publicRepoNames } from "../server/db.ts";33 import { type Html, html } from "./html.ts";44 import { openPublicRepo, type PageContext, type PublicRepo, plural } from "./layout.ts";55 import { repoIntro } from "./repo.ts";66 7-/** A public repo as the front page, browse and the profile show it. `saves` are oldest first. */8-export type LibraryRepo = PublicRepo & { stats: RepoStats; description: string; saves: Save[] };7+/** A public repo as the front page, browse and the profile show it. */8+export type LibraryRepo = PublicRepo & { stats: RepoStats; description: string };99 1010 export type RecentSave = { repo: string; save: Save };1111 @@ -15,14 +15,16 @@1515 		const repo = openPublicRepo(ctx, name);1616 		if (!repo) return [];1717 		const description = repoIntro(repo)?.text ?? "";18-		return [{ ...repo, stats: repoStats(repo.mirror), description, saves: timeline(repo.mirror) }];18+		return [{ ...repo, stats: repoStats(repo.mirror), description }];1919 	});2020 }2121 2222 /** The newest saves across the library, checkpoints never among them. */2323 export function recentSaves(library: LibraryRepo[], limit: number): RecentSave[] {2424 	return library25-		.flatMap((repo) => repo.saves.slice(-limit).map((save) => ({ repo: repo.settings.name, save })))25+		.flatMap((repo) =>26+			newestSaves(repo.mirror, limit).map((save) => ({ repo: repo.settings.name, save })),27+		)2628 		.sort((a, b) => b.save.created_at - a.save.created_at)2729 		.slice(0, limit);2830 }

@@ -1,5 +1,6 @@11 import type { Database } from "bun:sqlite";22 import type { Hono } from "hono";3+import { isText } from "../describe/describe.ts";34 import {45 	type FileEntry,56 	filesAt,@@ -10,7 +11,6 @@1011 	visibleParent,1112 } from "../mirror/queries.ts";1213 import { diffLines, type Edit, hunks } from "./diff.ts";13-import { isText } from "./files.ts";1414 import { highlightLines } from "./highlight.ts";1515 import { type Html, html } from "./html.ts";1616 import {

@@ -250,3 +250,28 @@250250 	}251251 	expect(blocks).toContain("animation: none;");252252 });253+254+test("a file two public repos share is one thing the library knows", async () => {255+	const repos = await makeRepos(["one", "two"]);256+	for (const repo of ["one", "two"]) {257+		await writeFile(join(repos.bucketDir, "..", "work", repo, "LICENSE"), "the same licence\n");258+		repos.koh(repo, "save", "add the licence");259+		repos.koh(repo, "offer");260+	}261+	const dir = await mkdtemp(join(tmpdir(), "kepr-front-"));262+	const db = openKeprDb(dir);263+	const bucket = openBucket({ kind: "folder", path: repos.bucketDir });264+	await syncAll({ bucket, kepr: db, mirrorDir: join(dir, "mirror") });265+	db.run("UPDATE repos SET public = 1");266+	const shared = createApp(267+		{268+			domain: "kepr.example",269+			dataDir: dir,270+			adminPasswordHash: "$argon2id$x",271+			bucket: { kind: "folder", path: repos.bucketDir },272+		},273+		db,274+	);275+	// Two READMEs that differ; koh's default .kohignore and the licence, each shared by both.276+	expect(await (await shared.request("/")).text()).toContain("The library knows<br>4 things.");277+});

@@ -155,17 +155,6 @@155155 	expect(saveByPrefix(mirror, "af96c81")?.message).toBe("made by Koh 3");156156 });157157 158-test("tells its target about each repo that synced cleanly, and not one that failed", async () => {159-	const fx = await makeKohBucket("told");160-	const t = await syncTarget(fx);161-	const synced: string[] = [];162-	await syncAll({ ...t, synced: (repo) => synced.push(repo) });163-	expect(synced).toEqual(["told"]);164-	await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");165-	await syncAll({ ...t, synced: (repo) => synced.push(repo) });166-	expect(synced).toEqual(["told"]);167-});168-169158 test("a sync lands history in batches, so one cut short resumes where it stopped", async () => {170159 	const fx = await makeKohBucket("batched");171160 	const bucket = openBucket({ kind: "folder", path: fx.bucketDir });@@ -343,16 +332,16 @@343332 });344333 345334 describe("the mirror's schema version", () => {346-	test("a version 3 mirror reopens and any other version is refused as disposable", async () => {335+	test("a version 4 mirror reopens and any other version is refused as disposable", async () => {347336 		const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));348337 		const created = openMirror(dir, "one");349338 		created.run("PRAGMA wal_checkpoint(TRUNCATE)");350339 		await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));351340 		expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({352-			user_version: 3,341+			user_version: 4,353342 		});354343 355-		for (const version of [1, 4]) {344+		for (const version of [1, 5]) {356345 			const other = new Database(join(dir, `v${version}.db`));357346 			other.run(`PRAGMA user_version = ${version}`);358347 			other.close();@@ -360,14 +349,16 @@360349 		}361350 	});362351 363-	test("a version 2 mirror moves to 3 in place, emptied only when it deleted hidden rows", async () => {352+	test("a version 2 mirror moves to 4 in place, emptied only when it deleted hidden rows", async () => {364353 		const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));365354 		for (const [name, hidden] of [366355 			["plain", ""],367356 			["hiding", "secret.txt"],368357 		] as const) {369358 			const db = openMirror(dir, name);370-			db.run("DROP TABLE hidden_files");359+			for (const table of ["hidden_files", "touches", "facts", "known_objects"]) {360+				db.run(`DROP TABLE ${table}`);361+			}371362 			db.run("PRAGMA user_version = 2");372363 			db.query("INSERT INTO state (key, value) VALUES ('head', 'abcdef123456'), ('hidden', ?)").run(373364 				hidden,@@ -379,10 +370,68 @@379370 		expect(head("hiding")).toBeNull();380371 		for (const name of ["plain", "hiding"]) {381372 			const db = openMirror(dir, name);382-			expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: 3 });373+			expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: 4 });383374 			expect(db.query("SELECT COUNT(*) AS n FROM hidden_files").get()).toEqual({ n: 0 });384375 		}385376 	});377+378+	test("touches recorded save by save match touches worked out all at once", async () => {379+		const fx = await makeKohBucket("stepwise");380+		const t = await syncTarget(fx);381+		await syncAll(t);382+		await writeFile(join(fx.repoDir, "src/main.ts"), "// Entry point.\nconsole.log('again');\n");383+		await writeFile(join(fx.repoDir, "added.txt"), "new\n");384+		Bun.spawnSync(["koh3", "save", "change main, add a file"], {385+			cwd: fx.repoDir,386+			env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },387+		});388+		Bun.spawnSync(["koh3", "offer"], {389+			cwd: fx.repoDir,390+			env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },391+		});392+		await syncAll(t);393+		const touches = () => t.mirror().query("SELECT path, n FROM touches ORDER BY n, path").all();394+		const stepwise = touches();395+		expect(stepwise).toContainEqual(expect.objectContaining({ path: "added.txt" }));396+		// Reopened as version 3, the mirror works every save's touches out again in one go.397+		for (const table of ["touches", "facts", "known_objects"]) {398+			t.mirror().run(`DROP TABLE ${table}`);399+		}400+		t.mirror().run("PRAGMA user_version = 3");401+		closeMirror(t.mirrorDir, fx.repo);402+		expect(touches()).toEqual(stepwise);403+	});404+405+	test("a recount cut short is done when the mirror next opens", async () => {406+		const fx = await makeKohBucket("recount");407+		const t = await syncTarget(fx);408+		await syncAll(t);409+		const known = () => t.mirror().query("SELECT COUNT(*) AS n FROM known_objects").get();410+		const counted = known();411+		// What a landing leaves if Kepr stops before its recount commits.412+		t.mirror().run("DELETE FROM known_objects");413+		t.mirror().run("DELETE FROM state WHERE key = 'stats_rule'");414+		closeMirror(t.mirrorDir, fx.repo);415+		expect(known()).toEqual(counted);416+	});417+418+	test("a version 3 mirror gains the touches and facts a sync would have recorded", async () => {419+		const fx = await makeKohBucket("derived");420+		const t = await syncTarget(fx);421+		await syncAll(t);422+		const derived = () => [423+			t.mirror().query("SELECT path, n FROM touches ORDER BY n, path").all(),424+			t.mirror().query("SELECT * FROM facts ORDER BY path").all(),425+		];426+		const recorded = derived();427+		expect((recorded[0] as unknown[]).length).toBeGreaterThan(0);428+		for (const table of ["touches", "facts", "known_objects"]) {429+			t.mirror().run(`DROP TABLE ${table}`);430+		}431+		t.mirror().run("PRAGMA user_version = 3");432+		closeMirror(t.mirrorDir, fx.repo);433+		expect(derived()).toEqual(recorded);434+	});386435 });387436 388437 test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => {@@ -488,7 +537,7 @@488537 	expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);489538 });490539 491-test("the line count skips a file over 8 MiB (A3)", async () => {540+test("the line count and the Atlas's facts skip a file over 8 MiB (A3)", async () => {492541 	const fx = await makeKohBucket("large-text");493542 	const t = await syncTarget(fx);494543 	await syncAll(t);@@ -500,4 +549,7 @@500549 	await syncAll(t);501550 	expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");502551 	expect(repoStats(t.mirror()).lines).toBe(before);552+	expect(553+		t.mirror().query("SELECT lines, description FROM facts WHERE path = 'big.txt'").get(),554+	).toEqual({ lines: null, description: null });503555 });