quarantine unsafe or oversized paths instead of stopping a repo's sync, cap object inflation at 512 MiB, recount stats outside the landing and past 8 MiB files; pin koh3 3.2.2 (v3.0.86)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.85",3+	"version": "3.0.86",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -1,7 +1,8 @@11 import type { Context, Hono } from "hono";22 import { syncAll } from "../bucket/sync.ts";3+import { openMirror, quarantinedPaths } from "../mirror/store.ts";34 import { html } from "../pages/html.ts";4-import { formatDate } from "../pages/layout.ts";5+import { formatDate, plural } from "../pages/layout.ts";56 import { type AdminContext, adminPage, formField, notice, textFields } from "./console.ts";67 import { type AdminRepo, allRepos, findRepo, type RepoEdit, updateRepo } from "./store.ts";78 @@ -54,11 +55,11 @@5455 <form method="post" action="/admin/sync">5556 ${formField(c)}<button class="issues-submit" type="submit">sync now</button></form>5657 ${repos.length === 0 && html`<p class="muted mono save-empty">no repos found in the bucket yet</p>`}57-${repos.map((repo) => repoForm(c, repo))}`;58+${repos.map((repo) => repoForm(c, repo, quarantinedPaths(openMirror(ctx.mirrorDir, repo.name))))}`;5859 	return adminPage(c, ctx, "repos", body);5960 }6061 61-function repoForm(c: Context, repo: AdminRepo) {62+function repoForm(c: Context, repo: AdminRepo, quarantined: string[]) {6263 	const input = (name: keyof RepoEdit, label: string) =>6364 		html`<label class="issues-label" for="${repo.name}-${name}">${label}</label>6465 <input class="issues-input" id="${repo.name}-${name}" name="${name}" maxlength="${MAX_FIELD}" value="${String(repo[name])}">`;@@ -71,6 +72,10 @@7172 <span>${repo.public ? "public" : "private"}</span><span class="history-when">${synced}</span></p>7273 ${7374 	repo.last_sync_error && html`<p class="issues-error mono">sync error: ${repo.last_sync_error}</p>`75+}76+${77+	quarantined.length > 0 &&78+	html`<p class="issues-error mono">${plural(quarantined.length, "path")} quarantined: ${quarantined.join(", ")}</p>`7479 }7580 <form class="issues-form" method="post" action="/admin/repos/${repo.name}">7681 ${formField(c)}

@@ -64,11 +64,19 @@6464 	"android-x86",6565 ] as const;6666 67+/** A save record as it arrived, less the files §5.5 quarantines, named here escaped. */68+export type ArrivingSave = SaveRecord & { quarantined: string[] };69+6770 /** An object's stored zlib bytes and the inflated bytes they were verified from. */6871 export type VerifiedObject = { compressed: Uint8Array; bytes: Uint8Array };6972 7073 export class BucketError extends Error {}7174 75+/** An object that inflates past Koh's own file limit; the paths pointing at it are quarantined. */76+export class OversizedObject extends BucketError {}77+78+const MAX_OBJECT = 512 * 1024 * 1024;79+7280 const SAVE_ID = /^[0-9a-f]{12}$/;7381 const REPO_NAME = /^[A-Za-z0-9._-]+$/;7482 const CONTROL_CHAR = /\p{Cc}/u;@@ -77,6 +85,7 @@7785 // A file name that is safe as one key segment and inside a quoted Content-Disposition filename.7886 const FILE_NAME = /^[^/\\"\p{Cc}]{1,255}$/u;7987 const decoder = new TextDecoder();88+const strictDecoder = new TextDecoder("utf-8", { fatal: true });8089 8190 // ponytail: one full listing per call; switch to a delimiter listing if the bucket grows large.8291 export async function listRepos(bucket: BucketReader): Promise<string[]> {@@ -124,14 +133,33 @@124133 	bucket: BucketReader,125134 	repo: string,126135 	id: string,127-): Promise<SaveRecord> {136+): Promise<ArrivingSave> {128137 	requireRepo(repo);129138 	requireSaveId(id);130139 	const bytes = await bucket.get(`${repo}/saves/${id}`);131140 	if (bytes === null) throw new BucketError(`${repo}/saves/${id} is missing`);132-	const record = parseSaveRecord(JSON.parse(decoder.decode(bytes)));141+	// Koh reads a record's bytes as they are and refuses a path that is not UTF-8. Decoding turns142+	// such bytes into U+FFFD, so only in a record that fails strict decoding does U+FFFD mark one.143+	// ponytail: a genuine U+FFFD name in that same broken record is quarantined too; Koh never144+	// writes such a record, so byte-exact recovery waits until one is seen.145+	let text: string;146+	let lossy = false;147+	try {148+		text = strictDecoder.decode(bytes);149+	} catch {150+		text = decoder.decode(bytes);151+		lossy = true;152+	}153+	const record = parseSaveRecord(JSON.parse(text));133154 	if (record.id !== id) throw new BucketError(`${repo}/saves/${id} holds save ${record.id}`);134-	return record;155+	const legal = (path: string) => isSafePath(path) && !(lossy && path.includes("\uFFFD"));156+	return {157+		...record,158+		files: record.files.filter((file) => legal(file.path)),159+		quarantined: record.files160+			.filter((file) => !legal(file.path))161+			.map((file) => escapePath(file.path)),162+	};135163 }136164 137165 /** Fetches an object, inflates it and checks its Blake3 against its key, as steal does (§9.4). */@@ -144,7 +172,15 @@144172 	if (!HEX64.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);145173 	const compressed = await bucket.get(`${repo}/objects/${hash}`);146174 	if (compressed === null) throw new BucketError(`${repo}/objects/${hash} is missing`);147-	const bytes = new Uint8Array(inflateSync(compressed));175+	let bytes: Uint8Array;176+	try {177+		bytes = new Uint8Array(inflateSync(compressed, { maxOutputLength: MAX_OBJECT }));178+	} catch (cause) {179+		if (cause instanceof RangeError) {180+			throw new OversizedObject(`${repo}/objects/${hash} inflates past 512 MiB`);181+		}182+		throw cause;183+	}148184 	if (bytesToHex(blake3(bytes)) !== hash) {149185 		throw new BucketError(`${repo}/objects/${hash} does not match its hash`);150186 	}@@ -324,22 +360,34 @@324360 	return Number.isSafeInteger(value) && (value as number) >= 0;325361 }326362 327-/** A repo-relative path that can't escape or alias: no empty, `.` or `..` segments, no backslashes or control characters. */363+/** §5.5's legal path, clause for clause with Koh's `legal_path`: no empty, `.` or `..` segments, no backslash or control character, well-formed UTF-8. */328364 export function isSafePath(path: string): boolean {329365 	return (366+		path.isWellFormed() &&330367 		!path.includes("\\") &&331368 		!CONTROL_CHAR.test(path) &&332369 		path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..")333370 	);334371 }335372 373+/** A name printable without its controls reaching the page, as Koh's `escape_path` prints it. */374+export function escapePath(path: string): string {375+	return path376+		.replace(/[\\\p{Cc}]/gu, (char) => {377+			const code = char.charCodeAt(0);378+			if (char === "\\") return "\\\\";379+			const hex = code.toString(16).padStart(2, "0");380+			return code >= 0x80 ? `\\u00${hex}` : `\\x${hex}`;381+		})382+		.toWellFormed();383+}384+336385 function isSaveFile(value: unknown): value is SaveFile {337386 	const f = value as Record<string, unknown>;338387 	return (339388 		typeof value === "object" &&340389 		value !== null &&341390 		typeof f.path === "string" &&342-		isSafePath(f.path) &&343391 		typeof f.hash === "string" &&344392 		HEX64.test(f.hash) &&345393 		typeof f.executable === "boolean"

@@ -17,9 +17,11 @@1717 	BucketError,1818 	checkFormat,1919 	type EventRecord,20+	escapePath,2021 	listEventIds,2122 	listReleaseVersions,2223 	listRepos,24+	OversizedObject,2325 	type ReleaseRecord,2426 	readEvent,2527 	readHead,@@ -77,7 +79,9 @@7779  * Koh's steal flow (§9.4) against a mirror: walk back from the remote head to a save the mirror has,7880  * fetch what is missing, and land it in one transaction. Events are listed every time, since a7981  * flag or a strip arrives without moving `head`; releases only when `head` or `releases/latest`80- * moved (§18), since publishing one moves both. Returns the events it landed.82+ * moved (§18), since publishing one moves both. A path §5.5 refuses, or one whose object inflates83+ * past Koh's file limit, is quarantined: the save lands without it, as if stripped. Returns the84+ * events it landed.8185  */8286 export async function syncRepo(8387 	bucket: BucketReader,@@ -89,16 +93,30 @@8993 	if (head === null) return [];9094 	const saves: SaveRecord[] = [];9195 	const objects = new Map<string, StoredObject>();96+	const oversized = new Set<string>();97+	const quarantined: string[] = [];9298 	const seen = new Set<string>();9399 	for (let cursor = head; cursor !== "" && !knownSave(mirror, cursor); ) {94100 		if (seen.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`);95101 		seen.add(cursor);96102 		const save = await readSave(bucket, repo, cursor);103+		quarantined.push(...save.quarantined);97104 		for (const file of save.files) {98-			if (objects.has(file.hash) || knownObject(mirror, file.hash)) continue;99-			const { compressed, bytes } = await readObject(bucket, repo, file.hash);100-			objects.set(file.hash, { compressed, size: bytes.length });105+			if (objects.has(file.hash) || oversized.has(file.hash) || knownObject(mirror, file.hash)) {106+				continue;107+			}108+			try {109+				const { compressed, bytes } = await readObject(bucket, repo, file.hash);110+				objects.set(file.hash, { compressed, size: bytes.length });111+			} catch (cause) {112+				if (!(cause instanceof OversizedObject)) throw cause;113+				oversized.add(file.hash);114+			}101115 		}116+		quarantined.push(117+			...save.files.filter((f) => oversized.has(f.hash)).map((f) => escapePath(f.path)),118+		);119+		save.files = save.files.filter((file) => !oversized.has(file.hash));102120 		saves.push(save);103121 		cursor = save.parent;104122 	}@@ -113,7 +131,15 @@113131 		mirrorState(mirror, "releases_latest") !== releasesLatest;114132 	const releases = moved ? await newReleases(bucket, mirror, repo) : [];115133 	if (!moved && events.length === 0) return [];116-	land(mirror, { head, releasesLatest, saves: saves.reverse(), objects, events, releases });134+	land(mirror, {135+		head,136+		releasesLatest,137+		saves: saves.reverse(),138+		objects,139+		events,140+		releases,141+		quarantined,142+	});117143 	return events;118144 }119145 

@@ -66,8 +66,13 @@6666 	objects: Map<string, StoredObject>;6767 	events: EventRecord[];6868 	releases: ReleaseRecord[];69+	/** Escaped names of the paths these saves arrived without (§5.5, A3). */70+	quarantined: string[];6971 };7072 73+// A recount inflates every file at the newest save; a file this large is data, not lines of code.74+const MAX_COUNTED = 8 * 1024 * 1024;75+7176 // Display names for the header's language, by extension; data and docs formats never count.7277 const LANGUAGES: Record<string, string> = {7378 	ts: "TypeScript",@@ -200,10 +205,18 @@200205 		applyStrips(db);201206 		setState.run("head", landing.head);202207 		setState.run("releases_latest", landing.releasesLatest);203-		refreshStats(db);208+		const quarantined = new Set([...quarantinedPaths(db), ...landing.quarantined]);209+		setState.run("quarantined", JSON.stringify([...quarantined].sort()));204210 	})();211+	// Outside the transaction: the recount can inflate megabytes and needs no lock to be correct.212+	refreshStats(db);205213 }206214 215+/** Escaped names of every path this mirror's saves arrived without. */216+export function quarantinedPaths(db: Database): string[] {217+	return JSON.parse(mirrorState(db, "quarantined") ?? "[]");218+}219+207220 // §10.2: a strip of one save reaches that save; a strip of every save bans its path from all of208221 // history, whenever a save was made or arrives. Re-applying every strip is harmless, which is how a209222 // save that arrives after its strip event is covered too.@@ -237,14 +250,14 @@237250 			)238251 			.get()?.id ?? "";239252 	const files = db240-		.query<{ path: string; data: Uint8Array }, [string]>(241-			"SELECT f.path, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",253+		.query<{ path: string; size: number; data: Uint8Array }, [string]>(254+			"SELECT f.path, o.size, o.data FROM save_files f JOIN objects o ON o.hash = f.hash WHERE f.save_id = ?",242255 		)243256 		.all(save);244257 	let lines = 0;245258 	const byLanguage = new Map<string, number>();246259 	for (const file of files) {247-		if (VENDORED.test(file.path)) continue;260+		if (VENDORED.test(file.path) || file.size > MAX_COUNTED) continue;248261 		const bytes = inflateSync(file.data);249262 		if (bytes.includes(0)) continue;250263 		let count = 0;

@@ -4,6 +4,7 @@44 import { openBucket } from "../src/bucket/reader.ts";55 import {66 	checkFormat,7+	escapePath,78 	listEventIds,89 	listRepos,910 	readEvent,@@ -155,6 +156,7 @@155156 		expect((await readTampered(() => {})).message).toBe("say hello");156157 	});157158 159+	// §5.5: a path Koh would refuse is quarantined, and the rest of the save still lands.158160 	test.each([159161 		"",160162 		"/etc/passwd",@@ -166,8 +168,40 @@166168 		"src\\main.ts",167169 		"src/ma\0in.ts",168170 		"src/ma\nin.ts",169-	])("refuses the file path %p", async (path) => {170-		await expect(readTampered(withPath(path))).rejects.toThrow("Koh 3 schema");171+		"Icon\r",172+		"a\u0085b",173+		"a\ud800b",174+	])("quarantines the file path %p", async (path) => {175+		const save = await readTampered(withPath(path));176+		expect(save.files.map((file) => file.path)).not.toContain(path);177+		expect(save.files.length).toBeGreaterThan(0);178+		expect(save.quarantined).toEqual([escapePath(path)]);179+	});180+181+	test("quarantines a path that is not valid UTF-8, and keeps a real U+FFFD in a valid record", async () => {182+		const id = fx.saveIds[0] as string;183+		const real = await bucket().get(`${fx.repo}/saves/${id}`);184+		const record = JSON.parse(new TextDecoder().decode(real as Uint8Array));185+		record.files[0].path = "bad-X-name";186+		const bytes = Buffer.from(JSON.stringify(record));187+		bytes[bytes.indexOf("bad-X-name") + 4] = 0xff;188+		const save = await readSave(189+			{ get: async () => bytes, list: async () => [], downloadUrl: () => "" },190+			fx.repo,191+			id,192+		);193+		expect(save.quarantined).toEqual(["bad-\ufffd-name"]);194+		expect((await readTampered(withPath("odd-\ufffd-name"))).quarantined).toEqual([]);195+	});196+197+	test("escapes a name as Koh's escape_path does", () => {198+		expect(["Icon\r", "a\\b", "a\u0085", "\x7f", "plain/ok"].map(escapePath)).toEqual([199+			"Icon\\x0d",200+			"a\\\\b",201+			"a\\u0085",202+			"\\x7f",203+			"plain/ok",204+		]);171205 	});172206 173207 	test("refuses duplicate paths in one record", async () => {

@@ -1 +1 @@1-3.1.51+3.2.2

@@ -1,8 +1,9 @@11 import { Database } from "bun:sqlite";22 import { beforeAll, describe, expect, test } from "bun:test";3-import { cp, mkdir, mkdtemp, writeFile } from "node:fs/promises";3+import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";44 import { tmpdir } from "node:os";55 import { join } from "node:path";6+import { deflateSync } from "node:zlib";67 import { openBucket } from "../src/bucket/reader.ts";78 import { syncAll } from "../src/bucket/sync.ts";89 import {@@ -16,8 +17,9 @@1617 	saveByPrefix,1718 	timeline,1819 } from "../src/mirror/queries.ts";19-import { openMirror } from "../src/mirror/store.ts";20+import { mirrorState, openMirror } from "../src/mirror/store.ts";2021 import { openKeprDb } from "../src/server/db.ts";22+import { Browser, instance } from "./browser.ts";2123 import { type Fixture, makeKohBucket } from "./fixture.ts";2224 2325 const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());@@ -227,4 +229,72 @@227229 228230 	for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();229231 	expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();232+});233+234+test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => {235+	const fx = await makeKohBucket("quarantine");236+	const head = fx.saveIds[0] as string;237+	const key = join(fx.bucketDir, fx.repo, "saves", head);238+	const record = JSON.parse(await readFile(key, "utf8"));239+	record.files.push({ ...record.files[0], path: "Icon\r" });240+	await writeFile(key, JSON.stringify(record));241+	const inst = await instance(fx.bucketDir);242+	const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo);243+	expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({244+		last_sync_error: null,245+	});246+	expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md");247+	expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r");248+249+	await writeFile(join(fx.repoDir, "later.txt"), "later\n");250+	const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };251+	Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env });252+	Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });253+	await syncAll({254+		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),255+		kepr: inst.kepr,256+		mirrorDir: join(inst.dataDir, "mirror"),257+		tarballDir: join(inst.dataDir, "tarballs"),258+	});259+	expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");260+261+	const browser = new Browser(inst.app);262+	await browser.signIn();263+	const page = await (await browser.get("/admin/repos")).text();264+	expect(page).toContain("1 path quarantined: Icon\\x0d");265+});266+267+test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {268+	const fx = await makeKohBucket("oversized");269+	const head = fx.saveIds[0] as string;270+	const key = join(fx.bucketDir, fx.repo, "saves", head);271+	const record = JSON.parse(await readFile(key, "utf8"));272+	const hash = "ab".repeat(32);273+	record.files.push({ path: "huge.bin", hash, executable: false });274+	await writeFile(key, JSON.stringify(record));275+	await writeFile(276+		join(fx.bucketDir, fx.repo, "objects", hash),277+		deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)),278+	);279+	const t = await syncTarget(fx);280+	await syncAll(t);281+	expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({282+		last_sync_error: null,283+	});284+	expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin");285+	expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);286+});287+288+test("the line count skips a file over 8 MiB (A3)", async () => {289+	const fx = await makeKohBucket("large-text");290+	const t = await syncTarget(fx);291+	await syncAll(t);292+	const before = repoStats(t.mirror()).lines;293+	await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024));294+	const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };295+	Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env });296+	Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });297+	await syncAll(t);298+	expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");299+	expect(repoStats(t.mirror()).lines).toBe(before);230300 });