treat large invalid utf-8 files as binary, check event kinds against their fields, and refuse unknown mirror schema versions (v3.0.8)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.7",3+	"version": "3.0.8",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -165,9 +165,12 @@165165 		value !== null &&166166 		typeof r.id === "string" &&167167 		SAVE_ID.test(r.id) &&168-		["flag", "clear", "strip"].includes(r.kind as string) &&169168 		isOptionalSaveId(r.save_id) &&170169 		(r.path === "" || (typeof r.path === "string" && isSafePath(r.path))) &&170+		// A flag or clear is about one save; a strip names a path, and one save or (with no ID) all.171+		(r.kind === "strip"172+			? r.path !== ""173+			: (r.kind === "flag" || r.kind === "clear") && r.save_id !== "") &&171174 		typeof r.reason === "string" &&172175 		typeof r.machine === "string" &&173176 		isCount(r.created_at);

@@ -104,6 +104,12 @@104104 	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {105105 		user_version: 0,106106 	};107+	if (user_version !== 0 && user_version !== 1) {108+		db.close();109+		throw new Error(110+			`${path} has mirror schema version ${user_version}; this Kepr reads version 1. The mirror is disposable: delete it and the next sync rebuilds it from the bucket.`,111+		);112+	}107113 	if (user_version === 0) {108114 		db.transaction(() => {109115 			db.run(SCHEMA);

@@ -72,11 +72,14 @@7272 function isText(bytes: Uint8Array): boolean {7373 	if (bytes.subarray(0, 8000).includes(0)) return false;7474 	try {75-		new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, SHOWN_BYTES));75+		// Streaming mode tolerates only a character cut at the 1 MB mark; invalid bytes still throw.76+		const cut = bytes.length > SHOWN_BYTES;77+		new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, SHOWN_BYTES), {78+			stream: cut,79+		});7680 		return true;7781 	} catch {78-		// A multi-byte character cut at the 1 MB mark is still text.79-		return bytes.length > SHOWN_BYTES;82+		return false;8083 	}8184 }8285 

@@ -197,7 +197,19 @@197197 			fx.saveIds[3] as string,198198 			"holds a secret",199199 		]);200-		for (const change of [{ kind: "delete" }, { path: "../x" }, { save_id: "nope" }]) {200+		const stripAll = { kind: "strip", save_id: "", path: "secret.txt" };201+		const accepted = new TextEncoder().encode(JSON.stringify({ ...event, ...stripAll }));202+		const reader = { get: async () => accepted, list: async () => [] };203+		expect((await readEvent(reader, fx.repo, event.id)).kind).toBe("strip");204+		for (const change of [205+			{ kind: "delete" },206+			{ path: "../x" },207+			{ save_id: "nope" },208+			{ kind: "flag", save_id: "" },209+			{ kind: "clear", save_id: "" },210+			{ kind: "strip", path: "" },211+			{ kind: "strip", save_id: "", path: "" },212+		]) {201213 			const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));202214 			const tampered = { get: async () => bytes, list: async () => [] };203215 			await expect(readEvent(tampered, fx.repo, event.id)).rejects.toThrow("Koh 3 schema");

@@ -1,5 +1,5 @@11 import { beforeAll, describe, expect, test } from "bun:test";2-import { mkdtemp } from "node:fs/promises";2+import { mkdtemp, writeFile } from "node:fs/promises";33 import { tmpdir } from "node:os";44 import { join } from "node:path";55 import type { Hono } from "hono";@@ -20,15 +20,27 @@2020 beforeAll(async () => {2121 	fx = await makeKohBucket();2222 	const secretRepo = await makeKohBucket("hidden");23+	const large = await makeKohBucket("large");24+	// Over 1 MB each: text with a two-byte "é" split by the 1 MB mark, and invalid UTF-8 with no NUL.25+	await writeFile(26+		join(large.repoDir, "big.txt"),27+		`${"a".repeat(1024 * 1024 - 1)}é${"b".repeat(1000)}\n`,28+	);29+	await writeFile(join(large.repoDir, "big.bin"), new Uint8Array(1024 * 1024 + 500).fill(0xff));30+	Bun.spawnSync(["koh3", "save", "add large files"], {31+		cwd: large.repoDir,32+		env: { PATH: process.env.PATH, HOME: join(large.repoDir, "../../home"), NO_COLOR: "1" },33+	});2334 	const dataDir = await mkdtemp(join(tmpdir(), "kepr-pages-"));2435 	mirrorDir = join(dataDir, "mirror");2536 	const kepr = openKeprDb(dataDir);26-	for (const bucketDir of [fx.bucketDir, secretRepo.bucketDir]) {37+	for (const bucketDir of [fx.bucketDir, secretRepo.bucketDir, large.bucketDir]) {2738 		await syncAll({ bucket: openBucket({ kind: "folder", path: bucketDir }), kepr, mirrorDir });2839 	}2940 	kepr.run(3041 		"UPDATE repos SET public = 1, install1 = 'nix profile add x', homepage = 'https://demo.example' WHERE name = 'demo'",3142 	);43+	kepr.run("UPDATE repos SET public = 1 WHERE name = 'large'");3244 	app = createApp(3345 		{3446 			domain: "kepr.example",@@ -127,6 +139,25 @@127139 128140 	test("an image displays inline from its raw file", async () => {129141 		expect(await body("/demo/files/docs/logo.png")).toContain('<img src="/demo/raw/docs/logo.png"');142+	});143+});144+145+describe("files over 1 MB", () => {146+	test("text shows its first 1 MB with a raw link, even with a character cut at the mark", async () => {147+		const page = await body("/large/files/big.txt");148+		expect(page).toContain("showing the first 1 MB");149+		expect(page).toContain('href="/large/raw/big.txt"');150+		expect(page).not.toContain("bbb");151+		const raw = await get("/large/raw/big.txt");152+		expect(raw.headers.get("content-type")).toBe("text/plain; charset=utf-8");153+	});154+155+	test("invalid UTF-8 without a NUL is binary, not text", async () => {156+		const page = await body("/large/files/big.bin");157+		expect(page).toContain("binary file");158+		expect(page).not.toContain('class="code"');159+		const raw = await get("/large/raw/big.bin");160+		expect(raw.headers.get("content-type")).toBe("application/octet-stream");130161 	});131162 });132163 

@@ -1,3 +1,4 @@1+import { Database } from "bun:sqlite";12 import { beforeAll, describe, expect, test } from "bun:test";23 import { mkdtemp, writeFile } from "node:fs/promises";34 import { tmpdir } from "node:os";@@ -122,4 +123,21 @@122123 		last_sync_error: string;123124 	};124125 	expect(row.last_sync_error).toContain('remote format "koh-remote 2"');126+});127+128+describe("the mirror's schema version", () => {129+	test("a version 1 mirror reopens and any other version is refused as disposable", async () => {130+		const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));131+		const created = openMirror(dir, "one");132+		created.run("PRAGMA wal_checkpoint(TRUNCATE)");133+		await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));134+		expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({135+			user_version: 1,136+		});137+138+		const future = new Database(join(dir, "future.db"));139+		future.run("PRAGMA user_version = 2");140+		future.close();141+		expect(() => openMirror(dir, "future")).toThrow("disposable");142+	});125143 });