read records as Koh does: unpaired surrogate escapes and bytes outside UTF-8 fail a path without refusing the record, names escape byte for byte like Koh's, and strips with illegal paths are ignored; pin koh3 3.2.4 (v3.0.90)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.89",3+	"version": "3.0.90",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -85,7 +85,13 @@8585 // A file name that is safe as one key segment and inside a quoted Content-Disposition filename.8686 const FILE_NAME = /^[^/\\"\p{Cc}]{1,255}$/u;8787 const decoder = new TextDecoder();88-const strictDecoder = new TextDecoder("utf-8", { fatal: true });88+// A run of whole UTF-8 sequences (RFC 3629), or one byte outside them, over a record read one89+// character per byte.90+const UTF8_RUN =91+	/((?:[^\x80-\xff]|[\xc2-\xdf][\x80-\xbf]|\xe0[\xa0-\xbf][\x80-\xbf]|[\xe1-\xec\xee\xef][\x80-\xbf]{2}|\xed[\x80-\x9f][\x80-\xbf]|\xf0[\x90-\xbf][\x80-\xbf]{2}|[\xf1-\xf3][\x80-\xbf]{3}|\xf4[\x80-\x8f][\x80-\xbf]{2})+)|[\s\S]/g;92+// A surrogate pair escape, a lone surrogate escape, or any other escape (`\\` included), whole.93+const JSON_ESCAPE =94+	/\\u[dD][89abAB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2}|\\u[dD][89a-fA-F][0-9a-fA-F]{2}|\\[\s\S]/g;8995 9096 // ponytail: one full listing per call; switch to a delimiter listing if the bucket grows large.9197 export async function listRepos(bucket: BucketReader): Promise<string[]> {@@ -138,30 +144,41 @@138144 	requireSaveId(id);139145 	const bytes = await bucket.get(`${repo}/saves/${id}`);140146 	if (bytes === null) throw new BucketError(`${repo}/saves/${id} is missing`);141-	// Koh reads a record's bytes as they are and refuses a path that is not UTF-8. Decoding turns142-	// such bytes into U+FFFD, so only in a record that fails strict decoding does U+FFFD mark one.143-	// ponytail: a genuine U+FFFD name in that same broken record is quarantined too; Koh never144-	// writes such a record, so byte-exact recovery waits until one is seen.145-	let text: string;146-	let lossy = false;147-	try {148-		text = strictDecoder.decode(bytes);149-	} catch {150-		text = decoder.decode(bytes);151-		lossy = true;152-	}153-	const record = parseSaveRecord(JSON.parse(text));147+	const record = parseSaveRecord(JSON.parse(decodeRecord(bytes)));154148 	if (record.id !== id) throw new BucketError(`${repo}/saves/${id} holds save ${record.id}`);155-	const legal = (path: string) => isSafePath(path) && !(lossy && path.includes("\uFFFD"));156149 	return {157150 		...record,158-		files: record.files.filter((file) => legal(file.path)),151+		files: record.files.filter((file) => isSafePath(file.path)),159152 		quarantined: record.files160-			.filter((file) => !legal(file.path))153+			.filter((file) => !isSafePath(file.path))161154 			.map((file) => escapePath(file.path)),162155 	};163156 }164157 158+/**159+ * A record's text, read as Koh reads it (\u00A75.5): an unpaired surrogate escape stands for its three160+ * bytes, and each byte outside a valid UTF-8 sequence becomes the lone surrogate U+DC00 + byte, so161+ * the path holding it fails `isWellFormed` while a genuine U+FFFD stays legal.162+ */163+function decodeRecord(bytes: Uint8Array): string {164+	const text = Buffer.from(bytes)165+		.toString("latin1")166+		.replace(JSON_ESCAPE, (sequence) => {167+			if (sequence.length !== 6 || sequence[1] !== "u") return sequence;168+			const unit = Number.parseInt(sequence.slice(2), 16);169+			return String.fromCharCode(170+				0xe0 | (unit >> 12),171+				0x80 | ((unit >> 6) & 0x3f),172+				0x80 | (unit & 0x3f),173+			);174+		});175+	return text.replace(UTF8_RUN, (run, valid: string | undefined) =>176+		valid === undefined177+			? String.fromCharCode(0xdc00 + run.charCodeAt(0))178+			: decoder.decode(Buffer.from(valid, "latin1")),179+	);180+}181+165182 /** Fetches an object, inflates it and checks its Blake3 against its key, as steal does (§9.4). */166183 export async function readObject(167184 	bucket: BucketReader,@@ -205,7 +222,7 @@205222 	requireSaveId(id);206223 	const bytes = await bucket.get(`${repo}/events/${id}`);207224 	if (bytes === null) throw new BucketError(`${repo}/events/${id} is missing`);208-	const record = parseEventRecord(JSON.parse(decoder.decode(bytes)));225+	const record = parseEventRecord(JSON.parse(decodeRecord(bytes)));209226 	if (record.id !== id) throw new BucketError(`${repo}/events/${id} holds event ${record.id}`);210227 	return record;211228 }@@ -339,12 +356,11 @@339356 		typeof r.id === "string" &&340357 		SAVE_ID.test(r.id) &&341358 		isOptionalSaveId(r.save_id) &&342-		(r.path === "" || (typeof r.path === "string" && isSafePath(r.path))) &&359+		typeof r.path === "string" &&343360 		// A flag or clear is about one save and no path; a strip names a path, and one save or344-		// (with no ID) all.345-		(r.kind === "strip"346-			? r.path !== ""347-			: (r.kind === "flag" || r.kind === "clear") && r.save_id !== "" && r.path === "") &&361+		// (with no ID) all. Sync ignores a strip whose path is illegal, as Koh does (§5.5).362+		(r.kind === "strip" ||363+			((r.kind === "flag" || r.kind === "clear") && r.save_id !== "" && r.path === "")) &&348364 		typeof r.reason === "string" &&349365 		typeof r.machine === "string" &&350366 		isCount(r.created_at);@@ -373,9 +389,11 @@373389 /** A name printable without its controls reaching the page, as Koh's `escape_path` prints it. */374390 export function escapePath(path: string): string {375391 	return path376-		.replace(/[\\\p{Cc}]/gu, (char) => {392+		.replace(/[\\\p{Cc}\udc80-\udcff]/gu, (char) => {377393 			const code = char.charCodeAt(0);378394 			if (char === "\\") return "\\\\";395+			// A byte `decodeRecord` found outside UTF-8.396+			if (code >= 0xdc80) return `\\x${(code - 0xdc00).toString(16)}`;379397 			const hex = code.toString(16).padStart(2, "0");380398 			return code >= 0x80 ? `\\u00${hex}` : `\\x${hex}`;381399 		})

@@ -18,6 +18,7 @@1818 	checkFormat,1919 	type EventRecord,2020 	escapePath,21+	isSafePath,2122 	listEventIds,2223 	listReleaseVersions,2324 	listRepos,@@ -123,7 +124,10 @@123124 	const known = knownEventIds(mirror);124125 	const events: EventRecord[] = [];125126 	for (const id of await listEventIds(bucket, repo)) {126-		if (!known.has(id)) events.push(await readEvent(bucket, repo, id));127+		if (known.has(id)) continue;128+		const event = await readEvent(bucket, repo, id);129+		// Koh ignores it too: no save can hold an illegal path (§5.5).130+		if (event.kind !== "strip" || isSafePath(event.path)) events.push(event);127131 	}128132 	const releasesLatest = (await readReleasesLatest(bucket, repo)) ?? "";129133 	const moved =

@@ -170,7 +170,6 @@170170 		"src/ma\nin.ts",171171 		"Icon\r",172172 		"a\u0085b",173-		"a\ud800b",174173 	])("quarantines the file path %p", async (path) => {175174 		const save = await readTampered(withPath(path));176175 		expect(save.files.map((file) => file.path)).not.toContain(path);@@ -178,11 +177,14 @@178177 		expect(save.quarantined).toEqual([escapePath(path)]);179178 	});180179 181-	test("quarantines a path that is not valid UTF-8, and keeps a real U+FFFD in a valid record", async () => {180+	test("quarantines bytes outside UTF-8 and unpaired surrogate escapes, and keeps a real U+FFFD", async () => {182181 		const id = fx.saveIds[0] as string;183182 		const real = await bucket().get(`${fx.repo}/saves/${id}`);184183 		const record = JSON.parse(new TextDecoder().decode(real as Uint8Array));185184 		record.files[0].path = "bad-X-name";185+		record.files[1].path = "odd-\ufffd-name";186+		record.files[2].path = "lead-\ud800";187+		record.files[3].path = "pair-\ud83d\ude00";186188 		const bytes = Buffer.from(JSON.stringify(record));187189 		bytes[bytes.indexOf("bad-X-name") + 4] = 0xff;188190 		const save = await readSave(@@ -190,8 +192,9 @@190192 			fx.repo,191193 			id,192194 		);193-		expect(save.quarantined).toEqual(["bad-\ufffd-name"]);194-		expect((await readTampered(withPath("odd-\ufffd-name"))).quarantined).toEqual([]);195+		expect(save.quarantined.sort()).toEqual(["bad-\\xff-name", "lead-\\xed\\xa0\\x80"]);196+		expect(save.files.map((file) => file.path)).toContain("odd-\ufffd-name");197+		expect(save.files.map((file) => file.path)).toContain("pair-\ud83d\ude00");195198 	});196199 197200 	test("escapes a name as Koh's escape_path does", () => {@@ -260,8 +263,7 @@260263 			{ kind: "clear", save_id: "" },261264 			{ kind: "flag", path: "secret.txt" },262265 			{ kind: "clear", path: "secret.txt" },263-			{ kind: "strip", path: "" },264-			{ kind: "strip", save_id: "", path: "" },266+			// A strip with an illegal path, "" included, parses; sync ignores it (sync.test.ts).265267 		]) {266268 			const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));267269 			const tampered = { get: async () => bytes, list: async () => [], downloadUrl: () => "" };

@@ -1 +1 @@1-3.2.31+3.2.4

@@ -17,7 +17,14 @@1717 	"back\\slash.txt",1818 	"ctl\u0001name.txt",1919 	"Icon\r",20+	"c1-\u0085",21+	// JSON.stringify writes this as an unpaired `\ud800` escape.22+	"lone-\ud800",23+	// Its X becomes the byte 0xff below: a name that is not UTF-8 at all.24+	"raw-X-byte",2025 ];26+// Strips that would reach `src/` if either side read their illegal paths loosely.27+const ILLEGAL_STRIPS = ["src//", "", "src\ud800"];2128 2229 test("Koh and Kepr drop the same unsafe paths and keep the same file list for every save (A3)", async () => {2330 	const fx = await makeKohBucket("joint-paths");@@ -26,7 +33,14 @@2633 	const record = JSON.parse(await readFile(key, "utf8"));2734 	const hash = record.files[0].hash;2835 	for (const path of UNSAFE) record.files.push({ path, hash, executable: false });29-	await writeFile(key, JSON.stringify(record));36+	const bytes = Buffer.from(JSON.stringify(record));37+	bytes[bytes.indexOf("raw-X-byte") + 4] = 0xff;38+	await writeFile(key, bytes);39+	for (const [i, path] of ILLEGAL_STRIPS.entries()) {40+		const id = `${i}`.repeat(12);41+		const strip = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };42+		await writeFile(join(fx.bucketDir, fx.repo, "events", id), JSON.stringify(strip));43+	}3044 3145 	const root = dirname(dirname(fx.repoDir));3246 	const home = join(root, "second-home");@@ -58,7 +72,7 @@5872 	const koh = new Database(join(work, fx.repo, ".koh"));5973 	const mirror = openMirror(join(dataDir, "mirror"), fx.repo);6074 	expect(fileLists(koh)).toEqual(fileLists(mirror));61-	expect(fileLists(koh).length).toBeGreaterThan(0);75+	expect(fileLists(koh)).toContainEqual({ save_id: head, path: "src/main.ts" });6276 	// Same verdict and the same escaped name on both sides, name for name.6377 	expect(dropped.length).toBe(UNSAFE.length);6478 	expect([...dropped].sort()).toEqual(quarantinedPaths(mirror));

@@ -264,6 +264,26 @@264264 	expect(page).toContain("1 path quarantined: Icon\\x0d");265265 });266266 267+test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => {268+	const fx = await makeKohBucket("illegal-strip");269+	for (const [id, path] of [270+		["aaaaaaaaaaaa", "src//"],271+		["bbbbbbbbbbbb", ""],272+		["cccccccccccc", "src\ud800"],273+	]) {274+		const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };275+		await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event));276+	}277+	const t = await syncTarget(fx);278+	await syncAll(t);279+	expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({280+		last_sync_error: null,281+	});282+	expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain(283+		"src/main.ts",284+	);285+});286+267287 test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {268288 	const fx = await makeKohBucket("oversized");269289 	const head = fx.saveIds[0] as string;