bucket.test.ts

1import { beforeAll, describe, expect, test } from "bun:test";2import { writeFile } from "node:fs/promises";3import { join } from "node:path";4import { openBucket } from "../src/bucket/reader.ts";5import {6	checkFormat,7	escapePath,8	listEventIds,9	listRepos,10	readEvent,11	readHead,12	readObject,13	readSave,14} from "../src/bucket/repo.ts";15import { type Fixture, makeKohBucket } from "./fixture.ts";1617let fx: Fixture;18beforeAll(async () => {19	fx = await makeKohBucket();20});2122const bucket = () => openBucket({ kind: "folder", path: fx.bucketDir });2324describe("reading a koh3-produced folder remote", () => {25	test("lists the repo", async () => {26		expect(await listRepos(bucket())).toEqual([fx.repo]);27	});2829	test("reads format, head, the save record and one verified object end to end", async () => {30		const b = bucket();31		await checkFormat(b, fx.repo);32		const head = await readHead(b, fx.repo);33		expect(head).toBe(fx.saveIds[0] as string);3435		const save = await readSave(b, fx.repo, head as string);36		expect(save.message).toBe("say hello");37		expect(save.is_checkpoint).toBe(false);38		const checkpoint = await readSave(b, fx.repo, save.parent);39		expect(checkpoint.is_checkpoint).toBe(true);40		const first = await readSave(b, fx.repo, checkpoint.parent);41		expect([first.agent, first.session]).toEqual(["test-agent", "fixture"]);4243		const main = save.files.find((file) => file.path === "src/main.ts");44		const { bytes } = await readObject(b, fx.repo, main?.hash as string);45		expect(new TextDecoder().decode(bytes)).toBe("// Entry point.\nconsole.log('hello');\n");46	});4748	test("a missing key reads as null and a listing is prefix-scoped", async () => {49		const b = bucket();50		expect(await b.get(`${fx.repo}/nothing-here`)).toBeNull();51		const saves = await b.list(`${fx.repo}/saves/`);52		expect(saves.sort()).toEqual(fx.saveIds.map((id) => `${fx.repo}/saves/${id}`).sort());53	});54});5556test("the S3 reader pages through listings with sizes and reads a missing key as null", async () => {57	const pageSize = 2;58	const server = Bun.serve({59		port: 0,60		async fetch(req) {61			const url = new URL(req.url);62			const folder = bucket();63			if (url.searchParams.get("list-type") === "2") {64				const sizes = await folder.sizes(url.searchParams.get("prefix") ?? "");65				const all = [...sizes.keys()];66				const start = Number(url.searchParams.get("continuation-token") ?? 0);67				const page = all.slice(start, start + pageSize);68				const more = start + pageSize < all.length;69				const contents = page70					.map((key) => `<Contents><Key>${key}</Key><Size>${sizes.get(key)}</Size></Contents>`)71					.join("");72				const next = more73					? `<NextContinuationToken>${start + pageSize}</NextContinuationToken>`74					: "";75				return new Response(76					`<ListBucketResult><IsTruncated>${more}</IsTruncated>${contents}${next}</ListBucketResult>`,77				);78			}79			const bytes = await folder.get(url.pathname.replace("/kohbucket/", ""));80			if (bytes) return new Response(bytes);81			return new Response("<Error><Code>NoSuchKey</Code></Error>", { status: 404 });82		},83	});84	try {85		const s3 = openBucket({86			kind: "s3",87			endpoint: `http://localhost:${server.port}`,88			name: "kohbucket",89			region: "fr-par",90			accessKeyId: "read",91			secretAccessKey: "only",92		});93		expect(await s3.list(`${fx.repo}/`)).toEqual(await bucket().list(`${fx.repo}/`));94		const sizes = await s3.sizes(`${fx.repo}/`);95		expect(sizes).toEqual(await bucket().sizes(`${fx.repo}/`));96		expect(sizes.get(`${fx.repo}/head`)).toBe(12);97		expect(await readHead(s3, fx.repo)).toBe(fx.saveIds[0] as string);98		// koh3 writes `<repo>/format` on its first offer (§9.2), so it is present;99		// a key that is not there is what reads as null.100		const format = await s3.get(`${fx.repo}/format`);101		expect(format === null ? null : new TextDecoder().decode(format)).toBe("koh-remote 1");102		expect(await s3.get(`${fx.repo}/no-such-key`)).toBeNull();103		const url = new URL(104			s3.downloadUrl("demo/releases/1.0/android-arm64/a b.apk", "a b.apk", "t/x"),105		);106		expect(url.pathname).toBe("/kohbucket/demo/releases/1.0/android-arm64/a%20b.apk");107		expect(url.searchParams.get("X-Amz-Expires")).toBe("300");108		expect(url.searchParams.get("response-content-disposition")).toBe(109			'attachment; filename="a b.apk"',110		);111		expect(url.searchParams.get("response-content-type")).toBe("t/x");112	} finally {113		server.stop();114	}115});116117describe("the format rule", () => {118	test("accepts koh-remote 1 and refuses anything else", async () => {119		const { bucketDir, repo } = await makeKohBucket("formatted");120		const b = openBucket({ kind: "folder", path: bucketDir });121		await writeFile(join(bucketDir, repo, "format"), "koh-remote 1");122		await checkFormat(b, repo);123		await writeFile(join(bucketDir, repo, "format"), "koh-remote 2");124		await expect(checkFormat(b, repo)).rejects.toThrow('remote format "koh-remote 2"');125	});126});127128describe("object verification", () => {129	test("refuses an object whose content does not match its key", async () => {130		const b = bucket();131		const save = await readSave(b, fx.repo, fx.saveIds[0] as string);132		const [one, two] = save.files;133		const swapped = {134			...b,135			get: (key: string) => b.get(key.replace(one?.hash ?? "", two?.hash ?? "")),136		};137		await expect(readObject(swapped, fx.repo, one?.hash as string)).rejects.toThrow(138			"does not match its hash",139		);140	});141});142143describe("untrusted input", () => {144	// Starts from a genuine koh3 record and corrupts one field, as a tampered bucket would.145	async function readTampered(change: (record: Record<string, unknown>) => void) {146		const id = fx.saveIds[0] as string;147		const real = await bucket().get(`${fx.repo}/saves/${id}`);148		const record = JSON.parse(new TextDecoder().decode(real as Uint8Array));149		change(record);150		const bytes = new TextEncoder().encode(JSON.stringify(record));151		return readSave(152			{153				get: async () => bytes,154				list: async () => [],155				sizes: async () => new Map(),156				downloadUrl: () => "",157				viewUrl: () => null,158				put: async () => {},159				remove: async () => {},160				copy: async () => {},161				removeRenamed: async () => {},162			},163			fx.repo,164			id,165		);166	}167	const withPath = (path: string) => (record: Record<string, unknown>) => {168		(record.files as { path: string }[])[0] = { ...(record.files as { path: string }[])[0], path };169	};170171	test("the genuine record is accepted", async () => {172		expect((await readTampered(() => {})).message).toBe("say hello");173	});174175	// §5.5: a path Koh would refuse is quarantined, and the rest of the save still lands.176	test.each([177		"",178		"/etc/passwd",179		"../escape",180		"src/../../escape",181		"./src/main.ts",182		"src//main.ts",183		"src/",184		"src\\main.ts",185		"src/ma\0in.ts",186		"src/ma\nin.ts",187		"Icon\r",188		"a\u0085b",189	])("quarantines the file path %p", async (path) => {190		const save = await readTampered(withPath(path));191		expect(save.files.map((file) => file.path)).not.toContain(path);192		expect(save.files.length).toBeGreaterThan(0);193		expect(save.quarantined).toEqual([escapePath(path)]);194	});195196	test("quarantines bytes outside UTF-8 and unpaired surrogate escapes, and keeps a real U+FFFD", async () => {197		const id = fx.saveIds[0] as string;198		const real = await bucket().get(`${fx.repo}/saves/${id}`);199		const record = JSON.parse(new TextDecoder().decode(real as Uint8Array));200		record.files[0].path = "bad-X-name";201		record.files[1].path = "odd-\ufffd-name";202		record.files[2].path = "lead-\ud800";203		record.files[3].path = "pair-\ud83d\ude00";204		const bytes = Buffer.from(JSON.stringify(record));205		bytes[bytes.indexOf("bad-X-name") + 4] = 0xff;206		const save = await readSave(207			{208				get: async () => bytes,209				list: async () => [],210				sizes: async () => new Map(),211				downloadUrl: () => "",212				viewUrl: () => null,213				put: async () => {},214				remove: async () => {},215				copy: async () => {},216				removeRenamed: async () => {},217			},218			fx.repo,219			id,220		);221		expect(save.quarantined.sort()).toEqual(["bad-\\xff-name", "lead-\\xed\\xa0\\x80"]);222		expect(save.files.map((file) => file.path)).toContain("odd-\ufffd-name");223		expect(save.files.map((file) => file.path)).toContain("pair-\ud83d\ude00");224	});225226	test("escapes a name as Koh's escape_path does", () => {227		expect(["Icon\r", "a\\b", "a\u0085", "\x7f", "plain/ok"].map(escapePath)).toEqual([228			"Icon\\x0d",229			"a\\\\b",230			"a\\u0085",231			"\\x7f",232			"plain/ok",233		]);234	});235236	test("refuses duplicate paths in one record", async () => {237		const dupe = (record: Record<string, unknown>) => {238			const files = record.files as { path: string }[];239			files[1] = { ...files[1], path: (files[0] as { path: string }).path } as { path: string };240		};241		await expect(readTampered(dupe)).rejects.toThrow("Koh 3 schema");242	});243244	test("accepts the six-character IDs that Koh 2's early saves keep through migration", async () => {245		expect((await readTampered((record) => (record.parent = "af96c8"))).parent).toBe("af96c8");246	});247248	test.each([249		["parent", "not-an-id"],250		["parent", "af96c"],251		["parent", "af96c8a"],252		["restores", "ABCDEF123456"],253		["replayed_from", "../../../x"],254		["created_at", -1],255		["created_at", 1.5],256		["size_bytes", -10],257		["size_bytes", "12"],258	])("refuses %s = %p", async (field, value) => {259		await expect(readTampered((record) => (record[field] = value))).rejects.toThrow("Koh 3 schema");260	});261262	test("refuses bad repo names and save IDs before building a key", async () => {263		const b = bucket();264		for (const repo of ["..", "a/b", "", "../demo"]) {265			await expect(checkFormat(b, repo)).rejects.toThrow("not a repo name");266			await expect(readHead(b, repo)).rejects.toThrow("not a repo name");267			await expect(readSave(b, repo, fx.saveIds[0] as string)).rejects.toThrow("not a repo name");268			await expect(readObject(b, repo, "0".repeat(64))).rejects.toThrow("not a repo name");269		}270		for (const id of ["../head", "abc", "ABCDEF123456", `${fx.saveIds[0]}0`]) {271			await expect(readSave(b, fx.repo, id)).rejects.toThrow("not a save ID");272		}273	});274});275276describe("event records", () => {277	test("reads a genuine koh3 flag event and refuses a tampered one", async () => {278		const b = bucket();279		const [id] = await listEventIds(b, fx.repo);280		const event = await readEvent(b, fx.repo, id as string);281		expect([event.kind, event.save_id, event.reason]).toEqual([282			"flag",283			fx.saveIds[3] as string,284			"holds a secret",285		]);286		const stripAll = { kind: "strip", save_id: "", path: "secret.txt" };287		const accepted = new TextEncoder().encode(JSON.stringify({ ...event, ...stripAll }));288		const reader = {289			get: async () => accepted,290			list: async () => [],291			sizes: async () => new Map(),292			downloadUrl: () => "",293			viewUrl: () => null,294			put: async () => {},295			remove: async () => {},296			copy: async () => {},297			removeRenamed: async () => {},298		};299		expect((await readEvent(reader, fx.repo, event.id)).kind).toBe("strip");300		for (const change of [301			{ kind: "delete" },302			{ path: "../x" },303			{ save_id: "nope" },304			{ kind: "flag", save_id: "" },305			{ kind: "clear", save_id: "" },306			{ kind: "flag", path: "secret.txt" },307			{ kind: "clear", path: "secret.txt" },308			// A strip with an illegal path, "" included, parses; sync ignores it (sync.test.ts).309		]) {310			const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));311			const tampered = {312				get: async () => bytes,313				list: async () => [],314				sizes: async () => new Map(),315				downloadUrl: () => "",316				viewUrl: () => null,317				put: async () => {},318				remove: async () => {},319				copy: async () => {},320				removeRenamed: async () => {},321			};322			await expect(readEvent(tampered, fx.repo, event.id)).rejects.toThrow("Koh 3 schema");323		}324	});325});