serve public repos read-only in Koh's remote layout (v3.0.64)

@@ -0,0 +1,165 @@1+import type { Database } from "bun:sqlite";2+import type { Context, Hono } from "hono";3+import type { EventRecord, SaveFile, SaveRecord } from "../bucket/repo.ts";4+import { openPublicRepo, type PageContext } from "../pages/layout.ts";5+import { filesAt, headId } from "./queries.ts";6+7+const JSON_TYPE = "application/json; charset=UTF-8";8+const TEXT_TYPE = "text/plain; charset=UTF-8";9+// §9.2's name for the layout. Koh trims whitespace before it compares, so no trailing newline is10+// needed to agree with a bucket.11+const FORMAT = "koh-remote 1";12+// A private repo and an unknown one answer alike, to a person and to Koh (§20).13+const NOT_FOUND = '{"error":"repo_not_found","message":"repo not found"}';14+const SAVE_ID = /^[0-9a-f]{12}$/;15+const OBJECT_HASH = /^[0-9a-f]{64}$/;16+// The addendum's lifetimes — a save's record can still be rewritten by a strip, so a minute, and an17+// object never changes once written but a strip can take it away, so a year rather than immutable.18+// `private`, as the pages are: a repo can be turned private, and no shared cache may keep serving19+// what it fetched while the repo was public (§20).20+const SAVE_CACHE = "private, max-age=60";21+const OBJECT_CACHE = "private, max-age=31536000";22+const DAY_CACHE = "private, max-age=86400";23+const NO_CACHE = "no-cache";24+25+type SaveRow = {26+	id: string;27+	parent: string | null;28+	message: string;29+	created_at: number;30+	machine: string;31+	is_checkpoint: number;32+	size_bytes: number;33+	agent: string | null;34+	session: string | null;35+	restores: string | null;36+	replayed_from: string | null;37+};38+39+type EventRow = {40+	id: string;41+	kind: EventRecord["kind"];42+	save_id: string | null;43+	path: string | null;44+	reason: string | null;45+	created_at: number;46+	machine: string;47+};48+49+const SAVE_COLUMNS = `id, parent, message, created_at, machine, is_checkpoint, size_bytes, agent,50+	session, restores, replayed_from`;51+const EVENT_COLUMNS = "id, kind, save_id, path, reason, created_at, machine";52+53+/** The one answer for a private repo, an unknown one, and a record the mirror does not hold. */54+function miss(c: Context): Response {55+	return c.body(NOT_FOUND, 404, { "Content-Type": JSON_TYPE, "Cache-Control": "no-store" });56+}57+58+/**59+ * §9.2's layout, read-only, for the repos a visitor can see: what `koh steal <host>/<repo>`60+ * fetches, so the paths, the bytes and the field names are Koh's, not Kepr's (A2, part 2).61+ */62+export function registerKohRemote(app: Hono, ctx: PageContext): void {63+	const serve = (c: Context, name: string, body: (mirror: Database) => Response): Response => {64+		const repo = openPublicRepo(ctx, name);65+		return repo ? body(repo.mirror) : miss(c);66+	};67+	const json = (c: Context, body: string, cache: string): Response =>68+		c.body(body, 200, { "Content-Type": JSON_TYPE, "Cache-Control": cache });69+	const text = (c: Context, body: string, cache: string): Response =>70+		c.body(body, 200, { "Content-Type": TEXT_TYPE, "Cache-Control": cache });71+72+	app.get("/:repo/koh/format", (c) =>73+		serve(c, c.req.param("repo"), () => text(c, FORMAT, DAY_CACHE)),74+	);75+76+	app.get("/:repo/koh/head", (c) =>77+		serve(c, c.req.param("repo"), (mirror) => {78+			const head = headId(mirror);79+			return head ? text(c, head, NO_CACHE) : miss(c);80+		}),81+	);82+83+	// Checkpoints are served like any other save: the parent links of the saves around them need84+	// them, even though the pages hide them (§21.4).85+	app.get("/:repo/koh/saves/:id", (c) =>86+		serve(c, c.req.param("repo"), (mirror) => {87+			const id = c.req.param("id");88+			if (!SAVE_ID.test(id)) return miss(c);89+			const row = mirror90+				.query<SaveRow, [string]>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE id = ?`)91+				.get(id);92+			if (!row) return miss(c);93+			// Field names and their order are Koh's, and every optional is an empty string, as94+			// `json2.encode` writes it: `koh steal` decodes this straight into its own record.95+			const record: SaveRecord = {96+				id: row.id,97+				parent: row.parent ?? "",98+				message: row.message,99+				created_at: row.created_at,100+				machine: row.machine,101+				is_checkpoint: row.is_checkpoint === 1,102+				size_bytes: row.size_bytes,103+				agent: row.agent ?? "",104+				session: row.session ?? "",105+				restores: row.restores ?? "",106+				replayed_from: row.replayed_from ?? "",107+				// Whatever the strips left: a path they removed is gone from the record too.108+				files: filesAt(mirror, row.id).map(109+					(file): SaveFile => ({ path: file.path, hash: file.hash, executable: file.executable }),110+				),111+			};112+			return json(c, JSON.stringify(record), SAVE_CACHE);113+		}),114+	);115+116+	// The object's stored bytes, zlib and all: `koh steal` inflates them itself and checks the hash.117+	app.get("/:repo/koh/objects/:hash", (c) =>118+		serve(c, c.req.param("repo"), (mirror) => {119+			const hash = c.req.param("hash");120+			if (!OBJECT_HASH.test(hash)) return miss(c);121+			const row = mirror122+				.query<{ data: Uint8Array<ArrayBuffer> }, [string]>(123+					"SELECT data FROM objects WHERE hash = ?",124+				)125+				.get(hash);126+			return row127+				? c.body(row.data, 200, {128+						"Content-Type": "application/octet-stream",129+						"Cache-Control": OBJECT_CACHE,130+					})131+				: miss(c);132+		}),133+	);134+135+	app.get("/:repo/koh/events", (c) =>136+		serve(c, c.req.param("repo"), (mirror) => {137+			const ids = mirror138+				.query<{ id: string }, []>("SELECT id FROM events ORDER BY id")139+				.all()140+				.map((row) => row.id);141+			return text(c, ids.length > 0 ? `${ids.join("\n")}\n` : "", NO_CACHE);142+		}),143+	);144+145+	app.get("/:repo/koh/events/:id", (c) =>146+		serve(c, c.req.param("repo"), (mirror) => {147+			const id = c.req.param("id");148+			if (!SAVE_ID.test(id)) return miss(c);149+			const row = mirror150+				.query<EventRow, [string]>(`SELECT ${EVENT_COLUMNS} FROM events WHERE id = ?`)151+				.get(id);152+			if (!row) return miss(c);153+			const record: EventRecord = {154+				id: row.id,155+				kind: row.kind,156+				save_id: row.save_id ?? "",157+				path: row.path ?? "",158+				reason: row.reason ?? "",159+				created_at: row.created_at,160+				machine: row.machine,161+			};162+			return json(c, JSON.stringify(record), DAY_CACHE);163+		}),164+	);165+}

@@ -0,0 +1,167 @@1+// Covers addendum A2 part 2: Koh's own remote layout, served read-only from a public repo's mirror.2+import { describe, expect, test } from "bun:test";3+import { join } from "node:path";4+import { inflateSync } from "node:zlib";5+import { blake3 } from "@noble/hashes/blake3.js";6+import { bytesToHex } from "@noble/hashes/utils.js";7+import { openBucket } from "../src/bucket/reader.ts";8+import { syncAll } from "../src/bucket/sync.ts";9+import { instance } from "./browser.ts";10+import { makeKohBucket } from "./fixture.ts";11+12+const REPO = "mirror";13+const NOT_FOUND = '{"error":"repo_not_found","message":"repo not found"}';14+const fixture = await makeKohBucket(REPO);15+const inst = await instance(fixture.bucketDir);16+inst.kepr.run("UPDATE repos SET public = 1");17+18+const head = fixture.saveIds[0] ?? "";19+const checkpoint = fixture.saveIds[1] ?? "";20+const oldest = fixture.saveIds[fixture.saveIds.length - 1] ?? "";21+const get = (path: string) => inst.app.request(`/${REPO}/koh/${path}`);22+const record = async (id: string) => JSON.parse(await (await get(`saves/${id}`)).text());23+24+async function resync() {25+	await syncAll({26+		bucket: openBucket(inst.config.bucket),27+		kepr: inst.kepr,28+		mirrorDir: join(inst.dataDir, "mirror"),29+		tarballDir: join(inst.dataDir, "tarballs"),30+	});31+}32+33+describe.serial("Koh's remote layout from the mirror", () => {34+	test("the format, the head and the caches the addendum names", async () => {35+		const format = await get("format");36+		expect(format.status).toBe(200);37+		expect(await format.text()).toBe("koh-remote 1");38+		expect(format.headers.get("cache-control")).toBe("private, max-age=86400");39+		const headResponse = await get("head");40+		expect(headResponse.status).toBe(200);41+		expect(await headResponse.text()).toBe(head);42+		expect(headResponse.headers.get("cache-control")).toBe("no-cache");43+	});44+45+	test("a save record is Koh's, field for field, and checkpoints are part of the chain", async () => {46+		const response = await get(`saves/${head}`);47+		expect(response.status).toBe(200);48+		expect(response.headers.get("cache-control")).toBe("private, max-age=60");49+		const save = await record(head);50+		expect(Object.keys(save)).toEqual([51+			"id",52+			"parent",53+			"message",54+			"created_at",55+			"machine",56+			"is_checkpoint",57+			"size_bytes",58+			"agent",59+			"session",60+			"restores",61+			"replayed_from",62+			"files",63+		]);64+		expect(save.id).toBe(head);65+		expect(save.parent).toBe(checkpoint);66+		// The mirror only keeps columns, so this is the proof that what it re-serves is what Koh67+		// wrote, byte shape and all: empty optionals as "", booleans as booleans, no extra member.68+		const written = await Bun.file(join(fixture.bucketDir, REPO, "saves", head)).json();69+		expect(save).toEqual(written);70+		expect(typeof save.size_bytes).toBe("number");71+		expect(typeof save.is_checkpoint).toBe("boolean");72+		expect(save.agent).toBe("");73+		expect(Object.keys(save.files[0])).toEqual(["path", "hash", "executable"]);74+		// The chain needs checkpoints, so these routes serve what the pages hide.75+		const chain = await get(`saves/${checkpoint}`);76+		expect(chain.status).toBe(200);77+		expect((await record(checkpoint)).is_checkpoint).toBe(true);78+		expect((await inst.app.request(`/${REPO}/saves/${checkpoint}`)).status).toBe(404);79+	});80+81+	test("an object is the stored zlib bytes and hashes back to its key", async () => {82+		const save = await record(head);83+		const file = save.files.find((entry: { path: string }) => entry.path === "src/main.ts");84+		const object = await get(`objects/${file.hash}`);85+		expect(object.status).toBe(200);86+		expect(object.headers.get("cache-control")).toBe("private, max-age=31536000");87+		expect(object.headers.get("cache-control")).not.toContain("immutable");88+		const bytes = new Uint8Array(await object.arrayBuffer());89+		expect(bytesToHex(blake3(inflateSync(bytes)))).toBe(file.hash);90+		expect(new TextDecoder().decode(inflateSync(bytes))).toContain("hello");91+	});92+93+	test("the events are IDs, one per line, and one event is Koh's record", async () => {94+		const events = await get("events");95+		expect(events.status).toBe(200);96+		expect(events.headers.get("cache-control")).toBe("no-cache");97+		const ids = (await events.text()).trim().split("\n");98+		expect(ids.length).toBeGreaterThan(0);99+		const one = await get(`events/${ids[0]}`);100+		expect(one.status).toBe(200);101+		expect(one.headers.get("cache-control")).toBe("private, max-age=86400");102+		const event = JSON.parse(await one.text());103+		expect(Object.keys(event)).toEqual([104+			"id",105+			"kind",106+			"save_id",107+			"path",108+			"reason",109+			"created_at",110+			"machine",111+		]);112+		const written = await Bun.file(join(fixture.bucketDir, REPO, "events", ids[0] ?? "")).json();113+		expect(event).toEqual(written);114+	});115+116+	test("a private repo answers exactly as an unknown one", async () => {117+		inst.kepr.run("UPDATE repos SET public = 0");118+		try {119+			for (const path of [120+				"format",121+				"head",122+				"events",123+				`saves/${head}`,124+				`objects/${"a".repeat(64)}`,125+				`events/${"b".repeat(12)}`,126+			]) {127+				const private_ = await get(path);128+				const unknown = await inst.app.request(`/no-such-repo/koh/${path}`);129+				const mine = `${private_.status} ${await private_.text()}`;130+				expect(mine).toBe(`${unknown.status} ${await unknown.text()}`);131+				expect(mine).toBe(`404 ${NOT_FOUND}`);132+				expect(private_.headers.get("content-type")).toBe(unknown.headers.get("content-type"));133+				expect(private_.headers.get("cache-control")).toBe(unknown.headers.get("cache-control"));134+			}135+		} finally {136+			inst.kepr.run("UPDATE repos SET public = 1");137+		}138+	});139+140+	test("an ID or hash the mirror does not hold answers the same 404", async () => {141+		for (const path of [142+			`saves/${"0".repeat(12)}`,143+			"saves/not-an-id",144+			`objects/${"f".repeat(64)}`,145+		]) {146+			const response = await get(path);147+			expect(`${response.status} ${await response.text()}`).toBe(`404 ${NOT_FOUND}`);148+		}149+	});150+151+	test("a stripped path leaves the record and the object the mirror serves", async () => {152+		const before = await record(oldest);153+		const secret = before.files.find((entry: { path: string }) => entry.path === "secret.txt");154+		expect(secret).toBeDefined();155+		await fixture.strip("secret.txt");156+		await resync();157+		const after = await record(oldest);158+		expect(after.files.some((entry: { path: string }) => entry.path === "secret.txt")).toBe(false);159+		expect((await get(`objects/${secret.hash}`)).status).toBe(404);160+		const events = await (await get("events")).text();161+		const strip = events162+			.trim()163+			.split("\n")164+			.filter((id) => id.length > 0);165+		expect(strip.length).toBeGreaterThan(0);166+	});167+});