serve a public repo's whole Koh layout in one streamed /koh/bundle response (v3.0.210)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.209",3+	"version": "3.0.210",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -88,27 +88,7 @@8888 			const row = mirror8989 				.query<SaveRow, [string]>(`SELECT ${SAVE_COLUMNS} FROM saves WHERE id = ?`)9090 				.get(id);91-			if (!row) return miss(c);92-			// Field names and their order are Koh's, and every optional is an empty string, as93-			// `json2.encode` writes it: `koh steal` decodes this straight into its own record.94-			const record: SaveRecord = {95-				id: row.id,96-				parent: row.parent ?? "",97-				message: row.message,98-				created_at: row.created_at,99-				machine: row.machine,100-				is_checkpoint: row.is_checkpoint === 1,101-				size_bytes: row.size_bytes,102-				agent: row.agent ?? "",103-				session: row.session ?? "",104-				restores: row.restores ?? "",105-				replayed_from: row.replayed_from ?? "",106-				// Whatever the strips left: a path they removed is gone from the record too.107-				files: filesAt(mirror, row.id).map(108-					(file): SaveFile => ({ path: file.path, hash: file.hash, executable: file.executable }),109-				),110-			};111-			return json(c, JSON.stringify(record), SAVE_CACHE);91+			return row ? json(c, saveRecord(mirror, row), SAVE_CACHE) : miss(c);11292 		}),11393 	);11494 @@ -178,17 +158,84 @@178158 			const row = mirror179159 				.query<EventRow, [string]>(`SELECT ${EVENT_COLUMNS} FROM events WHERE id = ?`)180160 				.get(id);181-			if (!row) return miss(c);182-			const record: EventRecord = {183-				id: row.id,184-				kind: row.kind,185-				save_id: row.save_id ?? "",186-				path: row.path ?? "",187-				reason: row.reason ?? "",188-				created_at: row.created_at,189-				machine: row.machine,190-			};191-			return json(c, JSON.stringify(record), DAY_CACHE);161+			return row ? json(c, eventRecord(row), DAY_CACHE) : miss(c);192162 		}),193163 	);164+165+	// The whole layout in one response, so a steal costs one request rather than one per key: a166+	// 2,071-save repo took an hour of round trips. Streamed, because it is as big as the repo.167+	app.get("/:repo/koh/bundle", (c) =>168+		serve(c, c.req.param("repo"), (mirror) =>169+			c.body(stream(bundle(mirror)), 200, {170+				"Content-Type": "application/octet-stream",171+				"Cache-Control": NO_CACHE,172+			}),173+		),174+	);175+}176+177+function stream(chunks: Generator<Uint8Array>): ReadableStream<Uint8Array> {178+	return new ReadableStream({179+		pull(controller) {180+			const next = chunks.next();181+			if (next.done) controller.close();182+			else controller.enqueue(next.value);183+		},184+	});185+}186+187+/** Each entry is `<key> <length>\n` and then that many bytes, keyed as the single routes are. */188+function* bundle(mirror: Database): Generator<Uint8Array> {189+	const encoder = new TextEncoder();190+	function* entry(key: string, bytes: Uint8Array): Generator<Uint8Array> {191+		yield encoder.encode(`${key} ${bytes.length}\n`);192+		yield bytes;193+	}194+	for (const row of mirror.query<SaveRow, []>(`SELECT ${SAVE_COLUMNS} FROM saves`).iterate()) {195+		yield* entry(`saves/${row.id}`, encoder.encode(saveRecord(mirror, row)));196+	}197+	for (const row of mirror.query<EventRow, []>(`SELECT ${EVENT_COLUMNS} FROM events`).iterate()) {198+		yield* entry(`events/${row.id}`, encoder.encode(eventRecord(row)));199+	}200+	const objects = mirror.query<{ hash: string; data: Uint8Array }, []>(201+		`SELECT hash, data FROM objects o202+		WHERE EXISTS (SELECT 1 FROM save_files f WHERE f.hash = o.hash)`,203+	);204+	for (const row of objects.iterate()) yield* entry(`objects/${row.hash}`, row.data);205+}206+207+// Field names and their order are Koh's, and every optional is an empty string, as `json2.encode`208+// writes it: `koh steal` decodes this straight into its own record.209+function saveRecord(mirror: Database, row: SaveRow): string {210+	const record: SaveRecord = {211+		id: row.id,212+		parent: row.parent ?? "",213+		message: row.message,214+		created_at: row.created_at,215+		machine: row.machine,216+		is_checkpoint: row.is_checkpoint === 1,217+		size_bytes: row.size_bytes,218+		agent: row.agent ?? "",219+		session: row.session ?? "",220+		restores: row.restores ?? "",221+		replayed_from: row.replayed_from ?? "",222+		// Whatever the strips left: a path they removed is gone from the record too.223+		files: filesAt(mirror, row.id).map(224+			(file): SaveFile => ({ path: file.path, hash: file.hash, executable: file.executable }),225+		),226+	};227+	return JSON.stringify(record);228+}229+230+function eventRecord(row: EventRow): string {231+	const record: EventRecord = {232+		id: row.id,233+		kind: row.kind,234+		save_id: row.save_id ?? "",235+		path: row.path ?? "",236+		reason: row.reason ?? "",237+		created_at: row.created_at,238+		machine: row.machine,239+	};240+	return JSON.stringify(record);194241 }

@@ -124,6 +124,38 @@124124 		expect(event).toEqual(written);125125 	});126126 127+	test("the bundle holds every save, event and object, byte for byte as the single routes", async () => {128+		const response = await get("bundle");129+		expect(response.status).toBe(200);130+		expect(response.headers.get("cache-control")).toBe("no-cache");131+		const body = new Uint8Array(await response.arrayBuffer());132+		const entries = new Map<string, Uint8Array>();133+		let at = 0;134+		while (at < body.length) {135+			const newline = body.indexOf(10, at);136+			const [key = "", length = ""] = new TextDecoder()137+				.decode(body.subarray(at, newline))138+				.split(" ");139+			at = newline + 1 + Number(length);140+			entries.set(key, body.subarray(newline + 1, at));141+		}142+		const listed = async (prefix: string) =>143+			(await (await get(prefix)).text())144+				.trim()145+				.split("\n")146+				.map((id) => `${prefix}/${id}`);147+		const keys = [148+			...(await listed("saves")),149+			...(await listed("events")),150+			...(await listed("objects")),151+		];152+		expect([...entries.keys()].sort()).toEqual(keys.sort());153+		for (const key of keys) {154+			const single = new Uint8Array(await (await get(key)).arrayBuffer());155+			expect(entries.get(key)).toEqual(single);156+		}157+	});158+127159 	test("a private repo answers exactly as an unknown one", async () => {128160 		inst.kepr.run("UPDATE repos SET public = 0");129161 		try {@@ -133,6 +165,7 @@133165 				"events",134166 				"saves",135167 				"objects",168+				"bundle",136169 				`saves/${head}`,137170 				`objects/${"a".repeat(64)}`,138171 				`events/${"b".repeat(12)}`,