add the read-only bucket reader with the koh-remote format check, verified objects and a koh3-built fixture (v3.0.2)

@@ -0,0 +1,61 @@1+import { readdir } from "node:fs/promises";2+import { join, relative } from "node:path";3+import type { BucketSource } from "../server/config.ts";4+5+/** Read-only view of a Koh 3 remote (§9.1): Kepr needs only these two of its five operations. */6+export type BucketReader = {7+	get(key: string): Promise<Uint8Array | null>;8+	list(prefix: string): Promise<string[]>;9+};10+11+export function openBucket(source: BucketSource): BucketReader {12+	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);13+}14+15+function s3Reader(source: Extract<BucketSource, { kind: "s3" }>): BucketReader {16+	const client = new Bun.S3Client({17+		endpoint: source.endpoint,18+		bucket: source.name,19+		region: source.region,20+		accessKeyId: source.accessKeyId,21+		secretAccessKey: source.secretAccessKey,22+	});23+	return {24+		async get(key) {25+			try {26+				return await client.file(key).bytes();27+			} catch (error) {28+				if ((error as { code?: string }).code === "NoSuchKey") return null;29+				throw error;30+			}31+		},32+		async list(prefix) {33+			const keys: string[] = [];34+			let continuationToken: string | undefined;35+			do {36+				const page = await client.list({ prefix, continuationToken });37+				for (const object of page.contents ?? []) keys.push(object.key);38+				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;39+			} while (continuationToken);40+			return keys;41+		},42+	};43+}44+45+function folderReader(root: string): BucketReader {46+	return {47+		async get(key) {48+			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);49+			const file = Bun.file(join(root, key));50+			return (await file.exists()) ? await file.bytes() : null;51+		},52+		async list(prefix) {53+			const entries = await readdir(root, { recursive: true, withFileTypes: true });54+			return entries55+				.filter((entry) => entry.isFile())56+				.map((entry) => relative(root, join(entry.parentPath, entry.name)))57+				.filter((key) => key.startsWith(prefix))58+				.sort();59+		},60+	};61+}

@@ -0,0 +1,122 @@1+import { inflateSync } from "node:zlib";2+import { blake3 } from "@noble/hashes/blake3.js";3+import { bytesToHex } from "@noble/hashes/utils.js";4+import type { BucketReader } from "./reader.ts";5+6+export type SaveFile = { path: string; hash: string; executable: boolean };7+8+/** The save record Koh 3 writes to `<repo>/saves/<id>` (§9.2). */9+export type SaveRecord = {10+	id: string;11+	parent: string;12+	message: string;13+	created_at: number;14+	machine: string;15+	is_checkpoint: boolean;16+	size_bytes: number;17+	agent: string;18+	session: string;19+	restores: string;20+	replayed_from: string;21+	files: SaveFile[];22+};23+24+export class BucketError extends Error {}25+26+const SAVE_ID = /^[0-9a-f]{12}$/;27+const OBJECT_HASH = /^[0-9a-f]{64}$/;28+const REPO_NAME = /^[A-Za-z0-9._-]+$/;29+const decoder = new TextDecoder();30+31+// ponytail: one full listing per call; switch to a delimiter listing if the bucket grows large.32+export async function listRepos(bucket: BucketReader): Promise<string[]> {33+	const keys = await bucket.list("");34+	return keys35+		.filter((key) => key.endsWith("/head") && key.split("/").length === 2)36+		.map((key) => key.slice(0, -"/head".length))37+		.filter((name) => REPO_NAME.test(name) && name !== "." && name !== "..");38+}39+40+/** Applies §9.2's format rule: a missing key is version 1; anything but `koh-remote 1` is refused. */41+export async function checkFormat(bucket: BucketReader, repo: string): Promise<void> {42+	const bytes = await bucket.get(`${repo}/format`);43+	if (bytes === null) return;44+	const format = decoder.decode(bytes).trim();45+	if (format !== "koh-remote 1") {46+		throw new BucketError(`${repo} uses remote format "${format}"; update Kepr to read it`);47+	}48+}49+50+export async function readHead(bucket: BucketReader, repo: string): Promise<string | null> {51+	const bytes = await bucket.get(`${repo}/head`);52+	if (bytes === null) return null;53+	const head = decoder.decode(bytes).trim();54+	if (!SAVE_ID.test(head)) throw new BucketError(`${repo}/head is not a save ID: "${head}"`);55+	return head;56+}57+58+export async function readSave(59+	bucket: BucketReader,60+	repo: string,61+	id: string,62+): Promise<SaveRecord> {63+	const bytes = await bucket.get(`${repo}/saves/${id}`);64+	if (bytes === null) throw new BucketError(`${repo}/saves/${id} is missing`);65+	const record = parseSaveRecord(JSON.parse(decoder.decode(bytes)));66+	if (record.id !== id) throw new BucketError(`${repo}/saves/${id} holds save ${record.id}`);67+	return record;68+}69+70+/** Fetches an object, inflates it and checks its Blake3 against its key, as steal does (§9.4). */71+export async function readObject(72+	bucket: BucketReader,73+	repo: string,74+	hash: string,75+): Promise<Uint8Array> {76+	if (!OBJECT_HASH.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);77+	const compressed = await bucket.get(`${repo}/objects/${hash}`);78+	if (compressed === null) throw new BucketError(`${repo}/objects/${hash} is missing`);79+	const bytes = new Uint8Array(inflateSync(compressed));80+	if (bytesToHex(blake3(bytes)) !== hash) {81+		throw new BucketError(`${repo}/objects/${hash} does not match its hash`);82+	}83+	return bytes;84+}85+86+function parseSaveRecord(value: unknown): SaveRecord {87+	const r = value as Record<string, unknown>;88+	const strings = [89+		"id",90+		"parent",91+		"message",92+		"machine",93+		"agent",94+		"session",95+		"restores",96+		"replayed_from",97+	];98+	const valid =99+		typeof value === "object" &&100+		value !== null &&101+		strings.every((key) => typeof r[key] === "string") &&102+		SAVE_ID.test(r.id as string) &&103+		typeof r.created_at === "number" &&104+		typeof r.size_bytes === "number" &&105+		typeof r.is_checkpoint === "boolean" &&106+		Array.isArray(r.files) &&107+		r.files.every(isSaveFile);108+	if (!valid) throw new BucketError("save record does not match the Koh 3 schema");109+	return value as SaveRecord;110+}111+112+function isSaveFile(value: unknown): value is SaveFile {113+	const f = value as Record<string, unknown>;114+	return (115+		typeof value === "object" &&116+		value !== null &&117+		typeof f.path === "string" &&118+		typeof f.hash === "string" &&119+		OBJECT_HASH.test(f.hash) &&120+		typeof f.executable === "boolean"121+	);122+}

@@ -0,0 +1,113 @@1+import { beforeAll, describe, expect, test } from "bun:test";2+import { writeFile } from "node:fs/promises";3+import { join } from "node:path";4+import { openBucket } from "../src/bucket/reader.ts";5+import { checkFormat, listRepos, readHead, readObject, readSave } from "../src/bucket/repo.ts";6+import { type Fixture, makeKohBucket } from "./fixture.ts";7+8+let fx: Fixture;9+beforeAll(async () => {10+	fx = await makeKohBucket();11+});12+13+const bucket = () => openBucket({ kind: "folder", path: fx.bucketDir });14+15+describe("reading a koh3-produced folder remote", () => {16+	test("lists the repo", async () => {17+		expect(await listRepos(bucket())).toEqual([fx.repo]);18+	});19+20+	test("reads format, head, the save record and one verified object end to end", async () => {21+		const b = bucket();22+		await checkFormat(b, fx.repo);23+		const head = await readHead(b, fx.repo);24+		expect(head).toBe(fx.saveIds[0] as string);25+26+		const save = await readSave(b, fx.repo, head as string);27+		expect(save.message).toBe("say hello");28+		expect(save.is_checkpoint).toBe(false);29+		const checkpoint = await readSave(b, fx.repo, save.parent);30+		expect(checkpoint.is_checkpoint).toBe(true);31+		const first = await readSave(b, fx.repo, checkpoint.parent);32+		expect([first.agent, first.session]).toEqual(["test-agent", "fixture"]);33+34+		const main = save.files.find((file) => file.path === "src/main.ts");35+		const bytes = await readObject(b, fx.repo, main?.hash as string);36+		expect(new TextDecoder().decode(bytes)).toBe("// Entry point.\nconsole.log('hello');\n");37+	});38+39+	test("a missing key reads as null and a listing is prefix-scoped", async () => {40+		const b = bucket();41+		expect(await b.get(`${fx.repo}/nothing-here`)).toBeNull();42+		const saves = await b.list(`${fx.repo}/saves/`);43+		expect(saves.sort()).toEqual(fx.saveIds.map((id) => `${fx.repo}/saves/${id}`).sort());44+	});45+});46+47+test("the S3 reader pages through listings and reads a missing key as null", async () => {48+	const pageSize = 2;49+	const server = Bun.serve({50+		port: 0,51+		async fetch(req) {52+			const url = new URL(req.url);53+			const folder = bucket();54+			if (url.searchParams.get("list-type") === "2") {55+				const all = await folder.list(url.searchParams.get("prefix") ?? "");56+				const start = Number(url.searchParams.get("continuation-token") ?? 0);57+				const page = all.slice(start, start + pageSize);58+				const more = start + pageSize < all.length;59+				const contents = page.map((key) => `<Contents><Key>${key}</Key></Contents>`).join("");60+				const next = more61+					? `<NextContinuationToken>${start + pageSize}</NextContinuationToken>`62+					: "";63+				return new Response(64+					`<ListBucketResult><IsTruncated>${more}</IsTruncated>${contents}${next}</ListBucketResult>`,65+				);66+			}67+			const bytes = await folder.get(url.pathname.replace("/kohbucket/", ""));68+			if (bytes) return new Response(bytes);69+			return new Response("<Error><Code>NoSuchKey</Code></Error>", { status: 404 });70+		},71+	});72+	try {73+		const s3 = openBucket({74+			kind: "s3",75+			endpoint: `http://localhost:${server.port}`,76+			name: "kohbucket",77+			region: "fr-par",78+			accessKeyId: "read",79+			secretAccessKey: "only",80+		});81+		expect(await s3.list(`${fx.repo}/`)).toEqual(await bucket().list(`${fx.repo}/`));82+		expect(await readHead(s3, fx.repo)).toBe(fx.saveIds[0] as string);83+		expect(await s3.get(`${fx.repo}/format`)).toBeNull();84+	} finally {85+		server.stop();86+	}87+});88+89+describe("the format rule", () => {90+	test("accepts koh-remote 1 and refuses anything else", async () => {91+		const { bucketDir, repo } = await makeKohBucket("formatted");92+		const b = openBucket({ kind: "folder", path: bucketDir });93+		await writeFile(join(bucketDir, repo, "format"), "koh-remote 1");94+		await checkFormat(b, repo);95+		await writeFile(join(bucketDir, repo, "format"), "koh-remote 2");96+		await expect(checkFormat(b, repo)).rejects.toThrow('remote format "koh-remote 2"');97+	});98+});99+100+describe("object verification", () => {101+	test("refuses an object whose content does not match its key", async () => {102+		const b = bucket();103+		const save = await readSave(b, fx.repo, fx.saveIds[0] as string);104+		const [one, two] = save.files;105+		const swapped = {106+			...b,107+			get: (key: string) => b.get(key.replace(one?.hash ?? "", two?.hash ?? "")),108+		};109+		await expect(readObject(swapped, fx.repo, one?.hash as string)).rejects.toThrow(110+			"does not match its hash",111+		);112+	});113+});

@@ -0,0 +1,42 @@1+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";2+import { tmpdir } from "node:os";3+import { join } from "node:path";4+5+export type Fixture = { bucketDir: string; repo: string; repoDir: string; saveIds: string[] };6+7+/** Builds a genuine Koh 3 folder remote with koh3 itself, under a throwaway HOME. */8+export async function makeKohBucket(repo = "demo"): Promise<Fixture> {9+	const root = await mkdtemp(join(tmpdir(), "kepr-fixture-"));10+	const home = join(root, "home");11+	const bucketDir = join(root, "bucket");12+	const repoDir = join(root, "work", repo);13+	await mkdir(home);14+	await mkdir(join(repoDir, "src"), { recursive: true });15+	const koh = (...args: string[]) => run(repoDir, home, args);16+17+	koh("init");18+	koh("remote", "folder", bucketDir);19+	await writeFile(join(repoDir, "README.md"), "# Demo\n\nA small repo for Kepr's tests.\n");20+	await writeFile(join(repoDir, "src/main.ts"), "// Entry point.\nconsole.log('hi');\n");21+	koh("save", "add readme and entry point", "--agent", "test-agent", "--session", "fixture");22+	koh("checkpoint", "before the second change");23+	await writeFile(join(repoDir, "src/main.ts"), "// Entry point.\nconsole.log('hello');\n");24+	koh("save", "say hello");25+	koh("offer");26+27+	const log = JSON.parse(koh("log", "--json")) as { saves: { id: string }[] };28+	return { bucketDir, repo, repoDir, saveIds: log.saves.map((save) => save.id) };29+}30+31+function run(cwd: string, home: string, args: string[]): string {32+	const result = Bun.spawnSync(["koh3", ...args], {33+		cwd,34+		env: { PATH: process.env.PATH, HOME: home, NO_COLOR: "1" },35+	});36+	if (result.exitCode !== 0) {37+		throw new Error(38+			`koh3 ${args.join(" ")} failed: ${result.stderr.toString()}${result.stdout.toString()}`,39+		);40+	}41+	return result.stdout.toString();42+}