reader.ts

1import { readdir, unlink } from "node:fs/promises";2import { join, relative } from "node:path";3import { pathToFileURL } from "node:url";4import type { BucketSource } from "../server/config.ts";56/**7 * A Koh 3 remote as Kepr uses it (§9.1): read-only on Koh's data, which needs two of its five8 * operations, plus a short-lived download URL so Kepr never streams release files itself (§22.2).9 * Kepr's own files (uploads) live under `KEPR_PREFIX` (June, 2026-09-29); no Koh repo can have10 * that name, since Kepr and Koh both need a `head` for one. Beyond it Kepr touches only builds,11 * which Koh has no way to delete (June, 2026-09-30): it deletes a version's files, rewrites its12 * record without a deleted one, and moves `releases/latest`.13 */14export type Bucket = {15	get(key: string): Promise<Uint8Array | null>;16	list(prefix: string): Promise<string[]>;17	/** The top-level folders' names: one listing, however many keys they hold. */18	folders(): Promise<string[]>;19	/** A key's size in bytes, asked without reading it. */20	size(key: string): Promise<number>;21	/** Every key under `prefix` with its size in bytes, for the console's storage view. */22	sizes(prefix: string): Promise<Map<string, number>>;23	/** A URL that serves `key` for five minutes, as an attachment with this filename and content type. */24	downloadUrl(key: string, filename: string, type: string): string;25	/** A five-minute URL that shows `key` in the page, or null when Kepr must serve the bytes itself. */26	viewUrl(key: string, type: string): string | null;27	/**28	 * Writes one of Kepr's own files, a repo's `releases/latest`, or a version's `release.json` when29	 * one of its files is deleted; any other key is refused.30	 */31	put(key: string, bytes: Uint8Array, type: string): Promise<void>;32	/** Deletes one of Kepr's own files, a release's file or `releases/latest`; nothing else. */33	remove(key: string): Promise<void>;34	/** Copies a key of one repo to another repo: a rename (June, 2026-10-01), and nothing else. */35	copy(from: string, to: string): Promise<void>;36	/** Deletes a key of a repo that a rename has copied to its new name. */37	removeRenamed(key: string): Promise<void>;38};3940export const KEPR_PREFIX = ".kepr/";4142/** `each` over `items`, `atOnce` of them in flight at a time; the results in order. */43export async function inTurn<T, R>(44	items: T[],45	atOnce: number,46	each: (item: T) => Promise<R>,47): Promise<R[]> {48	const results: R[] = [];49	for (let i = 0; i < items.length; i += atOnce) {50		results.push(...(await Promise.all(items.slice(i, i + atOnce).map(each))));51	}52	return results;53}5455/** A write Kepr can't make: outside its folder, or refused by the bucket. */56export class BucketWriteError extends Error {}5758const DOWNLOAD_SECONDS = 5 * 60;5960const RELEASE_WRITE = /^[^/]+\/releases\/(latest|[^/]+\/release\.json)$/;61const RELEASE_FILE = /^[^/]+\/releases\/(latest|[^/]+\/.+)$/;6263// A key inside a repo, as a rename moves them: never Kepr's own folder, never `..` or empty parts.64function requireRepoKey(key: string): void {65	if (66		key.startsWith(KEPR_PREFIX) ||67		!key.includes("/") ||68		key.split("/").some((p) => p === ".." || p === "")69	) {70		throw new BucketWriteError(`a rename moves only a repo's keys, not "${key}"`);71	}72}7374function requireWritable(key: string, releaseKey: RegExp): void {75	const allowed = key.startsWith(KEPR_PREFIX) || releaseKey.test(key);76	if (!allowed || key.split("/").some((part) => part === ".." || part === "")) {77		throw new BucketWriteError(`Kepr writes only under ${KEPR_PREFIX} and builds, not "${key}"`);78	}79}8081export function openBucket(source: BucketSource): Bucket {82	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);83}8485type S3Source = Extract<BucketSource, { kind: "s3" }>;8687/**88 * The console's connection test (§24), made before settings are saved: the key must list the89 * bucket and write Kepr's own test file. Returns what failed, or null.90 */91export async function checkS3Access(source: S3Source): Promise<string | null> {92	const reason = (error: unknown) => (error instanceof Error ? error.message : String(error));93	try {94		// One key is proof enough; listing the whole bucket could take minutes.95		await s3Client(source).list({ maxKeys: 1 });96	} catch (error) {97		return `The key cannot list the bucket: ${reason(error)}`;98	}99	try {100		const stamp = new TextEncoder().encode(new Date().toISOString());101		await s3Reader(source).put(`${KEPR_PREFIX}connection-test`, stamp, "text/plain");102	} catch (error) {103		return `The key cannot write ${KEPR_PREFIX}connection-test: ${reason(error)}`;104	}105	return null;106}107108function s3Client(source: S3Source): Bun.S3Client {109	return new Bun.S3Client({110		endpoint: source.endpoint,111		bucket: source.name,112		region: source.region,113		accessKeyId: source.accessKeyId,114		secretAccessKey: source.secretAccessKey,115	});116}117118function s3Reader(source: S3Source): Bucket {119	const client = s3Client(source);120	// A refused write is a BucketWriteError, which the console shows and a tarball cache skips.121	const write = async (key: string, run: (file: Bun.S3File) => Promise<unknown>) => {122		try {123			await run(client.file(key));124		} catch (error) {125			throw new BucketWriteError(`The bucket refused ${key}: ${error}`);126		}127	};128	return {129		async get(key) {130			// Through a presigned URL: Bun 1.3.14's S3File reads keep about one copy of every body131			// read, which a first sync of a big repo turns into hundreds of MB; fetch frees its own.132			const response = await fetch(client.presign(key, { method: "GET", expiresIn: 60 }));133			if (response.status === 404) return null;134			if (!response.ok) throw new Error(`S3 GET ${key} failed: HTTP ${response.status}`);135			return new Uint8Array(await response.arrayBuffer());136		},137		async list(prefix) {138			const keys: string[] = [];139			let continuationToken: string | undefined;140			do {141				const page = await client.list({ prefix, continuationToken });142				for (const object of page.contents ?? []) keys.push(object.key);143				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;144			} while (continuationToken);145			return keys;146		},147		size: (key) => client.size(key),148		async folders() {149			const names: string[] = [];150			let continuationToken: string | undefined;151			do {152				const page = await client.list({ delimiter: "/", continuationToken });153				for (const { prefix } of page.commonPrefixes ?? []) names.push(prefix.slice(0, -1));154				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;155			} while (continuationToken);156			return names;157		},158		async sizes(prefix) {159			const sizes = new Map<string, number>();160			let continuationToken: string | undefined;161			do {162				const page = await client.list({ prefix, continuationToken });163				for (const object of page.contents ?? []) sizes.set(object.key, object.size ?? 0);164				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;165			} while (continuationToken);166			return sizes;167		},168		downloadUrl(key, filename, type) {169			// Bun signs these as response-content-disposition and response-content-type (verified, v1.3.14).170			return client.presign(key, {171				method: "GET",172				expiresIn: DOWNLOAD_SECONDS,173				contentDisposition: `attachment; filename="${filename}"`,174				type,175			});176		},177		viewUrl(key, type) {178			return client.presign(key, { method: "GET", expiresIn: DOWNLOAD_SECONDS, type });179		},180		async put(key, bytes, type) {181			requireWritable(key, RELEASE_WRITE);182			await write(key, (file) => file.write(bytes, { type }));183		},184		async remove(key) {185			requireWritable(key, RELEASE_FILE);186			await write(key, (file) => file.delete());187		},188		async copy(from, to) {189			requireRepoKey(from);190			requireRepoKey(to);191			const bytes = await this.get(from);192			if (bytes === null) throw new Error(`${from} is missing`);193			await write(to, (file) => file.write(bytes));194		},195		async removeRenamed(key) {196			requireRepoKey(key);197			await write(key, (file) => file.delete());198		},199	};200}201202function folderReader(root: string): Bucket {203	const list = async (prefix: string) => {204		const entries = await readdir(root, { recursive: true, withFileTypes: true });205		return entries206			.filter((entry) => entry.isFile())207			.map((entry) => relative(root, join(entry.parentPath, entry.name)))208			.filter((key) => key.startsWith(prefix))209			.sort();210	};211	return {212		async get(key) {213			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);214			const file = Bun.file(join(root, key));215			return (await file.exists()) ? await file.bytes() : null;216		},217		list,218		async size(key) {219			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);220			return Bun.file(join(root, key)).size;221		},222		async folders() {223			const entries = await readdir(root, { withFileTypes: true });224			return entries225				.filter((entry) => entry.isDirectory())226				.map((entry) => entry.name)227				.sort();228		},229		async sizes(prefix) {230			return new Map((await list(prefix)).map((key) => [key, Bun.file(join(root, key)).size]));231		},232		// Development and tests only: a file:// URL can't carry a filename, a type or an expiry.233		downloadUrl(key) {234			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);235			return pathToFileURL(join(root, key)).href;236		},237		viewUrl: () => null,238		async put(key, bytes) {239			requireWritable(key, RELEASE_WRITE);240			await Bun.write(join(root, key), bytes);241		},242		async remove(key) {243			requireWritable(key, RELEASE_FILE);244			await unlink(join(root, key)).catch((error: NodeJS.ErrnoException) => {245				if (error.code !== "ENOENT") throw error;246			});247		},248		async copy(from, to) {249			requireRepoKey(from);250			requireRepoKey(to);251			await Bun.write(join(root, to), Bun.file(join(root, from)));252		},253		async removeRenamed(key) {254			requireRepoKey(key);255			await unlink(join(root, key));256		},257	};258}