reader.ts

1import { readdir, unlink } from "node:fs/promises";2import { join, relative } from "node:path";3import { pathToFileURL } from "node:url";4import type { BucketSource } from "../server/config.ts";56/**7 * A Koh 3 remote as Kepr uses it (§9.1): read-only on Koh's data, which needs two of its five8 * operations, plus a short-lived download URL so Kepr never streams release files itself (§22.2).9 * Kepr's own files (uploads) live under `KEPR_PREFIX` (June, 2026-09-29); no Koh repo can have10 * that name, since Kepr and Koh both need a `head` for one. Beyond it Kepr touches only builds,11 * which Koh has no way to delete (June, 2026-09-30): it deletes a version's files, rewrites its12 * record without a deleted one, and moves `releases/latest`.13 */14export type Bucket = {15	get(key: string): Promise<Uint8Array | null>;16	list(prefix: string): Promise<string[]>;17	/** The top-level folders' names: one listing, however many keys they hold. */18	folders(): Promise<string[]>;19	/** A key's size in bytes, asked without reading it. */20	size(key: string): Promise<number>;21	/** Every key under `prefix` with its size in bytes, for the console's storage view. */22	sizes(prefix: string): Promise<Map<string, number>>;23	/** A URL that serves `key` for five minutes, as an attachment with this filename and content type. */24	downloadUrl(key: string, filename: string, type: string): string;25	/** A five-minute URL that shows `key` in the page, or null when Kepr must serve the bytes itself. */26	viewUrl(key: string, type: string): string | null;27	/**28	 * Writes one of Kepr's own files, a repo's `releases/latest`, or a version's `release.json` when29	 * one of its files is deleted; any other key is refused.30	 */31	put(key: string, bytes: Uint8Array, type: string): Promise<void>;32	/** Deletes one of Kepr's own files, a release's file or `releases/latest`; nothing else. */33	remove(key: string): Promise<void>;34	/** Copies a key of one repo to another repo: a rename (June, 2026-10-01), and nothing else. */35	copy(from: string, to: string): Promise<void>;36	/** Deletes a key of a repo that a rename has copied to its new name. */37	removeRenamed(key: string): Promise<void>;38};3940export const KEPR_PREFIX = ".kepr/";4142/** `each` over `items`, `atOnce` of them in flight at a time; the results in order. */43export async function inTurn<T, R>(44	items: T[],45	atOnce: number,46	each: (item: T) => Promise<R>,47): Promise<R[]> {48	const results: R[] = [];49	for (let i = 0; i < items.length; i += atOnce) {50		results.push(...(await Promise.all(items.slice(i, i + atOnce).map(each))));51	}52	return results;53}5455/** A write Kepr can't make: outside its folder, or without the `[bucket.write]` key. */56export class BucketWriteError extends Error {}5758const DOWNLOAD_SECONDS = 5 * 60;5960const RELEASE_WRITE = /^[^/]+\/releases\/(latest|[^/]+\/release\.json)$/;61const RELEASE_FILE = /^[^/]+\/releases\/(latest|[^/]+\/.+)$/;6263// A key inside a repo, as a rename moves them: never Kepr's own folder, never `..` or empty parts.64function requireRepoKey(key: string): void {65	if (66		key.startsWith(KEPR_PREFIX) ||67		!key.includes("/") ||68		key.split("/").some((p) => p === ".." || p === "")69	) {70		throw new BucketWriteError(`a rename moves only a repo's keys, not "${key}"`);71	}72}7374function requireWritable(key: string, releaseKey: RegExp): void {75	const allowed = key.startsWith(KEPR_PREFIX) || releaseKey.test(key);76	if (!allowed || key.split("/").some((part) => part === ".." || part === "")) {77		throw new BucketWriteError(`Kepr writes only under ${KEPR_PREFIX} and builds, not "${key}"`);78	}79}8081export function openBucket(source: BucketSource): Bucket {82	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);83}8485type S3Source = Extract<BucketSource, { kind: "s3" }>;8687/**88 * The console's connection test (§24), made before settings are saved: the read key must list the89 * bucket, and a write key must write Kepr's own test file. Returns what failed, or null.90 */91export async function checkS3Access(source: S3Source): Promise<string | null> {92	const reason = (error: unknown) => (error instanceof Error ? error.message : String(error));93	try {94		// One key is proof enough; listing the whole bucket could take minutes.95		await s3Client(source).list({ maxKeys: 1 });96	} catch (error) {97		return `The read key cannot list the bucket: ${reason(error)}`;98	}99	if (!source.write) return null;100	try {101		const stamp = new TextEncoder().encode(new Date().toISOString());102		await s3Reader(source).put(`${KEPR_PREFIX}connection-test`, stamp, "text/plain");103	} catch (error) {104		return `The write key cannot write ${KEPR_PREFIX}connection-test: ${reason(error)}`;105	}106	return null;107}108109function s3Client(110	source: S3Source,111	keys: { accessKeyId: string; secretAccessKey: string } = source,112): Bun.S3Client {113	return new Bun.S3Client({114		endpoint: source.endpoint,115		bucket: source.name,116		region: source.region,117		accessKeyId: keys.accessKeyId,118		secretAccessKey: keys.secretAccessKey,119	});120}121122function s3Reader(source: S3Source): Bucket {123	const client = s3Client(source);124	// Reads keep the read-only key; only Kepr's own writes use the write key, when there is one.125	const writer = source.write && s3Client(source, source.write);126	const requireWriter = () => {127		if (!writer) {128			throw new BucketWriteError("Writing needs the bucket's write key: add [bucket.write].");129		}130		return writer;131	};132	return {133		async get(key) {134			// Through a presigned URL: Bun 1.3.14's S3File reads keep about one copy of every body135			// read, which a first sync of a big repo turns into hundreds of MB; fetch frees its own.136			const response = await fetch(client.presign(key, { method: "GET", expiresIn: 60 }));137			if (response.status === 404) return null;138			if (!response.ok) throw new Error(`S3 GET ${key} failed: HTTP ${response.status}`);139			return new Uint8Array(await response.arrayBuffer());140		},141		async list(prefix) {142			const keys: string[] = [];143			let continuationToken: string | undefined;144			do {145				const page = await client.list({ prefix, continuationToken });146				for (const object of page.contents ?? []) keys.push(object.key);147				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;148			} while (continuationToken);149			return keys;150		},151		size: (key) => client.size(key),152		async folders() {153			const names: string[] = [];154			let continuationToken: string | undefined;155			do {156				const page = await client.list({ delimiter: "/", continuationToken });157				for (const { prefix } of page.commonPrefixes ?? []) names.push(prefix.slice(0, -1));158				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;159			} while (continuationToken);160			return names;161		},162		async sizes(prefix) {163			const sizes = new Map<string, number>();164			let continuationToken: string | undefined;165			do {166				const page = await client.list({ prefix, continuationToken });167				for (const object of page.contents ?? []) sizes.set(object.key, object.size ?? 0);168				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;169			} while (continuationToken);170			return sizes;171		},172		downloadUrl(key, filename, type) {173			// Bun signs these as response-content-disposition and response-content-type (verified, v1.3.14).174			return client.presign(key, {175				method: "GET",176				expiresIn: DOWNLOAD_SECONDS,177				contentDisposition: `attachment; filename="${filename}"`,178				type,179			});180		},181		viewUrl(key, type) {182			return client.presign(key, { method: "GET", expiresIn: DOWNLOAD_SECONDS, type });183		},184		async put(key, bytes, type) {185			requireWritable(key, RELEASE_WRITE);186			await requireWriter().file(key).write(bytes, { type });187		},188		async remove(key) {189			requireWritable(key, RELEASE_FILE);190			await requireWriter().file(key).delete();191		},192		async copy(from, to) {193			requireRepoKey(from);194			requireRepoKey(to);195			const bytes = await this.get(from);196			if (bytes === null) throw new Error(`${from} is missing`);197			await requireWriter().file(to).write(bytes);198		},199		async removeRenamed(key) {200			requireRepoKey(key);201			await requireWriter().file(key).delete();202		},203	};204}205206function folderReader(root: string): Bucket {207	const list = async (prefix: string) => {208		const entries = await readdir(root, { recursive: true, withFileTypes: true });209		return entries210			.filter((entry) => entry.isFile())211			.map((entry) => relative(root, join(entry.parentPath, entry.name)))212			.filter((key) => key.startsWith(prefix))213			.sort();214	};215	return {216		async get(key) {217			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);218			const file = Bun.file(join(root, key));219			return (await file.exists()) ? await file.bytes() : null;220		},221		list,222		async size(key) {223			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);224			return Bun.file(join(root, key)).size;225		},226		async folders() {227			const entries = await readdir(root, { withFileTypes: true });228			return entries229				.filter((entry) => entry.isDirectory())230				.map((entry) => entry.name)231				.sort();232		},233		async sizes(prefix) {234			return new Map((await list(prefix)).map((key) => [key, Bun.file(join(root, key)).size]));235		},236		// Development and tests only: a file:// URL can't carry a filename, a type or an expiry.237		downloadUrl(key) {238			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);239			return pathToFileURL(join(root, key)).href;240		},241		viewUrl: () => null,242		async put(key, bytes) {243			requireWritable(key, RELEASE_WRITE);244			await Bun.write(join(root, key), bytes);245		},246		async remove(key) {247			requireWritable(key, RELEASE_FILE);248			await unlink(join(root, key)).catch((error: NodeJS.ErrnoException) => {249				if (error.code !== "ENOENT") throw error;250			});251		},252		async copy(from, to) {253			requireRepoKey(from);254			requireRepoKey(to);255			await Bun.write(join(root, to), Bun.file(join(root, from)));256		},257		async removeRenamed(key) {258			requireRepoKey(key);259			await unlink(join(root, key));260		},261	};262}