reader.ts

1import { readdir, unlink } from "node:fs/promises";2import { join, relative } from "node:path";3import { pathToFileURL } from "node:url";4import type { BucketSource } from "../server/config.ts";56/**7 * A Koh 3 remote as Kepr uses it (§9.1): read-only on Koh's data, which needs two of its five8 * operations, plus a short-lived download URL so Kepr never streams release files itself (§22.2).9 * Kepr's own files (uploads) live under `KEPR_PREFIX` (June, 2026-09-29); no Koh repo can have10 * that name, since Kepr and Koh both need a `head` for one. Beyond it Kepr touches only builds,11 * which Koh has no way to delete (June, 2026-09-30): it deletes a version's files, rewrites its12 * record without a deleted one, and moves `releases/latest`.13 */14export type Bucket = {15	get(key: string): Promise<Uint8Array | null>;16	list(prefix: string): Promise<string[]>;17	/** Every key under `prefix` with its size in bytes, for the console's storage view. */18	sizes(prefix: string): Promise<Map<string, number>>;19	/** A URL that serves `key` for five minutes, as an attachment with this filename and content type. */20	downloadUrl(key: string, filename: string, type: string): string;21	/** A five-minute URL that shows `key` in the page, or null when Kepr must serve the bytes itself. */22	viewUrl(key: string, type: string): string | null;23	/**24	 * Writes one of Kepr's own files, a repo's `releases/latest`, or a version's `release.json` when25	 * one of its files is deleted; any other key is refused.26	 */27	put(key: string, bytes: Uint8Array, type: string): Promise<void>;28	/** Deletes one of Kepr's own files, a release's file or `releases/latest`; nothing else. */29	remove(key: string): Promise<void>;30	/** Copies a key of one repo to another repo: a rename (June, 2026-10-01), and nothing else. */31	copy(from: string, to: string): Promise<void>;32	/** Deletes a key of a repo that a rename has copied to its new name. */33	removeRenamed(key: string): Promise<void>;34};3536export const KEPR_PREFIX = ".kepr/";3738/** A write Kepr can't make: outside its folder, or without the `[bucket.write]` key. */39export class BucketWriteError extends Error {}4041const DOWNLOAD_SECONDS = 5 * 60;4243const RELEASE_WRITE = /^[^/]+\/releases\/(latest|[^/]+\/release\.json)$/;44const RELEASE_FILE = /^[^/]+\/releases\/(latest|[^/]+\/.+)$/;4546// A key inside a repo, as a rename moves them: never Kepr's own folder, never `..` or empty parts.47function requireRepoKey(key: string): void {48	if (49		key.startsWith(KEPR_PREFIX) ||50		!key.includes("/") ||51		key.split("/").some((p) => p === ".." || p === "")52	) {53		throw new BucketWriteError(`a rename moves only a repo's keys, not "${key}"`);54	}55}5657function requireWritable(key: string, releaseKey: RegExp): void {58	const allowed = key.startsWith(KEPR_PREFIX) || releaseKey.test(key);59	if (!allowed || key.split("/").some((part) => part === ".." || part === "")) {60		throw new BucketWriteError(`Kepr writes only under ${KEPR_PREFIX} and builds, not "${key}"`);61	}62}6364export function openBucket(source: BucketSource): Bucket {65	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);66}6768type S3Source = Extract<BucketSource, { kind: "s3" }>;6970/**71 * The console's connection test (§24), made before settings are saved: the read key must list the72 * bucket, and a write key must write Kepr's own test file. Returns what failed, or null.73 */74export async function checkS3Access(source: S3Source): Promise<string | null> {75	const reason = (error: unknown) => (error instanceof Error ? error.message : String(error));76	try {77		// One key is proof enough; listing the whole bucket could take minutes.78		await s3Client(source).list({ maxKeys: 1 });79	} catch (error) {80		return `The read key cannot list the bucket: ${reason(error)}`;81	}82	if (!source.write) return null;83	try {84		const stamp = new TextEncoder().encode(new Date().toISOString());85		await s3Reader(source).put(`${KEPR_PREFIX}connection-test`, stamp, "text/plain");86	} catch (error) {87		return `The write key cannot write ${KEPR_PREFIX}connection-test: ${reason(error)}`;88	}89	return null;90}9192function s3Client(93	source: S3Source,94	keys: { accessKeyId: string; secretAccessKey: string } = source,95): Bun.S3Client {96	return new Bun.S3Client({97		endpoint: source.endpoint,98		bucket: source.name,99		region: source.region,100		accessKeyId: keys.accessKeyId,101		secretAccessKey: keys.secretAccessKey,102	});103}104105function s3Reader(source: S3Source): Bucket {106	const client = s3Client(source);107	// Reads keep the read-only key; only Kepr's own writes use the write key, when there is one.108	const writer = source.write && s3Client(source, source.write);109	const requireWriter = () => {110		if (!writer) {111			throw new BucketWriteError("Writing needs the bucket's write key: add [bucket.write].");112		}113		return writer;114	};115	return {116		async get(key) {117			// Through a presigned URL: Bun 1.3.14's S3File reads keep about one copy of every body118			// read, which a first sync of a big repo turns into hundreds of MB; fetch frees its own.119			const response = await fetch(client.presign(key, { method: "GET", expiresIn: 60 }));120			if (response.status === 404) return null;121			if (!response.ok) throw new Error(`S3 GET ${key} failed: HTTP ${response.status}`);122			return new Uint8Array(await response.arrayBuffer());123		},124		async list(prefix) {125			const keys: string[] = [];126			let continuationToken: string | undefined;127			do {128				const page = await client.list({ prefix, continuationToken });129				for (const object of page.contents ?? []) keys.push(object.key);130				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;131			} while (continuationToken);132			return keys;133		},134		async sizes(prefix) {135			const sizes = new Map<string, number>();136			let continuationToken: string | undefined;137			do {138				const page = await client.list({ prefix, continuationToken });139				for (const object of page.contents ?? []) sizes.set(object.key, object.size ?? 0);140				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;141			} while (continuationToken);142			return sizes;143		},144		downloadUrl(key, filename, type) {145			// Bun signs these as response-content-disposition and response-content-type (verified, v1.3.14).146			return client.presign(key, {147				method: "GET",148				expiresIn: DOWNLOAD_SECONDS,149				contentDisposition: `attachment; filename="${filename}"`,150				type,151			});152		},153		viewUrl(key, type) {154			return client.presign(key, { method: "GET", expiresIn: DOWNLOAD_SECONDS, type });155		},156		async put(key, bytes, type) {157			requireWritable(key, RELEASE_WRITE);158			await requireWriter().file(key).write(bytes, { type });159		},160		async remove(key) {161			requireWritable(key, RELEASE_FILE);162			await requireWriter().file(key).delete();163		},164		async copy(from, to) {165			requireRepoKey(from);166			requireRepoKey(to);167			const bytes = await this.get(from);168			if (bytes === null) throw new Error(`${from} is missing`);169			await requireWriter().file(to).write(bytes);170		},171		async removeRenamed(key) {172			requireRepoKey(key);173			await requireWriter().file(key).delete();174		},175	};176}177178function folderReader(root: string): Bucket {179	const list = async (prefix: string) => {180		const entries = await readdir(root, { recursive: true, withFileTypes: true });181		return entries182			.filter((entry) => entry.isFile())183			.map((entry) => relative(root, join(entry.parentPath, entry.name)))184			.filter((key) => key.startsWith(prefix))185			.sort();186	};187	return {188		async get(key) {189			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);190			const file = Bun.file(join(root, key));191			return (await file.exists()) ? await file.bytes() : null;192		},193		list,194		async sizes(prefix) {195			return new Map((await list(prefix)).map((key) => [key, Bun.file(join(root, key)).size]));196		},197		// Development and tests only: a file:// URL can't carry a filename, a type or an expiry.198		downloadUrl(key) {199			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);200			return pathToFileURL(join(root, key)).href;201		},202		viewUrl: () => null,203		async put(key, bytes) {204			requireWritable(key, RELEASE_WRITE);205			await Bun.write(join(root, key), bytes);206		},207		async remove(key) {208			requireWritable(key, RELEASE_FILE);209			await unlink(join(root, key)).catch((error: NodeJS.ErrnoException) => {210				if (error.code !== "ENOENT") throw error;211			});212		},213		async copy(from, to) {214			requireRepoKey(from);215			requireRepoKey(to);216			await Bun.write(join(root, to), Bun.file(join(root, from)));217		},218		async removeRenamed(key) {219			requireRepoKey(key);220			await unlink(join(root, key));221		},222	};223}