reader.ts

1import { readdir, unlink } from "node:fs/promises";2import { join, relative } from "node:path";3import { pathToFileURL } from "node:url";4import type { BucketSource } from "../server/config.ts";56/**7 * A Koh 3 remote as Kepr uses it (§9.1): read-only on Koh's data, which needs two of its five8 * operations, plus a short-lived download URL so Kepr never streams release files itself (§22.2).9 * Kepr's own files (uploads) live under `KEPR_PREFIX` (June, 2026-09-29); no Koh repo can have10 * that name, since Kepr and Koh both need a `head` for one. Beyond it Kepr touches only builds,11 * which Koh has no way to delete (June, 2026-09-30): it deletes a version's files, rewrites its12 * record without a deleted one, and moves `releases/latest`.13 */14export type Bucket = {15	get(key: string): Promise<Uint8Array | null>;16	list(prefix: string): Promise<string[]>;17	/** The top-level folders' names: one listing, however many keys they hold. */18	folders(): Promise<string[]>;19	/** A key's size in bytes, asked without reading it. */20	size(key: string): Promise<number>;21	/** Every key under `prefix` with its size in bytes, for the console's storage view. */22	sizes(prefix: string): Promise<Map<string, number>>;23	/** A URL that serves `key` for five minutes, as an attachment with this filename and content type. */24	downloadUrl(key: string, filename: string, type: string): string;25	/** A five-minute URL that shows `key` in the page, or null when Kepr must serve the bytes itself. */26	viewUrl(key: string, type: string): string | null;27	/**28	 * Writes one of Kepr's own files, a repo's `releases/latest`, or a version's `release.json` when29	 * one of its files is deleted; any other key is refused.30	 */31	put(key: string, bytes: Uint8Array, type: string): Promise<void>;32	/** Deletes one of Kepr's own files, a release's file or `releases/latest`; nothing else. */33	remove(key: string): Promise<void>;34	/** Copies a key of one repo to another repo: a rename (June, 2026-10-01), and nothing else. */35	copy(from: string, to: string): Promise<void>;36	/** Deletes a key of a repo that a rename has copied to its new name. */37	removeRenamed(key: string): Promise<void>;38	/**39	 * Every key that belongs to one repo (§24): its own, and Kepr's files for it under `.kepr/`.40	 * What deleting a repo has to remove, and nothing of any other repo's.41	 */42	repoKeys(repo: string): Promise<string[]>;43	/** Deletes one of those keys; anything not of that repo is refused. */44	removeRepoKey(repo: string, key: string): Promise<void>;45};4647export const KEPR_PREFIX = ".kepr/";4849/** `each` over `items`, `atOnce` of them in flight at a time; the results in order. */50export async function inTurn<T, R>(51	items: T[],52	atOnce: number,53	each: (item: T) => Promise<R>,54): Promise<R[]> {55	const results: R[] = [];56	for (let i = 0; i < items.length; i += atOnce) {57		results.push(...(await Promise.all(items.slice(i, i + atOnce).map(each))));58	}59	return results;60}6162/** A write Kepr can't make: outside its folder, or refused by the bucket. */63export class BucketWriteError extends Error {}6465const DOWNLOAD_SECONDS = 5 * 60;6667const RELEASE_WRITE = /^[^/]+\/releases\/(latest|[^/]+\/release\.json)$/;68const RELEASE_FILE = /^[^/]+\/releases\/(latest|[^/]+\/.+)$/;6970// A key inside a repo, as a rename moves them: never Kepr's own folder, never `..` or empty parts.71function requireRepoKey(key: string): void {72	if (73		key.startsWith(KEPR_PREFIX) ||74		!key.includes("/") ||75		key.split("/").some((p) => p === ".." || p === "")76	) {77		throw new BucketWriteError(`a rename moves only a repo's keys, not "${key}"`);78	}79}8081function requireWritable(key: string, releaseKey: RegExp): void {82	const allowed = key.startsWith(KEPR_PREFIX) || releaseKey.test(key);83	if (!allowed || key.split("/").some((part) => part === ".." || part === "")) {84		throw new BucketWriteError(`Kepr writes only under ${KEPR_PREFIX} and builds, not "${key}"`);85	}86}8788// Kepr's own files for one repo: uploaded media, kept tarballs, the hidden list, the rename record.89const keprRepoFolders = (repo: string) => [90	`${KEPR_PREFIX}media/${repo}/`,91	`${KEPR_PREFIX}tarballs/${repo}/`,92];93const keprRepoFiles = (repo: string) => [94	`${KEPR_PREFIX}hidden/${repo}`,95	`${KEPR_PREFIX}renamed/${repo}`,96];9798/**99 * Whether `key` belongs to `repo` alone: inside it, or one of the files Kepr keeps for it. An empty100 * name is nothing, and no key may escape through `..` or an empty part.101 */102export function isRepoKey(repo: string, key: string): boolean {103	if (repo === "" || key.split("/").some((part) => part === ".." || part === "")) return false;104	if (key.startsWith(`${repo}/`)) return true;105	if (keprRepoFolders(repo).some((folder) => key.startsWith(folder))) return true;106	return keprRepoFiles(repo).includes(key);107}108109/** Every key of one repo: what a bucket lists under it, plus the files Kepr keeps for it. */110async function listRepoKeys(111	list: (prefix: string) => Promise<string[]>,112	repo: string,113): Promise<string[]> {114	if (repo === "") return [];115	const keys = await list(`${repo}/`);116	for (const folder of keprRepoFolders(repo)) keys.push(...(await list(folder)));117	return [...keys, ...keprRepoFiles(repo)].filter((key) => isRepoKey(repo, key));118}119120export function openBucket(source: BucketSource): Bucket {121	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);122}123124type S3Source = Extract<BucketSource, { kind: "s3" }>;125126/**127 * The console's connection test (§24), made before settings are saved: the key must list the128 * bucket and write Kepr's own test file. Returns what failed, or null.129 */130export async function checkS3Access(source: S3Source): Promise<string | null> {131	const reason = (error: unknown) => (error instanceof Error ? error.message : String(error));132	try {133		// One key is proof enough; listing the whole bucket could take minutes.134		await s3Client(source).list({ maxKeys: 1 });135	} catch (error) {136		return `The key cannot list the bucket: ${reason(error)}`;137	}138	try {139		const stamp = new TextEncoder().encode(new Date().toISOString());140		await s3Reader(source).put(`${KEPR_PREFIX}connection-test`, stamp, "text/plain");141	} catch (error) {142		return `The key cannot write ${KEPR_PREFIX}connection-test: ${reason(error)}`;143	}144	return null;145}146147function s3Client(source: S3Source): Bun.S3Client {148	return new Bun.S3Client({149		endpoint: source.endpoint,150		bucket: source.name,151		region: source.region,152		accessKeyId: source.accessKeyId,153		secretAccessKey: source.secretAccessKey,154	});155}156157function s3Reader(source: S3Source): Bucket {158	const client = s3Client(source);159	// A refused write is a BucketWriteError, which the console shows and a tarball cache skips.160	const write = async (key: string, run: (file: Bun.S3File) => Promise<unknown>) => {161		try {162			await run(client.file(key));163		} catch (error) {164			throw new BucketWriteError(`The bucket refused ${key}: ${error}`);165		}166	};167	return {168		async get(key) {169			// Through a presigned URL: Bun 1.3.14's S3File reads keep about one copy of every body170			// read, which a first sync of a big repo turns into hundreds of MB; fetch frees its own.171			const response = await fetch(client.presign(key, { method: "GET", expiresIn: 60 }));172			if (response.status === 404) return null;173			if (!response.ok) throw new Error(`S3 GET ${key} failed: HTTP ${response.status}`);174			return new Uint8Array(await response.arrayBuffer());175		},176		async list(prefix) {177			const keys: string[] = [];178			let continuationToken: string | undefined;179			do {180				const page = await client.list({ prefix, continuationToken });181				for (const object of page.contents ?? []) keys.push(object.key);182				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;183			} while (continuationToken);184			return keys;185		},186		size: (key) => client.size(key),187		async folders() {188			const names: string[] = [];189			let continuationToken: string | undefined;190			do {191				const page = await client.list({ delimiter: "/", continuationToken });192				for (const { prefix } of page.commonPrefixes ?? []) names.push(prefix.slice(0, -1));193				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;194			} while (continuationToken);195			return names;196		},197		async sizes(prefix) {198			const sizes = new Map<string, number>();199			let continuationToken: string | undefined;200			do {201				const page = await client.list({ prefix, continuationToken });202				for (const object of page.contents ?? []) sizes.set(object.key, object.size ?? 0);203				continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;204			} while (continuationToken);205			return sizes;206		},207		downloadUrl(key, filename, type) {208			// Bun signs these as response-content-disposition and response-content-type (verified, v1.3.14).209			return client.presign(key, {210				method: "GET",211				expiresIn: DOWNLOAD_SECONDS,212				contentDisposition: `attachment; filename="${filename}"`,213				type,214			});215		},216		viewUrl(key, type) {217			return client.presign(key, { method: "GET", expiresIn: DOWNLOAD_SECONDS, type });218		},219		async put(key, bytes, type) {220			requireWritable(key, RELEASE_WRITE);221			await write(key, (file) => file.write(bytes, { type }));222		},223		async remove(key) {224			requireWritable(key, RELEASE_FILE);225			await write(key, (file) => file.delete());226		},227		async copy(from, to) {228			requireRepoKey(from);229			requireRepoKey(to);230			const bytes = await this.get(from);231			if (bytes === null) throw new Error(`${from} is missing`);232			await write(to, (file) => file.write(bytes));233		},234		async removeRenamed(key) {235			requireRepoKey(key);236			await write(key, (file) => file.delete());237		},238		async repoKeys(repo) {239			return listRepoKeys((prefix) => this.list(prefix), repo);240		},241		async removeRepoKey(repo, key) {242			if (!isRepoKey(repo, key)) {243				throw new BucketWriteError(`Kepr deletes only ${repo}'s own keys, not "${key}"`);244			}245			await write(key, (file) => file.delete());246		},247	};248}249250function folderReader(root: string): Bucket {251	const list = async (prefix: string) => {252		const entries = await readdir(root, { recursive: true, withFileTypes: true });253		return entries254			.filter((entry) => entry.isFile())255			.map((entry) => relative(root, join(entry.parentPath, entry.name)))256			.filter((key) => key.startsWith(prefix))257			.sort();258	};259	return {260		async get(key) {261			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);262			const file = Bun.file(join(root, key));263			return (await file.exists()) ? await file.bytes() : null;264		},265		list,266		async size(key) {267			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);268			return Bun.file(join(root, key)).size;269		},270		async folders() {271			const entries = await readdir(root, { withFileTypes: true });272			return entries273				.filter((entry) => entry.isDirectory())274				.map((entry) => entry.name)275				.sort();276		},277		async sizes(prefix) {278			return new Map((await list(prefix)).map((key) => [key, Bun.file(join(root, key)).size]));279		},280		// Development and tests only: a file:// URL can't carry a filename, a type or an expiry.281		downloadUrl(key) {282			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);283			return pathToFileURL(join(root, key)).href;284		},285		viewUrl: () => null,286		async put(key, bytes) {287			requireWritable(key, RELEASE_WRITE);288			await Bun.write(join(root, key), bytes);289		},290		async remove(key) {291			requireWritable(key, RELEASE_FILE);292			await unlink(join(root, key)).catch((error: NodeJS.ErrnoException) => {293				if (error.code !== "ENOENT") throw error;294			});295		},296		async copy(from, to) {297			requireRepoKey(from);298			requireRepoKey(to);299			await Bun.write(join(root, to), Bun.file(join(root, from)));300		},301		async removeRenamed(key) {302			requireRepoKey(key);303			await unlink(join(root, key));304		},305		async repoKeys(repo) {306			return listRepoKeys(list, repo);307		},308		async removeRepoKey(repo, key) {309			if (!isRepoKey(repo, key)) {310				throw new BucketWriteError(`Kepr deletes only ${repo}'s own keys, not "${key}"`);311			}312			await unlink(join(root, key)).catch((error: NodeJS.ErrnoException) => {313				if (error.code !== "ENOENT") throw error;314			});315		},316	};317}