keep Kepr's uploads in its own .kepr/ folder of the bucket, written only there with the write key and served by presigned link or with Range (v3.0.113)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.112",3+	"version": "3.0.113",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -1,7 +1,6 @@11 import { createHash } from "node:crypto";2-import { mkdirSync } from "node:fs";3-import { join } from "node:path";42 import type { Hono } from "hono";3+import { BucketWriteError, KEPR_PREFIX } from "../bucket/reader.ts";54 import { describeFile } from "../describe/describe.ts";65 import { openMirror } from "../mirror/store.ts";76 import { renderMarkdown, renderParagraph } from "../pages/markdown.ts";@@ -16,15 +15,17 @@1615 const MEDIA_ROUTE = "/static/assets/media/:repo/:name{[0-9a-f]{64}\\.(png|jpg|gif|webp|mp4|webm)}";1716 1817 // What an upload is, from its first bytes rather than its name or declared type.19-const MEDIA: [string, (bytes: Uint8Array) => boolean][] = [20-	["png", (b) => ascii(b, 1, 4) === "PNG" && b[0] === 0x89],21-	["jpg", (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff],22-	["gif", (b) => ascii(b, 0, 4) === "GIF8"],23-	["webp", (b) => ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 12) === "WEBP"],24-	["mp4", (b) => ascii(b, 4, 8) === "ftyp"],25-	["webm", (b) => b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3],18+const MEDIA: [ext: string, type: string, is: (bytes: Uint8Array) => boolean][] = [19+	["png", "image/png", (b) => ascii(b, 1, 4) === "PNG" && b[0] === 0x89],20+	["jpg", "image/jpeg", (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff],21+	["gif", "image/gif", (b) => ascii(b, 0, 4) === "GIF8"],22+	["webp", "image/webp", (b) => ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 12) === "WEBP"],23+	["mp4", "video/mp4", (b) => ascii(b, 4, 8) === "ftyp"],24+	["webm", "video/webm", (b) => b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3],2625 ];2726 27+const mediaKey = (repo: string, name: string) => `${KEPR_PREFIX}media/${repo}/${name}`;28+2829 function ascii(bytes: Uint8Array, from: number, to: number): string {2930 	return String.fromCharCode(...bytes.subarray(from, to));3031 }@@ -32,7 +33,7 @@3233 /**3334  * Editing a repo's description and details from its own page while signed in: the texts (with3435  * the README's as a starting point), a preview as the page will render it, saving, and pictures35- * and videos for the details, kept in `data/assets/media/<repo>/` so backups carry them.36+ * and videos for the details, kept in the bucket under Kepr's own folder, where Koh's data is.3637  */3738 export function registerRepoText(app: Hono, ctx: AdminContext): void {3839 	app.get("/admin/repos/:repo/text", (c) => {@@ -76,21 +77,31 @@7677 		);7778 	});7879 79-	// Named by their hash, so cached long; `private`, so no shared cache keeps one after the repo80-	// turns private (§20). Ranges are answered, which video players need.80+	// As private as the repo (§20). An S3 bucket serves the file itself, Range and all, through a81+	// short-lived link; a folder bucket's bytes are served here, with Range, which video players82+	// need. Named by their hash, so the bytes never change; `private`, so no shared cache keeps83+	// one after the repo turns private.8184 	app.get(MEDIA_ROUTE, async (c) => {82-		const repo = c.req.param("repo");83-		const file = Bun.file(join(ctx.dataDir, "assets", "media", repo, c.req.param("name")));84-		if (!publicRepo(ctx.kepr, repo) || !(await file.exists())) return c.notFound();85-		const headers = { "Content-Type": file.type, "Cache-Control": "private, max-age=86400" };85+		const [repo, name] = [c.req.param("repo"), c.req.param("name")];86+		if (!publicRepo(ctx.kepr, repo)) return c.notFound();87+		const type = MEDIA.find(([ext]) => name.endsWith(`.${ext}`))?.[1] as string;88+		const link = ctx.sync.bucket.viewUrl(mediaKey(repo, name), type);89+		if (link) {90+			c.header("Cache-Control", "private, max-age=60");91+			return c.redirect(link, 302);92+		}93+		const bytes = await ctx.sync.bucket.get(mediaKey(repo, name));94+		if (!bytes) return c.notFound();95+		const headers = { "Content-Type": type, "Cache-Control": "private, max-age=86400" };8696 		c.header("Accept-Ranges", "bytes");8797 		const [, from = "", to = ""] = /^bytes=(\d*)-(\d*)$/.exec(c.req.header("range") ?? "") ?? [];88-		if (from === "" && to === "") return c.body(await file.bytes(), 200, headers);89-		const start = from === "" ? Math.max(0, file.size - Number(to)) : Number(from);90-		const end = from === "" || to === "" ? file.size - 1 : Math.min(Number(to), file.size - 1);91-		if (start > end) return c.body(null, 416, { "Content-Range": `bytes */${file.size}` });92-		const part = await file.slice(start, end + 1).bytes();93-		return c.body(part, 206, { ...headers, "Content-Range": `bytes ${start}-${end}/${file.size}` });98+		if (from === "" && to === "") return c.body(bytes.slice(), 200, headers);99+		const start = from === "" ? Math.max(0, bytes.length - Number(to)) : Number(from);100+		const end =101+			from === "" || to === "" ? bytes.length - 1 : Math.min(Number(to), bytes.length - 1);102+		if (start > end) return c.body(null, 416, { "Content-Range": `bytes */${bytes.length}` });103+		const range = `bytes ${start}-${end}/${bytes.length}`;104+		return c.body(bytes.slice(start, end + 1), 206, { ...headers, "Content-Range": range });94105 	});95106 96107 	app.post("/admin/repos/:repo/media", async (c) => {@@ -99,12 +110,15 @@99110 		const file = (await c.req.parseBody()).file;100111 		if (!(file instanceof File)) return c.text("Choose a file to upload.", 400);101112 		const bytes = new Uint8Array(await file.arrayBuffer());102-		const kind = MEDIA.find(([, is]) => is(bytes))?.[0];113+		const kind = MEDIA.find(([, , is]) => is(bytes));103114 		if (!kind) return c.text("Only PNG, JPEG, GIF, WebP, MP4 and WebM files.", 400);104-		const name = `${createHash("sha256").update(bytes).digest("hex")}.${kind}`;105-		const dir = join(ctx.dataDir, "assets", "media", repo.name);106-		mkdirSync(dir, { recursive: true });107-		await Bun.write(join(dir, name), bytes);115+		const name = `${createHash("sha256").update(bytes).digest("hex")}.${kind[0]}`;116+		try {117+			await ctx.sync.bucket.put(mediaKey(repo.name, name), bytes, kind[1]);118+		} catch (error) {119+			if (error instanceof BucketWriteError) return c.text(error.message, 503);120+			throw error;121+		}108122 		return c.json({ url: `/static/assets/media/${repo.name}/${name}` });109123 	});110124 }

@@ -4,23 +4,40 @@44 import type { BucketSource } from "../server/config.ts";55 66 /**7- * Read-only view of a Koh 3 remote (§9.1): Kepr needs only two of its five operations, plus a8- * short-lived download URL so it never streams release files itself (§22.2).7+ * A Koh 3 remote as Kepr uses it (§9.1): read-only on Koh's data, which needs two of its five8+ * operations, plus a short-lived download URL so Kepr never streams release files itself (§22.2).9+ * Kepr's own files (uploads) live under `KEPR_PREFIX`, the only keys `put` writes (June,10+ * 2026-09-29); no Koh repo can have that name, since Kepr and Koh both need a `head` for one.911  */10-export type BucketReader = {12+export type Bucket = {1113 	get(key: string): Promise<Uint8Array | null>;1214 	list(prefix: string): Promise<string[]>;1315 	/** A URL that serves `key` for five minutes, as an attachment with this filename and content type. */1416 	downloadUrl(key: string, filename: string, type: string): string;17+	/** A five-minute URL that shows `key` in the page, or null when Kepr must serve the bytes itself. */18+	viewUrl(key: string, type: string): string | null;19+	/** Writes one of Kepr's own files; any key outside `KEPR_PREFIX` is refused. */20+	put(key: string, bytes: Uint8Array, type: string): Promise<void>;1521 };1622 23+export const KEPR_PREFIX = ".kepr/";24+25+/** A write Kepr can't make: outside its folder, or without the `[bucket.write]` key. */26+export class BucketWriteError extends Error {}27+1728 const DOWNLOAD_SECONDS = 5 * 60;1829 19-export function openBucket(source: BucketSource): BucketReader {30+function requireOwnKey(key: string): void {31+	if (!key.startsWith(KEPR_PREFIX) || key.split("/").some((part) => part === ".." || part === "")) {32+		throw new BucketWriteError(`Kepr writes only under ${KEPR_PREFIX}, not "${key}"`);33+	}34+}35+36+export function openBucket(source: BucketSource): Bucket {2037 	return source.kind === "s3" ? s3Reader(source) : folderReader(source.path);2138 }2239 23-function s3Reader(source: Extract<BucketSource, { kind: "s3" }>): BucketReader {40+function s3Reader(source: Extract<BucketSource, { kind: "s3" }>): Bucket {2441 	const client = new Bun.S3Client({2542 		endpoint: source.endpoint,2643 		bucket: source.name,@@ -28,6 +45,15 @@2845 		accessKeyId: source.accessKeyId,2946 		secretAccessKey: source.secretAccessKey,3047 	});48+	// Reads keep the read-only key; only Kepr's own writes use the write key, when there is one.49+	const writer =50+		source.write &&51+		new Bun.S3Client({52+			endpoint: source.endpoint,53+			bucket: source.name,54+			region: source.region,55+			...source.write,56+		});3157 	return {3258 		async get(key) {3359 			try {@@ -56,10 +82,20 @@5682 				type,5783 			});5884 		},85+		viewUrl(key, type) {86+			return client.presign(key, { method: "GET", expiresIn: DOWNLOAD_SECONDS, type });87+		},88+		async put(key, bytes, type) {89+			requireOwnKey(key);90+			if (!writer) {91+				throw new BucketWriteError("Uploads need the bucket's write key: add [bucket.write].");92+			}93+			await writer.file(key).write(bytes, { type });94+		},5995 	};6096 }6197 62-function folderReader(root: string): BucketReader {98+function folderReader(root: string): Bucket {6399 	return {64100 		async get(key) {65101 			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);@@ -78,6 +114,11 @@78114 		downloadUrl(key) {79115 			if (key.split("/").includes("..")) throw new Error(`bucket key escapes the folder: ${key}`);80116 			return pathToFileURL(join(root, key)).href;117+		},118+		viewUrl: () => null,119+		async put(key, bytes) {120+			requireOwnKey(key);121+			await Bun.write(join(root, key), bytes);81122 		},82123 	};83124 }

@@ -1,7 +1,7 @@11 import { inflateSync } from "node:zlib";22 import { blake3 } from "@noble/hashes/blake3.js";33 import { bytesToHex } from "@noble/hashes/utils.js";4-import type { BucketReader } from "./reader.ts";4+import type { Bucket } from "./reader.ts";55 66 export type SaveFile = { path: string; hash: string; executable: boolean };77 @@ -94,7 +94,7 @@9494 	/\\u[dD][89abAB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2}|\\u[dD][89a-fA-F][0-9a-fA-F]{2}|\\[\s\S]/g;9595 9696 // ponytail: one full listing per call; switch to a delimiter listing if the bucket grows large.97-export async function listRepos(bucket: BucketReader): Promise<string[]> {97+export async function listRepos(bucket: Bucket): Promise<string[]> {9898 	const keys = await bucket.list("");9999 	return keys100100 		.filter((key) => key.endsWith("/head") && key.split("/").length === 2)@@ -116,7 +116,7 @@116116 }117117 118118 /** Applies §9.2's format rule: a missing key is version 1; anything but `koh-remote 1` is refused. */119-export async function checkFormat(bucket: BucketReader, repo: string): Promise<void> {119+export async function checkFormat(bucket: Bucket, repo: string): Promise<void> {120120 	requireRepo(repo);121121 	const bytes = await bucket.get(`${repo}/format`);122122 	if (bytes === null) return;@@ -126,7 +126,7 @@126126 	}127127 }128128 129-export async function readHead(bucket: BucketReader, repo: string): Promise<string | null> {129+export async function readHead(bucket: Bucket, repo: string): Promise<string | null> {130130 	requireRepo(repo);131131 	const bytes = await bucket.get(`${repo}/head`);132132 	if (bytes === null) return null;@@ -135,11 +135,7 @@135135 	return head;136136 }137137 138-export async function readSave(139-	bucket: BucketReader,140-	repo: string,141-	id: string,142-): Promise<ArrivingSave> {138+export async function readSave(bucket: Bucket, repo: string, id: string): Promise<ArrivingSave> {143139 	requireRepo(repo);144140 	requireSaveId(id);145141 	const bytes = await bucket.get(`${repo}/saves/${id}`);@@ -181,7 +177,7 @@181177 182178 /** Fetches an object, inflates it and checks its Blake3 against its key, as steal does (§9.4). */183179 export async function readObject(184-	bucket: BucketReader,180+	bucket: Bucket,185181 	repo: string,186182 	hash: string,187183 ): Promise<VerifiedObject> {@@ -204,7 +200,7 @@204200 	return { compressed, bytes };205201 }206202 207-export async function listEventIds(bucket: BucketReader, repo: string): Promise<string[]> {203+export async function listEventIds(bucket: Bucket, repo: string): Promise<string[]> {208204 	requireRepo(repo);209205 	const prefix = `${repo}/events/`;210206 	// A folder remote's half-written temp files can show up in a listing; only IDs are events.@@ -213,11 +209,7 @@213209 		.filter((id) => SAVE_ID.test(id));214210 }215211 216-export async function readEvent(217-	bucket: BucketReader,218-	repo: string,219-	id: string,220-): Promise<EventRecord> {212+export async function readEvent(bucket: Bucket, repo: string, id: string): Promise<EventRecord> {221213 	requireRepo(repo);222214 	requireSaveId(id);223215 	const bytes = await bucket.get(`${repo}/events/${id}`);@@ -237,10 +229,7 @@237229 }238230 239231 /** The body of `<repo>/releases/latest`, trimmed, or null when the repo has never released. */240-export async function readReleasesLatest(241-	bucket: BucketReader,242-	repo: string,243-): Promise<string | null> {232+export async function readReleasesLatest(bucket: Bucket, repo: string): Promise<string | null> {244233 	requireRepo(repo);245234 	const bytes = await bucket.get(`${repo}/releases/latest`);246235 	if (bytes === null) return null;@@ -251,7 +240,7 @@251240 }252241 253242 /** Every version with a `release.json`: a version without one doesn't exist (§11.2). */254-export async function listReleaseVersions(bucket: BucketReader, repo: string): Promise<string[]> {243+export async function listReleaseVersions(bucket: Bucket, repo: string): Promise<string[]> {255244 	requireRepo(repo);256245 	return (await bucket.list(`${repo}/releases/`))257246 		.map((key) => key.split("/"))@@ -261,7 +250,7 @@261250 }262251 263252 export async function readRelease(264-	bucket: BucketReader,253+	bucket: Bucket,265254 	repo: string,266255 	version: string,267256 ): Promise<ReleaseRecord> {

@@ -12,7 +12,7 @@1212 import { recordSync, registerRepo } from "../server/db.ts";1313 import { RESERVED_NAMES } from "../server/reserved.ts";1414 import { dropTarballs } from "../tarball/cache.ts";15-import type { BucketReader } from "./reader.ts";15+import type { Bucket } from "./reader.ts";1616 import {1717 	BucketError,1818 	checkFormat,@@ -36,7 +36,7 @@3636 const INTERVAL_MS = 2 * 60 * 1000;3737 3838 export type SyncTarget = {39-	bucket: BucketReader;39+	bucket: Bucket;4040 	kepr: Database;4141 	mirrorDir: string;4242 	tarballDir: string;@@ -85,7 +85,7 @@8585  * events it landed.8686  */8787 export async function syncRepo(88-	bucket: BucketReader,88+	bucket: Bucket,8989 	mirror: Database,9090 	repo: string,9191 ): Promise<EventRecord[]> {@@ -148,7 +148,7 @@148148 }149149 150150 async function newReleases(151-	bucket: BucketReader,151+	bucket: Bucket,152152 	mirror: Database,153153 	repo: string,154154 ): Promise<ReleaseRecord[]> {

@@ -1,7 +1,7 @@11 import { readFileSync } from "node:fs";22 import { join } from "node:path";33 import type { Context, Hono } from "hono";4-import type { BucketReader } from "../bucket/reader.ts";4+import type { Bucket } from "../bucket/reader.ts";55 import { isPlatform, isVersion, type ReleaseFile } from "../bucket/repo.ts";66 import { headId, repoStats, saveByPrefix } from "../mirror/queries.ts";77 import { notFound, openPublicRepo, type PageContext, type PublicRepo } from "../pages/layout.ts";@@ -28,7 +28,7 @@2828 2929 type Feed = { repo: PublicRepo; releases: Release[] };3030 31-export function registerReleaseRoutes(app: Hono, ctx: PageContext, bucket: BucketReader): void {31+export function registerReleaseRoutes(app: Hono, ctx: PageContext, bucket: Bucket): void {3232 	const send = (c: Context, status: 200 | 404, body: string, type = JSON_TYPE) =>3333 		c.body(body, status, { "Content-Type": type });3434 	const feed = (name: string): Feed | null => {

@@ -40,7 +40,8 @@4040 /** The Hono app with §25's headers and body cap, and every route built so far. */4141 export function createApp(config: Config, kepr: Database): Hono {4242 	const app = new Hono();43-	// §25 allows images from the presigned host, which is the bucket endpoint's origin.43+	// §25 allows images from the presigned host, which is the bucket endpoint's origin; uploaded44+	// videos come from there too.4445 	const imgSrc =4546 		config.bucket.kind === "s3" ? ["'self'", new URL(config.bucket.endpoint).origin] : ["'self'"];4647 	app.use("*", async (c, next) => {@@ -59,6 +60,7 @@5960 				styleSrc: ["'self'", FONTS],6061 				fontSrc: [FONTS],6162 				imgSrc,63+				mediaSrc: imgSrc,6264 				objectSrc: ["'none'"],6365 				baseUri: ["'none'"],6466 				formAction: ["'self'"],

@@ -143,7 +143,13 @@143143 		change(record);144144 		const bytes = new TextEncoder().encode(JSON.stringify(record));145145 		return readSave(146-			{ get: async () => bytes, list: async () => [], downloadUrl: () => "" },146+			{147+				get: async () => bytes,148+				list: async () => [],149+				downloadUrl: () => "",150+				viewUrl: () => null,151+				put: async () => {},152+			},147153 			fx.repo,148154 			id,149155 		);@@ -188,7 +194,13 @@188194 		const bytes = Buffer.from(JSON.stringify(record));189195 		bytes[bytes.indexOf("bad-X-name") + 4] = 0xff;190196 		const save = await readSave(191-			{ get: async () => bytes, list: async () => [], downloadUrl: () => "" },197+			{198+				get: async () => bytes,199+				list: async () => [],200+				downloadUrl: () => "",201+				viewUrl: () => null,202+				put: async () => {},203+			},192204 			fx.repo,193205 			id,194206 		);@@ -253,7 +265,13 @@253265 		]);254266 		const stripAll = { kind: "strip", save_id: "", path: "secret.txt" };255267 		const accepted = new TextEncoder().encode(JSON.stringify({ ...event, ...stripAll }));256-		const reader = { get: async () => accepted, list: async () => [], downloadUrl: () => "" };268+		const reader = {269+			get: async () => accepted,270+			list: async () => [],271+			downloadUrl: () => "",272+			viewUrl: () => null,273+			put: async () => {},274+		};257275 		expect((await readEvent(reader, fx.repo, event.id)).kind).toBe("strip");258276 		for (const change of [259277 			{ kind: "delete" },@@ -266,7 +284,13 @@266284 			// A strip with an illegal path, "" included, parses; sync ignores it (sync.test.ts).267285 		]) {268286 			const bytes = new TextEncoder().encode(JSON.stringify({ ...event, ...change }));269-			const tampered = { get: async () => bytes, list: async () => [], downloadUrl: () => "" };287+			const tampered = {288+				get: async () => bytes,289+				list: async () => [],290+				downloadUrl: () => "",291+				viewUrl: () => null,292+				put: async () => {},293+			};270294 			await expect(readEvent(tampered, fx.repo, event.id)).rejects.toThrow("Koh 3 schema");271295 		}272296 	});

@@ -1,4 +1,7 @@11 import { beforeAll, describe, expect, test } from "bun:test";2+import { join } from "node:path";3+import { BucketWriteError, openBucket } from "../src/bucket/reader.ts";4+import { listRepos } from "../src/bucket/repo.ts";25 import { Browser, type Instance, instance } from "./browser.ts";36 import { makeRepos } from "./fixture.ts";47 @@ -8,6 +11,7 @@811 describe("editing a repo's description and details while signed in", () => {912 	let inst: Instance;1013 	let browser: Browser;14+	let bucketDir: string;1115 	const page = async (path: string) => (await inst.app.request(path)).text();1216 	const save = (field: string, value: string) =>1317 		browser.post("/admin/repos/app/text", { field, value });@@ -19,7 +23,8 @@1923 2024 	beforeAll(async () => {2125 		const repos = await makeRepos(["app", "shy"]);22-		inst = await instance(repos.bucketDir);26+		bucketDir = repos.bucketDir;27+		inst = await instance(bucketDir);2328 		inst.kepr.run("UPDATE repos SET public = 1 WHERE name = 'app'");2429 		browser = new Browser(inst.app);2530 	});@@ -102,6 +107,13 @@102107 		expect(png.status).toBe(200);103108 		const { url } = (await png.json()) as { url: string };104109 		expect(url).toMatch(/^\/static\/assets\/media\/app\/[0-9a-f]{64}\.png$/);110+		// Kept in the bucket, in Kepr's own folder, which no sync takes for a repo.111+		const key = `.kepr/media/app/${url.split("/").pop()}`;112+		expect(await Bun.file(join(bucketDir, key)).exists()).toBe(true);113+		expect(await listRepos(openBucket({ kind: "folder", path: bucketDir }))).toEqual([114+			"app",115+			"shy",116+		]);105117 		const served = await inst.app.request(url);106118 		expect([served.status, served.headers.get("content-type")]).toEqual([200, "image/png"]);107119 		const video = ((await (await upload(MP4, "demo.mp4")).json()) as { url: string }).url;@@ -122,4 +134,25 @@122134 		expect((await stranger.request("/admin/repos/app/text")).status).toBe(401);123135 		expect((await stranger.post("/admin/repos/app/media", new FormData())).status).toBe(401);124136 	});137+});138+139+test("Kepr writes only inside its own folder, and to S3 only with the write key", async () => {140+	const { bucketDir } = await makeRepos(["app"]);141+	const folder = openBucket({ kind: "folder", path: bucketDir });142+	for (const key of ["app/head", ".kepr/../app/head", ".kepr//x", "x/.kepr/y"]) {143+		await expect(folder.put(key, new Uint8Array([1]), "image/png")).rejects.toBeInstanceOf(144+			BucketWriteError,145+		);146+	}147+	const readOnly = openBucket({148+		kind: "s3",149+		endpoint: "http://127.0.0.1:9",150+		name: "b",151+		region: "r",152+		accessKeyId: "k",153+		secretAccessKey: "s",154+	});155+	await expect(156+		readOnly.put(".kepr/media/app/x.png", new Uint8Array([1]), "image/png"),157+	).rejects.toThrow("[bucket.write]");125158 });