read release records into the mirror when head or releases/latest moves, and keep hidden releases, notes overrides and download and install counts in kepr.db (v3.0.20)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.19",3+	"version": "3.0.20",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -32,15 +32,50 @@3232 	machine: string;3333 };3434 35+/** One built file of a release (§11.2). */36+export type ReleaseFile = {37+	platform: string;38+	name: string;39+	size: number;40+	blake3: string;41+	sha256: string;42+};43+44+/** The release record Koh 3 writes to `<repo>/releases/<version>/release.json` (§11.2). */45+export type ReleaseRecord = {46+	format: 1;47+	repo: string;48+	version: string;49+	save: string;50+	published_at: number;51+	machine: string;52+	notes: string;53+	files: ReleaseFile[];54+};55+56+/** §11.1's platform names, Kepr 2's exact list: Peach Android and existing build files depend on them. */57+export const PLATFORMS = [58+	"linux-x86_64",59+	"linux-aarch64",60+	"macos-aarch64",61+	"macos-x86_64",62+	"windows-x86_64",63+	"android-arm64",64+	"android-x86",65+] as const;66+3567 /** An object's stored zlib bytes and the inflated bytes they were verified from. */3668 export type VerifiedObject = { compressed: Uint8Array; bytes: Uint8Array };3769 3870 export class BucketError extends Error {}3971 4072 const SAVE_ID = /^[0-9a-f]{12}$/;41-const OBJECT_HASH = /^[0-9a-f]{64}$/;4273 const REPO_NAME = /^[A-Za-z0-9._-]+$/;4374 const CONTROL_CHAR = /\p{Cc}/u;75+const HEX64 = /^[0-9a-f]{64}$/;76+const VERSION = /^[A-Za-z0-9.-]{1,32}$/;77+// A file name that is safe as one key segment and inside a quoted Content-Disposition filename.78+const FILE_NAME = /^[^/\\"\p{Cc}]{1,255}$/u;4479 const decoder = new TextDecoder();4580 4681 // ponytail: one full listing per call; switch to a delimiter listing if the bucket grows large.@@ -106,7 +141,7 @@106141 	hash: string,107142 ): Promise<VerifiedObject> {108143 	requireRepo(repo);109-	if (!OBJECT_HASH.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);144+	if (!HEX64.test(hash)) throw new BucketError(`not an object hash: "${hash}"`);110145 	const compressed = await bucket.get(`${repo}/objects/${hash}`);111146 	if (compressed === null) throw new BucketError(`${repo}/objects/${hash} is missing`);112147 	const bytes = new Uint8Array(inflateSync(compressed));@@ -139,6 +174,92 @@139174 	return record;140175 }141176 177+/** §11.1's version rule, minus `.` and `..`, which would name a folder rather than a version. */178+export function isVersion(value: string): boolean {179+	return VERSION.test(value) && value !== "." && value !== "..";180+}181+182+export function isPlatform(value: string): boolean {183+	return (PLATFORMS as readonly string[]).includes(value);184+}185+186+/** The body of `<repo>/releases/latest`, trimmed, or null when the repo has never released. */187+export async function readReleasesLatest(188+	bucket: BucketReader,189+	repo: string,190+): Promise<string | null> {191+	requireRepo(repo);192+	const bytes = await bucket.get(`${repo}/releases/latest`);193+	return bytes === null ? null : decoder.decode(bytes).trim();194+}195+196+/** Every version with a `release.json`: a version without one doesn't exist (§11.2). */197+export async function listReleaseVersions(bucket: BucketReader, repo: string): Promise<string[]> {198+	requireRepo(repo);199+	return (await bucket.list(`${repo}/releases/`))200+		.map((key) => key.split("/"))201+		.filter((parts) => parts.length === 4 && parts[3] === "release.json")202+		.map((parts) => parts[2] as string)203+		.filter(isVersion);204+}205+206+export async function readRelease(207+	bucket: BucketReader,208+	repo: string,209+	version: string,210+): Promise<ReleaseRecord> {211+	requireRepo(repo);212+	if (!isVersion(version)) throw new BucketError(`not a release version: "${version}"`);213+	const key = `${repo}/releases/${version}/release.json`;214+	const bytes = await bucket.get(key);215+	if (bytes === null) throw new BucketError(`${key} is missing`);216+	const record = parseReleaseRecord(JSON.parse(decoder.decode(bytes)));217+	if (record.repo !== repo || record.version !== version) {218+		throw new BucketError(`${key} holds ${record.repo} ${record.version}`);219+	}220+	return record;221+}222+223+function parseReleaseRecord(value: unknown): ReleaseRecord {224+	const r = value as Record<string, unknown>;225+	const valid =226+		typeof value === "object" &&227+		value !== null &&228+		r.format === 1 &&229+		typeof r.repo === "string" &&230+		typeof r.version === "string" &&231+		typeof r.save === "string" &&232+		SAVE_ID.test(r.save) &&233+		isCount(r.published_at) &&234+		typeof r.machine === "string" &&235+		typeof r.notes === "string" &&236+		Array.isArray(r.files) &&237+		r.files.every(isReleaseFile) &&238+		// One file per platform per version (§11.2).239+		new Set(r.files.map((file) => file.platform)).size === r.files.length;240+	if (!valid) throw new BucketError("release record does not match §11.2");241+	return value as ReleaseRecord;242+}243+244+function isReleaseFile(value: unknown): value is ReleaseFile {245+	const f = value as Record<string, unknown>;246+	return (247+		typeof value === "object" &&248+		value !== null &&249+		typeof f.platform === "string" &&250+		isPlatform(f.platform) &&251+		typeof f.name === "string" &&252+		FILE_NAME.test(f.name) &&253+		f.name !== "." &&254+		f.name !== ".." &&255+		isCount(f.size) &&256+		typeof f.blake3 === "string" &&257+		HEX64.test(f.blake3) &&258+		typeof f.sha256 === "string" &&259+		HEX64.test(f.sha256)260+	);261+}262+142263 function parseSaveRecord(value: unknown): SaveRecord {143264 	const r = value as Record<string, unknown>;144265 	const valid =@@ -203,7 +324,7 @@203324 		typeof f.path === "string" &&204325 		isSafePath(f.path) &&205326 		typeof f.hash === "string" &&206-		OBJECT_HASH.test(f.hash) &&327+		HEX64.test(f.hash) &&207328 		typeof f.executable === "boolean"208329 	);209330 }

@@ -2,6 +2,7 @@22 import {33 	knownEventIds,44 	knownObject,5+	knownReleases,56 	knownSave,67 	land,78 	mirrorState,@@ -16,10 +17,14 @@1617 	checkFormat,1718 	type EventRecord,1819 	listEventIds,20+	listReleaseVersions,1921 	listRepos,22+	type ReleaseRecord,2023 	readEvent,2124 	readHead,2225 	readObject,26+	readRelease,27+	readReleasesLatest,2328 	readSave,2429 	type SaveRecord,2530 } from "./repo.ts";@@ -66,7 +71,8 @@6671 /**6772  * Koh's steal flow (§9.4) against a mirror: walk back from the remote head to a save the mirror has,6873  * fetch what is missing, and land it in one transaction. Events are listed every time, since a69- * flag or a strip arrives without moving `head`. Returns the events it landed.74+ * flag or a strip arrives without moving `head`; releases only when `head` or `releases/latest`75+ * moved (§18), since publishing one moves both. Returns the events it landed.7076  */7177 export async function syncRepo(7278 	bucket: BucketReader,@@ -96,7 +102,25 @@96102 	for (const id of await listEventIds(bucket, repo)) {97103 		if (!known.has(id)) events.push(await readEvent(bucket, repo, id));98104 	}99-	if (saves.length === 0 && events.length === 0 && mirrorState(mirror, "head") === head) return [];100-	land(mirror, { head, saves: saves.reverse(), objects, events });105+	const releasesLatest = (await readReleasesLatest(bucket, repo)) ?? "";106+	const moved =107+		mirrorState(mirror, "head") !== head ||108+		mirrorState(mirror, "releases_latest") !== releasesLatest;109+	const releases = moved ? await newReleases(bucket, mirror, repo) : [];110+	if (!moved && events.length === 0) return [];111+	land(mirror, { head, releasesLatest, saves: saves.reverse(), objects, events, releases });101112 	return events;113+}114+115+async function newReleases(116+	bucket: BucketReader,117+	mirror: Database,118+	repo: string,119+): Promise<ReleaseRecord[]> {120+	const known = knownReleases(mirror);121+	const releases: ReleaseRecord[] = [];122+	for (const version of await listReleaseVersions(bucket, repo)) {123+		if (!known.has(version)) releases.push(await readRelease(bucket, repo, version));124+	}125+	return releases;102126 }

@@ -1,5 +1,6 @@11 import type { Database } from "bun:sqlite";22 import { inflateSync } from "node:zlib";3+import type { ReleaseRecord } from "../bucket/repo.ts";34 import { mirrorState } from "./store.ts";45 56 export type Save = {@@ -159,6 +160,14 @@159160 		else byPath.set(path, [save]);160161 	}161162 	return { line: [...saves.values()], byPath };163+}164+165+/** Every release record the bucket holds, in no particular order. */166+export function releaseRecords(db: Database): ReleaseRecord[] {167+	return db168+		.query<{ record: string }, []>("SELECT record FROM releases")169+		.all()170+		.map((row) => JSON.parse(row.record) as ReleaseRecord);162171 }163172 164173 function asSave(row: Save): Save {

@@ -2,9 +2,12 @@22 import { mkdirSync } from "node:fs";33 import { join } from "node:path";44 import { inflateSync } from "node:zlib";5-import type { EventRecord, SaveRecord } from "../bucket/repo.ts";5+import type { EventRecord, ReleaseRecord, SaveRecord } from "../bucket/repo.ts";66 7-// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs.7+// Koh 3's schema (§5.1, `src/storage/storage.v`) minus `file_cache`, which only a working tree needs,8+// plus `releases`: each version's release.json (§11.2) as read from the bucket, since that is bucket9+// data too and pages and feeds need it on every request.10+const SCHEMA_VERSION = 2;811 const SCHEMA = `912 CREATE TABLE objects (1013 	hash TEXT PRIMARY KEY,@@ -44,6 +47,10 @@4447 	key   TEXT PRIMARY KEY,4548 	value TEXT NOT NULL4649 );50+CREATE TABLE releases (51+	version TEXT PRIMARY KEY,52+	record  TEXT NOT NULL53+);4754 CREATE INDEX saves_parent ON saves(parent);4855 CREATE INDEX save_files_hash ON save_files(hash);4956 `;@@ -54,9 +61,11 @@5461 /** What one sync brings in, landed in one transaction. Saves are oldest first. */5562 export type Landing = {5663 	head: string;64+	releasesLatest: string;5765 	saves: SaveRecord[];5866 	objects: Map<string, StoredObject>;5967 	events: EventRecord[];68+	releases: ReleaseRecord[];6069 };6170 6271 // Display names for the header's language, by extension; data and docs formats never count.@@ -104,16 +113,16 @@104113 	const { user_version } = db.query<{ user_version: number }, []>("PRAGMA user_version").get() ?? {105114 		user_version: 0,106115 	};107-	if (user_version !== 0 && user_version !== 1) {116+	if (user_version !== 0 && user_version !== SCHEMA_VERSION) {108117 		db.close();109118 		throw new Error(110-			`${path} has mirror schema version ${user_version}; this Kepr reads version 1. The mirror is disposable: delete it and the next sync rebuilds it from the bucket.`,119+			`${path} has mirror schema version ${user_version}; this Kepr reads version ${SCHEMA_VERSION}. The mirror is disposable: delete it and the next sync rebuilds it from the bucket.`,111120 		);112121 	}113122 	if (user_version === 0) {114123 		db.transaction(() => {115124 			db.run(SCHEMA);116-			db.run("PRAGMA user_version = 1");125+			db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);117126 		})();118127 	}119128 	open.set(path, db);@@ -140,6 +149,11 @@140149 	return new Set(rows.map((row) => row.id));141150 }142151 152+export function knownReleases(db: Database): Set<string> {153+	const rows = db.query<{ version: string }, []>("SELECT version FROM releases").all();154+	return new Set(rows.map((row) => row.version));155+}156+143157 /** Lands a sync like Koh's steal: objects, then events, then saves, then strips, all or nothing. */144158 export function land(db: Database, landing: Landing): void {145159 	const putObject = db.query("INSERT OR IGNORE INTO objects (hash, size, data) VALUES (?, ?, ?)");@@ -153,6 +167,8 @@153167 	const putFile = db.query(154168 		"INSERT OR IGNORE INTO save_files (save_id, path, hash, executable) VALUES (?, ?, ?, ?)",155169 	);170+	const putRelease = db.query("INSERT OR IGNORE INTO releases (version, record) VALUES (?, ?)");171+	const setState = db.query("INSERT OR REPLACE INTO state (key, value) VALUES (?, ?)");156172 	db.transaction(() => {157173 		for (const [hash, object] of landing.objects)158174 			putObject.run(hash, object.size, object.compressed);@@ -175,8 +191,10 @@175191 			);176192 			for (const f of s.files) putFile.run(s.id, f.path, f.hash, f.executable ? 1 : 0);177193 		}194+		for (const r of landing.releases) putRelease.run(r.version, JSON.stringify(r));178195 		applyStrips(db);179-		db.query("INSERT OR REPLACE INTO state (key, value) VALUES ('head', ?)").run(landing.head);196+		setState.run("head", landing.head);197+		setState.run("releases_latest", landing.releasesLatest);180198 		refreshStats(db, landing.head);181199 	})();182200 }

@@ -20,6 +20,26 @@2020 	last_sync_at    INTEGER,2121 	last_sync_error TEXT2222 );23+-- §24: the admin console hides a version or replaces its notes; NULL notes keep the record's.24+CREATE TABLE IF NOT EXISTS release_settings (25+	repo    TEXT NOT NULL,26+	version TEXT NOT NULL,27+	hidden  INTEGER NOT NULL DEFAULT 0,28+	notes   TEXT,29+	PRIMARY KEY (repo, version)30+);31+-- §22.2: one count per release file; installs are Nix tarball fetches, per repo.32+CREATE TABLE IF NOT EXISTS downloads (33+	repo     TEXT NOT NULL,34+	version  TEXT NOT NULL,35+	platform TEXT NOT NULL,36+	count    INTEGER NOT NULL DEFAULT 0,37+	PRIMARY KEY (repo, version, platform)38+);39+CREATE TABLE IF NOT EXISTS installs (40+	repo  TEXT PRIMARY KEY,41+	count INTEGER NOT NULL DEFAULT 042+);2343 `;2444 2545 export type RepoSettings = {

@@ -1,6 +1,9 @@11 import { mkdir, mkdtemp, writeFile } from "node:fs/promises";22 import { tmpdir } from "node:os";33 import { join } from "node:path";4+import { blake3 } from "@noble/hashes/blake3.js";5+import { bytesToHex } from "@noble/hashes/utils.js";6+import type { ReleaseFile } from "../src/bucket/repo.ts";47 58 export type Fixture = {69 	bucketDir: string;@@ -112,4 +115,78 @@112115 	const code = await proc.exited;113116 	proc.terminal?.close();114117 	if (code !== 0) throw new Error(`koh3 strip ${path} failed: ${output}`);118+}119+120+/** Repos with one koh3 save each, all offered to one folder remote: enough history for releases. */121+export async function makeRepos(122+	names: string[],123+): Promise<{ bucketDir: string; saves: Record<string, string> }> {124+	const root = await mkdtemp(join(tmpdir(), "kepr-repos-"));125+	const home = join(root, "home");126+	const bucketDir = join(root, "bucket");127+	await mkdir(home);128+	const saves: Record<string, string> = {};129+	for (const [i, name] of names.entries()) {130+		const repoDir = join(root, "work", name);131+		await mkdir(repoDir, { recursive: true });132+		run(repoDir, home, ["init"]);133+		if (i === 0) run(repoDir, home, ["remote", "folder", bucketDir]);134+		await writeFile(join(repoDir, "README.md"), `# ${name}\n\nThe ${name} project.\n`);135+		run(repoDir, home, ["save", `release ${name}`]);136+		saves[name] = JSON.parse(run(repoDir, home, ["log", "--json"])).head_id;137+	}138+	return { bucketDir, saves };139+}140+141+/** A release file with its bytes, or only as a record declares it (the bytes are then a stand-in). */142+export type FixtureFile = { platform: string; name: string; bytes: Uint8Array } | ReleaseFile;143+144+export type FixtureRelease = {145+	version: string;146+	save: string;147+	notes?: string;148+	publishedAt?: number;149+	files: FixtureFile[];150+};151+152+/**153+ * Publishes a release into a folder remote as `koh offer --builds` does (§11.2): the files, then154+ * release.json, then `releases/latest`. koh3 3.0.72 has no `--builds`, so this writes them itself.155+ */156+export async function writeRelease(157+	bucketDir: string,158+	repo: string,159+	release: FixtureRelease,160+	latest = true,161+): Promise<void> {162+	const dir = join(bucketDir, repo, "releases", release.version);163+	const files: ReleaseFile[] = [];164+	for (const file of release.files) {165+		const bytes = "bytes" in file ? file.bytes : new TextEncoder().encode(`stand-in ${file.name}`);166+		await mkdir(join(dir, file.platform), { recursive: true });167+		await writeFile(join(dir, file.platform, file.name), bytes);168+		files.push(169+			"bytes" in file170+				? {171+						platform: file.platform,172+						name: file.name,173+						size: bytes.length,174+						blake3: bytesToHex(blake3(bytes)),175+						sha256: new Bun.CryptoHasher("sha256").update(bytes).digest("hex"),176+					}177+				: file,178+		);179+	}180+	const record = {181+		format: 1,182+		repo,183+		version: release.version,184+		save: release.save,185+		published_at: release.publishedAt ?? 1790000000,186+		machine: "fixture",187+		notes: release.notes ?? "",188+		files,189+	};190+	await writeFile(join(dir, "release.json"), JSON.stringify(record, null, 2));191+	if (latest) await writeFile(join(bucketDir, repo, "releases", "latest"), release.version);115192 }

@@ -146,18 +146,20 @@146146 });147147 148148 describe("the mirror's schema version", () => {149-	test("a version 1 mirror reopens and any other version is refused as disposable", async () => {149+	test("a version 2 mirror reopens and any other version is refused as disposable", async () => {150150 		const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));151151 		const created = openMirror(dir, "one");152152 		created.run("PRAGMA wal_checkpoint(TRUNCATE)");153153 		await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));154154 		expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({155-			user_version: 1,155+			user_version: 2,156156 		});157157 158-		const future = new Database(join(dir, "future.db"));159-		future.run("PRAGMA user_version = 2");160-		future.close();161-		expect(() => openMirror(dir, "future")).toThrow("disposable");158+		for (const version of [1, 3]) {159+			const other = new Database(join(dir, `v${version}.db`));160+			other.run(`PRAGMA user_version = ${version}`);161+			other.close();162+			expect(() => openMirror(dir, `v${version}`)).toThrow("disposable");163+		}162164 	});163165 });