import-kepr2.ts

1// One-time Kepr 2 import (§28), deleted after cutover; the only bucket writer. Env: KEPR2_DATABASE_URL,2// KEPR2_DATA, KEPR_DATA, KOH_FOLDER or KOH_S3_* (write key at run time only). --dry-run writes nothing.3import { join } from "node:path";4import { blake3 } from "@noble/hashes/blake3.js";5import { bytesToHex } from "@noble/hashes/utils.js";6import { SQL } from "bun";7import { openBucket } from "../src/bucket/reader.ts";8import { checkRelease } from "../src/bucket/repo.ts";9import { compareVersions } from "../src/releases/releases.ts";10import { openKeprDb } from "../src/server/db.ts";1112type Row = Record<string, string | number | null>;13const env = (name: string) => process.env[name] ?? "";14const dry = process.argv.includes("--dry-run");15const pg = new SQL(env("KEPR2_DATABASE_URL"));16const kepr = openKeprDb(env("KEPR_DATA"));17const folder = env("KOH_FOLDER");18const VARS = "ENDPOINT BUCKET REGION ACCESS_KEY_ID SECRET_ACCESS_KEY".split(" ");19const [endpoint, name, region, accessKeyId, secretAccessKey] = VARS.map((v) => env(`KOH_S3_${v}`));20const s3 = { endpoint, region, accessKeyId, secretAccessKey };21const bucket = openBucket(folder ? { kind: "folder", path: folder } : { kind: "s3", name, ...s3 });22/** put_if_absent (§9.1): true when `key` holds `bytes` after; a key holding other bytes is kept. */23async function put(key: string, body: string | Uint8Array): Promise<boolean> {24	const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;25	const had = await bucket.get(key);26	const same = had === null || Bun.deepEquals(had, bytes);27	console.log(had === null ? (dry ? "would put" : "put") : same ? "have" : "differs, kept:", key);28	if (had !== null || dry) return same;29	if (folder) return Bun.write(join(folder, key), bytes).then(() => true);30	// Bun's S3 client has no conditional write, so a presigned PUT carries it.31	const url = new Bun.S3Client({ ...s3, bucket: name }).presign(key, { method: "PUT" });32	const res = await fetch(url, { method: "PUT", body: bytes, headers: { "If-None-Match": "*" } });33	if (!res.ok) throw new Error(`put ${key}: ${res.status}`);34	return true;35}36function insert(table: string, row: Row, tail: string): number | null {37	const cols = Object.keys(row);38	const sql = `INSERT INTO ${table} (${cols}) VALUES (${cols.map((c) => `$${c}`)}) ${tail} RETURNING rowid AS id`;39	const id = kepr.query<{ id: number }, [Row]>(sql).get(row)?.id ?? null;40	console.log(table, JSON.stringify(row).slice(0, 80), id ? "set" : "have");41	return id;42}43kepr.run("BEGIN");44// Kepr 3 keeps one category and has no description field; Kepr 2 had no homepage.45const upsert = "ON CONFLICT DO UPDATE SET public = excluded.public, category = excluded.category";46for (const repo of await pg<Row[]>`47	SELECT name, (private = 0)::int AS public, trim(split_part(COALESCE(categories, ''), ',', 1)) AS category FROM repos`)48	insert("repos", repo, upsert);49// Kepr 2's four statuses become two (Phase 5); "enhancement" is an idea. Any account's reply but the50// reporter's own is the operator's, since §23's thread has no other author.51for (const { replies, ...issue } of await pg<(Row & { replies: Row[] })[]>`52	SELECT i.repo, i.number, i.title, i.description AS body, CASE WHEN i.type IN ('bug', 'question') THEN i.type ELSE 'idea' END AS type,53		COALESCE(i.filed_ver, '') AS version, COALESCE(NULLIF(i.reporter_name, ''), '@' || a.username, '') AS reporter_name,54		COALESCE(NULLIF(i.contact_email, ''), i.contact, '') AS contact_email, i.filed_at,55		CASE WHEN i.status IN ('resolved', 'wontfix', 'closed') THEN 'closed' ELSE 'open' END AS status,56		CASE WHEN i.status IN ('resolved', 'wontfix', 'closed') THEN COALESCE(i.closed_at, i.filed_at) END AS closed_at,57		CASE WHEN i.closed_save ~ '^[0-9a-f]{12}$' THEN i.closed_save ELSE '' END AS fixed_in,58		(SELECT COALESCE(json_agg(json_build_object('author', CASE WHEN e.account_id = i.account_id THEN 'reporter' ELSE 'operator' END,59			'body', e.body, 'created_at', e.created_at) ORDER BY e.created_at, e.id), '[]') FROM issue_entries e WHERE e.issue_id = i.id AND e.entry_type = 'reply' AND e.body IS NOT NULL) AS replies60	FROM issues i LEFT JOIN accounts a ON a.id = i.account_id ORDER BY i.repo, i.number`) {61	const id = insert("issues", issue, "ON CONFLICT DO NOTHING");62	for (const reply of id ? replies : []) insert("issue_entries", { issue_id: id, ...reply }, "");63}64// Kepr 2 counted Nix fetches as installs (kept 90 days) and never counted release downloads.65for (const count of await pg<Row[]>`66	SELECT repo_id AS repo, COUNT(*) AS count FROM engagement_events WHERE event_type = 'nix_install' AND repo_id IS NOT NULL GROUP BY repo_id`)67	insert("installs", count, "ON CONFLICT DO NOTHING");68// As Kepr 2 served: successful and partial builds, a file stored under the repo's name as <repo>-<platform>,69// older uploads' files named by platform; builds off hid a repo's. Save, notes, time: the newest build's.70type Release = { repo: string; version: string; hidden: boolean; record: Row; stored: string[] };71const newest = new Map<string, string>();72for (const { repo, version, hidden, record, stored } of await pg<Release[]>`73	SELECT a.repo_id AS repo, a.version, bool_or(r.builds_enabled = 0) AS hidden, json_agg(a.filename ORDER BY a.platform) AS stored,74		json_build_object('format', 1, 'repo', a.repo_id, 'version', a.version, 'save', (array_agg(b.save_id ORDER BY b.started_at DESC))[1],75			'published_at', max(COALESCE(b.finished_at, a.built_at)), 'machine', 'kepr2', 'notes', COALESCE((array_agg(b.release_notes ORDER BY b.started_at DESC))[1], ''),76			'files', json_agg(json_build_object('platform', a.platform, 'name', CASE WHEN a.filename = a.repo_id THEN a.repo_id || '-' || a.platform ELSE a.filename END,77				'size', a.size_bytes, 'blake3', a.blake3, 'sha256', a.sha256) ORDER BY a.platform)) AS record78	FROM release_artifacts a JOIN build_runs b ON b.id = a.build_run JOIN repos r ON r.name = a.repo_id WHERE b.status IN ('success', 'partial') GROUP BY a.repo_id, a.version ORDER BY a.repo_id, a.version`) {79	const prefix = `${repo}/releases/${version}`;80	try {81		for (const [n, f] of checkRelease(repo, version, record).files.entries()) {82			const dir = join(env("KEPR2_DATA"), "releases", repo, version, f.platform);83			const path = join(dir, stored[n] ?? "");84			const bytes = await Bun.file((await Bun.file(path).exists()) ? path : dir).bytes();85			const kepr2s = bytesToHex(blake3(bytes)) === f.blake3;86			if (!kepr2s || !(await put(`${prefix}/${f.platform}/${f.name}`, bytes)))87				throw new Error(`${f.platform}: not Kepr 2's file, or another is there`);88		}89		if (!(await put(`${prefix}/release.json`, JSON.stringify(record)))) continue;90		if (hidden) insert("release_settings", { repo, version, hidden: 1 }, "ON CONFLICT DO NOTHING");91		if (compareVersions(version, newest.get(repo) ?? "") > 0) newest.set(repo, version);92	} catch (error) {93		console.log(`skip ${prefix}:`, error instanceof Error ? error.message : error);94	}95}96for (const [repo, version] of newest) await put(`${repo}/releases/latest`, version);97kepr.run(dry ? "ROLLBACK" : "COMMIT");98await pg.close();