1import { afterAll, beforeAll, expect, test } from "bun:test";2import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises";3import { tmpdir } from "node:os";4import { join } from "node:path";5import { blake3 } from "@noble/hashes/blake3.js";6import { bytesToHex } from "@noble/hashes/utils.js";7import { SQL } from "bun";8import { instance } from "./browser.ts";9import { makeRepos } from "./fixture.ts";101112const SCRIPT = join(import.meta.dir, "../scripts/import-kepr2.ts");13const root = await mkdtemp(join(tmpdir(), "kepr-import-"));14const pgDir = join(root, "pg");15const kepr2Data = join(root, "kepr2");16let url = "";17let bucketDir = "";18let save = "";1920const hex = (bytes: Uint8Array) => bytesToHex(blake3(bytes));21const sha = (bytes: Uint8Array) => new Bun.CryptoHasher("sha256").update(bytes).digest("hex");22const bytesOf = (text: string) => new TextEncoder().encode(text);23const run = (cmd: string[]) => {24 const out = Bun.spawnSync(cmd, { cwd: root, env: { PATH: process.env.PATH, LC_ALL: "C" } });25 if (out.exitCode !== 0) throw new Error(`${cmd.join(" ")}: ${out.stderr.toString()}`);26};2728async function kepr2File(path: string, text: string) {29 await mkdir(join(kepr2Data, "releases", path, ".."), { recursive: true });30 await writeFile(join(kepr2Data, "releases", path), text);31 return bytesOf(text);32}3334beforeAll(async () => {35 const repos = await makeRepos(["demo"]);36 bucketDir = repos.bucketDir;37 save = repos.saves.demo as string;38 const probe = Bun.serve({ port: 0, fetch: () => new Response() });39 const port = probe.port;40 probe.stop(true);41 run(["initdb", "-D", pgDir, "-U", "kepr", "-A", "trust"]);42 const options = `-p ${port} -k ${pgDir} -c listen_addresses=127.0.0.1`;43 run(["pg_ctl", "-D", pgDir, "-o", options, "-l", join(root, "pg.log"), "-w", "start"]);44 url = `postgres://[email protected]:${port}/postgres`;45 run([46 "psql",47 url,48 "-q",49 "-v",50 "ON_ERROR_STOP=1",51 "-f",52 join(import.meta.dir, "fixtures/kepr2/schema.sql"),53 ]);5455 const legacy = await kepr2File("demo/1.0.0/linux-x86_64/demo", "demo 1.0.0 linux");56 const byPlatform = await kepr2File("demo/1.1.0/macos-aarch64", "demo 1.1.0 mac");57 const apk = await kepr2File("demo/1.1.0/android-arm64/demo-1.1.0.apk", "demo 1.1.0 apk");58 const failed = await kepr2File("demo/1.2.0/linux-x86_64/demo", "demo 1.2.0 failed");59 await kepr2File("demo/1.3.0/linux-x86_64/demo", "demo 1.3.0 tampered");60 const secret = await kepr2File("secret/0.1.0/linux-x86_64/secret", "secret 0.1.0");61 const oldPlatform = await kepr2File(62 "demo/1.4.0/x86_64-linux/demo",63 "demo 1.4.0 old platform name",64 );65 const artifact = (66 id: string,67 run: string,68 repo: string,69 version: string,70 platform: string,71 name: string,72 bytes: Uint8Array,73 b3 = hex(bytes),74 ) =>75 `('${id}', '${run}', '${repo}', '${version}', '${platform}', '${name}', ${bytes.length}, '${sha(bytes)}', '${b3}', 5)`;76 const pg = new SQL(url);77 await pg.unsafe(`78 INSERT INTO repos (name, created_at, description, categories, private, builds_enabled) VALUES79 ('demo', 1, 'A demo.', 'tools, cli', 0, 1), ('secret', 1, NULL, '', 1, 0);80 INSERT INTO accounts (id, username, recovery_hash, created_at) VALUES ('acc-r', 'rita', 'x', 1), ('acc-o', 'olga', 'x', 1);81 INSERT INTO issues (id, repo, number, title, description, contact, contact_email, filed_at, filed_save, filed_ver, status, closed_at, closed_save, type, account_id, reporter_name) VALUES82 (1, 'demo', 7, 'Crash on start', 'It <b>crashes</b>.', NULL, '[email protected]', 1000, 'aaaaaaaaaaaa', '1.0.0', 'inprogress', NULL, NULL, 'bug', 'acc-r', NULL),83 (2, 'demo', 9, 'Dark mode', 'Please.', '[email protected]', NULL, 1500, 'aaaaaaaaaaaa', NULL, 'wontfix', 2000, '${save}', 'enhancement', NULL, 'Sam');84 INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES85 ('e1', 1, 'operator', 'reply', 'Looking.', 1100), ('e2', 1, 'acc-r', 'reply', 'Thanks!', 1200),86 ('e3', 1, 'operator', 'status_change', NULL, 1300), ('e4', 1, 'acc-o', 'reply', 'Me too.', 1400);87 INSERT INTO issue_notes (issue_id, body, posted_at, is_op) VALUES (1, 'internal only', 1250, 1);88 INSERT INTO engagement_events (id, event_type, repo_id, occurred_at) VALUES89 ('v1', 'nix_install', 'demo', 1), ('v2', 'nix_install', 'demo', 2), ('v3', 'nix_install', 'demo', 3),90 ('v4', 'archive_download', 'demo', 4), ('v5', 'nix_install', 'secret', 5);91 INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at, release_notes) VALUES92 ('b1', 'demo', '${save}', '1.0.0', 'manual', 'success', 10, 20, NULL),93 ('b2', 'demo', '${save}', '1.1.0', 'manual', 'partial', 30, 40, 'Starts twice as fast and no longer crashes on start.'),94 ('b3', 'demo', '${save}', '1.2.0', 'manual', 'failed', 50, 60, NULL),95 ('b4', 'demo', '${save}', '1.3.0', 'manual', 'success', 70, 80, NULL),96 ('b5', 'secret', '${save}', '0.1.0', 'manual', 'success', 90, 100, NULL),97 ('b6', 'demo', '${save}', '1.4.0', 'manual', 'success', 110, 120, NULL);98 INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES99 ${artifact("a1", "b1", "demo", "1.0.0", "linux-x86_64", "demo", legacy)},100 ${artifact("a2", "b2", "demo", "1.1.0", "macos-aarch64", "demo", byPlatform)},101 ${artifact("a3", "b2", "demo", "1.1.0", "android-arm64", "demo-1.1.0.apk", apk)},102 ${artifact("a4", "b3", "demo", "1.2.0", "linux-x86_64", "demo", failed)},103 ${artifact("a5", "b4", "demo", "1.3.0", "linux-x86_64", "demo", legacy, hex(legacy))},104 ${artifact("a6", "b5", "secret", "0.1.0", "linux-x86_64", "secret", secret)},105 ${artifact("a7", "b6", "demo", "1.4.0", "x86_64-linux", "demo", oldPlatform)};106 `);107 await pg.close();108});109110afterAll(() => run(["pg_ctl", "-D", pgDir, "-m", "fast", "-w", "stop"]));111112async function importKepr2(target: Record<string, string>, keprData: string, ...args: string[]) {113 const env = {114 PATH: process.env.PATH,115 KEPR2_DATABASE_URL: url,116 KEPR2_DATA: kepr2Data,117 KEPR_DATA: keprData,118 ...target,119 };120 121 const child = Bun.spawn(["bun", SCRIPT, ...args], { cwd: root, env, stderr: "pipe" });122 const [stdout, stderr, code] = await Promise.all([123 new Response(child.stdout).text(),124 new Response(child.stderr).text(),125 child.exited,126 ]);127 if (code !== 0) throw new Error(stderr);128 return stdout;129}130131async function files(dir: string): Promise<string[]> {132 return (await readdir(dir, { recursive: true, withFileTypes: true }))133 .filter((entry) => entry.isFile())134 .map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1))135 .sort();136}137138test("--dry-run prints the import and writes neither the bucket nor kepr.db", async () => {139 const before = await files(bucketDir);140 const keprData = await mkdtemp(join(tmpdir(), "kepr-import-dry-"));141 const out = await importKepr2({ KOH_FOLDER: bucketDir }, keprData, "--dry-run");142 expect(out).toContain("would put demo/releases/1.1.0/release.json");143 expect(out).toContain('issues {"repo":"demo","number":"7"');144 expect(await files(bucketDir)).toEqual(before);145 const inst = await instance(bucketDir, keprData);146 expect(inst.kepr.query("SELECT COUNT(*) AS n FROM issues").get()).toEqual({ n: 0 });147});148149test("the import lands Kepr 2's issues, counts, settings and releases, and a re-run changes nothing", async () => {150 const keprData = await mkdtemp(join(tmpdir(), "kepr-import-data-"));151 const first = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);152 expect(first).toContain("skip demo/releases/1.3.0: linux-x86_64: not Kepr 2's file");153 expect(first).toContain("skip demo/releases/1.4.0: release record does not match §11.2");154 expect(first).not.toContain("1.2.0");155 const landed = await files(bucketDir);156 expect(landed.filter((key) => key.includes("/releases/"))).toEqual([157 "demo/releases/1.0.0/linux-x86_64/demo-linux-x86_64",158 "demo/releases/1.0.0/release.json",159 "demo/releases/1.1.0/android-arm64/demo-1.1.0.apk",160 "demo/releases/1.1.0/macos-aarch64/demo-macos-aarch64",161 "demo/releases/1.1.0/release.json",162 "demo/releases/latest",163 "secret/releases/0.1.0/linux-x86_64/secret-linux-x86_64",164 "secret/releases/0.1.0/release.json",165 "secret/releases/latest",166 ]);167 const again = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);168 expect(again).not.toMatch(/^(put|would put|differs)/m);169 170 expect(171 again.split("\n").filter((line) => line.endsWith(" set") && !line.startsWith("repos ")),172 ).toEqual([]);173 expect(await files(bucketDir)).toEqual(landed);174175 const inst = await instance(bucketDir, keprData);176 const q = (sql: string) => inst.kepr.query(sql).all();177 expect(q("SELECT name, public, category FROM repos ORDER BY name")).toEqual([178 { name: "demo", public: 1, category: "tools" },179 { name: "secret", public: 0, category: "" },180 ]);181 expect(q("SELECT repo, count FROM installs ORDER BY repo")).toEqual([182 { repo: "demo", count: 3 },183 { repo: "secret", count: 1 },184 ]);185 expect(q("SELECT COUNT(*) AS n FROM downloads")).toEqual([{ n: 0 }]);186 expect(q("SELECT repo, version, hidden FROM release_settings")).toEqual([187 { repo: "secret", version: "0.1.0", hidden: 1 },188 ]);189 expect(190 q(191 "SELECT number, title, body, type, version, reporter_name, contact_email, filed_at, status, closed_at, fixed_in, token_hash FROM issues ORDER BY number",192 ),193 ).toEqual([194 {195 number: 7,196 title: "Crash on start",197 body: "It <b>crashes</b>.",198 type: "bug",199 version: "1.0.0",200 reporter_name: "@rita",201 contact_email: "[email protected]",202 filed_at: 1000,203 status: "open",204 closed_at: null,205 fixed_in: "",206 token_hash: null,207 },208 {209 number: 9,210 title: "Dark mode",211 body: "Please.",212 type: "idea",213 version: "",214 reporter_name: "Sam",215 contact_email: "[email protected]",216 filed_at: 1500,217 status: "closed",218 closed_at: 2000,219 fixed_in: save,220 token_hash: null,221 },222 ]);223 expect(q("SELECT author, body, created_at FROM issue_entries ORDER BY id")).toEqual([224 { author: "operator", body: "Looking.", created_at: 1100 },225 { author: "reporter", body: "Thanks!", created_at: 1200 },226 { author: "operator", body: "Me too.", created_at: 1400 },227 ]);228229 230 type Latest = { version: string; notes: string; platforms: Record<string, unknown> };231 const latest = (await (await inst.app.request("/api/demo/releases/latest")).json()) as Latest;232 expect(latest.version).toBe("1.1.0");233 expect(latest.notes).toBe("Starts twice as fast and no longer crashes on start.");234 expect(Object.keys(latest.platforms)).toEqual(["android-arm64", "macos-aarch64"]);235 const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).json();236 const legacy = bytesOf("demo 1.0.0 linux");237 expect(record).toEqual({238 format: 1,239 repo: "demo",240 version: "1.0.0",241 save,242 published_at: 20,243 machine: "kepr2",244 notes: "",245 files: [246 {247 platform: "linux-x86_64",248 name: "demo-linux-x86_64",249 size: legacy.length,250 blake3: hex(legacy),251 sha256: sha(legacy),252 },253 ],254 });255 expect(await (await inst.app.request("/demo/issues/7")).text()).toContain("Me too.");256 expect(inst.kepr.query("SELECT last_sync_error FROM repos WHERE name = 'demo'").get()).toEqual({257 last_sync_error: null,258 });259});260261test("into an S3 bucket, files go up as presigned PUTs with If-None-Match, never over an existing key", async () => {262 const s3Dir = await mkdtemp(join(tmpdir(), "kepr-import-s3-"));263 const puts: string[] = [];264 const server = Bun.serve({265 port: 0,266 async fetch(req) {267 const url = new URL(req.url);268 const path = join(s3Dir, decodeURIComponent(url.pathname.replace("/koh/", "")));269 const file = Bun.file(path);270 if (req.method === "GET")271 return (await file.exists())272 ? new Response(file)273 : new Response("<Error><Code>NoSuchKey</Code></Error>", { status: 404 });274 expect([req.headers.get("if-none-match"), url.searchParams.has("X-Amz-Signature")]).toEqual([275 "*",276 true,277 ]);278 if (await file.exists()) return new Response("", { status: 412 });279 puts.push(url.pathname);280 await Bun.write(path, await req.arrayBuffer());281 return new Response("");282 },283 });284 try {285 const target = {286 KOH_S3_ENDPOINT: `http://localhost:${server.port}`,287 KOH_S3_BUCKET: "koh",288 KOH_S3_REGION: "fr-par",289 KOH_S3_ACCESS_KEY_ID: "write",290 KOH_S3_SECRET_ACCESS_KEY: "key",291 };292 await importKepr2(target, await mkdtemp(join(tmpdir(), "kepr-import-s3data-")));293 expect(puts.length).toBe(9);294 for (const key of await files(s3Dir)) {295 expect(await Bun.file(join(s3Dir, key)).bytes()).toEqual(296 await Bun.file(join(bucketDir, key)).bytes(),297 );298 }299 await importKepr2(target, await mkdtemp(join(tmpdir(), "kepr-import-s3data-")));300 expect(puts.length).toBe(9);301 } finally {302 server.stop(true);303 }304});