import { afterAll, beforeAll, expect, test } from "bun:test"; import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { blake3 } from "@noble/hashes/blake3.js"; import { bytesToHex } from "@noble/hashes/utils.js"; import { SQL } from "bun"; import { instance } from "./browser.ts"; import { makeRepos } from "./fixture.ts"; // A throwaway Postgres holding Kepr 2's schema and a few synthetic rows, stopped after the tests. const SCRIPT = join(import.meta.dir, "../scripts/import-kepr2.ts"); const root = await mkdtemp(join(tmpdir(), "kepr-import-")); const pgDir = join(root, "pg"); const kepr2Data = join(root, "kepr2"); let url = ""; let bucketDir = ""; let save = ""; const hex = (bytes: Uint8Array) => bytesToHex(blake3(bytes)); const sha = (bytes: Uint8Array) => new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); const bytesOf = (text: string) => new TextEncoder().encode(text); const run = (cmd: string[]) => { const out = Bun.spawnSync(cmd, { cwd: root, env: { PATH: process.env.PATH, LC_ALL: "C" } }); if (out.exitCode !== 0) throw new Error(`${cmd.join(" ")}: ${out.stderr.toString()}`); }; const artifact = ( id: string, run: string, repo: string, version: string, platform: string, name: string, bytes: Uint8Array, b3 = hex(bytes), ) => `('${id}', '${run}', '${repo}', '${version}', '${platform}', '${name}', ${bytes.length}, '${sha(bytes)}', '${b3}', 5)`; async function kepr2File(path: string, text: string) { await mkdir(join(kepr2Data, "releases", path, ".."), { recursive: true }); await writeFile(join(kepr2Data, "releases", path), text); return bytesOf(text); } beforeAll(async () => { const repos = await makeRepos(["demo"]); bucketDir = repos.bucketDir; save = repos.saves.demo as string; const probe = Bun.serve({ port: 0, fetch: () => new Response() }); const port = probe.port; probe.stop(true); run(["initdb", "-D", pgDir, "-U", "kepr", "-A", "trust"]); const options = `-p ${port} -k ${pgDir} -c listen_addresses=127.0.0.1`; run(["pg_ctl", "-D", pgDir, "-o", options, "-l", join(root, "pg.log"), "-w", "start"]); url = `postgres://kepr@127.0.0.1:${port}/postgres`; run([ "psql", url, "-q", "-v", "ON_ERROR_STOP=1", "-f", join(import.meta.dir, "fixtures/kepr2/schema.sql"), ]); const legacy = await kepr2File("demo/1.0.0/linux-x86_64/demo", "demo 1.0.0 linux"); const byPlatform = await kepr2File("demo/1.1.0/macos-aarch64", "demo 1.1.0 mac"); const apk = await kepr2File("demo/1.1.0/android-arm64/demo-1.1.0.apk", "demo 1.1.0 apk"); const failed = await kepr2File("demo/1.2.0/linux-x86_64/demo", "demo 1.2.0 failed"); const secret = await kepr2File("secret/0.1.0/linux-x86_64/secret", "secret 0.1.0"); const pg = new SQL(url); await pg.unsafe(` INSERT INTO repos (name, created_at, description, categories, private, builds_enabled) VALUES ('demo', 1, 'A demo.', 'tools, cli', 0, 1), ('secret', 1, NULL, '', 1, 0); INSERT INTO accounts (id, username, recovery_hash, created_at, tier) VALUES ('acc-r', 'rita', 'x', 1, 'contributor'), ('acc-o', 'olga', 'x', 1, 'operator'), ('acc-t', 'tom', 'x', 1, 'team'); INSERT INTO issues (id, repo, number, title, description, contact, contact_email, filed_at, filed_save, filed_ver, status, closed_at, closed_save, type, account_id, reporter_name) VALUES (1, 'demo', 7, 'Crash on start', 'It crashes.', NULL, 'rita@example.com', 1000, 'aaaaaaaaaaaa', '1.0.0', 'inprogress', NULL, NULL, 'bug', 'acc-r', NULL), (2, 'demo', 9, 'Dark mode', 'Please.', 'sam@example.com', NULL, 1500, 'aaaaaaaaaaaa', NULL, 'wontfix', 2000, '${save}', 'enhancement', NULL, 'Sam'); INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES ('e1', 1, 'operator', 'reply', 'Looking.', 1100), ('e2', 1, 'acc-r', 'reply', 'Thanks!', 1200), ('e3', 1, 'operator', 'status_change', NULL, 1300), ('e4', 1, 'acc-o', 'reply', 'Me too.', 1400); INSERT INTO issue_notes (issue_id, body, posted_at, is_op) VALUES (1, 'internal only', 1250, 1); INSERT INTO engagement_events (id, event_type, repo_id, occurred_at) VALUES ('v1', 'nix_install', 'demo', 1), ('v2', 'nix_install', 'demo', 2), ('v3', 'nix_install', 'demo', 3), ('v4', 'archive_download', 'demo', 4), ('v5', 'nix_install', 'secret', 5); INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at, release_notes) VALUES ('b1', 'demo', '${save}', '1.0.0', 'manual', 'success', 10, 20, NULL), ('b2', 'demo', '${save}', '1.1.0', 'manual', 'partial', 30, 40, 'Starts twice as fast and no longer crashes on start.'), ('b3', 'demo', '${save}', '1.2.0', 'manual', 'failed', 50, 60, NULL), ('b5', 'secret', '${save}', '0.1.0', 'manual', 'success', 90, 100, NULL); INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES ${artifact("a1", "b1", "demo", "1.0.0", "linux-x86_64", "demo", legacy)}, ${artifact("a2", "b2", "demo", "1.1.0", "macos-aarch64", "demo", byPlatform)}, ${artifact("a3", "b2", "demo", "1.1.0", "android-arm64", "demo-1.1.0.apk", apk)}, ${artifact("a4", "b3", "demo", "1.2.0", "linux-x86_64", "demo", failed)}, ${artifact("a6", "b5", "secret", "0.1.0", "linux-x86_64", "secret", secret)}; `); await pg.close(); }); afterAll(() => run(["pg_ctl", "-D", pgDir, "-m", "fast", "-w", "stop"])); async function runImport(target: Record, keprData: string, ...args: string[]) { const env = { PATH: process.env.PATH, KEPR2_DATABASE_URL: url, KEPR2_DATA: kepr2Data, KEPR_DATA: keprData, ...target, }; // Not spawnSync: the S3 tests' stand-in server runs on this process's event loop. const child = Bun.spawn(["bun", SCRIPT, ...args], { cwd: root, env, stderr: "pipe" }); const [out, err, code] = await Promise.all([ new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited, ]); return { out, err, code }; } async function importKepr2(target: Record, keprData: string, ...args: string[]) { const { out, err, code } = await runImport(target, keprData, ...args); if (code !== 0) throw new Error(err); return out; } async function files(dir: string): Promise { return (await readdir(dir, { recursive: true, withFileTypes: true })) .filter((entry) => entry.isFile()) .map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1)) .sort(); } test("--dry-run prints the import and writes neither the bucket nor kepr.db", async () => { const before = await files(bucketDir); const keprData = await mkdtemp(join(tmpdir(), "kepr-import-dry-")); const out = await importKepr2({ KOH_FOLDER: bucketDir }, keprData, "--dry-run"); expect(out).toContain("would put demo/releases/1.1.0/release.json"); expect(out).toContain('issues {"repo":"demo","number":"7"'); expect(await files(bucketDir)).toEqual(before); const inst = await instance(bucketDir, keprData); expect(inst.kepr.query("SELECT COUNT(*) AS n FROM issues").get()).toEqual({ n: 0 }); }); test("the import lands Kepr 2's issues, counts, settings and releases, and a re-run changes nothing", async () => { const keprData = await mkdtemp(join(tmpdir(), "kepr-import-data-")); const first = await importKepr2({ KOH_FOLDER: bucketDir }, keprData); expect(first).not.toContain("1.2.0"); const landed = await files(bucketDir); expect(landed.filter((key) => key.includes("/releases/"))).toEqual([ "demo/releases/1.0.0/linux-x86_64/demo-linux-x86_64", "demo/releases/1.0.0/release.json", "demo/releases/1.1.0/android-arm64/demo-1.1.0.apk", "demo/releases/1.1.0/macos-aarch64/demo-macos-aarch64", "demo/releases/1.1.0/release.json", "demo/releases/latest", "secret/releases/0.1.0/linux-x86_64/secret-linux-x86_64", "secret/releases/0.1.0/release.json", "secret/releases/latest", ]); const again = await importKepr2({ KOH_FOLDER: bucketDir }, keprData); expect(again).not.toMatch(/^(put|would put)/m); // Only the repo settings are set again (to the same values); nothing else is added. expect( again.split("\n").filter((line) => line.endsWith(" set") && !line.startsWith("repos ")), ).toEqual([]); expect(await files(bucketDir)).toEqual(landed); const inst = await instance(bucketDir, keprData); const q = (sql: string) => inst.kepr.query(sql).all(); expect(q("SELECT name, public, category FROM repos ORDER BY name")).toEqual([ { name: "demo", public: 1, category: "tools" }, { name: "secret", public: 0, category: "" }, ]); expect(q("SELECT repo, count FROM installs ORDER BY repo")).toEqual([ { repo: "demo", count: 3 }, { repo: "secret", count: 1 }, ]); expect(q("SELECT COUNT(*) AS n FROM downloads")).toEqual([{ n: 0 }]); expect(q("SELECT repo, version, hidden FROM release_settings")).toEqual([ { repo: "secret", version: "0.1.0", hidden: 1 }, ]); expect( q( "SELECT number, title, body, type, version, reporter_name, contact_email, filed_at, status, closed_at, fixed_in FROM issues ORDER BY number", ), ).toEqual([ { number: 7, title: "Crash on start", body: "It crashes.", type: "bug", version: "1.0.0", reporter_name: "@rita", contact_email: "rita@example.com", filed_at: 1000, status: "open", closed_at: null, fixed_in: "", }, { number: 9, title: "Dark mode", body: "Please.", type: "idea", version: "", reporter_name: "Sam", contact_email: "sam@example.com", filed_at: 1500, status: "closed", closed_at: 2000, fixed_in: save, }, ]); expect(q("SELECT author, name, body, created_at FROM issue_entries ORDER BY id")).toEqual([ { author: "operator", name: "", body: "Looking.", created_at: 1100 }, { author: "reporter", name: "@rita", body: "Thanks!", created_at: 1200 }, { author: "operator", name: "", body: "Me too.", created_at: 1400 }, ]); // Kepr 3 reads what the import wrote as any Koh 3 release. type Latest = { version: string; notes: string; platforms: Record }; const latest = (await (await inst.app.request("/api/demo/releases/latest")).json()) as Latest; expect(latest.version).toBe("1.1.0"); expect(latest.notes).toBe("Starts twice as fast and no longer crashes on start."); expect(Object.keys(latest.platforms)).toEqual(["android-arm64", "macos-aarch64"]); const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).json(); const legacy = bytesOf("demo 1.0.0 linux"); expect(record).toEqual({ format: 1, repo: "demo", version: "1.0.0", save, published_at: 20, machine: "kepr2", notes: "", files: [ { platform: "linux-x86_64", name: "demo-linux-x86_64", size: legacy.length, blake3: hex(legacy), sha256: sha(legacy), }, ], }); expect(await (await inst.app.request("/demo/issues/7")).text()).toContain("Me too."); expect(inst.kepr.query("SELECT last_sync_error FROM repos WHERE name = 'demo'").get()).toEqual({ last_sync_error: null, }); }); type S3 = { dir: string; puts: string[]; target: Record; stop(): void }; /** A stand-in S3 over a folder. A key in `race` appears, holding those bytes, between our GET and PUT. */ async function standInS3(race = new Map()): Promise { const dir = await mkdtemp(join(tmpdir(), "kepr-import-s3-")); const puts: string[] = []; const server = Bun.serve({ port: 0, async fetch(req) { const url = new URL(req.url); const key = decodeURIComponent(url.pathname.replace("/koh/", "")); const file = Bun.file(join(dir, key)); if (req.method === "GET") return (await file.exists()) ? new Response(file) : new Response("NoSuchKey", { status: 404 }); expect([req.headers.get("if-none-match"), url.searchParams.has("X-Amz-Signature")]).toEqual([ "*", true, ]); const raced = race.get(key); if (raced !== undefined) await Bun.write(join(dir, key), raced); if (await file.exists()) return new Response("", { status: 412 }); puts.push(key); await Bun.write(join(dir, key), await req.arrayBuffer()); return new Response(""); }, }); const target = { KOH_S3_ENDPOINT: `http://localhost:${server.port}`, KOH_S3_BUCKET: "koh", KOH_S3_REGION: "fr-par", KOH_S3_ACCESS_KEY_ID: "write", KOH_S3_SECRET_ACCESS_KEY: "key", }; return { dir, puts, target, stop: () => server.stop(true) }; } const newKeprData = () => mkdtemp(join(tmpdir(), "kepr-import-s3data-")); test("into an S3 bucket, files go up as presigned PUTs with If-None-Match, never over an existing key", async () => { const s3 = await standInS3(); try { await importKepr2(s3.target, await newKeprData()); expect(s3.puts.length).toBe(9); for (const key of await files(s3.dir)) { expect(await Bun.file(join(s3.dir, key)).bytes()).toEqual( await Bun.file(join(bucketDir, key)).bytes(), ); } await importKepr2(s3.target, await newKeprData()); expect(s3.puts.length).toBe(9); } finally { s3.stop(); } }); test("a 412 from a key written since the read is success only when it holds the same bytes", async () => { const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).text(); const same = await standInS3(new Map([["demo/releases/1.0.0/release.json", record]])); try { await importKepr2(same.target, await newKeprData()); expect(same.puts).not.toContain("demo/releases/1.0.0/release.json"); expect(same.puts.length).toBe(8); } finally { same.stop(); } const other = await standInS3(new Map([["demo/releases/latest", "9.9.9"]])); try { const run = await runImport(other.target, await newKeprData()); expect(run.code).toBe(1); expect(run.err).toContain("demo/releases/latest holds other bytes"); expect(await Bun.file(join(other.dir, "demo/releases/latest")).text()).toBe("9.9.9"); } finally { other.stop(); } }); test("any imported row that fails a check fails the run: all are listed and nothing is written", async () => { await kepr2File("demo/1.3.0/linux-x86_64/demo", "demo 1.3.0 tampered"); const oldPlatform = await kepr2File("demo/1.4.0/x86_64-linux/demo", "demo 1.4.0 old name"); const legacy = bytesOf("demo 1.0.0 linux"); const pg = new SQL(url); await pg.unsafe(` INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at) VALUES ('b4', 'demo', '${save}', '1.3.0', 'manual', 'success', 70, 80), ('b6', 'demo', '${save}', '1.4.0', 'manual', 'success', 110, 120); INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES ${artifact("a5", "b4", "demo", "1.3.0", "linux-x86_64", "demo", legacy)}, ${artifact("a7", "b6", "demo", "1.4.0", "x86_64-linux", "demo", oldPlatform)}; INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES ('e5', 2, 'acc-t', 'reply', 'Team here.', 1600); `); await pg.close(); const before = await files(bucketDir); const keprData = await mkdtemp(join(tmpdir(), "kepr-import-bad-")); const run = await runImport({ KOH_FOLDER: bucketDir }, keprData); expect(run.code).toBe(1); expect(run.err).toContain("not imported, nothing written:"); expect(run.err).toContain("demo#9: third-party reply"); expect(run.err).toContain("demo/releases/1.3.0: linux-x86_64: hash mismatch"); expect(run.err).toContain("demo/releases/1.4.0: release record does not match ยง11.2"); expect(run.out).not.toMatch(/^put /m); expect(await files(bucketDir)).toEqual(before); const inst = await instance(bucketDir, keprData); for (const table of ["issues", "issue_entries", "installs", "release_settings"]) { expect([table, inst.kepr.query(`SELECT COUNT(*) AS n FROM ${table}`).get()]).toEqual([ table, { n: 0 }, ]); } expect(inst.kepr.query("SELECT COUNT(*) AS n FROM repos WHERE name = 'secret'").get()).toEqual({ n: 0, }); });