1import { afterAll, beforeAll, expect, test } from "bun:test";2import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises";3import { tmpdir } from "node:os";4import { join } from "node:path";5import { blake3 } from "@noble/hashes/blake3.js";6import { bytesToHex } from "@noble/hashes/utils.js";7import { SQL } from "bun";8import { instance } from "./browser.ts";9import { makeRepos } from "./fixture.ts";101112const SCRIPT = join(import.meta.dir, "../scripts/import-kepr2.ts");13const root = await mkdtemp(join(tmpdir(), "kepr-import-"));14const pgDir = join(root, "pg");15const kepr2Data = join(root, "kepr2");16let url = "";17let bucketDir = "";18let save = "";1920const hex = (bytes: Uint8Array) => bytesToHex(blake3(bytes));21const sha = (bytes: Uint8Array) => new Bun.CryptoHasher("sha256").update(bytes).digest("hex");22const bytesOf = (text: string) => new TextEncoder().encode(text);23const run = (cmd: string[]) => {24 const out = Bun.spawnSync(cmd, { cwd: root, env: { PATH: process.env.PATH, LC_ALL: "C" } });25 if (out.exitCode !== 0) throw new Error(`${cmd.join(" ")}: ${out.stderr.toString()}`);26};2728const artifact = (29 id: string,30 run: string,31 repo: string,32 version: string,33 platform: string,34 name: string,35 bytes: Uint8Array,36 b3 = hex(bytes),37) =>38 `('${id}', '${run}', '${repo}', '${version}', '${platform}', '${name}', ${bytes.length}, '${sha(bytes)}', '${b3}', 5)`;3940async function kepr2File(path: string, text: string) {41 await mkdir(join(kepr2Data, "releases", path, ".."), { recursive: true });42 await writeFile(join(kepr2Data, "releases", path), text);43 return bytesOf(text);44}4546beforeAll(async () => {47 const repos = await makeRepos(["demo"]);48 bucketDir = repos.bucketDir;49 save = repos.saves.demo as string;50 const probe = Bun.serve({ port: 0, fetch: () => new Response() });51 const port = probe.port;52 probe.stop(true);53 run(["initdb", "-D", pgDir, "-U", "kepr", "-A", "trust"]);54 const options = `-p ${port} -k ${pgDir} -c listen_addresses=127.0.0.1`;55 run(["pg_ctl", "-D", pgDir, "-o", options, "-l", join(root, "pg.log"), "-w", "start"]);56 url = `postgres://[email protected]:${port}/postgres`;57 run([58 "psql",59 url,60 "-q",61 "-v",62 "ON_ERROR_STOP=1",63 "-f",64 join(import.meta.dir, "fixtures/kepr2/schema.sql"),65 ]);6667 const legacy = await kepr2File("demo/1.0.0/linux-x86_64/demo", "demo 1.0.0 linux");68 const byPlatform = await kepr2File("demo/1.1.0/macos-aarch64", "demo 1.1.0 mac");69 const apk = await kepr2File("demo/1.1.0/android-arm64/demo-1.1.0.apk", "demo 1.1.0 apk");70 const failed = await kepr2File("demo/1.2.0/linux-x86_64/demo", "demo 1.2.0 failed");71 const secret = await kepr2File("secret/0.1.0/linux-x86_64/secret", "secret 0.1.0");72 const pg = new SQL(url);73 await pg.unsafe(`74 INSERT INTO repos (name, created_at, description, categories, private, builds_enabled) VALUES75 ('demo', 1, 'A demo.', 'tools, cli', 0, 1), ('secret', 1, NULL, '', 1, 0);76 INSERT INTO accounts (id, username, recovery_hash, created_at, tier) VALUES77 ('acc-r', 'rita', 'x', 1, 'contributor'), ('acc-o', 'olga', 'x', 1, 'operator'), ('acc-t', 'tom', 'x', 1, 'team');78 INSERT INTO issues (id, repo, number, title, description, contact, contact_email, filed_at, filed_save, filed_ver, status, closed_at, closed_save, type, account_id, reporter_name) VALUES79 (1, 'demo', 7, 'Crash on start', 'It <b>crashes</b>.', NULL, '[email protected]', 1000, 'aaaaaaaaaaaa', '1.0.0', 'inprogress', NULL, NULL, 'bug', 'acc-r', NULL),80 (2, 'demo', 9, 'Dark mode', 'Please.', '[email protected]', NULL, 1500, 'aaaaaaaaaaaa', NULL, 'wontfix', 2000, '${save}', 'enhancement', NULL, 'Sam');81 INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES82 ('e1', 1, 'operator', 'reply', 'Looking.', 1100), ('e2', 1, 'acc-r', 'reply', 'Thanks!', 1200),83 ('e3', 1, 'operator', 'status_change', NULL, 1300), ('e4', 1, 'acc-o', 'reply', 'Me too.', 1400);84 INSERT INTO issue_notes (issue_id, body, posted_at, is_op) VALUES (1, 'internal only', 1250, 1);85 INSERT INTO engagement_events (id, event_type, repo_id, occurred_at) VALUES86 ('v1', 'nix_install', 'demo', 1), ('v2', 'nix_install', 'demo', 2), ('v3', 'nix_install', 'demo', 3),87 ('v4', 'archive_download', 'demo', 4), ('v5', 'nix_install', 'secret', 5);88 INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at, release_notes) VALUES89 ('b1', 'demo', '${save}', '1.0.0', 'manual', 'success', 10, 20, NULL),90 ('b2', 'demo', '${save}', '1.1.0', 'manual', 'partial', 30, 40, 'Starts twice as fast and no longer crashes on start.'),91 ('b3', 'demo', '${save}', '1.2.0', 'manual', 'failed', 50, 60, NULL),92 ('b5', 'secret', '${save}', '0.1.0', 'manual', 'success', 90, 100, NULL);93 INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES94 ${artifact("a1", "b1", "demo", "1.0.0", "linux-x86_64", "demo", legacy)},95 ${artifact("a2", "b2", "demo", "1.1.0", "macos-aarch64", "demo", byPlatform)},96 ${artifact("a3", "b2", "demo", "1.1.0", "android-arm64", "demo-1.1.0.apk", apk)},97 ${artifact("a4", "b3", "demo", "1.2.0", "linux-x86_64", "demo", failed)},98 ${artifact("a6", "b5", "secret", "0.1.0", "linux-x86_64", "secret", secret)};99 `);100 await pg.close();101});102103afterAll(() => run(["pg_ctl", "-D", pgDir, "-m", "fast", "-w", "stop"]));104105async function runImport(target: Record<string, string>, keprData: string, ...args: string[]) {106 const env = {107 PATH: process.env.PATH,108 KEPR2_DATABASE_URL: url,109 KEPR2_DATA: kepr2Data,110 KEPR_DATA: keprData,111 ...target,112 };113 114 const child = Bun.spawn(["bun", SCRIPT, ...args], { cwd: root, env, stderr: "pipe" });115 const [out, err, code] = await Promise.all([116 new Response(child.stdout).text(),117 new Response(child.stderr).text(),118 child.exited,119 ]);120 return { out, err, code };121}122123async function importKepr2(target: Record<string, string>, keprData: string, ...args: string[]) {124 const { out, err, code } = await runImport(target, keprData, ...args);125 if (code !== 0) throw new Error(err);126 return out;127}128129async function files(dir: string): Promise<string[]> {130 return (await readdir(dir, { recursive: true, withFileTypes: true }))131 .filter((entry) => entry.isFile())132 .map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1))133 .sort();134}135136test("--dry-run prints the import and writes neither the bucket nor kepr.db", async () => {137 const before = await files(bucketDir);138 const keprData = await mkdtemp(join(tmpdir(), "kepr-import-dry-"));139 const out = await importKepr2({ KOH_FOLDER: bucketDir }, keprData, "--dry-run");140 expect(out).toContain("would put demo/releases/1.1.0/release.json");141 expect(out).toContain('issues {"repo":"demo","number":"7"');142 expect(await files(bucketDir)).toEqual(before);143 const inst = await instance(bucketDir, keprData);144 expect(inst.kepr.query("SELECT COUNT(*) AS n FROM issues").get()).toEqual({ n: 0 });145});146147test("the import lands Kepr 2's issues, counts, settings and releases, and a re-run changes nothing", async () => {148 const keprData = await mkdtemp(join(tmpdir(), "kepr-import-data-"));149 const first = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);150 expect(first).not.toContain("1.2.0");151 const landed = await files(bucketDir);152 expect(landed.filter((key) => key.includes("/releases/"))).toEqual([153 "demo/releases/1.0.0/linux-x86_64/demo-linux-x86_64",154 "demo/releases/1.0.0/release.json",155 "demo/releases/1.1.0/android-arm64/demo-1.1.0.apk",156 "demo/releases/1.1.0/macos-aarch64/demo-macos-aarch64",157 "demo/releases/1.1.0/release.json",158 "demo/releases/latest",159 "secret/releases/0.1.0/linux-x86_64/secret-linux-x86_64",160 "secret/releases/0.1.0/release.json",161 "secret/releases/latest",162 ]);163 const again = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);164 expect(again).not.toMatch(/^(put|would put)/m);165 166 expect(167 again.split("\n").filter((line) => line.endsWith(" set") && !line.startsWith("repos ")),168 ).toEqual([]);169 expect(await files(bucketDir)).toEqual(landed);170171 const inst = await instance(bucketDir, keprData);172 const q = (sql: string) => inst.kepr.query(sql).all();173 expect(q("SELECT name, public, category FROM repos ORDER BY name")).toEqual([174 { name: "demo", public: 1, category: "tools" },175 { name: "secret", public: 0, category: "" },176 ]);177 expect(q("SELECT repo, count FROM installs ORDER BY repo")).toEqual([178 { repo: "demo", count: 3 },179 { repo: "secret", count: 1 },180 ]);181 expect(q("SELECT COUNT(*) AS n FROM downloads")).toEqual([{ n: 0 }]);182 expect(q("SELECT repo, version, hidden FROM release_settings")).toEqual([183 { repo: "secret", version: "0.1.0", hidden: 1 },184 ]);185 expect(186 q(187 "SELECT number, title, body, type, version, reporter_name, contact_email, filed_at, status, closed_at, fixed_in, token_hash FROM issues ORDER BY number",188 ),189 ).toEqual([190 {191 number: 7,192 title: "Crash on start",193 body: "It <b>crashes</b>.",194 type: "bug",195 version: "1.0.0",196 reporter_name: "@rita",197 contact_email: "[email protected]",198 filed_at: 1000,199 status: "open",200 closed_at: null,201 fixed_in: "",202 token_hash: null,203 },204 {205 number: 9,206 title: "Dark mode",207 body: "Please.",208 type: "idea",209 version: "",210 reporter_name: "Sam",211 contact_email: "[email protected]",212 filed_at: 1500,213 status: "closed",214 closed_at: 2000,215 fixed_in: save,216 token_hash: null,217 },218 ]);219 expect(q("SELECT author, body, created_at FROM issue_entries ORDER BY id")).toEqual([220 { author: "operator", body: "Looking.", created_at: 1100 },221 { author: "reporter", body: "Thanks!", created_at: 1200 },222 { author: "operator", body: "Me too.", created_at: 1400 },223 ]);224225 226 type Latest = { version: string; notes: string; platforms: Record<string, unknown> };227 const latest = (await (await inst.app.request("/api/demo/releases/latest")).json()) as Latest;228 expect(latest.version).toBe("1.1.0");229 expect(latest.notes).toBe("Starts twice as fast and no longer crashes on start.");230 expect(Object.keys(latest.platforms)).toEqual(["android-arm64", "macos-aarch64"]);231 const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).json();232 const legacy = bytesOf("demo 1.0.0 linux");233 expect(record).toEqual({234 format: 1,235 repo: "demo",236 version: "1.0.0",237 save,238 published_at: 20,239 machine: "kepr2",240 notes: "",241 files: [242 {243 platform: "linux-x86_64",244 name: "demo-linux-x86_64",245 size: legacy.length,246 blake3: hex(legacy),247 sha256: sha(legacy),248 },249 ],250 });251 expect(await (await inst.app.request("/demo/issues/7")).text()).toContain("Me too.");252 expect(inst.kepr.query("SELECT last_sync_error FROM repos WHERE name = 'demo'").get()).toEqual({253 last_sync_error: null,254 });255});256257type S3 = { dir: string; puts: string[]; target: Record<string, string>; stop(): void };258259260async function standInS3(race = new Map<string, string>()): Promise<S3> {261 const dir = await mkdtemp(join(tmpdir(), "kepr-import-s3-"));262 const puts: string[] = [];263 const server = Bun.serve({264 port: 0,265 async fetch(req) {266 const url = new URL(req.url);267 const key = decodeURIComponent(url.pathname.replace("/koh/", ""));268 const file = Bun.file(join(dir, key));269 if (req.method === "GET")270 return (await file.exists())271 ? new Response(file)272 : new Response("<Error><Code>NoSuchKey</Code></Error>", { status: 404 });273 expect([req.headers.get("if-none-match"), url.searchParams.has("X-Amz-Signature")]).toEqual([274 "*",275 true,276 ]);277 const raced = race.get(key);278 if (raced !== undefined) await Bun.write(join(dir, key), raced);279 if (await file.exists()) return new Response("", { status: 412 });280 puts.push(key);281 await Bun.write(join(dir, key), await req.arrayBuffer());282 return new Response("");283 },284 });285 const target = {286 KOH_S3_ENDPOINT: `http://localhost:${server.port}`,287 KOH_S3_BUCKET: "koh",288 KOH_S3_REGION: "fr-par",289 KOH_S3_ACCESS_KEY_ID: "write",290 KOH_S3_SECRET_ACCESS_KEY: "key",291 };292 return { dir, puts, target, stop: () => server.stop(true) };293}294295const newKeprData = () => mkdtemp(join(tmpdir(), "kepr-import-s3data-"));296297test("into an S3 bucket, files go up as presigned PUTs with If-None-Match, never over an existing key", async () => {298 const s3 = await standInS3();299 try {300 await importKepr2(s3.target, await newKeprData());301 expect(s3.puts.length).toBe(9);302 for (const key of await files(s3.dir)) {303 expect(await Bun.file(join(s3.dir, key)).bytes()).toEqual(304 await Bun.file(join(bucketDir, key)).bytes(),305 );306 }307 await importKepr2(s3.target, await newKeprData());308 expect(s3.puts.length).toBe(9);309 } finally {310 s3.stop();311 }312});313314test("a 412 from a key written since the read is success only when it holds the same bytes", async () => {315 const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).text();316 const same = await standInS3(new Map([["demo/releases/1.0.0/release.json", record]]));317 try {318 await importKepr2(same.target, await newKeprData());319 expect(same.puts).not.toContain("demo/releases/1.0.0/release.json");320 expect(same.puts.length).toBe(8);321 } finally {322 same.stop();323 }324 const other = await standInS3(new Map([["demo/releases/latest", "9.9.9"]]));325 try {326 const run = await runImport(other.target, await newKeprData());327 expect(run.code).toBe(1);328 expect(run.err).toContain("demo/releases/latest holds other bytes");329 expect(await Bun.file(join(other.dir, "demo/releases/latest")).text()).toBe("9.9.9");330 } finally {331 other.stop();332 }333});334335test("any imported row that fails a check fails the run: all are listed and nothing is written", async () => {336 await kepr2File("demo/1.3.0/linux-x86_64/demo", "demo 1.3.0 tampered");337 const oldPlatform = await kepr2File("demo/1.4.0/x86_64-linux/demo", "demo 1.4.0 old name");338 const legacy = bytesOf("demo 1.0.0 linux");339 const pg = new SQL(url);340 await pg.unsafe(`341 INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at) VALUES342 ('b4', 'demo', '${save}', '1.3.0', 'manual', 'success', 70, 80), ('b6', 'demo', '${save}', '1.4.0', 'manual', 'success', 110, 120);343 INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES344 ${artifact("a5", "b4", "demo", "1.3.0", "linux-x86_64", "demo", legacy)},345 ${artifact("a7", "b6", "demo", "1.4.0", "x86_64-linux", "demo", oldPlatform)};346 INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES ('e5', 2, 'acc-t', 'reply', 'Team here.', 1600);347 `);348 await pg.close();349 const before = await files(bucketDir);350 const keprData = await mkdtemp(join(tmpdir(), "kepr-import-bad-"));351 const run = await runImport({ KOH_FOLDER: bucketDir }, keprData);352 expect(run.code).toBe(1);353 expect(run.err).toContain("not imported, nothing written:");354 expect(run.err).toContain("demo#9: third-party reply");355 expect(run.err).toContain("demo/releases/1.3.0: linux-x86_64: hash mismatch");356 expect(run.err).toContain("demo/releases/1.4.0: release record does not match §11.2");357 expect(run.out).not.toMatch(/^put /m);358 expect(await files(bucketDir)).toEqual(before);359 const inst = await instance(bucketDir, keprData);360 for (const table of ["issues", "issue_entries", "installs", "release_settings"]) {361 expect([table, inst.kepr.query(`SELECT COUNT(*) AS n FROM ${table}`).get()]).toEqual([362 table,363 { n: 0 },364 ]);365 }366 expect(inst.kepr.query("SELECT COUNT(*) AS n FROM repos WHERE name = 'secret'").get()).toEqual({367 n: 0,368 });369});