import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { blake3 } from "@noble/hashes/blake3.js";
import { bytesToHex } from "@noble/hashes/utils.js";
import { SQL } from "bun";
import { instance } from "./browser.ts";
import { makeRepos } from "./fixture.ts";
// A throwaway Postgres holding Kepr 2's schema and a few synthetic rows, stopped after the tests.
const SCRIPT = join(import.meta.dir, "../scripts/import-kepr2.ts");
const root = await mkdtemp(join(tmpdir(), "kepr-import-"));
const pgDir = join(root, "pg");
const kepr2Data = join(root, "kepr2");
let url = "";
let bucketDir = "";
let save = "";
const hex = (bytes: Uint8Array) => bytesToHex(blake3(bytes));
const sha = (bytes: Uint8Array) => new Bun.CryptoHasher("sha256").update(bytes).digest("hex");
const bytesOf = (text: string) => new TextEncoder().encode(text);
const run = (cmd: string[]) => {
const out = Bun.spawnSync(cmd, { cwd: root, env: { PATH: process.env.PATH, LC_ALL: "C" } });
if (out.exitCode !== 0) throw new Error(`${cmd.join(" ")}: ${out.stderr.toString()}`);
};
const artifact = (
id: string,
run: string,
repo: string,
version: string,
platform: string,
name: string,
bytes: Uint8Array,
b3 = hex(bytes),
) =>
`('${id}', '${run}', '${repo}', '${version}', '${platform}', '${name}', ${bytes.length}, '${sha(bytes)}', '${b3}', 5)`;
async function kepr2File(path: string, text: string) {
await mkdir(join(kepr2Data, "releases", path, ".."), { recursive: true });
await writeFile(join(kepr2Data, "releases", path), text);
return bytesOf(text);
}
beforeAll(async () => {
const repos = await makeRepos(["demo"]);
bucketDir = repos.bucketDir;
save = repos.saves.demo as string;
const probe = Bun.serve({ port: 0, fetch: () => new Response() });
const port = probe.port;
probe.stop(true);
run(["initdb", "-D", pgDir, "-U", "kepr", "-A", "trust"]);
const options = `-p ${port} -k ${pgDir} -c listen_addresses=127.0.0.1`;
run(["pg_ctl", "-D", pgDir, "-o", options, "-l", join(root, "pg.log"), "-w", "start"]);
url = `postgres://kepr@127.0.0.1:${port}/postgres`;
run([
"psql",
url,
"-q",
"-v",
"ON_ERROR_STOP=1",
"-f",
join(import.meta.dir, "fixtures/kepr2/schema.sql"),
]);
const legacy = await kepr2File("demo/1.0.0/linux-x86_64/demo", "demo 1.0.0 linux");
const byPlatform = await kepr2File("demo/1.1.0/macos-aarch64", "demo 1.1.0 mac");
const apk = await kepr2File("demo/1.1.0/android-arm64/demo-1.1.0.apk", "demo 1.1.0 apk");
const failed = await kepr2File("demo/1.2.0/linux-x86_64/demo", "demo 1.2.0 failed");
const secret = await kepr2File("secret/0.1.0/linux-x86_64/secret", "secret 0.1.0");
const pg = new SQL(url);
await pg.unsafe(`
INSERT INTO repos (name, created_at, description, categories, private, builds_enabled) VALUES
('demo', 1, 'A demo.', 'tools, cli', 0, 1), ('secret', 1, NULL, '', 1, 0);
INSERT INTO accounts (id, username, recovery_hash, created_at, tier) VALUES
('acc-r', 'rita', 'x', 1, 'contributor'), ('acc-o', 'olga', 'x', 1, 'operator'), ('acc-t', 'tom', 'x', 1, 'team');
INSERT INTO issues (id, repo, number, title, description, contact, contact_email, filed_at, filed_save, filed_ver, status, closed_at, closed_save, type, account_id, reporter_name) VALUES
(1, 'demo', 7, 'Crash on start', 'It crashes.', NULL, 'rita@example.com', 1000, 'aaaaaaaaaaaa', '1.0.0', 'inprogress', NULL, NULL, 'bug', 'acc-r', NULL),
(2, 'demo', 9, 'Dark mode', 'Please.', 'sam@example.com', NULL, 1500, 'aaaaaaaaaaaa', NULL, 'wontfix', 2000, '${save}', 'enhancement', NULL, 'Sam');
INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES
('e1', 1, 'operator', 'reply', 'Looking.', 1100), ('e2', 1, 'acc-r', 'reply', 'Thanks!', 1200),
('e3', 1, 'operator', 'status_change', NULL, 1300), ('e4', 1, 'acc-o', 'reply', 'Me too.', 1400);
INSERT INTO issue_notes (issue_id, body, posted_at, is_op) VALUES (1, 'internal only', 1250, 1);
INSERT INTO engagement_events (id, event_type, repo_id, occurred_at) VALUES
('v1', 'nix_install', 'demo', 1), ('v2', 'nix_install', 'demo', 2), ('v3', 'nix_install', 'demo', 3),
('v4', 'archive_download', 'demo', 4), ('v5', 'nix_install', 'secret', 5);
INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at, release_notes) VALUES
('b1', 'demo', '${save}', '1.0.0', 'manual', 'success', 10, 20, NULL),
('b2', 'demo', '${save}', '1.1.0', 'manual', 'partial', 30, 40, 'Starts twice as fast and no longer crashes on start.'),
('b3', 'demo', '${save}', '1.2.0', 'manual', 'failed', 50, 60, NULL),
('b5', 'secret', '${save}', '0.1.0', 'manual', 'success', 90, 100, NULL);
INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES
${artifact("a1", "b1", "demo", "1.0.0", "linux-x86_64", "demo", legacy)},
${artifact("a2", "b2", "demo", "1.1.0", "macos-aarch64", "demo", byPlatform)},
${artifact("a3", "b2", "demo", "1.1.0", "android-arm64", "demo-1.1.0.apk", apk)},
${artifact("a4", "b3", "demo", "1.2.0", "linux-x86_64", "demo", failed)},
${artifact("a6", "b5", "secret", "0.1.0", "linux-x86_64", "secret", secret)};
`);
await pg.close();
});
afterAll(() => run(["pg_ctl", "-D", pgDir, "-m", "fast", "-w", "stop"]));
async function runImport(target: Record, keprData: string, ...args: string[]) {
const env = {
PATH: process.env.PATH,
KEPR2_DATABASE_URL: url,
KEPR2_DATA: kepr2Data,
KEPR_DATA: keprData,
...target,
};
// Not spawnSync: the S3 tests' stand-in server runs on this process's event loop.
const child = Bun.spawn(["bun", SCRIPT, ...args], { cwd: root, env, stderr: "pipe" });
const [out, err, code] = await Promise.all([
new Response(child.stdout).text(),
new Response(child.stderr).text(),
child.exited,
]);
return { out, err, code };
}
async function importKepr2(target: Record, keprData: string, ...args: string[]) {
const { out, err, code } = await runImport(target, keprData, ...args);
if (code !== 0) throw new Error(err);
return out;
}
async function files(dir: string): Promise {
return (await readdir(dir, { recursive: true, withFileTypes: true }))
.filter((entry) => entry.isFile())
.map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1))
.sort();
}
test("--dry-run prints the import and writes neither the bucket nor kepr.db", async () => {
const before = await files(bucketDir);
const keprData = await mkdtemp(join(tmpdir(), "kepr-import-dry-"));
const out = await importKepr2({ KOH_FOLDER: bucketDir }, keprData, "--dry-run");
expect(out).toContain("would put demo/releases/1.1.0/release.json");
expect(out).toContain('issues {"repo":"demo","number":"7"');
expect(await files(bucketDir)).toEqual(before);
const inst = await instance(bucketDir, keprData);
expect(inst.kepr.query("SELECT COUNT(*) AS n FROM issues").get()).toEqual({ n: 0 });
});
test("the import lands Kepr 2's issues, counts, settings and releases, and a re-run changes nothing", async () => {
const keprData = await mkdtemp(join(tmpdir(), "kepr-import-data-"));
const first = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);
expect(first).not.toContain("1.2.0");
const landed = await files(bucketDir);
expect(landed.filter((key) => key.includes("/releases/"))).toEqual([
"demo/releases/1.0.0/linux-x86_64/demo-linux-x86_64",
"demo/releases/1.0.0/release.json",
"demo/releases/1.1.0/android-arm64/demo-1.1.0.apk",
"demo/releases/1.1.0/macos-aarch64/demo-macos-aarch64",
"demo/releases/1.1.0/release.json",
"demo/releases/latest",
"secret/releases/0.1.0/linux-x86_64/secret-linux-x86_64",
"secret/releases/0.1.0/release.json",
"secret/releases/latest",
]);
const again = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);
expect(again).not.toMatch(/^(put|would put)/m);
// Only the repo settings are set again (to the same values); nothing else is added.
expect(
again.split("\n").filter((line) => line.endsWith(" set") && !line.startsWith("repos ")),
).toEqual([]);
expect(await files(bucketDir)).toEqual(landed);
const inst = await instance(bucketDir, keprData);
const q = (sql: string) => inst.kepr.query(sql).all();
expect(q("SELECT name, public, category FROM repos ORDER BY name")).toEqual([
{ name: "demo", public: 1, category: "tools" },
{ name: "secret", public: 0, category: "" },
]);
expect(q("SELECT repo, count FROM installs ORDER BY repo")).toEqual([
{ repo: "demo", count: 3 },
{ repo: "secret", count: 1 },
]);
expect(q("SELECT COUNT(*) AS n FROM downloads")).toEqual([{ n: 0 }]);
expect(q("SELECT repo, version, hidden FROM release_settings")).toEqual([
{ repo: "secret", version: "0.1.0", hidden: 1 },
]);
expect(
q(
"SELECT number, title, body, type, version, reporter_name, contact_email, filed_at, status, closed_at, fixed_in, token_hash FROM issues ORDER BY number",
),
).toEqual([
{
number: 7,
title: "Crash on start",
body: "It crashes.",
type: "bug",
version: "1.0.0",
reporter_name: "@rita",
contact_email: "rita@example.com",
filed_at: 1000,
status: "open",
closed_at: null,
fixed_in: "",
token_hash: null,
},
{
number: 9,
title: "Dark mode",
body: "Please.",
type: "idea",
version: "",
reporter_name: "Sam",
contact_email: "sam@example.com",
filed_at: 1500,
status: "closed",
closed_at: 2000,
fixed_in: save,
token_hash: null,
},
]);
expect(q("SELECT author, body, created_at FROM issue_entries ORDER BY id")).toEqual([
{ author: "operator", body: "Looking.", created_at: 1100 },
{ author: "reporter", body: "Thanks!", created_at: 1200 },
{ author: "operator", body: "Me too.", created_at: 1400 },
]);
// Kepr 3 reads what the import wrote as any Koh 3 release.
type Latest = { version: string; notes: string; platforms: Record };
const latest = (await (await inst.app.request("/api/demo/releases/latest")).json()) as Latest;
expect(latest.version).toBe("1.1.0");
expect(latest.notes).toBe("Starts twice as fast and no longer crashes on start.");
expect(Object.keys(latest.platforms)).toEqual(["android-arm64", "macos-aarch64"]);
const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).json();
const legacy = bytesOf("demo 1.0.0 linux");
expect(record).toEqual({
format: 1,
repo: "demo",
version: "1.0.0",
save,
published_at: 20,
machine: "kepr2",
notes: "",
files: [
{
platform: "linux-x86_64",
name: "demo-linux-x86_64",
size: legacy.length,
blake3: hex(legacy),
sha256: sha(legacy),
},
],
});
expect(await (await inst.app.request("/demo/issues/7")).text()).toContain("Me too.");
expect(inst.kepr.query("SELECT last_sync_error FROM repos WHERE name = 'demo'").get()).toEqual({
last_sync_error: null,
});
});
type S3 = { dir: string; puts: string[]; target: Record; stop(): void };
/** A stand-in S3 over a folder. A key in `race` appears, holding those bytes, between our GET and PUT. */
async function standInS3(race = new Map()): Promise {
const dir = await mkdtemp(join(tmpdir(), "kepr-import-s3-"));
const puts: string[] = [];
const server = Bun.serve({
port: 0,
async fetch(req) {
const url = new URL(req.url);
const key = decodeURIComponent(url.pathname.replace("/koh/", ""));
const file = Bun.file(join(dir, key));
if (req.method === "GET")
return (await file.exists())
? new Response(file)
: new Response("NoSuchKey", { status: 404 });
expect([req.headers.get("if-none-match"), url.searchParams.has("X-Amz-Signature")]).toEqual([
"*",
true,
]);
const raced = race.get(key);
if (raced !== undefined) await Bun.write(join(dir, key), raced);
if (await file.exists()) return new Response("", { status: 412 });
puts.push(key);
await Bun.write(join(dir, key), await req.arrayBuffer());
return new Response("");
},
});
const target = {
KOH_S3_ENDPOINT: `http://localhost:${server.port}`,
KOH_S3_BUCKET: "koh",
KOH_S3_REGION: "fr-par",
KOH_S3_ACCESS_KEY_ID: "write",
KOH_S3_SECRET_ACCESS_KEY: "key",
};
return { dir, puts, target, stop: () => server.stop(true) };
}
const newKeprData = () => mkdtemp(join(tmpdir(), "kepr-import-s3data-"));
test("into an S3 bucket, files go up as presigned PUTs with If-None-Match, never over an existing key", async () => {
const s3 = await standInS3();
try {
await importKepr2(s3.target, await newKeprData());
expect(s3.puts.length).toBe(9);
for (const key of await files(s3.dir)) {
expect(await Bun.file(join(s3.dir, key)).bytes()).toEqual(
await Bun.file(join(bucketDir, key)).bytes(),
);
}
await importKepr2(s3.target, await newKeprData());
expect(s3.puts.length).toBe(9);
} finally {
s3.stop();
}
});
test("a 412 from a key written since the read is success only when it holds the same bytes", async () => {
const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).text();
const same = await standInS3(new Map([["demo/releases/1.0.0/release.json", record]]));
try {
await importKepr2(same.target, await newKeprData());
expect(same.puts).not.toContain("demo/releases/1.0.0/release.json");
expect(same.puts.length).toBe(8);
} finally {
same.stop();
}
const other = await standInS3(new Map([["demo/releases/latest", "9.9.9"]]));
try {
const run = await runImport(other.target, await newKeprData());
expect(run.code).toBe(1);
expect(run.err).toContain("demo/releases/latest holds other bytes");
expect(await Bun.file(join(other.dir, "demo/releases/latest")).text()).toBe("9.9.9");
} finally {
other.stop();
}
});
test("any imported row that fails a check fails the run: all are listed and nothing is written", async () => {
await kepr2File("demo/1.3.0/linux-x86_64/demo", "demo 1.3.0 tampered");
const oldPlatform = await kepr2File("demo/1.4.0/x86_64-linux/demo", "demo 1.4.0 old name");
const legacy = bytesOf("demo 1.0.0 linux");
const pg = new SQL(url);
await pg.unsafe(`
INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at) VALUES
('b4', 'demo', '${save}', '1.3.0', 'manual', 'success', 70, 80), ('b6', 'demo', '${save}', '1.4.0', 'manual', 'success', 110, 120);
INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES
${artifact("a5", "b4", "demo", "1.3.0", "linux-x86_64", "demo", legacy)},
${artifact("a7", "b6", "demo", "1.4.0", "x86_64-linux", "demo", oldPlatform)};
INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES ('e5', 2, 'acc-t', 'reply', 'Team here.', 1600);
`);
await pg.close();
const before = await files(bucketDir);
const keprData = await mkdtemp(join(tmpdir(), "kepr-import-bad-"));
const run = await runImport({ KOH_FOLDER: bucketDir }, keprData);
expect(run.code).toBe(1);
expect(run.err).toContain("not imported, nothing written:");
expect(run.err).toContain("demo#9: third-party reply");
expect(run.err).toContain("demo/releases/1.3.0: linux-x86_64: hash mismatch");
expect(run.err).toContain("demo/releases/1.4.0: release record does not match ยง11.2");
expect(run.out).not.toMatch(/^put /m);
expect(await files(bucketDir)).toEqual(before);
const inst = await instance(bucketDir, keprData);
for (const table of ["issues", "issue_entries", "installs", "release_settings"]) {
expect([table, inst.kepr.query(`SELECT COUNT(*) AS n FROM ${table}`).get()]).toEqual([
table,
{ n: 0 },
]);
}
expect(inst.kepr.query("SELECT COUNT(*) AS n FROM repos WHERE name = 'secret'").get()).toEqual({
n: 0,
});
});