946c9882a405

add the one-time Kepr 2 import tool, writing put-if-absent and checked by the sync's own release validator, with a local-Postgres test, and record phase 8 in continuity (v3.0.43)

time
2026-09-26 21:02 UTC · 2 weeks ago
machine
Junes-MacBook-Pro.local
agent
claude-opus-5-5
session
kepr 3 implementation
parent
7ec223c3528a
5 files package.json+1 −1scripts/import-kepr2.ts+98 −0src/bucket/repo.ts+16 −4tests/fixtures/kepr2/schema.sql+715 −0tests/import.test.ts+304 −0
package.json+1 −1
@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.42",3+	"version": "3.0.43",44 	"private": true,55 	"type": "module",66 	"scripts": {

view file

scripts/import-kepr2.ts+98 −0
@@ -0,0 +1,98 @@1+// One-time Kepr 2 import (§28), deleted after cutover; the only bucket writer. Env: KEPR2_DATABASE_URL,2+// KEPR2_DATA, KEPR_DATA, KOH_FOLDER or KOH_S3_* (write key at run time only). --dry-run writes nothing.3+import { join } from "node:path";4+import { blake3 } from "@noble/hashes/blake3.js";5+import { bytesToHex } from "@noble/hashes/utils.js";6+import { SQL } from "bun";7+import { openBucket } from "../src/bucket/reader.ts";8+import { checkRelease } from "../src/bucket/repo.ts";9+import { compareVersions } from "../src/releases/releases.ts";10+import { openKeprDb } from "../src/server/db.ts";11+12+type Row = Record<string, string | number | null>;13+const env = (name: string) => process.env[name] ?? "";14+const dry = process.argv.includes("--dry-run");15+const pg = new SQL(env("KEPR2_DATABASE_URL"));16+const kepr = openKeprDb(env("KEPR_DATA"));17+const folder = env("KOH_FOLDER");18+const VARS = "ENDPOINT BUCKET REGION ACCESS_KEY_ID SECRET_ACCESS_KEY".split(" ");19+const [endpoint, name, region, accessKeyId, secretAccessKey] = VARS.map((v) => env(`KOH_S3_${v}`));20+const s3 = { endpoint, region, accessKeyId, secretAccessKey };21+const bucket = openBucket(folder ? { kind: "folder", path: folder } : { kind: "s3", name, ...s3 });22+/** put_if_absent (§9.1): true when `key` holds `bytes` after; a key holding other bytes is kept. */23+async function put(key: string, body: string | Uint8Array): Promise<boolean> {24+	const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;25+	const had = await bucket.get(key);26+	const same = had === null || Bun.deepEquals(had, bytes);27+	console.log(had === null ? (dry ? "would put" : "put") : same ? "have" : "differs, kept:", key);28+	if (had !== null || dry) return same;29+	if (folder) return Bun.write(join(folder, key), bytes).then(() => true);30+	// Bun's S3 client has no conditional write, so a presigned PUT carries it.31+	const url = new Bun.S3Client({ ...s3, bucket: name }).presign(key, { method: "PUT" });32+	const res = await fetch(url, { method: "PUT", body: bytes, headers: { "If-None-Match": "*" } });33+	if (!res.ok) throw new Error(`put ${key}: ${res.status}`);34+	return true;35+}36+function insert(table: string, row: Row, tail: string): number | null {37+	const cols = Object.keys(row);38+	const sql = `INSERT INTO ${table} (${cols}) VALUES (${cols.map((c) => `$${c}`)}) ${tail} RETURNING rowid AS id`;39+	const id = kepr.query<{ id: number }, [Row]>(sql).get(row)?.id ?? null;40+	console.log(table, JSON.stringify(row).slice(0, 80), id ? "set" : "have");41+	return id;42+}43+kepr.run("BEGIN");44+// Kepr 3 keeps one category and has no description field; Kepr 2 had no homepage.45+const upsert = "ON CONFLICT DO UPDATE SET public = excluded.public, category = excluded.category";46+for (const repo of await pg<Row[]>`47+	SELECT name, (private = 0)::int AS public, trim(split_part(COALESCE(categories, ''), ',', 1)) AS category FROM repos`)48+	insert("repos", repo, upsert);49+// Kepr 2's four statuses become two (Phase 5); "enhancement" is an idea. Any account's reply but the50+// reporter's own is the operator's, since §23's thread has no other author.51+for (const { replies, ...issue } of await pg<(Row & { replies: Row[] })[]>`52+	SELECT i.repo, i.number, i.title, i.description AS body, CASE WHEN i.type IN ('bug', 'question') THEN i.type ELSE 'idea' END AS type,53+		COALESCE(i.filed_ver, '') AS version, COALESCE(NULLIF(i.reporter_name, ''), '@' || a.username, '') AS reporter_name,54+		COALESCE(NULLIF(i.contact_email, ''), i.contact, '') AS contact_email, i.filed_at,55+		CASE WHEN i.status IN ('resolved', 'wontfix', 'closed') THEN 'closed' ELSE 'open' END AS status,56+		CASE WHEN i.status IN ('resolved', 'wontfix', 'closed') THEN COALESCE(i.closed_at, i.filed_at) END AS closed_at,57+		CASE WHEN i.closed_save ~ '^[0-9a-f]{12}$' THEN i.closed_save ELSE '' END AS fixed_in,58+		(SELECT COALESCE(json_agg(json_build_object('author', CASE WHEN e.account_id = i.account_id THEN 'reporter' ELSE 'operator' END,59+			'body', e.body, 'created_at', e.created_at) ORDER BY e.created_at, e.id), '[]') FROM issue_entries e WHERE e.issue_id = i.id AND e.entry_type = 'reply' AND e.body IS NOT NULL) AS replies60+	FROM issues i LEFT JOIN accounts a ON a.id = i.account_id ORDER BY i.repo, i.number`) {61+	const id = insert("issues", issue, "ON CONFLICT DO NOTHING");62+	for (const reply of id ? replies : []) insert("issue_entries", { issue_id: id, ...reply }, "");63+}64+// Kepr 2 counted Nix fetches as installs (kept 90 days) and never counted release downloads.65+for (const count of await pg<Row[]>`66+	SELECT repo_id AS repo, COUNT(*) AS count FROM engagement_events WHERE event_type = 'nix_install' AND repo_id IS NOT NULL GROUP BY repo_id`)67+	insert("installs", count, "ON CONFLICT DO NOTHING");68+// As Kepr 2 served: successful and partial builds, a file stored under the repo's name as <repo>-<platform>,69+// older uploads' files named by platform; builds off hid a repo's. Save, notes, time: the newest build's.70+type Release = { repo: string; version: string; hidden: boolean; record: Row; stored: string[] };71+const newest = new Map<string, string>();72+for (const { repo, version, hidden, record, stored } of await pg<Release[]>`73+	SELECT a.repo_id AS repo, a.version, bool_or(r.builds_enabled = 0) AS hidden, json_agg(a.filename ORDER BY a.platform) AS stored,74+		json_build_object('format', 1, 'repo', a.repo_id, 'version', a.version, 'save', (array_agg(b.save_id ORDER BY b.started_at DESC))[1],75+			'published_at', max(COALESCE(b.finished_at, a.built_at)), 'machine', 'kepr2', 'notes', COALESCE((array_agg(b.release_notes ORDER BY b.started_at DESC))[1], ''),76+			'files', json_agg(json_build_object('platform', a.platform, 'name', CASE WHEN a.filename = a.repo_id THEN a.repo_id || '-' || a.platform ELSE a.filename END,77+				'size', a.size_bytes, 'blake3', a.blake3, 'sha256', a.sha256) ORDER BY a.platform)) AS record78+	FROM release_artifacts a JOIN build_runs b ON b.id = a.build_run JOIN repos r ON r.name = a.repo_id WHERE b.status IN ('success', 'partial') GROUP BY a.repo_id, a.version ORDER BY a.repo_id, a.version`) {79+	const prefix = `${repo}/releases/${version}`;80+	try {81+		for (const [n, f] of checkRelease(repo, version, record).files.entries()) {82+			const dir = join(env("KEPR2_DATA"), "releases", repo, version, f.platform);83+			const path = join(dir, stored[n] ?? "");84+			const bytes = await Bun.file((await Bun.file(path).exists()) ? path : dir).bytes();85+			const kepr2s = bytesToHex(blake3(bytes)) === f.blake3;86+			if (!kepr2s || !(await put(`${prefix}/${f.platform}/${f.name}`, bytes)))87+				throw new Error(`${f.platform}: not Kepr 2's file, or another is there`);88+		}89+		if (!(await put(`${prefix}/release.json`, JSON.stringify(record)))) continue;90+		if (hidden) insert("release_settings", { repo, version, hidden: 1 }, "ON CONFLICT DO NOTHING");91+		if (compareVersions(version, newest.get(repo) ?? "") > 0) newest.set(repo, version);92+	} catch (error) {93+		console.log(`skip ${prefix}:`, error instanceof Error ? error.message : error);94+	}95+}96+for (const [repo, version] of newest) await put(`${repo}/releases/latest`, version);97+kepr.run(dry ? "ROLLBACK" : "COMMIT");98+await pg.close();

view file

src/bucket/repo.ts+16 −4
@@ -212,16 +212,28 @@212212 	repo: string,213213 	version: string,214214 ): Promise<ReleaseRecord> {215-	requireRepo(repo);216-	if (!isVersion(version)) throw new BucketError(`not a release version: "${version}"`);215+	requireRelease(repo, version);217216 	const key = `${repo}/releases/${version}/release.json`;218217 	const bytes = await bucket.get(key);219218 	if (bytes === null) throw new BucketError(`${key} is missing`);220-	const record = parseReleaseRecord(JSON.parse(decoder.decode(bytes)));219+	return checkRelease(repo, version, JSON.parse(decoder.decode(bytes)));220+}221+222+/** A parsed `<repo>/releases/<version>/release.json`, if the sync would accept it (§11.2). */223+export function checkRelease(repo: string, version: string, value: unknown): ReleaseRecord {224+	requireRelease(repo, version);225+	const record = parseReleaseRecord(value);221226 	if (record.repo !== repo || record.version !== version) {222-		throw new BucketError(`${key} holds ${record.repo} ${record.version}`);227+		throw new BucketError(228+			`${repo}/releases/${version}/release.json holds ${record.repo} ${record.version}`,229+		);223230 	}224231 	return record;232+}233+234+function requireRelease(repo: string, version: string): void {235+	requireRepo(repo);236+	if (!isVersion(version)) throw new BucketError(`not a release version: "${version}"`);225237 }226238 227239 function parseReleaseRecord(value: unknown): ReleaseRecord {

view file

tests/fixtures/kepr2/schema.sql+715 −0
@@ -0,0 +1,715 @@1+-- Kepr Postgres schema — version 1.2+--3+-- A faithful translation of the cumulative SQLite schema (migrations 1..39,4+-- as materialised by db.applyAll) into idiomatic Postgres. It is NOT derived5+-- from prose: it was produced from `.schema` of a database with all 396+-- migrations applied, so every column, constraint, CHECK, and index matches7+-- what the live SQLite database actually holds.8+--9+-- Translation rules (chosen for an exact, value-preserving migration):10+--   * SQLite INTEGER  -> BIGINT          (SQLite ints are 64-bit; size_bytes11+--                                          and byte counts exceed int32)12+--   * SQLite REAL     -> DOUBLE PRECISION13+--   * SQLite TEXT     -> TEXT14+--   * Integer 0/1 booleans are KEPT as BIGINT 0/1 — not converted to BOOLEAN —15+--     so the migration is a verbatim value copy and db.zig reads them exactly16+--     as it does today (i64, != 0 == true).17+--   * INTEGER PRIMARY KEY AUTOINCREMENT -> BIGINT GENERATED BY DEFAULT AS18+--     IDENTITY (BY DEFAULT, not ALWAYS, so the migration tool can insert the19+--     existing ids; the sequence is advanced past MAX(id) after bulk load).20+--   * All timestamps remain Unix epoch seconds (BIGINT), unconverted.21+--22+-- Foreign keys are declared as in SQLite. SQLite did not enforce them at23+-- runtime, so the one-time migration tool loads with24+-- `SET session_replication_role = replica` to bypass FK/trigger checks, then25+-- the constraints guard all subsequently-written data.26+27+BEGIN;28+29+CREATE TABLE schema_version (30+    version INTEGER NOT NULL31+);32+INSERT INTO schema_version (version) VALUES (1);33+34+-- ── Repositories and saves ───────────────────────────────────────────35+36+CREATE TABLE repos (37+    name               TEXT PRIMARY KEY,38+    created_at         BIGINT NOT NULL,39+    description        TEXT,40+    languages          TEXT DEFAULT '',41+    details            TEXT,42+    categories         TEXT DEFAULT '',43+    count_extensions   TEXT DEFAULT '',44+    contributor_handle TEXT,45+    issues_enabled     BIGINT NOT NULL DEFAULT 1,46+    languages_cloud    TEXT,47+    project_id         TEXT,48+    trusted            BIGINT NOT NULL DEFAULT 0,49+    trust_computed_at  BIGINT,50+    private            BIGINT NOT NULL DEFAULT 0,51+    builds_enabled     BIGINT NOT NULL DEFAULT 1,52+    display_languages  TEXT53+);54+55+CREATE TABLE saves (56+    id            TEXT NOT NULL,57+    repo          TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,58+    lane          TEXT NOT NULL CHECK (lane IN ('main','dev')),59+    message       TEXT NOT NULL,60+    timestamp     BIGINT NOT NULL,61+    size_bytes    BIGINT NOT NULL,62+    parent_id     TEXT,63+    is_checkpoint BIGINT NOT NULL DEFAULT 0,64+    checksum      TEXT NOT NULL,65+    manifest_path TEXT NOT NULL,66+    file_count    BIGINT NOT NULL DEFAULT 0,67+    line_count    BIGINT NOT NULL DEFAULT 0,68+    session       TEXT,69+    agent         TEXT,70+    PRIMARY KEY (id, repo)71+);72+CREATE INDEX saves_repo_lane_ts ON saves(repo, lane, timestamp);73+74+CREATE TABLE object_refs (75+    hex_hash TEXT NOT NULL,76+    save_id  TEXT NOT NULL,77+    repo     TEXT NOT NULL,78+    PRIMARY KEY (hex_hash, save_id, repo)79+);80+CREATE INDEX object_refs_hash ON object_refs(hex_hash);81+82+-- Reserved advisory-lock id: Postgres advisory locks (pg_advisory_xact_lock /83+-- pg_advisory_xact_lock_shared) are a single flat bigint namespace per84+-- cluster, not scoped to any table, but the id kepr reserves here exists to85+-- protect this table specifically. storage GC's sweep (storage_admin.gc)86+-- holds it exclusive around deciding which objects/ files are unreferenced;87+-- every offer-complete transaction that promotes rows into object_refs88+-- (src/web/api.zig: handleComplete, acceptPending) takes it89+-- shared first, so a commit that references an object already on disk can90+-- never race a concurrent sweep into deleting that object (review F12,91+-- decision D3). Shared holders never block each other. See92+-- db_mod.STORAGE_GC_ADVISORY_LOCK_ID in src/db.zig for the actual id and the93+-- lockGcExclusive/lockGcShared helpers — if kepr ever reserves another94+-- advisory-lock id, pick a value nowhere near this one and document it here95+-- too.96+97+-- Content-addressed cache of a countable file's line count, keyed by its98+-- Blake3 hash. A hash's line count never changes, so once computed it never99+-- needs decompressing again on a later save that re-references the same100+-- unchanged file.101+CREATE TABLE object_line_counts (102+    hex_hash TEXT PRIMARY KEY,103+    lines    BIGINT NOT NULL104+);105+106+-- Operator-purged paths: the server-side equivalent of .kohignore (which kepr107+-- strips from manifests, so it cannot be edited server-side). A path here was108+-- stripped from every save's manifest by an operator; future offers that109+-- re-introduce it are filtered on ingest. Matches a file exactly or as a dir.110+CREATE TABLE repo_purged_paths (111+    repo_id   TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,112+    path      TEXT NOT NULL,113+    purged_at BIGINT NOT NULL,114+    PRIMARY KEY (repo_id, path)115+);116+117+CREATE TABLE offer_tokens (118+    offer_id    TEXT PRIMARY KEY,119+    repo        TEXT NOT NULL,120+    manifest_id TEXT NOT NULL,121+    missing     TEXT NOT NULL,122+    manifest    TEXT NOT NULL,123+    key_type    TEXT NOT NULL CHECK (key_type IN ('operator','contributor')),124+    expires_at  BIGINT NOT NULL,125+    -- Account id that negotiated this token; complete must be performed by the126+    -- same account (§8.9). NULL only for tokens predating protocol 2.0.127+    negotiated_account TEXT128+);129+130+CREATE TABLE pending_offers (131+    offer_id      TEXT PRIMARY KEY,132+    repo          TEXT NOT NULL,133+    manifest_id   TEXT NOT NULL,134+    manifest      TEXT NOT NULL,135+    contributor   TEXT,136+    submitted_at  BIGINT NOT NULL,137+    status        TEXT NOT NULL DEFAULT 'pending'138+        CHECK (status IN ('draft','pending','accepted','declined')),139+    reviewed_at   BIGINT,140+    review_note   TEXT,141+    agent_verdict TEXT,142+    agent_status  TEXT DEFAULT 'pending'143+        CHECK (agent_status IN ('pending','running','complete','failed','unavailable')),144+    findings_json TEXT,145+    scan_status   TEXT CHECK (scan_status IN ('ok','failed')),146+    -- Contributor's human-readable note to the reviewer (separate from save147+    -- messages); 'draft' status holds an offer out of the review queue until148+    -- the contributor promotes it.149+    description   TEXT150+);151+152+CREATE TABLE pending_offer_identity_quarantine (153+    offer_id           TEXT PRIMARY KEY REFERENCES pending_offers(offer_id) ON DELETE CASCADE,154+    legacy_fingerprint TEXT NOT NULL,155+    reason             TEXT NOT NULL,156+    recorded_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT157+);158+CREATE INDEX pending_offer_identity_quarantine_fp159+    ON pending_offer_identity_quarantine(legacy_fingerprint);160+161+CREATE TABLE steals (162+    id        TEXT PRIMARY KEY,163+    repo      TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,164+    timestamp BIGINT NOT NULL165+);166+CREATE INDEX steals_repo_ts ON steals(repo, timestamp);167+168+CREATE TABLE storage_snapshots (169+    date           TEXT PRIMARY KEY,170+    total_bytes    BIGINT NOT NULL,171+    object_count   BIGINT NOT NULL,172+    repo_count     BIGINT NOT NULL,173+    save_count     BIGINT NOT NULL,174+    languages      TEXT,175+    first_received BIGINT176+);177+178+CREATE TABLE instance_images (179+    filename     TEXT PRIMARY KEY,180+    created_at   BIGINT NOT NULL,181+    size_bytes   BIGINT NOT NULL,182+    checksum     TEXT NOT NULL,183+    repo_count   BIGINT NOT NULL,184+    save_count   BIGINT NOT NULL,185+    kepr_version TEXT NOT NULL186+);187+188+CREATE TABLE diff_cache (189+    repo        TEXT NOT NULL,190+    save_id     TEXT NOT NULL,191+    diff_json   TEXT NOT NULL,192+    computed_at BIGINT NOT NULL,193+    PRIMARY KEY (repo, save_id)194+);195+196+CREATE TABLE ops_credentials (197+    key   TEXT PRIMARY KEY,198+    value TEXT NOT NULL199+);200+201+-- ── Contributor handles ──────────────────────────────────────────────202+203+CREATE TABLE handles (204+    id            TEXT PRIMARY KEY,205+    handle        TEXT UNIQUE NOT NULL,206+    created_at    BIGINT NOT NULL,207+    recovery_hash TEXT NOT NULL208+);209+210+-- ── Assets, dependencies, builds, releases ───────────────────────────211+212+CREATE TABLE repo_assets (213+    id          TEXT PRIMARY KEY,214+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,215+    filename    TEXT NOT NULL,216+    mime_type   TEXT NOT NULL,217+    size_bytes  BIGINT NOT NULL,218+    uploaded_at BIGINT NOT NULL219+);220+221+CREATE TABLE repo_dependencies (222+    from_repo       TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,223+    to_repo         TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,224+    first_stolen_at BIGINT NOT NULL,225+    last_stolen_at  BIGINT NOT NULL,226+    PRIMARY KEY (from_repo, to_repo)227+);228+229+CREATE TABLE build_runs (230+    id               TEXT PRIMARY KEY,231+    repo_id          TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,232+    save_id          TEXT NOT NULL,233+    version          TEXT NOT NULL,234+    trigger          TEXT NOT NULL,235+    status           TEXT NOT NULL,236+    started_at       BIGINT NOT NULL,237+    finished_at      BIGINT,238+    log              TEXT,239+    release_notes    TEXT,240+    platform_results TEXT NOT NULL DEFAULT '{}',241+    duration_ms      BIGINT,242+    lane             TEXT NOT NULL DEFAULT 'main'243+);244+245+CREATE TABLE release_artifacts (246+    id          TEXT PRIMARY KEY,247+    build_run   TEXT NOT NULL REFERENCES build_runs(id),248+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,249+    version     TEXT NOT NULL,250+    platform    TEXT NOT NULL,251+    filename    TEXT NOT NULL,252+    size_bytes  BIGINT NOT NULL,253+    sha256      TEXT NOT NULL,254+    blake3      TEXT NOT NULL,255+    signature   TEXT NOT NULL DEFAULT '',256+    built_at    BIGINT NOT NULL257+);258+CREATE UNIQUE INDEX release_artifacts_repo_version_platform259+    ON release_artifacts (repo_id, version, platform);260+261+-- ── Issues ───────────────────────────────────────────────────────────262+263+CREATE TABLE issues (264+    id            BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,265+    repo          TEXT    NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,266+    title         TEXT    NOT NULL,267+    description   TEXT    NOT NULL,268+    contact       TEXT,269+    filed_at      BIGINT  NOT NULL,270+    filed_save    TEXT    NOT NULL,271+    filed_ver     TEXT,272+    status        TEXT    NOT NULL DEFAULT 'open',273+    closed_at     BIGINT,274+    closed_save   TEXT,275+    ip_hash       TEXT,276+    number        BIGINT  NOT NULL DEFAULT 0,277+    type          TEXT    NOT NULL DEFAULT 'bug',278+    severity      TEXT,279+    face_id       TEXT,280+    resolution_face TEXT,281+    account_id    TEXT,282+    contact_email TEXT,283+    updated_at    BIGINT  NOT NULL DEFAULT 0,284+    reporter_name TEXT285+);286+CREATE UNIQUE INDEX issues_repo_number ON issues(repo, number);287+CREATE INDEX issues_repo_status_created288+    ON issues(repo, status, filed_at DESC);289+CREATE INDEX issues_repo_type_status_created290+    ON issues(repo, type, status, filed_at DESC);291+292+CREATE TABLE issue_notes (293+    id        BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,294+    issue_id  BIGINT  NOT NULL REFERENCES issues(id),295+    body      TEXT    NOT NULL,296+    posted_at BIGINT  NOT NULL,297+    is_op     BIGINT  NOT NULL DEFAULT 0298+);299+300+CREATE TABLE issue_entries (301+    id         TEXT PRIMARY KEY,302+    issue_id   BIGINT NOT NULL REFERENCES issues(id) ON DELETE CASCADE,303+    account_id TEXT,304+    entry_type TEXT NOT NULL CHECK (entry_type IN ('reply','status_change','face_linked')),305+    body       TEXT,306+    created_at BIGINT NOT NULL307+);308+CREATE INDEX issue_entries_issue ON issue_entries(issue_id, created_at);309+310+CREATE TABLE issue_attachments (311+    id          TEXT PRIMARY KEY,312+    issue_id    BIGINT REFERENCES issues(id) ON DELETE CASCADE,313+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,314+    filename    TEXT NOT NULL,315+    mime_type   TEXT NOT NULL,316+    size_bytes  BIGINT NOT NULL,317+    path        TEXT NOT NULL,318+    tmp_token   TEXT,319+    tmp_expires BIGINT,320+    created_at  BIGINT NOT NULL321+);322+CREATE INDEX issue_attachments_issue ON issue_attachments(issue_id, created_at);323+CREATE INDEX issue_attachments_repo ON issue_attachments(repo_id, created_at);324+325+-- ── Caches and challenges ────────────────────────────────────────────326+327+CREATE TABLE graph_cache (328+    repo         TEXT NOT NULL,329+    head_save_id TEXT NOT NULL,330+    json         TEXT NOT NULL,331+    computed_at  BIGINT NOT NULL,332+    PRIMARY KEY (repo, head_save_id)333+);334+335+CREATE TABLE archive_cache (336+    repo_id    TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,337+    save_id    TEXT NOT NULL,338+    path       TEXT NOT NULL,339+    size_bytes BIGINT NOT NULL,340+    created_at BIGINT NOT NULL,341+    -- Vestigial: live archive nar hashes are kept in archive_hashes now. The342+    -- column still exists in the SQLite source schema, so it is mirrored here so343+    -- the one-time SQLite→Postgres migration (which copies every source column344+    -- via PRAGMA table_info) does not fail on a missing target column.345+    nar_hash   TEXT,346+    PRIMARY KEY (repo_id, save_id)347+);348+349+CREATE TABLE archive_hashes (350+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,351+    save_id     TEXT NOT NULL,352+    blake3      TEXT NOT NULL,353+    sha256      TEXT NOT NULL,354+    nar_hash    TEXT,355+    computed_at BIGINT NOT NULL,356+    PRIMARY KEY (repo_id, save_id)357+);358+359+CREATE TABLE pow_challenges (360+    id         TEXT PRIMARY KEY,361+    nonce      TEXT NOT NULL,362+    difficulty BIGINT NOT NULL DEFAULT 4,363+    created_at BIGINT NOT NULL,364+    expires_at BIGINT NOT NULL,365+    used       BIGINT NOT NULL DEFAULT 0366+);367+368+-- ── Accounts, sessions, auth ─────────────────────────────────────────369+370+CREATE TABLE accounts (371+    id                   TEXT PRIMARY KEY,372+    username             TEXT NOT NULL UNIQUE,373+    bio                  TEXT,374+    location             TEXT,375+    website              TEXT,376+    email                TEXT,377+    recovery_hash        TEXT NOT NULL,378+    created_at           BIGINT NOT NULL,379+    tier                 TEXT NOT NULL DEFAULT 'contributor'380+        CHECK (tier IN ('operator', 'team', 'contributor')),381+    -- Operator web-login credentials (bcrypt password + base32 TOTP secret).382+    -- NULL until the operator enrols at /login; only the operator-tier row uses them.383+    password_hash        TEXT,384+    totp_secret          TEXT,385+    notif_offer_accepted BIGINT NOT NULL DEFAULT 1,386+    notif_offer_rejected BIGINT NOT NULL DEFAULT 1,387+    notif_issue_response BIGINT NOT NULL DEFAULT 1,388+    notif_issue_resolved BIGINT NOT NULL DEFAULT 1,389+    notif_issue_status   BIGINT NOT NULL DEFAULT 1390+);391+CREATE INDEX accounts_username ON accounts(username);392+-- Partial unique: one account per non-null email, enforced at the database393+-- so the join handler's check-then-act can't lose a race under concurrent394+-- registrations (review F4). Existing databases get this via db.zig's395+-- ensureUniqueEmailIndex idempotent migration instead, which must detect396+-- pre-existing duplicates first (D4).397+CREATE UNIQUE INDEX accounts_email_unique ON accounts(email) WHERE email IS NOT NULL;398+399+-- ── Internal tickets ─────────────────────────────────────────────────400+-- Operator/team-only ticket tracker. created_by/assigned_to reference401+-- accounts(id) (TEXT, not the int ids the original brief assumed).402+-- resolving_save_id is a plain TEXT column with no FK: saves has a403+-- composite PRIMARY KEY (id, repo), so a single-column FK to saves(id)404+-- is not expressible; integrity is enforced in the application layer.405+406+CREATE TABLE tickets (407+    id                BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,408+    title             TEXT NOT NULL,409+    body              TEXT NOT NULL DEFAULT '',410+    state             TEXT NOT NULL DEFAULT 'open'411+        CHECK (state IN ('open', 'in_progress', 'closed')),412+    repo              TEXT REFERENCES repos(name) ON UPDATE CASCADE ON DELETE SET NULL,413+    resolving_save_id TEXT,414+    created_by        TEXT NOT NULL REFERENCES accounts(id),415+    assigned_to       TEXT REFERENCES accounts(id) ON DELETE SET NULL,416+    created_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,417+    updated_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,418+    closed_at         BIGINT419+);420+CREATE INDEX tickets_state ON tickets(state, created_at DESC);421+CREATE INDEX tickets_repo ON tickets(repo);422+CREATE INDEX tickets_assigned ON tickets(assigned_to);423+CREATE INDEX tickets_created_by ON tickets(created_by);424+425+CREATE TABLE ticket_attachments (426+    id          TEXT PRIMARY KEY,427+    ticket_id   BIGINT REFERENCES tickets(id) ON DELETE CASCADE,428+    repo_id     TEXT REFERENCES repos(name) ON UPDATE CASCADE,429+    filename    TEXT NOT NULL,430+    mime_type   TEXT NOT NULL,431+    size_bytes  BIGINT NOT NULL,432+    path        TEXT NOT NULL,433+    created_at  BIGINT NOT NULL434+);435+CREATE INDEX ticket_attachments_ticket ON ticket_attachments(ticket_id, created_at);436+CREATE INDEX ticket_attachments_repo ON ticket_attachments(repo_id, created_at);437+438+-- ── Collaboration layer + notifications ──────────────────────────────439+-- author/account columns reference accounts(id) (TEXT). Comment ids are440+-- BIGINT identity to match the project convention (the brief assumed441+-- SQLite-era INTEGER identity). save_comments carries both save_id and442+-- repo so a composite FK to saves(id, repo) is expressible; the other443+-- save-keyed tables hold only save_id (plain TEXT, no FK — saves has a444+-- composite PK) and rely on application-layer integrity. Boolean-ish445+-- columns are BIGINT 0/1 to match the existing schema (accounts.notif_*,446+-- repos.*), since the pg shim binds/reads integers, not SQL BOOLEAN.447+448+CREATE TABLE save_comments (449+    id                BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,450+    save_id           TEXT NOT NULL,451+    repo              TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE ON DELETE CASCADE,452+    author_account_id TEXT NOT NULL REFERENCES accounts(id),453+    body              TEXT NOT NULL,454+    created_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,455+    updated_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,456+    edited            BIGINT NOT NULL DEFAULT 0,457+    FOREIGN KEY (save_id, repo) REFERENCES saves(id, repo) ON DELETE CASCADE458+);459+CREATE INDEX save_comments_save_id ON save_comments(save_id, created_at);460+CREATE INDEX save_comments_repo ON save_comments(repo, created_at DESC);461+CREATE INDEX save_comments_author ON save_comments(author_account_id);462+463+CREATE TABLE save_acknowledgements (464+    save_id    TEXT NOT NULL,465+    account_id TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,466+    kind       TEXT NOT NULL DEFAULT 'ack'467+        CHECK (kind IN ('ack', 'flag')),468+    created_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,469+    PRIMARY KEY (save_id, account_id)470+);471+CREATE INDEX save_acks_save_id ON save_acknowledgements(save_id);472+473+CREATE TABLE draft_comments (474+    id                BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,475+    offer_id          TEXT NOT NULL REFERENCES pending_offers(offer_id) ON DELETE CASCADE,476+    author_account_id TEXT NOT NULL REFERENCES accounts(id),477+    body              TEXT NOT NULL,478+    created_at        BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT479+);480+CREATE INDEX draft_comments_offer ON draft_comments(offer_id, created_at);481+482+CREATE TABLE notifications (483+    id           BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,484+    account_id   TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,485+    kind         TEXT NOT NULL CHECK (kind IN (486+        'save_comment', 'save_comment_reply', 'save_mention', 'save_ack',487+        'save_flag', 'contribution_accepted', 'contribution_declined',488+        'contribution_pending', 'draft_comment', 'issue_filed', 'issue_reply',489+        'issue_response', 'issue_resolved', 'issue_status',490+        'ticket_assigned', 'ticket_state_changed', 'team_promoted',491+        'milestone_saves', 'milestone_downloads', 'milestone_first_steal',492+        'milestone_first_contributor', 'milestone_trusted'493+    )),494+    payload_json TEXT NOT NULL DEFAULT '{}',495+    preview      TEXT NOT NULL DEFAULT '',496+    href         TEXT NOT NULL DEFAULT '/',497+    read         BIGINT NOT NULL DEFAULT 0,498+    created_at   BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT,499+    read_at      BIGINT500+);501+CREATE INDEX notifications_account_unread ON notifications(account_id, read, created_at DESC);502+CREATE INDEX notifications_account_all ON notifications(account_id, created_at DESC);503+CREATE INDEX notifications_created_at ON notifications(created_at DESC);504+505+CREATE TABLE notification_preferences (506+    account_id TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,507+    kind       TEXT NOT NULL,508+    in_app     BIGINT NOT NULL DEFAULT 1,509+    email      BIGINT NOT NULL DEFAULT 0,510+    PRIMARY KEY (account_id, kind)511+);512+513+CREATE TABLE notification_category_prefs (514+    account_id TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,515+    category   TEXT NOT NULL CHECK (category IN (516+        'contributions', 'discussion', 'issues', 'tickets', 'milestones'517+    )),518+    in_app     BIGINT NOT NULL DEFAULT 1,519+    email      BIGINT NOT NULL DEFAULT 0,520+    PRIMARY KEY (account_id, category)521+);522+523+CREATE TABLE sessions (524+    id         TEXT PRIMARY KEY,525+    account_id TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,526+    created_at BIGINT NOT NULL,527+    expires_at BIGINT NOT NULL,528+    last_used  BIGINT NOT NULL,529+    user_agent TEXT530+);531+CREATE INDEX sessions_account ON sessions(account_id);532+CREATE INDEX sessions_expiry ON sessions(expires_at);533+534+CREATE TABLE account_recovery_codes (535+    id          TEXT PRIMARY KEY,536+    account_id  TEXT NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,537+    code_hash   TEXT NOT NULL,538+    created_at  BIGINT NOT NULL,539+    expires_at  BIGINT NOT NULL,540+    used        BIGINT NOT NULL DEFAULT 0541+);542+CREATE INDEX account_recovery_codes_account ON account_recovery_codes(account_id, expires_at);543+544+-- ── Health, safety, trust ────────────────────────────────────────────545+546+CREATE TABLE repo_health (547+    repo_id        TEXT PRIMARY KEY REFERENCES repos(name) ON UPDATE CASCADE,548+    save_id        TEXT NOT NULL,549+    computed_at    BIGINT NOT NULL,550+    secrets_status TEXT NOT NULL DEFAULT 'clean',551+    secrets_detail TEXT,552+    deps_status    TEXT NOT NULL DEFAULT 'none',553+    deps_detail    TEXT,554+    builds_status  TEXT NOT NULL DEFAULT 'none',555+    builds_detail  TEXT,556+    agents_status  TEXT NOT NULL DEFAULT 'none',557+    agents_detail  TEXT,558+    license_status TEXT NOT NULL DEFAULT 'none',559+    license_detail TEXT,560+    trust_status   TEXT NOT NULL DEFAULT 'clean',561+    trust_detail   TEXT,562+    collapsed_html TEXT NOT NULL DEFAULT ''563+);564+565+CREATE TABLE secret_scan_results (566+    id          TEXT PRIMARY KEY,567+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,568+    save_id     TEXT NOT NULL,569+    file_path   TEXT NOT NULL,570+    line_number BIGINT,571+    pattern     TEXT NOT NULL,572+    entropy     DOUBLE PRECISION,573+    severity    TEXT NOT NULL,574+    status      TEXT NOT NULL,575+    detected_at BIGINT NOT NULL576+);577+CREATE INDEX secret_scan_results_repo_save578+    ON secret_scan_results(repo_id, save_id, detected_at DESC);579+580+CREATE TABLE secret_false_positives (581+    repo_id   TEXT NOT NULL,582+    file_path TEXT NOT NULL,583+    marked_at BIGINT NOT NULL,584+    PRIMARY KEY (repo_id, file_path)585+);586+587+CREATE TABLE code_finding_false_positives (588+    repo_id   TEXT NOT NULL,589+    code      TEXT NOT NULL,590+    path      TEXT NOT NULL,591+    marked_at BIGINT NOT NULL,592+    PRIMARY KEY (repo_id, code, path)593+);594+595+-- Operator-dismissed safety findings. A suppressed finding is filtered from596+-- the public repo page and the admin panel; the dismissed list remains597+-- visible and reversible. Keyed by (repo_id, file_path, rule, fingerprint)598+-- so a suppression only matches when the same content fires the same rule.599+CREATE TABLE safety_suppressions (600+    id           TEXT PRIMARY KEY,601+    repo_id      TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE ON DELETE CASCADE,602+    file_path    TEXT NOT NULL,603+    rule         TEXT NOT NULL,604+    fingerprint  TEXT NOT NULL,605+    dismissed_at BIGINT NOT NULL,606+    dismissed_by TEXT NOT NULL REFERENCES accounts(id),607+    note         TEXT608+);609+CREATE INDEX safety_suppressions_repo ON safety_suppressions(repo_id, file_path, rule);610+611+CREATE TABLE dep_advisories (612+    id          TEXT PRIMARY KEY,613+    repo_id     TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,614+    save_id     TEXT NOT NULL,615+    dep_name    TEXT NOT NULL,616+    dep_version TEXT NOT NULL,617+    cve_id      TEXT,618+    severity    TEXT NOT NULL,619+    summary     TEXT NOT NULL,620+    fix_version TEXT,621+    detected_at BIGINT NOT NULL622+);623+CREATE INDEX dep_advisories_repo_save624+    ON dep_advisories(repo_id, save_id, detected_at DESC);625+626+CREATE TABLE binary_sizes (627+    id         TEXT PRIMARY KEY,628+    repo_id    TEXT NOT NULL REFERENCES repos(name) ON UPDATE CASCADE,629+    save_id    TEXT NOT NULL,630+    platform   TEXT NOT NULL,631+    size_bytes BIGINT NOT NULL,632+    built_at   BIGINT NOT NULL633+);634+CREATE INDEX binary_sizes_repo_save635+    ON binary_sizes(repo_id, save_id, built_at DESC);636+637+CREATE TABLE trust_results (638+    repo_id             TEXT PRIMARY KEY REFERENCES repos(name) ON UPDATE CASCADE,639+    save_id             TEXT NOT NULL,640+    trusted             BIGINT NOT NULL,641+    secrets_clean       BIGINT NOT NULL,642+    no_binary_blobs     BIGINT NOT NULL,643+    build_scripts_clean BIGINT NOT NULL,644+    no_trojan_source    BIGINT NOT NULL,645+    supply_chain_stable BIGINT NOT NULL,646+    licenses_compatible BIGINT NOT NULL,647+    has_attribution     BIGINT NOT NULL,648+    detail_json         TEXT NOT NULL DEFAULT '{}',649+    computed_at         BIGINT NOT NULL650+);651+652+-- ── Engagement and notifications ─────────────────────────────────────653+654+CREATE TABLE engagement_events (655+    id          TEXT PRIMARY KEY,656+    event_type  TEXT NOT NULL,657+    repo_id     TEXT,658+    save_id     TEXT,659+    platform    TEXT,660+    occurred_at BIGINT NOT NULL661+);662+CREATE INDEX eng_repo_type ON engagement_events(repo_id, event_type, occurred_at DESC);663+CREATE INDEX eng_type_time ON engagement_events(event_type, occurred_at DESC);664+665+CREATE TABLE notification_queue (666+    id         TEXT PRIMARY KEY,667+    account_id TEXT,668+    type       TEXT NOT NULL,669+    payload    TEXT NOT NULL,670+    status     TEXT NOT NULL DEFAULT 'pending',671+    attempts   BIGINT NOT NULL DEFAULT 0,672+    next_retry BIGINT,673+    created_at BIGINT NOT NULL,674+    sent_at    BIGINT675+);676+CREATE INDEX notification_queue_pending677+    ON notification_queue(status, next_retry);678+679+-- ┌─────────────────────────────────────────────────────────────────┐680+-- │  JOB QUEUE — Postgres-native, SELECT FOR UPDATE SKIP LOCKED.      │681+-- │  New in the Postgres architecture; has no SQLite predecessor.     │682+-- └─────────────────────────────────────────────────────────────────┘683+684+CREATE TABLE jobs (685+    id           BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,686+    kind         TEXT NOT NULL,687+    payload_json TEXT NOT NULL DEFAULT '{}',688+    status       TEXT NOT NULL DEFAULT 'pending'689+        CHECK (status IN ('pending','running','done','failed')),690+    -- Only one pending/running job per (kind, dedup_key) — collapses the691+    -- thundering herd of a 1000-save offer into one job of each kind per repo.692+    dedup_key    TEXT,693+    priority     BIGINT NOT NULL DEFAULT 5,694+    attempts     BIGINT NOT NULL DEFAULT 0,695+    max_attempts BIGINT NOT NULL DEFAULT 3,696+    created_at   BIGINT NOT NULL,697+    started_at   BIGINT,698+    completed_at BIGINT,699+    error_text   TEXT,700+    run_after    BIGINT NOT NULL DEFAULT 0701+);702+703+-- Worker poll path: lowest priority then oldest, pending only.704+CREATE INDEX jobs_pending ON jobs(priority, created_at)705+    WHERE status = 'pending';706+707+-- Deduplication: enforce one live job per (kind, dedup_key).708+CREATE UNIQUE INDEX jobs_dedup ON jobs(kind, dedup_key)709+    WHERE status IN ('pending','running') AND dedup_key IS NOT NULL;710+711+-- Cleanup scan for pruning finished jobs.712+CREATE INDEX jobs_completed_at ON jobs(completed_at)713+    WHERE status IN ('done','failed');714+715+COMMIT;

view file

tests/import.test.ts+304 −0
@@ -0,0 +1,304 @@1+import { afterAll, beforeAll, expect, test } from "bun:test";2+import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises";3+import { tmpdir } from "node:os";4+import { join } from "node:path";5+import { blake3 } from "@noble/hashes/blake3.js";6+import { bytesToHex } from "@noble/hashes/utils.js";7+import { SQL } from "bun";8+import { instance } from "./browser.ts";9+import { makeRepos } from "./fixture.ts";10+11+// A throwaway Postgres holding Kepr 2's schema and a few synthetic rows, stopped after the tests.12+const SCRIPT = join(import.meta.dir, "../scripts/import-kepr2.ts");13+const root = await mkdtemp(join(tmpdir(), "kepr-import-"));14+const pgDir = join(root, "pg");15+const kepr2Data = join(root, "kepr2");16+let url = "";17+let bucketDir = "";18+let save = "";19+20+const hex = (bytes: Uint8Array) => bytesToHex(blake3(bytes));21+const sha = (bytes: Uint8Array) => new Bun.CryptoHasher("sha256").update(bytes).digest("hex");22+const bytesOf = (text: string) => new TextEncoder().encode(text);23+const run = (cmd: string[]) => {24+	const out = Bun.spawnSync(cmd, { cwd: root, env: { PATH: process.env.PATH, LC_ALL: "C" } });25+	if (out.exitCode !== 0) throw new Error(`${cmd.join(" ")}: ${out.stderr.toString()}`);26+};27+28+async function kepr2File(path: string, text: string) {29+	await mkdir(join(kepr2Data, "releases", path, ".."), { recursive: true });30+	await writeFile(join(kepr2Data, "releases", path), text);31+	return bytesOf(text);32+}33+34+beforeAll(async () => {35+	const repos = await makeRepos(["demo"]);36+	bucketDir = repos.bucketDir;37+	save = repos.saves.demo as string;38+	const probe = Bun.serve({ port: 0, fetch: () => new Response() });39+	const port = probe.port;40+	probe.stop(true);41+	run(["initdb", "-D", pgDir, "-U", "kepr", "-A", "trust"]);42+	const options = `-p ${port} -k ${pgDir} -c listen_addresses=127.0.0.1`;43+	run(["pg_ctl", "-D", pgDir, "-o", options, "-l", join(root, "pg.log"), "-w", "start"]);44+	url = `postgres://[email protected]:${port}/postgres`;45+	run([46+		"psql",47+		url,48+		"-q",49+		"-v",50+		"ON_ERROR_STOP=1",51+		"-f",52+		join(import.meta.dir, "fixtures/kepr2/schema.sql"),53+	]);54+55+	const legacy = await kepr2File("demo/1.0.0/linux-x86_64/demo", "demo 1.0.0 linux");56+	const byPlatform = await kepr2File("demo/1.1.0/macos-aarch64", "demo 1.1.0 mac");57+	const apk = await kepr2File("demo/1.1.0/android-arm64/demo-1.1.0.apk", "demo 1.1.0 apk");58+	const failed = await kepr2File("demo/1.2.0/linux-x86_64/demo", "demo 1.2.0 failed");59+	await kepr2File("demo/1.3.0/linux-x86_64/demo", "demo 1.3.0 tampered");60+	const secret = await kepr2File("secret/0.1.0/linux-x86_64/secret", "secret 0.1.0");61+	const oldPlatform = await kepr2File(62+		"demo/1.4.0/x86_64-linux/demo",63+		"demo 1.4.0 old platform name",64+	);65+	const artifact = (66+		id: string,67+		run: string,68+		repo: string,69+		version: string,70+		platform: string,71+		name: string,72+		bytes: Uint8Array,73+		b3 = hex(bytes),74+	) =>75+		`('${id}', '${run}', '${repo}', '${version}', '${platform}', '${name}', ${bytes.length}, '${sha(bytes)}', '${b3}', 5)`;76+	const pg = new SQL(url);77+	await pg.unsafe(`78+		INSERT INTO repos (name, created_at, description, categories, private, builds_enabled) VALUES79+			('demo', 1, 'A demo.', 'tools, cli', 0, 1), ('secret', 1, NULL, '', 1, 0);80+		INSERT INTO accounts (id, username, recovery_hash, created_at) VALUES ('acc-r', 'rita', 'x', 1), ('acc-o', 'olga', 'x', 1);81+		INSERT INTO issues (id, repo, number, title, description, contact, contact_email, filed_at, filed_save, filed_ver, status, closed_at, closed_save, type, account_id, reporter_name) VALUES82+			(1, 'demo', 7, 'Crash on start', 'It <b>crashes</b>.', NULL, '[email protected]', 1000, 'aaaaaaaaaaaa', '1.0.0', 'inprogress', NULL, NULL, 'bug', 'acc-r', NULL),83+			(2, 'demo', 9, 'Dark mode', 'Please.', '[email protected]', NULL, 1500, 'aaaaaaaaaaaa', NULL, 'wontfix', 2000, '${save}', 'enhancement', NULL, 'Sam');84+		INSERT INTO issue_entries (id, issue_id, account_id, entry_type, body, created_at) VALUES85+			('e1', 1, 'operator', 'reply', 'Looking.', 1100), ('e2', 1, 'acc-r', 'reply', 'Thanks!', 1200),86+			('e3', 1, 'operator', 'status_change', NULL, 1300), ('e4', 1, 'acc-o', 'reply', 'Me too.', 1400);87+		INSERT INTO issue_notes (issue_id, body, posted_at, is_op) VALUES (1, 'internal only', 1250, 1);88+		INSERT INTO engagement_events (id, event_type, repo_id, occurred_at) VALUES89+			('v1', 'nix_install', 'demo', 1), ('v2', 'nix_install', 'demo', 2), ('v3', 'nix_install', 'demo', 3),90+			('v4', 'archive_download', 'demo', 4), ('v5', 'nix_install', 'secret', 5);91+		INSERT INTO build_runs (id, repo_id, save_id, version, trigger, status, started_at, finished_at, release_notes) VALUES92+			('b1', 'demo', '${save}', '1.0.0', 'manual', 'success', 10, 20, NULL),93+			('b2', 'demo', '${save}', '1.1.0', 'manual', 'partial', 30, 40, 'Starts twice as fast and no longer crashes on start.'),94+			('b3', 'demo', '${save}', '1.2.0', 'manual', 'failed', 50, 60, NULL),95+			('b4', 'demo', '${save}', '1.3.0', 'manual', 'success', 70, 80, NULL),96+			('b5', 'secret', '${save}', '0.1.0', 'manual', 'success', 90, 100, NULL),97+			('b6', 'demo', '${save}', '1.4.0', 'manual', 'success', 110, 120, NULL);98+		INSERT INTO release_artifacts (id, build_run, repo_id, version, platform, filename, size_bytes, sha256, blake3, built_at) VALUES99+			${artifact("a1", "b1", "demo", "1.0.0", "linux-x86_64", "demo", legacy)},100+			${artifact("a2", "b2", "demo", "1.1.0", "macos-aarch64", "demo", byPlatform)},101+			${artifact("a3", "b2", "demo", "1.1.0", "android-arm64", "demo-1.1.0.apk", apk)},102+			${artifact("a4", "b3", "demo", "1.2.0", "linux-x86_64", "demo", failed)},103+			${artifact("a5", "b4", "demo", "1.3.0", "linux-x86_64", "demo", legacy, hex(legacy))},104+			${artifact("a6", "b5", "secret", "0.1.0", "linux-x86_64", "secret", secret)},105+			${artifact("a7", "b6", "demo", "1.4.0", "x86_64-linux", "demo", oldPlatform)};106+	`);107+	await pg.close();108+});109+110+afterAll(() => run(["pg_ctl", "-D", pgDir, "-m", "fast", "-w", "stop"]));111+112+async function importKepr2(target: Record<string, string>, keprData: string, ...args: string[]) {113+	const env = {114+		PATH: process.env.PATH,115+		KEPR2_DATABASE_URL: url,116+		KEPR2_DATA: kepr2Data,117+		KEPR_DATA: keprData,118+		...target,119+	};120+	// Not spawnSync: the S3 test's stand-in server runs on this process's event loop.121+	const child = Bun.spawn(["bun", SCRIPT, ...args], { cwd: root, env, stderr: "pipe" });122+	const [stdout, stderr, code] = await Promise.all([123+		new Response(child.stdout).text(),124+		new Response(child.stderr).text(),125+		child.exited,126+	]);127+	if (code !== 0) throw new Error(stderr);128+	return stdout;129+}130+131+async function files(dir: string): Promise<string[]> {132+	return (await readdir(dir, { recursive: true, withFileTypes: true }))133+		.filter((entry) => entry.isFile())134+		.map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1))135+		.sort();136+}137+138+test("--dry-run prints the import and writes neither the bucket nor kepr.db", async () => {139+	const before = await files(bucketDir);140+	const keprData = await mkdtemp(join(tmpdir(), "kepr-import-dry-"));141+	const out = await importKepr2({ KOH_FOLDER: bucketDir }, keprData, "--dry-run");142+	expect(out).toContain("would put demo/releases/1.1.0/release.json");143+	expect(out).toContain('issues {"repo":"demo","number":"7"');144+	expect(await files(bucketDir)).toEqual(before);145+	const inst = await instance(bucketDir, keprData);146+	expect(inst.kepr.query("SELECT COUNT(*) AS n FROM issues").get()).toEqual({ n: 0 });147+});148+149+test("the import lands Kepr 2's issues, counts, settings and releases, and a re-run changes nothing", async () => {150+	const keprData = await mkdtemp(join(tmpdir(), "kepr-import-data-"));151+	const first = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);152+	expect(first).toContain("skip demo/releases/1.3.0: linux-x86_64: not Kepr 2's file");153+	expect(first).toContain("skip demo/releases/1.4.0: release record does not match §11.2");154+	expect(first).not.toContain("1.2.0");155+	const landed = await files(bucketDir);156+	expect(landed.filter((key) => key.includes("/releases/"))).toEqual([157+		"demo/releases/1.0.0/linux-x86_64/demo-linux-x86_64",158+		"demo/releases/1.0.0/release.json",159+		"demo/releases/1.1.0/android-arm64/demo-1.1.0.apk",160+		"demo/releases/1.1.0/macos-aarch64/demo-macos-aarch64",161+		"demo/releases/1.1.0/release.json",162+		"demo/releases/latest",163+		"secret/releases/0.1.0/linux-x86_64/secret-linux-x86_64",164+		"secret/releases/0.1.0/release.json",165+		"secret/releases/latest",166+	]);167+	const again = await importKepr2({ KOH_FOLDER: bucketDir }, keprData);168+	expect(again).not.toMatch(/^(put|would put|differs)/m);169+	// Only the repo settings are set again (to the same values); nothing else is added.170+	expect(171+		again.split("\n").filter((line) => line.endsWith(" set") && !line.startsWith("repos ")),172+	).toEqual([]);173+	expect(await files(bucketDir)).toEqual(landed);174+175+	const inst = await instance(bucketDir, keprData);176+	const q = (sql: string) => inst.kepr.query(sql).all();177+	expect(q("SELECT name, public, category FROM repos ORDER BY name")).toEqual([178+		{ name: "demo", public: 1, category: "tools" },179+		{ name: "secret", public: 0, category: "" },180+	]);181+	expect(q("SELECT repo, count FROM installs ORDER BY repo")).toEqual([182+		{ repo: "demo", count: 3 },183+		{ repo: "secret", count: 1 },184+	]);185+	expect(q("SELECT COUNT(*) AS n FROM downloads")).toEqual([{ n: 0 }]);186+	expect(q("SELECT repo, version, hidden FROM release_settings")).toEqual([187+		{ repo: "secret", version: "0.1.0", hidden: 1 },188+	]);189+	expect(190+		q(191+			"SELECT number, title, body, type, version, reporter_name, contact_email, filed_at, status, closed_at, fixed_in, token_hash FROM issues ORDER BY number",192+		),193+	).toEqual([194+		{195+			number: 7,196+			title: "Crash on start",197+			body: "It <b>crashes</b>.",198+			type: "bug",199+			version: "1.0.0",200+			reporter_name: "@rita",201+			contact_email: "[email protected]",202+			filed_at: 1000,203+			status: "open",204+			closed_at: null,205+			fixed_in: "",206+			token_hash: null,207+		},208+		{209+			number: 9,210+			title: "Dark mode",211+			body: "Please.",212+			type: "idea",213+			version: "",214+			reporter_name: "Sam",215+			contact_email: "[email protected]",216+			filed_at: 1500,217+			status: "closed",218+			closed_at: 2000,219+			fixed_in: save,220+			token_hash: null,221+		},222+	]);223+	expect(q("SELECT author, body, created_at FROM issue_entries ORDER BY id")).toEqual([224+		{ author: "operator", body: "Looking.", created_at: 1100 },225+		{ author: "reporter", body: "Thanks!", created_at: 1200 },226+		{ author: "operator", body: "Me too.", created_at: 1400 },227+	]);228+229+	// Kepr 3 reads what the import wrote as any Koh 3 release.230+	type Latest = { version: string; notes: string; platforms: Record<string, unknown> };231+	const latest = (await (await inst.app.request("/api/demo/releases/latest")).json()) as Latest;232+	expect(latest.version).toBe("1.1.0");233+	expect(latest.notes).toBe("Starts twice as fast and no longer crashes on start.");234+	expect(Object.keys(latest.platforms)).toEqual(["android-arm64", "macos-aarch64"]);235+	const record = await Bun.file(join(bucketDir, "demo/releases/1.0.0/release.json")).json();236+	const legacy = bytesOf("demo 1.0.0 linux");237+	expect(record).toEqual({238+		format: 1,239+		repo: "demo",240+		version: "1.0.0",241+		save,242+		published_at: 20,243+		machine: "kepr2",244+		notes: "",245+		files: [246+			{247+				platform: "linux-x86_64",248+				name: "demo-linux-x86_64",249+				size: legacy.length,250+				blake3: hex(legacy),251+				sha256: sha(legacy),252+			},253+		],254+	});255+	expect(await (await inst.app.request("/demo/issues/7")).text()).toContain("Me too.");256+	expect(inst.kepr.query("SELECT last_sync_error FROM repos WHERE name = 'demo'").get()).toEqual({257+		last_sync_error: null,258+	});259+});260+261+test("into an S3 bucket, files go up as presigned PUTs with If-None-Match, never over an existing key", async () => {262+	const s3Dir = await mkdtemp(join(tmpdir(), "kepr-import-s3-"));263+	const puts: string[] = [];264+	const server = Bun.serve({265+		port: 0,266+		async fetch(req) {267+			const url = new URL(req.url);268+			const path = join(s3Dir, decodeURIComponent(url.pathname.replace("/koh/", "")));269+			const file = Bun.file(path);270+			if (req.method === "GET")271+				return (await file.exists())272+					? new Response(file)273+					: new Response("<Error><Code>NoSuchKey</Code></Error>", { status: 404 });274+			expect([req.headers.get("if-none-match"), url.searchParams.has("X-Amz-Signature")]).toEqual([275+				"*",276+				true,277+			]);278+			if (await file.exists()) return new Response("", { status: 412 });279+			puts.push(url.pathname);280+			await Bun.write(path, await req.arrayBuffer());281+			return new Response("");282+		},283+	});284+	try {285+		const target = {286+			KOH_S3_ENDPOINT: `http://localhost:${server.port}`,287+			KOH_S3_BUCKET: "koh",288+			KOH_S3_REGION: "fr-par",289+			KOH_S3_ACCESS_KEY_ID: "write",290+			KOH_S3_SECRET_ACCESS_KEY: "key",291+		};292+		await importKepr2(target, await mkdtemp(join(tmpdir(), "kepr-import-s3data-")));293+		expect(puts.length).toBe(9);294+		for (const key of await files(s3Dir)) {295+			expect(await Bun.file(join(s3Dir, key)).bytes()).toEqual(296+				await Bun.file(join(bucketDir, key)).bytes(),297+			);298+		}299+		await importKepr2(target, await mkdtemp(join(tmpdir(), "kepr-import-s3data-")));300+		expect(puts.length).toBe(9);301+	} finally {302+		server.stop(true);303+	}304+});

view file