sync.test.ts

1import { Database } from "bun:sqlite";2import { beforeAll, describe, expect, test } from "bun:test";3import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";4import { tmpdir } from "node:os";5import { join } from "node:path";6import { deflateSync } from "node:zlib";7import { openBucket } from "../src/bucket/reader.ts";8import { syncAll } from "../src/bucket/sync.ts";9import {10	fileAt,11	filesAt,12	flagReason,13	headId,14	history,15	objectBytes,16	repoStats,17	saveByPrefix,18	timeline,19} from "../src/mirror/queries.ts";20import { mirrorState, openMirror } from "../src/mirror/store.ts";21import { openKeprDb } from "../src/server/db.ts";22import { Browser, instance } from "./browser.ts";23import { type Fixture, makeKohBucket } from "./fixture.ts";2425const text = (bytes: Uint8Array | null) => new TextDecoder().decode(bytes ?? new Uint8Array());2627async function syncTarget(fx: Fixture) {28	const dataDir = await mkdtemp(join(tmpdir(), "kepr-data-"));29	const target = {30		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),31		kepr: openKeprDb(dataDir),32		mirrorDir: join(dataDir, "mirror"),33	};34	return { ...target, mirror: () => openMirror(target.mirrorDir, fx.repo) };35}3637describe("a first sync of a koh3 bucket", () => {38	let fx: Fixture;39	let t: Awaited<ReturnType<typeof syncTarget>>;40	beforeAll(async () => {41		fx = await makeKohBucket();42		t = await syncTarget(fx);43		await syncAll(t);44	});45	const [head, checkpoint, second, first] = [0, 1, 2, 3].map((i) => () => fx.saveIds[i] as string);4647	test("registers the repo private, with a clean last sync", () => {48		const row = t.kepr.query("SELECT public, last_sync_at, last_sync_error FROM repos").get() as {49			public: number;50			last_sync_at: number;51			last_sync_error: string | null;52		};53		expect(row.public).toBe(0);54		expect(row.last_sync_error).toBeNull();55		expect(row.last_sync_at).toBeGreaterThan(0);56	});5758	test("history is newest first and leaves out the checkpoint", () => {59		expect(headId(t.mirror())).toBe(head());60		expect(history(t.mirror(), 10, 0).map((s) => s.id)).toEqual([head(), second(), first()]);61		expect(history(t.mirror(), 1, 1).map((s) => s.message)).toEqual(["add readme and entry point"]);62		expect(timeline(t.mirror()).map((s) => s.id)).toEqual([first(), second(), head()]);63		expect(saveByPrefix(t.mirror(), checkpoint())?.is_checkpoint).toBe(true);64	});6566	test("finds a save by a unique prefix of six or more characters", () => {67		const save = saveByPrefix(t.mirror(), second().slice(0, 6));68		expect([save?.id, save?.agent, save?.session]).toEqual([second(), "test-agent", "fixture"]);69		expect(saveByPrefix(t.mirror(), second().slice(0, 5))).toBeNull();70		expect(saveByPrefix(t.mirror(), "zzzzzz")).toBeNull();71	});7273	test("reads files and their bytes at any save", () => {74		const paths = filesAt(t.mirror(), head()).map((f) => f.path);75		expect(paths).toContain("src/lib/deep/util.ts");76		const now = fileAt(t.mirror(), head(), "src/main.ts");77		const then = fileAt(t.mirror(), second(), "src/main.ts");78		expect(text(objectBytes(t.mirror(), now?.hash as string))).toContain("hello");79		expect(text(objectBytes(t.mirror(), then?.hash as string))).toContain("'hi'");80		expect(fileAt(t.mirror(), first(), "src/main.ts")).toBeNull();81		expect(fileAt(t.mirror(), head(), "docs/logo.png")?.size).toBe(70);82	});8384	test("counts saves without checkpoints, and lines and language at head", () => {85		const stats = repoStats(t.mirror());86		expect(stats.saves).toBe(3);87		expect(stats.language).toBe("TypeScript");88		// Source lines only: util.ts 2 + main.ts 2. The README, guide, secret and .kohignore are not89		// code, and vendor/big.c's 100 lines of C are someone else's.90		expect(stats.lines).toBe(4);91		expect(stats.updated).toBe(saveByPrefix(t.mirror(), head())?.created_at ?? -1);92	});9394	test("stores the flag event", () => {95		const flag = t.mirror().query("SELECT kind, save_id, reason FROM events").get();96		expect(flag).toEqual({ kind: "flag", save_id: first(), reason: "holds a secret" });97	});9899	test("a save's flag state is its newest flag or clear event, by time and then ID", async () => {100		expect(flagReason(t.mirror(), first())).toBe("holds a secret");101		expect(flagReason(t.mirror(), head())).toBeNull();102		const mirror = openMirror(await mkdtemp(join(tmpdir(), "kepr-flags-")), "flags");103		const event = mirror.query(104			"INSERT INTO events (id, kind, save_id, reason, created_at, machine) VALUES (?, ?, 'aaaaaaaaaaaa', ?, ?, 'm')",105		);106		event.run("000000000001", "flag", "first", 10);107		expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("first");108		event.run("000000000002", "clear", null, 20);109		expect(flagReason(mirror, "aaaaaaaaaaaa")).toBeNull();110		// Same second: the larger ID is the newer.111		event.run("000000000003", "flag", "again", 20);112		expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");113		event.run("000000000000", "clear", null, 20);114		expect(flagReason(mirror, "aaaaaaaaaaaa")).toBe("again");115	});116117	test("a strip reaches the mirror and drops the object", async () => {118		const secret = fileAt(t.mirror(), head(), "secret.txt")?.hash as string;119		expect(objectBytes(t.mirror(), secret)).not.toBeNull();120		await fx.strip("secret.txt");121		await syncAll(t);122		for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();123		expect(objectBytes(t.mirror(), secret)).toBeNull();124		expect(fileAt(t.mirror(), head(), "src/main.ts")).not.toBeNull();125	});126});127128test("a new save syncs incrementally and a bad format is recorded as the last sync error", async () => {129	const fx = await makeKohBucket("growing");130	const t = await syncTarget(fx);131	await syncAll(t);132	await writeFile(join(fx.repoDir, "new.txt"), "new\n");133	Bun.spawnSync(["koh3", "save", "add new"], {134		cwd: fx.repoDir,135		env: { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" },136	});137	await syncAll(t);138	expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add new");139140	await writeFile(join(fx.bucketDir, fx.repo, "format"), "koh-remote 2");141	await syncAll(t);142	const row = t.kepr.query("SELECT last_sync_error FROM repos").get() as {143		last_sync_error: string;144	};145	expect(row.last_sync_error).toContain('remote format "koh-remote 2"');146});147148test("a whole-history strip reaches every save, those arriving after it too, whatever their time (§10.2)", async () => {149	const fx = await makeKohBucket("late");150	const view = await mkdtemp(join(tmpdir(), "kepr-view-"));151	const viewRepo = join(view, fx.repo);152	await cp(join(fx.bucketDir, fx.repo), viewRepo, { recursive: true });153	// A strip older than every save here: a strip bans its path from all of history, so no save154	// keeps it, however much later it was made or arrives.155	await mkdir(join(viewRepo, "events"), { recursive: true });156	await writeFile(157		join(viewRepo, "events", "0000000000bb"),158		JSON.stringify({159			id: "0000000000bb",160			kind: "strip",161			save_id: "",162			path: "secret.txt",163			reason: "a secret got saved",164			created_at: 1,165			machine: "elsewhere",166		}),167	);168	const oldest = fx.saveIds.at(-1) as string;169	const newest = fx.saveIds[0] as string;170	// First sync: the strip and the oldest save only.171	await writeFile(join(viewRepo, "head"), oldest);172	const t = await syncTarget({ ...fx, bucketDir: view });173	await syncAll(t);174	expect(fileAt(t.mirror(), oldest, "secret.txt")).toBeNull();175	// Then the rest arrive, each made long after the strip.176	await writeFile(join(viewRepo, "head"), newest);177	await syncAll(t);178	expect(headId(t.mirror())).toBe(newest);179	for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();180	expect(fileAt(t.mirror(), newest, "src/main.ts")).not.toBeNull();181});182183describe("the mirror's schema version", () => {184	test("a version 2 mirror reopens and any other version is refused as disposable", async () => {185		const dir = await mkdtemp(join(tmpdir(), "kepr-mirror-"));186		const created = openMirror(dir, "one");187		created.run("PRAGMA wal_checkpoint(TRUNCATE)");188		await Bun.write(join(dir, "copy.db"), Bun.file(join(dir, "one.db")));189		expect(openMirror(dir, "copy").query("PRAGMA user_version").get()).toEqual({190			user_version: 2,191		});192193		for (const version of [1, 3]) {194			const other = new Database(join(dir, `v${version}.db`));195			other.run(`PRAGMA user_version = ${version}`);196			other.close();197			expect(() => openMirror(dir, `v${version}`)).toThrow("disposable");198		}199	});200});201202test("a strip that arrives older than the saves already mirrored still strips them (§10.2, R6)", async () => {203	const fx = await makeKohBucket("older-strip");204	const t = await syncTarget(fx);205	await syncAll(t);206	// Every save in the mirror holds secret.txt …207	for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).not.toBeNull();208209	// … and now a strip arrives whose own time is older than all of them, which is what a210	// strip made on another machine and offered late looks like. Every save is reached, whatever211	// its time.212	const events = join(fx.bucketDir, fx.repo, "events");213	await mkdir(events, { recursive: true });214	await writeFile(215		join(events, "0000000000aa"),216		JSON.stringify({217			id: "0000000000aa",218			kind: "strip",219			save_id: "",220			path: "secret.txt",221			reason: "a secret got saved",222			created_at: 1,223			machine: "elsewhere",224		}),225	);226	await syncAll(t);227228	for (const id of fx.saveIds) expect(fileAt(t.mirror(), id, "secret.txt")).toBeNull();229	expect(fileAt(t.mirror(), fx.saveIds[0] as string, "src/main.ts")).not.toBeNull();230});231232test("a save with an unsafe path lands without it, keeps syncing, and the admin status names it (A3)", async () => {233	const fx = await makeKohBucket("quarantine");234	const head = fx.saveIds[0] as string;235	const key = join(fx.bucketDir, fx.repo, "saves", head);236	const record = JSON.parse(await readFile(key, "utf8"));237	record.files.push({ ...record.files[0], path: "Icon\r" });238	await writeFile(key, JSON.stringify(record));239	const inst = await instance(fx.bucketDir);240	const mirror = openMirror(join(inst.dataDir, "mirror"), fx.repo);241	expect(inst.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({242		last_sync_error: null,243	});244	expect(filesAt(mirror, head).map((file) => file.path)).toContain("README.md");245	expect(filesAt(mirror, head).map((file) => file.path)).not.toContain("Icon\r");246247	await writeFile(join(fx.repoDir, "later.txt"), "later\n");248	const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };249	Bun.spawnSync(["koh3", "save", "a later save"], { cwd: fx.repoDir, env });250	Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });251	await syncAll({252		bucket: openBucket({ kind: "folder", path: fx.bucketDir }),253		kepr: inst.kepr,254		mirrorDir: join(inst.dataDir, "mirror"),255	});256	expect(history(mirror, 1, 0)[0]?.message).toBe("a later save");257258	const browser = new Browser(inst.app);259	await browser.signIn();260	const page = await (await browser.get("/admin/repos")).text();261	expect(page).toContain("1 path quarantined: Icon\\x0d");262});263264test("a strip event with an illegal path is ignored, as Koh ignores it (§5.5)", async () => {265	const fx = await makeKohBucket("illegal-strip");266	for (const [id, path] of [267		["aaaaaaaaaaaa", "src//"],268		["bbbbbbbbbbbb", ""],269		["cccccccccccc", "src\ud800"],270	]) {271		const event = { id, kind: "strip", save_id: "", path, reason: "", created_at: 1, machine: "x" };272		await writeFile(join(fx.bucketDir, fx.repo, "events", id as string), JSON.stringify(event));273	}274	const t = await syncTarget(fx);275	await syncAll(t);276	expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({277		last_sync_error: null,278	});279	expect(filesAt(t.mirror(), fx.saveIds[0] as string).map((file) => file.path)).toContain(280		"src/main.ts",281	);282});283284test("an object that inflates past 512 MiB is quarantined without stopping the sync (A3)", async () => {285	const fx = await makeKohBucket("oversized");286	const head = fx.saveIds[0] as string;287	const key = join(fx.bucketDir, fx.repo, "saves", head);288	const record = JSON.parse(await readFile(key, "utf8"));289	const hash = "ab".repeat(32);290	record.files.push({ path: "huge.bin", hash, executable: false });291	await writeFile(key, JSON.stringify(record));292	await writeFile(293		join(fx.bucketDir, fx.repo, "objects", hash),294		deflateSync(new Uint8Array(512 * 1024 * 1024 + 1)),295	);296	const t = await syncTarget(fx);297	await syncAll(t);298	expect(t.kepr.query("SELECT last_sync_error FROM repos").get()).toEqual({299		last_sync_error: null,300	});301	expect(filesAt(t.mirror(), head).map((file) => file.path)).not.toContain("huge.bin");302	expect(JSON.parse(mirrorState(t.mirror(), "quarantined") ?? "[]")).toEqual(["huge.bin"]);303});304305test("the line count skips a file over 8 MiB (A3)", async () => {306	const fx = await makeKohBucket("large-text");307	const t = await syncTarget(fx);308	await syncAll(t);309	const before = repoStats(t.mirror()).lines;310	await writeFile(join(fx.repoDir, "big.txt"), "x\n".repeat(4.5 * 1024 * 1024));311	const env = { PATH: process.env.PATH, HOME: join(fx.repoDir, "../../home"), NO_COLOR: "1" };312	Bun.spawnSync(["koh3", "save", "add a big text file"], { cwd: fx.repoDir, env });313	Bun.spawnSync(["koh3", "offer"], { cwd: fx.repoDir, env });314	await syncAll(t);315	expect(history(t.mirror(), 1, 0)[0]?.message).toBe("add a big text file");316	expect(repoStats(t.mirror()).lines).toBe(before);317});