keep only save IDs while walking a sync and cap a batch at 100 saves, since each record lists its whole tree (v3.0.176)

@@ -1,6 +1,6 @@11 {22 	"name": "kepr",3-	"version": "3.0.175",3+	"version": "3.0.176",44 	"private": true,55 	"type": "module",66 	"scripts": {

@@ -15,7 +15,6 @@1515 import { dropTarballs } from "../tarball/cache.ts";1616 import type { Bucket } from "./reader.ts";1717 import {18-	type ArrivingSave,1918 	BucketError,2019 	checkFormat,2120 	type EventRecord,@@ -38,6 +37,7 @@3837 const INTERVAL_MS = 2 * 60 * 1000;3938 // Compressed bytes a sync holds before landing them; sized for a 2 GB host.4039 const LANDING_BYTES = 32 * 1024 * 1024;40+const LANDING_SAVES = 100;4141 4242 export type SyncTarget = {4343 	bucket: Bucket;@@ -105,23 +105,23 @@105105 	await checkFormat(bucket, repo);106106 	const head = await readHead(bucket, repo);107107 	if (head === null) return [];108-	const unknown: ArrivingSave[] = [];109-	const seen = new Set<string>();108+	const unknown = new Set<string>();110109 	for (let cursor = head; cursor !== "" && !knownSave(mirror, cursor); ) {111-		if (seen.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`);112-		seen.add(cursor);113-		const save = await readSave(bucket, repo, cursor);114-		unknown.push(save);115-		cursor = save.parent;110+		if (unknown.has(cursor)) throw new BucketError(`${repo}: save ${cursor} is its own ancestor`);111+		unknown.add(cursor);112+		cursor = (await readSave(bucket, repo, cursor)).parent;116113 	}117-	// Oldest first, landing whenever the fetched objects pass `landingBytes`: a first sync of a big118-	// repo never holds its whole history in memory, and one cut short resumes from the last batch.114+	// Oldest first, landing whenever the batch passes `landingBytes` of objects or LANDING_SAVES115+	// records: a first sync of a big repo never holds its whole history in memory, and one cut short116+	// resumes from the last batch. Records are read again here rather than kept from the walk, since117+	// each lists every file of its tree.119118 	let saves: SaveRecord[] = [];120119 	let objects = new Map<string, StoredObject>();121120 	let quarantined: string[] = [];122121 	let fetched = 0;123122 	const oversized = new Set<string>();124-	for (const save of unknown.reverse()) {123+	for (const id of [...unknown].reverse()) {124+		const save = await readSave(bucket, repo, id);125125 		quarantined.push(...save.quarantined);126126 		for (const file of save.files) {127127 			if (objects.has(file.hash) || oversized.has(file.hash) || knownObject(mirror, file.hash)) {@@ -141,7 +141,7 @@141141 		);142142 		save.files = save.files.filter((file) => !oversized.has(file.hash));143143 		saves.push(save);144-		if (fetched >= landingBytes && save.id !== head) {144+		if ((fetched >= landingBytes || saves.length >= LANDING_SAVES) && save.id !== head) {145145 			land(mirror, {146146 				head: save.id,147147 				releasesLatest: mirrorState(mirror, "releases_latest") ?? "",