atlas.ts

1import { describeFile, describeFolder, firstSentence } from "../describe/describe.ts";2import { isTestPath, type Outline, outline, repoIndex, resolveImports } from "../describe/links.ts";3import {4	filesAt,5	fileTouches,6	headId,7	objectBytes,8	type Save,9	type Touches,10	tarballGeneration,11} from "../mirror/queries.ts";12import { isText } from "./files.ts";13import { type Html, html } from "./html.ts";14import {15	formatDate,16	formatSize,17	openPublicRepo,18	type PageContext,19	type PublicRepo,20	pathUrl,21	plural,22	relativeTime,23} from "./layout.ts";24import { type MarkdownBase, renderLine, renderMarkdown } from "./markdown.ts";2526type AtlasFile = {27	name: string;28	path: string;29	size: number;30	lines: number | null;31	description: string | null;32	outline: Outline | null;33	/** The saves that touched it, oldest first. */34	saves: Save[];35};3637type AtlasFolder = {38	name: string;39	path: string;40	folders: AtlasFolder[];41	files: AtlasFile[];42	/** The saves that touched anything under it, removed files included, oldest first. */43	saves: Save[];44};4546/** Each file's resolved imports, and who imports each file or `folder/`. */47type Links = { imports: Map<string, string[]>; usedBy: Map<string, string[]> };4849type View = { repo: string; at: string | null; links: Links };5051type Facts = { lines: number | null; description: string | null; outline: Outline | null };5253// A panel lists this many names or links, then says how many more.54const LISTED = 12;5556// Measured: Koh 3's vendored 9 MB sqlite3.c cost every Atlas view ~25 ms to inflate and scan. An57// object never changes, so what it says about itself is kept for the life of the process, keyed by58// hash and file name (the name picks the comment syntax).59// ponytail: unbounded, one small entry per object viewed; an LRU if a bucket reaches millions.60const known = new Map<string, Facts>();6162function factsOf(db: PublicRepo["mirror"], hash: string, name: string): Facts {63	const key = `${hash} ${name}`;64	const cached = known.get(key);65	if (cached) return cached;66	const bytes = objectBytes(db, hash);67	const text = bytes && isText(bytes) ? new TextDecoder().decode(bytes) : null;68	const facts =69		text === null70			? { lines: null, description: null, outline: null }71			: {72					lines: countLines(text),73					description: describeFile(name, text),74					outline: outline(name, text),75				};76	known.set(key, facts);77	return facts;78}7980/** The tree at one save with each file's facts and the links between files. */81type AtlasModel = { root: AtlasFolder; links: Links; count: number };8283// Building a model reads every file's facts and every save that touched each path: about 165 ms84// warm for a 550-file repo on Tylee. Panels are fetched one at a time on hover, so the model is85// kept, keyed by everything that changes what it holds: the save, and the strips and hidden paths86// (`tarballGeneration`).87// ponytail: the newest MODELS models in insertion order, not an LRU; enough for a few repos at once.88const MODELS = 16;89const models = new Map<string, AtlasModel>();9091// In one read snapshot: the sync's thread can apply a hidden list between any two statements, and92// a model kept under the old generation must never hold a file the new one hides.93function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {94	return repo.mirror.transaction(() => buildAtlasModel(repo, saveId))();95}9697function buildAtlasModel(repo: PublicRepo, saveId: string): AtlasModel {98	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;99	const cached = models.get(key);100	if (cached) return cached;101	const touches = fileTouches(repo.mirror, saveId);102	const foldersTouches = folderTouches(touches);103	const root: AtlasFolder = { name: "", path: "", folders: [], files: [], saves: [] };104	// Koh's own rules file is bookkeeping, not part of the project; Koh never saves what it names.105	const entries = filesAt(repo.mirror, saveId).filter((entry) => entry.path !== ".kohignore");106	for (const entry of entries) {107		const parts = entry.path.split("/");108		let folder = root;109		for (const name of parts.slice(0, -1)) {110			let child = folder.folders.find((f) => f.name === name);111			if (!child) {112				const path = folder.path ? `${folder.path}/${name}` : name;113				child = { name, path, folders: [], files: [], saves: foldersTouches.get(path) ?? [] };114				folder.folders.push(child);115			}116			folder = child;117		}118		const name = parts[parts.length - 1] as string;119		folder.files.push({120			name,121			path: entry.path,122			size: entry.size,123			...factsOf(repo.mirror, entry.hash, name),124			saves: touches.byPath.get(entry.path) ?? [],125		});126	}127	const model = { root, links: linksOf(allFiles(root)), count: entries.length };128	if (models.size >= MODELS) models.delete(models.keys().next().value as string);129	models.set(key, model);130	return model;131}132133/** Builds a public repo's Atlas at its head ahead of the first visitor, as the sync calls it. */134export function warmAtlas(ctx: Pick<PageContext, "kepr" | "mirrorDir">, name: string): void {135	const repo = openPublicRepo(ctx, name);136	const head = repo && headId(repo.mirror);137	if (repo && head) atlasModel(repo, head);138}139140/**141 * §21.5's Atlas at one save: the folder tree as `<details>`, so it opens and links without142 * JavaScript. A row's detail panel is fetched from `atlasPanel` on hover or select (kepr.js), so143 * a big repo's page carries rows only. `HEAD` (or the save shown) opens Kepr 2's picker of saves144 * to view it at instead, its rows fetched from `atlasPickerRows` when first opened.145 */146export function atlasSection(repo: PublicRepo, saveId: string, at: string | null): Html {147	const model = atlasModel(repo, saveId);148	const view = { repo: repo.settings.name, at, links: model.links };149	return html`<section class="repo-section" id="atlas">150<details class="repo-section-toggle" open>151<summary class="section-head"><span class="section-label">atlas</span><span class="section-count">${plural(152		model.count,153		"file",154	)}</span></summary>155<div class="atlas-bar">156${picker(view)}157${at && html`<a class="atlas-head-link mono" href="/${view.repo}#atlas">back to HEAD</a>`}158<button type="button" class="atlas-fold-btn mono" hidden>expand all</button>159<button type="button" class="atlas-search-btn mono" hidden>search</button>160<input class="atlas-filter mono" type="search" placeholder="filter by filename" aria-label="Filter files by name" hidden>161</div>162<div class="atlas" data-repo="${view.repo}"${at && html` data-at="${at}"`}>163<div class="atlas-tree">${children(model.root, view)}</div>164<aside class="atlas-panel" hidden><p class="atlas-hint mono">hover or select a row for details</p></aside>165</div>166</details>167</section>`;168}169170/** One row's detail panel: a file's, or a folder's when `path` ends in a slash; null if absent. */171export function atlasPanel(172	repo: PublicRepo,173	saveId: string,174	at: string | null,175	path: string,176): Html | null {177	const model = atlasModel(repo, saveId);178	const view = { repo: repo.settings.name, at, links: model.links };179	const parts = path.replace(/\/$/, "").split("/");180	let folder: AtlasFolder | undefined = model.root;181	for (const name of path.endsWith("/") ? parts : parts.slice(0, -1)) {182		folder = folder?.folders.find((f) => f.name === name);183	}184	if (!folder) return null;185	if (path.endsWith("/")) return folderPanel(folder, view);186	const file = folder.files.find((f) => f.path === path);187	return file ? filePanel(file, view) : null;188}189190// Kepr 2's "explore through time". Its rows arrive when first opened (kepr.js): a big repo's191// every save was most of its page. Without JavaScript, History's save pages link to the Atlas.192function picker(view: View): Html {193	const query = view.at ? `?at=${view.at}` : "";194	return html`<details class="atlas-picker">195<summary class="atlas-ref-tab mono" title="explore through time">${view.at ?? "HEAD"}</summary>196<div class="atlas-picker-popover">197<input class="atlas-picker-filter mono" type="search" placeholder="search" aria-label="Search saves" hidden>198<div class="atlas-picker-list" data-src="/${view.repo}/atlas/saves${query}"><a class="atlas-picker-row" href="#history">pick a save from the history</a></div>199</div>200</details>`;201}202203/** The picker's rows: every save, newest first, `at` marked. kepr.js filters on their text. */204export function atlasPickerRows(repo: string, saves: Save[], at: string | null): Html[] {205	return saves.toReversed().map((save) => {206		const message = save.message.split("\n")[0] as string;207		return html`<a class="atlas-picker-row${save.id === at ? " active" : ""}" href="/${208			repo209		}?at=${save.id}#atlas"><span class="atlas-picker-id">${save.id}</span><span class="atlas-picker-msg">${message}</span><span class="atlas-picker-age">· ${relativeTime(save.created_at)}</span></a>`;210	});211}212213function children(folder: AtlasFolder, view: View): Html[] {214	const byName = (a: { name: string }, b: { name: string }) =>215		a.name.toLowerCase().localeCompare(b.name.toLowerCase()) || a.name.localeCompare(b.name);216	return [217		...folder.folders.sort(byName).map((child) => folderRow(child, view)),218		...folder.files.sort(byName).map((file) => fileRow(file, view)),219	];220}221222function fileRow(file: AtlasFile, view: View): Html {223	const href = `/${view.repo}/files/${pathUrl(file.path)}${view.at ? `?at=${view.at}` : ""}`;224	return html`<a class="atlas-row atlas-file" href="${225		href226	}" data-name="${file.name.toLowerCase()}" data-path="${file.path}">${row(file.name, file.description, markdownBase(view, file.path), file.size, file.saves)}</a>`;227}228229function filePanel(file: AtlasFile, view: View): Html {230	return panel(231		file.name,232		file.path,233		[234			["size", formatSize(file.size)],235			["lines", file.lines === null ? "binary" : file.lines.toLocaleString("en-US")],236			...saveFacts(file.saves, view),237		],238		file.description,239		markdownBase(view, file.path),240		fileMap(file, view),241	);242}243244// Kepr 2's map of the repo, only where a file has something to say: what it offers, what it245// leans on, and what leans on it, tests apart.246function fileMap(file: AtlasFile, view: View): [string, Html[]][] {247	const users = view.links.usedBy.get(file.path) ?? [];248	return [249		["exports", listed((file.outline?.exports ?? []).map((name) => html`${name}`))],250		["imports", targets(view.links.imports.get(file.path) ?? [], view)],251		[252			"used by",253			targets(254				users.filter((path) => !isTestPath(path)),255				view,256			),257		],258		["tested by", targets(users.filter(isTestPath), view)],259	];260}261262function targets(paths: string[], view: View): Html[] {263	return listed(paths.toSorted().map((path) => targetLink(path, view)));264}265266// A file links to its page and a folder to the Atlas with it open; kepr.js jumps to the row.267function targetLink(target: string, view: View): Html {268	const folder = target.endsWith("/");269	const path = folder ? target.slice(0, -1) : target;270	const query = new URLSearchParams(view.at ? { at: view.at } : {});271	if (folder) query.set("open", path);272	const href = folder273		? `/${view.repo}?${query}#atlas`274		: `/${view.repo}/files/${pathUrl(path)}${view.at ? `?${query}` : ""}`;275	const name = `${path.split("/").pop()}${folder ? "/" : ""}`;276	return html`<a class="atlas-jump" href="${href}" data-path="${target}" title="${path}">${name}</a>`;277}278279function listed(items: Html[]): Html[] {280	const more = items.length - LISTED;281	return more > 0 ? [...items.slice(0, LISTED), html`+${more} more`] : items;282}283284function linksOf(files: AtlasFile[]): Links {285	const index = repoIndex(files.map((file) => file.path));286	const imports = new Map<string, string[]>();287	const usedBy = new Map<string, string[]>();288	for (const file of files) {289		const found = resolveImports(file.path, file.outline?.imports ?? [], index);290		imports.set(file.path, found);291		for (const target of found) usedBy.set(target, [...(usedBy.get(target) ?? []), file.path]);292	}293	return { imports, usedBy };294}295296function folderRow(folder: AtlasFolder, view: View): Html {297	const files = allFiles(folder);298	const size = files.reduce((sum, f) => sum + f.size, 0);299	return html`<details class="atlas-folder" data-path="${folder.path}">300<summary class="atlas-row">${row(301		`${folder.name}/`,302		describeFolder(folder.files),303		markdownBase(view, `${folder.path}/`),304		size,305		folder.saves,306	)}</summary>307<div class="atlas-children">${children(folder, view)}</div>308</details>`;309}310311function folderPanel(folder: AtlasFolder, view: View): Html {312	const files = allFiles(folder);313	const size = files.reduce((sum, f) => sum + f.size, 0);314	const lines = files.reduce((sum, f) => sum + (f.lines ?? 0), 0);315	return panel(316		`${folder.name}/`,317		folder.path,318		[319			["size", formatSize(size)],320			["lines", lines.toLocaleString("en-US")],321			["files", files.length.toLocaleString("en-US")],322			...saveFacts(folder.saves, view),323		],324		describeFolder(folder.files),325		markdownBase(view, `${folder.path}/`),326		[["used by", targets(folderUsers(folder, files, view), view)]],327	);328}329330function row(331	name: string,332	description: string | null,333	base: MarkdownBase,334	size: number,335	saves: Save[],336): Html {337	const last = saves[saves.length - 1];338	const meta = [formatSize(size), last && relativeTime(last.created_at)]339		.filter(Boolean)340		.join(" · ");341	return html`<span class="atlas-name">${name}</span>${html`<span class="atlas-desc">${342		description && renderLine(firstSentence(description), base)343	}</span>`}${html`<span class="atlas-meta mono">${meta}</span>`}`;344}345346// Files outside a folder that import it as a module, or anything inside it.347function folderUsers(folder: AtlasFolder, files: AtlasFile[], view: View): string[] {348	const inside = (path: string) => path.startsWith(`${folder.path}/`);349	const users = new Set(view.links.usedBy.get(`${folder.path}/`) ?? []);350	for (const file of files) {351		for (const user of view.links.usedBy.get(file.path) ?? []) if (!inside(user)) users.add(user);352	}353	return [...users];354}355356function panel(357	name: string,358	path: string,359	facts: [string, Html | string][],360	description: string | null,361	base: MarkdownBase,362	map: [string, Html[]][],363): Html {364	const shown = map.filter(([, items]) => items.length > 0);365	return html`<h3 class="atlas-info-name">${name}</h3>366<p class="atlas-info-path mono">${path}</p>367<dl class="atlas-facts mono">${facts.map(368		([term, value]) => html`<dt>${term}</dt><dd>${value}</dd>`,369	)}</dl>370${371	description &&372	html`<div class="markdown atlas-info-desc">${renderMarkdown(description, base)}</div>`373}374${375	shown.length > 0 &&376	html`<dl class="atlas-facts atlas-map mono">${shown.map(377		([term, items]) =>378			html`<dt>${term}</dt><dd>${items.map((item, i) => html`${i > 0 && " · "}${item}`)}</dd>`,379	)}</dl>`380}`;381}382383function saveFacts(saves: Save[], view: View): [string, Html | string][] {384	const [first, last] = [saves[0], saves[saves.length - 1]];385	if (!first || !last) return [];386	return [387		["first save", saveLink(first, view)],388		["last save", saveLink(last, view)],389		["saves", saves.length.toLocaleString("en-US")],390	];391}392393function saveLink(save: Save, view: View): Html {394	return html`<a href="/${395		view.repo396	}/saves/${save.id}">${save.id}</a> ${formatDate(save.created_at)}`;397}398399// Each folder's saves are the line's saves that touched any path under it, in line order.400function folderTouches(touches: Touches): Map<string, Save[]> {401	const ids = new Map<string, Set<string>>();402	for (const [path, saves] of touches.byPath) {403		const parts = path.split("/");404		for (let depth = 1; depth < parts.length; depth++) {405			const folder = parts.slice(0, depth).join("/");406			const set = ids.get(folder) ?? new Set<string>();407			for (const save of saves) set.add(save.id);408			ids.set(folder, set);409		}410	}411	return new Map(412		[...ids].map(([folder, set]) => [folder, touches.line.filter((save) => set.has(save.id))]),413	);414}415416function allFiles(folder: AtlasFolder): AtlasFile[] {417	return [...folder.files, ...folder.folders.flatMap(allFiles)];418}419420// Relative links in a description resolve from the file's own folder, as in a README.421function markdownBase(view: View, path: string): MarkdownBase {422	return { repo: view.repo, dir: path.split("/").slice(0, -1).join("/"), at: view.at };423}424425function countLines(text: string): number {426	if (text === "") return 0;427	let count = 0;428	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;429	return text.endsWith("\n") ? count : count + 1;430}