atlas.ts

1import { describeFile, describeFolder, firstSentence } from "../describe/describe.ts";2import { isTestPath, type Outline, outline, repoIndex, resolveImports } from "../describe/links.ts";3import {4	filesAt,5	fileTouches,6	headId,7	objectBytes,8	type Save,9	type Touches,10	tarballGeneration,11} from "../mirror/queries.ts";12import { isText } from "./files.ts";13import { type Html, html } from "./html.ts";14import {15	formatDate,16	formatSize,17	openPublicRepo,18	type PageContext,19	type PublicRepo,20	pathUrl,21	plural,22	relativeTime,23} from "./layout.ts";24import { type MarkdownBase, renderLine, renderMarkdown } from "./markdown.ts";2526type AtlasFile = {27	name: string;28	path: string;29	size: number;30	lines: number | null;31	description: string | null;32	outline: Outline | null;33	/** The saves that touched it, oldest first. */34	saves: Save[];35};3637type AtlasFolder = {38	name: string;39	path: string;40	folders: AtlasFolder[];41	files: AtlasFile[];42	/** The saves that touched anything under it, removed files included, oldest first. */43	saves: Save[];44};4546/** Each file's resolved imports, and who imports each file or `folder/`. */47type Links = { imports: Map<string, string[]>; usedBy: Map<string, string[]> };4849type View = { repo: string; at: string | null; links: Links };5051type Facts = { lines: number | null; description: string | null; outline: Outline | null };5253// A panel lists this many names or links, then says how many more.54const LISTED = 12;5556// Measured: Koh 3's vendored 9 MB sqlite3.c cost every Atlas view ~25 ms to inflate and scan. An57// object never changes, so what it says about itself is kept for the life of the process, keyed by58// hash and file name (the name picks the comment syntax).59// ponytail: unbounded, one small entry per object viewed; an LRU if a bucket reaches millions.60const known = new Map<string, Facts>();6162function factsOf(db: PublicRepo["mirror"], hash: string, name: string): Facts {63	const key = `${hash} ${name}`;64	const cached = known.get(key);65	if (cached) return cached;66	const bytes = objectBytes(db, hash);67	const text = bytes && isText(bytes) ? new TextDecoder().decode(bytes) : null;68	const facts =69		text === null70			? { lines: null, description: null, outline: null }71			: {72					lines: countLines(text),73					description: describeFile(name, text),74					outline: outline(name, text),75				};76	known.set(key, facts);77	return facts;78}7980/** The tree at one save with each file's facts and the links between files. */81type AtlasModel = { root: AtlasFolder; links: Links; count: number };8283// Building a model reads every file's facts and every save that touched each path: about 165 ms84// warm for a 550-file repo on Tylee. Panels are fetched one at a time on hover, so the model is85// kept, keyed by everything that changes what it holds: the save, and the strips and hidden paths86// (`tarballGeneration`).87// ponytail: the newest MODELS models in insertion order, not an LRU; enough for a few repos at once.88const MODELS = 16;89const models = new Map<string, AtlasModel>();9091function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {92	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;93	const cached = models.get(key);94	if (cached) return cached;95	const touches = fileTouches(repo.mirror, saveId);96	const foldersTouches = folderTouches(touches);97	const root: AtlasFolder = { name: "", path: "", folders: [], files: [], saves: [] };98	// Koh's own rules file is bookkeeping, not part of the project; Koh never saves what it names.99	const entries = filesAt(repo.mirror, saveId).filter((entry) => entry.path !== ".kohignore");100	for (const entry of entries) {101		const parts = entry.path.split("/");102		let folder = root;103		for (const name of parts.slice(0, -1)) {104			let child = folder.folders.find((f) => f.name === name);105			if (!child) {106				const path = folder.path ? `${folder.path}/${name}` : name;107				child = { name, path, folders: [], files: [], saves: foldersTouches.get(path) ?? [] };108				folder.folders.push(child);109			}110			folder = child;111		}112		const name = parts[parts.length - 1] as string;113		folder.files.push({114			name,115			path: entry.path,116			size: entry.size,117			...factsOf(repo.mirror, entry.hash, name),118			saves: touches.byPath.get(entry.path) ?? [],119		});120	}121	const model = { root, links: linksOf(allFiles(root)), count: entries.length };122	if (models.size >= MODELS) models.delete(models.keys().next().value as string);123	models.set(key, model);124	return model;125}126127/** Builds a public repo's Atlas at its head ahead of the first visitor, as the sync calls it. */128export function warmAtlas(ctx: Pick<PageContext, "kepr" | "mirrorDir">, name: string): void {129	const repo = openPublicRepo(ctx, name);130	const head = repo && headId(repo.mirror);131	if (repo && head) atlasModel(repo, head);132}133134/**135 * §21.5's Atlas at one save: the folder tree as `<details>`, so it opens and links without136 * JavaScript. A row's detail panel is fetched from `atlasPanel` on hover or select (kepr.js), so137 * a big repo's page carries rows only. `HEAD` (or the save shown) opens Kepr 2's picker of saves138 * to view it at instead.139 */140export function atlasSection(141	repo: PublicRepo,142	saveId: string,143	at: string | null,144	saves: Save[],145): Html {146	const model = atlasModel(repo, saveId);147	const view = { repo: repo.settings.name, at, links: model.links };148	return html`<section class="repo-section" id="atlas">149<details class="repo-section-toggle" open>150<summary class="section-head"><span class="section-label">atlas</span><span class="section-count">${plural(151		model.count,152		"file",153	)}</span></summary>154<div class="atlas-bar">155${picker(view, saves)}156${at && html`<a class="atlas-head-link mono" href="/${view.repo}#atlas">back to HEAD</a>`}157<button type="button" class="atlas-fold-btn mono" hidden>expand all</button>158<button type="button" class="atlas-search-btn mono" hidden>search</button>159<input class="atlas-filter mono" type="search" placeholder="filter by filename" aria-label="Filter files by name" hidden>160</div>161<div class="atlas" data-repo="${view.repo}"${at && html` data-at="${at}"`}>162<div class="atlas-tree">${children(model.root, view)}</div>163<aside class="atlas-panel" hidden><p class="atlas-hint mono">hover or select a row for details</p></aside>164</div>165</details>166</section>`;167}168169/** One row's detail panel: a file's, or a folder's when `path` ends in a slash; null if absent. */170export function atlasPanel(171	repo: PublicRepo,172	saveId: string,173	at: string | null,174	path: string,175): Html | null {176	const model = atlasModel(repo, saveId);177	const view = { repo: repo.settings.name, at, links: model.links };178	const parts = path.replace(/\/$/, "").split("/");179	let folder: AtlasFolder | undefined = model.root;180	for (const name of path.endsWith("/") ? parts : parts.slice(0, -1)) {181		folder = folder?.folders.find((f) => f.name === name);182	}183	if (!folder) return null;184	if (path.endsWith("/")) return folderPanel(folder, view);185	const file = folder.files.find((f) => f.path === path);186	return file ? filePanel(file, view) : null;187}188189// Kepr 2's "explore through time": every save, newest first. The filter is kepr.js's.190function picker(view: View, saves: Save[]): Html {191	return html`<details class="atlas-picker">192<summary class="atlas-ref-tab mono" title="explore through time">${view.at ?? "HEAD"}</summary>193<div class="atlas-picker-popover">194<input class="atlas-picker-filter mono" type="search" placeholder="search" aria-label="Search saves" hidden>195<div class="atlas-picker-list">${saves.toReversed().map((save) => {196		const message = save.message.split("\n")[0] as string;197		return html`<a class="atlas-picker-row${save.id === view.at ? " active" : ""}" href="/${198			view.repo199		}?at=${save.id}#atlas" data-q="${`${save.id} ${message}`.toLowerCase()}"><span class="atlas-picker-id">${save.id}</span><span class="atlas-picker-msg">${message}</span><span class="atlas-picker-age">· ${relativeTime(save.created_at)}</span></a>`;200	})}</div>201</div>202</details>`;203}204205function children(folder: AtlasFolder, view: View): Html[] {206	const byName = (a: { name: string }, b: { name: string }) =>207		a.name.toLowerCase().localeCompare(b.name.toLowerCase()) || a.name.localeCompare(b.name);208	return [209		...folder.folders.sort(byName).map((child) => folderRow(child, view)),210		...folder.files.sort(byName).map((file) => fileRow(file, view)),211	];212}213214function fileRow(file: AtlasFile, view: View): Html {215	const href = `/${view.repo}/files/${pathUrl(file.path)}${view.at ? `?at=${view.at}` : ""}`;216	return html`<a class="atlas-row atlas-file" href="${217		href218	}" data-name="${file.name.toLowerCase()}" data-path="${file.path}">${row(file.name, file.description, markdownBase(view, file.path), file.size, file.saves)}</a>`;219}220221function filePanel(file: AtlasFile, view: View): Html {222	return panel(223		file.name,224		file.path,225		[226			["size", formatSize(file.size)],227			["lines", file.lines === null ? "binary" : file.lines.toLocaleString("en-US")],228			...saveFacts(file.saves, view),229		],230		file.description,231		markdownBase(view, file.path),232		fileMap(file, view),233	);234}235236// Kepr 2's map of the repo, only where a file has something to say: what it offers, what it237// leans on, and what leans on it, tests apart.238function fileMap(file: AtlasFile, view: View): [string, Html[]][] {239	const users = view.links.usedBy.get(file.path) ?? [];240	return [241		["exports", listed((file.outline?.exports ?? []).map((name) => html`${name}`))],242		["imports", targets(view.links.imports.get(file.path) ?? [], view)],243		[244			"used by",245			targets(246				users.filter((path) => !isTestPath(path)),247				view,248			),249		],250		["tested by", targets(users.filter(isTestPath), view)],251	];252}253254function targets(paths: string[], view: View): Html[] {255	return listed(paths.toSorted().map((path) => targetLink(path, view)));256}257258// A file links to its page and a folder to the Atlas with it open; kepr.js jumps to the row.259function targetLink(target: string, view: View): Html {260	const folder = target.endsWith("/");261	const path = folder ? target.slice(0, -1) : target;262	const query = new URLSearchParams(view.at ? { at: view.at } : {});263	if (folder) query.set("open", path);264	const href = folder265		? `/${view.repo}?${query}#atlas`266		: `/${view.repo}/files/${pathUrl(path)}${view.at ? `?${query}` : ""}`;267	const name = `${path.split("/").pop()}${folder ? "/" : ""}`;268	return html`<a class="atlas-jump" href="${href}" data-path="${target}" title="${path}">${name}</a>`;269}270271function listed(items: Html[]): Html[] {272	const more = items.length - LISTED;273	return more > 0 ? [...items.slice(0, LISTED), html`+${more} more`] : items;274}275276function linksOf(files: AtlasFile[]): Links {277	const index = repoIndex(files.map((file) => file.path));278	const imports = new Map<string, string[]>();279	const usedBy = new Map<string, string[]>();280	for (const file of files) {281		const found = resolveImports(file.path, file.outline?.imports ?? [], index);282		imports.set(file.path, found);283		for (const target of found) usedBy.set(target, [...(usedBy.get(target) ?? []), file.path]);284	}285	return { imports, usedBy };286}287288function folderRow(folder: AtlasFolder, view: View): Html {289	const files = allFiles(folder);290	const size = files.reduce((sum, f) => sum + f.size, 0);291	return html`<details class="atlas-folder" data-path="${folder.path}">292<summary class="atlas-row">${row(293		`${folder.name}/`,294		describeFolder(folder.files),295		markdownBase(view, `${folder.path}/`),296		size,297		folder.saves,298	)}</summary>299<div class="atlas-children">${children(folder, view)}</div>300</details>`;301}302303function folderPanel(folder: AtlasFolder, view: View): Html {304	const files = allFiles(folder);305	const size = files.reduce((sum, f) => sum + f.size, 0);306	const lines = files.reduce((sum, f) => sum + (f.lines ?? 0), 0);307	return panel(308		`${folder.name}/`,309		folder.path,310		[311			["size", formatSize(size)],312			["lines", lines.toLocaleString("en-US")],313			["files", files.length.toLocaleString("en-US")],314			...saveFacts(folder.saves, view),315		],316		describeFolder(folder.files),317		markdownBase(view, `${folder.path}/`),318		[["used by", targets(folderUsers(folder, files, view), view)]],319	);320}321322function row(323	name: string,324	description: string | null,325	base: MarkdownBase,326	size: number,327	saves: Save[],328): Html {329	const last = saves[saves.length - 1];330	const meta = [formatSize(size), last && relativeTime(last.created_at)]331		.filter(Boolean)332		.join(" · ");333	return html`<span class="atlas-name">${name}</span>${html`<span class="atlas-desc">${334		description && renderLine(firstSentence(description), base)335	}</span>`}${html`<span class="atlas-meta mono">${meta}</span>`}`;336}337338// Files outside a folder that import it as a module, or anything inside it.339function folderUsers(folder: AtlasFolder, files: AtlasFile[], view: View): string[] {340	const inside = (path: string) => path.startsWith(`${folder.path}/`);341	const users = new Set(view.links.usedBy.get(`${folder.path}/`) ?? []);342	for (const file of files) {343		for (const user of view.links.usedBy.get(file.path) ?? []) if (!inside(user)) users.add(user);344	}345	return [...users];346}347348function panel(349	name: string,350	path: string,351	facts: [string, Html | string][],352	description: string | null,353	base: MarkdownBase,354	map: [string, Html[]][],355): Html {356	const shown = map.filter(([, items]) => items.length > 0);357	return html`<h3 class="atlas-info-name">${name}</h3>358<p class="atlas-info-path mono">${path}</p>359<dl class="atlas-facts mono">${facts.map(360		([term, value]) => html`<dt>${term}</dt><dd>${value}</dd>`,361	)}</dl>362${363	description &&364	html`<div class="markdown atlas-info-desc">${renderMarkdown(description, base)}</div>`365}366${367	shown.length > 0 &&368	html`<dl class="atlas-facts atlas-map mono">${shown.map(369		([term, items]) =>370			html`<dt>${term}</dt><dd>${items.map((item, i) => html`${i > 0 && " · "}${item}`)}</dd>`,371	)}</dl>`372}`;373}374375function saveFacts(saves: Save[], view: View): [string, Html | string][] {376	const [first, last] = [saves[0], saves[saves.length - 1]];377	if (!first || !last) return [];378	return [379		["first save", saveLink(first, view)],380		["last save", saveLink(last, view)],381		["saves", saves.length.toLocaleString("en-US")],382	];383}384385function saveLink(save: Save, view: View): Html {386	return html`<a href="/${387		view.repo388	}/saves/${save.id}">${save.id}</a> ${formatDate(save.created_at)}`;389}390391// Each folder's saves are the line's saves that touched any path under it, in line order.392function folderTouches(touches: Touches): Map<string, Save[]> {393	const ids = new Map<string, Set<string>>();394	for (const [path, saves] of touches.byPath) {395		const parts = path.split("/");396		for (let depth = 1; depth < parts.length; depth++) {397			const folder = parts.slice(0, depth).join("/");398			const set = ids.get(folder) ?? new Set<string>();399			for (const save of saves) set.add(save.id);400			ids.set(folder, set);401		}402	}403	return new Map(404		[...ids].map(([folder, set]) => [folder, touches.line.filter((save) => set.has(save.id))]),405	);406}407408function allFiles(folder: AtlasFolder): AtlasFile[] {409	return [...folder.files, ...folder.folders.flatMap(allFiles)];410}411412// Relative links in a description resolve from the file's own folder, as in a README.413function markdownBase(view: View, path: string): MarkdownBase {414	return { repo: view.repo, dir: path.split("/").slice(0, -1).join("/"), at: view.at };415}416417function countLines(text: string): number {418	if (text === "") return 0;419	let count = 0;420	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;421	return text.endsWith("\n") ? count : count + 1;422}