atlas.ts

1import { describeFile, describeFolder, firstSentence } from "../describe/describe.ts";2import { isTestPath, type Outline, outline, repoIndex, resolveImports } from "../describe/links.ts";3import {4	filesAt,5	fileTouches,6	objectBytes,7	type Save,8	type Touches,9	tarballGeneration,10} from "../mirror/queries.ts";11import { isText } from "./files.ts";12import { type Html, html } from "./html.ts";13import {14	formatDate,15	formatSize,16	type PublicRepo,17	pathUrl,18	plural,19	relativeTime,20} from "./layout.ts";21import { type MarkdownBase, renderLine, renderMarkdown } from "./markdown.ts";2223type AtlasFile = {24	name: string;25	path: string;26	size: number;27	lines: number | null;28	description: string | null;29	outline: Outline | null;30	/** The saves that touched it, oldest first. */31	saves: Save[];32};3334type AtlasFolder = {35	name: string;36	path: string;37	folders: AtlasFolder[];38	files: AtlasFile[];39	/** The saves that touched anything under it, removed files included, oldest first. */40	saves: Save[];41};4243/** Each file's resolved imports, and who imports each file or `folder/`. */44type Links = { imports: Map<string, string[]>; usedBy: Map<string, string[]> };4546type View = { repo: string; at: string | null; links: Links };4748type Facts = { lines: number | null; description: string | null; outline: Outline | null };4950// A panel lists this many names or links, then says how many more.51const LISTED = 12;5253// Measured: Koh 3's vendored 9 MB sqlite3.c cost every Atlas view ~25 ms to inflate and scan. An54// object never changes, so what it says about itself is kept for the life of the process, keyed by55// hash and file name (the name picks the comment syntax).56// ponytail: unbounded, one small entry per object viewed; an LRU if a bucket reaches millions.57const known = new Map<string, Facts>();5859function factsOf(db: PublicRepo["mirror"], hash: string, name: string): Facts {60	const key = `${hash} ${name}`;61	const cached = known.get(key);62	if (cached) return cached;63	const bytes = objectBytes(db, hash);64	const text = bytes && isText(bytes) ? new TextDecoder().decode(bytes) : null;65	const facts =66		text === null67			? { lines: null, description: null, outline: null }68			: {69					lines: countLines(text),70					description: describeFile(name, text),71					outline: outline(name, text),72				};73	known.set(key, facts);74	return facts;75}7677/** The tree at one save with each file's facts and the links between files. */78type AtlasModel = { root: AtlasFolder; links: Links; count: number };7980// Building a model reads every file's facts and every save that touched each path: about 165 ms81// warm for a 550-file repo on Tylee. Panels are fetched one at a time on hover, so the model is82// kept, keyed by everything that changes what it holds: the save, and the strips and hidden paths83// (`tarballGeneration`).84// ponytail: the newest MODELS models in insertion order, not an LRU; enough for a few repos at once.85const MODELS = 16;86const models = new Map<string, AtlasModel>();8788function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {89	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;90	const cached = models.get(key);91	if (cached) return cached;92	const touches = fileTouches(repo.mirror, saveId);93	const foldersTouches = folderTouches(touches);94	const root: AtlasFolder = { name: "", path: "", folders: [], files: [], saves: [] };95	// Koh's own rules file is bookkeeping, not part of the project; Koh never saves what it names.96	const entries = filesAt(repo.mirror, saveId).filter((entry) => entry.path !== ".kohignore");97	for (const entry of entries) {98		const parts = entry.path.split("/");99		let folder = root;100		for (const name of parts.slice(0, -1)) {101			let child = folder.folders.find((f) => f.name === name);102			if (!child) {103				const path = folder.path ? `${folder.path}/${name}` : name;104				child = { name, path, folders: [], files: [], saves: foldersTouches.get(path) ?? [] };105				folder.folders.push(child);106			}107			folder = child;108		}109		const name = parts[parts.length - 1] as string;110		folder.files.push({111			name,112			path: entry.path,113			size: entry.size,114			...factsOf(repo.mirror, entry.hash, name),115			saves: touches.byPath.get(entry.path) ?? [],116		});117	}118	const model = { root, links: linksOf(allFiles(root)), count: entries.length };119	if (models.size >= MODELS) models.delete(models.keys().next().value as string);120	models.set(key, model);121	return model;122}123124/**125 * §21.5's Atlas at one save: the folder tree as `<details>`, so it opens and links without126 * JavaScript. A row's detail panel is fetched from `atlasPanel` on hover or select (kepr.js), so127 * a big repo's page carries rows only. `HEAD` (or the save shown) opens Kepr 2's picker of saves128 * to view it at instead.129 */130export function atlasSection(131	repo: PublicRepo,132	saveId: string,133	at: string | null,134	saves: Save[],135): Html {136	const model = atlasModel(repo, saveId);137	const view = { repo: repo.settings.name, at, links: model.links };138	return html`<section class="repo-section" id="atlas">139<details class="repo-section-toggle" open>140<summary class="section-head"><span class="section-label">atlas</span><span class="section-count">${plural(141		model.count,142		"file",143	)}</span></summary>144<div class="atlas-bar">145${picker(view, saves)}146${at && html`<a class="atlas-head-link mono" href="/${view.repo}#atlas">back to HEAD</a>`}147<button type="button" class="atlas-fold-btn mono" hidden>expand all</button>148<button type="button" class="atlas-search-btn mono" hidden>search</button>149<input class="atlas-filter mono" type="search" placeholder="filter by filename" aria-label="Filter files by name" hidden>150</div>151<div class="atlas" data-repo="${view.repo}"${at && html` data-at="${at}"`}>152<div class="atlas-tree">${children(model.root, view)}</div>153<aside class="atlas-panel" hidden><p class="atlas-hint mono">hover or select a row for details</p></aside>154</div>155</details>156</section>`;157}158159/** One row's detail panel: a file's, or a folder's when `path` ends in a slash; null if absent. */160export function atlasPanel(161	repo: PublicRepo,162	saveId: string,163	at: string | null,164	path: string,165): Html | null {166	const model = atlasModel(repo, saveId);167	const view = { repo: repo.settings.name, at, links: model.links };168	const parts = path.replace(/\/$/, "").split("/");169	let folder: AtlasFolder | undefined = model.root;170	for (const name of path.endsWith("/") ? parts : parts.slice(0, -1)) {171		folder = folder?.folders.find((f) => f.name === name);172	}173	if (!folder) return null;174	if (path.endsWith("/")) return folderPanel(folder, view);175	const file = folder.files.find((f) => f.path === path);176	return file ? filePanel(file, view) : null;177}178179// Kepr 2's "explore through time": every save, newest first. The filter is kepr.js's.180function picker(view: View, saves: Save[]): Html {181	return html`<details class="atlas-picker">182<summary class="atlas-ref-tab mono" title="explore through time">${view.at ?? "HEAD"}</summary>183<div class="atlas-picker-popover">184<input class="atlas-picker-filter mono" type="search" placeholder="search" aria-label="Search saves" hidden>185<div class="atlas-picker-list">${saves.toReversed().map((save) => {186		const message = save.message.split("\n")[0] as string;187		return html`<a class="atlas-picker-row${save.id === view.at ? " active" : ""}" href="/${188			view.repo189		}?at=${save.id}#atlas" data-q="${`${save.id} ${message}`.toLowerCase()}"><span class="atlas-picker-id">${save.id}</span><span class="atlas-picker-msg">${message}</span><span class="atlas-picker-age">· ${relativeTime(save.created_at)}</span></a>`;190	})}</div>191</div>192</details>`;193}194195function children(folder: AtlasFolder, view: View): Html[] {196	const byName = (a: { name: string }, b: { name: string }) =>197		a.name.toLowerCase().localeCompare(b.name.toLowerCase()) || a.name.localeCompare(b.name);198	return [199		...folder.folders.sort(byName).map((child) => folderRow(child, view)),200		...folder.files.sort(byName).map((file) => fileRow(file, view)),201	];202}203204function fileRow(file: AtlasFile, view: View): Html {205	const href = `/${view.repo}/files/${pathUrl(file.path)}${view.at ? `?at=${view.at}` : ""}`;206	return html`<a class="atlas-row atlas-file" href="${207		href208	}" data-name="${file.name.toLowerCase()}" data-path="${file.path}">${row(file.name, file.description, markdownBase(view, file.path), file.size, file.saves)}</a>`;209}210211function filePanel(file: AtlasFile, view: View): Html {212	return panel(213		file.name,214		file.path,215		[216			["size", formatSize(file.size)],217			["lines", file.lines === null ? "binary" : file.lines.toLocaleString("en-US")],218			...saveFacts(file.saves, view),219		],220		file.description,221		markdownBase(view, file.path),222		fileMap(file, view),223	);224}225226// Kepr 2's map of the repo, only where a file has something to say: what it offers, what it227// leans on, and what leans on it, tests apart.228function fileMap(file: AtlasFile, view: View): [string, Html[]][] {229	const users = view.links.usedBy.get(file.path) ?? [];230	return [231		["exports", listed((file.outline?.exports ?? []).map((name) => html`${name}`))],232		["imports", targets(view.links.imports.get(file.path) ?? [], view)],233		[234			"used by",235			targets(236				users.filter((path) => !isTestPath(path)),237				view,238			),239		],240		["tested by", targets(users.filter(isTestPath), view)],241	];242}243244function targets(paths: string[], view: View): Html[] {245	return listed(paths.toSorted().map((path) => targetLink(path, view)));246}247248// A file links to its page and a folder to the Atlas with it open; kepr.js jumps to the row.249function targetLink(target: string, view: View): Html {250	const folder = target.endsWith("/");251	const path = folder ? target.slice(0, -1) : target;252	const query = new URLSearchParams(view.at ? { at: view.at } : {});253	if (folder) query.set("open", path);254	const href = folder255		? `/${view.repo}?${query}#atlas`256		: `/${view.repo}/files/${pathUrl(path)}${view.at ? `?${query}` : ""}`;257	const name = `${path.split("/").pop()}${folder ? "/" : ""}`;258	return html`<a class="atlas-jump" href="${href}" data-path="${target}" title="${path}">${name}</a>`;259}260261function listed(items: Html[]): Html[] {262	const more = items.length - LISTED;263	return more > 0 ? [...items.slice(0, LISTED), html`+${more} more`] : items;264}265266function linksOf(files: AtlasFile[]): Links {267	const index = repoIndex(files.map((file) => file.path));268	const imports = new Map<string, string[]>();269	const usedBy = new Map<string, string[]>();270	for (const file of files) {271		const found = resolveImports(file.path, file.outline?.imports ?? [], index);272		imports.set(file.path, found);273		for (const target of found) usedBy.set(target, [...(usedBy.get(target) ?? []), file.path]);274	}275	return { imports, usedBy };276}277278function folderRow(folder: AtlasFolder, view: View): Html {279	const files = allFiles(folder);280	const size = files.reduce((sum, f) => sum + f.size, 0);281	return html`<details class="atlas-folder" data-path="${folder.path}">282<summary class="atlas-row">${row(283		`${folder.name}/`,284		describeFolder(folder.files),285		markdownBase(view, `${folder.path}/`),286		size,287		folder.saves,288	)}</summary>289<div class="atlas-children">${children(folder, view)}</div>290</details>`;291}292293function folderPanel(folder: AtlasFolder, view: View): Html {294	const files = allFiles(folder);295	const size = files.reduce((sum, f) => sum + f.size, 0);296	const lines = files.reduce((sum, f) => sum + (f.lines ?? 0), 0);297	return panel(298		`${folder.name}/`,299		folder.path,300		[301			["size", formatSize(size)],302			["lines", lines.toLocaleString("en-US")],303			["files", files.length.toLocaleString("en-US")],304			...saveFacts(folder.saves, view),305		],306		describeFolder(folder.files),307		markdownBase(view, `${folder.path}/`),308		[["used by", targets(folderUsers(folder, files, view), view)]],309	);310}311312function row(313	name: string,314	description: string | null,315	base: MarkdownBase,316	size: number,317	saves: Save[],318): Html {319	const last = saves[saves.length - 1];320	const meta = [formatSize(size), last && relativeTime(last.created_at)]321		.filter(Boolean)322		.join(" · ");323	return html`<span class="atlas-name">${name}</span>${html`<span class="atlas-desc">${324		description && renderLine(firstSentence(description), base)325	}</span>`}${html`<span class="atlas-meta mono">${meta}</span>`}`;326}327328// Files outside a folder that import it as a module, or anything inside it.329function folderUsers(folder: AtlasFolder, files: AtlasFile[], view: View): string[] {330	const inside = (path: string) => path.startsWith(`${folder.path}/`);331	const users = new Set(view.links.usedBy.get(`${folder.path}/`) ?? []);332	for (const file of files) {333		for (const user of view.links.usedBy.get(file.path) ?? []) if (!inside(user)) users.add(user);334	}335	return [...users];336}337338function panel(339	name: string,340	path: string,341	facts: [string, Html | string][],342	description: string | null,343	base: MarkdownBase,344	map: [string, Html[]][],345): Html {346	const shown = map.filter(([, items]) => items.length > 0);347	return html`<h3 class="atlas-info-name">${name}</h3>348<p class="atlas-info-path mono">${path}</p>349<dl class="atlas-facts mono">${facts.map(350		([term, value]) => html`<dt>${term}</dt><dd>${value}</dd>`,351	)}</dl>352${353	description &&354	html`<div class="markdown atlas-info-desc">${renderMarkdown(description, base)}</div>`355}356${357	shown.length > 0 &&358	html`<dl class="atlas-facts atlas-map mono">${shown.map(359		([term, items]) =>360			html`<dt>${term}</dt><dd>${items.map((item, i) => html`${i > 0 && " · "}${item}`)}</dd>`,361	)}</dl>`362}`;363}364365function saveFacts(saves: Save[], view: View): [string, Html | string][] {366	const [first, last] = [saves[0], saves[saves.length - 1]];367	if (!first || !last) return [];368	return [369		["first save", saveLink(first, view)],370		["last save", saveLink(last, view)],371		["saves", saves.length.toLocaleString("en-US")],372	];373}374375function saveLink(save: Save, view: View): Html {376	return html`<a href="/${377		view.repo378	}/saves/${save.id}">${save.id}</a> ${formatDate(save.created_at)}`;379}380381// Each folder's saves are the line's saves that touched any path under it, in line order.382function folderTouches(touches: Touches): Map<string, Save[]> {383	const ids = new Map<string, Set<string>>();384	for (const [path, saves] of touches.byPath) {385		const parts = path.split("/");386		for (let depth = 1; depth < parts.length; depth++) {387			const folder = parts.slice(0, depth).join("/");388			const set = ids.get(folder) ?? new Set<string>();389			for (const save of saves) set.add(save.id);390			ids.set(folder, set);391		}392	}393	return new Map(394		[...ids].map(([folder, set]) => [folder, touches.line.filter((save) => set.has(save.id))]),395	);396}397398function allFiles(folder: AtlasFolder): AtlasFile[] {399	return [...folder.files, ...folder.folders.flatMap(allFiles)];400}401402// Relative links in a description resolve from the file's own folder, as in a README.403function markdownBase(view: View, path: string): MarkdownBase {404	return { repo: view.repo, dir: path.split("/").slice(0, -1).join("/"), at: view.at };405}406407function countLines(text: string): number {408	if (text === "") return 0;409	let count = 0;410	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;411	return text.endsWith("\n") ? count : count + 1;412}