atlas.ts

1import { describeFile, describeFolder, firstSentence } from "../describe/describe.ts";2import { isTestPath, type Outline, outline, repoIndex, resolveImports } from "../describe/links.ts";3import {4	filesAt,5	fileTouches,6	headId,7	objectBytes,8	type Save,9	type Touches,10	tarballGeneration,11} from "../mirror/queries.ts";12import { isText } from "./files.ts";13import { type Html, html } from "./html.ts";14import {15	formatDate,16	formatSize,17	openPublicRepo,18	type PageContext,19	type PublicRepo,20	pathUrl,21	plural,22	relativeTime,23} from "./layout.ts";24import { type MarkdownBase, renderLine, renderMarkdown } from "./markdown.ts";2526type AtlasFile = {27	name: string;28	path: string;29	size: number;30	lines: number | null;31	description: string | null;32	outline: Outline | null;33	/** The saves that touched it, oldest first. */34	saves: Save[];35};3637type AtlasFolder = {38	name: string;39	path: string;40	folders: AtlasFolder[];41	files: AtlasFile[];42	/** The saves that touched anything under it, removed files included, oldest first. */43	saves: Save[];44};4546/** Each file's resolved imports, and who imports each file or `folder/`. */47type Links = { imports: Map<string, string[]>; usedBy: Map<string, string[]> };4849type View = { repo: string; at: string | null; links: Links };5051type Facts = { lines: number | null; description: string | null; outline: Outline | null };5253// A panel lists this many names or links, then says how many more.54const LISTED = 12;5556// Measured: Koh 3's vendored 9 MB sqlite3.c cost every Atlas view ~25 ms to inflate and scan. An57// object never changes, so what it says about itself is kept for the life of the process, keyed by58// hash and file name (the name picks the comment syntax).59// ponytail: unbounded, one small entry per object viewed; an LRU if a bucket reaches millions.60const known = new Map<string, Facts>();6162function factsOf(db: PublicRepo["mirror"], hash: string, name: string): Facts {63	const key = `${hash} ${name}`;64	const cached = known.get(key);65	if (cached) return cached;66	const bytes = objectBytes(db, hash);67	const text = bytes && isText(bytes) ? new TextDecoder().decode(bytes) : null;68	const facts =69		text === null70			? { lines: null, description: null, outline: null }71			: {72					lines: countLines(text),73					description: describeFile(name, text),74					outline: outline(name, text),75				};76	known.set(key, facts);77	return facts;78}7980/** The tree at one save with each file's facts and the links between files. */81type AtlasModel = { root: AtlasFolder; links: Links; count: number };8283// Building a model reads every file's facts and every save that touched each path: about 165 ms84// warm for a 550-file repo on Tylee. Panels are fetched one at a time on hover, so the model is85// kept, keyed by everything that changes what it holds: the save, and the strips and hidden paths86// (`tarballGeneration`).87// ponytail: the newest MODELS models in insertion order, not an LRU; enough for a few repos at once.88const MODELS = 16;89const models = new Map<string, AtlasModel>();9091function atlasModel(repo: PublicRepo, saveId: string): AtlasModel {92	const key = `${repo.settings.name} ${saveId} ${tarballGeneration(repo.mirror)}`;93	const cached = models.get(key);94	if (cached) return cached;95	const touches = fileTouches(repo.mirror, saveId);96	const foldersTouches = folderTouches(touches);97	const root: AtlasFolder = { name: "", path: "", folders: [], files: [], saves: [] };98	// Koh's own rules file is bookkeeping, not part of the project; Koh never saves what it names.99	const entries = filesAt(repo.mirror, saveId).filter((entry) => entry.path !== ".kohignore");100	for (const entry of entries) {101		const parts = entry.path.split("/");102		let folder = root;103		for (const name of parts.slice(0, -1)) {104			let child = folder.folders.find((f) => f.name === name);105			if (!child) {106				const path = folder.path ? `${folder.path}/${name}` : name;107				child = { name, path, folders: [], files: [], saves: foldersTouches.get(path) ?? [] };108				folder.folders.push(child);109			}110			folder = child;111		}112		const name = parts[parts.length - 1] as string;113		folder.files.push({114			name,115			path: entry.path,116			size: entry.size,117			...factsOf(repo.mirror, entry.hash, name),118			saves: touches.byPath.get(entry.path) ?? [],119		});120	}121	const model = { root, links: linksOf(allFiles(root)), count: entries.length };122	if (models.size >= MODELS) models.delete(models.keys().next().value as string);123	models.set(key, model);124	return model;125}126127/** Builds a public repo's Atlas at its head ahead of the first visitor, as the sync calls it. */128export function warmAtlas(ctx: Pick<PageContext, "kepr" | "mirrorDir">, name: string): void {129	const repo = openPublicRepo(ctx, name);130	const head = repo && headId(repo.mirror);131	if (repo && head) atlasModel(repo, head);132}133134/**135 * §21.5's Atlas at one save: the folder tree as `<details>`, so it opens and links without136 * JavaScript. A row's detail panel is fetched from `atlasPanel` on hover or select (kepr.js), so137 * a big repo's page carries rows only. `HEAD` (or the save shown) opens Kepr 2's picker of saves138 * to view it at instead, its rows fetched from `atlasPickerRows` when first opened.139 */140export function atlasSection(repo: PublicRepo, saveId: string, at: string | null): Html {141	const model = atlasModel(repo, saveId);142	const view = { repo: repo.settings.name, at, links: model.links };143	return html`<section class="repo-section" id="atlas">144<details class="repo-section-toggle" open>145<summary class="section-head"><span class="section-label">atlas</span><span class="section-count">${plural(146		model.count,147		"file",148	)}</span></summary>149<div class="atlas-bar">150${picker(view)}151${at && html`<a class="atlas-head-link mono" href="/${view.repo}#atlas">back to HEAD</a>`}152<button type="button" class="atlas-fold-btn mono" hidden>expand all</button>153<button type="button" class="atlas-search-btn mono" hidden>search</button>154<input class="atlas-filter mono" type="search" placeholder="filter by filename" aria-label="Filter files by name" hidden>155</div>156<div class="atlas" data-repo="${view.repo}"${at && html` data-at="${at}"`}>157<div class="atlas-tree">${children(model.root, view)}</div>158<aside class="atlas-panel" hidden><p class="atlas-hint mono">hover or select a row for details</p></aside>159</div>160</details>161</section>`;162}163164/** One row's detail panel: a file's, or a folder's when `path` ends in a slash; null if absent. */165export function atlasPanel(166	repo: PublicRepo,167	saveId: string,168	at: string | null,169	path: string,170): Html | null {171	const model = atlasModel(repo, saveId);172	const view = { repo: repo.settings.name, at, links: model.links };173	const parts = path.replace(/\/$/, "").split("/");174	let folder: AtlasFolder | undefined = model.root;175	for (const name of path.endsWith("/") ? parts : parts.slice(0, -1)) {176		folder = folder?.folders.find((f) => f.name === name);177	}178	if (!folder) return null;179	if (path.endsWith("/")) return folderPanel(folder, view);180	const file = folder.files.find((f) => f.path === path);181	return file ? filePanel(file, view) : null;182}183184// Kepr 2's "explore through time". Its rows arrive when first opened (kepr.js): a big repo's185// every save was most of its page. Without JavaScript, History's save pages link to the Atlas.186function picker(view: View): Html {187	const query = view.at ? `?at=${view.at}` : "";188	return html`<details class="atlas-picker">189<summary class="atlas-ref-tab mono" title="explore through time">${view.at ?? "HEAD"}</summary>190<div class="atlas-picker-popover">191<input class="atlas-picker-filter mono" type="search" placeholder="search" aria-label="Search saves" hidden>192<div class="atlas-picker-list" data-src="/${view.repo}/atlas/saves${query}"><a class="atlas-picker-row" href="#history">pick a save from the history</a></div>193</div>194</details>`;195}196197/** The picker's rows: every save, newest first, `at` marked. kepr.js filters on their text. */198export function atlasPickerRows(repo: string, saves: Save[], at: string | null): Html[] {199	return saves.toReversed().map((save) => {200		const message = save.message.split("\n")[0] as string;201		return html`<a class="atlas-picker-row${save.id === at ? " active" : ""}" href="/${202			repo203		}?at=${save.id}#atlas"><span class="atlas-picker-id">${save.id}</span><span class="atlas-picker-msg">${message}</span><span class="atlas-picker-age">· ${relativeTime(save.created_at)}</span></a>`;204	});205}206207function children(folder: AtlasFolder, view: View): Html[] {208	const byName = (a: { name: string }, b: { name: string }) =>209		a.name.toLowerCase().localeCompare(b.name.toLowerCase()) || a.name.localeCompare(b.name);210	return [211		...folder.folders.sort(byName).map((child) => folderRow(child, view)),212		...folder.files.sort(byName).map((file) => fileRow(file, view)),213	];214}215216function fileRow(file: AtlasFile, view: View): Html {217	const href = `/${view.repo}/files/${pathUrl(file.path)}${view.at ? `?at=${view.at}` : ""}`;218	return html`<a class="atlas-row atlas-file" href="${219		href220	}" data-name="${file.name.toLowerCase()}" data-path="${file.path}">${row(file.name, file.description, markdownBase(view, file.path), file.size, file.saves)}</a>`;221}222223function filePanel(file: AtlasFile, view: View): Html {224	return panel(225		file.name,226		file.path,227		[228			["size", formatSize(file.size)],229			["lines", file.lines === null ? "binary" : file.lines.toLocaleString("en-US")],230			...saveFacts(file.saves, view),231		],232		file.description,233		markdownBase(view, file.path),234		fileMap(file, view),235	);236}237238// Kepr 2's map of the repo, only where a file has something to say: what it offers, what it239// leans on, and what leans on it, tests apart.240function fileMap(file: AtlasFile, view: View): [string, Html[]][] {241	const users = view.links.usedBy.get(file.path) ?? [];242	return [243		["exports", listed((file.outline?.exports ?? []).map((name) => html`${name}`))],244		["imports", targets(view.links.imports.get(file.path) ?? [], view)],245		[246			"used by",247			targets(248				users.filter((path) => !isTestPath(path)),249				view,250			),251		],252		["tested by", targets(users.filter(isTestPath), view)],253	];254}255256function targets(paths: string[], view: View): Html[] {257	return listed(paths.toSorted().map((path) => targetLink(path, view)));258}259260// A file links to its page and a folder to the Atlas with it open; kepr.js jumps to the row.261function targetLink(target: string, view: View): Html {262	const folder = target.endsWith("/");263	const path = folder ? target.slice(0, -1) : target;264	const query = new URLSearchParams(view.at ? { at: view.at } : {});265	if (folder) query.set("open", path);266	const href = folder267		? `/${view.repo}?${query}#atlas`268		: `/${view.repo}/files/${pathUrl(path)}${view.at ? `?${query}` : ""}`;269	const name = `${path.split("/").pop()}${folder ? "/" : ""}`;270	return html`<a class="atlas-jump" href="${href}" data-path="${target}" title="${path}">${name}</a>`;271}272273function listed(items: Html[]): Html[] {274	const more = items.length - LISTED;275	return more > 0 ? [...items.slice(0, LISTED), html`+${more} more`] : items;276}277278function linksOf(files: AtlasFile[]): Links {279	const index = repoIndex(files.map((file) => file.path));280	const imports = new Map<string, string[]>();281	const usedBy = new Map<string, string[]>();282	for (const file of files) {283		const found = resolveImports(file.path, file.outline?.imports ?? [], index);284		imports.set(file.path, found);285		for (const target of found) usedBy.set(target, [...(usedBy.get(target) ?? []), file.path]);286	}287	return { imports, usedBy };288}289290function folderRow(folder: AtlasFolder, view: View): Html {291	const files = allFiles(folder);292	const size = files.reduce((sum, f) => sum + f.size, 0);293	return html`<details class="atlas-folder" data-path="${folder.path}">294<summary class="atlas-row">${row(295		`${folder.name}/`,296		describeFolder(folder.files),297		markdownBase(view, `${folder.path}/`),298		size,299		folder.saves,300	)}</summary>301<div class="atlas-children">${children(folder, view)}</div>302</details>`;303}304305function folderPanel(folder: AtlasFolder, view: View): Html {306	const files = allFiles(folder);307	const size = files.reduce((sum, f) => sum + f.size, 0);308	const lines = files.reduce((sum, f) => sum + (f.lines ?? 0), 0);309	return panel(310		`${folder.name}/`,311		folder.path,312		[313			["size", formatSize(size)],314			["lines", lines.toLocaleString("en-US")],315			["files", files.length.toLocaleString("en-US")],316			...saveFacts(folder.saves, view),317		],318		describeFolder(folder.files),319		markdownBase(view, `${folder.path}/`),320		[["used by", targets(folderUsers(folder, files, view), view)]],321	);322}323324function row(325	name: string,326	description: string | null,327	base: MarkdownBase,328	size: number,329	saves: Save[],330): Html {331	const last = saves[saves.length - 1];332	const meta = [formatSize(size), last && relativeTime(last.created_at)]333		.filter(Boolean)334		.join(" · ");335	return html`<span class="atlas-name">${name}</span>${html`<span class="atlas-desc">${336		description && renderLine(firstSentence(description), base)337	}</span>`}${html`<span class="atlas-meta mono">${meta}</span>`}`;338}339340// Files outside a folder that import it as a module, or anything inside it.341function folderUsers(folder: AtlasFolder, files: AtlasFile[], view: View): string[] {342	const inside = (path: string) => path.startsWith(`${folder.path}/`);343	const users = new Set(view.links.usedBy.get(`${folder.path}/`) ?? []);344	for (const file of files) {345		for (const user of view.links.usedBy.get(file.path) ?? []) if (!inside(user)) users.add(user);346	}347	return [...users];348}349350function panel(351	name: string,352	path: string,353	facts: [string, Html | string][],354	description: string | null,355	base: MarkdownBase,356	map: [string, Html[]][],357): Html {358	const shown = map.filter(([, items]) => items.length > 0);359	return html`<h3 class="atlas-info-name">${name}</h3>360<p class="atlas-info-path mono">${path}</p>361<dl class="atlas-facts mono">${facts.map(362		([term, value]) => html`<dt>${term}</dt><dd>${value}</dd>`,363	)}</dl>364${365	description &&366	html`<div class="markdown atlas-info-desc">${renderMarkdown(description, base)}</div>`367}368${369	shown.length > 0 &&370	html`<dl class="atlas-facts atlas-map mono">${shown.map(371		([term, items]) =>372			html`<dt>${term}</dt><dd>${items.map((item, i) => html`${i > 0 && " · "}${item}`)}</dd>`,373	)}</dl>`374}`;375}376377function saveFacts(saves: Save[], view: View): [string, Html | string][] {378	const [first, last] = [saves[0], saves[saves.length - 1]];379	if (!first || !last) return [];380	return [381		["first save", saveLink(first, view)],382		["last save", saveLink(last, view)],383		["saves", saves.length.toLocaleString("en-US")],384	];385}386387function saveLink(save: Save, view: View): Html {388	return html`<a href="/${389		view.repo390	}/saves/${save.id}">${save.id}</a> ${formatDate(save.created_at)}`;391}392393// Each folder's saves are the line's saves that touched any path under it, in line order.394function folderTouches(touches: Touches): Map<string, Save[]> {395	const ids = new Map<string, Set<string>>();396	for (const [path, saves] of touches.byPath) {397		const parts = path.split("/");398		for (let depth = 1; depth < parts.length; depth++) {399			const folder = parts.slice(0, depth).join("/");400			const set = ids.get(folder) ?? new Set<string>();401			for (const save of saves) set.add(save.id);402			ids.set(folder, set);403		}404	}405	return new Map(406		[...ids].map(([folder, set]) => [folder, touches.line.filter((save) => set.has(save.id))]),407	);408}409410function allFiles(folder: AtlasFolder): AtlasFile[] {411	return [...folder.files, ...folder.folders.flatMap(allFiles)];412}413414// Relative links in a description resolve from the file's own folder, as in a README.415function markdownBase(view: View, path: string): MarkdownBase {416	return { repo: view.repo, dir: path.split("/").slice(0, -1).join("/"), at: view.at };417}418419function countLines(text: string): number {420	if (text === "") return 0;421	let count = 0;422	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;423	return text.endsWith("\n") ? count : count + 1;424}