describe.ts

1// §21.5: a description is the file's own words, found by syntax alone; no comment means none.2// Every function returns Markdown source (hard-wrapped lines kept) or null.34import MarkdownIt from "markdown-it";5import { type Outline, outline } from "./links.ts";67/** A file page shows a file's first megabyte, and that megabyte decides whether it is text. */8export const SHOWN_BYTES = 1024 * 1024;910// What a file says about itself is worked out only up to this size; a larger file is data.11const MAX_DESCRIBED = 8 * 1024 * 1024;1213/** What a file says about itself, as the Atlas shows it; all null for binary or very large files. */14export type FileFacts = {15	lines: number | null;16	description: string | null;17	outline: Outline | null;18};1920/** `read` is called only for a file small enough to describe, so a large one is never inflated. */21export function fileFacts(path: string, size: number, read: () => Uint8Array): FileFacts {22	const bytes = size > MAX_DESCRIBED ? null : read();23	if (!bytes || !isText(bytes)) return { lines: null, description: null, outline: null };24	const text = new TextDecoder().decode(bytes);25	return {26		lines: countLines(text),27		description: describeFile(path, text),28		outline: outline(path, text),29	};30}3132export function isText(bytes: Uint8Array): boolean {33	if (bytes.subarray(0, 8000).includes(0)) return false;34	try {35		// Streaming mode tolerates only a character cut at the 1 MB mark; invalid bytes still throw.36		const cut = bytes.length > SHOWN_BYTES;37		new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, SHOWN_BYTES), {38			stream: cut,39		});40		return true;41	} catch {42		return false;43	}44}4546function countLines(text: string): number {47	if (text === "") return 0;48	let count = 0;49	for (let i = text.indexOf("\n"); i >= 0; i = text.indexOf("\n", i + 1)) count++;50	return text.endsWith("\n") ? count : count + 1;51}5253type Style = "bang" | "slash" | "jvm" | "hash" | "python" | "markdown";5455const STYLES: Record<string, Style> = {56	zig: "bang",57	rs: "bang",58	py: "python",59	md: "markdown",60	markdown: "markdown",61};62for (const ext of "v go c h cc cpp cxx hpp hh js mjs cjs jsx ts tsx mts cts".split(" ")) {63	STYLES[ext] = "slash";64}65for (const ext of "kt kts java swift".split(" ")) STYLES[ext] = "jvm";66for (const ext of "sh bash zsh nix toml yaml yml".split(" ")) STYLES[ext] = "hash";6768const SHELL_SHEBANG = /^#!.*\b(ba|z|da|k)?sh\b/;69const MODULE_FILES = ["mod.zig", "mod.rs", "index.ts", "__init__.py"];70const README = /^readme\.(md|markdown)$/i;7172/**73 * A license header is the first comment block when it has a license header's shape: an SPDX tag,74 * "Copyright (c)", "Copyright ©", "Copyright <year>", "© <year>", "licensed under" or "all rights75 * reserved". Only that one block is skipped, and the word "copyright" alone is not enough, so76 * `// Renders copyright notices.` keeps its description.77 */78const LICENSE = new RegExp(79	String.raw`SPDX-License-Identifier|\bcopyright\s*(\(c\)|©|\d{4})|©\s*\d{4}|` +80		String.raw`\blicensed under\b|\ball rights reserved\b`,81	"i",82);83/**84 * Kotlin, Java and Swift put a file's words on its main declaration, after `package` and85 * `import`; those lines only name the file's place, so they are passed over, not counted as code.86 */87const PREAMBLE = /^(package|import)\s|^@file:/;88// Go build constraints are comments that describe nothing.89const DIRECTIVE = /^\/\/(go:|\s*\+build\b)/;9091// Parsing only: with HTML on, a leading `<!-- -->` or `<p align>` block is not taken for a paragraph.92const markdown = new MarkdownIt({ html: true });9394/** The description of one file from its text, by the kind its name says it is. */95export function describeFile(path: string, text: string): string | null {96	const name = path.split("/").pop() ?? path;97	const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/);98	const ext = name.includes(".") ? (name.split(".").pop() as string).toLowerCase() : "";99	const style = STYLES[ext] ?? (ext === "" && SHELL_SHEBANG.test(lines[0] ?? "") ? "hash" : null);100	if (style === "bang") return bangLines(lines);101	if (style === "slash") return slashBlock(lines);102	if (style === "jvm") return slashBlock(lines.filter((line) => !PREAMBLE.test(line.trim())));103	if (style === "hash") return hashBlock(lines);104	if (style === "python") return docstring(lines);105	if (style === "markdown") return firstParagraph(text);106	return null;107}108109/** A folder's description: its README's, else its module file's, given each child's own. */110export function describeFolder(111	children: { name: string; description: string | null }[],112): string | null {113	const readme = children.find((child) => README.test(child.name))?.description;114	const moduleFile = MODULE_FILES.map((name) => children.find((c) => c.name === name)).find(115		Boolean,116	);117	return readme ?? moduleFile?.description ?? null;118}119120/** The first sentence of the first paragraph, with hard-wrapped lines joined: the row's one-liner. */121export function firstSentence(description: string): string {122	const paragraph = (description.trim().split(/\n\s*\n/)[0] as string).replace(/\s*\n\s*/g, " ");123	let inCode = false;124	for (let i = 0; i < paragraph.length; i++) {125		const c = paragraph[i] as string;126		if (c === "`") inCode = !inCode;127		const ends = i + 1 === paragraph.length || /\s/.test(paragraph[i + 1] as string);128		if (!inCode && ".!?".includes(c) && ends && !ABBREVIATION.test(paragraph.slice(0, i + 1))) {129			return paragraph.slice(0, i + 1);130		}131	}132	return paragraph;133}134135const ABBREVIATION = /\b(e\.g|i\.e|etc|vs|cf)\.$/i;136137function joined(lines: string[]): string | null {138	const text = lines.join("\n").trim();139	return text === "" ? null : text;140}141142// Zig and Rust: the `//!` lines at the top, after ordinary comments such as a license header.143function bangLines(lines: string[]): string | null {144	const body: string[] = [];145	for (const raw of lines) {146		const line = raw.trim();147		if (line.startsWith("//!")) body.push(line.replace(/^\/\/! ?/, ""));148		else if (body.length > 0 || !(line === "" || line.startsWith("//"))) break;149	}150	return joined(body);151}152153function slashBlock(lines: string[]): string | null {154	const block = commentBlocks(lines, (line) => line.startsWith("//"), true).find(155		(b) => !b.lines.every((line) => DIRECTIVE.test(line)),156	);157	if (!block) return null;158	if (!block.starred) return joined(block.lines.map((line) => line.replace(/^\/\/[/!]? ?/, "")));159	return joined(160		block.lines.map((line) => line.replace(/\s*\*\/$/, "").replace(/^(\/\*[*!]?|\*(?!\/)) ?/, "")),161	);162}163164// Shell, Nix, TOML, YAML: `#` lines at the top, after a shebang and a license header.165function hashBlock(lines: string[]): string | null {166	const rest = lines[0]?.startsWith("#!") ? lines.slice(1) : lines;167	const block = commentBlocks(rest, (line) => line.startsWith("#"))[0];168	return block ? joined(block.lines.map((line) => line.replace(/^#+ ?/, ""))) : null;169}170171type Block = { lines: string[]; starred: boolean };172173/**174 * The comment blocks before the first line of code, license header dropped. A block is a run of175 * line comments, or with `starred` one `/* … *\/` comment; blank lines separate blocks.176 */177function commentBlocks(178	lines: string[],179	isComment: (trimmed: string) => boolean,180	starred = false,181): Block[] {182	const blocks: Block[] = [];183	let i = 0;184	while (i < lines.length) {185		const line = (lines[i] as string).trim();186		if (line === "") {187			i++;188		} else if (starred && line.startsWith("/*")) {189			const start = i;190			while (i < lines.length && !(lines[i] as string).includes("*/")) i++;191			if (i === lines.length) break;192			const body = lines.slice(start, ++i).map((l) => l.trim());193			const last = body.length - 1;194			// Code may follow the `*/` on its closing line.195			body[last] = (body[last] as string).slice(0, (body[last] as string).indexOf("*/") + 2);196			blocks.push({ lines: body, starred: true });197			if (!(lines[i - 1] as string).trim().endsWith("*/")) break;198		} else if (isComment(line)) {199			const body: string[] = [];200			while (i < lines.length && isComment((lines[i] as string).trim())) {201				body.push((lines[i++] as string).trim());202			}203			blocks.push({ lines: body, starred: false });204		} else {205			break;206		}207	}208	if (blocks[0] && LICENSE.test(blocks[0].lines.join("\n"))) blocks.shift();209	return blocks;210}211212// Python: the module docstring, the first statement when it is a string; comments may precede it.213function docstring(lines: string[]): string | null {214	let i = 0;215	while (i < lines.length && /^\s*(#.*)?$/.test(lines[i] as string)) i++;216	const open = /^[rRuU]?("""|'''|"|')/.exec(lines[i] ?? "");217	if (!open) return null;218	const quote = open[1] as string;219	const first = (lines[i] as string).slice(open[0].length);220	const body: string[] = [];221	for (let line = first, n = i; n < lines.length; line = lines[++n] ?? "") {222		const close = line.indexOf(quote);223		if (close >= 0) {224			body.push(line.slice(0, close));225			return joined(dedent(body));226		}227		if (quote.length === 1) return null;228		body.push(line);229	}230	return null;231}232233// Like Python's inspect.cleandoc: continuation lines lose their common indent.234function dedent(lines: string[]): string[] {235	const rest = lines.slice(1).filter((line) => line.trim() !== "");236	const indent = Math.min(...rest.map((line) => (/^\s*/.exec(line) as RegExpExecArray)[0].length));237	return [238		(lines[0] as string).trim(),239		...lines.slice(1).map((line) => line.slice(Number.isFinite(indent) ? indent : 0)),240	];241}242243function firstParagraph(text: string): string | null {244	const tokens = markdown.parse(text, {});245	const map = tokens.find((token) => token.type === "paragraph_open" && token.level === 0)?.map;246	return map ? joined(text.split(/\r?\n/).slice(map[0], map[1])) : null;247}