add descriptions from comments per §21.5, with license-header and shebang skips and first-sentence one-liners (v3.0.9)

@@ -0,0 +1,193 @@1+import MarkdownIt from "markdown-it";2+3+// §21.5: a description is the file's own words, found by syntax alone; no comment means none.4+// Every function returns Markdown source (hard-wrapped lines kept) or null.5+6+type Style = "bang" | "slash" | "jvm" | "hash" | "python" | "markdown";7+8+const STYLES: Record<string, Style> = {9+	zig: "bang",10+	rs: "bang",11+	py: "python",12+	md: "markdown",13+	markdown: "markdown",14+};15+for (const ext of "v go c h cc cpp cxx hpp hh js mjs cjs jsx ts tsx mts cts".split(" ")) {16+	STYLES[ext] = "slash";17+}18+for (const ext of "kt kts java swift".split(" ")) STYLES[ext] = "jvm";19+for (const ext of "sh bash zsh nix toml yaml yml".split(" ")) STYLES[ext] = "hash";20+21+const SHELL_SHEBANG = /^#!.*\b(ba|z|da|k)?sh\b/;22+const MODULE_FILES = ["mod.zig", "mod.rs", "index.ts", "__init__.py"];23+const README = /^readme(\.md|\.markdown)?$/i;24+25+/**26+ * A license header is the first comment block when it carries an SPDX tag, a copyright line or27+ * "licensed under"/"all rights reserved". Only that one block is skipped, so a description that28+ * merely mentions a license elsewhere is kept.29+ */30+const LICENSE =31+	/SPDX-License-Identifier|\bcopyright\b|©|\blicensed under\b|\ball rights reserved\b/i;32+/**33+ * Kotlin, Java and Swift put a file's words on its main declaration, after `package` and34+ * `import`; those lines only name the file's place, so they are passed over, not counted as code.35+ */36+const PREAMBLE = /^(package|import)\s|^@file:/;37+// Go build constraints are comments that describe nothing.38+const DIRECTIVE = /^\/\/(go:|\s*\+build\b)/;39+40+// Parsing only: with HTML on, a leading `<!-- -->` or `<p align>` block is not taken for a paragraph.41+const markdown = new MarkdownIt({ html: true });42+43+/** The description of one file from its text, by the kind its name says it is. */44+export function describeFile(path: string, text: string): string | null {45+	const name = path.split("/").pop() ?? path;46+	const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/);47+	const ext = name.includes(".") ? (name.split(".").pop() as string).toLowerCase() : "";48+	const style = STYLES[ext] ?? (ext === "" && SHELL_SHEBANG.test(lines[0] ?? "") ? "hash" : null);49+	if (style === "bang") return bangLines(lines);50+	if (style === "slash") return slashBlock(lines);51+	if (style === "jvm") return slashBlock(lines.filter((line) => !PREAMBLE.test(line.trim())));52+	if (style === "hash") return hashBlock(lines);53+	if (style === "python") return docstring(lines);54+	if (style === "markdown") return firstParagraph(text);55+	return null;56+}57+58+/** A folder's description: its README's first paragraph, else its module file's header. */59+export function describeFolder(children: { name: string; text: string }[]): string | null {60+	const readme = children.find((child) => README.test(child.name));61+	if (readme) return firstParagraph(readme.text);62+	for (const moduleFile of MODULE_FILES) {63+		const child = children.find((c) => c.name === moduleFile);64+		if (child) return describeFile(child.name, child.text);65+	}66+	return null;67+}68+69+/** The first sentence of the first paragraph, with hard-wrapped lines joined: the row's one-liner. */70+export function firstSentence(description: string): string {71+	const paragraph = (description.trim().split(/\n\s*\n/)[0] as string).replace(/\s*\n\s*/g, " ");72+	let inCode = false;73+	for (let i = 0; i < paragraph.length; i++) {74+		const c = paragraph[i] as string;75+		if (c === "`") inCode = !inCode;76+		const ends = i + 1 === paragraph.length || /\s/.test(paragraph[i + 1] as string);77+		if (!inCode && ".!?".includes(c) && ends && !ABBREVIATION.test(paragraph.slice(0, i + 1))) {78+			return paragraph.slice(0, i + 1);79+		}80+	}81+	return paragraph;82+}83+84+const ABBREVIATION = /\b(e\.g|i\.e|etc|vs|cf)\.$/i;85+86+function joined(lines: string[]): string | null {87+	const text = lines.join("\n").trim();88+	return text === "" ? null : text;89+}90+91+// Zig and Rust: the `//!` lines at the top, after ordinary comments such as a license header.92+function bangLines(lines: string[]): string | null {93+	const body: string[] = [];94+	for (const raw of lines) {95+		const line = raw.trim();96+		if (line.startsWith("//!")) body.push(line.replace(/^\/\/! ?/, ""));97+		else if (body.length > 0 || !(line === "" || line.startsWith("//"))) break;98+	}99+	return joined(body);100+}101+102+function slashBlock(lines: string[]): string | null {103+	const block = commentBlocks(lines, (line) => line.startsWith("//"), true).find(104+		(b) => !b.lines.every((line) => DIRECTIVE.test(line)),105+	);106+	if (!block) return null;107+	if (!block.starred) return joined(block.lines.map((line) => line.replace(/^\/\/[/!]? ?/, "")));108+	return joined(109+		block.lines.map((line) => line.replace(/\s*\*\/$/, "").replace(/^(\/\*[*!]?|\*(?!\/)) ?/, "")),110+	);111+}112+113+// Shell, Nix, TOML, YAML: `#` lines at the top, after a shebang and a license header.114+function hashBlock(lines: string[]): string | null {115+	const rest = lines[0]?.startsWith("#!") ? lines.slice(1) : lines;116+	const block = commentBlocks(rest, (line) => line.startsWith("#"))[0];117+	return block ? joined(block.lines.map((line) => line.replace(/^#+ ?/, ""))) : null;118+}119+120+type Block = { lines: string[]; starred: boolean };121+122+/**123+ * The comment blocks before the first line of code, license header dropped. A block is a run of124+ * line comments, or with `starred` one `/* … *\/` comment; blank lines separate blocks.125+ */126+function commentBlocks(127+	lines: string[],128+	isComment: (trimmed: string) => boolean,129+	starred = false,130+): Block[] {131+	const blocks: Block[] = [];132+	let i = 0;133+	while (i < lines.length) {134+		const line = (lines[i] as string).trim();135+		if (line === "") {136+			i++;137+		} else if (starred && line.startsWith("/*")) {138+			const start = i;139+			while (i < lines.length && !(lines[i] as string).includes("*/")) i++;140+			if (i === lines.length) break;141+			const body = lines.slice(start, ++i).map((l) => l.trim());142+			blocks.push({ lines: body, starred: true });143+			if (!(lines[i - 1] as string).trim().endsWith("*/")) break;144+		} else if (isComment(line)) {145+			const body: string[] = [];146+			while (i < lines.length && isComment((lines[i] as string).trim())) {147+				body.push((lines[i++] as string).trim());148+			}149+			blocks.push({ lines: body, starred: false });150+		} else {151+			break;152+		}153+	}154+	if (blocks[0] && LICENSE.test(blocks[0].lines.join("\n"))) blocks.shift();155+	return blocks;156+}157+158+// Python: the module docstring, the first statement when it is a string; comments may precede it.159+function docstring(lines: string[]): string | null {160+	let i = 0;161+	while (i < lines.length && /^\s*(#.*)?$/.test(lines[i] as string)) i++;162+	const open = /^[rRuU]?("""|'''|"|')/.exec(lines[i] ?? "");163+	if (!open) return null;164+	const quote = open[1] as string;165+	const first = (lines[i] as string).slice(open[0].length);166+	const body: string[] = [];167+	for (let line = first, n = i; n < lines.length; line = lines[++n] ?? "") {168+		const close = line.indexOf(quote);169+		if (close >= 0) {170+			body.push(line.slice(0, close));171+			return joined(dedent(body));172+		}173+		if (quote.length === 1) return null;174+		body.push(line);175+	}176+	return null;177+}178+179+// Like Python's inspect.cleandoc: continuation lines lose their common indent.180+function dedent(lines: string[]): string[] {181+	const rest = lines.slice(1).filter((line) => line.trim() !== "");182+	const indent = Math.min(...rest.map((line) => (/^\s*/.exec(line) as RegExpExecArray)[0].length));183+	return [184+		(lines[0] as string).trim(),185+		...lines.slice(1).map((line) => line.slice(Number.isFinite(indent) ? indent : 0)),186+	];187+}188+189+function firstParagraph(text: string): string | null {190+	const tokens = markdown.parse(text, {});191+	const map = tokens.find((token) => token.type === "paragraph_open" && token.level === 0)?.map;192+	return map ? joined(text.split(/\r?\n/).slice(map[0], map[1])) : null;193+}

@@ -0,0 +1,159 @@1+import { describe, expect, test } from "bun:test";2+import { describeFile, describeFolder, firstSentence } from "../src/describe/describe.ts";3+4+describe("descriptions by kind (§21.5)", () => {5+	test("Zig and Rust take the leading //! lines, not /// or //", () => {6+		const zig = '//! Tree rendering.\n//!\n//! Second paragraph.\n\nconst std = @import("std");\n';7+		expect(describeFile("src/tree.zig", zig)).toBe("Tree rendering.\n\nSecond paragraph.");8+		const rust = "// Copyright 2026 June\n\n//! Parses things.\nuse std::io;\n";9+		expect(describeFile("src/lib.rs", rust)).toBe("Parses things.");10+		expect(describeFile("a.zig", "// main.zig — entry point\nconst std = 1;\n")).toBeNull();11+		expect(describeFile("a.rs", "/// Doc of a function.\nfn main() {}\n")).toBeNull();12+	});13+14+	test("V, Go, C, C++, Swift, Kotlin, Java, JS and TS take the leading // block", () => {15+		for (const ext of "v go c h cpp swift kt java js ts".split(" ")) {16+			expect(describeFile(`x.${ext}`, "// Reads the bucket.\n// Two lines.\n\ncode();\n")).toBe(17+				"Reads the bucket.\nTwo lines.",18+			);19+		}20+		expect(describeFile("cli.v", "//! Command line parsing.\n\nmodule cli\n")).toBe(21+			"Command line parsing.",22+		);23+	});24+25+	test("the /** */ block counts, with its stars and delimiters removed", () => {26+		const ts = "/**\n * Markup that is `safe`.\n *\n * More.\n */\nexport class Html {}\n";27+		expect(describeFile("html.ts", ts)).toBe("Markup that is `safe`.\n\nMore.");28+		expect(describeFile("x.java", "/** One line. */\nclass X {}\n")).toBe("One line.");29+	});30+31+	test("a comment after the first code does not count", () => {32+		expect(describeFile("x.kt", "val x = 1\n/** Too late. */\nobject X\n")).toBeNull();33+		expect(describeFile("x.go", "package main\n// Too late.\n")).toBeNull();34+		expect(describeFile("x.ts", "import x from 'y';\n// Too late.\n")).toBeNull();35+	});36+37+	test("Kotlin, Java and Swift pass over package and import lines first", () => {38+		const kt =39+			"// SPDX-License-Identifier: MIT\npackage com.peach\n\nimport a.B\n\n/**\n * Idle auto-lock timer.\n */\nclass T\n";40+		expect(describeFile("T.kt", kt)).toBe("Idle auto-lock timer.");41+		expect(describeFile("X.java", "package a;\nimport b.C;\n// Reads C.\nclass X {}\n")).toBe(42+			"Reads C.",43+		);44+		expect(describeFile("X.swift", "import Foundation\n\n/// A view.\nstruct X {}\n")).toBe(45+			"A view.",46+		);47+	});48+49+	test("Go build constraints are not a description", () => {50+		const go = "//go:build linux\n\n// Package tty talks to terminals.\npackage tty\n";51+		expect(describeFile("tty.go", go)).toBe("Package tty talks to terminals.");52+	});53+54+	test("Python takes the module docstring, after comments and a shebang", () => {55+		const py =56+			'#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n"""Sync tool.\n\n    Indented body.\n"""\nimport os\n';57+		expect(describeFile("tool.py", py)).toBe("Sync tool.\n\nIndented body.");58+		expect(describeFile("one.py", "'''One line.'''\n")).toBe("One line.");59+		expect(describeFile("x.py", 'import os\n"""Not a docstring."""\n')).toBeNull();60+		expect(describeFile("x.py", "# Just a comment.\nimport os\n")).toBeNull();61+	});62+63+	test("Shell, Nix, TOML and YAML take the leading # lines after any shebang", () => {64+		for (const ext of "sh nix toml yaml".split(" ")) {65+			expect(describeFile(`x.${ext}`, "# Builds the thing.\n# Quickly.\nvalue = 1\n")).toBe(66+				"Builds the thing.\nQuickly.",67+			);68+		}69+		expect(describeFile("install.sh", "#!/bin/sh\n# Installs kepr.\nset -e\n")).toBe(70+			"Installs kepr.",71+		);72+		expect(describeFile("bin/run", "#!/usr/bin/env bash\n\n# Runs it.\nexec x\n")).toBe("Runs it.");73+		expect(describeFile("install.sh", "#!/bin/sh\nset -e\n# Too late.\n")).toBeNull();74+		expect(describeFile("bin/tool", "#!/usr/bin/env python3\n# Not shell.\n")).toBeNull();75+	});76+77+	test("Markdown takes the first paragraph, not a heading", () => {78+		const md = "# Guide\n\nRead **this** first,\nthen the rest.\n\nMore.\n";79+		expect(describeFile("docs/guide.md", md)).toBe("Read **this** first,\nthen the rest.");80+		expect(describeFile("empty.md", "# Only a heading\n")).toBeNull();81+	});82+83+	test("no comment, an unknown kind or a binary file means no description", () => {84+		expect(describeFile("x.ts", "export const one = 1;\n")).toBeNull();85+		expect(describeFile("x.ts", "")).toBeNull();86+		expect(describeFile("logo.png", "// looks like a comment\n")).toBeNull();87+		expect(describeFile("x.c", "//\n//\nint x;\n")).toBeNull();88+	});89+});90+91+describe("license headers", () => {92+	test("the first block is skipped when it carries SPDX, copyright or licence wording", () => {93+		const headers = [94+			"// SPDX-License-Identifier: MIT\n",95+			"/*\n * Copyright (c) 2026 June\n */\n",96+			"// Licensed under the Apache License, Version 2.0.\n// See LICENSE.\n",97+			"/* © 2026 Asha. All rights reserved. */\n",98+		];99+		for (const header of headers) {100+			expect(describeFile("x.ts", `${header}\n// Real description.\ncode();\n`)).toBe(101+				"Real description.",102+			);103+			expect(describeFile("x.ts", `${header}\ncode();\n`)).toBeNull();104+		}105+		expect(describeFile("x.nix", "# SPDX-License-Identifier: MIT\n\n# The flake.\n{ }\n")).toBe(106+			"The flake.",107+		);108+	});109+110+	test("only the first block, so a description mentioning a licence survives", () => {111+		expect(describeFile("x.ts", "// Checks each repo's license file.\ncode();\n")).toBe(112+			"Checks each repo's license file.",113+		);114+		const two = "// Copyright 2026.\n\n// Copyright notices, rendered.\ncode();\n";115+		expect(describeFile("x.ts", two)).toBe("Copyright notices, rendered.");116+	});117+});118+119+describe("folders", () => {120+	test("a README's first paragraph wins over the module file", () => {121+		const children = [122+			{ name: "mod.zig", text: "//! From the module.\n" },123+			{ name: "README.md", text: "# Pages\n\nFrom the readme.\n" },124+		];125+		expect(describeFolder(children)).toBe("From the readme.");126+	});127+128+	test("else the module file's header, else nothing", () => {129+		expect(describeFolder([{ name: "mod.rs", text: "//! Rust module.\n" }])).toBe("Rust module.");130+		expect(describeFolder([{ name: "index.ts", text: "// TS module.\nx();\n" }])).toBe(131+			"TS module.",132+		);133+		expect(describeFolder([{ name: "__init__.py", text: '"""Py package."""\n' }])).toBe(134+			"Py package.",135+		);136+		expect(describeFolder([{ name: "other.ts", text: "// Not a module file.\n" }])).toBeNull();137+	});138+});139+140+describe("the one-line description", () => {141+	test("joins hard-wrapped lines and stops at the first sentence", () => {142+		expect(firstSentence("Reads the\nbucket. Then more.\n\nNext paragraph.")).toBe(143+			"Reads the bucket.",144+		);145+	});146+147+	test("ignores full stops inside code, versions and common abbreviations", () => {148+		expect(firstSentence("Reads `koh.build`, e.g. v3.0.72 files. More.")).toBe(149+			"Reads `koh.build`, e.g. v3.0.72 files.",150+		);151+		expect(firstSentence("`a. b` is code! Next.")).toBe("`a. b` is code!");152+	});153+154+	test("a paragraph without a full stop is the whole line", () => {155+		expect(firstSentence("Command line entry: dispatch table")).toBe(156+			"Command line entry: dispatch table",157+		);158+	});159+});