From ddb9391fd8314d1f66e1d6faea99b70920c26c06 Mon Sep 17 00:00:00 2001 From: Martin Beckert Date: Fri, 2 Oct 2026 09:31:00 +0200 Subject: [PATCH 1/2] feat(context): report documentation ingestion outcomes (#143) --- .changeset/bright-documents-report.md | 5 + packages/context/README.md | 31 ++ packages/context/src/cli.ts | 94 ++++- .../context/src/fixtures/ingestion/guide.html | 7 + .../context/src/fixtures/ingestion/guide.md | 9 + .../context/src/fixtures/ingestion/guide.rst | 11 + packages/context/src/git.ts | 207 ++++++----- packages/context/src/index.ts | 9 + packages/context/src/ingestion.test.ts | 322 ++++++++++++++++++ packages/context/src/ingestion.ts | 131 +++++++ packages/context/src/package-builder.ts | 93 +++-- packages/registry/src/build.ts | 47 ++- packages/registry/src/cli.ts | 55 ++- packages/registry/src/glob.ts | 12 +- .../registry/src/html-index-build.test.ts | 1 + packages/registry/src/html-index.ts | 56 ++- packages/registry/src/index.ts | 2 + packages/registry/src/ingestion.test.ts | 297 ++++++++++++++++ packages/registry/src/zip.ts | 49 ++- registry/README.md | 29 ++ 20 files changed, 1307 insertions(+), 160 deletions(-) create mode 100644 .changeset/bright-documents-report.md create mode 100644 packages/context/src/fixtures/ingestion/guide.html create mode 100644 packages/context/src/fixtures/ingestion/guide.md create mode 100644 packages/context/src/fixtures/ingestion/guide.rst create mode 100644 packages/context/src/ingestion.test.ts create mode 100644 packages/context/src/ingestion.ts create mode 100644 packages/registry/src/ingestion.test.ts diff --git a/.changeset/bright-documents-report.md b/.changeset/bright-documents-report.md new file mode 100644 index 0000000..e354c66 --- /dev/null +++ b/.changeset/bright-documents-report.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": minor +--- + +Report documentation ingestion outcomes with relative paths and reasons, distinguishing exclusions, duplicate content, read/parse failures, empty documents, and indexed documents. Add JSON reports and strict validation to documentation builds, preserving installed packages when validation rejects unexpected document loss. diff --git a/packages/context/README.md b/packages/context/README.md index b1d8631..61521aa 100644 --- a/packages/context/README.md +++ b/packages/context/README.md @@ -297,6 +297,37 @@ context add ./my-project --name my-lib --pkg-version 2.0 --save ./packages/ context add ./packages/my-lib@2.0.db ``` +Builds print an ingestion summary: discovered documentation files, selected files, +indexed documents and sections, plus exclusions, duplicates, read/parse failures, +and empty documents. To inspect paths and reasons, write a JSON report: + +```bash +context add ./my-project --diagnostics ingestion.json +context add ./my-project --strict --diagnostics ingestion.json +``` + +`--strict` rejects unreadable files/directories, parser failures, and documents +that produce no sections before changing the installed package. Intentional +exclusions and duplicate content remain acceptable. The JSON report is also +written when strict validation fails. These options apply to documentation +builds, not installation of prebuilt `.db` files. + +The report has `schemaVersion`, `summary`, and per-path `entries`. Discovered +files are documentation candidates actually visited within the selected source; +selected files exclude intentional exclusions and duplicates. Indexed sections +count only sections stored after deduplication. Pruned directories appear once +with their exclusion reason; their contents are not scanned or counted. Unrelated +source-code files are omitted. Git/local paths are relative to the repository +root, including an explicit docs path. + +Library callers receive the report as `BuildResult.diagnostics`; the existing +`skippedFiles` field still counts only split/parse failures. To include source +selection and read outcomes, pass the same `IngestionDiagnostic[]` to +`readLocalDocsFiles(..., { diagnostics })` and +`buildPackage(..., { name, version, diagnostics, strict })`. Strict failures throw +`IngestionError`, whose `diagnostics` property contains the report. Library code +does not log diagnostics. + --- ## :whale: Docker diff --git a/packages/context/src/cli.ts b/packages/context/src/cli.ts index 34baa43..1243e5d 100644 --- a/packages/context/src/cli.ts +++ b/packages/context/src/cli.ts @@ -9,6 +9,7 @@ import { renameSync, statSync, unlinkSync, + writeFileSync, } from "node:fs"; import { createRequire } from "node:module"; import { homedir } from "node:os"; @@ -49,6 +50,11 @@ import { NO_DOCUMENTATION_FOUND_MESSAGE, SEARCH_PACKAGES_NAME_DESCRIPTION, } from "./guidance.js"; +import { + formatIngestionSummary, + type IngestionDiagnostic, + IngestionError, +} from "./ingestion.js"; import { fetchLinkedDocs } from "./llms-txt.js"; import { type BuildResult, @@ -353,6 +359,7 @@ async function addFromWebsite( console.log(`Building package...`); const result = buildPackage(outputPath, files, { + strict: options.strict, name: packageName, version: versionLabel, sourceUrl: source, @@ -364,7 +371,13 @@ async function addFromWebsite( ); } - reportBuilt(result, packageName, versionLabel, outputPath); + reportBuilt( + result, + packageName, + versionLabel, + outputPath, + options.diagnostics, + ); if (options.save) { savePackageCopy(outputPath, options.save, packageName, versionLabel); @@ -414,6 +427,7 @@ async function addFromWebsite( console.log(`Building package...`); const result = buildPackage(outputPath, files, { + strict: options.strict, name: packageName, version: versionLabel, sourceUrl: source, @@ -425,7 +439,13 @@ async function addFromWebsite( ); } - reportBuilt(result, packageName, versionLabel, outputPath); + reportBuilt( + result, + packageName, + versionLabel, + outputPath, + options.diagnostics, + ); // Save to custom path if specified if (options.save) { @@ -540,9 +560,16 @@ function reportBuilt( name: string, version: string, outputPath: string, + diagnosticsPath?: string, ): void { console.log(`✓ Built package: ${name}@${version}`); console.log(`✓ Saved to ${outputPath}`); + console.log(formatIngestionSummary(result.diagnostics)); + if (diagnosticsPath) + writeFileSync( + resolve(diagnosticsPath), + `${JSON.stringify(result.diagnostics, null, 2)}\n`, + ); if (result.skippedFiles > 0) { console.log( @@ -719,6 +746,8 @@ export interface AddFromGitOptions { name?: string; save?: string; lang?: string; + strict?: boolean; + diagnostics?: string; } /** @@ -895,17 +924,20 @@ async function addFromGitClone( } // Read all markdown files (filtered by language) + const diagnostics: IngestionDiagnostic[] = []; const files = readLocalDocsFiles(tempDir, { + diagnostics, path: docsPath, lang: options.lang, }); if (files.length === 0) { - throw new Error( - `No markdown files found${docsPath ? ` in ${docsPath}` : ""}. Use --path to specify or --lang all to include all languages.`, + throw new IngestionError( + `No documentation files found${docsPath ? ` in ${docsPath}` : ""}. Use --path to specify or --lang all to include all languages.`, + diagnostics, ); } console.log( - `✓ Found ${files.length} markdown files${options.lang ? ` (lang: ${options.lang})` : ""}`, + `✓ Found ${files.length} documentation files${options.lang ? ` (lang: ${options.lang})` : ""}`, ); // Build the package @@ -917,12 +949,20 @@ async function addFromGitClone( console.log(`Building package...`); const result = buildPackage(outputPath, files, { + strict: options.strict, name: packageName, version: versionLabel, sourceUrl: url, + diagnostics, }); - reportBuilt(result, packageName, versionLabel, outputPath); + reportBuilt( + result, + packageName, + versionLabel, + outputPath, + options.diagnostics, + ); // Save to custom path if specified if (options.save) { @@ -974,17 +1014,20 @@ async function addFromLocalDir( } // Read all markdown files (filtered by language) + const diagnostics: IngestionDiagnostic[] = []; const files = readLocalDocsFiles(dirPath, { + diagnostics, path: docsPath, lang: options.lang, }); if (files.length === 0) { - throw new Error( - `No markdown files found${docsPath ? ` in ${docsPath}` : ""}. Use --path to specify or --lang all to include all languages.`, + throw new IngestionError( + `No documentation files found${docsPath ? ` in ${docsPath}` : ""}. Use --path to specify or --lang all to include all languages.`, + diagnostics, ); } console.log( - `✓ Found ${files.length} markdown files${options.lang ? ` (lang: ${options.lang})` : ""}`, + `✓ Found ${files.length} documentation files${options.lang ? ` (lang: ${options.lang})` : ""}`, ); // Build the package @@ -996,12 +1039,20 @@ async function addFromLocalDir( console.log(`Building package...`); const result = buildPackage(outputPath, files, { + strict: options.strict, name: packageName, version: versionLabel, sourceUrl: dirPath, + diagnostics, }); - reportBuilt(result, packageName, versionLabel, outputPath); + reportBuilt( + result, + packageName, + versionLabel, + outputPath, + options.diagnostics, + ); // Save to custom path if specified if (options.save) { @@ -1029,6 +1080,11 @@ program "", "Package source: local .db file, URL (.db), GitHub URL, git URL, website URL (auto-fetches llms.txt), or local directory", ) + .option("--strict", "Fail on unreadable, unparseable, or empty documentation") + .option( + "--diagnostics ", + "Write detailed ingestion diagnostics as JSON", + ) .option("--tag ", "Git tag to checkout (for git repos)") .option("--pkg-version ", "Custom version label") .option("--path ", "Path to docs folder in repo/directory") @@ -1048,10 +1104,20 @@ program name?: string; save?: string; lang?: string; + strict?: boolean; + diagnostics?: string; }, ) => { try { const sourceType = detectSourceType(source); + if ( + (options.strict || options.diagnostics) && + (sourceType === "file" || sourceType === "url") + ) { + throw new Error( + "Ingestion diagnostics and strict validation require a documentation source, not a prebuilt .db package.", + ); + } // Map pkgVersion to version for internal use const internalOptions = { @@ -1077,6 +1143,14 @@ program break; } } catch (err) { + if (err instanceof IngestionError) { + console.error(formatIngestionSummary(err.diagnostics)); + if (options.diagnostics) + writeFileSync( + resolve(options.diagnostics), + `${JSON.stringify(err.diagnostics, null, 2)}\n`, + ); + } console.error(`Error: ${err instanceof Error ? err.message : err}`); process.exit(1); } diff --git a/packages/context/src/fixtures/ingestion/guide.html b/packages/context/src/fixtures/ingestion/guide.html new file mode 100644 index 0000000..f2b6ee1 --- /dev/null +++ b/packages/context/src/fixtures/ingestion/guide.html @@ -0,0 +1,7 @@ + + +

HTML Guide

+

Configure HTML

+

Configure the HTML service with this example:

+
run-html-example --port 8081
+ diff --git a/packages/context/src/fixtures/ingestion/guide.md b/packages/context/src/fixtures/ingestion/guide.md new file mode 100644 index 0000000..1e9ee4e --- /dev/null +++ b/packages/context/src/fixtures/ingestion/guide.md @@ -0,0 +1,9 @@ +# Markdown Guide + +## Configure Markdown + +Configure the Markdown service with this example: + +```sh +run-markdown-example --port 8080 +``` diff --git a/packages/context/src/fixtures/ingestion/guide.rst b/packages/context/src/fixtures/ingestion/guide.rst new file mode 100644 index 0000000..2c435a2 --- /dev/null +++ b/packages/context/src/fixtures/ingestion/guide.rst @@ -0,0 +1,11 @@ +RST Guide +========= + +Configure RST +------------- + +Configure the RST service with this example: + +.. code-block:: sh + + run-rst-example --port 8082 diff --git a/packages/context/src/git.ts b/packages/context/src/git.ts index df93da1..f5cc1ce 100755 --- a/packages/context/src/git.ts +++ b/packages/context/src/git.ts @@ -12,6 +12,7 @@ import { } from "node:child_process"; import { createHash } from "node:crypto"; import { + type Dirent, existsSync, mkdtempSync, readdirSync, @@ -21,6 +22,11 @@ import { import { tmpdir } from "node:os"; import { join, posix } from "node:path"; import ignore, { type Ignore } from "ignore"; +import { + type IngestionDiagnostic, + IngestionError, + ingestionErrorReason, +} from "./ingestion.js"; /** * Generate a content hash for deduplication. @@ -421,6 +427,8 @@ export interface FindMarkdownOptions { lang?: string; /** True when the scan starts at the repo root rather than inside a docs folder */ atRepoRoot?: boolean; + diagnostics?: IngestionDiagnostic[]; + diagnosticRoot?: string; } /** @@ -436,96 +444,95 @@ function findMarkdownFiles( ): string[] { const files: string[] = []; const lang = options.lang?.toLowerCase(); - + let entries: Dirent[]; try { - const entries = readdirSync(dirPath, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = join(dirPath, entry.name); - // Stored paths always use "/" so packages built on Windows match the rest - const relativePath = basePath - ? posix.join(basePath, entry.name) - : entry.name; - - // Skip hidden entries - if (entry.name.startsWith(".")) continue; - - // Check gitignore (directories need trailing slash for gitignore matching) - const pathToCheck = entry.isDirectory() - ? `${relativePath}/` - : relativePath; - if (ig.ignores(pathToCheck)) continue; - - if (entry.isDirectory()) { - // Skip test, internal, and other non-doc directories - const dirName = entry.name.toLowerCase(); - if ( - IGNORED_DIRS.has(dirName) || - (options.atRepoRoot && REPO_ROOT_IGNORED_DIRS.has(dirName)) - ) { - continue; - } + entries = readdirSync(dirPath, { withFileTypes: true }); + } catch (error) { + options.diagnostics?.push({ + path: posix.join(options.diagnosticRoot ?? "", basePath) || ".", + kind: "directory", + outcome: "read-error", + reason: ingestionErrorReason(error), + }); + return files; + } - // Filter locale directories unless --lang all or specific lang matches - if (isLocaleDir(entry.name)) { - // Include if: all languages, matching lang, or default to English - if ( - lang === "all" || - lang === dirName || - (!lang && dirName === "en") - ) { - files.push( - ...findMarkdownFiles(fullPath, ig, relativePath, options), - ); - } - // Skip other locales by default - } else { - files.push(...findMarkdownFiles(fullPath, ig, relativePath, options)); - } - } else if (entry.isFile()) { - const lowerName = entry.name.toLowerCase(); - const hasDocumentationExtension = DOCUMENTATION_EXTENSIONS.some((ext) => - lowerName.endsWith(ext), - ); + for (const entry of entries) { + const fullPath = join(dirPath, entry.name); + const relativePath = basePath + ? posix.join(basePath, entry.name) + : entry.name; + const matchingExt = DOCUMENTATION_EXTENSIONS.find((ext) => + entry.name.toLowerCase().endsWith(ext), + ); + const exclude = (reason: string) => { + // Do not flood the report with unrelated source-code files. + if (entry.isDirectory() || matchingExt) + options.diagnostics?.push({ + path: posix.join(options.diagnosticRoot ?? "", relativePath), + kind: entry.isDirectory() ? "directory" : "file", + outcome: "excluded", + reason, + }); + }; + if (entry.name.startsWith(".")) { + exclude("hidden"); + continue; + } + const pathToCheck = entry.isDirectory() ? `${relativePath}/` : relativePath; + if (ig.ignores(pathToCheck)) { + exclude("gitignore"); + continue; + } - if (hasDocumentationExtension) { - // Skip non-doc markdown files - // Find matching extension to remove it for checking ignored files - const matchingExt = DOCUMENTATION_EXTENSIONS.find((ext) => - lowerName.endsWith(ext), - ); - // Only at the repo root: these names mean repo housekeeping there, - // but anywhere in a docs tree they are ordinary pages — forgejo's - // docs/admin/actions/security.md documents Actions security, and was - // being dropped as if it were a SECURITY.md policy file. A docs - // folder is not the repo root even though the walk starts there. - if (matchingExt && basePath === "" && options.atRepoRoot) { - const baseName = lowerName.slice(0, -matchingExt.length); - if (IGNORED_FILES.has(baseName)) continue; - } - // Skip test fixture files (e.g., component.expect.md, hook.test.md) - if (matchingExt) { - const nameWithoutExt = lowerName.slice(0, -matchingExt.length); - const nameParts = nameWithoutExt.split("."); - if (nameParts.length > 1) { - const lastPart = nameParts[nameParts.length - 1] || ""; - if (FIXTURE_SUFFIXES.includes(lastPart)) { - continue; - } - } - } - files.push(relativePath); - } + if (entry.isDirectory()) { + const dirName = entry.name.toLowerCase(); + if ( + IGNORED_DIRS.has(dirName) || + (options.atRepoRoot && REPO_ROOT_IGNORED_DIRS.has(dirName)) + ) { + exclude("directory-filter"); + continue; + } + if ( + isLocaleDir(entry.name) && + lang !== "all" && + lang !== dirName && + !(!lang && dirName === "en") + ) { + exclude("language-filter"); + continue; + } + files.push(...findMarkdownFiles(fullPath, ig, relativePath, options)); + } else if (entry.isFile() && matchingExt) { + const baseName = entry.name.toLowerCase().slice(0, -matchingExt.length); + if ( + basePath === "" && + options.atRepoRoot && + IGNORED_FILES.has(baseName) + ) { + exclude("repo-metadata"); + continue; + } + const nameParts = baseName.split("."); + if ( + nameParts.length > 1 && + FIXTURE_SUFFIXES.includes(nameParts[nameParts.length - 1] || "") + ) { + exclude("test-fixture"); + continue; } + files.push(relativePath); + } else if (matchingExt) { + exclude("not-a-regular-file"); } - } catch { - // Directory read failed } - return files; } export interface ReadLocalDocsOptions { + /** Collect exclusions, duplicates and read failures without logging. */ + diagnostics?: IngestionDiagnostic[]; /** Path to docs folder within the repository */ path?: string; /** Language filter: "all" includes everything, specific code (e.g., "en") includes only that locale */ @@ -542,11 +549,21 @@ export function readLocalDocsFiles( basePath: string, options: ReadLocalDocsOptions = {}, ): Array<{ path: string; content: string }> { - const { path: docsPath, lang } = options; + const { path: docsPath, lang, diagnostics } = options; const searchPath = docsPath ? join(basePath, docsPath) : basePath; if (!existsSync(searchPath)) { - throw new Error(`Directory not found: ${searchPath}`); + const entry: IngestionDiagnostic = { + path: docsPath ?? ".", + kind: "directory", + outcome: "read-error", + reason: "ENOENT", + }; + diagnostics?.push(entry); + throw new IngestionError( + `Directory not found: ${searchPath}`, + diagnostics ?? [entry], + ); } // Load gitignore from repo root @@ -555,11 +572,14 @@ export function readLocalDocsFiles( const markdownFiles = findMarkdownFiles(searchPath, ig, "", { lang, atRepoRoot: !docsPath, + diagnostics, + diagnosticRoot: docsPath, }); const files: Array<{ path: string; content: string }> = []; - const seenHashes = new Set(); + const seenHashes = new Map(); for (const filePath of markdownFiles) { + const storagePath = docsPath ? posix.join(docsPath, filePath) : filePath; try { const fullPath = join(searchPath, filePath); const content = readFileSync(fullPath, "utf-8"); @@ -567,15 +587,24 @@ export function readLocalDocsFiles( // Skip duplicate content (keep first occurrence) const hash = contentHash(content); if (seenHashes.has(hash)) { + diagnostics?.push({ + path: storagePath, + kind: "file", + outcome: "duplicate", + reason: "Identical document content", + duplicateOf: seenHashes.get(hash), + }); continue; } - seenHashes.add(hash); - - // Use relative path from docs folder for storage - const storagePath = docsPath ? posix.join(docsPath, filePath) : filePath; + seenHashes.set(hash, storagePath); files.push({ path: storagePath, content }); - } catch { - // Skip files that can't be read + } catch (error) { + diagnostics?.push({ + path: storagePath, + kind: "file", + outcome: "read-error", + reason: ingestionErrorReason(error), + }); } } diff --git a/packages/context/src/index.ts b/packages/context/src/index.ts index d2aa3a9..9471bd5 100644 --- a/packages/context/src/index.ts +++ b/packages/context/src/index.ts @@ -45,6 +45,15 @@ export { readLocalDocsFiles, } from "./git.js"; export { parseHtml } from "./html.js"; +export { + createIngestionReport, + formatIngestionSummary, + type IngestionDiagnostic, + IngestionError, + type IngestionReport, + ingestionErrorReason, + validateIngestion, +} from "./ingestion.js"; export { type BuildResult, buildPackage, diff --git a/packages/context/src/ingestion.test.ts b/packages/context/src/ingestion.test.ts new file mode 100644 index 0000000..f5e4078 --- /dev/null +++ b/packages/context/src/ingestion.test.ts @@ -0,0 +1,322 @@ +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { afterEach, beforeAll, beforeEach, expect, it, vi } from "vitest"; +import { parseDocument } from "./build.js"; +import { initDatabase, openDatabase } from "./database.js"; +import { readLocalDocsFiles } from "./git.js"; +import { type IngestionDiagnostic, IngestionError } from "./ingestion.js"; +import { buildPackage } from "./package-builder.js"; + +vi.mock("node:fs", async (original) => { + const fs = await original(); + return { + ...fs, + readFileSync: vi.fn(fs.readFileSync), + readdirSync: vi.fn(fs.readdirSync), + }; +}); +vi.mock("./build.js", async (original) => { + const build = await original(); + return { ...build, parseDocument: vi.fn(build.parseDocument) }; +}); + +const fixtures = resolve(import.meta.dirname, "fixtures/ingestion"); +const options = { name: "ingestion-fixture", version: "1" }; +let directory: string; +let output: string; + +beforeAll(async () => { + await initDatabase(); +}); +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), "context-ingestion-")); + output = join(directory, "package.db"); + mkdirSync(join(directory, "docs")); +}); +afterEach(() => { + vi.resetAllMocks(); + rmSync(directory, { recursive: true, force: true }); +}); + +it("reports distinct source and parser outcomes while preserving headings and examples", async () => { + const fs = await vi.importActual("node:fs"); + const build = + await vi.importActual("./build.js"); + for (const extension of ["md", "html", "rst"]) { + copyFileSync( + join(fixtures, `guide.${extension}`), + join(directory, "docs", `guide.${extension}`), + ); + } + copyFileSync( + join(fixtures, "guide.md"), + join(directory, "docs", "z-duplicate.md"), + ); + writeFileSync(join(directory, ".gitignore"), "ignored.md\n"); + for (const file of [ + "ignored.md", + ".hidden.md", + "unreadable.md", + "broken.md", + ]) { + writeFileSync( + join(directory, "docs", file), + `## ${file}\n\nUnique documentation for ${file}.`, + ); + } + writeFileSync(join(directory, "docs", "empty.md"), ""); + writeFileSync( + join(directory, "docs", "source.ts"), + "export const unrelated = 1;", + ); + mkdirSync(join(directory, "docs", "de")); + mkdirSync(join(directory, "docs", "unreadable")); + const denied = () => + Object.assign(new Error("Permission denied"), { code: "EACCES" }); + vi.mocked(readFileSync).mockImplementation((path, opts) => { + if (basename(String(path)) === "unreadable.md") throw denied(); + return fs.readFileSync(path, opts); + }); + vi.mocked(readdirSync).mockImplementation((path, opts) => { + if (basename(String(path)) === "unreadable") throw denied(); + return fs.readdirSync(path, opts); + }); + vi.mocked(parseDocument).mockImplementation((content, path) => { + if (path.endsWith("broken.md")) throw new Error("Parser rejected document"); + return build.parseDocument(content, path); + }); + + const diagnostics: IngestionDiagnostic[] = []; + const files = readLocalDocsFiles(directory, { path: "docs", diagnostics }); + const result = buildPackage(output, files, { ...options, diagnostics }); + expect(result.skippedFiles).toBe(1); + expect(result.diagnostics.summary).toEqual({ + discoveredFiles: 9, + selectedFiles: 6, + indexedDocuments: 3, + indexedSections: result.sectionCount, + excludedFiles: 2, + excludedDirectories: 1, + duplicateFiles: 1, + unreadableFiles: 1, + unreadableDirectories: 1, + parseFailures: 1, + emptyFiles: 1, + }); + expect(result.diagnostics.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "docs/ignored.md", + outcome: "excluded", + reason: "gitignore", + }), + expect.objectContaining({ + path: "docs/de", + kind: "directory", + outcome: "excluded", + reason: "language-filter", + }), + expect.objectContaining({ + path: "docs/z-duplicate.md", + outcome: "duplicate", + duplicateOf: "docs/guide.md", + }), + expect.objectContaining({ + path: "docs/unreadable.md", + outcome: "read-error", + reason: "EACCES", + }), + expect.objectContaining({ + path: "docs/unreadable", + kind: "directory", + outcome: "read-error", + }), + expect.objectContaining({ + path: "docs/broken.md", + outcome: "parse-error", + reason: "Parser rejected document", + }), + expect.objectContaining({ path: "docs/empty.md", outcome: "empty" }), + ]), + ); + expect(JSON.stringify(result.diagnostics)).not.toContain(directory); + expect( + result.diagnostics.entries.some((entry) => + entry.path.endsWith("source.ts"), + ), + ).toBe(false); + const db = openDatabase(output, { readonly: true }); + try { + for (const [extension, title, example] of [ + ["md", "Configure Markdown", "run-markdown-example --port 8080"], + ["html", "Configure HTML", "run-html-example --port 8081"], + ["rst", "Configure RST", "run-rst-example --port 8082"], + ] as const) { + expect( + db + .prepare( + "SELECT section_title, content FROM chunks WHERE doc_path = ?", + ) + .all(`docs/guide.${extension}`), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + section_title: title, + content: expect.stringContaining(example), + }), + ]), + ); + } + } finally { + db.close(); + } +}); + +it.each([ + "read-error", + "parse-error", + "empty", +] as const)("strict validation rejects %s without replacing an installed package", (outcome) => { + const valid = { + path: "guide.md", + content: "## Guide\n\nOriginal installed documentation.", + }; + buildPackage(output, [valid], options); + const original = readFileSync(output); + const diagnostics: IngestionDiagnostic[] = + outcome === "read-error" + ? [{ path: "missing.md", kind: "file", outcome, reason: "ENOENT" }] + : []; + const files = + outcome === "read-error" + ? [valid] + : [ + valid, + { + path: "bad.md", + content: + outcome === "empty" ? "" : (undefined as unknown as string), + }, + ]; + expect(() => + buildPackage(output, files, { ...options, diagnostics, strict: true }), + ).toThrow(IngestionError); + expect(readFileSync(output)).toEqual(original); +}); + +it("strict validation rejects unreadable directories even when no files were discovered", () => { + expect(() => + buildPackage(output, [], { + ...options, + strict: true, + diagnostics: [ + { + path: "docs", + kind: "directory", + outcome: "read-error", + reason: "EACCES", + }, + ], + }), + ).toThrow(IngestionError); + expect(existsSync(output)).toBe(false); +}); + +it("allows intentional exclusions and duplicates in strict mode, counting only inserted sections", () => { + const content = + "## Guide\n\nIntentional duplicate content is not document loss."; + const result = buildPackage( + output, + [ + { path: "a.md", content }, + { path: "b.md", content }, + ], + { + ...options, + strict: true, + diagnostics: [ + { + path: "ignored.md", + kind: "file", + outcome: "excluded", + reason: "exclude_paths", + }, + ], + }, + ); + expect(result.diagnostics.summary).toMatchObject({ + discoveredFiles: 3, + selectedFiles: 1, + indexedDocuments: 1, + indexedSections: 1, + excludedFiles: 1, + duplicateFiles: 1, + }); + expect(result.diagnostics.entries).toContainEqual({ + path: "b.md", + kind: "file", + outcome: "duplicate", + reason: "All sections duplicate earlier content", + duplicateOf: "a.md", + }); +}); + +it("writes context add reports on success and strict failure without changing the installed package", () => { + const home = join(directory, "isolated-user"); + const preload = join(directory, "isolated-os.mjs"); + // Override homedir only inside the test child; never touch the user's packages. + writeFileSync( + preload, + `import os from 'node:os'; import { syncBuiltinESMExports } from 'node:module'; os.homedir = () => ${JSON.stringify(home)}; syncBuiltinESMExports();`, + ); + copyFileSync(join(fixtures, "guide.md"), join(directory, "docs", "guide.md")); + writeFileSync(join(directory, "docs", "empty.md"), ""); + const report = join(directory, "report.json"); + const args = [ + "--import", + preload, + resolve(import.meta.dirname, "../dist/cli.js"), + "add", + directory, + "--path", + "docs", + "--name", + "fixture", + "--pkg-version", + "1", + "--diagnostics", + report, + ]; + const normal = spawnSync(process.execPath, args, { encoding: "utf8" }); + expect(normal.status, normal.stderr).toBe(0); + expect(normal.stdout).toContain("Ingestion:"); + expect(JSON.parse(readFileSync(report, "utf8")).summary).toMatchObject({ + discoveredFiles: 2, + indexedDocuments: 1, + emptyFiles: 1, + }); + const installed = join(home, ".context", "packages", "fixture@1.db"); + const original = readFileSync(installed); + const strict = spawnSync(process.execPath, [...args, "--strict"], { + encoding: "utf8", + }); + expect(strict.status).toBe(1); + expect(strict.stderr).toContain("Strict ingestion validation failed"); + expect(readFileSync(installed)).toEqual(original); + expect(JSON.parse(readFileSync(report, "utf8")).entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "docs/empty.md", outcome: "empty" }), + ]), + ); +}, 15_000); diff --git a/packages/context/src/ingestion.ts b/packages/context/src/ingestion.ts new file mode 100644 index 0000000..8da2d63 --- /dev/null +++ b/packages/context/src/ingestion.ts @@ -0,0 +1,131 @@ +/** Per-document outcomes shared by source readers and package builders. */ +export interface IngestionDiagnostic { + /** Path relative to the source root (never the temporary checkout path). */ + path: string; + kind: "file" | "directory"; + outcome: + | "excluded" + | "duplicate" + | "read-error" + | "parse-error" + | "empty" + | "indexed"; + reason?: string; + duplicateOf?: string; + /** Sections actually added to the package after section deduplication. */ + sections?: number; + duplicateSections?: number; +} + +export interface IngestionReport { + schemaVersion: 1; + summary: { + discoveredFiles: number; + selectedFiles: number; + indexedDocuments: number; + indexedSections: number; + excludedFiles: number; + excludedDirectories: number; + duplicateFiles: number; + unreadableFiles: number; + unreadableDirectories: number; + parseFailures: number; + emptyFiles: number; + }; + entries: IngestionDiagnostic[]; +} + +export function createIngestionReport( + diagnostics: readonly IngestionDiagnostic[], +): IngestionReport { + const summary: IngestionReport["summary"] = { + discoveredFiles: 0, + selectedFiles: 0, + indexedDocuments: 0, + indexedSections: 0, + excludedFiles: 0, + excludedDirectories: 0, + duplicateFiles: 0, + unreadableFiles: 0, + unreadableDirectories: 0, + parseFailures: 0, + emptyFiles: 0, + }; + for (const entry of diagnostics) { + if (entry.kind === "directory") { + if (entry.outcome === "excluded") summary.excludedDirectories++; + if (entry.outcome === "read-error") summary.unreadableDirectories++; + continue; + } + summary.discoveredFiles++; + if (entry.outcome !== "excluded" && entry.outcome !== "duplicate") { + summary.selectedFiles++; + } + switch (entry.outcome) { + case "excluded": + summary.excludedFiles++; + break; + case "duplicate": + summary.duplicateFiles++; + break; + case "read-error": + summary.unreadableFiles++; + break; + case "parse-error": + summary.parseFailures++; + break; + case "empty": + summary.emptyFiles++; + break; + case "indexed": + summary.indexedDocuments++; + summary.indexedSections += entry.sections ?? 0; + break; + } + } + const entries = diagnostics + .map((entry) => ({ ...entry })) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + return { schemaVersion: 1, summary, entries }; +} + +/** Keep filesystem error codes useful without leaking checkout paths. */ +export function ingestionErrorReason(error: unknown): string { + const code = (error as NodeJS.ErrnoException | null)?.code; + return typeof code === "string" + ? code + : error instanceof Error + ? error.message + : String(error); +} + +export function formatIngestionSummary(report: IngestionReport): string { + const s = report.summary; + return `Ingestion: ${s.discoveredFiles} documentation files discovered, ${s.selectedFiles} selected, ${s.indexedDocuments} indexed, ${s.indexedSections} sections; excluded ${s.excludedFiles} files/${s.excludedDirectories} directories, ${s.duplicateFiles} duplicates, unreadable ${s.unreadableFiles} files/${s.unreadableDirectories} directories, ${s.parseFailures} parse failures, ${s.emptyFiles} empty files`; +} + +export class IngestionError extends Error { + readonly diagnostics: IngestionReport; + + constructor(message: string, diagnostics: readonly IngestionDiagnostic[]) { + super(message); + this.name = "IngestionError"; + this.diagnostics = createIngestionReport(diagnostics); + } +} + +export function validateIngestion(report: IngestionReport): void { + const s = report.summary; + if ( + s.unreadableFiles + + s.unreadableDirectories + + s.parseFailures + + s.emptyFiles > + 0 + ) { + throw new IngestionError( + "Strict ingestion validation failed: unreadable, unparseable, or empty documentation", + report.entries, + ); + } +} diff --git a/packages/context/src/package-builder.ts b/packages/context/src/package-builder.ts index a336c75..965a5bb 100644 --- a/packages/context/src/package-builder.ts +++ b/packages/context/src/package-builder.ts @@ -7,6 +7,13 @@ import { existsSync, unlinkSync } from "node:fs"; import { type DocSection, parseDocument } from "./build.js"; import { openDatabase } from "./database.js"; import { REMOVED_TAGS } from "./html.js"; +import { + createIngestionReport, + type IngestionDiagnostic, + type IngestionReport, + ingestionErrorReason, + validateIngestion, +} from "./ingestion.js"; /** * Generate a content hash for section deduplication. @@ -23,6 +30,10 @@ export interface PackageBuildOptions { sourceUrl?: string; /** Git commit SHA used to build this package (for skip-if-unchanged checks) */ sourceCommit?: string; + /** Outcomes collected while selecting and reading source documents. */ + diagnostics?: readonly IngestionDiagnostic[]; + /** Reject read/parse failures and empty documents before changing outputPath. */ + strict?: boolean; } export interface MarkdownFile { @@ -36,6 +47,7 @@ export interface BuildResult { totalTokens: number; /** Files dropped whole because splitting or parsing them threw. */ skippedFiles: number; + diagnostics: IngestionReport; } /** @@ -569,6 +581,58 @@ export function buildPackage( files: MarkdownFile[], options: PackageBuildOptions, ): BuildResult { + const allSections: DocSection[] = []; + const seenHashes = new Map(); + const entries: IngestionDiagnostic[] = [...(options.diagnostics ?? [])]; + let skippedFiles = 0; + + // Parse before opening the output so strict validation never replaces an + // installed package with an incomplete build. Keep only one file's split + // chunks alive at a time, as before. + for (const file of files) { + try { + const sections = parseChunks(file); + let indexed = 0; + let duplicateOf: string | undefined; + for (const section of sections) { + const hash = contentHash(section.content); + if (seenHashes.has(hash)) { + duplicateOf ??= seenHashes.get(hash); + } else { + seenHashes.set(hash, file.path); + allSections.push(section); + indexed++; + } + } + entries.push({ + path: file.path, + kind: "file", + outcome: !sections.length ? "empty" : indexed ? "indexed" : "duplicate", + ...(!sections.length + ? { reason: "Document produced no sections" } + : indexed + ? { + sections: indexed, + duplicateSections: sections.length - indexed, + } + : { + reason: "All sections duplicate earlier content", + duplicateOf, + }), + }); + } catch (error) { + skippedFiles++; + entries.push({ + path: file.path, + kind: "file", + outcome: "parse-error", + reason: ingestionErrorReason(error), + }); + } + } + const diagnostics = createIngestionReport(entries); + if (options.strict) validateIngestion(diagnostics); + // Remove existing file if present if (existsSync(outputPath)) { unlinkSync(outputPath); @@ -618,34 +682,6 @@ export function buildPackage( VALUES (?, ?, ?, ?, ?, ?) `); - const allSections: DocSection[] = []; - const seenHashes = new Set(); - let skippedFiles = 0; - - for (const file of files) { - try { - // Splitting is inside the guard on purpose: it walks untrusted markup, so a - // throw there has to cost one file rather than the whole registry build. Doing - // it per file also keeps only one file's chunks alive at a time. - const sections = parseChunks(file); - - for (const section of sections) { - // Deduplicate sections with identical content (ignore titles) - const hash = contentHash(section.content); - if (!seenHashes.has(hash)) { - seenHashes.add(hash); - allSections.push(section); - } - } - } catch { - // A file that cannot be split or parsed is dropped whole, so a half-indexed - // document never reaches the package. The failure is counted rather than - // logged: `buildPackage` has no logger, and a silent skip is how a registry - // build loses documents without anyone noticing. - skippedFiles++; - } - } - // Insert all sections in a transaction const insertAll = db.transaction((sections: DocSection[]) => { for (const section of sections) { @@ -672,6 +708,7 @@ export function buildPackage( sectionCount: allSections.length, totalTokens, skippedFiles, + diagnostics, }; } finally { db.close(); diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts index adb70ba..f93a6a3 100644 --- a/packages/registry/src/build.ts +++ b/packages/registry/src/build.ts @@ -14,6 +14,8 @@ import { type BuildResult, buildPackage, cloneRepository, + type IngestionDiagnostic, + IngestionError, readLocalDocsFiles, } from "@neuledge/context"; import { @@ -36,6 +38,11 @@ export interface RegistryBuildResult extends BuildResult { sourceCommit?: string; } +export interface RegistryBuildOptions { + strict?: boolean; + diagnostics?: IngestionDiagnostic[]; +} + /** * Get the HEAD commit SHA of a remote repository without cloning. * Uses `git ls-remote` which makes a single HTTP call. @@ -64,7 +71,9 @@ export async function buildFromDefinition( definition: VersionedDefinition, version: string, outputDir: string, + options: RegistryBuildOptions = {}, ): Promise { + const diagnostics = options.diagnostics ?? []; const entry = resolveVersionEntry(definition, version); if (!entry) { throw new Error( @@ -89,6 +98,7 @@ export async function buildFromDefinition( outputPath, definition, version, + { ...options, diagnostics }, ); } @@ -96,21 +106,25 @@ export async function buildFromDefinition( const url = resolveUrl(entry.source.url, version); const files = entry.source.type === "html-index" - ? await downloadHtmlIndex(entry.source, version) + ? await downloadHtmlIndex(entry.source, version, { diagnostics }) : await downloadAndExtractZip(url, { docsPath: entry.source.docs_path ? resolveUrl(entry.source.docs_path, version) : undefined, + diagnostics, excludePaths: entry.source.exclude_paths, }); if (files.length === 0) { - throw new Error( + throw new IngestionError( `No documentation files found in ${entry.source.type} source from ${url}`, + diagnostics, ); } const result = buildPackage(outputPath, files, { + diagnostics, + strict: options.strict, name: definition.name, version, description: definition.description, @@ -133,7 +147,9 @@ export async function buildFromDefinition( export async function buildUnversioned( definition: UnversionedDefinition, outputDir: string, + options: RegistryBuildOptions = {}, ): Promise { + const diagnostics = options.diagnostics ?? []; const version = "latest"; const { source } = definition; const safeName = definition.name.replace(/\//g, "-"); @@ -145,14 +161,20 @@ export async function buildUnversioned( if (source.type === "zip") { const files = await downloadAndExtractZip(source.url, { docsPath: source.docs_path, + diagnostics, excludePaths: source.exclude_paths, }); if (files.length === 0) { - throw new Error(`No documentation files found in ZIP from ${source.url}`); + throw new IngestionError( + `No documentation files found in ZIP from ${source.url}`, + diagnostics, + ); } const result = buildPackage(outputPath, files, { + diagnostics, + strict: options.strict, name: definition.name, version, description: definition.description, @@ -184,18 +206,23 @@ export async function buildUnversioned( readLocalDocsFiles(tempDir, { path: source.docs_path, lang: source.lang, + diagnostics, }), source.exclude_paths, source.docs_path, + diagnostics, ); if (files.length === 0) { - throw new Error( + throw new IngestionError( `No documentation files found in ${source.url} (default branch)`, + diagnostics, ); } const result = buildPackage(outputPath, files, { + diagnostics, + strict: options.strict, name: definition.name, version, description: definition.description, @@ -225,23 +252,31 @@ function buildFromGit( outputPath: string, definition: VersionedDefinition, version: string, + options: RegistryBuildOptions, ): RegistryBuildResult { + const diagnostics = options.diagnostics ?? []; const { tempDir, cleanup } = cloneRepository(url, tag); try { // Filter before the emptiness check, so an over-broad exclude_paths fails // loudly here instead of publishing an empty package. const files = excludeFiles( - readLocalDocsFiles(tempDir, { path: docsPath, lang }), + readLocalDocsFiles(tempDir, { path: docsPath, lang, diagnostics }), excludePaths, docsPath, + diagnostics, ); if (files.length === 0) { - throw new Error(`No documentation files found in ${url} at tag ${tag}`); + throw new IngestionError( + `No documentation files found in ${url} at tag ${tag}`, + diagnostics, + ); } const result = buildPackage(outputPath, files, { + diagnostics, + strict: options.strict, name: definition.name, version, description: definition.description, diff --git a/packages/registry/src/cli.ts b/packages/registry/src/cli.ts index b92e6dd..6da1148 100644 --- a/packages/registry/src/cli.ts +++ b/packages/registry/src/cli.ts @@ -5,9 +5,17 @@ * Not shipped to users — used for building and publishing context packages. */ -import { mkdirSync, rmSync } from "node:fs"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; -import { type BuildResult, isMissingRefError } from "@neuledge/context"; +import { + type BuildResult, + createIngestionReport, + formatIngestionSummary, + type IngestionDiagnostic, + IngestionError, + type IngestionReport, + isMissingRefError, +} from "@neuledge/context"; import { Command } from "commander"; import { buildFromDefinition, @@ -42,7 +50,7 @@ const program = new Command() function formatBuilt(result: BuildResult): string { const skipped = result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; - return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}`; + return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}${result.diagnostics ? `; ${formatIngestionSummary(result.diagnostics)}` : ""}`; } program @@ -108,25 +116,44 @@ program .description("Build a .db package for a specific version") .option("--dir ", "Registry directory", DEFAULT_REGISTRY_DIR) .option("--output ", "Output directory", "./dist-packages") + .option("--strict", "Fail on unreadable, unparseable, or empty documentation") + .option( + "--diagnostics ", + "Write detailed ingestion diagnostics as JSON", + ) .action(async (name, version, opts) => { - const def = findDefinition(opts.dir, name); - mkdirSync(opts.output, { recursive: true }); - - if (isVersioned(def)) { - if (!version) { + const diagnostics: IngestionDiagnostic[] = []; + let report: IngestionReport | undefined; + try { + const def = findDefinition(opts.dir, name); + mkdirSync(opts.output, { recursive: true }); + if (isVersioned(def) && !version) { throw new Error( `Version required for versioned package "${name}". Use: registry build ${name} `, ); } - console.log(`Building ${def.registry}/${def.name}@${version}...`); - const result = await buildFromDefinition(def, version, opts.output); - console.log(`Built: ${result.path} (${formatBuilt(result)})`); - } else { console.log( - `Building ${def.registry}/${def.name}@latest (unversioned)...`, + `Building ${def.registry}/${def.name}@${version ?? "latest"}...`, ); - const result = await buildUnversioned(def, opts.output); + const options = { strict: opts.strict, diagnostics }; + const result = isVersioned(def) + ? await buildFromDefinition(def, version, opts.output, options) + : await buildUnversioned(def, opts.output, options); + report = result.diagnostics; console.log(`Built: ${result.path} (${formatBuilt(result)})`); + } catch (error) { + report = + error instanceof IngestionError + ? error.diagnostics + : createIngestionReport(diagnostics); + console.error(formatIngestionSummary(report)); + throw error; + } finally { + if (opts.diagnostics) + writeFileSync( + resolve(opts.diagnostics), + `${JSON.stringify(report ?? createIngestionReport(diagnostics), null, 2)}\n`, + ); } }); diff --git a/packages/registry/src/glob.ts b/packages/registry/src/glob.ts index f1fabf8..1153d28 100644 --- a/packages/registry/src/glob.ts +++ b/packages/registry/src/glob.ts @@ -1,6 +1,7 @@ /** * Glob matching for `exclude_paths`, shared by the zip and git source builders. */ +import type { IngestionDiagnostic } from "@neuledge/context"; /** * Compile a simple glob pattern to a RegExp. @@ -26,6 +27,7 @@ export function excludeFiles( files: T[], excludePaths: string[] | undefined, docsPath?: string, + diagnostics?: IngestionDiagnostic[], ): T[] { if (!excludePaths?.length) return files; @@ -37,6 +39,14 @@ export function excludeFiles( prefix && file.path.startsWith(prefix) ? file.path.slice(prefix.length) : file.path; - return !patterns.some((re) => re.test(relative)); + const excluded = patterns.some((re) => re.test(relative)); + if (excluded) + diagnostics?.push({ + path: file.path, + kind: "file", + outcome: "excluded", + reason: "exclude_paths", + }); + return !excluded; }); } diff --git a/packages/registry/src/html-index-build.test.ts b/packages/registry/src/html-index-build.test.ts index 7a6afa1..c0ab581 100644 --- a/packages/registry/src/html-index-build.test.ts +++ b/packages/registry/src/html-index-build.test.ts @@ -135,6 +135,7 @@ describe("HTML index registry integration", () => { expect(downloadHtmlIndex).toHaveBeenCalledWith( expect.objectContaining({ type: "html-index" }), "258", + { diagnostics: expect.any(Array) }, ); }); diff --git a/packages/registry/src/html-index.ts b/packages/registry/src/html-index.ts index feb609c..ae8e9f5 100644 --- a/packages/registry/src/html-index.ts +++ b/packages/registry/src/html-index.ts @@ -3,6 +3,11 @@ import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; +import { + type IngestionDiagnostic, + IngestionError, + ingestionErrorReason, +} from "@neuledge/context"; import { parseHTML } from "linkedom"; import type { HtmlIndexSource } from "./definition.js"; import { compileGlob } from "./glob.js"; @@ -62,16 +67,28 @@ function indexLinks( index: URL, root: URL, source: HtmlIndexSource, + diagnostics?: IngestionDiagnostic[], ): URL[] { const { document } = parseHTML(html); const excluded = source.exclude_paths?.map(compileGlob) ?? []; const urls = new Set(); + const seenLinks = new Set(); for (const anchor of document.querySelectorAll("a[href]")) { const url = scopedUrl(anchor.getAttribute("href") ?? "", index, root); if (!url || !HTML_PATH.test(url.pathname) || url.href === index.href) continue; + if (seenLinks.has(url.href)) continue; + seenLinks.add(url.href); const path = decodeURIComponent(url.pathname.slice(root.pathname.length)); - if (excluded.some((pattern) => pattern.test(path))) continue; + if (excluded.some((pattern) => pattern.test(path))) { + diagnostics?.push({ + path, + kind: "file", + outcome: "excluded", + reason: "exclude_paths", + }); + continue; + } urls.add(url.href); if (urls.size > source.max_pages) { throw new Error( @@ -253,7 +270,7 @@ async function fetchPage( export async function downloadHtmlIndex( source: HtmlIndexSource, version: string, - options: { cacheDir?: string } = {}, + options: { cacheDir?: string; diagnostics?: IngestionDiagnostic[] } = {}, ): Promise> { const index = resolveIndexUrl(source.url, version); const root = new URL(".", index); @@ -266,16 +283,19 @@ export async function downloadHtmlIndex( new URL(indexPage.finalUrl), root, source, + options.diagnostics, ); const pages = new Map(); let totalBytes = Buffer.byteLength(indexPage.content); let cursor = 0; let failure: unknown; const worker = async () => { + let path = "."; try { while (!controller.signal.aborted) { const url = urls[cursor++]; if (!url) return; + path = decodeURIComponent(url.pathname.slice(root.pathname.length)); const page = await fetchPage(url, root, cacheDir, controller.signal); totalBytes += Buffer.byteLength(page.content); if (totalBytes > MAX_TOTAL_BYTES) @@ -283,7 +303,19 @@ export async function downloadHtmlIndex( pages.set(url.href, page); } } catch (error) { - if (!controller.signal.aborted) failure = error; + if (!controller.signal.aborted) { + const diagnostic: IngestionDiagnostic = { + path, + kind: "file", + outcome: "read-error", + reason: ingestionErrorReason(error), + }; + options.diagnostics?.push(diagnostic); + failure = new IngestionError( + ingestionErrorReason(error), + options.diagnostics ?? [diagnostic], + ); + } controller.abort(); } }; @@ -291,17 +323,27 @@ export async function downloadHtmlIndex( Array.from({ length: Math.min(source.concurrency, urls.length) }, worker), ); if (failure) throw failure; - const seen = new Set(); + const seen = new Map(); const files: Array<{ path: string; content: string }> = []; // Stable order chooses the same alias regardless of download completion order. for (const url of urls) { const page = pages.get(url.href); if (!page) throw new Error(`HTML page was not downloaded: ${url}`); const hash = digest(page.content); - if (seen.has(hash)) continue; - seen.add(hash); + const path = decodeURIComponent(url.pathname.slice(root.pathname.length)); + if (seen.has(hash)) { + options.diagnostics?.push({ + path, + kind: "file", + outcome: "duplicate", + reason: "Identical document content", + duplicateOf: seen.get(hash), + }); + continue; + } + seen.set(hash, path); files.push({ - path: decodeURIComponent(url.pathname.slice(root.pathname.length)), + path, content: page.content, }); } diff --git a/packages/registry/src/index.ts b/packages/registry/src/index.ts index 72c54db..ddd93b9 100644 --- a/packages/registry/src/index.ts +++ b/packages/registry/src/index.ts @@ -2,6 +2,8 @@ export { buildFromDefinition, buildUnversioned, getHeadCommit, + type RegistryBuildOptions, + type RegistryBuildResult, } from "./build.js"; export { constructTag, diff --git a/packages/registry/src/ingestion.test.ts b/packages/registry/src/ingestion.test.ts new file mode 100644 index 0000000..046f53c --- /dev/null +++ b/packages/registry/src/ingestion.test.ts @@ -0,0 +1,297 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { + buildPackage, + type IngestionDiagnostic, + IngestionError, + initDatabase, +} from "@neuledge/context"; +import { afterEach, beforeAll, beforeEach, expect, it, vi } from "vitest"; +import { buildFromDefinition } from "./build.js"; +import type { VersionedDefinition } from "./definition.js"; +import { downloadHtmlIndex } from "./html-index.js"; + +let directory: string; +beforeAll(async () => { + await initDatabase(); +}); +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), "registry-ingestion-")); +}); +afterEach(() => { + vi.unstubAllGlobals(); + rmSync(directory, { recursive: true, force: true }); +}); + +/** Small stored ZIP fixture, without compression or third-party tooling. */ +function zip(files: Record): Uint8Array { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const [path, text] of Object.entries(files)) { + const name = Buffer.from(path); + const content = Buffer.from(text); + const header = Buffer.alloc(30); + header.writeUInt32LE(0x04034b50, 0); + header.writeUInt32LE(content.length, 18); + header.writeUInt32LE(content.length, 22); + header.writeUInt16LE(name.length, 26); + local.push(header, name, content); + const entry = Buffer.alloc(46); + entry.writeUInt32LE(0x02014b50, 0); + entry.writeUInt32LE(content.length, 20); + entry.writeUInt32LE(content.length, 24); + entry.writeUInt16LE(name.length, 28); + entry.writeUInt32LE(offset, 42); + central.push(entry, name); + offset += header.length + name.length + content.length; + } + const index = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(Object.keys(files).length, 8); + end.writeUInt16LE(Object.keys(files).length, 10); + end.writeUInt32LE(index.length, 12); + end.writeUInt32LE(offset, 16); + return new Uint8Array(Buffer.concat([...local, index, end])); +} + +it("carries ZIP exclusions, duplicate sections and empty documents into the build report", async () => { + const content = "## ZIP Guide\n\nRun the archived documentation example."; + const archive = zip({ + "docs/guide.md": content, + "docs/alias.md": content, + "docs/excluded.md": "Intentionally excluded.", + "docs/empty.md": "", + "docs/search.html": "Generated navigation.", + "docs/source.ts": "Unrelated source code.", + "outside.md": "Outside selected docs_path.", + }); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(archive)), + ); + const definition: VersionedDefinition = { + registry: "test", + name: "zip-docs", + versions: [ + { + versions: ["1"], + source: { + type: "zip", + url: "https://fixture.example/docs.zip", + docs_path: "docs", + exclude_paths: ["excluded.md"], + lang: "en", + }, + }, + ], + }; + const result = await buildFromDefinition(definition, "1", directory); + expect(result.diagnostics.summary).toMatchObject({ + discoveredFiles: 5, + selectedFiles: 2, + indexedDocuments: 1, + indexedSections: 1, + duplicateFiles: 1, + excludedFiles: 2, + emptyFiles: 1, + }); + expect(result.diagnostics.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "excluded.md", + outcome: "excluded", + reason: "exclude_paths", + }), + expect.objectContaining({ + path: "search.html", + outcome: "excluded", + reason: "generated-navigation", + }), + expect.objectContaining({ + path: "alias.md", + outcome: "duplicate", + duplicateOf: "guide.md", + }), + ]), + ); + const original = readFileSync(result.path); + await expect( + buildFromDefinition(definition, "1", directory, { strict: true }), + ).rejects.toBeInstanceOf(IngestionError); + expect(readFileSync(result.path)).toEqual(original); +}); + +it("uses the same outcomes for HTML index exclusions, aliases and empty pages", async () => { + const page = + "

HTML reference

Example

Run the HTML example to configure the service.

"; + const pages: Record = { + "/docs/1/": + 'GuideAliasEmptyExcludedRepeated excluded link', + "/docs/1/a.html": page, + "/docs/1/b.html": page, + "/docs/1/empty.html": "", + }; + const fetchMock = vi.fn( + async (url: URL) => + new Response(pages[url.pathname], { + headers: { "content-type": "text/html" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const diagnostics: IngestionDiagnostic[] = []; + const files = await downloadHtmlIndex( + { + type: "html-index", + url: "https://fixture.example/docs/{version}/", + exclude_paths: ["excluded.html"], + max_pages: 10, + concurrency: 2, + }, + "1", + { cacheDir: join(directory, "cache"), diagnostics }, + ); + const result = buildPackage(join(directory, "html.db"), files, { + name: "html-docs", + version: "1", + diagnostics, + }); + expect(result.diagnostics.summary).toMatchObject({ + discoveredFiles: 4, + selectedFiles: 2, + indexedDocuments: 1, + duplicateFiles: 1, + excludedFiles: 1, + emptyFiles: 1, + }); + expect(result.diagnostics.entries).toContainEqual({ + path: "b.html", + kind: "file", + outcome: "duplicate", + reason: "Identical document content", + duplicateOf: "a.html", + }); + expect(fetchMock).toHaveBeenCalledTimes(4); +}); + +it("keeps failed HTML downloads fatal and attaches their relative path", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (url: URL) => + url.pathname.endsWith("missing.html") + ? new Response("Missing", { status: 404 }) + : new Response('Missing', { + headers: { "content-type": "text/html" }, + }), + ), + ); + await expect( + downloadHtmlIndex( + { + type: "html-index", + url: "https://fixture.example/docs/{version}/", + max_pages: 10, + concurrency: 1, + }, + "1", + { cacheDir: join(directory, "cache") }, + ), + ).rejects.toMatchObject({ + diagnostics: { + entries: [ + expect.objectContaining({ + path: "missing.html", + kind: "file", + outcome: "read-error", + }), + ], + }, + }); +}); + +it("writes CLI JSON diagnostics for Git builds and strict failures without changing the package", () => { + const repository = join(directory, "source"); + const definitions = join(directory, "registry"); + const output = join(directory, "output"); + const report = join(directory, "report.json"); + mkdirSync(repository); + mkdirSync(join(definitions, "test"), { recursive: true }); + writeFileSync( + join(repository, "guide.md"), + "## Git Guide\n\nRun the locally cloned documentation example.", + ); + writeFileSync(join(repository, "empty.md"), ""); + writeFileSync(join(repository, "excluded.md"), "Excluded source document."); + execFileSync("git", ["init", "--quiet", repository]); + execFileSync("git", ["add", "."], { cwd: repository }); + execFileSync( + "git", + [ + "-c", + "user.name=Ingestion Test", + "-c", + "user.email=ingestion@example.invalid", + "commit", + "--quiet", + "-m", + "Fixture", + ], + { cwd: repository }, + ); + writeFileSync( + join(definitions, "test", "fixture.yaml"), + `name: fixture\nsource:\n type: git\n url: "${pathToFileURL(repository).href}"\n exclude_paths: [excluded.md]\n`, + ); + const args = [ + resolve(import.meta.dirname, "../dist/cli.js"), + "build", + "fixture", + "--dir", + definitions, + "--output", + output, + "--diagnostics", + report, + ]; + const normal = spawnSync(process.execPath, args, { encoding: "utf8" }); + expect(normal.status, normal.stderr).toBe(0); + expect(normal.stdout).toContain("Ingestion:"); + expect(JSON.parse(readFileSync(report, "utf8"))).toMatchObject({ + schemaVersion: 1, + summary: { + discoveredFiles: 3, + selectedFiles: 2, + indexedDocuments: 1, + excludedFiles: 1, + emptyFiles: 1, + }, + }); + const packagePath = join(output, "test-fixture@latest.db"); + const original = readFileSync(packagePath); + const strict = spawnSync(process.execPath, [...args, "--strict"], { + encoding: "utf8", + }); + expect(strict.status).not.toBe(0); + expect(strict.stderr).toContain("Strict ingestion validation failed"); + expect(readFileSync(packagePath)).toEqual(original); + expect(JSON.parse(readFileSync(report, "utf8")).entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "empty.md", outcome: "empty" }), + expect.objectContaining({ + path: "excluded.md", + outcome: "excluded", + reason: "exclude_paths", + }), + ]), + ); +}, 20_000); diff --git a/packages/registry/src/zip.ts b/packages/registry/src/zip.ts index c8ad7e9..36b479a 100644 --- a/packages/registry/src/zip.ts +++ b/packages/registry/src/zip.ts @@ -7,6 +7,11 @@ */ import { inflateRawSync } from "node:zlib"; +import { + type IngestionDiagnostic, + IngestionError, + ingestionErrorReason, +} from "@neuledge/context"; import { compileGlob } from "./glob.js"; const DOCUMENTATION_EXTENSIONS = [ @@ -57,7 +62,11 @@ interface ZipEntry { */ export async function downloadAndExtractZip( url: string, - options?: { docsPath?: string; excludePaths?: string[] }, + options?: { + docsPath?: string; + excludePaths?: string[]; + diagnostics?: IngestionDiagnostic[]; + }, ): Promise> { const response = await fetch(url); if (!response.ok) { @@ -88,13 +97,43 @@ export async function downloadAndExtractZip( // Skip default ignored files (by basename) const basename = relativePath.split("/").pop() ?? ""; - if (IGNORED_FILES.has(basename)) continue; + if (IGNORED_FILES.has(basename)) { + options?.diagnostics?.push({ + path: relativePath, + kind: "file", + outcome: "excluded", + reason: "generated-navigation", + }); + continue; + } // Apply custom exclude patterns against relative path - if (excludePatterns?.some((re) => re.test(relativePath))) continue; + if (excludePatterns?.some((re) => re.test(relativePath))) { + options?.diagnostics?.push({ + path: relativePath, + kind: "file", + outcome: "excluded", + reason: "exclude_paths", + }); + continue; + } - const content = extractEntry(buffer, entry); - files.push({ path: relativePath, content }); + try { + const content = extractEntry(buffer, entry); + files.push({ path: relativePath, content }); + } catch (error) { + const diagnostic: IngestionDiagnostic = { + path: relativePath, + kind: "file", + outcome: "read-error", + reason: ingestionErrorReason(error), + }; + options?.diagnostics?.push(diagnostic); + throw new IngestionError( + `Could not extract ${relativePath}: ${ingestionErrorReason(error)}`, + options?.diagnostics ?? [diagnostic], + ); + } } return files; diff --git a/registry/README.md b/registry/README.md index 14410a9..da27226 100644 --- a/registry/README.md +++ b/registry/README.md @@ -205,5 +205,34 @@ pnpm --filter @neuledge/registry registry build A healthy build reports a few hundred sections. A handful usually means `docs_path` is pointing at the wrong directory. +Build and publication logs also summarize ingestion outcomes. For a detailed JSON +report, or to reject unexpected document loss during local validation, use: + +```bash +pnpm --filter @neuledge/registry registry build [version] --diagnostics ingestion.json +pnpm --filter @neuledge/registry registry build [version] --strict --diagnostics ingestion.json +``` + +Git, ZIP, and HTML-index sources share the same report vocabulary: `excluded`, +`duplicate`, `read-error`, `parse-error`, `empty`, and `indexed`. Entries name +relative paths and give reasons; duplicate documents identify the retained path +when available. Git paths are relative to the checkout, ZIP paths to `docs_path`, +and HTML paths to the pinned index directory. Directory exclusions are reported +without walking their descendants, and unrelated source-code files are omitted. + +The summary distinguishes discovered documentation candidates, selected files +(excluding intentional exclusions and duplicates), indexed documents, and stored +sections after deduplication. Counts cover the selected source scope, not files +outside `docs_path` or links outside the pinned HTML directory. Failed source +downloads still abort the build; failure reports may contain only outcomes +collected before the abort. + +`--strict` fails on read errors, parse errors, and documents producing no sections. +It does not reject exclusions or duplicates, and validation failures leave any +existing output package unchanged. The report is written on strict failure too. +The default remains a tolerant build with visible diagnostics. For library use, +`buildFromDefinition` and `buildUnversioned` accept an optional final +`{ strict, diagnostics }` argument and return `diagnostics` with the build result. + Then open the PR. New definitions are welcome — the registry is only as good as its coverage. From d0363562fc3321101f205a5b7f5c46189c0a1150 Mon Sep 17 00:00:00 2001 From: Martin Beckert Date: Fri, 2 Oct 2026 11:54:56 +0200 Subject: [PATCH 2/2] test(context): use file URL for Windows ingestion preload --- packages/context/src/ingestion.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/context/src/ingestion.test.ts b/packages/context/src/ingestion.test.ts index f5e4078..4eb1105 100644 --- a/packages/context/src/ingestion.test.ts +++ b/packages/context/src/ingestion.test.ts @@ -11,6 +11,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; import { afterEach, beforeAll, beforeEach, expect, it, vi } from "vitest"; import { parseDocument } from "./build.js"; import { initDatabase, openDatabase } from "./database.js"; @@ -285,7 +286,7 @@ it("writes context add reports on success and strict failure without changing th const report = join(directory, "report.json"); const args = [ "--import", - preload, + pathToFileURL(preload).href, resolve(import.meta.dirname, "../dist/cli.js"), "add", directory,