diff --git a/.changeset/fresh-documentation-inputs.md b/.changeset/fresh-documentation-inputs.md new file mode 100644 index 0000000..b370000 --- /dev/null +++ b/.changeset/fresh-documentation-inputs.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Allow package builders to store build fingerprints and automatic ingestion revisions in package metadata for registry freshness checks. diff --git a/SERVER_SPEC.md b/SERVER_SPEC.md index d90a57f..118969e 100644 --- a/SERVER_SPEC.md +++ b/SERVER_SPEC.md @@ -67,7 +67,7 @@ Returns an empty array `[]` when no packages match. Results are sorted by versio GET /packages/// ``` -Check if a package version exists and return its metadata. Used by the publish pipeline for idempotency (skip already-published versions) and by unversioned packages to compare `source_commit`. +Check if a package version exists and return its metadata. The publish pipeline compares `build_fingerprint` to skip packages whose source, resolved definition, and ingestion implementation are unchanged. **Response `200 OK`:** @@ -76,7 +76,9 @@ Check if a package version exists and return its metadata. Used by the publish p "registry": "npm", "name": "nextjs", "version": "15.1.0", - "source_commit": "abc1234" + "source_commit": "abc1234", + "build_fingerprint": "", + "ingestion_revision": "" } ``` @@ -87,7 +89,22 @@ Check if a package version exists and return its metadata. Used by the publish p | `registry` | string | Package manager | | `name` | string | Package name | | `version` | string | Semver version or `"latest"` | -| `source_commit` | string? | Git SHA for unversioned packages | +| `source_commit` | string? | Git commit SHA for versioned or unversioned Git packages | +| `build_fingerprint` | string? | Opaque SHA-256 copied from the uploaded database's `meta` table | +| `ingestion_revision` | string? | Automatic ingestion revision copied from the uploaded database's `meta` table | + +Servers supporting automatic freshness checks must extract and persist these optional metadata values on every upload, including replacements, and return them here. The client treats fingerprints as opaque; servers should not recompute them. Older packages and servers can omit these fields: versioned packages retain existence-based skipping, unversioned Git packages retain `source_commit` comparison, and unversioned ZIP packages continue rebuilding. The CLI identifies legacy skips and offers `--force` for migration. Returning the fingerprint after a successful rebuild enables subsequent automatic checks; a server that continues omitting it retains the legacy behavior. + +Server adoption must be verified in the server deployment; this repository only +contains the client and protocol contract. Verify that an initial upload returns +the database's fingerprint and revision, that replacing it updates both fields, +and that a subsequent metadata request returns the replacement's values. + +After an upload returns `409 Conflict`, the client requests metadata to determine +whether an earlier attempt already succeeded. It accepts success only when the +package identity, `build_fingerprint`, and `ingestion_revision` match the uploaded +artifact. Missing or different metadata remains a conflict; an existing version +alone is not sufficient evidence of a successful upload. **Response `404 Not Found`:** @@ -135,6 +152,8 @@ Upload a new documentation package. Requires a valid API key. **Response `409 Conflict`** — Package version already exists (optional; servers may also allow overwrites). +The publisher uploads stale or explicitly forced builds to the same registry/name/version. `--force` bypasses client freshness checks only; it does not grant replacement permission or change the API request. If this endpoint rejects replacement with HTTP 409, the publisher reports the conflict and preserves the rebuilt local `.db`. Updating an immutable release requires a server-supported replacement or artifact revision policy; no alternative version is invented by the client. + ## Package format (`.db` file) Packages are SQLite databases with the following schema: @@ -178,7 +197,9 @@ CREATE VIRTUAL TABLE chunks_fts USING fts5( |-----|-------------| | `description` | Short package description | | `source_url` | URL of the source repository | -| `source_commit` | Git commit SHA (used for unversioned packages to detect changes) | +| `source_commit` | Actual Git commit SHA used to build the package | +| `build_fingerprint` | Deterministic SHA-256 of resolved package build inputs | +| `ingestion_revision` | SHA-256 generated from ingestion code and locked dependencies at build time | ## Error format diff --git a/packages/context/src/package-builder.ts b/packages/context/src/package-builder.ts index e82e88f..d12534a 100644 --- a/packages/context/src/package-builder.ts +++ b/packages/context/src/package-builder.ts @@ -24,6 +24,10 @@ export interface PackageBuildOptions { sourceUrl?: string; /** Git commit SHA used to build this package (for skip-if-unchanged checks) */ sourceCommit?: string; + /** Deterministic registry build inputs, used for publication freshness. */ + buildFingerprint?: string; + /** Automatically generated revision of the ingestion implementation. */ + ingestionRevision?: string; } export interface MarkdownFile { @@ -623,6 +627,12 @@ function writePackage( if (options.sourceCommit) { insertMeta.run("source_commit", options.sourceCommit); } + if (options.buildFingerprint) { + insertMeta.run("build_fingerprint", options.buildFingerprint); + } + if (options.ingestionRevision) { + insertMeta.run("ingestion_revision", options.ingestionRevision); + } // Parse and insert chunks const insertChunk = db.prepare(` diff --git a/packages/registry/package.json b/packages/registry/package.json index 6d90568..45b00d1 100644 --- a/packages/registry/package.json +++ b/packages/registry/package.json @@ -14,10 +14,10 @@ ".": "./dist/index.js" }, "scripts": { - "build": "rimraf --glob dist/** && tsc -p tsconfig.build.json", + "build": "rimraf --glob dist/** && node ../../scripts/generate-ingestion-revision.mjs && tsc -p tsconfig.build.json", "test": "vitest run", "test:watch": "vitest", - "registry": "tsx src/cli.ts", + "registry": "node ../../scripts/generate-ingestion-revision.mjs && tsx src/cli.ts", "test-registry": "tsx src/test-registry.ts", "lint": "npx biome ci --error-on-warnings", "fix": "npx biome check --write" diff --git a/packages/registry/src/build.test.ts b/packages/registry/src/build.test.ts new file mode 100644 index 0000000..1b360f0 --- /dev/null +++ b/packages/registry/src/build.test.ts @@ -0,0 +1,68 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { getHeadCommit, MissingSourceRefError } from "./build.js"; + +describe("remote Git commit resolution", () => { + let root: string; + let url: string; + let commit: string; + const git = (...args: string[]) => + execFileSync("git", args, { + cwd: root, + encoding: "utf8", + stdio: "pipe", + }).trim(); + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "registry-refs-")); + url = pathToFileURL(root).href; + git("init", "--initial-branch=main"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.com"); + writeFileSync(join(root, "doc.md"), "documentation"); + git("add", "."); + git("commit", "-m", "docs"); + commit = git("rev-parse", "HEAD"); + git("tag", "-a", "annotated", "-m", "release"); + git("tag", "lightweight"); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + it.each([ + undefined, + "main", + "refs/heads/main", + "annotated", + "refs/tags/annotated", + "lightweight", + "refs/tags/lightweight", + ])("resolves %s to the cloned commit", (ref) => { + expect(getHeadCommit(url, ref)).toBe(commit); + }); + + it("prefers a branch over a same-named annotated tag", () => { + git("branch", "annotated"); + git("checkout", "annotated"); + writeFileSync(join(root, "doc.md"), "updated documentation"); + git("commit", "-am", "update"); + expect(getHeadCommit(url, "annotated")).toBe(git("rev-parse", "HEAD")); + expect(getHeadCommit(url, "refs/tags/annotated")).toBe(commit); + }); + + it("distinguishes missing refs from an inaccessible repository", () => { + expect(() => getHeadCommit(url, "missing-tag")).toThrow( + MissingSourceRefError, + ); + expect(() => + getHeadCommit(pathToFileURL(join(root, "missing-repo")).href, "main"), + ).toThrow(); + try { + getHeadCommit(pathToFileURL(join(root, "missing-repo")).href, "main"); + } catch (error) { + expect(error).not.toBeInstanceOf(MissingSourceRefError); + } + }); +}); diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts index adb70ba..7916f14 100644 --- a/packages/registry/src/build.ts +++ b/packages/registry/src/build.ts @@ -8,26 +8,30 @@ * (clone default branch) definitions. Supports git, zip and HTML index sources. */ -import { execSync } from "node:child_process"; +import { execFileSync } from "node:child_process"; import { join } from "node:path"; import { type BuildResult, buildPackage, cloneRepository, + initDatabase, readLocalDocsFiles, } from "@neuledge/context"; -import { - constructTag, - isGitVersionEntry, - resolveUrl, - resolveVersionEntry, - type UnversionedDefinition, - type VersionedDefinition, +import type { + GitSource, + PackageDefinition, + UnversionedDefinition, + VersionedDefinition, } from "./definition.js"; +import { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; import { excludeFiles } from "./glob.js"; import { downloadHtmlIndex } from "./html-index.js"; +import { resolveBuildSource } from "./source.js"; import { downloadAndExtractZip } from "./zip.js"; +/** Only an absent ref is skippable; transport failures keep their original error. */ +export class MissingSourceRefError extends Error {} + export interface RegistryBuildResult extends BuildResult { name: string; registry: string; @@ -44,19 +48,54 @@ export function getHeadCommit(url: string, ref?: string): string { // Must match the ref the package is built from. Asking for HEAD while building // a branch compares two unrelated commits, so the skip-if-unchanged check never // fires and the package is rebuilt and republished on every run. - const output = execSync(`git ls-remote ${url} ${ref ?? "HEAD"}`, { - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - }).trim(); - - // Format: "\tHEAD" - const sha = output.split("\t")[0]; + const requested = ref ?? "HEAD"; + const output = execFileSync( + "git", + ["ls-remote", url, requested, `${requested}^{}`], + { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }, + ).trim(); + + const refs = new Map( + output.split("\n").map((line) => { + const [sha, name] = line.split("\t"); + return [name, sha]; + }), + ); + // Match clone --branch: prefer a branch, and peel annotated tags to commits. + const candidates = requested.startsWith("refs/") + ? [requested] + : [`refs/heads/${requested}`, `refs/tags/${requested}`, requested]; + const sha = candidates + .map((name) => refs.get(`${name}^{}`) ?? refs.get(name)) + .find(Boolean); if (!sha) { - throw new Error(`Failed to get HEAD commit for ${url}`); + throw new MissingSourceRefError( + `Git reference ${requested} not found in upstream ${url}`, + ); } return sha; } +function fingerprintOptions( + definition: PackageDefinition, + version: string, + sourceCommit?: string, +) { + const ingestionRevision = getIngestionRevision(); + return { + ingestionRevision, + buildFingerprint: createBuildFingerprint( + definition, + version, + sourceCommit, + ingestionRevision, + ), + }; +} + /** * Build a .db package for a specific version of a versioned definition. */ @@ -65,12 +104,8 @@ export async function buildFromDefinition( version: string, outputDir: string, ): Promise { - const entry = resolveVersionEntry(definition, version); - if (!entry) { - throw new Error( - `No version entry matches ${version} in ${definition.name}`, - ); - } + await initDatabase(); + const source = resolveBuildSource(definition, version); // Replace / in scoped names (e.g., @trpc/server → @trpc-server) for valid filenames const safeName = definition.name.replace(/\//g, "-"); @@ -79,34 +114,21 @@ export async function buildFromDefinition( `${definition.registry}-${safeName}@${version}.db`, ); - if (isGitVersionEntry(entry)) { - return buildFromGit( - entry.source.url, - constructTag(entry.tag_pattern, version), - entry.source.docs_path, - entry.source.exclude_paths, - entry.source.lang, - outputPath, - definition, - version, - ); + if (source.type === "git") { + return buildFromGit(source, outputPath, definition, version); } - // Explicit releases download an archive or a pinned HTML index. - const url = resolveUrl(entry.source.url, version); const files = - entry.source.type === "html-index" - ? await downloadHtmlIndex(entry.source, version) - : await downloadAndExtractZip(url, { - docsPath: entry.source.docs_path - ? resolveUrl(entry.source.docs_path, version) - : undefined, - excludePaths: entry.source.exclude_paths, + source.type === "html-index" + ? await downloadHtmlIndex(source, version) + : await downloadAndExtractZip(source.url, { + docsPath: source.docs_path, + excludePaths: source.exclude_paths, }); if (files.length === 0) { throw new Error( - `No documentation files found in ${entry.source.type} source from ${url}`, + `No documentation files found in ${source.type} source from ${source.url}`, ); } @@ -114,7 +136,8 @@ export async function buildFromDefinition( name: definition.name, version, description: definition.description, - sourceUrl: definition.repository ?? url, + sourceUrl: definition.repository ?? source.url, + ...fingerprintOptions(definition, version), }); return { @@ -134,8 +157,9 @@ export async function buildUnversioned( definition: UnversionedDefinition, outputDir: string, ): Promise { + await initDatabase(); const version = "latest"; - const { source } = definition; + const source = resolveBuildSource(definition, version); const safeName = definition.name.replace(/\//g, "-"); const outputPath = join( outputDir, @@ -157,6 +181,7 @@ export async function buildUnversioned( version, description: definition.description, sourceUrl: definition.repository ?? source.url, + ...fingerprintOptions(definition, version), }); return { @@ -167,17 +192,26 @@ export async function buildUnversioned( }; } - // Git source: clone and read + if (source.type !== "git") + throw new Error("Unversioned HTML sources are unsupported"); + return buildFromGit(source, outputPath, definition, version); +} + +/** Build from a git source (clone at tag, read docs, build package). */ +function buildFromGit( + source: GitSource, + outputPath: string, + definition: PackageDefinition, + version: string, +): RegistryBuildResult { const { tempDir, cleanup } = cloneRepository(source.url, source.ref); try { - // Get the commit SHA of the cloned HEAD - const sourceCommit = execSync("git rev-parse HEAD", { + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: tempDir, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], }).trim(); - // Filter before the emptiness check, so an over-broad exclude_paths fails // loudly here instead of publishing an empty package. const files = excludeFiles( @@ -191,7 +225,7 @@ export async function buildUnversioned( if (files.length === 0) { throw new Error( - `No documentation files found in ${source.url} (default branch)`, + `No documentation files found in ${source.url} at ref ${source.ref ?? "HEAD"}`, ); } @@ -201,6 +235,7 @@ export async function buildUnversioned( description: definition.description, sourceUrl: definition.repository ?? source.url, sourceCommit, + ...fingerprintOptions(definition, version, sourceCommit), }); return { @@ -214,47 +249,3 @@ export async function buildUnversioned( cleanup(); } } - -/** Build from a git source (clone at tag, read docs, build package). */ -function buildFromGit( - url: string, - tag: string, - docsPath: string | undefined, - excludePaths: string[] | undefined, - lang: string, - outputPath: string, - definition: VersionedDefinition, - version: string, -): RegistryBuildResult { - const { tempDir, cleanup } = cloneRepository(url, tag); - - try { - // Filter before the emptiness check, so an over-broad exclude_paths fails - // loudly here instead of publishing an empty package. - const files = excludeFiles( - readLocalDocsFiles(tempDir, { path: docsPath, lang }), - excludePaths, - docsPath, - ); - - if (files.length === 0) { - throw new Error(`No documentation files found in ${url} at tag ${tag}`); - } - - const result = buildPackage(outputPath, files, { - name: definition.name, - version, - description: definition.description, - sourceUrl: definition.repository ?? url, - }); - - return { - ...result, - name: definition.name, - registry: definition.registry, - version, - }; - } finally { - cleanup(); - } -} diff --git a/packages/registry/src/cli.ts b/packages/registry/src/cli.ts index b92e6dd..cc09b24 100644 --- a/packages/registry/src/cli.ts +++ b/packages/registry/src/cli.ts @@ -7,19 +7,20 @@ import { mkdirSync, rmSync } from "node:fs"; import { resolve } from "node:path"; -import { type BuildResult, isMissingRefError } from "@neuledge/context"; +import { isMissingRefError } from "@neuledge/context"; import { Command } from "commander"; import { buildFromDefinition, buildUnversioned, - getHeadCommit, + MissingSourceRefError, } from "./build.js"; import { isExplicitVersionEntry, isVersioned, listDefinitions, } from "./definition.js"; -import { checkPackageExists, publishPackage } from "./publish.js"; +import { formatBuilt } from "./format.js"; +import { publishDefinition } from "./publication.js"; import { type AvailableVersion, discoverVersions } from "./version-check.js"; const DEFAULT_REGISTRY_DIR = resolve( @@ -32,19 +33,6 @@ const program = new Command() .name("registry") .description("Build context documentation packages from definitions"); -/** - * Describe a finished build. - * - * Skipped files are named rather than counted silently: a file too malformed to - * parse is dropped on its own so one bad document can't fail the build, which is - * only safe if the count reaches whoever is reading the log. - */ -function formatBuilt(result: BuildResult): string { - const skipped = - result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; - return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}`; -} - program .command("list") .description("List all package definitions") @@ -139,69 +127,34 @@ program "Output directory for build artifacts", "./dist-packages", ) + .option( + "--force", + "Rebuild and upload even when already published or unchanged", + ) .action(async (name, version, opts) => { const def = findDefinition(opts.dir, name); mkdirSync(opts.output, { recursive: true }); - if (isVersioned(def)) { - if (!version) { - throw new Error( - `Version required for versioned package "${name}". Use: registry publish ${name} `, - ); - } - - // Check if already published - const existing = await checkPackageExists( - def.registry, - def.name, - version, - ); - if (existing) { - console.log( - `Already published: ${def.registry}/${def.name}@${version}`, - ); - return; - } - - console.log(`Building ${def.registry}/${def.name}@${version}...`); - const result = await buildFromDefinition(def, version, opts.output); - console.log(`Built: ${result.path} (${formatBuilt(result)})`); - - console.log(`Publishing ${def.registry}/${def.name}@${version}...`); - await publishPackage(def.registry, def.name, version, result.path); - console.log(`Published: ${def.registry}/${def.name}@${version}`); - } else { - // Unversioned: check source_commit to skip if unchanged - const existing = await checkPackageExists( - def.registry, - def.name, - "latest", + if (isVersioned(def) && !version) { + throw new Error( + `Version required for versioned package "${name}". Use: registry publish ${name} `, ); - if (existing?.source_commit && def.source.type === "git") { - const currentCommit = getHeadCommit(def.source.url, def.source.ref); - if (currentCommit === existing.source_commit) { - console.log( - `Skipping ${def.registry}/${def.name}@latest (source unchanged: ${currentCommit.slice(0, 8)})`, - ); - return; - } - } - - console.log( - `Building ${def.registry}/${def.name}@latest (unversioned)...`, - ); - const result = await buildUnversioned(def, opts.output); - console.log(`Built: ${result.path} (${formatBuilt(result)})`); - - console.log(`Publishing ${def.registry}/${def.name}@latest...`); - await publishPackage(def.registry, def.name, "latest", result.path); - console.log(`Published: ${def.registry}/${def.name}@latest`); } + await publishDefinition( + def, + isVersioned(def) ? version : "latest", + opts.output, + { + force: opts.force, + }, + ); }); program .command("publish-all") - .description("Check all definitions, build and publish missing versions") + .description( + "Check all definitions, build and publish missing or stale versions", + ) .option("--dir ", "Registry directory", DEFAULT_REGISTRY_DIR) .option( "--output ", @@ -216,12 +169,21 @@ program "--latest ", "Only the N most recent minor versions per package", ) + .option( + "--force", + "Rebuild and upload even when already published or unchanged", + ) .action(async (opts) => { const definitions = listDefinitions(opts.dir); mkdirSync(opts.output, { recursive: true }); let succeeded = 0; let skipped = 0; + const skipReasons = new Map(); + const recordSkip = (reason: string) => { + skipped++; + skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1); + }; const failures: { id: string; error: string }[] = []; for (const def of definitions) { @@ -248,74 +210,34 @@ program for (const ver of versions) { const id = `${def.registry}/${def.name}@${ver.version}`; try { - if (isVersioned(def)) { - // Check if already published - const existing = await checkPackageExists( - def.registry, - def.name, - ver.version, - ); - if (existing) { - skipped++; - continue; - } - - console.log(`Building ${id}...`); - const result = await buildFromDefinition( - def, - ver.version, - opts.output, - ); - console.log(` Built (${formatBuilt(result)})`); - - console.log(` Publishing...`); - await publishPackage( - def.registry, - def.name, - ver.version, - result.path, - ); - console.log(` Published`); - - // Clean up build artifact to save disk space - rmSync(result.path, { force: true }); - } else { - // Unversioned: check source_commit - const existing = await checkPackageExists( - def.registry, - def.name, - "latest", - ); - if (existing?.source_commit && def.source.type === "git") { - const currentCommit = getHeadCommit( - def.source.url, - def.source.ref, - ); - if (currentCommit === existing.source_commit) { - skipped++; - continue; - } - } - - console.log(`Building ${id}...`); - const result = await buildUnversioned(def, opts.output); - console.log(` Built (${formatBuilt(result)})`); - - console.log(` Publishing...`); - await publishPackage(def.registry, def.name, "latest", result.path); - console.log(` Published`); - - rmSync(result.path, { force: true }); + const result = await publishDefinition( + def, + ver.version, + opts.output, + { + force: opts.force, + quietSkips: true, + onSkip: recordSkip, + }, + ); + if (!result) { + continue; } + // Keep failed uploads on disk for recovery; remove successful artifacts. + rmSync(result.path, { force: true }); succeeded++; } catch (err) { const message = err instanceof Error ? err.message : String(err); - // A registry can publish a version before its git tag is pushed. - // Skip (don't fail) — the next run picks it up once the tag lands. - if (isMissingRefError(message)) { - console.log(` Skipping ${id} (git tag not published yet)`); - skipped++; + // Tags can disappear after publication or not have been pushed yet. + if ( + err instanceof MissingSourceRefError || + isMissingRefError(message) + ) { + console.warn( + ` WARNING ${id}: source tag unavailable; skipping (${message})`, + ); + recordSkip("source tag unavailable"); continue; } console.error(` FAILED ${id}: ${message}`); @@ -327,7 +249,10 @@ program // Summary console.log(`\n--- Summary ---`); console.log(`Succeeded: ${succeeded}`); - console.log(`Skipped (already published): ${skipped}`); + console.log(`Skipped: ${skipped}`); + for (const [reason, count] of skipReasons) { + console.log(` ${reason}: ${count}`); + } console.log(`Failed: ${failures.length}`); if (failures.length > 0) { diff --git a/packages/registry/src/fingerprint.test.ts b/packages/registry/src/fingerprint.test.ts new file mode 100644 index 0000000..ea2f5d3 --- /dev/null +++ b/packages/registry/src/fingerprint.test.ts @@ -0,0 +1,215 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + loadDefinition, + type UnversionedDefinition, + type VersionedDefinition, +} from "./definition.js"; +import { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; +import { publicationSkipReason } from "./publication.js"; + +const git: UnversionedDefinition = { + registry: "custom", + name: "docs", + description: "Example docs", + source: { + type: "git", + url: "https://example.com/docs.git", + docs_path: "docs", + lang: "en", + }, +}; +const zip: VersionedDefinition = { + registry: "custom", + name: "docs", + versions: [ + { + versions: ["1.0"], + source: { + type: "zip", + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "en", + }, + }, + ], +}; + +describe("build fingerprints", () => { + const fingerprint = (def = git, commit = "commit", revision = "pipeline") => + createBuildFingerprint(def, "latest", commit, revision); + + it("is deterministic and includes the source commit and pipeline revision", () => { + expect(fingerprint()).toBe(fingerprint()); + expect(fingerprint(git, "changed")).not.toBe(fingerprint()); + expect(fingerprint(git, "commit", "changed")).not.toBe(fingerprint()); + expect(getIngestionRevision()).toMatch(/^[a-f0-9]{64}$/); + }); + + it.each([ + { docs_path: "guides" }, + { exclude_paths: ["private/**"] }, + { url: "https://example.com/other.git" }, + { ref: "stable" }, + { lang: "de" }, + ])("invalidates changed source settings: %j", (change) => { + expect( + fingerprint({ ...git, source: { ...git.source, ...change } }), + ).not.toBe(fingerprint()); + }); + + it("normalizes exclusion order, duplicates and omitted defaults", () => { + const a = { + ...git, + source: { ...git.source, exclude_paths: ["a/**", "b/**"] }, + }; + const b = { + ...git, + source: { ...git.source, exclude_paths: ["b/**", "a/**", "a/**"] }, + }; + expect(fingerprint(a)).toBe(fingerprint(b)); + expect( + fingerprint({ ...git, source: { ...git.source, exclude_paths: [] } }), + ).toBe(fingerprint()); + }); + + it("ignores YAML formatting and property order", () => { + const dir = mkdtempSync(join(tmpdir(), "fingerprint-yaml-")); + const path = join(dir, "docs.yaml"); + try { + writeFileSync( + path, + "name: docs\nsource:\n type: git\n url: https://example.com/docs.git\n docs_path: docs\n", + ); + const a = loadDefinition(path); + writeFileSync( + path, + "# same settings\nsource: {docs_path: docs, url: https://example.com/docs.git, lang: en, type: git}\nname: docs\n", + ); + expect(createBuildFingerprint(a, "latest", "commit", "pipeline")).toBe( + createBuildFingerprint( + loadDefinition(path), + "latest", + "commit", + "pipeline", + ), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("hashes the effective release entry rather than unrelated releases", () => { + const changed = { + ...zip, + versions: [ + ...zip.versions, + { + source: { + type: "zip" as const, + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "en", + }, + versions: ["2.0"], + }, + ], + }; + expect(createBuildFingerprint(zip, "1.0")).toBe( + createBuildFingerprint(changed, "1.0"), + ); + const resolved = { + ...zip, + versions: [ + { + versions: ["1.0"], + source: { + type: "zip" as const, + url: "https://example.com/1.0.zip", + docs_path: "docs-1.0", + lang: "en", + }, + }, + ], + }; + expect(createBuildFingerprint(zip, "1.0")).toBe( + createBuildFingerprint(resolved, "1.0"), + ); + }); + + it("ignores language settings that ZIP ingestion does not use", () => { + const changed: VersionedDefinition = { + ...zip, + versions: [ + { + versions: ["1.0"], + source: { + type: "zip", + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "de", + }, + }, + ], + }; + expect(createBuildFingerprint(changed, "1.0")).toBe( + createBuildFingerprint(zip, "1.0"), + ); + }); + + it("skips an existing explicit release only when its fingerprint matches", () => { + const existing = { + registry: "custom", + name: "docs", + version: "1.0", + build_fingerprint: createBuildFingerprint(zip, "1.0"), + }; + expect(publicationSkipReason(zip, "1.0", existing)).toBe( + "build inputs unchanged", + ); + expect( + publicationSkipReason( + { ...zip, description: "changed" }, + "1.0", + existing, + ), + ).toBeUndefined(); + expect( + publicationSkipReason(zip, "1.0", { + ...existing, + build_fingerprint: "old-pipeline", + }), + ).toBeUndefined(); + expect(publicationSkipReason(zip, "1.0", existing, true)).toBeUndefined(); + }); + + it("preserves legacy versioned skipping and allows an explicit rebuild", () => { + const legacy = { registry: "custom", name: "docs", version: "1.0" }; + expect(publicationSkipReason(zip, "1.0", legacy)).toContain( + "legacy metadata", + ); + expect(publicationSkipReason(zip, "1.0", legacy, true)).toBeUndefined(); + expect(publicationSkipReason(zip, "1.0", null)).toBeUndefined(); + }); + + it("keeps rebuilding unversioned archives with no immutable revision", () => { + const def: UnversionedDefinition = { + ...git, + source: { + type: "zip", + url: "https://example.com/latest.zip", + lang: "en", + }, + }; + expect( + publicationSkipReason(def, "latest", { + registry: "custom", + name: "docs", + version: "latest", + build_fingerprint: createBuildFingerprint(def, "latest"), + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/registry/src/fingerprint.ts b/packages/registry/src/fingerprint.ts new file mode 100644 index 0000000..1340ca5 --- /dev/null +++ b/packages/registry/src/fingerprint.ts @@ -0,0 +1,51 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { PackageDefinition } from "./definition.js"; +import { resolveBuildSource } from "./source.js"; + +export function getIngestionRevision(): string { + // Works from src (tsx) and dist. Runtime needs only the built artifact. + const { revision } = JSON.parse( + readFileSync( + new URL("../dist/ingestion-revision.json", import.meta.url), + "utf8", + ), + ) as { revision: string }; + return revision; +} + +export function createBuildFingerprint( + definition: PackageDefinition, + version: string, + sourceCommit?: string, + ingestionRevision = getIngestionRevision(), +): string { + const source = resolveBuildSource(definition, version); + if (source.type === "git" && !sourceCommit) { + throw new Error( + "A resolved Git commit is required for a build fingerprint", + ); + } + const effectiveSource = { + type: source.type, + url: source.url, + ref: source.type === "git" ? (source.ref ?? "HEAD") : undefined, + docs_path: "docs_path" in source ? source.docs_path : undefined, + exclude_paths: [...new Set(source.exclude_paths ?? [])].sort(), + lang: source.type === "git" ? (source.lang ?? "en") : undefined, + max_pages: + source.type === "html-index" ? (source.max_pages ?? 2000) : undefined, + }; + const inputs = { + schema_version: 1, + registry: definition.registry, + name: definition.name, + version, + description: definition.description, + source_url: definition.repository ?? source.url, + source: effectiveSource, + source_revision: sourceCommit ?? version, + ingestion_revision: ingestionRevision, + }; + return createHash("sha256").update(JSON.stringify(inputs)).digest("hex"); +} diff --git a/packages/registry/src/fixtures/freshness.zip b/packages/registry/src/fixtures/freshness.zip new file mode 100644 index 0000000..25da895 Binary files /dev/null and b/packages/registry/src/fixtures/freshness.zip differ diff --git a/packages/registry/src/format.ts b/packages/registry/src/format.ts new file mode 100644 index 0000000..838ff40 --- /dev/null +++ b/packages/registry/src/format.ts @@ -0,0 +1,8 @@ +import type { BuildResult } from "@neuledge/context"; + +/** Include skipped files so malformed documents are visible to operators. */ +export function formatBuilt(result: BuildResult): string { + const skipped = + result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; + return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}`; +} diff --git a/packages/registry/src/index.ts b/packages/registry/src/index.ts index fb20b2c..d3670d5 100644 --- a/packages/registry/src/index.ts +++ b/packages/registry/src/index.ts @@ -23,7 +23,9 @@ export { type ZipSource, type ZipVersionEntry, } from "./definition.js"; +export { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; export { downloadHtmlIndex } from "./html-index.js"; +export { publishDefinition } from "./publication.js"; export { checkPackageExists, type PackageMetadata, diff --git a/packages/registry/src/ingestion-revision.test.ts b/packages/registry/src/ingestion-revision.test.ts new file mode 100644 index 0000000..3cbd832 --- /dev/null +++ b/packages/registry/src/ingestion-revision.test.ts @@ -0,0 +1,187 @@ +import { execFileSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +describe("automatic ingestion revision", () => { + let root: string; + const script = resolve( + import.meta.dirname, + "../../../scripts/generate-ingestion-revision.mjs", + ); + function write(path: string, content: string) { + const file = resolve(root, path); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, content); + } + function revision() { + execFileSync(process.execPath, [script, root]); + return JSON.parse( + readFileSync( + resolve(root, "packages/registry/dist/ingestion-revision.json"), + "utf8", + ), + ) as { revision: string; files: string[]; dependencies: string[] }; + } + const lock = (version = "1.0") => + JSON.stringify({ + importers: { + "packages/registry": { + devDependencies: { typescript: { version: "5.9" } }, + }, + "packages/context": { dependencies: { parser: { version: "1.0" } } }, + }, + packages: { + "parser@1.0": { resolution: { integrity: "parser" } }, + [`helper@${version}`]: { + resolution: { integrity: `helper-${version}` }, + }, + "typescript@5.9": { resolution: { integrity: "ts" } }, + }, + snapshots: { + "parser@1.0": { dependencies: { helper: version } }, + [`helper@${version}`]: {}, + "typescript@5.9": {}, + }, + }); + + beforeEach(() => { + root = mkdtempSync(resolve(tmpdir(), "ingestion-revision-")); + write( + "packages/registry/src/build.ts", + 'import { buildPackage } from "@neuledge/context";\n', + ); + write( + "packages/context/src/index.ts", + 'export { buildPackage } from "./package-builder.js";\nexport { server } from "./server.js";\n', + ); + write( + "packages/context/src/package-builder.ts", + 'import parser from "parser";\nimport { helper } from "./helper.js";\nexport function buildPackage() { return helper(parser); }\n', + ); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x;\n", + ); + write("pnpm-lock.yaml", lock()); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + it("follows helpers and workspace exports while ignoring unrelated files and line endings", () => { + const original = revision(); + expect(original.files).toContain("packages/context/src/helper.ts"); + expect(original.files).not.toContain("packages/context/src/server.ts"); + write("README.md", "unrelated documentation"); + write("packages/context/src/helper.test.ts", "unrelated test"); + write("packages/context/src/server.ts", "unrelated server"); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x;\r\n", + ); + expect(revision().revision).toBe(original.revision); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x + 1;\n", + ); + expect(revision().revision).not.toBe(original.revision); + }); + + it("invalidates transitive dependency changes but ignores lockfile serialization", () => { + const original = revision(); + expect(original.dependencies).toContain("helper@1.0"); + write("pnpm-lock.yaml", JSON.stringify(JSON.parse(lock()), null, 2)); + expect(revision().revision).toBe(original.revision); + write("pnpm-lock.yaml", lock("2.0")); + expect(revision().revision).not.toBe(original.revision); + }); + + it("discovers newly imported helpers without a manually maintained file list", () => { + const original = revision(); + write( + "packages/context/src/helper.ts", + 'export { helper } from "./nested.js";\n', + ); + write( + "packages/context/src/nested.ts", + "export const helper = (x) => x;\n", + ); + const changed = revision(); + expect(changed.files).toContain("packages/context/src/nested.ts"); + expect(changed.revision).not.toBe(original.revision); + }); + + it.each([ + false, + true, + ])("follows peer-suffixed dependencies with suffixed package records: %s", (suffixedPackages) => { + const data = JSON.parse(lock()); + data.importers["packages/context"].dependencies.parser.version = + "1.0(peer@1.0)"; + for (const name of ["parser", "helper"]) { + const key = `${name}@1.0`; + const peerKey = `${key}(peer@1.0)`; + data.snapshots[peerKey] = data.snapshots[key]; + delete data.snapshots[key]; + if (suffixedPackages) { + data.packages[peerKey] = data.packages[key]; + delete data.packages[key]; + } + } + data.snapshots["parser@1.0(peer@1.0)"].dependencies.helper = + "1.0(peer@1.0)"; + write("pnpm-lock.yaml", JSON.stringify(data)); + const original = revision(); + expect(original.dependencies).toContain("helper@1.0(peer@1.0)"); + const key = suffixedPackages ? "helper@1.0(peer@1.0)" : "helper@1.0"; + data.packages[key].resolution.integrity = "changed-peer-dependency"; + write("pnpm-lock.yaml", JSON.stringify(data)); + expect(revision().revision).not.toBe(original.revision); + }); + + it("covers real ingestion inputs and agrees with source and built revisions", () => { + const repository = resolve(import.meta.dirname, "../../.."); + const generated = JSON.parse( + execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { generateIngestionRevision } from ${JSON.stringify(new URL("../../../scripts/generate-ingestion-revision.mjs", import.meta.url).href)}; console.log(JSON.stringify(generateIngestionRevision()));`, + ], + { cwd: repository, encoding: "utf8" }, + ), + ); + expect(generated.files).toEqual( + expect.arrayContaining([ + "packages/registry/src/build.ts", + "packages/registry/src/source.ts", + "packages/registry/src/html-index.ts", + "packages/registry/src/zip.ts", + "packages/context/src/package-builder.ts", + "packages/context/src/build.ts", + "packages/context/src/html.ts", + ]), + ); + expect(generated.files).not.toContain("packages/context/src/cli.ts"); + for (const entry of ["src/fingerprint.ts", "dist/fingerprint.js"]) { + const revision = execFileSync( + process.execPath, + [ + ...(entry.startsWith("src/") ? ["--import", "tsx"] : []), + "--input-type=module", + "--eval", + `import { getIngestionRevision } from ${JSON.stringify(new URL(entry, new URL("../", import.meta.url)).href)}; console.log(getIngestionRevision());`, + ], + { encoding: "utf8" }, + ).trim(); + expect(revision).toBe(generated.revision); + } + }); +}); diff --git a/packages/registry/src/publication.test.ts b/packages/registry/src/publication.test.ts new file mode 100644 index 0000000..1c059ea --- /dev/null +++ b/packages/registry/src/publication.test.ts @@ -0,0 +1,417 @@ +import { execFile, execFileSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer, type Server } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { initDatabase, openDatabase } from "@neuledge/context"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { getHeadCommit } from "./build.js"; +import { getIngestionRevision } from "./fingerprint.js"; +import type { PackageMetadata } from "./publish.js"; + +const run = promisify(execFile); + +describe("publication freshness through the CLI", () => { + let root: string; + let server: Server; + let url: string; + let metadata: PackageMetadata | null; + let uploads: number; + let archiveDownloads: number; + let conflict: boolean; + let legacy: boolean; + let dropUploadResponse: boolean; + let metadataReads: number; + let entryPoint: string; + let preload: string | undefined; + let repoUrl: string; + const docs = + "# Documentation\n\n## Getting started\n\nThis documentation explains how to configure and run the application with a complete example.\n"; + + beforeAll(() => initDatabase()); + beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "publication-freshness-")); + mkdirSync(join(root, "definitions", "custom"), { recursive: true }); + mkdirSync(join(root, "output")); + const repo = join(root, "repo"); + mkdirSync(join(repo, "docs"), { recursive: true }); + mkdirSync(join(repo, "guides")); + writeFileSync(join(repo, "docs", "intro.md"), docs); + writeFileSync( + join(repo, "docs", "extra.md"), + docs.replace("Getting started", "Extra"), + ); + writeFileSync( + join(repo, "guides", "intro.md"), + docs.replace("Getting started", "Guide"), + ); + const git = (args: string[]) => + execFileSync("git", args, { cwd: repo, stdio: "pipe" }); + git(["init", "--initial-branch=main"]); + git(["add", "."]); + git([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "docs", + ]); + git([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "tag", + "-a", + "v1.0", + "-m", + "release", + ]); + repoUrl = pathToFileURL(repo).href; + metadata = null; + uploads = 0; + archiveDownloads = 0; + conflict = false; + legacy = false; + dropUploadResponse = false; + metadataReads = 0; + entryPoint = "src/cli.ts"; + preload = undefined; + + server = createServer((request, response) => { + if (request.url?.startsWith("/source/")) { + archiveDownloads++; + response.end( + readFileSync(new URL("./fixtures/freshness.zip", import.meta.url)), + ); + return; + } + if (request.method !== "POST") { + metadataReads++; + response.writeHead(metadata ? 200 : 404, { + "Content-Type": "application/json", + }); + const body = + metadata && legacy + ? { + ...metadata, + build_fingerprint: undefined, + ingestion_revision: undefined, + } + : metadata; + response.end(JSON.stringify(body ?? { error: "not found" })); + return; + } + uploads++; + const buffers: Buffer[] = []; + request.on("data", (data: Buffer) => buffers.push(data)); + request.on("end", () => { + if (conflict) { + response.writeHead(409); + response.end("immutable release"); + return; + } + const dbPath = join(root, "uploaded.db"); + writeFileSync(dbPath, Buffer.concat(buffers)); + const db = openDatabase(dbPath, { readonly: true }); + try { + const values = Object.fromEntries( + ( + db.prepare("SELECT key, value FROM meta").all() as { + key: string; + value: string; + }[] + ).map((row) => [row.key, row.value]), + ); + if (!values.name || !values.version) + throw new Error("Missing package identity"); + metadata = { + registry: decodeURIComponent( + request.url?.split("/")[2] ?? "custom", + ), + name: values.name, + version: values.version, + source_commit: values.source_commit, + build_fingerprint: values.build_fingerprint, + ingestion_revision: values.ingestion_revision, + }; + } finally { + db.close(); + } + if (dropUploadResponse) { + dropUploadResponse = false; + conflict = true; + request.socket.destroy(); + return; + } + response.end("{}"); + }); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing test server address"); + url = `http://127.0.0.1:${address.port}`; + }); + afterEach(async () => { + await new Promise((done, reject) => + server.close((error) => (error ? reject(error) : done())), + ); + rmSync(root, { recursive: true, force: true }); + }); + + function currentMetadata(): PackageMetadata { + if (!metadata) throw new Error("No package was uploaded"); + return metadata; + } + + function defineGit(path = "docs", excludes: string[] = []) { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nsource:\n type: git\n url: ${repoUrl}\n ref: main\n docs_path: ${path}\n exclude_paths: ${JSON.stringify(excludes)}\n`, + ); + } + function defineArchive(path = "docs") { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nversions:\n - versions: ["1.0"]\n source:\n type: zip\n url: ${url}/source/{version}.zip\n docs_path: ${path}\n`, + ); + } + async function cli(command: string, ...args: string[]) { + try { + const result = await run( + process.execPath, + [ + ...(entryPoint.startsWith("src/") ? ["--import", "tsx"] : []), + ...(preload ? ["--import", pathToFileURL(preload).href] : []), + entryPoint, + command, + ...args, + "--dir", + join(root, "definitions"), + "--output", + join(root, "output"), + ], + { + cwd: process.cwd(), + timeout: 30_000, + env: { + ...process.env, + REGISTRY_SERVER_URL: url, + REGISTRY_PUBLISH_KEY: "test-key", + }, + }, + ); + return { status: 0, output: result.stdout + result.stderr }; + } catch (error) { + const result = error as { + code?: number; + stdout?: string; + stderr?: string; + }; + return { + status: result.code ?? 1, + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + }; + } + } + + it("skips unchanged Git, then rebuilds changed docs_path, exclusions and pipeline at the same commit", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + const commit = metadata?.source_commit; + expect(commit).toBe(getHeadCommit(repoUrl, "main")); + expect(metadata?.ingestion_revision).toBe(getIngestionRevision()); + expect(metadata?.build_fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect((await cli("publish", "docs")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + defineGit("guides"); + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(2); + expect(metadata?.source_commit).toBe(commit); + defineGit("docs", ["extra.md"]); + expect((await cli("publish", "docs")).status).toBe(0); + expect(uploads).toBe(3); + metadata = { + ...currentMetadata(), + build_fingerprint: "old-ingestion-pipeline", + }; + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(4); + }, 90_000); + + it("compares fingerprints for already-published explicit versions in both commands", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect((await cli("publish-all")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + expect(archiveDownloads).toBe(1); + defineArchive("guides"); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect(uploads).toBe(2); + metadata = { + ...currentMetadata(), + build_fingerprint: "old-ingestion-pipeline", + }; + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(3); + expect((await cli("publish-all", "--force")).status).toBe(0); + expect(uploads).toBe(4); + }, 90_000); + + it("retains legacy Git skipping and migrates with --force", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + legacy = true; + defineGit("guides"); + expect((await cli("publish", "docs")).output).toContain("legacy metadata"); + expect(uploads).toBe(1); + expect((await cli("publish", "docs", "--force")).status).toBe(0); + expect(uploads).toBe(2); + legacy = false; + expect((await cli("publish", "docs")).output).toContain( + "build inputs unchanged", + ); + }, 90_000); + + it("reports immutable-version conflicts and preserves the rebuilt artifact", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + legacy = true; + expect((await cli("publish-all")).output).toContain("legacy metadata"); + expect(uploads).toBe(1); + conflict = true; + const failed = await cli("publish-all", "--force"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("409 Conflict"); + expect(failed.output).toContain("immutable release"); + expect(failed.output).toContain("rebuilt artifact is preserved"); + expect(failed.output).toContain("Failed: 1"); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(true); + }, 90_000); + + it("resolves annotated tags to the commit recorded by a versioned Git build", async () => { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nversions:\n - min_version: "1.0"\n source:\n type: git\n url: ${repoUrl}\n docs_path: docs\n`, + ); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect(metadata?.source_commit).toBe(getHeadCommit(repoUrl, "v1.0")); + expect((await cli("publish", "docs", "1.0")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + }, 90_000); + + it("recovers a lost upload response followed by a conflict only after verifying metadata", async () => { + defineArchive(); + dropUploadResponse = true; + const published = await cli("publish-all"); + expect(published.status).toBe(0); + expect(published.output).toContain("Succeeded: 1"); + expect(uploads).toBe(2); + expect(metadataReads).toBe(2); + expect(currentMetadata().build_fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(false); + }, 60_000); + + it.each([ + "build_fingerprint", + "ingestion_revision", + ] as const)("rejects a conflicting upload with mismatched %s", async (field) => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + metadata = { ...currentMetadata(), [field]: "different-artifact" }; + conflict = true; + const failed = await cli("publish-all", "--force"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("409 Conflict"); + expect(failed.output).toContain("rebuilt artifact is preserved"); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(true); + }, 60_000); + + it("summarizes unchanged versions without printing each skip", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + const skipped = await cli("publish-all"); + expect(skipped.status).toBe(0); + expect(skipped.output).toContain("Skipped: 1"); + expect(skipped.output).toContain("build inputs unchanged: 1"); + expect(skipped.output).not.toContain("Skipping custom/docs@1.0"); + }, 60_000); + + it("does not request metadata when forcing publication", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + const previousReads = metadataReads; + expect((await cli("publish-all", "--force")).status).toBe(0); + expect(metadataReads).toBe(previousReads); + expect(uploads).toBe(2); + }, 60_000); + + it("builds identical fingerprints through the source and built CLIs", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + const sourceMetadata = currentMetadata(); + entryPoint = "dist/cli.js"; + expect((await cli("publish", "docs", "--force")).status).toBe(0); + expect(currentMetadata()).toEqual(sourceMetadata); + expect(uploads).toBe(2); + }, 60_000); + + it("warns and skips removed published tags while publishing subsequent versions", async () => { + mkdirSync(join(root, "definitions", "npm")); + writeFileSync( + join(root, "definitions", "npm", "docs.yaml"), + `name: docs\nversions:\n - min_version: "1.0"\n source:\n type: git\n url: ${repoUrl}\n docs_path: docs\n`, + ); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + execFileSync("git", ["tag", "-d", "v1.0"], { cwd: join(root, "repo") }); + preload = join(root, "versions.mjs"); + writeFileSync( + preload, + `const original = globalThis.fetch; +globalThis.fetch = (url, init) => String(url).startsWith("https://registry.npmjs.org/") + ? Promise.resolve(new Response(JSON.stringify({ versions: { "1.0": {} } }))) + : original(url, init);\n`, + ); + writeFileSync( + join(root, "definitions", "npm", "zzz.yaml"), + `name: zzz\nversions:\n - versions: ["1.0"]\n source:\n type: zip\n url: ${url}/source/{version}.zip\n docs_path: docs\n`, + ); + const published = await cli("publish-all"); + expect(published.status).toBe(0); + expect(published.output).toContain("WARNING npm/docs@1.0"); + expect(published.output).toContain("source tag unavailable: 1"); + expect(published.output).toContain("Succeeded: 1"); + expect(published.output).toContain("Skipped: 1"); + expect(published.output).toContain("Failed: 0"); + expect(uploads).toBe(2); + }, 60_000); + + it("reports unreachable source repositories as failures", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + repoUrl = pathToFileURL(join(root, "missing-repository")).href; + defineGit(); + const failed = await cli("publish-all"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("Failed: 1"); + expect(failed.output).not.toContain("source tag unavailable"); + }, 60_000); +}); diff --git a/packages/registry/src/publication.ts b/packages/registry/src/publication.ts new file mode 100644 index 0000000..b2244be --- /dev/null +++ b/packages/registry/src/publication.ts @@ -0,0 +1,86 @@ +import type { BuildResult } from "@neuledge/context"; +import { + buildFromDefinition, + buildUnversioned, + getHeadCommit, +} from "./build.js"; +import { isVersioned, type PackageDefinition } from "./definition.js"; +import { createBuildFingerprint } from "./fingerprint.js"; +import { formatBuilt } from "./format.js"; +import { + checkPackageExists, + type PackageMetadata, + publishPackage, +} from "./publish.js"; +import { resolveBuildSource } from "./source.js"; + +/** Legacy metadata retains the old skip policy until a forced migration. */ +export function publicationSkipReason( + definition: PackageDefinition, + version: string, + existing: PackageMetadata | null, + force = false, +): string | undefined { + if (force || !existing) return; + if (!existing.build_fingerprint && isVersioned(definition)) { + return "already published; legacy metadata (use --force to rebuild)"; + } + const source = resolveBuildSource(definition, version); + // An unversioned archive has no immutable source revision to compare. + if (!isVersioned(definition) && source.type === "zip") return; + const commit = + source.type === "git" ? getHeadCommit(source.url, source.ref) : undefined; + if (existing.build_fingerprint) { + if ( + existing.build_fingerprint === + createBuildFingerprint(definition, version, commit) + ) { + return "build inputs unchanged"; + } + } else if (commit && commit === existing.source_commit) { + return "source unchanged; legacy metadata (use --force to rebuild)"; + } +} + +export async function publishDefinition( + definition: PackageDefinition, + version: string, + outputDir: string, + options: { + force?: boolean; + log?: (message: string) => void; + quietSkips?: boolean; + onSkip?: (reason: string) => void; + } = {}, +): Promise { + const log = options.log ?? console.log; + const id = `${definition.registry}/${definition.name}@${version}`; + const existing = options.force + ? null + : await checkPackageExists(definition.registry, definition.name, version); + const reason = publicationSkipReason( + definition, + version, + existing, + options.force, + ); + if (reason) { + options.onSkip?.(reason); + if (!options.quietSkips) log(`Skipping ${id} (${reason})`); + return; + } + log(`Building ${id}...`); + const result = isVersioned(definition) + ? await buildFromDefinition(definition, version, outputDir) + : await buildUnversioned(definition, outputDir); + log(`Built: ${result.path} (${formatBuilt(result)})`); + log(`Publishing ${id}...`); + await publishPackage( + definition.registry, + definition.name, + version, + result.path, + ); + log(`Published: ${id}`); + return result; +} diff --git a/packages/registry/src/publish.test.ts b/packages/registry/src/publish.test.ts index 33b3f93..88163c8 100644 --- a/packages/registry/src/publish.test.ts +++ b/packages/registry/src/publish.test.ts @@ -70,8 +70,21 @@ describe("publish", () => { process.env.REGISTRY_SERVER_URL = `http://127.0.0.1:${port}`; process.env.REGISTRY_PUBLISH_KEY = "test-key"; + const published = publishPackage("npm", "preact", "latest", dbPath); + await expect(published).rejects.toThrow(/403 Forbidden — bad key/); + await expect(published).rejects.toThrow( + `rebuilt artifact is preserved at ${dbPath}`, + ); + }); + + it("does not treat a missing upload endpoint as a successful publication", async () => { + const { server, url, hits } = await flakyServer(0, 404); + running = server; + process.env.REGISTRY_SERVER_URL = url; + process.env.REGISTRY_PUBLISH_KEY = "test-key"; await expect( publishPackage("npm", "preact", "latest", dbPath), - ).rejects.toThrow(/403 Forbidden — bad key/); + ).rejects.toThrow("404"); + expect(hits()).toBe(1); }); }); diff --git a/packages/registry/src/publish.ts b/packages/registry/src/publish.ts index 51bf7b4..3f316a4 100644 --- a/packages/registry/src/publish.ts +++ b/packages/registry/src/publish.ts @@ -7,10 +7,20 @@ */ import { readFileSync } from "node:fs"; +import { initDatabase, openDatabase } from "@neuledge/context"; import pRetry, { AbortError } from "p-retry"; const DEFAULT_SERVER_URL = "https://api.context.neuledge.com"; +class RegistryRequestError extends Error { + constructor( + public readonly status: number, + message: string, + ) { + super(message); + } +} + /** * The registry server occasionally drops a connection or returns 5xx under load. * A single blip used to fail the whole nightly publish — 48 packages succeed and @@ -25,10 +35,12 @@ function requestWithRetry( return pRetry( async () => { const response = await fetch(url, init); - if (response.ok || response.status === 404) return response; + if (response.ok || (response.status === 404 && init.method !== "POST")) + return response; const body = await response.text().catch(() => ""); - const error = new Error( + const error = new RegistryRequestError( + response.status, `${describe()}: ${response.status} ${response.statusText}${body ? ` — ${body}` : ""}`, ); if (response.status < 500) throw new AbortError(error); @@ -57,6 +69,8 @@ export interface PackageMetadata { name: string; version: string; source_commit?: string; + build_fingerprint?: string; + ingestion_revision?: string; } /** @@ -101,19 +115,69 @@ export async function publishPackage( const url = `${getServerUrl()}/packages/${encodeURIComponent(registry)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`; const body = readFileSync(dbPath); - // Re-uploading an identical package is safe: the server keys on - // registry/name/version, so a retry after a dropped connection overwrites - // rather than duplicating. - await requestWithRetry( - url, - { - method: "POST", - headers: { - Authorization: `Bearer ${getPublishKey()}`, - "Content-Type": "application/octet-stream", + try { + await requestWithRetry( + url, + { + method: "POST", + headers: { + Authorization: `Bearer ${getPublishKey()}`, + "Content-Type": "application/octet-stream", + }, + body, }, - body, - }, - () => `Failed to publish ${registry}/${name}@${version}`, + () => `Failed to publish ${registry}/${name}@${version}`, + ); + } catch (error) { + let message = error instanceof Error ? error.message : String(error); + if (error instanceof RegistryRequestError && error.status === 409) { + try { + if (await matchesPublishedArtifact(registry, name, version, dbPath)) + return; + } catch (verificationError) { + message += `. Could not verify published metadata: ${verificationError instanceof Error ? verificationError.message : String(verificationError)}`; + } + message += + ". The registry rejected replacement of this published version. Use a registry-supported replacement or artifact revision; --force cannot override the server's policy"; + } + throw new Error( + `${message}. The rebuilt artifact is preserved at ${dbPath}.`, + { + cause: error, + }, + ); + } +} + +/** A lost upload response is recoverable only with matching artifact metadata. */ +async function matchesPublishedArtifact( + registry: string, + name: string, + version: string, + dbPath: string, +): Promise { + await initDatabase(); + const db = openDatabase(dbPath, { readonly: true }); + let values: Record; + try { + values = Object.fromEntries( + ( + db.prepare("SELECT key, value FROM meta").all() as { + key: string; + value: string; + }[] + ).map(({ key, value }) => [key, value]), + ); + } finally { + db.close(); + } + if (!values.build_fingerprint || !values.ingestion_revision) return false; + const published = await checkPackageExists(registry, name, version); + return ( + published?.registry === registry && + published.name === name && + published.version === version && + published.build_fingerprint === values.build_fingerprint && + published.ingestion_revision === values.ingestion_revision ); } diff --git a/packages/registry/src/source.ts b/packages/registry/src/source.ts new file mode 100644 index 0000000..7546744 --- /dev/null +++ b/packages/registry/src/source.ts @@ -0,0 +1,32 @@ +import { + constructTag, + isGitVersionEntry, + isVersioned, + type PackageDefinition, + resolveUrl, + resolveVersionEntry, +} from "./definition.js"; + +/** Resolve the same release inputs for both building and fingerprinting. */ +export function resolveBuildSource( + definition: PackageDefinition, + version: string, +) { + if (!isVersioned(definition)) return definition.source; + const entry = resolveVersionEntry(definition, version); + if (!entry) { + throw new Error( + `No version entry matches ${version} in ${definition.name}`, + ); + } + if (isGitVersionEntry(entry)) { + return { ...entry.source, ref: constructTag(entry.tag_pattern, version) }; + } + return { + ...entry.source, + url: resolveUrl(entry.source.url, version), + ...(entry.source.type === "zip" && entry.source.docs_path + ? { docs_path: resolveUrl(entry.source.docs_path, version) } + : {}), + }; +} diff --git a/registry/README.md b/registry/README.md index 4aad632..837cfce 100644 --- a/registry/README.md +++ b/registry/README.md @@ -160,7 +160,8 @@ indexed once, which avoids duplicate man-page aliases. Pinned downloads are reused from `.cache/context/html-index` across builds. Delete that directory to refetch a corrected upstream release. The nightly -publisher skips releases already present in the registry. Check the publisher's crawling policy +publisher compares build fingerprints to skip unchanged releases. Packages with +legacy metadata retain the skip behavior described below. Check the publisher's crawling policy before adding an index source. For systemd, `systemd/systemd` contains the versioned reference manuals and @@ -197,6 +198,65 @@ content you want is already isolated in its own directory. Markdown (`.md`, `.mdx`), HTML, AsciiDoc (`.adoc`) and reStructuredText (`.rst`). Point `docs_path` at the directory holding them; everything else in the repo is ignored. +## Publication freshness and rebuilding + +`registry publish` and `registry publish-all` compare `build_fingerprint` metadata +before building. A matching fingerprint skips work; changed build inputs cause a +rebuild and an upload to the same package version. Inputs include the actual Git +commit (or pinned explicit release), the effective source settings and package +metadata, and an automatically generated ingestion revision. `docs_path`, +exclusions, Git language filters, source URL/ref, description, or ingestion changes can +therefore rebuild a package even when the source commit is unchanged. YAML key +order, comments, exclusion order/duplicates, and unrelated release entries do not +change the fingerprint. + +The ingestion revision is generated during `pnpm build` and before the +`pnpm --filter @neuledge/registry registry ...` source CLI runs. The generator starts +from the registry builder, follows runtime imports and the context APIs it uses, +and hashes those modules plus their locked runtime dependencies, including +transitive and optional dependencies and the compiler version. New helper imports +are discovered automatically. Repository documentation, tests, and unrelated CLI +or server modules are excluded. Line endings and lockfile key order are normalized. +Harmless ingestion refactoring can conservatively trigger a rebuild. + +The generated revision ships in `dist/ingestion-revision.json`. The built publisher +needs neither Git history nor TypeScript source files to compute the ingestion +revision; Git commands are still needed to resolve and build Git sources. Rebuild +after editing ingestion code if running `src/cli.ts` directly instead of the +package script. Unversioned ZIP sources continue rebuilding on each publication, +because their URLs provide no immutable source revision. Pinned archives/HTML +releases are assumed immutable; refreshing upstream download caches is separate +from derived-package freshness. + +Servers must expose the uploaded database's `build_fingerprint` on the metadata +endpoint (see [SERVER_SPEC.md](../SERVER_SPEC.md)). Without that field, versioned +packages retain the previous existence-based skip behavior, unversioned Git +packages retain commit comparison, and skips explicitly mention legacy metadata. +Use `--force` to migrate old packages after the server supports fingerprints: + +```bash +pnpm --filter @neuledge/registry registry publish [version] --force +pnpm --filter @neuledge/registry registry publish-all --force +``` + +`--force` bypasses freshness checks and rebuilds; it does not override the server's +replacement policy. Both automatic rebuilds and forced builds upload to the same +registry/name/version. Servers that allow replacement accept the new artifact; +servers that return HTTP 409 are checked for a matching package identity, +`build_fingerprint`, and `ingestion_revision`. A match confirms that the upload +already landed, even if its response was lost and the retry returned 409. Missing +or mismatched metadata produces a clear failure. All failed uploads report the +preserved `.db` path; successful `publish-all` uploads remove the local artifact. + +`--force` rebuilds the derived package using the current source cache; it does not +refresh downloaded HTML. Delete `.cache/context/html-index` before rebuilding if +the publisher corrected an existing release's pages. + +Bulk publishing summarizes skip reasons instead of logging every unchanged +version. Missing Git tags are warned about and skipped, including tags removed +after a package was published. Unreachable repositories and other transport +failures remain errors. + ## Before opening a PR Build your definition locally to confirm it produces real content: diff --git a/scripts/generate-ingestion-revision.mjs b/scripts/generate-ingestion-revision.mjs new file mode 100644 index 0000000..553044b --- /dev/null +++ b/scripts/generate-ingestion-revision.mjs @@ -0,0 +1,203 @@ +/** Build-time fingerprint of ingestion modules and their locked runtime dependencies. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptPath = fileURLToPath(import.meta.url); +const repository = resolve(dirname(scriptPath), ".."); +const require = createRequire( + resolve(repository, "packages/registry/package.json"), +); +const ts = require("typescript"); +const { parse } = require("yaml"); + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]), + ); + } + return value; +} + +function lockedDependency(state, name, version) { + const key = `${name}@${version}`; + if (state.dependencies.has(key)) return; + const snapshot = state.lock.snapshots[key]; + // pnpm snapshots retain peer suffixes; package records normally omit them. + // Also accept suffix-bearing package records rather than assuming one layout. + const pkg = + state.lock.packages[key] ?? state.lock.packages[key.replace(/\(.*$/, "")]; + if (!snapshot || !pkg) + throw new Error(`Missing locked ingestion dependency: ${key}`); + state.dependencies.set(key, { package: pkg, snapshot }); + for (const [child, childVersion] of Object.entries({ + ...snapshot.dependencies, + ...snapshot.optionalDependencies, + })) { + lockedDependency(state, child, childVersion); + } +} + +function sourceFile(file, content = readFileSync(file, "utf8")) { + return ts.createSourceFile(file, content, ts.ScriptTarget.Latest, true); +} + +function followContextExports(state, names) { + const index = resolve(state.root, "packages/context/src/index.ts"); + const unresolved = names && new Set(names); + for (const statement of sourceFile(index).statements) { + if ( + !ts.isExportDeclaration(statement) || + statement.isTypeOnly || + !statement.moduleSpecifier + ) + continue; + const exported = + statement.exportClause && ts.isNamedExports(statement.exportClause) + ? statement.exportClause.elements + .filter((item) => !item.isTypeOnly) + .map((item) => item.name.text) + : undefined; + if (!names || !exported || exported.some((name) => names.includes(name))) { + followImport(state, index, statement.moduleSpecifier.text); + for (const name of exported ?? []) unresolved?.delete(name); + } + } + if (unresolved?.size) + throw new Error( + `Unresolved ingestion exports: ${[...unresolved].join(", ")}`, + ); +} + +function followImport(state, file, specifier, names) { + if (specifier.startsWith("node:")) return; + if (specifier.startsWith(".")) { + visitSource( + state, + resolve(dirname(file), specifier.replace(/\.js$/, ".ts")), + ); + return; + } + const packageName = specifier.startsWith("@") + ? specifier.split("/").slice(0, 2).join("/") + : specifier.split("/")[0]; + if (packageName === "@neuledge/context") { + if (specifier === packageName) { + followContextExports(state, names); + } else { + visitSource( + state, + resolve( + state.root, + "packages/context/src", + specifier.slice(packageName.length + 1).replace(/\.js$/, ".ts"), + ), + ); + } + return; + } + const importer = relative(state.root, file) + .replaceAll("\\", "/") + .split("/") + .slice(0, 2) + .join("/"); + const manifest = state.lock.importers[importer]; + const entry = + manifest.dependencies?.[packageName] ?? + manifest.optionalDependencies?.[packageName]; + if (!entry) + throw new Error( + `Undeclared ingestion dependency: ${packageName} in ${file}`, + ); + lockedDependency(state, packageName, entry.version); +} + +function visitNode(state, file, source, node) { + if (ts.isImportDeclaration(node)) { + const clause = node.importClause; + if (clause?.isTypeOnly) return; + const bindings = clause?.namedBindings; + const names = + bindings && ts.isNamedImports(bindings) + ? bindings.elements + .filter((item) => !item.isTypeOnly) + .map((item) => (item.propertyName ?? item.name).text) + : undefined; + if (names?.length === 0 && !clause?.name) return; + followImport(state, file, node.moduleSpecifier.text, names); + } else if ( + ts.isExportDeclaration(node) && + !node.isTypeOnly && + node.moduleSpecifier + ) { + followImport(state, file, node.moduleSpecifier.text); + } else if ( + ts.isCallExpression(node) && + node.arguments[0] && + ts.isStringLiteral(node.arguments[0]) + ) { + const expression = node.expression.getText(source); + if ( + node.expression.kind === ts.SyntaxKind.ImportKeyword || + /^(?:_?require)(?:\.resolve)?$/.test(expression) + ) { + followImport(state, file, node.arguments[0].text); + } + } + ts.forEachChild(node, (child) => visitNode(state, file, source, child)); +} + +function visitSource(state, file) { + const path = relative(state.root, file).replaceAll("\\", "/"); + if (state.files.has(path)) return; + const content = readFileSync(file, "utf8").replaceAll("\r\n", "\n"); + state.files.set(path, content); + const source = sourceFile(file, content); + visitNode(state, file, source, source); +} + +export function generateIngestionRevision(root = repository) { + const state = { + root, + files: new Map(), + dependencies: new Map(), + lock: parse(readFileSync(resolve(root, "pnpm-lock.yaml"), "utf8")), + }; + visitSource(state, resolve(root, "packages/registry/src/build.ts")); + lockedDependency( + state, + "typescript", + state.lock.importers["packages/registry"].devDependencies.typescript + .version, + ); + // Include the algorithm itself so changes in what we hash also invalidate. + state.files.set( + "scripts/generate-ingestion-revision.mjs", + readFileSync(scriptPath, "utf8").replaceAll("\r\n", "\n"), + ); + const inputs = canonical({ + files: Object.fromEntries(state.files), + dependencies: Object.fromEntries(state.dependencies), + }); + return { + revision: createHash("sha256").update(JSON.stringify(inputs)).digest("hex"), + files: Object.keys(inputs.files), + dependencies: Object.keys(inputs.dependencies), + }; +} + +if (process.argv[1] && resolve(process.argv[1]) === scriptPath) { + const root = process.argv[2] ? resolve(process.argv[2]) : repository; + const output = resolve(root, "packages/registry/dist"); + mkdirSync(output, { recursive: true }); + writeFileSync( + resolve(output, "ingestion-revision.json"), + `${JSON.stringify(generateIngestionRevision(root), null, 2)}\n`, + ); +} diff --git a/turbo.json b/turbo.json index 8b1f417..e368888 100644 --- a/turbo.json +++ b/turbo.json @@ -5,6 +5,14 @@ "dependsOn": ["^build"], "outputs": ["dist/**"] }, + "@neuledge/registry#build": { + "dependsOn": ["^build"], + "outputs": ["dist/**"], + "inputs": [ + "$TURBO_DEFAULT$", + "$TURBO_ROOT$/scripts/generate-ingestion-revision.mjs" + ] + }, "test": { "dependsOn": ["build"] },