From 0ea0b0f8f098860c99eab9a37f5faab12ffcab66 Mon Sep 17 00:00:00 2001 From: Martin Beckert Date: Fri, 2 Oct 2026 10:28:59 +0200 Subject: [PATCH 1/2] fix(registry): invalidate stale documentation builds automatically --- .changeset/fresh-documentation-inputs.md | 5 + SERVER_SPEC.md | 18 +- packages/context/src/package-builder.ts | 10 + packages/registry/package.json | 4 +- packages/registry/src/build.ts | 67 +++- packages/registry/src/cli.ts | 156 +++------ packages/registry/src/fingerprint.test.ts | 195 +++++++++++ packages/registry/src/fingerprint.ts | 80 +++++ packages/registry/src/fixtures/freshness.zip | Bin 0 -> 632 bytes packages/registry/src/index.ts | 2 + .../registry/src/ingestion-revision.test.ts | 119 +++++++ packages/registry/src/publication.test.ts | 303 ++++++++++++++++++ packages/registry/src/publication.ts | 84 +++++ packages/registry/src/publish.ts | 38 ++- registry/README.md | 51 ++- scripts/generate-ingestion-revision.mjs | 201 ++++++++++++ turbo.json | 1 + 17 files changed, 1186 insertions(+), 148 deletions(-) create mode 100644 .changeset/fresh-documentation-inputs.md create mode 100644 packages/registry/src/fingerprint.test.ts create mode 100644 packages/registry/src/fingerprint.ts create mode 100644 packages/registry/src/fixtures/freshness.zip create mode 100644 packages/registry/src/ingestion-revision.test.ts create mode 100644 packages/registry/src/publication.test.ts create mode 100644 packages/registry/src/publication.ts create mode 100644 scripts/generate-ingestion-revision.mjs diff --git a/.changeset/fresh-documentation-inputs.md b/.changeset/fresh-documentation-inputs.md new file mode 100644 index 0000000..b370000 --- /dev/null +++ b/.changeset/fresh-documentation-inputs.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Allow package builders to store build fingerprints and automatic ingestion revisions in package metadata for registry freshness checks. diff --git a/SERVER_SPEC.md b/SERVER_SPEC.md index d90a57f..19132f1 100644 --- a/SERVER_SPEC.md +++ b/SERVER_SPEC.md @@ -67,7 +67,7 @@ Returns an empty array `[]` when no packages match. Results are sorted by versio GET /packages/// ``` -Check if a package version exists and return its metadata. Used by the publish pipeline for idempotency (skip already-published versions) and by unversioned packages to compare `source_commit`. +Check if a package version exists and return its metadata. The publish pipeline compares `build_fingerprint` to skip packages whose source, resolved definition, and ingestion implementation are unchanged. **Response `200 OK`:** @@ -76,7 +76,9 @@ Check if a package version exists and return its metadata. Used by the publish p "registry": "npm", "name": "nextjs", "version": "15.1.0", - "source_commit": "abc1234" + "source_commit": "abc1234", + "build_fingerprint": "", + "ingestion_revision": "" } ``` @@ -87,7 +89,11 @@ Check if a package version exists and return its metadata. Used by the publish p | `registry` | string | Package manager | | `name` | string | Package name | | `version` | string | Semver version or `"latest"` | -| `source_commit` | string? | Git SHA for unversioned packages | +| `source_commit` | string? | Git commit SHA for versioned or unversioned Git packages | +| `build_fingerprint` | string? | Opaque SHA-256 copied from the uploaded database's `meta` table | +| `ingestion_revision` | string? | Automatic ingestion revision copied from the uploaded database's `meta` table | + +Servers supporting automatic freshness checks must extract and persist these optional metadata values on every upload, including replacements, and return them here. The client treats fingerprints as opaque; servers should not recompute them. Older packages and servers can omit these fields: versioned packages retain existence-based skipping, unversioned Git packages retain `source_commit` comparison, and unversioned ZIP packages continue rebuilding. The CLI identifies legacy skips and offers `--force` for migration. Returning the fingerprint after a successful rebuild enables subsequent automatic checks; a server that continues omitting it retains the legacy behavior. **Response `404 Not Found`:** @@ -135,6 +141,8 @@ Upload a new documentation package. Requires a valid API key. **Response `409 Conflict`** — Package version already exists (optional; servers may also allow overwrites). +The publisher uploads stale or explicitly forced builds to the same registry/name/version. `--force` bypasses client freshness checks only; it does not grant replacement permission or change the API request. If this endpoint rejects replacement with HTTP 409, the publisher reports the conflict and preserves the rebuilt local `.db`. Updating an immutable release requires a server-supported replacement or artifact revision policy; no alternative version is invented by the client. + ## Package format (`.db` file) Packages are SQLite databases with the following schema: @@ -178,7 +186,9 @@ CREATE VIRTUAL TABLE chunks_fts USING fts5( |-----|-------------| | `description` | Short package description | | `source_url` | URL of the source repository | -| `source_commit` | Git commit SHA (used for unversioned packages to detect changes) | +| `source_commit` | Actual Git commit SHA used to build the package | +| `build_fingerprint` | Deterministic SHA-256 of resolved package build inputs | +| `ingestion_revision` | SHA-256 generated from ingestion code and locked dependencies at build time | ## Error format diff --git a/packages/context/src/package-builder.ts b/packages/context/src/package-builder.ts index a336c75..13b3bc6 100644 --- a/packages/context/src/package-builder.ts +++ b/packages/context/src/package-builder.ts @@ -23,6 +23,10 @@ export interface PackageBuildOptions { sourceUrl?: string; /** Git commit SHA used to build this package (for skip-if-unchanged checks) */ sourceCommit?: string; + /** Deterministic registry build inputs, used for publication freshness. */ + buildFingerprint?: string; + /** Automatically generated revision of the ingestion implementation. */ + ingestionRevision?: string; } export interface MarkdownFile { @@ -611,6 +615,12 @@ export function buildPackage( if (options.sourceCommit) { insertMeta.run("source_commit", options.sourceCommit); } + if (options.buildFingerprint) { + insertMeta.run("build_fingerprint", options.buildFingerprint); + } + if (options.ingestionRevision) { + insertMeta.run("ingestion_revision", options.ingestionRevision); + } // Parse and insert chunks const insertChunk = db.prepare(` diff --git a/packages/registry/package.json b/packages/registry/package.json index de8d8d1..b211a7a 100644 --- a/packages/registry/package.json +++ b/packages/registry/package.json @@ -14,10 +14,10 @@ ".": "./dist/index.js" }, "scripts": { - "build": "rimraf --glob dist/** && tsc -p tsconfig.build.json", + "build": "rimraf --glob dist/** && node ../../scripts/generate-ingestion-revision.mjs && tsc -p tsconfig.build.json", "test": "vitest run", "test:watch": "vitest", - "registry": "tsx src/cli.ts", + "registry": "node ../../scripts/generate-ingestion-revision.mjs && tsx src/cli.ts", "test-registry": "tsx src/test-registry.ts", "lint": "npx biome ci --error-on-warnings", "fix": "npx biome check --write" diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts index adb70ba..f86526b 100644 --- a/packages/registry/src/build.ts +++ b/packages/registry/src/build.ts @@ -8,22 +8,25 @@ * (clone default branch) definitions. Supports git, zip and HTML index sources. */ -import { execSync } from "node:child_process"; +import { execFileSync } from "node:child_process"; import { join } from "node:path"; import { type BuildResult, buildPackage, cloneRepository, + initDatabase, readLocalDocsFiles, } from "@neuledge/context"; import { constructTag, isGitVersionEntry, + type PackageDefinition, resolveUrl, resolveVersionEntry, type UnversionedDefinition, type VersionedDefinition, } from "./definition.js"; +import { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; import { excludeFiles } from "./glob.js"; import { downloadHtmlIndex } from "./html-index.js"; import { downloadAndExtractZip } from "./zip.js"; @@ -44,19 +47,52 @@ export function getHeadCommit(url: string, ref?: string): string { // Must match the ref the package is built from. Asking for HEAD while building // a branch compares two unrelated commits, so the skip-if-unchanged check never // fires and the package is rebuilt and republished on every run. - const output = execSync(`git ls-remote ${url} ${ref ?? "HEAD"}`, { - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - }).trim(); + const requested = ref ?? "HEAD"; + const output = execFileSync( + "git", + ["ls-remote", url, requested, `${requested}^{}`], + { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }, + ).trim(); - // Format: "\tHEAD" - const sha = output.split("\t")[0]; + const refs = new Map( + output.split("\n").map((line) => { + const [sha, name] = line.split("\t"); + return [name, sha]; + }), + ); + // Match clone --branch: prefer a branch, and peel annotated tags to commits. + const sha = + refs.get(`refs/heads/${requested}`) ?? + refs.get(`${requested}^{}`) ?? + refs.get(`refs/tags/${requested}^{}`) ?? + refs.get(requested) ?? + refs.get(`refs/tags/${requested}`); if (!sha) { - throw new Error(`Failed to get HEAD commit for ${url}`); + throw new Error(`Git reference ${requested} not found in upstream ${url}`); } return sha; } +function fingerprintOptions( + definition: PackageDefinition, + version: string, + sourceCommit?: string, +) { + const ingestionRevision = getIngestionRevision(); + return { + ingestionRevision, + buildFingerprint: createBuildFingerprint( + definition, + version, + sourceCommit, + ingestionRevision, + ), + }; +} + /** * Build a .db package for a specific version of a versioned definition. */ @@ -65,6 +101,7 @@ export async function buildFromDefinition( version: string, outputDir: string, ): Promise { + await initDatabase(); const entry = resolveVersionEntry(definition, version); if (!entry) { throw new Error( @@ -115,6 +152,7 @@ export async function buildFromDefinition( version, description: definition.description, sourceUrl: definition.repository ?? url, + ...fingerprintOptions(definition, version), }); return { @@ -134,6 +172,7 @@ export async function buildUnversioned( definition: UnversionedDefinition, outputDir: string, ): Promise { + await initDatabase(); const version = "latest"; const { source } = definition; const safeName = definition.name.replace(/\//g, "-"); @@ -157,6 +196,7 @@ export async function buildUnversioned( version, description: definition.description, sourceUrl: definition.repository ?? source.url, + ...fingerprintOptions(definition, version), }); return { @@ -172,7 +212,7 @@ export async function buildUnversioned( try { // Get the commit SHA of the cloned HEAD - const sourceCommit = execSync("git rev-parse HEAD", { + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: tempDir, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], @@ -201,6 +241,7 @@ export async function buildUnversioned( description: definition.description, sourceUrl: definition.repository ?? source.url, sourceCommit, + ...fingerprintOptions(definition, version, sourceCommit), }); return { @@ -229,6 +270,11 @@ function buildFromGit( const { tempDir, cleanup } = cloneRepository(url, tag); try { + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: tempDir, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }).trim(); // Filter before the emptiness check, so an over-broad exclude_paths fails // loudly here instead of publishing an empty package. const files = excludeFiles( @@ -246,6 +292,8 @@ function buildFromGit( version, description: definition.description, sourceUrl: definition.repository ?? url, + sourceCommit, + ...fingerprintOptions(definition, version, sourceCommit), }); return { @@ -253,6 +301,7 @@ function buildFromGit( name: definition.name, registry: definition.registry, version, + sourceCommit, }; } finally { cleanup(); diff --git a/packages/registry/src/cli.ts b/packages/registry/src/cli.ts index b92e6dd..d8e6867 100644 --- a/packages/registry/src/cli.ts +++ b/packages/registry/src/cli.ts @@ -9,17 +9,13 @@ import { mkdirSync, rmSync } from "node:fs"; import { resolve } from "node:path"; import { type BuildResult, isMissingRefError } from "@neuledge/context"; import { Command } from "commander"; -import { - buildFromDefinition, - buildUnversioned, - getHeadCommit, -} from "./build.js"; +import { buildFromDefinition, buildUnversioned } from "./build.js"; import { isExplicitVersionEntry, isVersioned, listDefinitions, } from "./definition.js"; -import { checkPackageExists, publishPackage } from "./publish.js"; +import { publishDefinition } from "./publication.js"; import { type AvailableVersion, discoverVersions } from "./version-check.js"; const DEFAULT_REGISTRY_DIR = resolve( @@ -139,69 +135,34 @@ program "Output directory for build artifacts", "./dist-packages", ) + .option( + "--force", + "Rebuild and upload even when already published or unchanged", + ) .action(async (name, version, opts) => { const def = findDefinition(opts.dir, name); mkdirSync(opts.output, { recursive: true }); - if (isVersioned(def)) { - if (!version) { - throw new Error( - `Version required for versioned package "${name}". Use: registry publish ${name} `, - ); - } - - // Check if already published - const existing = await checkPackageExists( - def.registry, - def.name, - version, - ); - if (existing) { - console.log( - `Already published: ${def.registry}/${def.name}@${version}`, - ); - return; - } - - console.log(`Building ${def.registry}/${def.name}@${version}...`); - const result = await buildFromDefinition(def, version, opts.output); - console.log(`Built: ${result.path} (${formatBuilt(result)})`); - - console.log(`Publishing ${def.registry}/${def.name}@${version}...`); - await publishPackage(def.registry, def.name, version, result.path); - console.log(`Published: ${def.registry}/${def.name}@${version}`); - } else { - // Unversioned: check source_commit to skip if unchanged - const existing = await checkPackageExists( - def.registry, - def.name, - "latest", + if (isVersioned(def) && !version) { + throw new Error( + `Version required for versioned package "${name}". Use: registry publish ${name} `, ); - if (existing?.source_commit && def.source.type === "git") { - const currentCommit = getHeadCommit(def.source.url, def.source.ref); - if (currentCommit === existing.source_commit) { - console.log( - `Skipping ${def.registry}/${def.name}@latest (source unchanged: ${currentCommit.slice(0, 8)})`, - ); - return; - } - } - - console.log( - `Building ${def.registry}/${def.name}@latest (unversioned)...`, - ); - const result = await buildUnversioned(def, opts.output); - console.log(`Built: ${result.path} (${formatBuilt(result)})`); - - console.log(`Publishing ${def.registry}/${def.name}@latest...`); - await publishPackage(def.registry, def.name, "latest", result.path); - console.log(`Published: ${def.registry}/${def.name}@latest`); } + await publishDefinition( + def, + isVersioned(def) ? version : "latest", + opts.output, + { + force: opts.force, + }, + ); }); program .command("publish-all") - .description("Check all definitions, build and publish missing versions") + .description( + "Check all definitions, build and publish missing or stale versions", + ) .option("--dir ", "Registry directory", DEFAULT_REGISTRY_DIR) .option( "--output ", @@ -216,6 +177,10 @@ program "--latest ", "Only the N most recent minor versions per package", ) + .option( + "--force", + "Rebuild and upload even when already published or unchanged", + ) .action(async (opts) => { const definitions = listDefinitions(opts.dir); mkdirSync(opts.output, { recursive: true }); @@ -248,65 +213,20 @@ program for (const ver of versions) { const id = `${def.registry}/${def.name}@${ver.version}`; try { - if (isVersioned(def)) { - // Check if already published - const existing = await checkPackageExists( - def.registry, - def.name, - ver.version, - ); - if (existing) { - skipped++; - continue; - } - - console.log(`Building ${id}...`); - const result = await buildFromDefinition( - def, - ver.version, - opts.output, - ); - console.log(` Built (${formatBuilt(result)})`); - - console.log(` Publishing...`); - await publishPackage( - def.registry, - def.name, - ver.version, - result.path, - ); - console.log(` Published`); - - // Clean up build artifact to save disk space - rmSync(result.path, { force: true }); - } else { - // Unversioned: check source_commit - const existing = await checkPackageExists( - def.registry, - def.name, - "latest", - ); - if (existing?.source_commit && def.source.type === "git") { - const currentCommit = getHeadCommit( - def.source.url, - def.source.ref, - ); - if (currentCommit === existing.source_commit) { - skipped++; - continue; - } - } - - console.log(`Building ${id}...`); - const result = await buildUnversioned(def, opts.output); - console.log(` Built (${formatBuilt(result)})`); - - console.log(` Publishing...`); - await publishPackage(def.registry, def.name, "latest", result.path); - console.log(` Published`); - - rmSync(result.path, { force: true }); + const result = await publishDefinition( + def, + ver.version, + opts.output, + { + force: opts.force, + }, + ); + if (!result) { + skipped++; + continue; } + // Keep failed uploads on disk for recovery; remove successful artifacts. + rmSync(result.path, { force: true }); succeeded++; } catch (err) { @@ -327,7 +247,7 @@ program // Summary console.log(`\n--- Summary ---`); console.log(`Succeeded: ${succeeded}`); - console.log(`Skipped (already published): ${skipped}`); + console.log(`Skipped (up to date or legacy metadata): ${skipped}`); console.log(`Failed: ${failures.length}`); if (failures.length > 0) { diff --git a/packages/registry/src/fingerprint.test.ts b/packages/registry/src/fingerprint.test.ts new file mode 100644 index 0000000..0767f3e --- /dev/null +++ b/packages/registry/src/fingerprint.test.ts @@ -0,0 +1,195 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + loadDefinition, + type UnversionedDefinition, + type VersionedDefinition, +} from "./definition.js"; +import { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; +import { publicationSkipReason } from "./publication.js"; + +const git: UnversionedDefinition = { + registry: "custom", + name: "docs", + description: "Example docs", + source: { + type: "git", + url: "https://example.com/docs.git", + docs_path: "docs", + lang: "en", + }, +}; +const zip: VersionedDefinition = { + registry: "custom", + name: "docs", + versions: [ + { + versions: ["1.0"], + source: { + type: "zip", + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "en", + }, + }, + ], +}; + +describe("build fingerprints", () => { + const fingerprint = (def = git, commit = "commit", revision = "pipeline") => + createBuildFingerprint(def, "latest", commit, revision); + + it("is deterministic and includes the source commit and pipeline revision", () => { + expect(fingerprint()).toBe(fingerprint()); + expect(fingerprint(git, "changed")).not.toBe(fingerprint()); + expect(fingerprint(git, "commit", "changed")).not.toBe(fingerprint()); + expect(getIngestionRevision()).toMatch(/^[a-f0-9]{64}$/); + }); + + it.each([ + { docs_path: "guides" }, + { exclude_paths: ["private/**"] }, + { url: "https://example.com/other.git" }, + { ref: "stable" }, + { lang: "de" }, + ])("invalidates changed source settings: %j", (change) => { + expect( + fingerprint({ ...git, source: { ...git.source, ...change } }), + ).not.toBe(fingerprint()); + }); + + it("normalizes exclusion order, duplicates and omitted defaults", () => { + const a = { + ...git, + source: { ...git.source, exclude_paths: ["a/**", "b/**"] }, + }; + const b = { + ...git, + source: { ...git.source, exclude_paths: ["b/**", "a/**", "a/**"] }, + }; + expect(fingerprint(a)).toBe(fingerprint(b)); + expect( + fingerprint({ ...git, source: { ...git.source, exclude_paths: [] } }), + ).toBe(fingerprint()); + }); + + it("ignores YAML formatting and property order", () => { + const dir = mkdtempSync(join(tmpdir(), "fingerprint-yaml-")); + const path = join(dir, "docs.yaml"); + try { + writeFileSync( + path, + "name: docs\nsource:\n type: git\n url: https://example.com/docs.git\n docs_path: docs\n", + ); + const a = loadDefinition(path); + writeFileSync( + path, + "# same settings\nsource: {docs_path: docs, url: https://example.com/docs.git, lang: en, type: git}\nname: docs\n", + ); + expect(createBuildFingerprint(a, "latest", "commit", "pipeline")).toBe( + createBuildFingerprint( + loadDefinition(path), + "latest", + "commit", + "pipeline", + ), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("hashes the effective release entry rather than unrelated releases", () => { + const changed = { + ...zip, + versions: [ + ...zip.versions, + { + source: { + type: "zip" as const, + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "en", + }, + versions: ["2.0"], + }, + ], + }; + expect(createBuildFingerprint(zip, "1.0")).toBe( + createBuildFingerprint(changed, "1.0"), + ); + const resolved = { + ...zip, + versions: [ + { + versions: ["1.0"], + source: { + type: "zip" as const, + url: "https://example.com/1.0.zip", + docs_path: "docs-1.0", + lang: "en", + }, + }, + ], + }; + expect(createBuildFingerprint(zip, "1.0")).toBe( + createBuildFingerprint(resolved, "1.0"), + ); + }); + + it("skips an existing explicit release only when its fingerprint matches", () => { + const existing = { + registry: "custom", + name: "docs", + version: "1.0", + build_fingerprint: createBuildFingerprint(zip, "1.0"), + }; + expect(publicationSkipReason(zip, "1.0", existing)).toBe( + "build inputs unchanged", + ); + expect( + publicationSkipReason( + { ...zip, description: "changed" }, + "1.0", + existing, + ), + ).toBeUndefined(); + expect( + publicationSkipReason(zip, "1.0", { + ...existing, + build_fingerprint: "old-pipeline", + }), + ).toBeUndefined(); + expect(publicationSkipReason(zip, "1.0", existing, true)).toBeUndefined(); + }); + + it("preserves legacy versioned skipping and allows an explicit rebuild", () => { + const legacy = { registry: "custom", name: "docs", version: "1.0" }; + expect(publicationSkipReason(zip, "1.0", legacy)).toContain( + "legacy metadata", + ); + expect(publicationSkipReason(zip, "1.0", legacy, true)).toBeUndefined(); + expect(publicationSkipReason(zip, "1.0", null)).toBeUndefined(); + }); + + it("keeps rebuilding unversioned archives with no immutable revision", () => { + const def: UnversionedDefinition = { + ...git, + source: { + type: "zip", + url: "https://example.com/latest.zip", + lang: "en", + }, + }; + expect( + publicationSkipReason(def, "latest", { + registry: "custom", + name: "docs", + version: "latest", + build_fingerprint: createBuildFingerprint(def, "latest"), + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/registry/src/fingerprint.ts b/packages/registry/src/fingerprint.ts new file mode 100644 index 0000000..1ecb289 --- /dev/null +++ b/packages/registry/src/fingerprint.ts @@ -0,0 +1,80 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { + constructTag, + isGitVersionEntry, + isVersioned, + type PackageDefinition, + resolveUrl, + resolveVersionEntry, +} from "./definition.js"; + +export function getIngestionRevision(): string { + // Works from src (tsx) and dist. Runtime needs only the built artifact. + const { revision } = JSON.parse( + readFileSync( + new URL("../dist/ingestion-revision.json", import.meta.url), + "utf8", + ), + ) as { revision: string }; + return revision; +} + +/** Only the entry selected for this release contributes to its fingerprint. */ +export function resolveBuildSource( + definition: PackageDefinition, + version: string, +) { + if (!isVersioned(definition)) return definition.source; + const entry = resolveVersionEntry(definition, version); + if (!entry) + throw new Error( + `No version entry matches ${version} in ${definition.name}`, + ); + if (isGitVersionEntry(entry)) { + return { ...entry.source, ref: constructTag(entry.tag_pattern, version) }; + } + return { + ...entry.source, + url: resolveUrl(entry.source.url, version), + ...(entry.source.type === "zip" && entry.source.docs_path + ? { docs_path: resolveUrl(entry.source.docs_path, version) } + : {}), + }; +} + +export function createBuildFingerprint( + definition: PackageDefinition, + version: string, + sourceCommit?: string, + ingestionRevision = getIngestionRevision(), +): string { + const source = resolveBuildSource(definition, version); + if (source.type === "git" && !sourceCommit) { + throw new Error( + "A resolved Git commit is required for a build fingerprint", + ); + } + const effectiveSource = { + type: source.type, + url: source.url, + ref: source.type === "git" ? (source.ref ?? "HEAD") : undefined, + docs_path: "docs_path" in source ? source.docs_path : undefined, + exclude_paths: [...new Set(source.exclude_paths ?? [])].sort(), + lang: "lang" in source ? (source.lang ?? "en") : undefined, + max_pages: + source.type === "html-index" ? (source.max_pages ?? 2000) : undefined, + }; + const inputs = { + schema_version: 1, + registry: definition.registry, + name: definition.name, + version, + description: definition.description, + source_url: definition.repository ?? source.url, + source: effectiveSource, + source_revision: sourceCommit ?? version, + ingestion_revision: ingestionRevision, + }; + return createHash("sha256").update(JSON.stringify(inputs)).digest("hex"); +} diff --git a/packages/registry/src/fixtures/freshness.zip b/packages/registry/src/fixtures/freshness.zip new file mode 100644 index 0000000000000000000000000000000000000000..25da895cb0529a328fe0486a2b92a97a8ab8f197 GIT binary patch literal 632 zcmWIWW@Zs#U|`^2m=WX@n|XC{Un-E-48*)ZoRXhhte=@zQk1Wko8sHEk=MzPhqc=O z%fZ585ZI+GqDz*u}a0=v~!2w(Em+EUxrwsM@GFy*IvofYs>=wI$3PG?cHz6lEa!TxhU`5-pqfZ)7@C}KfdvG`qsF=WHEmLihB}gcIl)6 z-O+~Zp45txqC~KJ4stRZ^0540mvr#|7Xc5>Ootm@Iv+eA=@e_)oBe!Z&TTyV;|3%C zYYPM}PboUKH|%D!)fw}iDf4{UUhU`lSD}8fF-`nXfKU9VdFstu9?j{>=)SvqN_?TG z$ps<)_cD9rGA`@ZGNL-D`n7|(|smMX($aUC&hvj|M(Wdv6t_$T) zu(zze#uRPT>GSK)t}l~rA2{cC=A=p-;`U<6Sp(TA=T tJ!~Nw85kNEC&0AALK { + let root: string; + const script = resolve( + import.meta.dirname, + "../../../scripts/generate-ingestion-revision.mjs", + ); + function write(path: string, content: string) { + const file = resolve(root, path); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, content); + } + function revision() { + execFileSync(process.execPath, [script, root]); + return JSON.parse( + readFileSync( + resolve(root, "packages/registry/dist/ingestion-revision.json"), + "utf8", + ), + ) as { revision: string; files: string[]; dependencies: string[] }; + } + const lock = (version = "1.0") => + JSON.stringify({ + importers: { + "packages/registry": { + devDependencies: { typescript: { version: "5.9" } }, + }, + "packages/context": { dependencies: { parser: { version: "1.0" } } }, + }, + packages: { + "parser@1.0": { resolution: { integrity: "parser" } }, + [`helper@${version}`]: { + resolution: { integrity: `helper-${version}` }, + }, + "typescript@5.9": { resolution: { integrity: "ts" } }, + }, + snapshots: { + "parser@1.0": { dependencies: { helper: version } }, + [`helper@${version}`]: {}, + "typescript@5.9": {}, + }, + }); + + beforeEach(() => { + root = mkdtempSync(resolve(tmpdir(), "ingestion-revision-")); + write( + "packages/registry/src/build.ts", + 'import { buildPackage } from "@neuledge/context";\n', + ); + write( + "packages/context/src/index.ts", + 'export { buildPackage } from "./package-builder.js";\nexport { server } from "./server.js";\n', + ); + write( + "packages/context/src/package-builder.ts", + 'import parser from "parser";\nimport { helper } from "./helper.js";\nexport function buildPackage() { return helper(parser); }\n', + ); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x;\n", + ); + write("pnpm-lock.yaml", lock()); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + it("follows helpers and workspace exports while ignoring unrelated files and line endings", () => { + const original = revision(); + expect(original.files).toContain("packages/context/src/helper.ts"); + expect(original.files).not.toContain("packages/context/src/server.ts"); + write("README.md", "unrelated documentation"); + write("packages/context/src/helper.test.ts", "unrelated test"); + write("packages/context/src/server.ts", "unrelated server"); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x;\r\n", + ); + expect(revision().revision).toBe(original.revision); + write( + "packages/context/src/helper.ts", + "export const helper = (x) => x + 1;\n", + ); + expect(revision().revision).not.toBe(original.revision); + }); + + it("invalidates transitive dependency changes but ignores lockfile serialization", () => { + const original = revision(); + expect(original.dependencies).toContain("helper@1.0"); + write("pnpm-lock.yaml", JSON.stringify(JSON.parse(lock()), null, 2)); + expect(revision().revision).toBe(original.revision); + write("pnpm-lock.yaml", lock("2.0")); + expect(revision().revision).not.toBe(original.revision); + }); + + it("discovers newly imported helpers without a manually maintained file list", () => { + const original = revision(); + write( + "packages/context/src/helper.ts", + 'export { helper } from "./nested.js";\n', + ); + write( + "packages/context/src/nested.ts", + "export const helper = (x) => x;\n", + ); + const changed = revision(); + expect(changed.files).toContain("packages/context/src/nested.ts"); + expect(changed.revision).not.toBe(original.revision); + }); +}); diff --git a/packages/registry/src/publication.test.ts b/packages/registry/src/publication.test.ts new file mode 100644 index 0000000..64f155f --- /dev/null +++ b/packages/registry/src/publication.test.ts @@ -0,0 +1,303 @@ +import { execFile, execFileSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer, type Server } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { initDatabase, openDatabase } from "@neuledge/context"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { getHeadCommit } from "./build.js"; +import { getIngestionRevision } from "./fingerprint.js"; +import type { PackageMetadata } from "./publish.js"; + +const run = promisify(execFile); + +describe("publication freshness through the CLI", () => { + let root: string; + let server: Server; + let url: string; + let metadata: PackageMetadata | null; + let uploads: number; + let archiveDownloads: number; + let conflict: boolean; + let legacy: boolean; + let repoUrl: string; + const docs = + "# Documentation\n\n## Getting started\n\nThis documentation explains how to configure and run the application with a complete example.\n"; + + beforeAll(() => initDatabase()); + beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "publication-freshness-")); + mkdirSync(join(root, "definitions", "custom"), { recursive: true }); + mkdirSync(join(root, "output")); + const repo = join(root, "repo"); + mkdirSync(join(repo, "docs"), { recursive: true }); + mkdirSync(join(repo, "guides")); + writeFileSync(join(repo, "docs", "intro.md"), docs); + writeFileSync( + join(repo, "docs", "extra.md"), + docs.replace("Getting started", "Extra"), + ); + writeFileSync( + join(repo, "guides", "intro.md"), + docs.replace("Getting started", "Guide"), + ); + const git = (args: string[]) => + execFileSync("git", args, { cwd: repo, stdio: "pipe" }); + git(["init", "--initial-branch=main"]); + git(["add", "."]); + git([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "docs", + ]); + git([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "tag", + "-a", + "v1.0", + "-m", + "release", + ]); + repoUrl = pathToFileURL(repo).href; + metadata = null; + uploads = 0; + archiveDownloads = 0; + conflict = false; + legacy = false; + + server = createServer((request, response) => { + if (request.url?.startsWith("/source/")) { + archiveDownloads++; + response.end( + readFileSync(new URL("./fixtures/freshness.zip", import.meta.url)), + ); + return; + } + if (request.method !== "POST") { + response.writeHead(metadata ? 200 : 404, { + "Content-Type": "application/json", + }); + const body = + metadata && legacy + ? { + ...metadata, + build_fingerprint: undefined, + ingestion_revision: undefined, + } + : metadata; + response.end(JSON.stringify(body ?? { error: "not found" })); + return; + } + uploads++; + const buffers: Buffer[] = []; + request.on("data", (data: Buffer) => buffers.push(data)); + request.on("end", () => { + if (conflict) { + response.writeHead(409); + response.end("immutable release"); + return; + } + const dbPath = join(root, "uploaded.db"); + writeFileSync(dbPath, Buffer.concat(buffers)); + const db = openDatabase(dbPath, { readonly: true }); + try { + const values = Object.fromEntries( + ( + db.prepare("SELECT key, value FROM meta").all() as { + key: string; + value: string; + }[] + ).map((row) => [row.key, row.value]), + ); + if (!values.name || !values.version) + throw new Error("Missing package identity"); + metadata = { + registry: "custom", + name: values.name, + version: values.version, + source_commit: values.source_commit, + build_fingerprint: values.build_fingerprint, + ingestion_revision: values.ingestion_revision, + }; + } finally { + db.close(); + } + response.end("{}"); + }); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing test server address"); + url = `http://127.0.0.1:${address.port}`; + }); + afterEach(async () => { + await new Promise((done, reject) => + server.close((error) => (error ? reject(error) : done())), + ); + rmSync(root, { recursive: true, force: true }); + }); + + function currentMetadata(): PackageMetadata { + if (!metadata) throw new Error("No package was uploaded"); + return metadata; + } + + function defineGit(path = "docs", excludes: string[] = []) { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nsource:\n type: git\n url: ${repoUrl}\n ref: main\n docs_path: ${path}\n exclude_paths: ${JSON.stringify(excludes)}\n`, + ); + } + function defineArchive(path = "docs") { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nversions:\n - versions: ["1.0"]\n source:\n type: zip\n url: ${url}/source/{version}.zip\n docs_path: ${path}\n`, + ); + } + async function cli(command: string, ...args: string[]) { + try { + const result = await run( + process.execPath, + [ + "--import", + "tsx", + "src/cli.ts", + command, + ...args, + "--dir", + join(root, "definitions"), + "--output", + join(root, "output"), + ], + { + cwd: process.cwd(), + timeout: 30_000, + env: { + ...process.env, + REGISTRY_SERVER_URL: url, + REGISTRY_PUBLISH_KEY: "test-key", + }, + }, + ); + return { status: 0, output: result.stdout + result.stderr }; + } catch (error) { + const result = error as { + code?: number; + stdout?: string; + stderr?: string; + }; + return { + status: result.code ?? 1, + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + }; + } + } + + it("skips unchanged Git, then rebuilds changed docs_path, exclusions and pipeline at the same commit", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + const commit = metadata?.source_commit; + expect(commit).toBe(getHeadCommit(repoUrl, "main")); + expect(metadata?.ingestion_revision).toBe(getIngestionRevision()); + expect(metadata?.build_fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect((await cli("publish", "docs")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + defineGit("guides"); + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(2); + expect(metadata?.source_commit).toBe(commit); + defineGit("docs", ["extra.md"]); + expect((await cli("publish", "docs")).status).toBe(0); + expect(uploads).toBe(3); + metadata = { + ...currentMetadata(), + build_fingerprint: "old-ingestion-pipeline", + }; + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(4); + }, 90_000); + + it("compares fingerprints for already-published explicit versions in both commands", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect((await cli("publish-all")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + expect(archiveDownloads).toBe(1); + defineArchive("guides"); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect(uploads).toBe(2); + metadata = { + ...currentMetadata(), + build_fingerprint: "old-ingestion-pipeline", + }; + expect((await cli("publish-all")).status).toBe(0); + expect(uploads).toBe(3); + expect((await cli("publish-all", "--force")).status).toBe(0); + expect(uploads).toBe(4); + }, 90_000); + + it("retains legacy Git skipping and migrates with --force", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + legacy = true; + defineGit("guides"); + expect((await cli("publish", "docs")).output).toContain("legacy metadata"); + expect(uploads).toBe(1); + expect((await cli("publish", "docs", "--force")).status).toBe(0); + expect(uploads).toBe(2); + legacy = false; + expect((await cli("publish", "docs")).output).toContain( + "build inputs unchanged", + ); + }, 90_000); + + it("reports immutable-version conflicts and preserves the rebuilt artifact", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + legacy = true; + expect((await cli("publish-all")).output).toContain("legacy metadata"); + expect(uploads).toBe(1); + conflict = true; + const failed = await cli("publish-all", "--force"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("409 Conflict"); + expect(failed.output).toContain("immutable release"); + expect(failed.output).toContain("rebuilt artifact is preserved"); + expect(failed.output).toContain("Failed: 1"); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(true); + }, 90_000); + + it("resolves annotated tags to the commit recorded by a versioned Git build", async () => { + writeFileSync( + join(root, "definitions", "custom", "docs.yaml"), + `name: docs\nversions:\n - min_version: "1.0"\n source:\n type: git\n url: ${repoUrl}\n docs_path: docs\n`, + ); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + expect(metadata?.source_commit).toBe(getHeadCommit(repoUrl, "v1.0")); + expect((await cli("publish", "docs", "1.0")).output).toContain( + "build inputs unchanged", + ); + expect(uploads).toBe(1); + }, 90_000); +}); diff --git a/packages/registry/src/publication.ts b/packages/registry/src/publication.ts new file mode 100644 index 0000000..d291e52 --- /dev/null +++ b/packages/registry/src/publication.ts @@ -0,0 +1,84 @@ +import type { BuildResult } from "@neuledge/context"; +import { + buildFromDefinition, + buildUnversioned, + getHeadCommit, +} from "./build.js"; +import { isVersioned, type PackageDefinition } from "./definition.js"; +import { createBuildFingerprint, resolveBuildSource } from "./fingerprint.js"; +import { + checkPackageExists, + type PackageMetadata, + publishPackage, +} from "./publish.js"; + +/** Legacy metadata retains the old skip policy until a forced migration. */ +export function publicationSkipReason( + definition: PackageDefinition, + version: string, + existing: PackageMetadata | null, + force = false, +): string | undefined { + if (force || !existing) return; + if (!existing.build_fingerprint && isVersioned(definition)) { + return "already published; legacy metadata (use --force to rebuild)"; + } + const source = resolveBuildSource(definition, version); + // An unversioned archive has no immutable source revision to compare. + if (!isVersioned(definition) && source.type === "zip") return; + const commit = + source.type === "git" ? getHeadCommit(source.url, source.ref) : undefined; + if (existing.build_fingerprint) { + if ( + existing.build_fingerprint === + createBuildFingerprint(definition, version, commit) + ) { + return "build inputs unchanged"; + } + } else if (commit && commit === existing.source_commit) { + return "source unchanged; legacy metadata (use --force to rebuild)"; + } +} + +export async function publishDefinition( + definition: PackageDefinition, + version: string, + outputDir: string, + options: { force?: boolean; log?: (message: string) => void } = {}, +): Promise { + const log = options.log ?? console.log; + const id = `${definition.registry}/${definition.name}@${version}`; + const existing = await checkPackageExists( + definition.registry, + definition.name, + version, + ); + const reason = publicationSkipReason( + definition, + version, + existing, + options.force, + ); + if (reason) { + log(`Skipping ${id} (${reason})`); + return; + } + log(`Building ${id}...`); + const result = isVersioned(definition) + ? await buildFromDefinition(definition, version, outputDir) + : await buildUnversioned(definition, outputDir); + const skipped = + result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; + log( + `Built: ${result.path} (${result.sectionCount} sections, ${result.totalTokens} tokens${skipped})`, + ); + log(`Publishing ${id}...`); + await publishPackage( + definition.registry, + definition.name, + version, + result.path, + ); + log(`Published: ${id}`); + return result; +} diff --git a/packages/registry/src/publish.ts b/packages/registry/src/publish.ts index 51bf7b4..d0fe030 100644 --- a/packages/registry/src/publish.ts +++ b/packages/registry/src/publish.ts @@ -57,6 +57,8 @@ export interface PackageMetadata { name: string; version: string; source_commit?: string; + build_fingerprint?: string; + ingestion_revision?: string; } /** @@ -101,19 +103,27 @@ export async function publishPackage( const url = `${getServerUrl()}/packages/${encodeURIComponent(registry)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`; const body = readFileSync(dbPath); - // Re-uploading an identical package is safe: the server keys on - // registry/name/version, so a retry after a dropped connection overwrites - // rather than duplicating. - await requestWithRetry( - url, - { - method: "POST", - headers: { - Authorization: `Bearer ${getPublishKey()}`, - "Content-Type": "application/octet-stream", + try { + await requestWithRetry( + url, + { + method: "POST", + headers: { + Authorization: `Bearer ${getPublishKey()}`, + "Content-Type": "application/octet-stream", + }, + body, }, - body, - }, - () => `Failed to publish ${registry}/${name}@${version}`, - ); + () => `Failed to publish ${registry}/${name}@${version}`, + ); + } catch (error) { + if (error instanceof Error && error.message.includes(": 409 ")) { + throw new Error( + `${error.message}. The registry rejected replacement of this published version. ` + + `The rebuilt artifact is preserved at ${dbPath}. Use a registry-supported replacement or artifact revision; --force cannot override the server's policy.`, + { cause: error }, + ); + } + throw error; + } } diff --git a/registry/README.md b/registry/README.md index 14410a9..9c2c17b 100644 --- a/registry/README.md +++ b/registry/README.md @@ -156,7 +156,8 @@ indexed once, which avoids duplicate man-page aliases. Pinned downloads are reused from `.cache/context/html-index` across builds. Delete that directory to refetch a corrected upstream release. The nightly -publisher skips releases already present in the registry. Check the publisher's crawling policy +publisher compares build fingerprints to skip unchanged releases. Packages with +legacy metadata retain the skip behavior described below. Check the publisher's crawling policy before adding an index source. For systemd, `systemd/systemd` contains the versioned reference manuals and @@ -193,6 +194,54 @@ content you want is already isolated in its own directory. Markdown (`.md`, `.mdx`), HTML, AsciiDoc (`.adoc`) and reStructuredText (`.rst`). Point `docs_path` at the directory holding them; everything else in the repo is ignored. +## Publication freshness and rebuilding + +`registry publish` and `registry publish-all` compare `build_fingerprint` metadata +before building. A matching fingerprint skips work; changed build inputs cause a +rebuild and an upload to the same package version. Inputs include the actual Git +commit (or pinned explicit release), the effective source settings and package +metadata, and an automatically generated ingestion revision. `docs_path`, +exclusions, language, source URL/ref, description, or ingestion changes can +therefore rebuild a package even when the source commit is unchanged. YAML key +order, comments, exclusion order/duplicates, and unrelated release entries do not +change the fingerprint. + +The ingestion revision is generated during `pnpm build` and before the +`pnpm --filter @neuledge/registry registry ...` source CLI runs. The generator starts +from the registry builder, follows runtime imports and the context APIs it uses, +and hashes those modules plus their locked runtime dependencies, including +transitive and optional dependencies and the compiler version. New helper imports +are discovered automatically. Repository documentation, tests, and unrelated CLI +or server modules are excluded. Line endings and lockfile key order are normalized. +Harmless ingestion refactoring can conservatively trigger a rebuild. + +The generated revision ships in `dist/ingestion-revision.json`. The built publisher +needs neither Git history nor TypeScript source files to compute the ingestion +revision; Git commands are still needed to resolve and build Git sources. Rebuild +after editing ingestion code if running `src/cli.ts` directly instead of the +package script. Unversioned ZIP sources continue rebuilding on each publication, +because their URLs provide no immutable source revision. Pinned archives/HTML +releases are assumed immutable; refreshing upstream download caches is separate +from derived-package freshness. + +Servers must expose the uploaded database's `build_fingerprint` on the metadata +endpoint (see [SERVER_SPEC.md](../SERVER_SPEC.md)). Without that field, versioned +packages retain the previous existence-based skip behavior, unversioned Git +packages retain commit comparison, and skips explicitly mention legacy metadata. +Use `--force` to migrate old packages after the server supports fingerprints: + +```bash +pnpm --filter @neuledge/registry registry publish [version] --force +pnpm --filter @neuledge/registry registry publish-all --force +``` + +`--force` bypasses freshness checks and rebuilds; it does not override the server's +replacement policy. Both automatic rebuilds and forced builds upload to the same +registry/name/version. Servers that allow replacement accept the new artifact; +servers that return HTTP 409 produce a clear failure and leave the rebuilt `.db` +on disk. Such servers need their own supported replacement or artifact revision +policy. Successful `publish-all` uploads remove local build artifacts as before. + ## Before opening a PR Build your definition locally to confirm it produces real content: diff --git a/scripts/generate-ingestion-revision.mjs b/scripts/generate-ingestion-revision.mjs new file mode 100644 index 0000000..ec6da1c --- /dev/null +++ b/scripts/generate-ingestion-revision.mjs @@ -0,0 +1,201 @@ +/** Build-time fingerprint of ingestion modules and their locked runtime dependencies. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptPath = fileURLToPath(import.meta.url); +const repository = resolve(dirname(scriptPath), ".."); +const require = createRequire( + resolve(repository, "packages/registry/package.json"), +); +const ts = require("typescript"); +const { parse } = require("yaml"); + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]), + ); + } + return value; +} + +export function generateIngestionRevision(root = repository) { + const files = new Map(); + const dependencies = new Map(); + const lock = parse(readFileSync(resolve(root, "pnpm-lock.yaml"), "utf8")); + + function dependency(name, version) { + const key = `${name}@${version}`; + if (dependencies.has(key)) return; + const snapshot = lock.snapshots[key]; + const packageKey = key.replace(/\(.*$/, ""); + const pkg = lock.packages[packageKey]; + if (!snapshot || !pkg) + throw new Error(`Missing locked ingestion dependency: ${key}`); + dependencies.set(key, { package: pkg, snapshot }); + for (const [child, childVersion] of Object.entries({ + ...snapshot.dependencies, + ...snapshot.optionalDependencies, + })) { + dependency(child, childVersion); + } + } + + function follow(file, specifier, names) { + if (specifier.startsWith("node:")) return; + if (specifier.startsWith(".")) { + visit(resolve(dirname(file), specifier.replace(/\.js$/, ".ts"))); + return; + } + const packageName = specifier.startsWith("@") + ? specifier.split("/").slice(0, 2).join("/") + : specifier.split("/")[0]; + if (packageName === "@neuledge/context") { + // Resolve the APIs actually imported from the workspace barrel. Changes + // to the CLI or server must not invalidate documentation packages. + const index = resolve(root, "packages/context/src/index.ts"); + if (specifier !== packageName) { + visit( + resolve( + root, + "packages/context/src", + specifier.slice(packageName.length + 1).replace(/\.js$/, ".ts"), + ), + ); + return; + } + const source = ts.createSourceFile( + index, + readFileSync(index, "utf8"), + ts.ScriptTarget.Latest, + true, + ); + const unresolved = names && new Set(names); + for (const statement of source.statements) { + if ( + !ts.isExportDeclaration(statement) || + statement.isTypeOnly || + !statement.moduleSpecifier + ) + continue; + const exported = + statement.exportClause && ts.isNamedExports(statement.exportClause) + ? statement.exportClause.elements + .filter((item) => !item.isTypeOnly) + .map((item) => item.name.text) + : undefined; + if ( + !names || + !exported || + exported.some((name) => names.includes(name)) + ) { + follow(index, statement.moduleSpecifier.text); + for (const name of exported ?? []) unresolved?.delete(name); + } + } + if (unresolved?.size) + throw new Error( + `Unresolved ingestion exports: ${[...unresolved].join(", ")}`, + ); + return; + } + const importer = relative(root, file) + .replaceAll("\\", "/") + .split("/") + .slice(0, 2) + .join("/"); + const manifest = lock.importers[importer]; + const entry = + manifest.dependencies?.[packageName] ?? + manifest.optionalDependencies?.[packageName]; + if (!entry) + throw new Error( + `Undeclared ingestion dependency: ${packageName} in ${file}`, + ); + dependency(packageName, entry.version); + } + + function visit(file) { + const path = relative(root, file).replaceAll("\\", "/"); + if (files.has(path)) return; + const content = readFileSync(file, "utf8").replaceAll("\r\n", "\n"); + files.set(path, content); + const source = ts.createSourceFile( + file, + content, + ts.ScriptTarget.Latest, + true, + ); + function walk(node) { + if (ts.isImportDeclaration(node)) { + const clause = node.importClause; + if (clause?.isTypeOnly) return; + const bindings = clause?.namedBindings; + const names = + bindings && ts.isNamedImports(bindings) + ? bindings.elements + .filter((item) => !item.isTypeOnly) + .map((item) => (item.propertyName ?? item.name).text) + : undefined; + if (names?.length === 0 && !clause?.name) return; + follow(file, node.moduleSpecifier.text, names); + } else if ( + ts.isExportDeclaration(node) && + !node.isTypeOnly && + node.moduleSpecifier + ) { + follow(file, node.moduleSpecifier.text); + } else if ( + ts.isCallExpression(node) && + node.arguments[0] && + ts.isStringLiteral(node.arguments[0]) + ) { + const expression = node.expression.getText(source); + if ( + node.expression.kind === ts.SyntaxKind.ImportKeyword || + /^(?:_?require)(?:\.resolve)?$/.test(expression) + ) { + follow(file, node.arguments[0].text); + } + } + ts.forEachChild(node, walk); + } + walk(source); + } + + visit(resolve(root, "packages/registry/src/build.ts")); + dependency( + "typescript", + lock.importers["packages/registry"].devDependencies.typescript.version, + ); + // Include the algorithm itself so changes in what we hash also invalidate. + files.set( + "scripts/generate-ingestion-revision.mjs", + readFileSync(scriptPath, "utf8").replaceAll("\r\n", "\n"), + ); + const inputs = canonical({ + files: Object.fromEntries(files), + dependencies: Object.fromEntries(dependencies), + }); + return { + revision: createHash("sha256").update(JSON.stringify(inputs)).digest("hex"), + files: Object.keys(inputs.files), + dependencies: Object.keys(inputs.dependencies), + }; +} + +if (process.argv[1] && resolve(process.argv[1]) === scriptPath) { + const root = process.argv[2] ? resolve(process.argv[2]) : repository; + const output = resolve(root, "packages/registry/dist"); + mkdirSync(output, { recursive: true }); + writeFileSync( + resolve(output, "ingestion-revision.json"), + `${JSON.stringify(generateIngestionRevision(root), null, 2)}\n`, + ); +} diff --git a/turbo.json b/turbo.json index 8b1f417..65f2bea 100644 --- a/turbo.json +++ b/turbo.json @@ -1,5 +1,6 @@ { "$schema": "https://turborepo.com/schema.json", + "globalDependencies": ["scripts/generate-ingestion-revision.mjs"], "tasks": { "build": { "dependsOn": ["^build"], From b5bbe75e26a236e52f293b8af2c950bab15d5427 Mon Sep 17 00:00:00 2001 From: Martin Beckert Date: Thu, 8 Oct 2026 13:50:11 +0200 Subject: [PATCH 2/2] fix(registry): address freshness publication review cases --- SERVER_SPEC.md | 11 + packages/registry/src/build.test.ts | 68 +++++ packages/registry/src/build.ts | 142 +++------ packages/registry/src/cli.ts | 49 +-- packages/registry/src/fingerprint.test.ts | 20 ++ packages/registry/src/fingerprint.ts | 35 +-- packages/registry/src/format.ts | 8 + .../registry/src/ingestion-revision.test.ts | 68 +++++ packages/registry/src/publication.test.ts | 122 +++++++- packages/registry/src/publication.ts | 28 +- packages/registry/src/publish.test.ts | 15 +- packages/registry/src/publish.ts | 72 ++++- packages/registry/src/source.ts | 32 ++ registry/README.md | 19 +- scripts/generate-ingestion-revision.mjs | 280 +++++++++--------- turbo.json | 9 +- 16 files changed, 653 insertions(+), 325 deletions(-) create mode 100644 packages/registry/src/build.test.ts create mode 100644 packages/registry/src/format.ts create mode 100644 packages/registry/src/source.ts diff --git a/SERVER_SPEC.md b/SERVER_SPEC.md index 19132f1..118969e 100644 --- a/SERVER_SPEC.md +++ b/SERVER_SPEC.md @@ -95,6 +95,17 @@ Check if a package version exists and return its metadata. The publish pipeline Servers supporting automatic freshness checks must extract and persist these optional metadata values on every upload, including replacements, and return them here. The client treats fingerprints as opaque; servers should not recompute them. Older packages and servers can omit these fields: versioned packages retain existence-based skipping, unversioned Git packages retain `source_commit` comparison, and unversioned ZIP packages continue rebuilding. The CLI identifies legacy skips and offers `--force` for migration. Returning the fingerprint after a successful rebuild enables subsequent automatic checks; a server that continues omitting it retains the legacy behavior. +Server adoption must be verified in the server deployment; this repository only +contains the client and protocol contract. Verify that an initial upload returns +the database's fingerprint and revision, that replacing it updates both fields, +and that a subsequent metadata request returns the replacement's values. + +After an upload returns `409 Conflict`, the client requests metadata to determine +whether an earlier attempt already succeeded. It accepts success only when the +package identity, `build_fingerprint`, and `ingestion_revision` match the uploaded +artifact. Missing or different metadata remains a conflict; an existing version +alone is not sufficient evidence of a successful upload. + **Response `404 Not Found`:** ```json diff --git a/packages/registry/src/build.test.ts b/packages/registry/src/build.test.ts new file mode 100644 index 0000000..1b360f0 --- /dev/null +++ b/packages/registry/src/build.test.ts @@ -0,0 +1,68 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { getHeadCommit, MissingSourceRefError } from "./build.js"; + +describe("remote Git commit resolution", () => { + let root: string; + let url: string; + let commit: string; + const git = (...args: string[]) => + execFileSync("git", args, { + cwd: root, + encoding: "utf8", + stdio: "pipe", + }).trim(); + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "registry-refs-")); + url = pathToFileURL(root).href; + git("init", "--initial-branch=main"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.com"); + writeFileSync(join(root, "doc.md"), "documentation"); + git("add", "."); + git("commit", "-m", "docs"); + commit = git("rev-parse", "HEAD"); + git("tag", "-a", "annotated", "-m", "release"); + git("tag", "lightweight"); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + it.each([ + undefined, + "main", + "refs/heads/main", + "annotated", + "refs/tags/annotated", + "lightweight", + "refs/tags/lightweight", + ])("resolves %s to the cloned commit", (ref) => { + expect(getHeadCommit(url, ref)).toBe(commit); + }); + + it("prefers a branch over a same-named annotated tag", () => { + git("branch", "annotated"); + git("checkout", "annotated"); + writeFileSync(join(root, "doc.md"), "updated documentation"); + git("commit", "-am", "update"); + expect(getHeadCommit(url, "annotated")).toBe(git("rev-parse", "HEAD")); + expect(getHeadCommit(url, "refs/tags/annotated")).toBe(commit); + }); + + it("distinguishes missing refs from an inaccessible repository", () => { + expect(() => getHeadCommit(url, "missing-tag")).toThrow( + MissingSourceRefError, + ); + expect(() => + getHeadCommit(pathToFileURL(join(root, "missing-repo")).href, "main"), + ).toThrow(); + try { + getHeadCommit(pathToFileURL(join(root, "missing-repo")).href, "main"); + } catch (error) { + expect(error).not.toBeInstanceOf(MissingSourceRefError); + } + }); +}); diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts index f86526b..7916f14 100644 --- a/packages/registry/src/build.ts +++ b/packages/registry/src/build.ts @@ -17,20 +17,21 @@ import { initDatabase, readLocalDocsFiles, } from "@neuledge/context"; -import { - constructTag, - isGitVersionEntry, - type PackageDefinition, - resolveUrl, - resolveVersionEntry, - type UnversionedDefinition, - type VersionedDefinition, +import type { + GitSource, + PackageDefinition, + UnversionedDefinition, + VersionedDefinition, } from "./definition.js"; import { createBuildFingerprint, getIngestionRevision } from "./fingerprint.js"; import { excludeFiles } from "./glob.js"; import { downloadHtmlIndex } from "./html-index.js"; +import { resolveBuildSource } from "./source.js"; import { downloadAndExtractZip } from "./zip.js"; +/** Only an absent ref is skippable; transport failures keep their original error. */ +export class MissingSourceRefError extends Error {} + export interface RegistryBuildResult extends BuildResult { name: string; registry: string; @@ -64,14 +65,16 @@ export function getHeadCommit(url: string, ref?: string): string { }), ); // Match clone --branch: prefer a branch, and peel annotated tags to commits. - const sha = - refs.get(`refs/heads/${requested}`) ?? - refs.get(`${requested}^{}`) ?? - refs.get(`refs/tags/${requested}^{}`) ?? - refs.get(requested) ?? - refs.get(`refs/tags/${requested}`); + const candidates = requested.startsWith("refs/") + ? [requested] + : [`refs/heads/${requested}`, `refs/tags/${requested}`, requested]; + const sha = candidates + .map((name) => refs.get(`${name}^{}`) ?? refs.get(name)) + .find(Boolean); if (!sha) { - throw new Error(`Git reference ${requested} not found in upstream ${url}`); + throw new MissingSourceRefError( + `Git reference ${requested} not found in upstream ${url}`, + ); } return sha; } @@ -102,12 +105,7 @@ export async function buildFromDefinition( outputDir: string, ): Promise { await initDatabase(); - const entry = resolveVersionEntry(definition, version); - if (!entry) { - throw new Error( - `No version entry matches ${version} in ${definition.name}`, - ); - } + const source = resolveBuildSource(definition, version); // Replace / in scoped names (e.g., @trpc/server → @trpc-server) for valid filenames const safeName = definition.name.replace(/\//g, "-"); @@ -116,34 +114,21 @@ export async function buildFromDefinition( `${definition.registry}-${safeName}@${version}.db`, ); - if (isGitVersionEntry(entry)) { - return buildFromGit( - entry.source.url, - constructTag(entry.tag_pattern, version), - entry.source.docs_path, - entry.source.exclude_paths, - entry.source.lang, - outputPath, - definition, - version, - ); + if (source.type === "git") { + return buildFromGit(source, outputPath, definition, version); } - // Explicit releases download an archive or a pinned HTML index. - const url = resolveUrl(entry.source.url, version); const files = - entry.source.type === "html-index" - ? await downloadHtmlIndex(entry.source, version) - : await downloadAndExtractZip(url, { - docsPath: entry.source.docs_path - ? resolveUrl(entry.source.docs_path, version) - : undefined, - excludePaths: entry.source.exclude_paths, + source.type === "html-index" + ? await downloadHtmlIndex(source, version) + : await downloadAndExtractZip(source.url, { + docsPath: source.docs_path, + excludePaths: source.exclude_paths, }); if (files.length === 0) { throw new Error( - `No documentation files found in ${entry.source.type} source from ${url}`, + `No documentation files found in ${source.type} source from ${source.url}`, ); } @@ -151,7 +136,7 @@ export async function buildFromDefinition( name: definition.name, version, description: definition.description, - sourceUrl: definition.repository ?? url, + sourceUrl: definition.repository ?? source.url, ...fingerprintOptions(definition, version), }); @@ -174,7 +159,7 @@ export async function buildUnversioned( ): Promise { await initDatabase(); const version = "latest"; - const { source } = definition; + const source = resolveBuildSource(definition, version); const safeName = definition.name.replace(/\//g, "-"); const outputPath = join( outputDir, @@ -207,17 +192,26 @@ export async function buildUnversioned( }; } - // Git source: clone and read + if (source.type !== "git") + throw new Error("Unversioned HTML sources are unsupported"); + return buildFromGit(source, outputPath, definition, version); +} + +/** Build from a git source (clone at tag, read docs, build package). */ +function buildFromGit( + source: GitSource, + outputPath: string, + definition: PackageDefinition, + version: string, +): RegistryBuildResult { const { tempDir, cleanup } = cloneRepository(source.url, source.ref); try { - // Get the commit SHA of the cloned HEAD const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: tempDir, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], }).trim(); - // Filter before the emptiness check, so an over-broad exclude_paths fails // loudly here instead of publishing an empty package. const files = excludeFiles( @@ -231,7 +225,7 @@ export async function buildUnversioned( if (files.length === 0) { throw new Error( - `No documentation files found in ${source.url} (default branch)`, + `No documentation files found in ${source.url} at ref ${source.ref ?? "HEAD"}`, ); } @@ -255,55 +249,3 @@ export async function buildUnversioned( cleanup(); } } - -/** Build from a git source (clone at tag, read docs, build package). */ -function buildFromGit( - url: string, - tag: string, - docsPath: string | undefined, - excludePaths: string[] | undefined, - lang: string, - outputPath: string, - definition: VersionedDefinition, - version: string, -): RegistryBuildResult { - const { tempDir, cleanup } = cloneRepository(url, tag); - - try { - const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { - cwd: tempDir, - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - }).trim(); - // Filter before the emptiness check, so an over-broad exclude_paths fails - // loudly here instead of publishing an empty package. - const files = excludeFiles( - readLocalDocsFiles(tempDir, { path: docsPath, lang }), - excludePaths, - docsPath, - ); - - if (files.length === 0) { - throw new Error(`No documentation files found in ${url} at tag ${tag}`); - } - - const result = buildPackage(outputPath, files, { - name: definition.name, - version, - description: definition.description, - sourceUrl: definition.repository ?? url, - sourceCommit, - ...fingerprintOptions(definition, version, sourceCommit), - }); - - return { - ...result, - name: definition.name, - registry: definition.registry, - version, - sourceCommit, - }; - } finally { - cleanup(); - } -} diff --git a/packages/registry/src/cli.ts b/packages/registry/src/cli.ts index d8e6867..cc09b24 100644 --- a/packages/registry/src/cli.ts +++ b/packages/registry/src/cli.ts @@ -7,14 +7,19 @@ import { mkdirSync, rmSync } from "node:fs"; import { resolve } from "node:path"; -import { type BuildResult, isMissingRefError } from "@neuledge/context"; +import { isMissingRefError } from "@neuledge/context"; import { Command } from "commander"; -import { buildFromDefinition, buildUnversioned } from "./build.js"; +import { + buildFromDefinition, + buildUnversioned, + MissingSourceRefError, +} from "./build.js"; import { isExplicitVersionEntry, isVersioned, listDefinitions, } from "./definition.js"; +import { formatBuilt } from "./format.js"; import { publishDefinition } from "./publication.js"; import { type AvailableVersion, discoverVersions } from "./version-check.js"; @@ -28,19 +33,6 @@ const program = new Command() .name("registry") .description("Build context documentation packages from definitions"); -/** - * Describe a finished build. - * - * Skipped files are named rather than counted silently: a file too malformed to - * parse is dropped on its own so one bad document can't fail the build, which is - * only safe if the count reaches whoever is reading the log. - */ -function formatBuilt(result: BuildResult): string { - const skipped = - result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; - return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}`; -} - program .command("list") .description("List all package definitions") @@ -187,6 +179,11 @@ program let succeeded = 0; let skipped = 0; + const skipReasons = new Map(); + const recordSkip = (reason: string) => { + skipped++; + skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1); + }; const failures: { id: string; error: string }[] = []; for (const def of definitions) { @@ -219,10 +216,11 @@ program opts.output, { force: opts.force, + quietSkips: true, + onSkip: recordSkip, }, ); if (!result) { - skipped++; continue; } // Keep failed uploads on disk for recovery; remove successful artifacts. @@ -231,11 +229,15 @@ program succeeded++; } catch (err) { const message = err instanceof Error ? err.message : String(err); - // A registry can publish a version before its git tag is pushed. - // Skip (don't fail) — the next run picks it up once the tag lands. - if (isMissingRefError(message)) { - console.log(` Skipping ${id} (git tag not published yet)`); - skipped++; + // Tags can disappear after publication or not have been pushed yet. + if ( + err instanceof MissingSourceRefError || + isMissingRefError(message) + ) { + console.warn( + ` WARNING ${id}: source tag unavailable; skipping (${message})`, + ); + recordSkip("source tag unavailable"); continue; } console.error(` FAILED ${id}: ${message}`); @@ -247,7 +249,10 @@ program // Summary console.log(`\n--- Summary ---`); console.log(`Succeeded: ${succeeded}`); - console.log(`Skipped (up to date or legacy metadata): ${skipped}`); + console.log(`Skipped: ${skipped}`); + for (const [reason, count] of skipReasons) { + console.log(` ${reason}: ${count}`); + } console.log(`Failed: ${failures.length}`); if (failures.length > 0) { diff --git a/packages/registry/src/fingerprint.test.ts b/packages/registry/src/fingerprint.test.ts index 0767f3e..ea2f5d3 100644 --- a/packages/registry/src/fingerprint.test.ts +++ b/packages/registry/src/fingerprint.test.ts @@ -139,6 +139,26 @@ describe("build fingerprints", () => { ); }); + it("ignores language settings that ZIP ingestion does not use", () => { + const changed: VersionedDefinition = { + ...zip, + versions: [ + { + versions: ["1.0"], + source: { + type: "zip", + url: "https://example.com/{version}.zip", + docs_path: "docs-{version}", + lang: "de", + }, + }, + ], + }; + expect(createBuildFingerprint(changed, "1.0")).toBe( + createBuildFingerprint(zip, "1.0"), + ); + }); + it("skips an existing explicit release only when its fingerprint matches", () => { const existing = { registry: "custom", diff --git a/packages/registry/src/fingerprint.ts b/packages/registry/src/fingerprint.ts index 1ecb289..1340ca5 100644 --- a/packages/registry/src/fingerprint.ts +++ b/packages/registry/src/fingerprint.ts @@ -1,13 +1,7 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; -import { - constructTag, - isGitVersionEntry, - isVersioned, - type PackageDefinition, - resolveUrl, - resolveVersionEntry, -} from "./definition.js"; +import type { PackageDefinition } from "./definition.js"; +import { resolveBuildSource } from "./source.js"; export function getIngestionRevision(): string { // Works from src (tsx) and dist. Runtime needs only the built artifact. @@ -20,29 +14,6 @@ export function getIngestionRevision(): string { return revision; } -/** Only the entry selected for this release contributes to its fingerprint. */ -export function resolveBuildSource( - definition: PackageDefinition, - version: string, -) { - if (!isVersioned(definition)) return definition.source; - const entry = resolveVersionEntry(definition, version); - if (!entry) - throw new Error( - `No version entry matches ${version} in ${definition.name}`, - ); - if (isGitVersionEntry(entry)) { - return { ...entry.source, ref: constructTag(entry.tag_pattern, version) }; - } - return { - ...entry.source, - url: resolveUrl(entry.source.url, version), - ...(entry.source.type === "zip" && entry.source.docs_path - ? { docs_path: resolveUrl(entry.source.docs_path, version) } - : {}), - }; -} - export function createBuildFingerprint( definition: PackageDefinition, version: string, @@ -61,7 +32,7 @@ export function createBuildFingerprint( ref: source.type === "git" ? (source.ref ?? "HEAD") : undefined, docs_path: "docs_path" in source ? source.docs_path : undefined, exclude_paths: [...new Set(source.exclude_paths ?? [])].sort(), - lang: "lang" in source ? (source.lang ?? "en") : undefined, + lang: source.type === "git" ? (source.lang ?? "en") : undefined, max_pages: source.type === "html-index" ? (source.max_pages ?? 2000) : undefined, }; diff --git a/packages/registry/src/format.ts b/packages/registry/src/format.ts new file mode 100644 index 0000000..838ff40 --- /dev/null +++ b/packages/registry/src/format.ts @@ -0,0 +1,8 @@ +import type { BuildResult } from "@neuledge/context"; + +/** Include skipped files so malformed documents are visible to operators. */ +export function formatBuilt(result: BuildResult): string { + const skipped = + result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; + return `${result.sectionCount} sections, ${result.totalTokens} tokens${skipped}`; +} diff --git a/packages/registry/src/ingestion-revision.test.ts b/packages/registry/src/ingestion-revision.test.ts index 4bf4fff..3cbd832 100644 --- a/packages/registry/src/ingestion-revision.test.ts +++ b/packages/registry/src/ingestion-revision.test.ts @@ -116,4 +116,72 @@ describe("automatic ingestion revision", () => { expect(changed.files).toContain("packages/context/src/nested.ts"); expect(changed.revision).not.toBe(original.revision); }); + + it.each([ + false, + true, + ])("follows peer-suffixed dependencies with suffixed package records: %s", (suffixedPackages) => { + const data = JSON.parse(lock()); + data.importers["packages/context"].dependencies.parser.version = + "1.0(peer@1.0)"; + for (const name of ["parser", "helper"]) { + const key = `${name}@1.0`; + const peerKey = `${key}(peer@1.0)`; + data.snapshots[peerKey] = data.snapshots[key]; + delete data.snapshots[key]; + if (suffixedPackages) { + data.packages[peerKey] = data.packages[key]; + delete data.packages[key]; + } + } + data.snapshots["parser@1.0(peer@1.0)"].dependencies.helper = + "1.0(peer@1.0)"; + write("pnpm-lock.yaml", JSON.stringify(data)); + const original = revision(); + expect(original.dependencies).toContain("helper@1.0(peer@1.0)"); + const key = suffixedPackages ? "helper@1.0(peer@1.0)" : "helper@1.0"; + data.packages[key].resolution.integrity = "changed-peer-dependency"; + write("pnpm-lock.yaml", JSON.stringify(data)); + expect(revision().revision).not.toBe(original.revision); + }); + + it("covers real ingestion inputs and agrees with source and built revisions", () => { + const repository = resolve(import.meta.dirname, "../../.."); + const generated = JSON.parse( + execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { generateIngestionRevision } from ${JSON.stringify(new URL("../../../scripts/generate-ingestion-revision.mjs", import.meta.url).href)}; console.log(JSON.stringify(generateIngestionRevision()));`, + ], + { cwd: repository, encoding: "utf8" }, + ), + ); + expect(generated.files).toEqual( + expect.arrayContaining([ + "packages/registry/src/build.ts", + "packages/registry/src/source.ts", + "packages/registry/src/html-index.ts", + "packages/registry/src/zip.ts", + "packages/context/src/package-builder.ts", + "packages/context/src/build.ts", + "packages/context/src/html.ts", + ]), + ); + expect(generated.files).not.toContain("packages/context/src/cli.ts"); + for (const entry of ["src/fingerprint.ts", "dist/fingerprint.js"]) { + const revision = execFileSync( + process.execPath, + [ + ...(entry.startsWith("src/") ? ["--import", "tsx"] : []), + "--input-type=module", + "--eval", + `import { getIngestionRevision } from ${JSON.stringify(new URL(entry, new URL("../", import.meta.url)).href)}; console.log(getIngestionRevision());`, + ], + { encoding: "utf8" }, + ).trim(); + expect(revision).toBe(generated.revision); + } + }); }); diff --git a/packages/registry/src/publication.test.ts b/packages/registry/src/publication.test.ts index 64f155f..1c059ea 100644 --- a/packages/registry/src/publication.test.ts +++ b/packages/registry/src/publication.test.ts @@ -29,6 +29,10 @@ describe("publication freshness through the CLI", () => { let archiveDownloads: number; let conflict: boolean; let legacy: boolean; + let dropUploadResponse: boolean; + let metadataReads: number; + let entryPoint: string; + let preload: string | undefined; let repoUrl: string; const docs = "# Documentation\n\n## Getting started\n\nThis documentation explains how to configure and run the application with a complete example.\n"; @@ -80,6 +84,10 @@ describe("publication freshness through the CLI", () => { archiveDownloads = 0; conflict = false; legacy = false; + dropUploadResponse = false; + metadataReads = 0; + entryPoint = "src/cli.ts"; + preload = undefined; server = createServer((request, response) => { if (request.url?.startsWith("/source/")) { @@ -90,6 +98,7 @@ describe("publication freshness through the CLI", () => { return; } if (request.method !== "POST") { + metadataReads++; response.writeHead(metadata ? 200 : 404, { "Content-Type": "application/json", }); @@ -128,7 +137,9 @@ describe("publication freshness through the CLI", () => { if (!values.name || !values.version) throw new Error("Missing package identity"); metadata = { - registry: "custom", + registry: decodeURIComponent( + request.url?.split("/")[2] ?? "custom", + ), name: values.name, version: values.version, source_commit: values.source_commit, @@ -138,6 +149,12 @@ describe("publication freshness through the CLI", () => { } finally { db.close(); } + if (dropUploadResponse) { + dropUploadResponse = false; + conflict = true; + request.socket.destroy(); + return; + } response.end("{}"); }); }); @@ -176,9 +193,9 @@ describe("publication freshness through the CLI", () => { const result = await run( process.execPath, [ - "--import", - "tsx", - "src/cli.ts", + ...(entryPoint.startsWith("src/") ? ["--import", "tsx"] : []), + ...(preload ? ["--import", pathToFileURL(preload).href] : []), + entryPoint, command, ...args, "--dir", @@ -300,4 +317,101 @@ describe("publication freshness through the CLI", () => { ); expect(uploads).toBe(1); }, 90_000); + + it("recovers a lost upload response followed by a conflict only after verifying metadata", async () => { + defineArchive(); + dropUploadResponse = true; + const published = await cli("publish-all"); + expect(published.status).toBe(0); + expect(published.output).toContain("Succeeded: 1"); + expect(uploads).toBe(2); + expect(metadataReads).toBe(2); + expect(currentMetadata().build_fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(false); + }, 60_000); + + it.each([ + "build_fingerprint", + "ingestion_revision", + ] as const)("rejects a conflicting upload with mismatched %s", async (field) => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + metadata = { ...currentMetadata(), [field]: "different-artifact" }; + conflict = true; + const failed = await cli("publish-all", "--force"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("409 Conflict"); + expect(failed.output).toContain("rebuilt artifact is preserved"); + expect(existsSync(join(root, "output", "custom-docs@1.0.db"))).toBe(true); + }, 60_000); + + it("summarizes unchanged versions without printing each skip", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + const skipped = await cli("publish-all"); + expect(skipped.status).toBe(0); + expect(skipped.output).toContain("Skipped: 1"); + expect(skipped.output).toContain("build inputs unchanged: 1"); + expect(skipped.output).not.toContain("Skipping custom/docs@1.0"); + }, 60_000); + + it("does not request metadata when forcing publication", async () => { + defineArchive(); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + const previousReads = metadataReads; + expect((await cli("publish-all", "--force")).status).toBe(0); + expect(metadataReads).toBe(previousReads); + expect(uploads).toBe(2); + }, 60_000); + + it("builds identical fingerprints through the source and built CLIs", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + const sourceMetadata = currentMetadata(); + entryPoint = "dist/cli.js"; + expect((await cli("publish", "docs", "--force")).status).toBe(0); + expect(currentMetadata()).toEqual(sourceMetadata); + expect(uploads).toBe(2); + }, 60_000); + + it("warns and skips removed published tags while publishing subsequent versions", async () => { + mkdirSync(join(root, "definitions", "npm")); + writeFileSync( + join(root, "definitions", "npm", "docs.yaml"), + `name: docs\nversions:\n - min_version: "1.0"\n source:\n type: git\n url: ${repoUrl}\n docs_path: docs\n`, + ); + expect((await cli("publish", "docs", "1.0")).status).toBe(0); + execFileSync("git", ["tag", "-d", "v1.0"], { cwd: join(root, "repo") }); + preload = join(root, "versions.mjs"); + writeFileSync( + preload, + `const original = globalThis.fetch; +globalThis.fetch = (url, init) => String(url).startsWith("https://registry.npmjs.org/") + ? Promise.resolve(new Response(JSON.stringify({ versions: { "1.0": {} } }))) + : original(url, init);\n`, + ); + writeFileSync( + join(root, "definitions", "npm", "zzz.yaml"), + `name: zzz\nversions:\n - versions: ["1.0"]\n source:\n type: zip\n url: ${url}/source/{version}.zip\n docs_path: docs\n`, + ); + const published = await cli("publish-all"); + expect(published.status).toBe(0); + expect(published.output).toContain("WARNING npm/docs@1.0"); + expect(published.output).toContain("source tag unavailable: 1"); + expect(published.output).toContain("Succeeded: 1"); + expect(published.output).toContain("Skipped: 1"); + expect(published.output).toContain("Failed: 0"); + expect(uploads).toBe(2); + }, 60_000); + + it("reports unreachable source repositories as failures", async () => { + defineGit(); + expect((await cli("publish", "docs")).status).toBe(0); + repoUrl = pathToFileURL(join(root, "missing-repository")).href; + defineGit(); + const failed = await cli("publish-all"); + expect(failed.status).not.toBe(0); + expect(failed.output).toContain("Failed: 1"); + expect(failed.output).not.toContain("source tag unavailable"); + }, 60_000); }); diff --git a/packages/registry/src/publication.ts b/packages/registry/src/publication.ts index d291e52..b2244be 100644 --- a/packages/registry/src/publication.ts +++ b/packages/registry/src/publication.ts @@ -5,12 +5,14 @@ import { getHeadCommit, } from "./build.js"; import { isVersioned, type PackageDefinition } from "./definition.js"; -import { createBuildFingerprint, resolveBuildSource } from "./fingerprint.js"; +import { createBuildFingerprint } from "./fingerprint.js"; +import { formatBuilt } from "./format.js"; import { checkPackageExists, type PackageMetadata, publishPackage, } from "./publish.js"; +import { resolveBuildSource } from "./source.js"; /** Legacy metadata retains the old skip policy until a forced migration. */ export function publicationSkipReason( @@ -44,15 +46,18 @@ export async function publishDefinition( definition: PackageDefinition, version: string, outputDir: string, - options: { force?: boolean; log?: (message: string) => void } = {}, + options: { + force?: boolean; + log?: (message: string) => void; + quietSkips?: boolean; + onSkip?: (reason: string) => void; + } = {}, ): Promise { const log = options.log ?? console.log; const id = `${definition.registry}/${definition.name}@${version}`; - const existing = await checkPackageExists( - definition.registry, - definition.name, - version, - ); + const existing = options.force + ? null + : await checkPackageExists(definition.registry, definition.name, version); const reason = publicationSkipReason( definition, version, @@ -60,18 +65,15 @@ export async function publishDefinition( options.force, ); if (reason) { - log(`Skipping ${id} (${reason})`); + options.onSkip?.(reason); + if (!options.quietSkips) log(`Skipping ${id} (${reason})`); return; } log(`Building ${id}...`); const result = isVersioned(definition) ? await buildFromDefinition(definition, version, outputDir) : await buildUnversioned(definition, outputDir); - const skipped = - result.skippedFiles > 0 ? `, ${result.skippedFiles} files skipped` : ""; - log( - `Built: ${result.path} (${result.sectionCount} sections, ${result.totalTokens} tokens${skipped})`, - ); + log(`Built: ${result.path} (${formatBuilt(result)})`); log(`Publishing ${id}...`); await publishPackage( definition.registry, diff --git a/packages/registry/src/publish.test.ts b/packages/registry/src/publish.test.ts index 33b3f93..88163c8 100644 --- a/packages/registry/src/publish.test.ts +++ b/packages/registry/src/publish.test.ts @@ -70,8 +70,21 @@ describe("publish", () => { process.env.REGISTRY_SERVER_URL = `http://127.0.0.1:${port}`; process.env.REGISTRY_PUBLISH_KEY = "test-key"; + const published = publishPackage("npm", "preact", "latest", dbPath); + await expect(published).rejects.toThrow(/403 Forbidden — bad key/); + await expect(published).rejects.toThrow( + `rebuilt artifact is preserved at ${dbPath}`, + ); + }); + + it("does not treat a missing upload endpoint as a successful publication", async () => { + const { server, url, hits } = await flakyServer(0, 404); + running = server; + process.env.REGISTRY_SERVER_URL = url; + process.env.REGISTRY_PUBLISH_KEY = "test-key"; await expect( publishPackage("npm", "preact", "latest", dbPath), - ).rejects.toThrow(/403 Forbidden — bad key/); + ).rejects.toThrow("404"); + expect(hits()).toBe(1); }); }); diff --git a/packages/registry/src/publish.ts b/packages/registry/src/publish.ts index d0fe030..3f316a4 100644 --- a/packages/registry/src/publish.ts +++ b/packages/registry/src/publish.ts @@ -7,10 +7,20 @@ */ import { readFileSync } from "node:fs"; +import { initDatabase, openDatabase } from "@neuledge/context"; import pRetry, { AbortError } from "p-retry"; const DEFAULT_SERVER_URL = "https://api.context.neuledge.com"; +class RegistryRequestError extends Error { + constructor( + public readonly status: number, + message: string, + ) { + super(message); + } +} + /** * The registry server occasionally drops a connection or returns 5xx under load. * A single blip used to fail the whole nightly publish — 48 packages succeed and @@ -25,10 +35,12 @@ function requestWithRetry( return pRetry( async () => { const response = await fetch(url, init); - if (response.ok || response.status === 404) return response; + if (response.ok || (response.status === 404 && init.method !== "POST")) + return response; const body = await response.text().catch(() => ""); - const error = new Error( + const error = new RegistryRequestError( + response.status, `${describe()}: ${response.status} ${response.statusText}${body ? ` — ${body}` : ""}`, ); if (response.status < 500) throw new AbortError(error); @@ -117,13 +129,55 @@ export async function publishPackage( () => `Failed to publish ${registry}/${name}@${version}`, ); } catch (error) { - if (error instanceof Error && error.message.includes(": 409 ")) { - throw new Error( - `${error.message}. The registry rejected replacement of this published version. ` + - `The rebuilt artifact is preserved at ${dbPath}. Use a registry-supported replacement or artifact revision; --force cannot override the server's policy.`, - { cause: error }, - ); + let message = error instanceof Error ? error.message : String(error); + if (error instanceof RegistryRequestError && error.status === 409) { + try { + if (await matchesPublishedArtifact(registry, name, version, dbPath)) + return; + } catch (verificationError) { + message += `. Could not verify published metadata: ${verificationError instanceof Error ? verificationError.message : String(verificationError)}`; + } + message += + ". The registry rejected replacement of this published version. Use a registry-supported replacement or artifact revision; --force cannot override the server's policy"; } - throw error; + throw new Error( + `${message}. The rebuilt artifact is preserved at ${dbPath}.`, + { + cause: error, + }, + ); + } +} + +/** A lost upload response is recoverable only with matching artifact metadata. */ +async function matchesPublishedArtifact( + registry: string, + name: string, + version: string, + dbPath: string, +): Promise { + await initDatabase(); + const db = openDatabase(dbPath, { readonly: true }); + let values: Record; + try { + values = Object.fromEntries( + ( + db.prepare("SELECT key, value FROM meta").all() as { + key: string; + value: string; + }[] + ).map(({ key, value }) => [key, value]), + ); + } finally { + db.close(); } + if (!values.build_fingerprint || !values.ingestion_revision) return false; + const published = await checkPackageExists(registry, name, version); + return ( + published?.registry === registry && + published.name === name && + published.version === version && + published.build_fingerprint === values.build_fingerprint && + published.ingestion_revision === values.ingestion_revision + ); } diff --git a/packages/registry/src/source.ts b/packages/registry/src/source.ts new file mode 100644 index 0000000..7546744 --- /dev/null +++ b/packages/registry/src/source.ts @@ -0,0 +1,32 @@ +import { + constructTag, + isGitVersionEntry, + isVersioned, + type PackageDefinition, + resolveUrl, + resolveVersionEntry, +} from "./definition.js"; + +/** Resolve the same release inputs for both building and fingerprinting. */ +export function resolveBuildSource( + definition: PackageDefinition, + version: string, +) { + if (!isVersioned(definition)) return definition.source; + const entry = resolveVersionEntry(definition, version); + if (!entry) { + throw new Error( + `No version entry matches ${version} in ${definition.name}`, + ); + } + if (isGitVersionEntry(entry)) { + return { ...entry.source, ref: constructTag(entry.tag_pattern, version) }; + } + return { + ...entry.source, + url: resolveUrl(entry.source.url, version), + ...(entry.source.type === "zip" && entry.source.docs_path + ? { docs_path: resolveUrl(entry.source.docs_path, version) } + : {}), + }; +} diff --git a/registry/README.md b/registry/README.md index e78c9f2..837cfce 100644 --- a/registry/README.md +++ b/registry/README.md @@ -205,7 +205,7 @@ before building. A matching fingerprint skips work; changed build inputs cause a rebuild and an upload to the same package version. Inputs include the actual Git commit (or pinned explicit release), the effective source settings and package metadata, and an automatically generated ingestion revision. `docs_path`, -exclusions, language, source URL/ref, description, or ingestion changes can +exclusions, Git language filters, source URL/ref, description, or ingestion changes can therefore rebuild a package even when the source commit is unchanged. YAML key order, comments, exclusion order/duplicates, and unrelated release entries do not change the fingerprint. @@ -242,9 +242,20 @@ pnpm --filter @neuledge/registry registry publish-all --force `--force` bypasses freshness checks and rebuilds; it does not override the server's replacement policy. Both automatic rebuilds and forced builds upload to the same registry/name/version. Servers that allow replacement accept the new artifact; -servers that return HTTP 409 produce a clear failure and leave the rebuilt `.db` -on disk. Such servers need their own supported replacement or artifact revision -policy. Successful `publish-all` uploads remove local build artifacts as before. +servers that return HTTP 409 are checked for a matching package identity, +`build_fingerprint`, and `ingestion_revision`. A match confirms that the upload +already landed, even if its response was lost and the retry returned 409. Missing +or mismatched metadata produces a clear failure. All failed uploads report the +preserved `.db` path; successful `publish-all` uploads remove the local artifact. + +`--force` rebuilds the derived package using the current source cache; it does not +refresh downloaded HTML. Delete `.cache/context/html-index` before rebuilding if +the publisher corrected an existing release's pages. + +Bulk publishing summarizes skip reasons instead of logging every unchanged +version. Missing Git tags are warned about and skipped, including tags removed +after a package was published. Unreachable repositories and other transport +failures remain errors. ## Before opening a PR diff --git a/scripts/generate-ingestion-revision.mjs b/scripts/generate-ingestion-revision.mjs index ec6da1c..553044b 100644 --- a/scripts/generate-ingestion-revision.mjs +++ b/scripts/generate-ingestion-revision.mjs @@ -25,163 +25,165 @@ function canonical(value) { return value; } -export function generateIngestionRevision(root = repository) { - const files = new Map(); - const dependencies = new Map(); - const lock = parse(readFileSync(resolve(root, "pnpm-lock.yaml"), "utf8")); +function lockedDependency(state, name, version) { + const key = `${name}@${version}`; + if (state.dependencies.has(key)) return; + const snapshot = state.lock.snapshots[key]; + // pnpm snapshots retain peer suffixes; package records normally omit them. + // Also accept suffix-bearing package records rather than assuming one layout. + const pkg = + state.lock.packages[key] ?? state.lock.packages[key.replace(/\(.*$/, "")]; + if (!snapshot || !pkg) + throw new Error(`Missing locked ingestion dependency: ${key}`); + state.dependencies.set(key, { package: pkg, snapshot }); + for (const [child, childVersion] of Object.entries({ + ...snapshot.dependencies, + ...snapshot.optionalDependencies, + })) { + lockedDependency(state, child, childVersion); + } +} + +function sourceFile(file, content = readFileSync(file, "utf8")) { + return ts.createSourceFile(file, content, ts.ScriptTarget.Latest, true); +} - function dependency(name, version) { - const key = `${name}@${version}`; - if (dependencies.has(key)) return; - const snapshot = lock.snapshots[key]; - const packageKey = key.replace(/\(.*$/, ""); - const pkg = lock.packages[packageKey]; - if (!snapshot || !pkg) - throw new Error(`Missing locked ingestion dependency: ${key}`); - dependencies.set(key, { package: pkg, snapshot }); - for (const [child, childVersion] of Object.entries({ - ...snapshot.dependencies, - ...snapshot.optionalDependencies, - })) { - dependency(child, childVersion); +function followContextExports(state, names) { + const index = resolve(state.root, "packages/context/src/index.ts"); + const unresolved = names && new Set(names); + for (const statement of sourceFile(index).statements) { + if ( + !ts.isExportDeclaration(statement) || + statement.isTypeOnly || + !statement.moduleSpecifier + ) + continue; + const exported = + statement.exportClause && ts.isNamedExports(statement.exportClause) + ? statement.exportClause.elements + .filter((item) => !item.isTypeOnly) + .map((item) => item.name.text) + : undefined; + if (!names || !exported || exported.some((name) => names.includes(name))) { + followImport(state, index, statement.moduleSpecifier.text); + for (const name of exported ?? []) unresolved?.delete(name); } } + if (unresolved?.size) + throw new Error( + `Unresolved ingestion exports: ${[...unresolved].join(", ")}`, + ); +} - function follow(file, specifier, names) { - if (specifier.startsWith("node:")) return; - if (specifier.startsWith(".")) { - visit(resolve(dirname(file), specifier.replace(/\.js$/, ".ts"))); - return; - } - const packageName = specifier.startsWith("@") - ? specifier.split("/").slice(0, 2).join("/") - : specifier.split("/")[0]; - if (packageName === "@neuledge/context") { - // Resolve the APIs actually imported from the workspace barrel. Changes - // to the CLI or server must not invalidate documentation packages. - const index = resolve(root, "packages/context/src/index.ts"); - if (specifier !== packageName) { - visit( - resolve( - root, - "packages/context/src", - specifier.slice(packageName.length + 1).replace(/\.js$/, ".ts"), - ), - ); - return; - } - const source = ts.createSourceFile( - index, - readFileSync(index, "utf8"), - ts.ScriptTarget.Latest, - true, +function followImport(state, file, specifier, names) { + if (specifier.startsWith("node:")) return; + if (specifier.startsWith(".")) { + visitSource( + state, + resolve(dirname(file), specifier.replace(/\.js$/, ".ts")), + ); + return; + } + const packageName = specifier.startsWith("@") + ? specifier.split("/").slice(0, 2).join("/") + : specifier.split("/")[0]; + if (packageName === "@neuledge/context") { + if (specifier === packageName) { + followContextExports(state, names); + } else { + visitSource( + state, + resolve( + state.root, + "packages/context/src", + specifier.slice(packageName.length + 1).replace(/\.js$/, ".ts"), + ), ); - const unresolved = names && new Set(names); - for (const statement of source.statements) { - if ( - !ts.isExportDeclaration(statement) || - statement.isTypeOnly || - !statement.moduleSpecifier - ) - continue; - const exported = - statement.exportClause && ts.isNamedExports(statement.exportClause) - ? statement.exportClause.elements - .filter((item) => !item.isTypeOnly) - .map((item) => item.name.text) - : undefined; - if ( - !names || - !exported || - exported.some((name) => names.includes(name)) - ) { - follow(index, statement.moduleSpecifier.text); - for (const name of exported ?? []) unresolved?.delete(name); - } - } - if (unresolved?.size) - throw new Error( - `Unresolved ingestion exports: ${[...unresolved].join(", ")}`, - ); - return; } - const importer = relative(root, file) - .replaceAll("\\", "/") - .split("/") - .slice(0, 2) - .join("/"); - const manifest = lock.importers[importer]; - const entry = - manifest.dependencies?.[packageName] ?? - manifest.optionalDependencies?.[packageName]; - if (!entry) - throw new Error( - `Undeclared ingestion dependency: ${packageName} in ${file}`, - ); - dependency(packageName, entry.version); + return; } - - function visit(file) { - const path = relative(root, file).replaceAll("\\", "/"); - if (files.has(path)) return; - const content = readFileSync(file, "utf8").replaceAll("\r\n", "\n"); - files.set(path, content); - const source = ts.createSourceFile( - file, - content, - ts.ScriptTarget.Latest, - true, + const importer = relative(state.root, file) + .replaceAll("\\", "/") + .split("/") + .slice(0, 2) + .join("/"); + const manifest = state.lock.importers[importer]; + const entry = + manifest.dependencies?.[packageName] ?? + manifest.optionalDependencies?.[packageName]; + if (!entry) + throw new Error( + `Undeclared ingestion dependency: ${packageName} in ${file}`, ); - function walk(node) { - if (ts.isImportDeclaration(node)) { - const clause = node.importClause; - if (clause?.isTypeOnly) return; - const bindings = clause?.namedBindings; - const names = - bindings && ts.isNamedImports(bindings) - ? bindings.elements - .filter((item) => !item.isTypeOnly) - .map((item) => (item.propertyName ?? item.name).text) - : undefined; - if (names?.length === 0 && !clause?.name) return; - follow(file, node.moduleSpecifier.text, names); - } else if ( - ts.isExportDeclaration(node) && - !node.isTypeOnly && - node.moduleSpecifier - ) { - follow(file, node.moduleSpecifier.text); - } else if ( - ts.isCallExpression(node) && - node.arguments[0] && - ts.isStringLiteral(node.arguments[0]) - ) { - const expression = node.expression.getText(source); - if ( - node.expression.kind === ts.SyntaxKind.ImportKeyword || - /^(?:_?require)(?:\.resolve)?$/.test(expression) - ) { - follow(file, node.arguments[0].text); - } - } - ts.forEachChild(node, walk); + lockedDependency(state, packageName, entry.version); +} + +function visitNode(state, file, source, node) { + if (ts.isImportDeclaration(node)) { + const clause = node.importClause; + if (clause?.isTypeOnly) return; + const bindings = clause?.namedBindings; + const names = + bindings && ts.isNamedImports(bindings) + ? bindings.elements + .filter((item) => !item.isTypeOnly) + .map((item) => (item.propertyName ?? item.name).text) + : undefined; + if (names?.length === 0 && !clause?.name) return; + followImport(state, file, node.moduleSpecifier.text, names); + } else if ( + ts.isExportDeclaration(node) && + !node.isTypeOnly && + node.moduleSpecifier + ) { + followImport(state, file, node.moduleSpecifier.text); + } else if ( + ts.isCallExpression(node) && + node.arguments[0] && + ts.isStringLiteral(node.arguments[0]) + ) { + const expression = node.expression.getText(source); + if ( + node.expression.kind === ts.SyntaxKind.ImportKeyword || + /^(?:_?require)(?:\.resolve)?$/.test(expression) + ) { + followImport(state, file, node.arguments[0].text); } - walk(source); } + ts.forEachChild(node, (child) => visitNode(state, file, source, child)); +} - visit(resolve(root, "packages/registry/src/build.ts")); - dependency( +function visitSource(state, file) { + const path = relative(state.root, file).replaceAll("\\", "/"); + if (state.files.has(path)) return; + const content = readFileSync(file, "utf8").replaceAll("\r\n", "\n"); + state.files.set(path, content); + const source = sourceFile(file, content); + visitNode(state, file, source, source); +} + +export function generateIngestionRevision(root = repository) { + const state = { + root, + files: new Map(), + dependencies: new Map(), + lock: parse(readFileSync(resolve(root, "pnpm-lock.yaml"), "utf8")), + }; + visitSource(state, resolve(root, "packages/registry/src/build.ts")); + lockedDependency( + state, "typescript", - lock.importers["packages/registry"].devDependencies.typescript.version, + state.lock.importers["packages/registry"].devDependencies.typescript + .version, ); // Include the algorithm itself so changes in what we hash also invalidate. - files.set( + state.files.set( "scripts/generate-ingestion-revision.mjs", readFileSync(scriptPath, "utf8").replaceAll("\r\n", "\n"), ); const inputs = canonical({ - files: Object.fromEntries(files), - dependencies: Object.fromEntries(dependencies), + files: Object.fromEntries(state.files), + dependencies: Object.fromEntries(state.dependencies), }); return { revision: createHash("sha256").update(JSON.stringify(inputs)).digest("hex"), diff --git a/turbo.json b/turbo.json index 65f2bea..e368888 100644 --- a/turbo.json +++ b/turbo.json @@ -1,11 +1,18 @@ { "$schema": "https://turborepo.com/schema.json", - "globalDependencies": ["scripts/generate-ingestion-revision.mjs"], "tasks": { "build": { "dependsOn": ["^build"], "outputs": ["dist/**"] }, + "@neuledge/registry#build": { + "dependsOn": ["^build"], + "outputs": ["dist/**"], + "inputs": [ + "$TURBO_DEFAULT$", + "$TURBO_ROOT$/scripts/generate-ingestion-revision.mjs" + ] + }, "test": { "dependsOn": ["build"] },