From b2bbe8d2a684dd1567a5eb5bad8d944e7edc7194 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Mon, 2 Nov 2026 17:26:03 +0100 Subject: [PATCH 1/9] Warn when links to the API docs point at pages that do not exist Links from narrative docs and redirects into /docs/api/doc/ were never checked, so a removed endpoint broke them silently. The assembler now records those links, compares them with the pages the API Explorer generated, warns per broken link, and writes api-link-report.json. Co-Authored-By: Claude Sonnet 5.5 --- docs/syntax/links.md | 10 +++ src/Elastic.ApiExplorer/OpenApiGenerator.cs | 10 +++ .../Diagnostics/ApiLinkReference.cs | 40 ++++++++++ .../Diagnostics/DiagnosticsCollector.cs | 4 + .../Diagnostics/IDiagnosticsCollector.cs | 2 + src/Elastic.Markdown/IO/DocumentationSet.cs | 3 + .../Myst/Directives/DirectiveLinkValidator.cs | 16 ++++ .../DiagnosticLinkInlineParser.cs | 17 +++++ .../Building/ApiLinkValidator.cs | 73 +++++++++++++++++++ .../Building/AssemblerBuildService.cs | 13 +++- .../Building/AssemblerOpenApiBuildStep.cs | 14 +++- .../OpenApiGeneratorCatalogSplitTests.cs | 22 ++++++ .../ApiLinkValidatorTests.cs | 59 +++++++++++++++ .../Diagnostics/ApiLinkMatcherTests.cs | 31 ++++++++ .../Inline/ApiLinkTests.cs | 38 ++++++++++ 15 files changed, 347 insertions(+), 5 deletions(-) create mode 100644 src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs create mode 100644 src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs create mode 100644 tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs create mode 100644 tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs create mode 100644 tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs diff --git a/docs/syntax/links.md b/docs/syntax/links.md index 78ea3b894a..d31ca5d835 100644 --- a/docs/syntax/links.md +++ b/docs/syntax/links.md @@ -113,6 +113,16 @@ Link to websites and resources outside the Elastic docs: External links display an arrow indicator next to the link text. +### Links to API reference pages + +Links to `https://www.elastic.co/docs/api/doc/...` pages, including `redirects.yml` targets, are checked when the assembler builds with the API Explorer enabled. Each link is compared with the API pages that build generates. A link that doesn't match produces a warning, and the assembler writes the full list to `api-link-report.json` next to `redirects.json`. + +```markdown +[Search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) +``` + +If an endpoint no longer exists in the current API docs, point the link at the versioned tree that still has it, for example `https://www.elastic.co/docs/api/doc/kibana/v8/...`. Only the page is checked, not the `#anchor`. Old bump.sh URLs that resolve only through edge redirects are also reported, so link to the canonical URL. + ### Linked images Wrap an image in a link to make the image itself clickable. Combine the image syntax with the link syntax: diff --git a/src/Elastic.ApiExplorer/OpenApiGenerator.cs b/src/Elastic.ApiExplorer/OpenApiGenerator.cs index 96a47f8010..e5864b158a 100644 --- a/src/Elastic.ApiExplorer/OpenApiGenerator.cs +++ b/src/Elastic.ApiExplorer/OpenApiGenerator.cs @@ -15,6 +15,7 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Products; using Elastic.Documentation.Configuration.Toc; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Navigation; using Elastic.Documentation.Site.FileProviders; using Elastic.Documentation.Site.Navigation; @@ -66,6 +67,10 @@ public class OpenApiGenerator( private readonly VersionIndexClient _versionIndexClient = versionIndexClient ?? new VersionIndexClient(); private readonly IOpenApiSpecificationReader _openApiReader = openApiReader ?? OpenApiReader.Instance; private readonly ConcurrentDictionary> _documents = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _generatedUrls = new(StringComparer.Ordinal); + + /// Normalized URL path of every page this generator rendered, including redirect stubs for collapsed operations. + public IReadOnlyCollection GeneratedUrls => [.. _generatedUrls.Keys]; public LandingNavigationItem CreateNavigation( string apiUrlSuffix, @@ -580,6 +585,8 @@ Cancel ctx { var pages = new List<(INavigationItem Item, IApiModel Model)>(); CollectPages(root, pages); + foreach (var (item, _) in pages) + RecordUrl(item.Url); // Pages of one version render in parallel. When two items share a URL the last one wins, // as it did when pages were written sequentially in depth-first order. @@ -596,12 +603,15 @@ Cancel ctx }).ConfigureAwait(false); } + private void RecordUrl(string url) => _ = _generatedUrls.TryAdd(ApiLinkMatcher.Normalize(url), 0); + /// Former per-operation URLs of a collapsed page forward to it, keeping any #fragment. private async Task WriteRedirects(OperationNavigationItem page, Cancel ctx) { var html = ApiRedirectPage.Html(page.Url); foreach (var alias in page.AliasUrls) { + RecordUrl(alias); await WriteSpecSibling( ApiOutputPaths.RelativeHtmlFile(alias, context.UrlPathPrefix), async (stream, c) => diff --git a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs new file mode 100644 index 0000000000..50edd6fe91 --- /dev/null +++ b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs @@ -0,0 +1,40 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Documentation.Diagnostics; + +/// A link to an API reference page, recorded during the Markdown build and checked once the API pages exist. +public readonly record struct ApiLinkReference(string Url, string File, int Line, int Column, int Length); + +public static class ApiLinkMatcher +{ + public const string ApiDocPrefix = "/docs/api/doc/"; + + /// + /// Returns the normalized page path (no query, fragment or trailing slash, lowercase) when + /// targets an API reference page on elastic.co. + /// + public static bool TryGetApiPath(string? url, out string path) + { + path = string.Empty; + if (string.IsNullOrWhiteSpace(url) || !Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri)) + return false; + if (uri.Scheme is not ("http" or "https")) + return false; + if ( + !uri.Host.Equals("www.elastic.co", StringComparison.OrdinalIgnoreCase) + && !uri.Host.Equals("elastic.co", StringComparison.OrdinalIgnoreCase) + ) + return false; + + var normalized = Normalize(uri.AbsolutePath); + if (!normalized.StartsWith(ApiDocPrefix, StringComparison.Ordinal)) + return false; + + path = normalized; + return true; + } + + public static string Normalize(string path) => Uri.UnescapeDataString(path).TrimEnd('/').ToLowerInvariant(); +} diff --git a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs index 86bca5be89..6f8747a6f9 100644 --- a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs @@ -41,6 +41,8 @@ public class DiagnosticsCollector( public ConcurrentBag CrossLinks { get; } = []; + public ConcurrentBag ApiLinks { get; } = []; + public bool NoHints { get; set; } public bool IsStarted => _readerStarted; @@ -140,6 +142,8 @@ public virtual async Task StopAsync(Cancel cancellationToken) public void EmitCrossLink(string link) => CrossLinks.Add(link); + public void EmitApiLink(ApiLinkReference link) => ApiLinks.Add(link); + public virtual void Write(Diagnostic diagnostic) { // IncrementSeverityCount MUST run unconditionally and before Channel.Write. diff --git a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs index 4622e521af..c16b74e10a 100644 --- a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs @@ -17,6 +17,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable DiagnosticsChannel Channel { get; } ConcurrentBag CrossLinks { get; } + ConcurrentBag ApiLinks { get; } ConcurrentDictionary OffendingFiles { get; } ConcurrentDictionary InUseSubstitutionKeys { get; } @@ -49,6 +50,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable void Write(Diagnostic diagnostic); void CollectUsedSubstitutionKey(ReadOnlySpan key); void EmitCrossLink(string link); + void EmitApiLink(ApiLinkReference link); void EmitError(IFileInfo file, string message, Exception? e = null) => EmitError(file.FullName, message, e); diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index b40d79b2de..62a062d2d1 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -13,6 +13,7 @@ using Elastic.Documentation.Configuration.ReleaseNotes; using Elastic.Documentation.Configuration.Toc; using Elastic.Documentation.Configuration.Toc.CliReference; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Documentation.Links.CrossLinks; using Elastic.Documentation.Navigation; @@ -181,6 +182,8 @@ void ValidateExists(string from, string to, IReadOnlyDictionary { if (!Uri.TryCreate(to, UriKind.Absolute, out _)) Context.EmitError(Configuration.SourceFile, $"Redirect {from} points to {to} which is not a valid URI"); + else if (ApiLinkMatcher.TryGetApiPath(to, out _)) + Context.Collector.EmitApiLink(new ApiLinkReference(to, Configuration.SourceFile.FullName, 1, 1, to.Length)); return; } diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs index 63b6c39a98..7cc2a55f74 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; using Markdig; @@ -90,7 +91,22 @@ internal static class DirectiveLinkValidator return url; if (IsExternal(trimmed)) + { + if (ApiLinkMatcher.TryGetApiPath(trimmed, out _)) + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference( + trimmed, + block.CurrentFile.FullName, + block.Line + 1, + System.Math.Max(block.Column, 1), + trimmed.Length + ) + ); return url; + } if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) && CrossLinkValidator.IsCrossLink(uri)) return ResolveCrossLink(url, uri, block, context); diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index b9fbf1112b..d81661e54d 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; @@ -153,11 +154,27 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc link.SetData("isCrossLink", false); if (ValidateExternalUri(link, processor, uri)) + { + RecordApiLink(link, context, url); return; + } ProcessInternalLink(link, processor, context); } + private static void RecordApiLink(LinkInline link, ParserContext context, string? url) + { + if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _)) + return; + + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, link.Line + 1, Math.Max(link.Column, 1), url!.Length) + ); + } + private static bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url) { if (string.IsNullOrEmpty(url)) diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs new file mode 100644 index 0000000000..dfd4230a7f --- /dev/null +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -0,0 +1,73 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Assembler.Building; + +public sealed record ApiLinkReportEntry(string Url, string Source, int Line); + +public sealed record ApiLinkReport(int Checked, IReadOnlyList Broken); + +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(ApiLinkReport))] +internal sealed partial class ApiLinkReportJsonContext : JsonSerializerContext; + +/// Checks links recorded during the Markdown build against the API pages the assembler generated. +public static class ApiLinkValidator +{ + public const string ReportFileName = "api-link-report.json"; + + public static IReadOnlyList FindBroken(IEnumerable links, IReadOnlySet generatedUrls) => + [ + .. links + .Where(l => ApiLinkMatcher.TryGetApiPath(l.Url, out var path) && !generatedUrls.Contains(path)) + .OrderBy(l => l.Url, StringComparer.Ordinal) + .ThenBy(l => l.File, StringComparer.Ordinal) + .ThenBy(l => l.Line) + ]; + + /// Emits a warning per broken link and writes the report; returns the report path, or null when nothing was checked. + public static async Task ValidateAsync( + IDiagnosticsCollector collector, + IReadOnlySet generatedUrls, + IFileSystem fileSystem, + string outputDirectory, + Cancel ctx + ) + { + // Without generated pages every API link would look broken, so skip instead of flooding the build. + if (generatedUrls.Count == 0) + return null; + + var links = collector.ApiLinks.ToArray(); + if (links.Length == 0) + return null; + + var broken = FindBroken(links, generatedUrls); + foreach (var link in broken) + { + collector.Write(new Diagnostic + { + Severity = Severity.Warning, + File = link.File, + Line = link.Line, + Column = link.Column, + Length = link.Length, + Message = $"API link '{link.Url}' does not resolve to a generated API page. Point it at an existing page " + + "(for older endpoints use a versioned tree such as /docs/api/doc//v8/...) or confirm the endpoint was removed intentionally." + }); + } + + var report = new ApiLinkReport(links.Length, [.. broken.Select(b => new ApiLinkReportEntry(b.Url, b.File, b.Line))]); + var path = fileSystem.Path.Join(outputDirectory, ReportFileName); + var json = JsonSerializer.Serialize(report, ApiLinkReportJsonContext.Default.ApiLinkReport); + await fileSystem.File.WriteAllTextAsync(path, json, Encoding.UTF8, ctx).ConfigureAwait(false); + return path; + } +} diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs index e2b6dce932..6dbf6fa5d4 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs @@ -176,16 +176,27 @@ await fileSystem.File.ReadAllTextAsync(navigationFileInfo.FullName, ctx), if (exporters.Contains(Exporter.Html)) { var openApiStopwatch = Stopwatch.StartNew(); - catalogEntries = await AssemblerOpenApiBuildStep.BuildAsync( + var openApiResult = await AssemblerOpenApiBuildStep.BuildAsync( logFactory, assembleContext, assembleSources, navigation.TopNav, ctx ); + catalogEntries = openApiResult.CatalogEntries; openApiStopwatch.Stop(); _logger.LogInformation("OpenAPI build step completed in {DurationMs} ms", openApiStopwatch.ElapsedMilliseconds); + var apiLinkReportPath = await ApiLinkValidator.ValidateAsync( + collector, + openApiResult.GeneratedUrls, + assembleContext.WriteFileSystem, + assembleContext.OutputDirectory.FullName, + ctx + ); + if (apiLinkReportPath is not null) + await githubActionsService.SetOutputAsync("api-link-report-path", apiLinkReportPath); + // Build-time sitemap uses current date as placeholder for backwards compatibility. // Production sitemap with correct content_last_updated dates is generated via // `assembler sitemap` after ES indexing, which overwrites this file. diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs index dbf4c21dba..d8407faa97 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs @@ -24,7 +24,7 @@ namespace Elastic.Documentation.Assembler.Building; /// public static class AssemblerOpenApiBuildStep { - public static async Task> BuildAsync( + public static async Task BuildAsync( ILoggerFactory logFactory, AssembleContext assembleContext, AssembleSources assembleSources, @@ -41,14 +41,14 @@ Cancel ctx if (!features.AssemblerApiExplorerEnabled) { logger.LogInformation("Skipping OpenAPI generation: assembler-api-explorer feature flag is disabled"); - return []; + return AssemblerOpenApiResult.Empty; } var owners = DiscoverApiOwners(assembleSources.AssembleSets, assembleContext.Collector); if (owners.Count == 0) { logger.LogInformation("Skipping OpenAPI generation: no API declarations found in assembled docsets"); - return []; + return AssemblerOpenApiResult.Empty; } var stopwatch = Stopwatch.StartNew(); @@ -105,7 +105,8 @@ Cancel ctx assembleContext.OutputWithPathPrefixDirectory.FullName, stopwatch.ElapsedMilliseconds ); - return catalogEntries; + var generatedUrls = planned.SelectMany(p => p.Generator.GeneratedUrls).ToHashSet(StringComparer.Ordinal); + return new AssemblerOpenApiResult(catalogEntries, generatedUrls); } private static async Task WriteApiLlmsTxt(AssembleContext assembleContext, IReadOnlyList catalogEntries, Cancel ctx) @@ -165,3 +166,8 @@ private static void ApplyFeatureFlags(AssemblerDocumentationSet set, IReadOnlyDi } internal sealed record AssemblerApiOwner(AssemblerDocumentationSet Set); + +public sealed record AssemblerOpenApiResult(IReadOnlyList CatalogEntries, IReadOnlySet GeneratedUrls) +{ + public static AssemblerOpenApiResult Empty { get; } = new([], new HashSet()); +} diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs index 980fdc78d5..1d459266fb 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs @@ -56,6 +56,28 @@ public async Task GenerateProducts_DoesNotWriteCatalogPage() context.WriteFileSystem.File.Exists(Path.Join(outputRoot, "api", "index.html")).Should().BeFalse(); } + [Test] + public async Task GenerateProducts_RecordsEveryRenderedPageInGeneratedUrls() + { + var outputRoot = Path.Join(Paths.WorkingDirectoryRoot.FullName, $"api-catalog-split-{Guid.NewGuid():N}"); + var context = CreateGenerateContext(outputRoot); + using var versionIndexClient = new VersionIndexClient(BaseUri, MultiVersionHandler(), sleep: (_, _) => Task.CompletedTask); + var generator = new OpenApiGenerator( + NullLoggerFactory.Instance, + context, + NoopMarkdownStringRenderer.Instance, + versionIndexClient, + CreateSequentialReader(SpecDocument("Elasticsearch main")) + ); + + _ = await generator.GenerateProducts(ctx: TestContext.Current!.Execution.CancellationToken); + + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch"); + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch/group/endpoint-core"); + generator.GeneratedUrls.Should().OnlyContain(url => url == url.ToLowerInvariant() && !url.EndsWith('/')); + generator.GeneratedUrls.Should().Contain(url => url.Contains("/operation/operation-")); + } + [Test] public async Task ResolveCanonicalDocuments_ConfiguredApi_ReturnsSpecUnderItsExplorerKey() { diff --git a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs new file mode 100644 index 0000000000..ef7752e459 --- /dev/null +++ b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Documentation.Assembler.Building; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Build.Tests; + +public class ApiLinkValidatorTests +{ + private static readonly IReadOnlySet Generated = new HashSet + { + "/docs/api/doc/kibana", + "/docs/api/doc/kibana/v8/operation/operation-upgrade", + "/docs/api/doc/kibana/operation/operation-alias" + }; + + private static ApiLinkReference Link(string url, string file = "a.md", int line = 1) => new(url, file, line, 1, url.Length); + + [Test] + public void FindBroken_ExistingPage_IsNotReported() => + ApiLinkValidator.FindBroken([Link("https://www.elastic.co/docs/api/doc/kibana/")], Generated).Should().BeEmpty(); + + [Test] + public void FindBroken_PageOnlyInOlderVersion_ReportsMainTreeLink() + { + var broken = ApiLinkValidator.FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/kibana/operation/operation-upgrade"), + Link("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade#responses") + ], + Generated + ); + broken.Should().ContainSingle().Which.Url.Should().EndWith("/kibana/operation/operation-upgrade"); + } + + [Test] + public void FindBroken_AliasUrl_Resolves() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/kibana/operation/operation-alias")], Generated) + .Should() + .BeEmpty(); + + [Test] + public void FindBroken_MultipleBroken_AreOrderedByUrlThenSource() + { + var broken = ApiLinkValidator.FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/kibana/zzz", "b.md"), + Link("https://www.elastic.co/docs/api/doc/kibana/aaa", "b.md"), + Link("https://www.elastic.co/docs/api/doc/kibana/aaa", "a.md") + ], + Generated + ); + broken.Select(b => (b.Url.Split('/')[^1], b.File)).Should().Equal(("aaa", "a.md"), ("aaa", "b.md"), ("zzz", "b.md")); + } +} diff --git a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs new file mode 100644 index 0000000000..8d89574005 --- /dev/null +++ b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Tests.Diagnostics; + +public class ApiLinkMatcherTests +{ + [Test] + [Arguments("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-x", "/docs/api/doc/kibana/v8/operation/operation-x")] + [Arguments("https://elastic.co/docs/api/doc/kibana/", "/docs/api/doc/kibana")] + [Arguments("http://www.elastic.co/docs/api/doc/Kibana/Group/Endpoint-Foo?x=1#frag", "/docs/api/doc/kibana/group/endpoint-foo")] + public void TryGetApiPath_ApiUrl_ReturnsNormalizedPath(string url, string expected) + { + ApiLinkMatcher.TryGetApiPath(url, out var path).Should().BeTrue(); + path.Should().Be(expected); + } + + [Test] + [Arguments("https://www.elastic.co/docs/solutions/search")] + [Arguments("https://example.com/docs/api/doc/kibana")] + [Arguments("https://www.elastic.co/docs/api/")] + [Arguments("/docs/api/doc/kibana")] + [Arguments("mailto:a@elastic.co")] + [Arguments("")] + [Arguments(null)] + public void TryGetApiPath_NonApiUrl_ReturnsFalse(string? url) => ApiLinkMatcher.TryGetApiPath(url, out _).Should().BeFalse(); +} diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs new file mode 100644 index 0000000000..9c4229c227 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using JetBrains.Annotations; +using Markdig.Syntax.Inlines; + +namespace Elastic.Markdown.Tests.Inline; + +[InheritsTests] +public class ApiLinkRecordedTests() : LinkTestBase( + """ +[Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params) +""" +) +{ + [Test] + public void RecordsApiLinkWithLocation() + { + var link = Collector.ApiLinks.Should().ContainSingle().Subject; + link.Url.Should().Be("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params"); + link.Line.Should().BePositive(); + link.File.Should().EndWith(".md"); + } + + [Test] + public void HasNoErrors() => Collector.Errors.Should().Be(0); +} + +[InheritsTests] +public class NonApiElasticLinkTests() : LinkTestBase(""" +[Docs](https://www.elastic.co/docs/solutions/search) +""") +{ + [Test] + public void RecordsNoApiLink() => Collector.ApiLinks.Should().BeEmpty(); +} From c4dcaac927a040717f3aa0580bf2d59eacf98dfa Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Mon, 2 Nov 2026 17:29:39 +0100 Subject: [PATCH 2/9] Report API links in redirects.yml against the redirects file The redirect check recorded API targets against docset.yml, so a warning pointed at the wrong file. ConfigurationFile now exposes the redirects file and a test covers the recorded location. Co-Authored-By: Claude Sonnet 5.5 --- .../Builder/ConfigurationFile.cs | 3 ++ src/Elastic.Markdown/IO/DocumentationSet.cs | 4 +- .../RedirectApiLinkTests.cs | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs diff --git a/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs b/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs index 625806b40c..63f8d221b0 100644 --- a/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs +++ b/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs @@ -55,6 +55,8 @@ public record ConfigurationFile public Dictionary? Redirects { get; } + public IFileInfo? RedirectsSource { get; } + public HashSet Products { get; private set; } = []; private readonly Dictionary _substitutions = [with(StringComparer.OrdinalIgnoreCase)]; @@ -138,6 +140,7 @@ ProductsConfiguration productsConfig var redirectFile = new RedirectFile(_context); Redirects = redirectFile.Redirects; + RedirectsSource = redirectFile.Source; try { diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 62a062d2d1..b0ba0e74e6 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -183,7 +183,9 @@ void ValidateExists(string from, string to, IReadOnlyDictionary if (!Uri.TryCreate(to, UriKind.Absolute, out _)) Context.EmitError(Configuration.SourceFile, $"Redirect {from} points to {to} which is not a valid URI"); else if (ApiLinkMatcher.TryGetApiPath(to, out _)) - Context.Collector.EmitApiLink(new ApiLinkReference(to, Configuration.SourceFile.FullName, 1, 1, to.Length)); + Context.Collector.EmitApiLink( + new ApiLinkReference(to, (Configuration.RedirectsSource ?? Configuration.SourceFile).FullName, 1, 1, to.Length) + ); return; } diff --git a/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs b/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs new file mode 100644 index 0000000000..67b24a0366 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using AwesomeAssertions; +using Elastic.Documentation.Configuration; +using Elastic.Markdown.IO; + +namespace Elastic.Markdown.Tests; + +public class RedirectApiLinkTests +{ + [Test] + public void RedirectToApiPage_RecordsApiLinkAtTheRedirectsFile() + { + var collector = BuildDocSet( + """ + redirects: + 'old-api.md': 'https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade' + 'old-other.md': 'https://www.elastic.co/docs/solutions/search' + """ + ); + + var link = collector.ApiLinks.Should().ContainSingle().Subject; + link.Url.Should().Be("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade"); + link.File.Should().EndWith("redirects.yml"); + collector.Errors.Should().Be(0); + } + + private static TestDiagnosticsCollector BuildDocSet(string redirects) + { + var fileSystem = new MockFileSystem( + new Dictionary + { + { "docs/docset.yml", new MockFileData("project: test\ntoc:\n- file: index.md\n") }, + { "docs/index.md", new MockFileData("# Index") }, + { "docs/redirects.yml", new MockFileData(redirects) } + }, + new MockFileSystemOptions { CurrentDirectory = Paths.WorkingDirectoryRoot.FullName } + ); + var collector = new TestDiagnosticsCollector(); + _ = collector.StartAsync(TestContext.Current!.Execution.CancellationToken); + var configurationContext = TestHelpers.CreateConfigurationContext(fileSystem); + var context = new BuildContext(collector, TestHelpers.CreateDocumentationFileSystem(fileSystem), configurationContext); + _ = new DocumentationSet(context, new TestLoggerFactory(), new TestCrossLinkResolver()); + return collector; + } +} From cb684bfd25caf1eb7d0baf2cf45d255d6700dd90 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Mon, 2 Nov 2026 17:37:31 +0100 Subject: [PATCH 3/9] Collapse duplicate slashes in API links and write the report without a BOM An assembler run found 52 links written as .../elasticsearch//operation/... that the validator reported as broken. Normalizing repeated slashes treats them like the single-slash form. The report also began with a UTF-8 byte order mark, which strict JSON parsers reject. Co-Authored-By: Claude Sonnet 5.5 --- .../Diagnostics/ApiLinkReference.cs | 10 ++++++++-- .../Building/ApiLinkValidator.cs | 5 ++++- .../Diagnostics/ApiLinkMatcherTests.cs | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs index 50edd6fe91..347bab0d80 100644 --- a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs +++ b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs @@ -2,12 +2,14 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Text.RegularExpressions; + namespace Elastic.Documentation.Diagnostics; /// A link to an API reference page, recorded during the Markdown build and checked once the API pages exist. public readonly record struct ApiLinkReference(string Url, string File, int Line, int Column, int Length); -public static class ApiLinkMatcher +public static partial class ApiLinkMatcher { public const string ApiDocPrefix = "/docs/api/doc/"; @@ -36,5 +38,9 @@ public static bool TryGetApiPath(string? url, out string path) return true; } - public static string Normalize(string path) => Uri.UnescapeDataString(path).TrimEnd('/').ToLowerInvariant(); + public static string Normalize(string path) => + DuplicateSlashes().Replace(Uri.UnescapeDataString(path), "/").TrimEnd('/').ToLowerInvariant(); + + [GeneratedRegex("/{2,}")] + private static partial Regex DuplicateSlashes(); } diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs index dfd4230a7f..e8bf881283 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -67,7 +67,10 @@ Cancel ctx var report = new ApiLinkReport(links.Length, [.. broken.Select(b => new ApiLinkReportEntry(b.Url, b.File, b.Line))]); var path = fileSystem.Path.Join(outputDirectory, ReportFileName); var json = JsonSerializer.Serialize(report, ApiLinkReportJsonContext.Default.ApiLinkReport); - await fileSystem.File.WriteAllTextAsync(path, json, Encoding.UTF8, ctx).ConfigureAwait(false); + await fileSystem + .File + .WriteAllTextAsync(path, json, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), ctx) + .ConfigureAwait(false); return path; } } diff --git a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs index 8d89574005..8c27c1b89a 100644 --- a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs +++ b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs @@ -13,6 +13,7 @@ public class ApiLinkMatcherTests [Arguments("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-x", "/docs/api/doc/kibana/v8/operation/operation-x")] [Arguments("https://elastic.co/docs/api/doc/kibana/", "/docs/api/doc/kibana")] [Arguments("http://www.elastic.co/docs/api/doc/Kibana/Group/Endpoint-Foo?x=1#frag", "/docs/api/doc/kibana/group/endpoint-foo")] + [Arguments("https://www.elastic.co/docs/api/doc/elasticsearch//operation/operation-bulk", "/docs/api/doc/elasticsearch/operation/operation-bulk")] public void TryGetApiPath_ApiUrl_ReturnsNormalizedPath(string url, string expected) { ApiLinkMatcher.TryGetApiPath(url, out var path).Should().BeTrue(); From 616bd056ad7af44f59410dadcecb1de856bec8c4 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Tue, 3 Nov 2026 11:51:56 +0100 Subject: [PATCH 4/9] Record API links with the outbound link extractor instead of parse hooks The three parse-time hooks missed autolinks and recorded link cards twice. An ApiLinkExporter now reuses the document each page already parsed and the OutboundLinkExtractor from the search indexer, which now also returns link positions. Redirect targets are still recorded separately. Co-Authored-By: Claude Sonnet 5.5 --- .../Exporters/ApiLinkExporter.cs | 40 ++++++++++++++ .../Exporters/OutboundLinkExtractor.cs | 54 +++++++++++++------ .../Myst/Directives/DirectiveLinkValidator.cs | 16 ------ .../DiagnosticLinkInlineParser.cs | 17 ------ .../Building/AssemblerBuilder.cs | 4 +- .../PlainText/OutboundLinkExtractorTests.cs | 49 +++++++++++++++++ .../Inline/ApiLinkTests.cs | 38 ------------- 7 files changed, 130 insertions(+), 88 deletions(-) create mode 100644 src/Elastic.Markdown/Exporters/ApiLinkExporter.cs delete mode 100644 tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs diff --git a/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs b/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs new file mode 100644 index 0000000000..39bda5a3ba --- /dev/null +++ b/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs @@ -0,0 +1,40 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Markdown.Exporters; + +/// +/// Records every link to an API reference page so the assembler can check it once those pages exist. +/// Reuses the document each page already parsed for rendering, so it costs one tree walk per page. +/// +public class ApiLinkExporter : IMarkdownExporter +{ + /// + public ValueTask StartAsync(Cancel ctx = default) => default; + + /// + public ValueTask StopAsync(Cancel ctx = default) => default; + + /// + public ValueTask ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx) + { + var file = fileContext.SourceFile.SourceFile.FullName; + foreach (var link in OutboundLinkExtractor.ExtractWithLocations(fileContext.Document, file)) + { + if (ApiLinkMatcher.TryGetApiPath(link.Url, out _)) + fileContext + .BuildContext + .Collector + .EmitApiLink(new ApiLinkReference(link.Url, link.File, link.Line, link.Column, link.Url.Length)); + } + + return ValueTask.FromResult(true); + } + + /// + public ValueTask FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx) => ValueTask.FromResult(true); +} diff --git a/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs b/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs index 876baddb92..8fc2f4424f 100644 --- a/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs +++ b/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs @@ -11,54 +11,76 @@ namespace Elastic.Markdown.Exporters; +/// A link target with the file and 1-based position it was written at. +public readonly record struct OutboundLink(string Url, string File, int Line, int Column); + /// -/// Collects the link targets of a parsed markdown document for the search index. +/// Collects the link targets of a parsed markdown document for the search index and for link checks. /// public static class OutboundLinkExtractor { /// /// Returns the distinct link targets in document order, or null when the page has none. - /// Covers inline links, {page-card} and {link-card} targets, and the content of + /// Covers inline links, autolinks, {page-card} and {link-card} targets, and the content of /// {include} snippets. Images, same-page anchors and links inside comments are skipped. /// public static string[]? Extract(MarkdownDocument document) { - var links = new List(); - Collect(document, links, []); - var distinct = links - .OfType() - .Where(url => !string.IsNullOrWhiteSpace(url) && !url.StartsWith('#')) - .Distinct(StringComparer.Ordinal) - .ToArray(); + var distinct = ExtractWithLocations(document, string.Empty).Select(link => link.Url).Distinct(StringComparer.Ordinal).ToArray(); return distinct.Length > 0 ? distinct : null; } - private static void Collect(MarkdownDocument document, List links, HashSet visitedIncludes) + /// + /// Returns every link occurrence in document order with the file it was written in. Links inside an + /// {include} snippet report the snippet file and the line within it. + /// + public static IReadOnlyList ExtractWithLocations(MarkdownDocument document, string file) + { + var links = new List(); + Collect(document, file, links, []); + return [.. links.Where(link => !string.IsNullOrWhiteSpace(link.Url) && !link.Url.StartsWith('#'))]; + } + + private static void Collect(MarkdownDocument document, string file, List links, HashSet visitedIncludes) { foreach (var obj in document.Descendants()) { switch (obj) { case LinkInline { IsImage: false } link when !IsInComment(link): - links.Add(link.Url); + Add(links, link.Url, file, link); break; case AutolinkInline autolink when !IsInComment(autolink): - links.Add(autolink.Url); + Add(links, autolink.Url, file, autolink); break; case PageCardBlock card: - links.Add(card.ResolvedUrl); + Add(links, card.ResolvedUrl, file, card); break; case LinkCardBlock card: - links.Add(card.Data.Link); - links.AddRange(card.Data.Links.Select(l => l.Url)); + AddCardTarget(links, card.Data.Link, file, card); + foreach (var linkItem in card.Data.Links) + AddCardTarget(links, linkItem.Url, file, card); break; case IncludeBlock { Found: true, Literal: false, IncludePath: { } path } include when visitedIncludes.Add(path): - Collect(include.ParseSnippet(), links, visitedIncludes); + Collect(include.ParseSnippet(), path, links, visitedIncludes); break; } } } + // The autolink scan already reports https:// targets in a card's YAML body at their exact position. + private static void AddCardTarget(List links, string? url, string file, MarkdownObject card) + { + if (url is not null && !url.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + Add(links, url, file, card); + } + + private static void Add(List links, string? url, string file, MarkdownObject source) + { + if (url is not null) + links.Add(new OutboundLink(url, file, source.Line + 1, Math.Max(source.Column, 1))); + } + private static bool IsInComment(Inline inline) { var container = inline as ContainerInline ?? inline.Parent; diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs index 7cc2a55f74..63b6c39a98 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information using Elastic.Documentation; -using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; using Markdig; @@ -91,22 +90,7 @@ internal static class DirectiveLinkValidator return url; if (IsExternal(trimmed)) - { - if (ApiLinkMatcher.TryGetApiPath(trimmed, out _)) - context - .Build - .Collector - .EmitApiLink( - new ApiLinkReference( - trimmed, - block.CurrentFile.FullName, - block.Line + 1, - System.Math.Max(block.Column, 1), - trimmed.Length - ) - ); return url; - } if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) && CrossLinkValidator.IsCrossLink(uri)) return ResolveCrossLink(url, uri, block, context); diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index d81661e54d..b9fbf1112b 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -7,7 +7,6 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Elastic.Documentation; -using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; @@ -154,27 +153,11 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc link.SetData("isCrossLink", false); if (ValidateExternalUri(link, processor, uri)) - { - RecordApiLink(link, context, url); return; - } ProcessInternalLink(link, processor, context); } - private static void RecordApiLink(LinkInline link, ParserContext context, string? url) - { - if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _)) - return; - - context - .Build - .Collector - .EmitApiLink( - new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, link.Line + 1, Math.Max(link.Column, 1), url!.Length) - ); - } - private static bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url) { if (string.IsNullOrEmpty(url)) diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index dc9382ca6d..677f7a23bf 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -53,7 +53,9 @@ Cancel ctx var buildTimes = new List<(string Name, int FileCount, TimeSpan Duration)>(); // Create exporters without inferrer - inferrer is created per-repository - var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context); + var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context).ToList(); + if (exportOptions.Contains(Exporter.Html)) + markdownExporters.Add(new ApiLinkExporter()); var tasks = markdownExporters.Select(async e => await e.StartAsync(ctx)); await Task.WhenAll(tasks); diff --git a/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs b/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs index 0974327ac2..41608c0f7f 100644 --- a/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs +++ b/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs @@ -96,3 +96,52 @@ public async Task Extract_PageWithoutLinks_ReturnsNull() OutboundLinkExtractor.Extract(result.Document).Should().BeNull(); } } + +public class OutboundLinkLocations : GeneratorTest +{ + protected override IReadOnlyCollection Files => + [ + Index( + """ + [Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) and + . + + % [note](https://www.elastic.co/docs/api/doc/hidden) + + :::{link-card} + title: Bulk + link: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk + links: + - label: Get + url: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get + ::: + + :::{include} _snippets/api.md + ::: + """ + ), + Snippet("_snippets/api.md", "Read the [guide](https://www.elastic.co/docs/api/doc/kibana/operation/operation-snippet)."), + ]; + + [Test] + public async Task ExtractWithLocations_PageWithApiLinks_ReturnsEveryOccurrenceWithItsFile() + { + var result = await Docs.Converts("index.md").Inner; + + var links = OutboundLinkExtractor.ExtractWithLocations(result.Document, "index.md"); + + links + .Select(l => l.Url) + .Should() + .Equal( + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search", + "https://www.elastic.co/docs/api/doc/kibana/operation/operation-auto", + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk", + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get", + "https://www.elastic.co/docs/api/doc/kibana/operation/operation-snippet" + ); + links.Should().OnlyContain(l => l.Line > 0 && l.Column > 0); + links.Take(4).Should().OnlyContain(l => l.File == "index.md"); + links[4].File.Should().EndWith("api.md"); + } +} diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs deleted file mode 100644 index 9c4229c227..0000000000 --- a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using AwesomeAssertions; -using JetBrains.Annotations; -using Markdig.Syntax.Inlines; - -namespace Elastic.Markdown.Tests.Inline; - -[InheritsTests] -public class ApiLinkRecordedTests() : LinkTestBase( - """ -[Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params) -""" -) -{ - [Test] - public void RecordsApiLinkWithLocation() - { - var link = Collector.ApiLinks.Should().ContainSingle().Subject; - link.Url.Should().Be("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params"); - link.Line.Should().BePositive(); - link.File.Should().EndWith(".md"); - } - - [Test] - public void HasNoErrors() => Collector.Errors.Should().Be(0); -} - -[InheritsTests] -public class NonApiElasticLinkTests() : LinkTestBase(""" -[Docs](https://www.elastic.co/docs/solutions/search) -""") -{ - [Test] - public void RecordsNoApiLink() => Collector.ApiLinks.Should().BeEmpty(); -} From ecabff4c4d89a75cec9a041eeb2c8514aa26b8c4 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Tue, 3 Nov 2026 12:13:56 +0100 Subject: [PATCH 5/9] Address review: record API links where links are parsed, fix preview prefix The exporter route missed API children pages and the hero, card-group and what's-new fields. Links are recorded again at the inline, autolink and directive parse points, which every page and directive passes through. A PR preview serves pages under another path prefix, so the generated URLs are now recorded under /docs. Each URL is reported once per file. Co-Authored-By: Claude Sonnet 5.5 --- src/Elastic.ApiExplorer/OpenApiGenerator.cs | 9 +- .../Exporters/ApiLinkExporter.cs | 40 -------- .../Exporters/OutboundLinkExtractor.cs | 54 ++++------- .../Myst/Directives/DirectiveLinkValidator.cs | 16 ++++ .../InlineParsers/AutoLinkInlineParser.cs | 2 + .../DiagnosticLinkInlineParser.cs | 17 ++++ .../Building/ApiLinkValidator.cs | 10 +- .../Building/AssemblerBuilder.cs | 4 +- .../OpenApiGeneratorCatalogSplitTests.cs | 25 ++++- .../PlainText/OutboundLinkExtractorTests.cs | 49 ---------- .../ApiLinkValidatorTests.cs | 92 +++++++++++++++++++ .../Directives/DirectiveApiLinkTests.cs | 56 +++++++++++ .../Inline/ApiLinkTests.cs | 57 ++++++++++++ 13 files changed, 297 insertions(+), 134 deletions(-) delete mode 100644 src/Elastic.Markdown/Exporters/ApiLinkExporter.cs create mode 100644 tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs create mode 100644 tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs diff --git a/src/Elastic.ApiExplorer/OpenApiGenerator.cs b/src/Elastic.ApiExplorer/OpenApiGenerator.cs index e5864b158a..8ac09cd9ec 100644 --- a/src/Elastic.ApiExplorer/OpenApiGenerator.cs +++ b/src/Elastic.ApiExplorer/OpenApiGenerator.cs @@ -603,7 +603,14 @@ Cancel ctx }).ConfigureAwait(false); } - private void RecordUrl(string url) => _ = _generatedUrls.TryAdd(ApiLinkMatcher.Normalize(url), 0); + // Narrative links use the production path (/docs/api/doc/...), while a build such as a PR preview serves the same + // pages under another prefix, so record each page under the production path. + private void RecordUrl(string url) + { + var prefix = context.UrlPathPrefix ?? string.Empty; + var relative = prefix.Length > 0 && url.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) ? url[prefix.Length..] : url; + _ = _generatedUrls.TryAdd(ApiLinkMatcher.Normalize($"/docs{relative}"), 0); + } /// Former per-operation URLs of a collapsed page forward to it, keeping any #fragment. private async Task WriteRedirects(OperationNavigationItem page, Cancel ctx) diff --git a/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs b/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs deleted file mode 100644 index 39bda5a3ba..0000000000 --- a/src/Elastic.Markdown/Exporters/ApiLinkExporter.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.IO.Abstractions; -using Elastic.Documentation.Diagnostics; - -namespace Elastic.Markdown.Exporters; - -/// -/// Records every link to an API reference page so the assembler can check it once those pages exist. -/// Reuses the document each page already parsed for rendering, so it costs one tree walk per page. -/// -public class ApiLinkExporter : IMarkdownExporter -{ - /// - public ValueTask StartAsync(Cancel ctx = default) => default; - - /// - public ValueTask StopAsync(Cancel ctx = default) => default; - - /// - public ValueTask ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx) - { - var file = fileContext.SourceFile.SourceFile.FullName; - foreach (var link in OutboundLinkExtractor.ExtractWithLocations(fileContext.Document, file)) - { - if (ApiLinkMatcher.TryGetApiPath(link.Url, out _)) - fileContext - .BuildContext - .Collector - .EmitApiLink(new ApiLinkReference(link.Url, link.File, link.Line, link.Column, link.Url.Length)); - } - - return ValueTask.FromResult(true); - } - - /// - public ValueTask FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx) => ValueTask.FromResult(true); -} diff --git a/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs b/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs index 8fc2f4424f..876baddb92 100644 --- a/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs +++ b/src/Elastic.Markdown/Exporters/OutboundLinkExtractor.cs @@ -11,76 +11,54 @@ namespace Elastic.Markdown.Exporters; -/// A link target with the file and 1-based position it was written at. -public readonly record struct OutboundLink(string Url, string File, int Line, int Column); - /// -/// Collects the link targets of a parsed markdown document for the search index and for link checks. +/// Collects the link targets of a parsed markdown document for the search index. /// public static class OutboundLinkExtractor { /// /// Returns the distinct link targets in document order, or null when the page has none. - /// Covers inline links, autolinks, {page-card} and {link-card} targets, and the content of + /// Covers inline links, {page-card} and {link-card} targets, and the content of /// {include} snippets. Images, same-page anchors and links inside comments are skipped. /// public static string[]? Extract(MarkdownDocument document) { - var distinct = ExtractWithLocations(document, string.Empty).Select(link => link.Url).Distinct(StringComparer.Ordinal).ToArray(); + var links = new List(); + Collect(document, links, []); + var distinct = links + .OfType() + .Where(url => !string.IsNullOrWhiteSpace(url) && !url.StartsWith('#')) + .Distinct(StringComparer.Ordinal) + .ToArray(); return distinct.Length > 0 ? distinct : null; } - /// - /// Returns every link occurrence in document order with the file it was written in. Links inside an - /// {include} snippet report the snippet file and the line within it. - /// - public static IReadOnlyList ExtractWithLocations(MarkdownDocument document, string file) - { - var links = new List(); - Collect(document, file, links, []); - return [.. links.Where(link => !string.IsNullOrWhiteSpace(link.Url) && !link.Url.StartsWith('#'))]; - } - - private static void Collect(MarkdownDocument document, string file, List links, HashSet visitedIncludes) + private static void Collect(MarkdownDocument document, List links, HashSet visitedIncludes) { foreach (var obj in document.Descendants()) { switch (obj) { case LinkInline { IsImage: false } link when !IsInComment(link): - Add(links, link.Url, file, link); + links.Add(link.Url); break; case AutolinkInline autolink when !IsInComment(autolink): - Add(links, autolink.Url, file, autolink); + links.Add(autolink.Url); break; case PageCardBlock card: - Add(links, card.ResolvedUrl, file, card); + links.Add(card.ResolvedUrl); break; case LinkCardBlock card: - AddCardTarget(links, card.Data.Link, file, card); - foreach (var linkItem in card.Data.Links) - AddCardTarget(links, linkItem.Url, file, card); + links.Add(card.Data.Link); + links.AddRange(card.Data.Links.Select(l => l.Url)); break; case IncludeBlock { Found: true, Literal: false, IncludePath: { } path } include when visitedIncludes.Add(path): - Collect(include.ParseSnippet(), path, links, visitedIncludes); + Collect(include.ParseSnippet(), links, visitedIncludes); break; } } } - // The autolink scan already reports https:// targets in a card's YAML body at their exact position. - private static void AddCardTarget(List links, string? url, string file, MarkdownObject card) - { - if (url is not null && !url.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - Add(links, url, file, card); - } - - private static void Add(List links, string? url, string file, MarkdownObject source) - { - if (url is not null) - links.Add(new OutboundLink(url, file, source.Line + 1, Math.Max(source.Column, 1))); - } - private static bool IsInComment(Inline inline) { var container = inline as ContainerInline ?? inline.Parent; diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs index 63b6c39a98..7cc2a55f74 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; using Markdig; @@ -90,7 +91,22 @@ internal static class DirectiveLinkValidator return url; if (IsExternal(trimmed)) + { + if (ApiLinkMatcher.TryGetApiPath(trimmed, out _)) + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference( + trimmed, + block.CurrentFile.FullName, + block.Line + 1, + System.Math.Max(block.Column, 1), + trimmed.Length + ) + ); return url; + } if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) && CrossLinkValidator.IsCrossLink(uri)) return ResolveCrossLink(url, uri, block, context); diff --git a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs index f9e4c1529c..555f41236e 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs @@ -86,6 +86,8 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) processor.Inline = linkInline; + DiagnosticLinkInlineParser.RecordApiLink(linkInline, context, url); + // Emit hint for elastic.co/docs URLs (after setting Inline so position is correct) if (url.Contains("elastic.co/docs", StringComparison.OrdinalIgnoreCase)) processor.EmitHint( diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index b9fbf1112b..00294f6716 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; @@ -153,11 +154,27 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc link.SetData("isCrossLink", false); if (ValidateExternalUri(link, processor, uri)) + { + RecordApiLink(link, context, url); return; + } ProcessInternalLink(link, processor, context); } + internal static void RecordApiLink(LinkInline link, ParserContext context, string? url) + { + if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _)) + return; + + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, link.Line + 1, Math.Max(link.Column, 1), url!.Length) + ); + } + private static bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url) { if (string.IsNullOrEmpty(url)) diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs index e8bf881283..86610eee6a 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -45,7 +45,15 @@ Cancel ctx if (generatedUrls.Count == 0) return null; - var links = collector.ApiLinks.ToArray(); + // A snippet is parsed once per page that includes it, and a card's targets are seen by more than one parser, + // so report each URL once per file, at its first position. + var links = collector + .ApiLinks + .OrderBy(l => l.File, StringComparer.Ordinal) + .ThenBy(l => l.Line) + .ThenBy(l => l.Column) + .DistinctBy(l => (l.Url, l.File)) + .ToArray(); if (links.Length == 0) return null; diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index 677f7a23bf..dc9382ca6d 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -53,9 +53,7 @@ Cancel ctx var buildTimes = new List<(string Name, int FileCount, TimeSpan Duration)>(); // Create exporters without inferrer - inferrer is created per-repository - var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context).ToList(); - if (exportOptions.Contains(Exporter.Html)) - markdownExporters.Add(new ApiLinkExporter()); + var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context); var tasks = markdownExporters.Select(async e => await e.StartAsync(ctx)); await Task.WhenAll(tasks); diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs index 1d459266fb..f83d81f0b7 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs @@ -78,6 +78,26 @@ public async Task GenerateProducts_RecordsEveryRenderedPageInGeneratedUrls() generator.GeneratedUrls.Should().Contain(url => url.Contains("/operation/operation-")); } + [Test] + public async Task GenerateProducts_PreviewPathPrefix_RecordsUrlsUnderTheProductionPath() + { + var outputRoot = Path.Join(Paths.WorkingDirectoryRoot.FullName, $"api-catalog-split-{Guid.NewGuid():N}"); + var context = CreateGenerateContext(outputRoot, urlPathPrefix: "elastic/docs-builder/docs/123"); + using var versionIndexClient = new VersionIndexClient(BaseUri, MultiVersionHandler(), sleep: (_, _) => Task.CompletedTask); + var generator = new OpenApiGenerator( + NullLoggerFactory.Instance, + context, + NoopMarkdownStringRenderer.Instance, + versionIndexClient, + CreateSequentialReader(SpecDocument("Elasticsearch main")) + ); + + _ = await generator.GenerateProducts(ctx: TestContext.Current!.Execution.CancellationToken); + + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch"); + generator.GeneratedUrls.Should().OnlyContain(url => url.StartsWith("/docs/api/doc/")); + } + [Test] public async Task ResolveCanonicalDocuments_ConfiguredApi_ReturnsSpecUnderItsExplorerKey() { @@ -628,7 +648,8 @@ private static BuildContext CreateGenerateContext( string outputRoot, string? docsetYaml = null, string? displayName = null, - BuildType buildType = BuildType.Isolated + BuildType buildType = BuildType.Isolated, + string urlPathPrefix = "docs" ) { var collector = new DiagnosticsCollector([]); @@ -680,7 +701,7 @@ private static BuildContext CreateGenerateContext( ), configurationContext ) - { UrlPathPrefix = "docs", BuildType = buildType }; + { UrlPathPrefix = urlPathPrefix, BuildType = buildType }; } private static OpenApiDocument SpecDocument(string title) => diff --git a/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs b/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs index 41608c0f7f..0974327ac2 100644 --- a/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs +++ b/tests/Elastic.Authoring.Tests/PlainText/OutboundLinkExtractorTests.cs @@ -96,52 +96,3 @@ public async Task Extract_PageWithoutLinks_ReturnsNull() OutboundLinkExtractor.Extract(result.Document).Should().BeNull(); } } - -public class OutboundLinkLocations : GeneratorTest -{ - protected override IReadOnlyCollection Files => - [ - Index( - """ - [Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) and - . - - % [note](https://www.elastic.co/docs/api/doc/hidden) - - :::{link-card} - title: Bulk - link: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk - links: - - label: Get - url: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get - ::: - - :::{include} _snippets/api.md - ::: - """ - ), - Snippet("_snippets/api.md", "Read the [guide](https://www.elastic.co/docs/api/doc/kibana/operation/operation-snippet)."), - ]; - - [Test] - public async Task ExtractWithLocations_PageWithApiLinks_ReturnsEveryOccurrenceWithItsFile() - { - var result = await Docs.Converts("index.md").Inner; - - var links = OutboundLinkExtractor.ExtractWithLocations(result.Document, "index.md"); - - links - .Select(l => l.Url) - .Should() - .Equal( - "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search", - "https://www.elastic.co/docs/api/doc/kibana/operation/operation-auto", - "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk", - "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get", - "https://www.elastic.co/docs/api/doc/kibana/operation/operation-snippet" - ); - links.Should().OnlyContain(l => l.Line > 0 && l.Column > 0); - links.Take(4).Should().OnlyContain(l => l.File == "index.md"); - links[4].File.Should().EndWith("api.md"); - } -} diff --git a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs index ef7752e459..db84bed752 100644 --- a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs @@ -2,6 +2,8 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Abstractions.TestingHelpers; +using System.Text.Json; using AwesomeAssertions; using Elastic.Documentation.Assembler.Building; using Elastic.Documentation.Diagnostics; @@ -56,4 +58,94 @@ public void FindBroken_MultipleBroken_AreOrderedByUrlThenSource() ); broken.Select(b => (b.Url.Split('/')[^1], b.File)).Should().Equal(("aaa", "a.md"), ("aaa", "b.md"), ("zzz", "b.md")); } + + private static (DiagnosticsCollector Collector, MockFileSystem FileSystem) Setup(params ApiLinkReference[] links) + { + var collector = new DiagnosticsCollector([]); + foreach (var link in links) + collector.EmitApiLink(link); + var fileSystem = new MockFileSystem(); + fileSystem.AddDirectory("/out"); + return (collector, fileSystem); + } + + [Test] + public async Task ValidateAsync_BrokenLink_WarnsAtItsPositionAndWritesTheReport() + { + var (collector, fileSystem) = Setup( + new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/", "a.md", 4, 7, 40), + new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/operation/missing#x", "b.md", 9, 3, 55) + ); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + collector.Warnings.Should().Be(1); + path.Should().EndWith("api-link-report.json"); + var bytes = await fileSystem.File.ReadAllBytesAsync(path!); + bytes.Take(3).Should().NotEqual([(byte)0xEF, (byte)0xBB, (byte)0xBF], "the report is plain UTF-8 without a BOM"); + using var report = JsonDocument.Parse(bytes); + report.RootElement.GetProperty("checked").GetInt32().Should().Be(2); + var broken = report.RootElement.GetProperty("broken").EnumerateArray().Single(); + broken.GetProperty("source").GetString().Should().Be("b.md"); + broken.GetProperty("line").GetInt32().Should().Be(9); + } + + [Test] + public async Task ValidateAsync_SameLinkRecordedByEveryIncludingPage_ReportsOnce() + { + var link = new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/gone", "_snippets/s.md", 2, 1, 30); + var (collector, fileSystem) = Setup(link, link, link); + + _ = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + collector.Warnings.Should().Be(1); + using var report = JsonDocument.Parse(await fileSystem.File.ReadAllTextAsync("/out/api-link-report.json")); + report.RootElement.GetProperty("checked").GetInt32().Should().Be(1); + } + + [Test] + public async Task ValidateAsync_NoGeneratedPages_SkipsWithoutWarningsOrReport() + { + var (collector, fileSystem) = Setup(new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/gone", "a.md", 1, 1, 30)); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + new HashSet(), + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + path.Should().BeNull(); + collector.Warnings.Should().Be(0); + fileSystem.File.Exists("/out/api-link-report.json").Should().BeFalse(); + } + + [Test] + public async Task ValidateAsync_NoRecordedLinks_ReturnsNull() + { + var (collector, fileSystem) = Setup(); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + path.Should().BeNull(); + } } diff --git a/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs new file mode 100644 index 0000000000..ebde2f0ab6 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Markdown.Myst.Directives.Hub; + +namespace Elastic.Markdown.Tests.Directives; + +[InheritsTests] +public class HeroApiLinkTests() : DirectiveTest( + """ + :::{hero} + :title: Elasticsearch + :description: Start with the [search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search). + :primary-action: [Bulk](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) + :secondary-action: [Install](https://www.elastic.co/downloads/elasticsearch) + ::: + """ +) +{ + [Test] + public void RecordsApiLinksFromActionsAndDescriptionOnly() => + Collector + .ApiLinks + .Select(l => l.Url) + .Distinct() + .Should() + .BeEquivalentTo([ + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search", + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk" + ]); +} + +[InheritsTests] +public class LinkCardApiLinkTests() : DirectiveTest( + """ + :::{link-card} + title: Search + link: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search + links: + - label: Elsewhere + url: https://www.elastic.co/docs/solutions/search + ::: + """ +) +{ + [Test] + public void RecordsOnlyTheApiTarget() => + Collector + .ApiLinks + .Select(l => l.Url) + .Distinct() + .Should() + .Equal("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search"); +} diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs new file mode 100644 index 0000000000..688ef194d2 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using JetBrains.Annotations; +using Markdig.Syntax.Inlines; + +namespace Elastic.Markdown.Tests.Inline; + +[InheritsTests] +public class ApiLinkRecordedTests() : LinkTestBase( + """ +[Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params) +""" +) +{ + [Test] + public void RecordsApiLinkWithLocation() + { + var link = Collector.ApiLinks.Should().ContainSingle().Subject; + link.Url.Should().Be("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params"); + link.Line.Should().BePositive(); + link.File.Should().EndWith(".md"); + } + + [Test] + public void HasNoErrors() => Collector.Errors.Should().Be(0); +} + +[InheritsTests] +public class NonApiElasticLinkTests() : LinkTestBase(""" +[Docs](https://www.elastic.co/docs/solutions/search) +""") +{ + [Test] + public void RecordsNoApiLink() => Collector.ApiLinks.Should().BeEmpty(); +} + +[InheritsTests] +public class ApiAutoLinkTests() : InlineTest( + """ +See https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing for details. +""" +) +{ + [Test] + public void RecordsBareUrlAsApiLink() => + Collector + .ApiLinks + .Should() + .ContainSingle() + .Which + .Url + .Should() + .Be("https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing"); +} From 5cf92e9db1781e6ea9536cfeb1f6e72f70e82654 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Tue, 3 Nov 2026 12:26:24 +0100 Subject: [PATCH 6/9] Resolve old bump.sh API URLs through the edge redirects before checking About 70 reported links used the old serverless and elastic-cloud-serverless roots, which the edge redirects. The validator now follows the same redirect table and checks the target page, so a link that works today is not flagged but one whose target is gone still is. The table moves out of ApiSitemapParityTests, which now uses it. Co-Authored-By: Claude Sonnet 5.5 --- .../Building/ApiLegacyRedirects.cs | 77 +++++++++++++++++++ .../Building/ApiLinkValidator.cs | 2 +- .../ApiLinkValidatorTests.cs | 31 +++++++- .../ApiSitemapParityTests.cs | 65 +--------------- 4 files changed, 112 insertions(+), 63 deletions(-) create mode 100644 src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs new file mode 100644 index 0000000000..747f4a0fb8 --- /dev/null +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs @@ -0,0 +1,77 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Assembler.Building; + +/// +/// The redirects from old bump.sh API URLs. They mirror modules/elastic-docs-v3/api-root-redirects.json in +/// elastic/docs-infra (#459), which the native viewer resolves exact match first. Keep this table in sync with that file. +/// +// TODO: Delete this class once the API Explorer migration (elastic/docs-eng-team#436) is finalised. +public static class ApiLegacyRedirects +{ + private static readonly IReadOnlyDictionary RedirectedRoots = new Dictionary + { + ["serverless"] = "kibana-serverless", + ["elastic-cloud-serverless"] = "cloud-serverless" + }; + + private static readonly IReadOnlySet GoneRoots = new HashSet { "observability-serverless" }; + + // Exact page redirects from the same file. Bump.sh served a Servers page for specs that declare no servers, + // Cloud Billing renamed its Budgets tag, Kibana removed Streams operations from its 9.6 spec, and some bump.sh topic + // pages are a children: page or the /authentication page in the native viewer. + private static readonly IReadOnlyDictionary RedirectedPages = new Dictionary + { + ["/docs/api/doc/elasticsearch/servers"] = "/docs/api/doc/elasticsearch", + ["/docs/api/doc/elasticsearch/v8/servers"] = "/docs/api/doc/elasticsearch/v8", + ["/docs/api/doc/elasticsearch/v9/servers"] = "/docs/api/doc/elasticsearch/v9", + ["/docs/api/doc/elasticsearch-serverless/servers"] = "/docs/api/doc/elasticsearch-serverless", + ["/docs/api/doc/cloud-enterprise/servers"] = "/docs/api/doc/cloud-enterprise", + ["/docs/api/doc/cloud-enterprise/v3/servers"] = "/docs/api/doc/cloud-enterprise/v3", + ["/docs/api/doc/cloud-billing/group/endpoint-budgets"] = "/docs/api/doc/cloud-billing/group/endpoint-billing-budgets", + ["/docs/api/doc/kibana/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/kibana-api-overview", + ["/docs/api/doc/kibana/v8/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v8/kibana-api-overview", + ["/docs/api/doc/kibana/v9/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v9/kibana-api-overview", + ["/docs/api/doc/cloud/topic/topic-authentication"] = "/docs/api/doc/cloud/authentication", + ["/docs/api/doc/cloud-billing/topic/topic-authentication"] = "/docs/api/doc/cloud-billing/authentication", + ["/docs/api/doc/kibana/v9/group/endpoint-significant_events"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-delete-streams-name-queries-queryid"] = + "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-queries"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-significant-events"] = + "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-post-streams-name-queries-bulk"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-put-streams-name-queries-queryid"] = "/docs/api/doc/kibana/v9/group/endpoint-streams" + }; + + /// True for a root with no pages at all, so a link to it has nowhere to land. + public static bool IsGone(string path) => GoneRoots.Contains(RootOf(path)); + + /// True when the edge redirects this path to another one. + public static bool IsRedirected(string path) => RedirectedPages.ContainsKey(path) || RedirectedRoots.ContainsKey(RootOf(path)); + + /// The path the edge serves for : its redirect target, or the path itself. + public static string Resolve(string path) + { + if (RedirectedPages.TryGetValue(path, out var page)) + return page; + + var root = RootOf(path); + return RedirectedRoots.TryGetValue(root, out var newRoot) + ? string.Concat(ApiLinkMatcher.ApiDocPrefix, newRoot, path.AsSpan(ApiLinkMatcher.ApiDocPrefix.Length + root.Length)) + : path; + } + + private static string RootOf(string path) + { + if (!path.StartsWith(ApiLinkMatcher.ApiDocPrefix, StringComparison.Ordinal)) + return string.Empty; + var rest = path[ApiLinkMatcher.ApiDocPrefix.Length..]; + var slash = rest.IndexOf('/'); + return slash < 0 ? rest : rest[..slash]; + } +} diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs index 86610eee6a..709756b6ca 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -26,7 +26,7 @@ public static class ApiLinkValidator public static IReadOnlyList FindBroken(IEnumerable links, IReadOnlySet generatedUrls) => [ .. links - .Where(l => ApiLinkMatcher.TryGetApiPath(l.Url, out var path) && !generatedUrls.Contains(path)) + .Where(l => ApiLinkMatcher.TryGetApiPath(l.Url, out var path) && !generatedUrls.Contains(ApiLegacyRedirects.Resolve(path))) .OrderBy(l => l.Url, StringComparer.Ordinal) .ThenBy(l => l.File, StringComparer.Ordinal) .ThenBy(l => l.Line) diff --git a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs index db84bed752..e63e85c828 100644 --- a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs @@ -16,7 +16,9 @@ public class ApiLinkValidatorTests { "/docs/api/doc/kibana", "/docs/api/doc/kibana/v8/operation/operation-upgrade", - "/docs/api/doc/kibana/operation/operation-alias" + "/docs/api/doc/kibana/operation/operation-alias", + "/docs/api/doc/kibana-serverless/operation/operation-live", + "/docs/api/doc/kibana/kibana-api-overview" }; private static ApiLinkReference Link(string url, string file = "a.md", int line = 1) => new(url, file, line, 1, url.Length); @@ -38,6 +40,33 @@ public void FindBroken_PageOnlyInOlderVersion_ReportsMainTreeLink() broken.Should().ContainSingle().Which.Url.Should().EndWith("/kibana/operation/operation-upgrade"); } + [Test] + public void FindBroken_OldBumpShRootWithLivePage_IsNotReported() => + ApiLinkValidator + .FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/serverless/operation/operation-live"), + Link("https://www.elastic.co/docs/api/doc/kibana/topic/topic-kibana-spaces") + ], + Generated + ) + .Should() + .BeEmpty(); + + [Test] + public void FindBroken_OldBumpShRootWithMissingPage_IsStillReported() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/serverless/operation/operation-removed")], Generated) + .Should() + .ContainSingle(); + + [Test] + public void FindBroken_GoneBumpShRoot_IsReported() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/observability-serverless/operation/operation-x")], Generated) + .Should() + .ContainSingle(); + [Test] public void FindBroken_AliasUrl_Resolves() => ApiLinkValidator diff --git a/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs index 2c54b35e4c..27686a5fb5 100644 --- a/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Xml.Linq; +using Elastic.Documentation.Assembler.Building; using Elastic.Documentation.Configuration; namespace Elastic.Documentation.Build.Tests; @@ -10,41 +11,6 @@ namespace Elastic.Documentation.Build.Tests; // TODO: Delete this class once the API Explorer migration (elastic/docs-eng-team#436) is finalised. public class ApiSitemapParityTests { - // Mirrors modules/elastic-docs-v3/api-root-redirects.json in elastic/docs-infra (#459), which the native viewer resolves exact match first. - private const string ApiDocPrefix = "/docs/api/doc/"; - private static readonly IReadOnlyDictionary RedirectedRoots = new Dictionary - { - ["serverless"] = "kibana-serverless", - ["elastic-cloud-serverless"] = "cloud-serverless" - }; - private static readonly IReadOnlySet GoneRoots = new HashSet { "observability-serverless" }; - - // Exact page redirects from the same file. Bump.sh served a Servers page for specs that declare no servers, - // Cloud Billing renamed its Budgets tag, Kibana removed Streams operations from its 9.6 spec, and some bump.sh topic - // pages are a children: page or the /authentication page in the native viewer. - private static readonly IReadOnlyDictionary RedirectedPages = new Dictionary - { - ["/docs/api/doc/elasticsearch/servers"] = "/docs/api/doc/elasticsearch", - ["/docs/api/doc/elasticsearch/v8/servers"] = "/docs/api/doc/elasticsearch/v8", - ["/docs/api/doc/elasticsearch/v9/servers"] = "/docs/api/doc/elasticsearch/v9", - ["/docs/api/doc/elasticsearch-serverless/servers"] = "/docs/api/doc/elasticsearch-serverless", - ["/docs/api/doc/cloud-enterprise/servers"] = "/docs/api/doc/cloud-enterprise", - ["/docs/api/doc/cloud-enterprise/v3/servers"] = "/docs/api/doc/cloud-enterprise/v3", - ["/docs/api/doc/cloud-billing/group/endpoint-budgets"] = "/docs/api/doc/cloud-billing/group/endpoint-billing-budgets", - ["/docs/api/doc/kibana/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/kibana-api-overview", - ["/docs/api/doc/kibana/v8/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v8/kibana-api-overview", - ["/docs/api/doc/kibana/v9/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v9/kibana-api-overview", - ["/docs/api/doc/cloud/topic/topic-authentication"] = "/docs/api/doc/cloud/authentication", - ["/docs/api/doc/cloud-billing/topic/topic-authentication"] = "/docs/api/doc/cloud-billing/authentication", - ["/docs/api/doc/kibana/v9/group/endpoint-significant_events"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-delete-streams-name-queries-queryid"] = - "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-queries"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-significant-events"] = - "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-post-streams-name-queries-bulk"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-put-streams-name-queries-queryid"] = "/docs/api/doc/kibana/v9/group/endpoint-streams" - }; private static readonly Uri SitemapIndex = new("https://www.elastic.co/docs/api/sitemap_index.xml"); private static readonly XNamespace SitemapNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9"; @@ -61,12 +27,10 @@ public async Task Build_ContainsEveryProductionApiSitemapPage(CancellationToken var productionUrls = await DownloadPageUrls(client, SitemapIndex, cancellationToken); await Assert.That(productionUrls).IsNotEmpty(); - var gone = productionUrls.Where(url => GoneRoots.Contains(RootOf(url))).ToArray(); - var redirected = productionUrls.Where( - url => RedirectedPages.ContainsKey(url.AbsolutePath) || RedirectedRoots.ContainsKey(RootOf(url)) - ).ToArray(); + var gone = productionUrls.Where(url => ApiLegacyRedirects.IsGone(url.AbsolutePath)).ToArray(); + var redirected = productionUrls.Where(url => ApiLegacyRedirects.IsRedirected(url.AbsolutePath)).ToArray(); // Redirected pages are served by an exact or prefix redirect, so verify the page exists at its redirect target. - var auditedPaths = productionUrls.Except(gone).Select(RedirectTarget).Distinct().ToArray(); + var auditedPaths = productionUrls.Except(gone).Select(url => ApiLegacyRedirects.Resolve(url.AbsolutePath)).Distinct().ToArray(); var outputRoot = Environment.GetEnvironmentVariable("API_SITEMAP_PARITY_OUTPUT") ?? Path.Join(Paths.WorkingDirectoryRoot.FullName, ".artifacts", "assembly"); var missing = auditedPaths @@ -87,27 +51,6 @@ public async Task Build_ContainsEveryProductionApiSitemapPage(CancellationToken File.Delete(reportPath); } - private static string RootOf(Uri url) - { - var path = url.AbsolutePath; - if (!path.StartsWith(ApiDocPrefix, StringComparison.Ordinal)) - return string.Empty; - var rest = path[ApiDocPrefix.Length..]; - var slash = rest.IndexOf('/'); - return slash < 0 ? rest : rest[..slash]; - } - - private static string RedirectTarget(Uri url) - { - if (RedirectedPages.TryGetValue(url.AbsolutePath, out var page)) - return page; - - var root = RootOf(url); - return RedirectedRoots.TryGetValue(root, out var newRoot) - ? string.Concat(ApiDocPrefix, newRoot, url.AbsolutePath.AsSpan(ApiDocPrefix.Length + root.Length)) - : url.AbsolutePath; - } - private static async Task> DownloadPageUrls(HttpClient client, Uri sitemap, CancellationToken ctx) { await using var stream = await client.GetStreamAsync(sitemap, ctx); From 4c2a6bfa3573d808b3c7fd46924abc0647d00d30 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Wed, 4 Nov 2026 15:09:24 +0100 Subject: [PATCH 7/9] Record angle-bracket API autolinks and skip API links inside comments Markdig parses links itself, so they never reached the API link hooks. A recorder in front of its autolink parser now sees them without consuming input. The bare-URL and angle-bracket paths skip % comment lines, as the inline link parser already does. Co-Authored-By: Claude Sonnet 5.5 --- .../InlineParsers/AutoLinkInlineParser.cs | 42 +++++++++++++++++-- .../DiagnosticLinkInlineParser.cs | 6 +-- .../Inline/ApiLinkTests.cs | 34 +++++++++++++++ 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs index 555f41236e..1aa02d6090 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs @@ -4,6 +4,7 @@ using Elastic.Documentation.Diagnostics; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.Myst.Comments; using Markdig; using Markdig.Helpers; using Markdig.Parsers; @@ -24,8 +25,11 @@ public static MarkdownPipelineBuilder UseAutoLinks(this MarkdownPipelineBuilder public class AutoLinkBuilderExtension : IMarkdownExtension { - public void Setup(MarkdownPipelineBuilder pipeline) => - pipeline.InlineParsers.InsertBefore(new AutoLinkInlineParser()); + public void Setup(MarkdownPipelineBuilder pipeline) + { + _ = pipeline.InlineParsers.InsertBefore(new AutoLinkInlineParser()); + _ = pipeline.InlineParsers.InsertBefore(new AngleBracketApiLinkRecorder()); + } public void Setup(MarkdownPipeline pipeline, Markdig.Renderers.IMarkdownRenderer renderer) { @@ -86,7 +90,8 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) processor.Inline = linkInline; - DiagnosticLinkInlineParser.RecordApiLink(linkInline, context, url); + if (processor.Block is not CommentBlock) + DiagnosticLinkInlineParser.RecordApiLink(context, url, line, column); // Emit hint for elastic.co/docs URLs (after setting Inline so position is correct) if (url.Contains("elastic.co/docs", StringComparison.OrdinalIgnoreCase)) @@ -228,3 +233,34 @@ private static bool IsAllDigits(ReadOnlySpan span) return true; } } + +/// +/// Records <https://...> links that point at API reference pages. It never consumes input, so Markdig's own +/// autolink parser still creates the link. +/// +public class AngleBracketApiLinkRecorder : InlineParser +{ + public AngleBracketApiLinkRecorder() => OpeningCharacters = ['<']; + + public override bool Match(InlineProcessor processor, ref StringSlice slice) + { + if (processor.Block is CommentBlock) + return false; + + var span = slice.AsSpan(); + if (!span.StartsWith("'); + if (end < 0) + return false; + + var url = span[1..end].ToString(); + if (url.AsSpan().ContainsAny(' ', '<', '\n')) + return false; + + _ = processor.GetSourcePosition(slice.Start, out var line, out var column); + DiagnosticLinkInlineParser.RecordApiLink(processor.GetContext(), url, line, column); + return false; + } +} diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index 00294f6716..e8b17289e5 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -155,14 +155,14 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc if (ValidateExternalUri(link, processor, uri)) { - RecordApiLink(link, context, url); + RecordApiLink(context, url, link.Line, link.Column); return; } ProcessInternalLink(link, processor, context); } - internal static void RecordApiLink(LinkInline link, ParserContext context, string? url) + internal static void RecordApiLink(ParserContext context, string? url, int zeroBasedLine, int column) { if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _)) return; @@ -171,7 +171,7 @@ internal static void RecordApiLink(LinkInline link, ParserContext context, strin .Build .Collector .EmitApiLink( - new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, link.Line + 1, Math.Max(link.Column, 1), url!.Length) + new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, zeroBasedLine + 1, Math.Max(column, 1), url!.Length) ); } diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs index 688ef194d2..bf61533e4f 100644 --- a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs @@ -55,3 +55,37 @@ public void RecordsBareUrlAsApiLink() => .Should() .Be("https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing"); } + +[InheritsTests] +public class ApiAngleBracketAutoLinkTests() : InlineTest( + """ +See for details. +""" +) +{ + [Test] + public void RecordsAngleBracketAutolinkAsApiLink() => + Collector + .ApiLinks + .Should() + .ContainSingle() + .Which + .Url + .Should() + .Be("https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing"); +} + +[InheritsTests] +public class ApiLinksInCommentsTests() : InlineTest( + """ +% https://www.elastic.co/docs/api/doc/kibana/operation/old-bare +% +% [old](https://www.elastic.co/docs/api/doc/kibana/operation/old-inline) + +Text. +""" +) +{ + [Test] + public void RecordsNothingFromComments() => Collector.ApiLinks.Should().BeEmpty(); +} From ae861f3b928b6f616c49e2b25088c26240017f7a Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Wed, 4 Nov 2026 15:13:21 +0100 Subject: [PATCH 8/9] Fix the angle-bracket autolink test base Markdig builds an AutolinkInline for , not a LinkInline, so the generic test base failed its own non-null check. Co-Authored-By: Claude Sonnet 5.5 --- tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs index bf61533e4f..96ad73ee12 100644 --- a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs @@ -57,7 +57,7 @@ public void RecordsBareUrlAsApiLink() => } [InheritsTests] -public class ApiAngleBracketAutoLinkTests() : InlineTest( +public class ApiAngleBracketAutoLinkTests() : InlineTest( """ See for details. """ From ed50b61fd71b28723a9749314c01e78d4fc144f8 Mon Sep 17 00:00:00 2001 From: Andrea De Pirro Date: Thu, 5 Nov 2026 10:15:17 +0100 Subject: [PATCH 9/9] Check links inside API descriptions against the generated pages API descriptions parse without link validation, so the links between API pages that come from the OpenAPI specs were never checked. HtmlWriter now records them, using the outbound link extractor, and spec download files count as generated pages. A run on the staging checkouts finds 10 broken description links across 5 targets. Co-Authored-By: Claude Sonnet 5.5 --- docs/syntax/links.md | 2 +- src/Elastic.ApiExplorer/OpenApiGenerator.cs | 2 + .../Diagnostics/ApiLinkReference.cs | 24 ++++++++++++ src/Elastic.Markdown/HtmlWriter.cs | 14 +++++++ .../Building/ApiLegacyRedirects.cs | 7 +++- .../Building/ApiLinkValidator.cs | 3 +- .../OpenApiGeneratorCatalogSplitTests.cs | 1 + .../ApiLinkValidatorTests.cs | 8 ++++ .../Diagnostics/ApiLinkMatcherTests.cs | 18 +++++++++ .../Inline/ApiDescriptionLinkTests.cs | 38 +++++++++++++++++++ 10 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs diff --git a/docs/syntax/links.md b/docs/syntax/links.md index d31ca5d835..e2f4091bf2 100644 --- a/docs/syntax/links.md +++ b/docs/syntax/links.md @@ -121,7 +121,7 @@ Links to `https://www.elastic.co/docs/api/doc/...` pages, including `redirects.y [Search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) ``` -If an endpoint no longer exists in the current API docs, point the link at the versioned tree that still has it, for example `https://www.elastic.co/docs/api/doc/kibana/v8/...`. Only the page is checked, not the `#anchor`. Old bump.sh URLs that resolve only through edge redirects are also reported, so link to the canonical URL. +If an endpoint no longer exists in the current API docs, point the link at the versioned tree that still has it, for example `https://www.elastic.co/docs/api/doc/kibana/v8/...`. The links between API pages that come from the OpenAPI spec descriptions are checked the same way; fix those in the spec. Only the page is checked, not the `#anchor`. Old bump.sh URLs that resolve only through edge redirects are also reported, so link to the canonical URL. ### Linked images diff --git a/src/Elastic.ApiExplorer/OpenApiGenerator.cs b/src/Elastic.ApiExplorer/OpenApiGenerator.cs index 8ac09cd9ec..44dc7d709b 100644 --- a/src/Elastic.ApiExplorer/OpenApiGenerator.cs +++ b/src/Elastic.ApiExplorer/OpenApiGenerator.cs @@ -507,6 +507,8 @@ private async Task GenerateApiProduct(ApiProductGeneration generation, Cancel ct private async Task WriteSpecDownloads(INavigationItem landing, OpenApiDocument document, Cancel ctx) { + RecordUrl($"{landing.Url.TrimEnd('/')}.json"); + RecordUrl($"{landing.Url.TrimEnd('/')}.yaml"); await WriteSpecSibling( ApiOutputPaths.RelativeJsonFile(landing.Url, context.UrlPathPrefix), (stream, token) => document.SerializeAsJsonAsync(stream, OpenApiSpecVersion.OpenApi3_1, token), diff --git a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs index 347bab0d80..faafb9cc4b 100644 --- a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs +++ b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs @@ -38,6 +38,30 @@ public static bool TryGetApiPath(string? url, out string path) return true; } + /// + /// Returns as an absolute elastic.co API URL when it targets an API page, either as such or as a + /// site path under , which is how the API Explorer writes links between its own pages. + /// + public static bool TryCanonicalize(string? url, string? urlPathPrefix, out string canonical) + { + canonical = string.Empty; + if (string.IsNullOrWhiteSpace(url)) + return false; + + if (TryGetApiPath(url, out _)) + { + canonical = url; + return true; + } + + var prefix = urlPathPrefix?.TrimEnd('/') ?? string.Empty; + if (!url.StartsWith($"{prefix}/api/doc/", StringComparison.OrdinalIgnoreCase)) + return false; + + canonical = $"https://www.elastic.co/docs{url[prefix.Length..]}"; + return true; + } + public static string Normalize(string path) => DuplicateSlashes().Replace(Uri.UnescapeDataString(path), "/").TrimEnd('/').ToLowerInvariant(); diff --git a/src/Elastic.Markdown/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs index 65feed874f..4272b4ca7a 100644 --- a/src/Elastic.Markdown/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -8,11 +8,13 @@ using Elastic.Documentation.Configuration.LegacyUrlMappings; using Elastic.Documentation.Configuration.Products; using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Navigation; using Elastic.Documentation.Site; using Elastic.Documentation.Site.FileProviders; using Elastic.Documentation.Site.Navigation; +using Elastic.Markdown.Exporters; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.Helpers; using Elastic.Markdown.IO; @@ -71,9 +73,21 @@ public string RenderApiDescription(string markdown, IFileInfo? source) { source ??= DocumentationSet.Context.ConfigurationPath; var parsed = DocumentationSet.MarkdownParser.ParseApiDescriptionString(markdown, source); + RecordApiLinks(parsed, source); return MarkdownFile.CreateHtml(parsed, stripFirstHeadingLevel1: true, MarkdownParser.ApiDescriptionPipeline); } + // Descriptions parse without link validation, so the links they hold between API pages are recorded here for the API link check. + private void RecordApiLinks(MarkdownDocument parsed, IFileInfo source) + { + var context = DocumentationSet.Context; + foreach (var url in OutboundLinkExtractor.Extract(parsed) ?? []) + { + if (ApiLinkMatcher.TryCanonicalize(url, context.UrlPathPrefix, out var canonical)) + context.Collector.EmitApiLink(new ApiLinkReference(canonical, source.FullName, 1, 1, canonical.Length)); + } + } + /// public IReadOnlyList ReadApiDescriptionHeadings(string markdown, IFileInfo? source) { diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs index 747f4a0fb8..1cf1f72b50 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs @@ -72,6 +72,11 @@ private static string RootOf(string path) return string.Empty; var rest = path[ApiLinkMatcher.ApiDocPrefix.Length..]; var slash = rest.IndexOf('/'); - return slash < 0 ? rest : rest[..slash]; + if (slash >= 0) + return rest[..slash]; + + // A product root also has spec downloads next to it: kibana.json and kibana.yaml. + var extension = Path.GetExtension(rest); + return extension is ".json" or ".yaml" ? rest[..^extension.Length] : rest; } } diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs index 709756b6ca..345eec2756 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -68,7 +68,8 @@ Cancel ctx Column = link.Column, Length = link.Length, Message = $"API link '{link.Url}' does not resolve to a generated API page. Point it at an existing page " + - "(for older endpoints use a versioned tree such as /docs/api/doc//v8/...) or confirm the endpoint was removed intentionally." + "(for older endpoints use a versioned tree such as /docs/api/doc//v8/...) or confirm the endpoint was removed intentionally. " + + "Links in API descriptions are fixed in the OpenAPI spec." }); } diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs index f83d81f0b7..cddedc1164 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs @@ -74,6 +74,7 @@ public async Task GenerateProducts_RecordsEveryRenderedPageInGeneratedUrls() generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch"); generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch/group/endpoint-core"); + generator.GeneratedUrls.Should().Contain(["/docs/api/doc/elasticsearch.json", "/docs/api/doc/elasticsearch.yaml"]); generator.GeneratedUrls.Should().OnlyContain(url => url == url.ToLowerInvariant() && !url.EndsWith('/')); generator.GeneratedUrls.Should().Contain(url => url.Contains("/operation/operation-")); } diff --git a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs index e63e85c828..bb9f4d7765 100644 --- a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs @@ -18,6 +18,7 @@ public class ApiLinkValidatorTests "/docs/api/doc/kibana/v8/operation/operation-upgrade", "/docs/api/doc/kibana/operation/operation-alias", "/docs/api/doc/kibana-serverless/operation/operation-live", + "/docs/api/doc/cloud-serverless.yaml", "/docs/api/doc/kibana/kibana-api-overview" }; @@ -53,6 +54,13 @@ public void FindBroken_OldBumpShRootWithLivePage_IsNotReported() => .Should() .BeEmpty(); + [Test] + public void FindBroken_OldBumpShRootSpecDownload_ResolvesToTheNewRootDownload() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/elastic-cloud-serverless.yaml")], Generated) + .Should() + .BeEmpty(); + [Test] public void FindBroken_OldBumpShRootWithMissingPage_IsStillReported() => ApiLinkValidator diff --git a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs index 8c27c1b89a..be6f4f8b3c 100644 --- a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs +++ b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs @@ -29,4 +29,22 @@ public void TryGetApiPath_ApiUrl_ReturnsNormalizedPath(string url, string expect [Arguments("")] [Arguments(null)] public void TryGetApiPath_NonApiUrl_ReturnsFalse(string? url) => ApiLinkMatcher.TryGetApiPath(url, out _).Should().BeFalse(); + + [Test] + [Arguments("/docs/api/doc/kibana/operation/operation-x", "/docs", "https://www.elastic.co/docs/api/doc/kibana/operation/operation-x")] + [Arguments("/elastic/docs-builder/docs/123/api/doc/kibana/group/endpoint-a", "/elastic/docs-builder/docs/123", "https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a")] + [Arguments("https://www.elastic.co/docs/api/doc/kibana", "/docs", "https://www.elastic.co/docs/api/doc/kibana")] + public void TryCanonicalize_ApiPageLink_ReturnsAbsoluteProductionUrl(string url, string prefix, string expected) + { + ApiLinkMatcher.TryCanonicalize(url, prefix, out var canonical).Should().BeTrue(); + canonical.Should().Be(expected); + } + + [Test] + [Arguments("/docs/solutions/search", "/docs")] + [Arguments("../operation/x", "/docs")] + [Arguments("https://example.com/docs/api/doc/kibana", "/docs")] + [Arguments(null, "/docs")] + public void TryCanonicalize_OtherLink_ReturnsFalse(string? url, string prefix) => + ApiLinkMatcher.TryCanonicalize(url, prefix, out _).Should().BeFalse(); } diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs new file mode 100644 index 0000000000..f13aabe62b --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; + +namespace Elastic.Markdown.Tests.Inline; + +[InheritsTests] +public class ApiDescriptionLinkTests() : InlineTest(""" +Placeholder. +""") +{ + [Test] + public void RenderApiDescription_LinksBetweenApiPages_AreRecordedWithTheirCanonicalUrl() + { + var writer = new HtmlWriter(Set, FileSystem, new DescriptionGenerator()); + var source = FileSystem.FileInfo.New("docs/api/doc/kibana/operation/operation-x/description.md"); + + _ = writer.RenderApiDescription( + """ + See [the rule](/docs/api/doc/kibana/operation/operation-find-rules), [groups](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a) + and [the guide](https://www.elastic.co/docs/solutions/search) or [elsewhere](https://example.com). + """, + source + ); + + Collector + .ApiLinks + .Select(l => l.Url) + .Should() + .BeEquivalentTo([ + "https://www.elastic.co/docs/api/doc/kibana/operation/operation-find-rules", + "https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a" + ]); + Collector.ApiLinks.Should().OnlyContain(l => l.File.EndsWith("description.md")); + } +}