Warn when links to the API docs point at missing pages

docs/syntax/links.md

@@ -113,6 +113,16 @@ Link to websites and resources outside the Elastic docs:
 
 External links display an arrow indicator next to the link text.
 
+### Links to API reference pages
+
+Links to `https://www.elastic.co/docs/api/doc/...` pages, including `redirects.yml` targets, are checked when the assembler builds with the API Explorer enabled. Each link is compared with the API pages that build generates. A link that doesn't match produces a warning, and the assembler writes the full list to `api-link-report.json` next to `redirects.json`.
+
+```markdown
+[Search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search)
+```
+
+If an endpoint no longer exists in the current API docs, point the link at the versioned tree that still has it, for example `https://www.elastic.co/docs/api/doc/kibana/v8/...`. The links between API pages that come from the OpenAPI spec descriptions are checked the same way; fix those in the spec. Only the page is checked, not the `#anchor`. Old bump.sh URLs that resolve only through edge redirects are also reported, so link to the canonical URL.
+
 ### Linked images
 
 Wrap an image in a link to make the image itself clickable. Combine the image syntax with the link syntax:

src/Elastic.ApiExplorer/OpenApiGenerator.cs

@@ -15,6 +15,7 @@
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Products;
 using Elastic.Documentation.Configuration.Toc;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Navigation;
 using Elastic.Documentation.Site.FileProviders;
 using Elastic.Documentation.Site.Navigation;
@@ -66,6 +67,10 @@ public class OpenApiGenerator(
 	private readonly VersionIndexClient _versionIndexClient = versionIndexClient ?? new VersionIndexClient();
 	private readonly IOpenApiSpecificationReader _openApiReader = openApiReader ?? OpenApiReader.Instance;
 	private readonly ConcurrentDictionary<string, Task<ResolvedProductDocuments>> _documents = new(StringComparer.OrdinalIgnoreCase);
+	private readonly ConcurrentDictionary<string, byte> _generatedUrls = new(StringComparer.Ordinal);
+
+	/// <summary>Normalized URL path of every page this generator rendered, including redirect stubs for collapsed operations.</summary>
+	public IReadOnlyCollection<string> GeneratedUrls => [.. _generatedUrls.Keys];
 
 	public LandingNavigationItem CreateNavigation(
 		string apiUrlSuffix,
@@ -502,6 +507,8 @@ private async Task GenerateApiProduct(ApiProductGeneration generation, Cancel ct
 
 	private async Task WriteSpecDownloads(INavigationItem landing, OpenApiDocument document, Cancel ctx)
 	{
+		RecordUrl($"{landing.Url.TrimEnd('/')}.json");
+		RecordUrl($"{landing.Url.TrimEnd('/')}.yaml");
 		await WriteSpecSibling(
 			ApiOutputPaths.RelativeJsonFile(landing.Url, context.UrlPathPrefix),
 			(stream, token) => document.SerializeAsJsonAsync(stream, OpenApiSpecVersion.OpenApi3_1, token),
@@ -580,6 +587,8 @@ Cancel ctx
 	{
 		var pages = new List<(INavigationItem Item, IApiModel Model)>();
 		CollectPages(root, pages);
+		foreach (var (item, _) in pages)
+			RecordUrl(item.Url);
 
 		// Pages of one version render in parallel. When two items share a URL the last one wins,
 		// as it did when pages were written sequentially in depth-first order.
@@ -596,12 +605,22 @@ Cancel ctx
 		}).ConfigureAwait(false);
 	}
 
+	// Narrative links use the production path (/docs/api/doc/...), while a build such as a PR preview serves the same
+	// pages under another prefix, so record each page under the production path.
+	private void RecordUrl(string url)
+	{
+		var prefix = context.UrlPathPrefix ?? string.Empty;
+		var relative = prefix.Length > 0 && url.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) ? url[prefix.Length..] : url;
+		_ = _generatedUrls.TryAdd(ApiLinkMatcher.Normalize($"/docs{relative}"), 0);
+	}
+
 	/// <summary>Former per-operation URLs of a collapsed page forward to it, keeping any <c>#fragment</c>.</summary>
 	private async Task WriteRedirects(OperationNavigationItem page, Cancel ctx)
 	{
 		var html = ApiRedirectPage.Html(page.Url);
 		foreach (var alias in page.AliasUrls)
 		{
+			RecordUrl(alias);
 			await WriteSpecSibling(
 				ApiOutputPaths.RelativeHtmlFile(alias, context.UrlPathPrefix),
 				async (stream, c) =>

src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs

@@ -55,6 +55,8 @@ public record ConfigurationFile
 
 	public Dictionary<string, LinkRedirect>? Redirects { get; }
 
+	public IFileInfo? RedirectsSource { get; }
+
 	public HashSet<Product> Products { get; private set; } = [];
 
 	private readonly Dictionary<string, string> _substitutions = [with(StringComparer.OrdinalIgnoreCase)];
@@ -138,6 +140,7 @@ ProductsConfiguration productsConfig
 
 		var redirectFile = new RedirectFile(_context);
 		Redirects = redirectFile.Redirects;
+		RedirectsSource = redirectFile.Source;
 
 		try
 		{

src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs

@@ -0,0 +1,70 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.RegularExpressions;
+
+namespace Elastic.Documentation.Diagnostics;
+
+/// <summary>A link to an API reference page, recorded during the Markdown build and checked once the API pages exist.</summary>
+public readonly record struct ApiLinkReference(string Url, string File, int Line, int Column, int Length);
+
+public static partial class ApiLinkMatcher
+{
+	public const string ApiDocPrefix = "/docs/api/doc/";
+
+	/// <summary>
+	/// Returns the normalized page path (no query, fragment or trailing slash, lowercase) when
+	/// <paramref name="url"/> targets an API reference page on elastic.co.
+	/// </summary>
+	public static bool TryGetApiPath(string? url, out string path)
+	{
+		path = string.Empty;
+		if (string.IsNullOrWhiteSpace(url) || !Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri))
+			return false;
+		if (uri.Scheme is not ("http" or "https"))
+			return false;
+		if (
+			!uri.Host.Equals("www.elastic.co", StringComparison.OrdinalIgnoreCase)
+			&& !uri.Host.Equals("elastic.co", StringComparison.OrdinalIgnoreCase)
+		)
+			return false;
+
+		var normalized = Normalize(uri.AbsolutePath);
+		if (!normalized.StartsWith(ApiDocPrefix, StringComparison.Ordinal))
+			return false;
+
+		path = normalized;
+		return true;
+	}
+
+	/// <summary>
+	/// Returns <paramref name="url"/> as an absolute elastic.co API URL when it targets an API page, either as such or as a
+	/// site path under <paramref name="urlPathPrefix"/>, which is how the API Explorer writes links between its own pages.
+	/// </summary>
+	public static bool TryCanonicalize(string? url, string? urlPathPrefix, out string canonical)
+	{
+		canonical = string.Empty;
+		if (string.IsNullOrWhiteSpace(url))
+			return false;
+
+		if (TryGetApiPath(url, out _))
+		{
+			canonical = url;
+			return true;
+		}
+
+		var prefix = urlPathPrefix?.TrimEnd('/') ?? string.Empty;
+		if (!url.StartsWith($"{prefix}/api/doc/", StringComparison.OrdinalIgnoreCase))
+			return false;
+
+		canonical = $"https://www.elastic.co/docs{url[prefix.Length..]}";
+		return true;
+	}
+
+	public static string Normalize(string path) =>
+		DuplicateSlashes().Replace(Uri.UnescapeDataString(path), "/").TrimEnd('/').ToLowerInvariant();
+
+	[GeneratedRegex("/{2,}")]
+	private static partial Regex DuplicateSlashes();
+}

src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs

@@ -41,6 +41,8 @@ public class DiagnosticsCollector(
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
+	public ConcurrentBag<ApiLinkReference> ApiLinks { get; } = [];
+
 	public bool NoHints { get; set; }
 
 	public bool IsStarted => _readerStarted;
@@ -140,6 +142,8 @@ public virtual async Task StopAsync(Cancel cancellationToken)
 
 	public void EmitCrossLink(string link) => CrossLinks.Add(link);
 
+	public void EmitApiLink(ApiLinkReference link) => ApiLinks.Add(link);
+
 	public virtual void Write(Diagnostic diagnostic)
 	{
 		// IncrementSeverityCount MUST run unconditionally and before Channel.Write.

src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs

@@ -17,6 +17,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable
 
 	DiagnosticsChannel Channel { get; }
 	ConcurrentBag<string> CrossLinks { get; }
+	ConcurrentBag<ApiLinkReference> ApiLinks { get; }
 	ConcurrentDictionary<string, bool> OffendingFiles { get; }
 	ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; }
 
@@ -49,6 +50,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable
 	void Write(Diagnostic diagnostic);
 	void CollectUsedSubstitutionKey(ReadOnlySpan<char> key);
 	void EmitCrossLink(string link);
+	void EmitApiLink(ApiLinkReference link);
 
 	void EmitError(IFileInfo file, string message, Exception? e = null) => EmitError(file.FullName, message, e);
 

src/Elastic.Markdown/HtmlWriter.cs

@@ -8,11 +8,13 @@
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
 using Elastic.Documentation.Configuration.Versions;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Extensions;
 using Elastic.Documentation.Navigation;
 using Elastic.Documentation.Site;
 using Elastic.Documentation.Site.FileProviders;
 using Elastic.Documentation.Site.Navigation;
+using Elastic.Markdown.Exporters;
 using Elastic.Markdown.Extensions.DetectionRules;
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
@@ -71,9 +73,21 @@ public string RenderApiDescription(string markdown, IFileInfo? source)
 	{
 		source ??= DocumentationSet.Context.ConfigurationPath;
 		var parsed = DocumentationSet.MarkdownParser.ParseApiDescriptionString(markdown, source);
+		RecordApiLinks(parsed, source);
 		return MarkdownFile.CreateHtml(parsed, stripFirstHeadingLevel1: true, MarkdownParser.ApiDescriptionPipeline);
 	}
 
+	// Descriptions parse without link validation, so the links they hold between API pages are recorded here for the API link check.
+	private void RecordApiLinks(MarkdownDocument parsed, IFileInfo source)
+	{
+		var context = DocumentationSet.Context;
+		foreach (var url in OutboundLinkExtractor.Extract(parsed) ?? [])
+		{
+			if (ApiLinkMatcher.TryCanonicalize(url, context.UrlPathPrefix, out var canonical))
+				context.Collector.EmitApiLink(new ApiLinkReference(canonical, source.FullName, 1, 1, canonical.Length));
+		}
+	}
+
 	/// <inheritdoc />
 	public IReadOnlyList<TocItem> ReadApiDescriptionHeadings(string markdown, IFileInfo? source)
 	{

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -13,6 +13,7 @@
 using Elastic.Documentation.Configuration.ReleaseNotes;
 using Elastic.Documentation.Configuration.Toc;
 using Elastic.Documentation.Configuration.Toc.CliReference;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Links;
 using Elastic.Documentation.Links.CrossLinks;
 using Elastic.Documentation.Navigation;
@@ -181,6 +182,10 @@ void ValidateExists(string from, string to, IReadOnlyDictionary<string, string?>
 			{
 				if (!Uri.TryCreate(to, UriKind.Absolute, out _))
 					Context.EmitError(Configuration.SourceFile, $"Redirect {from} points to {to} which is not a valid URI");
+				else if (ApiLinkMatcher.TryGetApiPath(to, out _))
+					Context.Collector.EmitApiLink(
+						new ApiLinkReference(to, (Configuration.RedirectsSource ?? Configuration.SourceFile).FullName, 1, 1, to.Length)
+					);
 
 				return;
 			}

src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using Elastic.Documentation;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.Diagnostics;
 using Markdig;
@@ -90,7 +91,22 @@ internal static class DirectiveLinkValidator
 			return url;
 
 		if (IsExternal(trimmed))
+		{
+			if (ApiLinkMatcher.TryGetApiPath(trimmed, out _))
+				context
+					.Build
+					.Collector
+					.EmitApiLink(
+						new ApiLinkReference(
+							trimmed,
+							block.CurrentFile.FullName,
+							block.Line + 1,
+							System.Math.Max(block.Column, 1),
+							trimmed.Length
+						)
+					);
 			return url;
+		}
 
 		if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) && CrossLinkValidator.IsCrossLink(uri))
 			return ResolveCrossLink(url, uri, block, context);

src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs

@@ -4,6 +4,7 @@
 
 using Elastic.Documentation.Diagnostics;
 using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.Myst.Comments;
 using Markdig;
 using Markdig.Helpers;
 using Markdig.Parsers;
@@ -24,8 +25,11 @@ public static MarkdownPipelineBuilder UseAutoLinks(this MarkdownPipelineBuilder
 
 public class AutoLinkBuilderExtension : IMarkdownExtension
 {
-	public void Setup(MarkdownPipelineBuilder pipeline) =>
-		pipeline.InlineParsers.InsertBefore<LinkInlineParser>(new AutoLinkInlineParser());
+	public void Setup(MarkdownPipelineBuilder pipeline)
+	{
+		_ = pipeline.InlineParsers.InsertBefore<LinkInlineParser>(new AutoLinkInlineParser());
+		_ = pipeline.InlineParsers.InsertBefore<AutolinkInlineParser>(new AngleBracketApiLinkRecorder());
+	}
 
 	public void Setup(MarkdownPipeline pipeline, Markdig.Renderers.IMarkdownRenderer renderer)
 	{
@@ -86,6 +90,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 
 		processor.Inline = linkInline;
 
+		if (processor.Block is not CommentBlock)
+			DiagnosticLinkInlineParser.RecordApiLink(context, url, line, column);
+
 		// Emit hint for elastic.co/docs URLs (after setting Inline so position is correct)
 		if (url.Contains("elastic.co/docs", StringComparison.OrdinalIgnoreCase))
 			processor.EmitHint(
@@ -226,3 +233,34 @@ private static bool IsAllDigits(ReadOnlySpan<char> span)
 		return true;
 	}
 }
+
+/// <summary>
+/// Records <c>&lt;https://...&gt;</c> links that point at API reference pages. It never consumes input, so Markdig's own
+/// autolink parser still creates the link.
+/// </summary>
+public class AngleBracketApiLinkRecorder : InlineParser
+{
+	public AngleBracketApiLinkRecorder() => OpeningCharacters = ['<'];
+
+	public override bool Match(InlineProcessor processor, ref StringSlice slice)
+	{
+		if (processor.Block is CommentBlock)
+			return false;
+
+		var span = slice.AsSpan();
+		if (!span.StartsWith("<http", StringComparison.OrdinalIgnoreCase))
+			return false;
+
+		var end = span.IndexOf('>');
+		if (end < 0)
+			return false;
+
+		var url = span[1..end].ToString();
+		if (url.AsSpan().ContainsAny(' ', '<', '\n'))
+			return false;
+
+		_ = processor.GetSourcePosition(slice.Start, out var line, out var column);
+		DiagnosticLinkInlineParser.RecordApiLink(processor.GetContext(), url, line, column);
+		return false;
+	}
+}

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -7,6 +7,7 @@
 using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using Elastic.Documentation;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Extensions;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.Diagnostics;
@@ -153,11 +154,27 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc
 		link.SetData("isCrossLink", false);
 
 		if (ValidateExternalUri(link, processor, uri))
+		{
+			RecordApiLink(context, url, link.Line, link.Column);
 			return;
+		}
 
 		ProcessInternalLink(link, processor, context);
 	}
 
+	internal static void RecordApiLink(ParserContext context, string? url, int zeroBasedLine, int column)
+	{
+		if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _))
+			return;
+
+		context
+			.Build
+			.Collector
+			.EmitApiLink(
+				new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, zeroBasedLine + 1, Math.Max(column, 1), url!.Length)
+			);
+	}
+
 	private static bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url)
 	{
 		if (string.IsNullOrEmpty(url))