diff --git a/docs/syntax/links.md b/docs/syntax/links.md index 78ea3b894a..e2f4091bf2 100644 --- a/docs/syntax/links.md +++ b/docs/syntax/links.md @@ -113,6 +113,16 @@ Link to websites and resources outside the Elastic docs: External links display an arrow indicator next to the link text. +### Links to API reference pages + +Links to `https://www.elastic.co/docs/api/doc/...` pages, including `redirects.yml` targets, are checked when the assembler builds with the API Explorer enabled. Each link is compared with the API pages that build generates. A link that doesn't match produces a warning, and the assembler writes the full list to `api-link-report.json` next to `redirects.json`. + +```markdown +[Search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) +``` + +If an endpoint no longer exists in the current API docs, point the link at the versioned tree that still has it, for example `https://www.elastic.co/docs/api/doc/kibana/v8/...`. The links between API pages that come from the OpenAPI spec descriptions are checked the same way; fix those in the spec. Only the page is checked, not the `#anchor`. Old bump.sh URLs that resolve only through edge redirects are also reported, so link to the canonical URL. + ### Linked images Wrap an image in a link to make the image itself clickable. Combine the image syntax with the link syntax: diff --git a/src/Elastic.ApiExplorer/OpenApiGenerator.cs b/src/Elastic.ApiExplorer/OpenApiGenerator.cs index 96a47f8010..44dc7d709b 100644 --- a/src/Elastic.ApiExplorer/OpenApiGenerator.cs +++ b/src/Elastic.ApiExplorer/OpenApiGenerator.cs @@ -15,6 +15,7 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Products; using Elastic.Documentation.Configuration.Toc; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Navigation; using Elastic.Documentation.Site.FileProviders; using Elastic.Documentation.Site.Navigation; @@ -66,6 +67,10 @@ public class OpenApiGenerator( private readonly VersionIndexClient _versionIndexClient = versionIndexClient ?? new VersionIndexClient(); private readonly IOpenApiSpecificationReader _openApiReader = openApiReader ?? OpenApiReader.Instance; private readonly ConcurrentDictionary> _documents = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _generatedUrls = new(StringComparer.Ordinal); + + /// Normalized URL path of every page this generator rendered, including redirect stubs for collapsed operations. + public IReadOnlyCollection GeneratedUrls => [.. _generatedUrls.Keys]; public LandingNavigationItem CreateNavigation( string apiUrlSuffix, @@ -502,6 +507,8 @@ private async Task GenerateApiProduct(ApiProductGeneration generation, Cancel ct private async Task WriteSpecDownloads(INavigationItem landing, OpenApiDocument document, Cancel ctx) { + RecordUrl($"{landing.Url.TrimEnd('/')}.json"); + RecordUrl($"{landing.Url.TrimEnd('/')}.yaml"); await WriteSpecSibling( ApiOutputPaths.RelativeJsonFile(landing.Url, context.UrlPathPrefix), (stream, token) => document.SerializeAsJsonAsync(stream, OpenApiSpecVersion.OpenApi3_1, token), @@ -580,6 +587,8 @@ Cancel ctx { var pages = new List<(INavigationItem Item, IApiModel Model)>(); CollectPages(root, pages); + foreach (var (item, _) in pages) + RecordUrl(item.Url); // Pages of one version render in parallel. When two items share a URL the last one wins, // as it did when pages were written sequentially in depth-first order. @@ -596,12 +605,22 @@ Cancel ctx }).ConfigureAwait(false); } + // Narrative links use the production path (/docs/api/doc/...), while a build such as a PR preview serves the same + // pages under another prefix, so record each page under the production path. + private void RecordUrl(string url) + { + var prefix = context.UrlPathPrefix ?? string.Empty; + var relative = prefix.Length > 0 && url.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) ? url[prefix.Length..] : url; + _ = _generatedUrls.TryAdd(ApiLinkMatcher.Normalize($"/docs{relative}"), 0); + } + /// Former per-operation URLs of a collapsed page forward to it, keeping any #fragment. private async Task WriteRedirects(OperationNavigationItem page, Cancel ctx) { var html = ApiRedirectPage.Html(page.Url); foreach (var alias in page.AliasUrls) { + RecordUrl(alias); await WriteSpecSibling( ApiOutputPaths.RelativeHtmlFile(alias, context.UrlPathPrefix), async (stream, c) => diff --git a/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs b/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs index 625806b40c..63f8d221b0 100644 --- a/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs +++ b/src/Elastic.Documentation.Configuration/Builder/ConfigurationFile.cs @@ -55,6 +55,8 @@ public record ConfigurationFile public Dictionary? Redirects { get; } + public IFileInfo? RedirectsSource { get; } + public HashSet Products { get; private set; } = []; private readonly Dictionary _substitutions = [with(StringComparer.OrdinalIgnoreCase)]; @@ -138,6 +140,7 @@ ProductsConfiguration productsConfig var redirectFile = new RedirectFile(_context); Redirects = redirectFile.Redirects; + RedirectsSource = redirectFile.Source; try { diff --git a/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs new file mode 100644 index 0000000000..faafb9cc4b --- /dev/null +++ b/src/Elastic.Documentation/Diagnostics/ApiLinkReference.cs @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.RegularExpressions; + +namespace Elastic.Documentation.Diagnostics; + +/// A link to an API reference page, recorded during the Markdown build and checked once the API pages exist. +public readonly record struct ApiLinkReference(string Url, string File, int Line, int Column, int Length); + +public static partial class ApiLinkMatcher +{ + public const string ApiDocPrefix = "/docs/api/doc/"; + + /// + /// Returns the normalized page path (no query, fragment or trailing slash, lowercase) when + /// targets an API reference page on elastic.co. + /// + public static bool TryGetApiPath(string? url, out string path) + { + path = string.Empty; + if (string.IsNullOrWhiteSpace(url) || !Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri)) + return false; + if (uri.Scheme is not ("http" or "https")) + return false; + if ( + !uri.Host.Equals("www.elastic.co", StringComparison.OrdinalIgnoreCase) + && !uri.Host.Equals("elastic.co", StringComparison.OrdinalIgnoreCase) + ) + return false; + + var normalized = Normalize(uri.AbsolutePath); + if (!normalized.StartsWith(ApiDocPrefix, StringComparison.Ordinal)) + return false; + + path = normalized; + return true; + } + + /// + /// Returns as an absolute elastic.co API URL when it targets an API page, either as such or as a + /// site path under , which is how the API Explorer writes links between its own pages. + /// + public static bool TryCanonicalize(string? url, string? urlPathPrefix, out string canonical) + { + canonical = string.Empty; + if (string.IsNullOrWhiteSpace(url)) + return false; + + if (TryGetApiPath(url, out _)) + { + canonical = url; + return true; + } + + var prefix = urlPathPrefix?.TrimEnd('/') ?? string.Empty; + if (!url.StartsWith($"{prefix}/api/doc/", StringComparison.OrdinalIgnoreCase)) + return false; + + canonical = $"https://www.elastic.co/docs{url[prefix.Length..]}"; + return true; + } + + public static string Normalize(string path) => + DuplicateSlashes().Replace(Uri.UnescapeDataString(path), "/").TrimEnd('/').ToLowerInvariant(); + + [GeneratedRegex("/{2,}")] + private static partial Regex DuplicateSlashes(); +} diff --git a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs index 86bca5be89..6f8747a6f9 100644 --- a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs @@ -41,6 +41,8 @@ public class DiagnosticsCollector( public ConcurrentBag CrossLinks { get; } = []; + public ConcurrentBag ApiLinks { get; } = []; + public bool NoHints { get; set; } public bool IsStarted => _readerStarted; @@ -140,6 +142,8 @@ public virtual async Task StopAsync(Cancel cancellationToken) public void EmitCrossLink(string link) => CrossLinks.Add(link); + public void EmitApiLink(ApiLinkReference link) => ApiLinks.Add(link); + public virtual void Write(Diagnostic diagnostic) { // IncrementSeverityCount MUST run unconditionally and before Channel.Write. diff --git a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs index 4622e521af..c16b74e10a 100644 --- a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs @@ -17,6 +17,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable DiagnosticsChannel Channel { get; } ConcurrentBag CrossLinks { get; } + ConcurrentBag ApiLinks { get; } ConcurrentDictionary OffendingFiles { get; } ConcurrentDictionary InUseSubstitutionKeys { get; } @@ -49,6 +50,7 @@ public interface IDiagnosticsCollector : IAsyncDisposable void Write(Diagnostic diagnostic); void CollectUsedSubstitutionKey(ReadOnlySpan key); void EmitCrossLink(string link); + void EmitApiLink(ApiLinkReference link); void EmitError(IFileInfo file, string message, Exception? e = null) => EmitError(file.FullName, message, e); diff --git a/src/Elastic.Markdown/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs index 65feed874f..4272b4ca7a 100644 --- a/src/Elastic.Markdown/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -8,11 +8,13 @@ using Elastic.Documentation.Configuration.LegacyUrlMappings; using Elastic.Documentation.Configuration.Products; using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Navigation; using Elastic.Documentation.Site; using Elastic.Documentation.Site.FileProviders; using Elastic.Documentation.Site.Navigation; +using Elastic.Markdown.Exporters; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.Helpers; using Elastic.Markdown.IO; @@ -71,9 +73,21 @@ public string RenderApiDescription(string markdown, IFileInfo? source) { source ??= DocumentationSet.Context.ConfigurationPath; var parsed = DocumentationSet.MarkdownParser.ParseApiDescriptionString(markdown, source); + RecordApiLinks(parsed, source); return MarkdownFile.CreateHtml(parsed, stripFirstHeadingLevel1: true, MarkdownParser.ApiDescriptionPipeline); } + // Descriptions parse without link validation, so the links they hold between API pages are recorded here for the API link check. + private void RecordApiLinks(MarkdownDocument parsed, IFileInfo source) + { + var context = DocumentationSet.Context; + foreach (var url in OutboundLinkExtractor.Extract(parsed) ?? []) + { + if (ApiLinkMatcher.TryCanonicalize(url, context.UrlPathPrefix, out var canonical)) + context.Collector.EmitApiLink(new ApiLinkReference(canonical, source.FullName, 1, 1, canonical.Length)); + } + } + /// public IReadOnlyList ReadApiDescriptionHeadings(string markdown, IFileInfo? source) { diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index b40d79b2de..b0ba0e74e6 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -13,6 +13,7 @@ using Elastic.Documentation.Configuration.ReleaseNotes; using Elastic.Documentation.Configuration.Toc; using Elastic.Documentation.Configuration.Toc.CliReference; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Documentation.Links.CrossLinks; using Elastic.Documentation.Navigation; @@ -181,6 +182,10 @@ void ValidateExists(string from, string to, IReadOnlyDictionary { if (!Uri.TryCreate(to, UriKind.Absolute, out _)) Context.EmitError(Configuration.SourceFile, $"Redirect {from} points to {to} which is not a valid URI"); + else if (ApiLinkMatcher.TryGetApiPath(to, out _)) + Context.Collector.EmitApiLink( + new ApiLinkReference(to, (Configuration.RedirectsSource ?? Configuration.SourceFile).FullName, 1, 1, to.Length) + ); return; } diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs index 63b6c39a98..7cc2a55f74 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveLinkValidator.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; using Markdig; @@ -90,7 +91,22 @@ internal static class DirectiveLinkValidator return url; if (IsExternal(trimmed)) + { + if (ApiLinkMatcher.TryGetApiPath(trimmed, out _)) + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference( + trimmed, + block.CurrentFile.FullName, + block.Line + 1, + System.Math.Max(block.Column, 1), + trimmed.Length + ) + ); return url; + } if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) && CrossLinkValidator.IsCrossLink(uri)) return ResolveCrossLink(url, uri, block, context); diff --git a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs index f9e4c1529c..1aa02d6090 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/AutoLinkInlineParser.cs @@ -4,6 +4,7 @@ using Elastic.Documentation.Diagnostics; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.Myst.Comments; using Markdig; using Markdig.Helpers; using Markdig.Parsers; @@ -24,8 +25,11 @@ public static MarkdownPipelineBuilder UseAutoLinks(this MarkdownPipelineBuilder public class AutoLinkBuilderExtension : IMarkdownExtension { - public void Setup(MarkdownPipelineBuilder pipeline) => - pipeline.InlineParsers.InsertBefore(new AutoLinkInlineParser()); + public void Setup(MarkdownPipelineBuilder pipeline) + { + _ = pipeline.InlineParsers.InsertBefore(new AutoLinkInlineParser()); + _ = pipeline.InlineParsers.InsertBefore(new AngleBracketApiLinkRecorder()); + } public void Setup(MarkdownPipeline pipeline, Markdig.Renderers.IMarkdownRenderer renderer) { @@ -86,6 +90,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) processor.Inline = linkInline; + if (processor.Block is not CommentBlock) + DiagnosticLinkInlineParser.RecordApiLink(context, url, line, column); + // Emit hint for elastic.co/docs URLs (after setting Inline so position is correct) if (url.Contains("elastic.co/docs", StringComparison.OrdinalIgnoreCase)) processor.EmitHint( @@ -226,3 +233,34 @@ private static bool IsAllDigits(ReadOnlySpan span) return true; } } + +/// +/// Records <https://...> links that point at API reference pages. It never consumes input, so Markdig's own +/// autolink parser still creates the link. +/// +public class AngleBracketApiLinkRecorder : InlineParser +{ + public AngleBracketApiLinkRecorder() => OpeningCharacters = ['<']; + + public override bool Match(InlineProcessor processor, ref StringSlice slice) + { + if (processor.Block is CommentBlock) + return false; + + var span = slice.AsSpan(); + if (!span.StartsWith("'); + if (end < 0) + return false; + + var url = span[1..end].ToString(); + if (url.AsSpan().ContainsAny(' ', '<', '\n')) + return false; + + _ = processor.GetSourcePosition(slice.Start, out var line, out var column); + DiagnosticLinkInlineParser.RecordApiLink(processor.GetContext(), url, line, column); + return false; + } +} diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index b9fbf1112b..e8b17289e5 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Elastic.Documentation.Links; using Elastic.Markdown.Diagnostics; @@ -153,11 +154,27 @@ private static void ValidateAndProcessLink(LinkInline link, InlineProcessor proc link.SetData("isCrossLink", false); if (ValidateExternalUri(link, processor, uri)) + { + RecordApiLink(context, url, link.Line, link.Column); return; + } ProcessInternalLink(link, processor, context); } + internal static void RecordApiLink(ParserContext context, string? url, int zeroBasedLine, int column) + { + if (context.SkipValidation || !ApiLinkMatcher.TryGetApiPath(url, out _)) + return; + + context + .Build + .Collector + .EmitApiLink( + new ApiLinkReference(url!, context.MarkdownSourcePath.FullName, zeroBasedLine + 1, Math.Max(column, 1), url!.Length) + ); + } + private static bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url) { if (string.IsNullOrEmpty(url)) diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs new file mode 100644 index 0000000000..1cf1f72b50 --- /dev/null +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLegacyRedirects.cs @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Assembler.Building; + +/// +/// The redirects from old bump.sh API URLs. They mirror modules/elastic-docs-v3/api-root-redirects.json in +/// elastic/docs-infra (#459), which the native viewer resolves exact match first. Keep this table in sync with that file. +/// +// TODO: Delete this class once the API Explorer migration (elastic/docs-eng-team#436) is finalised. +public static class ApiLegacyRedirects +{ + private static readonly IReadOnlyDictionary RedirectedRoots = new Dictionary + { + ["serverless"] = "kibana-serverless", + ["elastic-cloud-serverless"] = "cloud-serverless" + }; + + private static readonly IReadOnlySet GoneRoots = new HashSet { "observability-serverless" }; + + // Exact page redirects from the same file. Bump.sh served a Servers page for specs that declare no servers, + // Cloud Billing renamed its Budgets tag, Kibana removed Streams operations from its 9.6 spec, and some bump.sh topic + // pages are a children: page or the /authentication page in the native viewer. + private static readonly IReadOnlyDictionary RedirectedPages = new Dictionary + { + ["/docs/api/doc/elasticsearch/servers"] = "/docs/api/doc/elasticsearch", + ["/docs/api/doc/elasticsearch/v8/servers"] = "/docs/api/doc/elasticsearch/v8", + ["/docs/api/doc/elasticsearch/v9/servers"] = "/docs/api/doc/elasticsearch/v9", + ["/docs/api/doc/elasticsearch-serverless/servers"] = "/docs/api/doc/elasticsearch-serverless", + ["/docs/api/doc/cloud-enterprise/servers"] = "/docs/api/doc/cloud-enterprise", + ["/docs/api/doc/cloud-enterprise/v3/servers"] = "/docs/api/doc/cloud-enterprise/v3", + ["/docs/api/doc/cloud-billing/group/endpoint-budgets"] = "/docs/api/doc/cloud-billing/group/endpoint-billing-budgets", + ["/docs/api/doc/kibana/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/kibana-api-overview", + ["/docs/api/doc/kibana/v8/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v8/kibana-api-overview", + ["/docs/api/doc/kibana/v9/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v9/kibana-api-overview", + ["/docs/api/doc/cloud/topic/topic-authentication"] = "/docs/api/doc/cloud/authentication", + ["/docs/api/doc/cloud-billing/topic/topic-authentication"] = "/docs/api/doc/cloud-billing/authentication", + ["/docs/api/doc/kibana/v9/group/endpoint-significant_events"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-delete-streams-name-queries-queryid"] = + "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-queries"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-significant-events"] = + "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-post-streams-name-queries-bulk"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", + ["/docs/api/doc/kibana/v9/operation/operation-put-streams-name-queries-queryid"] = "/docs/api/doc/kibana/v9/group/endpoint-streams" + }; + + /// True for a root with no pages at all, so a link to it has nowhere to land. + public static bool IsGone(string path) => GoneRoots.Contains(RootOf(path)); + + /// True when the edge redirects this path to another one. + public static bool IsRedirected(string path) => RedirectedPages.ContainsKey(path) || RedirectedRoots.ContainsKey(RootOf(path)); + + /// The path the edge serves for : its redirect target, or the path itself. + public static string Resolve(string path) + { + if (RedirectedPages.TryGetValue(path, out var page)) + return page; + + var root = RootOf(path); + return RedirectedRoots.TryGetValue(root, out var newRoot) + ? string.Concat(ApiLinkMatcher.ApiDocPrefix, newRoot, path.AsSpan(ApiLinkMatcher.ApiDocPrefix.Length + root.Length)) + : path; + } + + private static string RootOf(string path) + { + if (!path.StartsWith(ApiLinkMatcher.ApiDocPrefix, StringComparison.Ordinal)) + return string.Empty; + var rest = path[ApiLinkMatcher.ApiDocPrefix.Length..]; + var slash = rest.IndexOf('/'); + if (slash >= 0) + return rest[..slash]; + + // A product root also has spec downloads next to it: kibana.json and kibana.yaml. + var extension = Path.GetExtension(rest); + return extension is ".json" or ".yaml" ? rest[..^extension.Length] : rest; + } +} diff --git a/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs new file mode 100644 index 0000000000..345eec2756 --- /dev/null +++ b/src/services/Elastic.Documentation.Assembler/Building/ApiLinkValidator.cs @@ -0,0 +1,85 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Assembler.Building; + +public sealed record ApiLinkReportEntry(string Url, string Source, int Line); + +public sealed record ApiLinkReport(int Checked, IReadOnlyList Broken); + +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(ApiLinkReport))] +internal sealed partial class ApiLinkReportJsonContext : JsonSerializerContext; + +/// Checks links recorded during the Markdown build against the API pages the assembler generated. +public static class ApiLinkValidator +{ + public const string ReportFileName = "api-link-report.json"; + + public static IReadOnlyList FindBroken(IEnumerable links, IReadOnlySet generatedUrls) => + [ + .. links + .Where(l => ApiLinkMatcher.TryGetApiPath(l.Url, out var path) && !generatedUrls.Contains(ApiLegacyRedirects.Resolve(path))) + .OrderBy(l => l.Url, StringComparer.Ordinal) + .ThenBy(l => l.File, StringComparer.Ordinal) + .ThenBy(l => l.Line) + ]; + + /// Emits a warning per broken link and writes the report; returns the report path, or null when nothing was checked. + public static async Task ValidateAsync( + IDiagnosticsCollector collector, + IReadOnlySet generatedUrls, + IFileSystem fileSystem, + string outputDirectory, + Cancel ctx + ) + { + // Without generated pages every API link would look broken, so skip instead of flooding the build. + if (generatedUrls.Count == 0) + return null; + + // A snippet is parsed once per page that includes it, and a card's targets are seen by more than one parser, + // so report each URL once per file, at its first position. + var links = collector + .ApiLinks + .OrderBy(l => l.File, StringComparer.Ordinal) + .ThenBy(l => l.Line) + .ThenBy(l => l.Column) + .DistinctBy(l => (l.Url, l.File)) + .ToArray(); + if (links.Length == 0) + return null; + + var broken = FindBroken(links, generatedUrls); + foreach (var link in broken) + { + collector.Write(new Diagnostic + { + Severity = Severity.Warning, + File = link.File, + Line = link.Line, + Column = link.Column, + Length = link.Length, + Message = $"API link '{link.Url}' does not resolve to a generated API page. Point it at an existing page " + + "(for older endpoints use a versioned tree such as /docs/api/doc//v8/...) or confirm the endpoint was removed intentionally. " + + "Links in API descriptions are fixed in the OpenAPI spec." + }); + } + + var report = new ApiLinkReport(links.Length, [.. broken.Select(b => new ApiLinkReportEntry(b.Url, b.File, b.Line))]); + var path = fileSystem.Path.Join(outputDirectory, ReportFileName); + var json = JsonSerializer.Serialize(report, ApiLinkReportJsonContext.Default.ApiLinkReport); + await fileSystem + .File + .WriteAllTextAsync(path, json, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), ctx) + .ConfigureAwait(false); + return path; + } +} diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs index e2b6dce932..6dbf6fa5d4 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs @@ -176,16 +176,27 @@ await fileSystem.File.ReadAllTextAsync(navigationFileInfo.FullName, ctx), if (exporters.Contains(Exporter.Html)) { var openApiStopwatch = Stopwatch.StartNew(); - catalogEntries = await AssemblerOpenApiBuildStep.BuildAsync( + var openApiResult = await AssemblerOpenApiBuildStep.BuildAsync( logFactory, assembleContext, assembleSources, navigation.TopNav, ctx ); + catalogEntries = openApiResult.CatalogEntries; openApiStopwatch.Stop(); _logger.LogInformation("OpenAPI build step completed in {DurationMs} ms", openApiStopwatch.ElapsedMilliseconds); + var apiLinkReportPath = await ApiLinkValidator.ValidateAsync( + collector, + openApiResult.GeneratedUrls, + assembleContext.WriteFileSystem, + assembleContext.OutputDirectory.FullName, + ctx + ); + if (apiLinkReportPath is not null) + await githubActionsService.SetOutputAsync("api-link-report-path", apiLinkReportPath); + // Build-time sitemap uses current date as placeholder for backwards compatibility. // Production sitemap with correct content_last_updated dates is generated via // `assembler sitemap` after ES indexing, which overwrites this file. diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs index dbf4c21dba..d8407faa97 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs @@ -24,7 +24,7 @@ namespace Elastic.Documentation.Assembler.Building; /// public static class AssemblerOpenApiBuildStep { - public static async Task> BuildAsync( + public static async Task BuildAsync( ILoggerFactory logFactory, AssembleContext assembleContext, AssembleSources assembleSources, @@ -41,14 +41,14 @@ Cancel ctx if (!features.AssemblerApiExplorerEnabled) { logger.LogInformation("Skipping OpenAPI generation: assembler-api-explorer feature flag is disabled"); - return []; + return AssemblerOpenApiResult.Empty; } var owners = DiscoverApiOwners(assembleSources.AssembleSets, assembleContext.Collector); if (owners.Count == 0) { logger.LogInformation("Skipping OpenAPI generation: no API declarations found in assembled docsets"); - return []; + return AssemblerOpenApiResult.Empty; } var stopwatch = Stopwatch.StartNew(); @@ -105,7 +105,8 @@ Cancel ctx assembleContext.OutputWithPathPrefixDirectory.FullName, stopwatch.ElapsedMilliseconds ); - return catalogEntries; + var generatedUrls = planned.SelectMany(p => p.Generator.GeneratedUrls).ToHashSet(StringComparer.Ordinal); + return new AssemblerOpenApiResult(catalogEntries, generatedUrls); } private static async Task WriteApiLlmsTxt(AssembleContext assembleContext, IReadOnlyList catalogEntries, Cancel ctx) @@ -165,3 +166,8 @@ private static void ApplyFeatureFlags(AssemblerDocumentationSet set, IReadOnlyDi } internal sealed record AssemblerApiOwner(AssemblerDocumentationSet Set); + +public sealed record AssemblerOpenApiResult(IReadOnlyList CatalogEntries, IReadOnlySet GeneratedUrls) +{ + public static AssemblerOpenApiResult Empty { get; } = new([], new HashSet()); +} diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs index f499ccb957..8add68dc34 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorCatalogSplitTests.cs @@ -55,6 +55,49 @@ public async Task GenerateProducts_DoesNotWriteCatalogPage() context.WriteFileSystem.File.Exists(Path.Join(outputRoot, "api", "index.html")).Should().BeFalse(); } + [Test] + public async Task GenerateProducts_RecordsEveryRenderedPageInGeneratedUrls() + { + var outputRoot = Path.Join(Paths.WorkingDirectoryRoot.FullName, $"api-catalog-split-{Guid.NewGuid():N}"); + var context = CreateGenerateContext(outputRoot); + using var versionIndexClient = new VersionIndexClient(BaseUri, MultiVersionHandler(), sleep: (_, _) => Task.CompletedTask); + var generator = new OpenApiGenerator( + NullLoggerFactory.Instance, + context, + NoopMarkdownStringRenderer.Instance, + versionIndexClient, + CreateSequentialReader(SpecDocument("Elasticsearch main")) + ); + + _ = await generator.GenerateProducts(ctx: TestContext.Current!.Execution.CancellationToken); + + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch"); + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch/group/endpoint-core"); + generator.GeneratedUrls.Should().Contain(["/docs/api/doc/elasticsearch.json", "/docs/api/doc/elasticsearch.yaml"]); + generator.GeneratedUrls.Should().OnlyContain(url => url == url.ToLowerInvariant() && !url.EndsWith('/')); + generator.GeneratedUrls.Should().Contain(url => url.Contains("/operation/operation-")); + } + + [Test] + public async Task GenerateProducts_PreviewPathPrefix_RecordsUrlsUnderTheProductionPath() + { + var outputRoot = Path.Join(Paths.WorkingDirectoryRoot.FullName, $"api-catalog-split-{Guid.NewGuid():N}"); + var context = CreateGenerateContext(outputRoot, urlPathPrefix: "elastic/docs-builder/docs/123"); + using var versionIndexClient = new VersionIndexClient(BaseUri, MultiVersionHandler(), sleep: (_, _) => Task.CompletedTask); + var generator = new OpenApiGenerator( + NullLoggerFactory.Instance, + context, + NoopMarkdownStringRenderer.Instance, + versionIndexClient, + CreateSequentialReader(SpecDocument("Elasticsearch main")) + ); + + _ = await generator.GenerateProducts(ctx: TestContext.Current!.Execution.CancellationToken); + + generator.GeneratedUrls.Should().Contain("/docs/api/doc/elasticsearch"); + generator.GeneratedUrls.Should().OnlyContain(url => url.StartsWith("/docs/api/doc/")); + } + [Test] public async Task ResolveCanonicalDocuments_ConfiguredApi_ReturnsSpecUnderItsExplorerKey() { @@ -605,7 +648,8 @@ private static BuildContext CreateGenerateContext( string outputRoot, string? docsetYaml = null, string? displayName = null, - BuildType buildType = BuildType.Isolated + BuildType buildType = BuildType.Isolated, + string urlPathPrefix = "docs" ) { var collector = new DiagnosticsCollector([]); @@ -657,7 +701,7 @@ private static BuildContext CreateGenerateContext( ), configurationContext ) - { UrlPathPrefix = "docs", BuildType = buildType }; + { UrlPathPrefix = urlPathPrefix, BuildType = buildType }; } private static OpenApiDocument SpecDocument(string title) => diff --git a/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs new file mode 100644 index 0000000000..bb9f4d7765 --- /dev/null +++ b/tests/Elastic.Documentation.Build.Tests/ApiLinkValidatorTests.cs @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Documentation.Assembler.Building; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Build.Tests; + +public class ApiLinkValidatorTests +{ + private static readonly IReadOnlySet Generated = new HashSet + { + "/docs/api/doc/kibana", + "/docs/api/doc/kibana/v8/operation/operation-upgrade", + "/docs/api/doc/kibana/operation/operation-alias", + "/docs/api/doc/kibana-serverless/operation/operation-live", + "/docs/api/doc/cloud-serverless.yaml", + "/docs/api/doc/kibana/kibana-api-overview" + }; + + private static ApiLinkReference Link(string url, string file = "a.md", int line = 1) => new(url, file, line, 1, url.Length); + + [Test] + public void FindBroken_ExistingPage_IsNotReported() => + ApiLinkValidator.FindBroken([Link("https://www.elastic.co/docs/api/doc/kibana/")], Generated).Should().BeEmpty(); + + [Test] + public void FindBroken_PageOnlyInOlderVersion_ReportsMainTreeLink() + { + var broken = ApiLinkValidator.FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/kibana/operation/operation-upgrade"), + Link("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade#responses") + ], + Generated + ); + broken.Should().ContainSingle().Which.Url.Should().EndWith("/kibana/operation/operation-upgrade"); + } + + [Test] + public void FindBroken_OldBumpShRootWithLivePage_IsNotReported() => + ApiLinkValidator + .FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/serverless/operation/operation-live"), + Link("https://www.elastic.co/docs/api/doc/kibana/topic/topic-kibana-spaces") + ], + Generated + ) + .Should() + .BeEmpty(); + + [Test] + public void FindBroken_OldBumpShRootSpecDownload_ResolvesToTheNewRootDownload() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/elastic-cloud-serverless.yaml")], Generated) + .Should() + .BeEmpty(); + + [Test] + public void FindBroken_OldBumpShRootWithMissingPage_IsStillReported() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/serverless/operation/operation-removed")], Generated) + .Should() + .ContainSingle(); + + [Test] + public void FindBroken_GoneBumpShRoot_IsReported() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/observability-serverless/operation/operation-x")], Generated) + .Should() + .ContainSingle(); + + [Test] + public void FindBroken_AliasUrl_Resolves() => + ApiLinkValidator + .FindBroken([Link("https://www.elastic.co/docs/api/doc/kibana/operation/operation-alias")], Generated) + .Should() + .BeEmpty(); + + [Test] + public void FindBroken_MultipleBroken_AreOrderedByUrlThenSource() + { + var broken = ApiLinkValidator.FindBroken( + [ + Link("https://www.elastic.co/docs/api/doc/kibana/zzz", "b.md"), + Link("https://www.elastic.co/docs/api/doc/kibana/aaa", "b.md"), + Link("https://www.elastic.co/docs/api/doc/kibana/aaa", "a.md") + ], + Generated + ); + broken.Select(b => (b.Url.Split('/')[^1], b.File)).Should().Equal(("aaa", "a.md"), ("aaa", "b.md"), ("zzz", "b.md")); + } + + private static (DiagnosticsCollector Collector, MockFileSystem FileSystem) Setup(params ApiLinkReference[] links) + { + var collector = new DiagnosticsCollector([]); + foreach (var link in links) + collector.EmitApiLink(link); + var fileSystem = new MockFileSystem(); + fileSystem.AddDirectory("/out"); + return (collector, fileSystem); + } + + [Test] + public async Task ValidateAsync_BrokenLink_WarnsAtItsPositionAndWritesTheReport() + { + var (collector, fileSystem) = Setup( + new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/", "a.md", 4, 7, 40), + new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/operation/missing#x", "b.md", 9, 3, 55) + ); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + collector.Warnings.Should().Be(1); + path.Should().EndWith("api-link-report.json"); + var bytes = await fileSystem.File.ReadAllBytesAsync(path!); + bytes.Take(3).Should().NotEqual([(byte)0xEF, (byte)0xBB, (byte)0xBF], "the report is plain UTF-8 without a BOM"); + using var report = JsonDocument.Parse(bytes); + report.RootElement.GetProperty("checked").GetInt32().Should().Be(2); + var broken = report.RootElement.GetProperty("broken").EnumerateArray().Single(); + broken.GetProperty("source").GetString().Should().Be("b.md"); + broken.GetProperty("line").GetInt32().Should().Be(9); + } + + [Test] + public async Task ValidateAsync_SameLinkRecordedByEveryIncludingPage_ReportsOnce() + { + var link = new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/gone", "_snippets/s.md", 2, 1, 30); + var (collector, fileSystem) = Setup(link, link, link); + + _ = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + collector.Warnings.Should().Be(1); + using var report = JsonDocument.Parse(await fileSystem.File.ReadAllTextAsync("/out/api-link-report.json")); + report.RootElement.GetProperty("checked").GetInt32().Should().Be(1); + } + + [Test] + public async Task ValidateAsync_NoGeneratedPages_SkipsWithoutWarningsOrReport() + { + var (collector, fileSystem) = Setup(new ApiLinkReference("https://www.elastic.co/docs/api/doc/kibana/gone", "a.md", 1, 1, 30)); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + new HashSet(), + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + path.Should().BeNull(); + collector.Warnings.Should().Be(0); + fileSystem.File.Exists("/out/api-link-report.json").Should().BeFalse(); + } + + [Test] + public async Task ValidateAsync_NoRecordedLinks_ReturnsNull() + { + var (collector, fileSystem) = Setup(); + + var path = await ApiLinkValidator.ValidateAsync( + collector, + Generated, + fileSystem, + "/out", + TestContext.Current!.Execution.CancellationToken + ); + + path.Should().BeNull(); + } +} diff --git a/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs b/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs index 2c54b35e4c..27686a5fb5 100644 --- a/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/ApiSitemapParityTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Xml.Linq; +using Elastic.Documentation.Assembler.Building; using Elastic.Documentation.Configuration; namespace Elastic.Documentation.Build.Tests; @@ -10,41 +11,6 @@ namespace Elastic.Documentation.Build.Tests; // TODO: Delete this class once the API Explorer migration (elastic/docs-eng-team#436) is finalised. public class ApiSitemapParityTests { - // Mirrors modules/elastic-docs-v3/api-root-redirects.json in elastic/docs-infra (#459), which the native viewer resolves exact match first. - private const string ApiDocPrefix = "/docs/api/doc/"; - private static readonly IReadOnlyDictionary RedirectedRoots = new Dictionary - { - ["serverless"] = "kibana-serverless", - ["elastic-cloud-serverless"] = "cloud-serverless" - }; - private static readonly IReadOnlySet GoneRoots = new HashSet { "observability-serverless" }; - - // Exact page redirects from the same file. Bump.sh served a Servers page for specs that declare no servers, - // Cloud Billing renamed its Budgets tag, Kibana removed Streams operations from its 9.6 spec, and some bump.sh topic - // pages are a children: page or the /authentication page in the native viewer. - private static readonly IReadOnlyDictionary RedirectedPages = new Dictionary - { - ["/docs/api/doc/elasticsearch/servers"] = "/docs/api/doc/elasticsearch", - ["/docs/api/doc/elasticsearch/v8/servers"] = "/docs/api/doc/elasticsearch/v8", - ["/docs/api/doc/elasticsearch/v9/servers"] = "/docs/api/doc/elasticsearch/v9", - ["/docs/api/doc/elasticsearch-serverless/servers"] = "/docs/api/doc/elasticsearch-serverless", - ["/docs/api/doc/cloud-enterprise/servers"] = "/docs/api/doc/cloud-enterprise", - ["/docs/api/doc/cloud-enterprise/v3/servers"] = "/docs/api/doc/cloud-enterprise/v3", - ["/docs/api/doc/cloud-billing/group/endpoint-budgets"] = "/docs/api/doc/cloud-billing/group/endpoint-billing-budgets", - ["/docs/api/doc/kibana/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/kibana-api-overview", - ["/docs/api/doc/kibana/v8/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v8/kibana-api-overview", - ["/docs/api/doc/kibana/v9/topic/topic-kibana-spaces"] = "/docs/api/doc/kibana/v9/kibana-api-overview", - ["/docs/api/doc/cloud/topic/topic-authentication"] = "/docs/api/doc/cloud/authentication", - ["/docs/api/doc/cloud-billing/topic/topic-authentication"] = "/docs/api/doc/cloud-billing/authentication", - ["/docs/api/doc/kibana/v9/group/endpoint-significant_events"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-delete-streams-name-queries-queryid"] = - "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-queries"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-get-streams-name-significant-events"] = - "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-post-streams-name-queries-bulk"] = "/docs/api/doc/kibana/v9/group/endpoint-streams", - ["/docs/api/doc/kibana/v9/operation/operation-put-streams-name-queries-queryid"] = "/docs/api/doc/kibana/v9/group/endpoint-streams" - }; private static readonly Uri SitemapIndex = new("https://www.elastic.co/docs/api/sitemap_index.xml"); private static readonly XNamespace SitemapNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9"; @@ -61,12 +27,10 @@ public async Task Build_ContainsEveryProductionApiSitemapPage(CancellationToken var productionUrls = await DownloadPageUrls(client, SitemapIndex, cancellationToken); await Assert.That(productionUrls).IsNotEmpty(); - var gone = productionUrls.Where(url => GoneRoots.Contains(RootOf(url))).ToArray(); - var redirected = productionUrls.Where( - url => RedirectedPages.ContainsKey(url.AbsolutePath) || RedirectedRoots.ContainsKey(RootOf(url)) - ).ToArray(); + var gone = productionUrls.Where(url => ApiLegacyRedirects.IsGone(url.AbsolutePath)).ToArray(); + var redirected = productionUrls.Where(url => ApiLegacyRedirects.IsRedirected(url.AbsolutePath)).ToArray(); // Redirected pages are served by an exact or prefix redirect, so verify the page exists at its redirect target. - var auditedPaths = productionUrls.Except(gone).Select(RedirectTarget).Distinct().ToArray(); + var auditedPaths = productionUrls.Except(gone).Select(url => ApiLegacyRedirects.Resolve(url.AbsolutePath)).Distinct().ToArray(); var outputRoot = Environment.GetEnvironmentVariable("API_SITEMAP_PARITY_OUTPUT") ?? Path.Join(Paths.WorkingDirectoryRoot.FullName, ".artifacts", "assembly"); var missing = auditedPaths @@ -87,27 +51,6 @@ public async Task Build_ContainsEveryProductionApiSitemapPage(CancellationToken File.Delete(reportPath); } - private static string RootOf(Uri url) - { - var path = url.AbsolutePath; - if (!path.StartsWith(ApiDocPrefix, StringComparison.Ordinal)) - return string.Empty; - var rest = path[ApiDocPrefix.Length..]; - var slash = rest.IndexOf('/'); - return slash < 0 ? rest : rest[..slash]; - } - - private static string RedirectTarget(Uri url) - { - if (RedirectedPages.TryGetValue(url.AbsolutePath, out var page)) - return page; - - var root = RootOf(url); - return RedirectedRoots.TryGetValue(root, out var newRoot) - ? string.Concat(ApiDocPrefix, newRoot, url.AbsolutePath.AsSpan(ApiDocPrefix.Length + root.Length)) - : url.AbsolutePath; - } - private static async Task> DownloadPageUrls(HttpClient client, Uri sitemap, CancellationToken ctx) { await using var stream = await client.GetStreamAsync(sitemap, ctx); diff --git a/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs new file mode 100644 index 0000000000..be6f4f8b3c --- /dev/null +++ b/tests/Elastic.Documentation.Tests/Diagnostics/ApiLinkMatcherTests.cs @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Documentation.Diagnostics; + +namespace Elastic.Documentation.Tests.Diagnostics; + +public class ApiLinkMatcherTests +{ + [Test] + [Arguments("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-x", "/docs/api/doc/kibana/v8/operation/operation-x")] + [Arguments("https://elastic.co/docs/api/doc/kibana/", "/docs/api/doc/kibana")] + [Arguments("http://www.elastic.co/docs/api/doc/Kibana/Group/Endpoint-Foo?x=1#frag", "/docs/api/doc/kibana/group/endpoint-foo")] + [Arguments("https://www.elastic.co/docs/api/doc/elasticsearch//operation/operation-bulk", "/docs/api/doc/elasticsearch/operation/operation-bulk")] + public void TryGetApiPath_ApiUrl_ReturnsNormalizedPath(string url, string expected) + { + ApiLinkMatcher.TryGetApiPath(url, out var path).Should().BeTrue(); + path.Should().Be(expected); + } + + [Test] + [Arguments("https://www.elastic.co/docs/solutions/search")] + [Arguments("https://example.com/docs/api/doc/kibana")] + [Arguments("https://www.elastic.co/docs/api/")] + [Arguments("/docs/api/doc/kibana")] + [Arguments("mailto:a@elastic.co")] + [Arguments("")] + [Arguments(null)] + public void TryGetApiPath_NonApiUrl_ReturnsFalse(string? url) => ApiLinkMatcher.TryGetApiPath(url, out _).Should().BeFalse(); + + [Test] + [Arguments("/docs/api/doc/kibana/operation/operation-x", "/docs", "https://www.elastic.co/docs/api/doc/kibana/operation/operation-x")] + [Arguments("/elastic/docs-builder/docs/123/api/doc/kibana/group/endpoint-a", "/elastic/docs-builder/docs/123", "https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a")] + [Arguments("https://www.elastic.co/docs/api/doc/kibana", "/docs", "https://www.elastic.co/docs/api/doc/kibana")] + public void TryCanonicalize_ApiPageLink_ReturnsAbsoluteProductionUrl(string url, string prefix, string expected) + { + ApiLinkMatcher.TryCanonicalize(url, prefix, out var canonical).Should().BeTrue(); + canonical.Should().Be(expected); + } + + [Test] + [Arguments("/docs/solutions/search", "/docs")] + [Arguments("../operation/x", "/docs")] + [Arguments("https://example.com/docs/api/doc/kibana", "/docs")] + [Arguments(null, "/docs")] + public void TryCanonicalize_OtherLink_ReturnsFalse(string? url, string prefix) => + ApiLinkMatcher.TryCanonicalize(url, prefix, out _).Should().BeFalse(); +} diff --git a/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs new file mode 100644 index 0000000000..ebde2f0ab6 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Directives/DirectiveApiLinkTests.cs @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Markdown.Myst.Directives.Hub; + +namespace Elastic.Markdown.Tests.Directives; + +[InheritsTests] +public class HeroApiLinkTests() : DirectiveTest( + """ + :::{hero} + :title: Elasticsearch + :description: Start with the [search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search). + :primary-action: [Bulk](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) + :secondary-action: [Install](https://www.elastic.co/downloads/elasticsearch) + ::: + """ +) +{ + [Test] + public void RecordsApiLinksFromActionsAndDescriptionOnly() => + Collector + .ApiLinks + .Select(l => l.Url) + .Distinct() + .Should() + .BeEquivalentTo([ + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search", + "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk" + ]); +} + +[InheritsTests] +public class LinkCardApiLinkTests() : DirectiveTest( + """ + :::{link-card} + title: Search + link: https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search + links: + - label: Elsewhere + url: https://www.elastic.co/docs/solutions/search + ::: + """ +) +{ + [Test] + public void RecordsOnlyTheApiTarget() => + Collector + .ApiLinks + .Select(l => l.Url) + .Distinct() + .Should() + .Equal("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search"); +} diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs new file mode 100644 index 0000000000..f13aabe62b --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/ApiDescriptionLinkTests.cs @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; + +namespace Elastic.Markdown.Tests.Inline; + +[InheritsTests] +public class ApiDescriptionLinkTests() : InlineTest(""" +Placeholder. +""") +{ + [Test] + public void RenderApiDescription_LinksBetweenApiPages_AreRecordedWithTheirCanonicalUrl() + { + var writer = new HtmlWriter(Set, FileSystem, new DescriptionGenerator()); + var source = FileSystem.FileInfo.New("docs/api/doc/kibana/operation/operation-x/description.md"); + + _ = writer.RenderApiDescription( + """ + See [the rule](/docs/api/doc/kibana/operation/operation-find-rules), [groups](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a) + and [the guide](https://www.elastic.co/docs/solutions/search) or [elsewhere](https://example.com). + """, + source + ); + + Collector + .ApiLinks + .Select(l => l.Url) + .Should() + .BeEquivalentTo([ + "https://www.elastic.co/docs/api/doc/kibana/operation/operation-find-rules", + "https://www.elastic.co/docs/api/doc/kibana/group/endpoint-a" + ]); + Collector.ApiLinks.Should().OnlyContain(l => l.File.EndsWith("description.md")); + } +} diff --git a/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs new file mode 100644 index 0000000000..96ad73ee12 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/ApiLinkTests.cs @@ -0,0 +1,91 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using JetBrains.Annotations; +using Markdig.Syntax.Inlines; + +namespace Elastic.Markdown.Tests.Inline; + +[InheritsTests] +public class ApiLinkRecordedTests() : LinkTestBase( + """ +[Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params) +""" +) +{ + [Test] + public void RecordsApiLinkWithLocation() + { + var link = Collector.ApiLinks.Should().ContainSingle().Subject; + link.Url.Should().Be("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search#query-params"); + link.Line.Should().BePositive(); + link.File.Should().EndWith(".md"); + } + + [Test] + public void HasNoErrors() => Collector.Errors.Should().Be(0); +} + +[InheritsTests] +public class NonApiElasticLinkTests() : LinkTestBase(""" +[Docs](https://www.elastic.co/docs/solutions/search) +""") +{ + [Test] + public void RecordsNoApiLink() => Collector.ApiLinks.Should().BeEmpty(); +} + +[InheritsTests] +public class ApiAutoLinkTests() : InlineTest( + """ +See https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing for details. +""" +) +{ + [Test] + public void RecordsBareUrlAsApiLink() => + Collector + .ApiLinks + .Should() + .ContainSingle() + .Which + .Url + .Should() + .Be("https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing"); +} + +[InheritsTests] +public class ApiAngleBracketAutoLinkTests() : InlineTest( + """ +See for details. +""" +) +{ + [Test] + public void RecordsAngleBracketAutolinkAsApiLink() => + Collector + .ApiLinks + .Should() + .ContainSingle() + .Which + .Url + .Should() + .Be("https://www.elastic.co/docs/api/doc/kibana/operation/operation-missing"); +} + +[InheritsTests] +public class ApiLinksInCommentsTests() : InlineTest( + """ +% https://www.elastic.co/docs/api/doc/kibana/operation/old-bare +% +% [old](https://www.elastic.co/docs/api/doc/kibana/operation/old-inline) + +Text. +""" +) +{ + [Test] + public void RecordsNothingFromComments() => Collector.ApiLinks.Should().BeEmpty(); +} diff --git a/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs b/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs new file mode 100644 index 0000000000..67b24a0366 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/RedirectApiLinkTests.cs @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using AwesomeAssertions; +using Elastic.Documentation.Configuration; +using Elastic.Markdown.IO; + +namespace Elastic.Markdown.Tests; + +public class RedirectApiLinkTests +{ + [Test] + public void RedirectToApiPage_RecordsApiLinkAtTheRedirectsFile() + { + var collector = BuildDocSet( + """ + redirects: + 'old-api.md': 'https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade' + 'old-other.md': 'https://www.elastic.co/docs/solutions/search' + """ + ); + + var link = collector.ApiLinks.Should().ContainSingle().Subject; + link.Url.Should().Be("https://www.elastic.co/docs/api/doc/kibana/v8/operation/operation-upgrade"); + link.File.Should().EndWith("redirects.yml"); + collector.Errors.Should().Be(0); + } + + private static TestDiagnosticsCollector BuildDocSet(string redirects) + { + var fileSystem = new MockFileSystem( + new Dictionary + { + { "docs/docset.yml", new MockFileData("project: test\ntoc:\n- file: index.md\n") }, + { "docs/index.md", new MockFileData("# Index") }, + { "docs/redirects.yml", new MockFileData(redirects) } + }, + new MockFileSystemOptions { CurrentDirectory = Paths.WorkingDirectoryRoot.FullName } + ); + var collector = new TestDiagnosticsCollector(); + _ = collector.StartAsync(TestContext.Current!.Execution.CancellationToken); + var configurationContext = TestHelpers.CreateConfigurationContext(fileSystem); + var context = new BuildContext(collector, TestHelpers.CreateDocumentationFileSystem(fileSystem), configurationContext); + _ = new DocumentationSet(context, new TestLoggerFactory(), new TestCrossLinkResolver()); + return collector; + } +}