diff --git a/AGENTS.md b/AGENTS.md index 3587a94a9..a1b443622 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,8 +12,8 @@ sets the working practice for both. When there is nothing to do here, it goes ba `build` runs `src/NaiveStaticGenerator`, which wraps every page in `src/content` in `src/content/_templates/top.html` and `bottom.html` and writes `.output/final`. `run` builds and opens the home page. -- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt` and a `sitemap.xml` that - the generator writes from the pages it produced. +- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt`, and a `sitemap.xml` and an + `llms.txt` that the generator writes from the pages it produced. - A pull request builds the site on Windows, Linux and macOS (`build-test.yml`). - **A merge to master is live.** `deployment.yml` builds the site and pushes `.output/final` to the `gh-pages` branch, and GitHub Pages publishes it a few minutes later. Check the live page after a diff --git a/src/NaiveStaticGenerator/Program.cs b/src/NaiveStaticGenerator/Program.cs index 46f0d2111..33704e76f 100644 --- a/src/NaiveStaticGenerator/Program.cs +++ b/src/NaiveStaticGenerator/Program.cs @@ -32,6 +32,7 @@ CopyCName(); CopyRobots(); GenerateSitemap(); +GenerateLlmsTxt(); static string GetNearestRoot(string name, string current) => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current)); @@ -300,6 +301,63 @@ void GenerateSitemap() File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString()); } +// The website for language models, as llmstxt.org proposes it: the library's name and description, +// a link to each page, and the documentation of the types and members, thousands of pages, as one +// link to its index. The description is the one the pages give search engines, so it is said once. +void GenerateLlmsTxt() +{ + var host = File.ReadAllText(RootP._("CNAME")).Trim(); + var top = File.ReadAllText(ContentP._("_templates")._("top.html")); + var description = System.Text.RegularExpressions.Regex.Match(top, " + { + [""] = "Home", ["quickstart/"] = "Quickstart", ["whatsnew/"] = "What's new", + ["why/"] = "Why AngouriMath", ["research/"] = "Research", ["demo/"] = "Demo", ["wiki/"] = "Wiki", + }; + var pages = new List<(string Path, string Url)>(); + var wiki = new List(); + foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal)) + { + var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/'); + if (relative.StartsWith("docs/", StringComparison.Ordinal)) + continue; + if (relative == "index.html") + relative = ""; + else if (relative.EndsWith("/index.html", StringComparison.Ordinal)) + relative = relative[..^"index.html".Length]; + var url = $"https://{host}/{string.Join("/", relative.Split('/').Select(Uri.EscapeDataString))}"; + // A page of the wiki by its name; `_Footer` and `_Sidebar` are parts of the wiki's pages. + if (relative.StartsWith("wiki/", StringComparison.Ordinal) && relative != "wiki/") + { + var name = Path.GetFileNameWithoutExtension(relative); + if (!name.StartsWith("_", StringComparison.Ordinal)) + wiki.Add($"- [{name.Replace('-', ' ')}]({url})"); + } + else + pages.Add((relative, url)); + } + var llms = new StringBuilder(); + llms.AppendLine("# AngouriMath"); + llms.AppendLine(); + llms.AppendLine($"> {System.Net.WebUtility.HtmlDecode(description)}"); + llms.AppendLine(); + llms.AppendLine("## Pages"); + llms.AppendLine(); + // The pages in the order the map names them, any other after them. + var order = named.Keys.ToList(); + foreach (var (path, url) in pages.OrderBy(page => order.IndexOf(page.Path) is var at && at >= 0 ? at : order.Count)) + llms.AppendLine($"- [{(named.TryGetValue(path, out var name) ? name : path.TrimEnd('/'))}]({url})"); + llms.AppendLine(); + llms.AppendLine("## Wiki"); + llms.AppendLine(); + wiki.ForEach(line => llms.AppendLine(line)); + llms.AppendLine(); + llms.AppendLine("## Optional"); + llms.AppendLine(); + llms.AppendLine($"- [Almanac](https://{host}/docs/namespaces.html): every namespace, type and member, from the library's XML documentation"); + File.WriteAllText(FinalOutputP._("llms.txt"), llms.ToString()); +} + public sealed class PageSaverAndCounter : IPageSave {