From 31663d6d5c810df8f586fccbbed3a090da185e99 Mon Sep 17 00:00:00 2001
From: Rafael Vuijk <r.vuijk@sownet.nl>
Date: Fri, 2 Oct 2026 19:39:10 +0000
Subject: [PATCH] robots.txt, a sitemap, and a description for search engines

robots.txt allows everything and names sitemap.xml, which the generator
writes after the pages: every page of the website, a folder's index.html
listed as the folder. Each page also says its character set and carries
a description of the library.

Part of #21.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012sonx8iAspMiwRwokT1Ura
---
 robots.txt                          |  4 ++++
 src/NaiveStaticGenerator/Program.cs | 29 +++++++++++++++++++++++++++++
 src/content/_templates/top.html     |  2 ++
 3 files changed, 35 insertions(+)
 create mode 100644 robots.txt

diff --git a/robots.txt b/robots.txt
new file mode 100644
index 000000000..b049379c1
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://am.angouri.org/sitemap.xml
diff --git a/src/NaiveStaticGenerator/Program.cs b/src/NaiveStaticGenerator/Program.cs
index 89343c62b..46f0d2111 100644
--- a/src/NaiveStaticGenerator/Program.cs
+++ b/src/NaiveStaticGenerator/Program.cs
@@ -30,6 +30,8 @@
 CopyCssFilesToFinalWebsite();
 CopyImgFolderToFinalWebsite();
 CopyCName();
+CopyRobots();
+GenerateSitemap();
 
 static string GetNearestRoot(string name, string current)
     => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current));
@@ -271,6 +273,33 @@ void CopyCName()
     File.Copy(RootP._("CNAME"), FinalOutputP._("CNAME"), true);
 }
 
+void CopyRobots()
+{
+    File.Copy(RootP._("robots.txt"), FinalOutputP._("robots.txt"), true);
+}
+
+// Every page of the website, for search engines, which robots.txt points to. A page that is a
+// folder's index.html is served at the folder, so it is listed as the folder.
+void GenerateSitemap()
+{
+    var host = File.ReadAllText(RootP._("CNAME")).Trim();
+    var sitemap = new StringBuilder();
+    sitemap.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+    sitemap.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
+    foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal))
+    {
+        var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/');
+        if (relative == "index.html")
+            relative = "";
+        else if (relative.EndsWith("/index.html", StringComparison.Ordinal))
+            relative = relative[..^"index.html".Length];
+        var path = string.Join("/", relative.Split('/').Select(Uri.EscapeDataString));
+        sitemap.AppendLine($"  <url><loc>{System.Security.SecurityElement.Escape($"https://{host}/{path}")}</loc></url>");
+    }
+    sitemap.AppendLine("</urlset>");
+    File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString());
+}
+
 
 public sealed class PageSaverAndCounter : IPageSave
 {
diff --git a/src/content/_templates/top.html b/src/content/_templates/top.html
index 9281cb094..3aba0f890 100644
--- a/src/content/_templates/top.html
+++ b/src/content/_templates/top.html
@@ -1,6 +1,8 @@
 <!doctype html>
 <html lang="en">
 <head>
+    <meta charset="utf-8">
+    <meta name="description" content="AngouriMath, an open-source symbolic algebra library for .NET: solve equations and systems, simplify, differentiate, integrate, take limits, and compile expressions.">
     <title>##page_title##</title>    
     <link rel="shortcut icon" href="/img/favicon.ico" type="image/x-icon">
     <link rel="stylesheet" href="{{RELATIVE_PATH}}/styles.css">