robots.txt, a sitemap, and a description for search engines

robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://am.angouri.org/sitemap.xml

src/NaiveStaticGenerator/Program.cs

@@ -30,6 +30,8 @@
 CopyCssFilesToFinalWebsite();
 CopyImgFolderToFinalWebsite();
 CopyCName();
+CopyRobots();
+GenerateSitemap();
 
 static string GetNearestRoot(string name, string current)
     => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current));
@@ -204,7 +206,7 @@
     }
 
 
     static string PathUp(string content, string relativeName, string subPath)
         => content.Replace("/" + subPath,
             string.Join("",
                 Enumerable.Range(0, relativeName.Count(c => c == '\\')
@@ -251,7 +253,7 @@
     }
 }
 
 void CopyWikiImagesToFinalWebsite()
 {
     DirectoryCopy(RootP._("wiki"), FinalOutputP._("wiki"), copySubDirs: false, _ => true);    
 }
@@ -271,6 +273,33 @@
     File.Copy(RootP._("CNAME"), FinalOutputP._("CNAME"), true);
 }
 
+void CopyRobots()
+{
+    File.Copy(RootP._("robots.txt"), FinalOutputP._("robots.txt"), true);
+}
+
+// Every page of the website, for search engines, which robots.txt points to. A page that is a
+// folder's index.html is served at the folder, so it is listed as the folder.
+void GenerateSitemap()
+{
+    var host = File.ReadAllText(RootP._("CNAME")).Trim();
+    var sitemap = new StringBuilder();
+    sitemap.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+    sitemap.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
+    foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal))
+    {
+        var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/');
+        if (relative == "index.html")
+            relative = "";
+        else if (relative.EndsWith("/index.html", StringComparison.Ordinal))
+            relative = relative[..^"index.html".Length];
+        var path = string.Join("/", relative.Split('/').Select(Uri.EscapeDataString));
+        sitemap.AppendLine($"  <url><loc>{System.Security.SecurityElement.Escape($"https://{host}/{path}")}</loc></url>");
+    }
+    sitemap.AppendLine("</urlset>");
+    File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString());
+}
+
 
 public sealed class PageSaverAndCounter : IPageSave
 {

src/content/_templates/top.html

@@ -1,6 +1,8 @@
 <!doctype html>
 <html lang="en">
 <head>
+    <meta charset="utf-8">
+    <meta name="description" content="AngouriMath, an open-source symbolic algebra library for .NET: solve equations and systems, simplify, differentiate, integrate, take limits, and compile expressions.">
     <title>##page_title##</title>    
     <link rel="shortcut icon" href="/img/favicon.ico" type="image/x-icon">
     <link rel="stylesheet" href="{{RELATIVE_PATH}}/styles.css">