diff --git a/robots.txt b/robots.txt new file mode 100644 index 000000000..b049379c1 --- /dev/null +++ b/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://am.angouri.org/sitemap.xml diff --git a/src/NaiveStaticGenerator/Program.cs b/src/NaiveStaticGenerator/Program.cs index 89343c62b..46f0d2111 100644 --- a/src/NaiveStaticGenerator/Program.cs +++ b/src/NaiveStaticGenerator/Program.cs @@ -30,6 +30,8 @@ CopyCssFilesToFinalWebsite(); CopyImgFolderToFinalWebsite(); CopyCName(); +CopyRobots(); +GenerateSitemap(); static string GetNearestRoot(string name, string current) => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current)); @@ -271,6 +273,33 @@ void CopyCName() File.Copy(RootP._("CNAME"), FinalOutputP._("CNAME"), true); } +void CopyRobots() +{ + File.Copy(RootP._("robots.txt"), FinalOutputP._("robots.txt"), true); +} + +// Every page of the website, for search engines, which robots.txt points to. A page that is a +// folder's index.html is served at the folder, so it is listed as the folder. +void GenerateSitemap() +{ + var host = File.ReadAllText(RootP._("CNAME")).Trim(); + var sitemap = new StringBuilder(); + sitemap.AppendLine(""); + sitemap.AppendLine(""); + foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal)) + { + var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/'); + if (relative == "index.html") + relative = ""; + else if (relative.EndsWith("/index.html", StringComparison.Ordinal)) + relative = relative[..^"index.html".Length]; + var path = string.Join("/", relative.Split('/').Select(Uri.EscapeDataString)); + sitemap.AppendLine($" {System.Security.SecurityElement.Escape($"https://{host}/{path}")}"); + } + sitemap.AppendLine(""); + File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString()); +} + public sealed class PageSaverAndCounter : IPageSave { diff --git a/src/content/_templates/top.html b/src/content/_templates/top.html index 9281cb094..3aba0f890 100644 --- a/src/content/_templates/top.html +++ b/src/content/_templates/top.html @@ -1,6 +1,8 @@ + + ##page_title##