".repeat(130), url)).toThrow("128 levels");
+ expect(() => parseSitemap("
" + "".repeat(65), 100)).toThrow(
+ "64-level",
+ );
+});
diff --git a/site-crawl-plugin/src/seo.ts b/site-crawl-plugin/src/seo.ts
new file mode 100644
index 0000000..04af9b0
--- /dev/null
+++ b/site-crawl-plugin/src/seo.ts
@@ -0,0 +1,245 @@
+// SPDX-FileCopyrightText: 2024-2026 Temps Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+import { Parser } from "htmlparser2";
+import { normalize } from "./http";
+import { CrawlError, type Issue } from "./types";
+export function analyzeHtml(html: string, url: string, xRobots = "") {
+ const fields = extractHtml(html);
+ const issues: Issue[] = [];
+ const add = (
+ code: string,
+ severity: Issue["severity"],
+ message: string,
+ fix: string,
+ ) => issues.push({ code, severity, message, fix });
+ const title = fields.title.trim();
+ const description = fields.description.trim();
+ if (!title)
+ add(
+ "missing_title",
+ "warning",
+ "No page title.",
+ "Add a descriptive, unique in the HTML head.",
+ );
+ if (!description)
+ add(
+ "missing_description",
+ "warning",
+ "No meta description.",
+ "Describe this page in a meta description. Search engines may choose a different snippet.",
+ );
+ if (!fields.hasH1)
+ add(
+ "missing_h1",
+ "info",
+ "No primary heading.",
+ "Add a clear heading that describes this page.",
+ );
+ if (!fields.language.trim())
+ add(
+ "missing_language",
+ "info",
+ "Document language is not declared.",
+ "Set the appropriate lang attribute on <html>.",
+ );
+ const robots = [
+ fields.noindex ? "noindex" : "",
+ fields.nofollow ? "nofollow" : "",
+ xRobots,
+ ]
+ .join(",")
+ .toLowerCase();
+ if (/\b(noindex|none)\b/.test(robots))
+ add(
+ "noindex",
+ "warning",
+ "Indexing is disabled by a robots directive.",
+ "If this page should appear in search, remove its noindex directive. Keep it for intentionally private or excluded pages.",
+ );
+
+ let canonical: string | null = null;
+ if (fields.canonicalCount > 1)
+ add(
+ "multiple_canonicals",
+ "warning",
+ "Multiple canonical URLs are declared.",
+ "Keep one consistent canonical URL for this page.",
+ );
+ const raw = fields.canonical;
+ if (raw) {
+ try {
+ canonical = normalize(raw, url).href;
+ if (canonical !== url)
+ add(
+ "alternate_canonical",
+ "info",
+ "This page points to a different canonical URL.",
+ "Verify the target is the intended preferred page and is reachable.",
+ );
+ } catch {
+ add(
+ "invalid_canonical",
+ "warning",
+ "Canonical URL is invalid or unsupported.",
+ "Use a valid public HTTP or HTTPS canonical URL.",
+ );
+ }
+ } else
+ add(
+ "missing_canonical",
+ "info",
+ "No canonical URL is declared.",
+ "Consider a canonical URL where duplicate URLs could exist. This is not automatically an indexing error.",
+ );
+ let base = url;
+ try {
+ const rawBase = fields.base;
+ if (rawBase) base = normalize(rawBase, url).href;
+ } catch {
+ /* invalid base falls back to the document */
+ }
+ const links: string[] = [];
+ const nofollow = /\b(nofollow|none)\b/.test(robots);
+ if (!nofollow)
+ for (const link of fields.links) {
+ if (/\bnofollow\b/i.test(link.rel)) continue;
+ try {
+ links.push(normalize(link.href, base).href);
+ } catch {
+ /* unsupported crawl target */
+ }
+ }
+ if (fields.linkCount > 2000)
+ add(
+ "link_limit",
+ "info",
+ "Only the first 2,000 links were inspected.",
+ "Split very large navigation lists across smaller pages.",
+ );
+ return { title, description, canonical, links: [...new Set(links)], issues };
+}
+
+/** Tokenizer callbacks retain only bounded SEO fields, never a document tree. */
+function extractHtml(html: string) {
+ const fields = {
+ title: "",
+ description: "",
+ hasH1: false,
+ language: "",
+ noindex: false,
+ nofollow: false,
+ canonicalCount: 0,
+ canonical: undefined as string | undefined,
+ base: undefined as string | undefined,
+ links: [] as { href: string; rel: string }[],
+ linkCount: 0,
+ };
+ let inHead = true,
+ titleOpen = false,
+ titleSeen = false,
+ descriptionSeen = false,
+ languageSeen = false,
+ baseSeen = false;
+ let ignored = 0;
+ let depth = 0;
+ const suppressed = new Set([
+ "script",
+ "style",
+ "noscript",
+ "svg",
+ "math",
+ "template",
+ ]);
+ const headTags = new Set([
+ "html",
+ "head",
+ "title",
+ "base",
+ "link",
+ "meta",
+ "script",
+ "style",
+ "noscript",
+ "template",
+ ]);
+ const parser = new Parser(
+ {
+ onopentag(name, attrs) {
+ if (++depth > 128)
+ throw new CrawlError(
+ "markup_depth",
+ "Inspection incomplete: HTML nesting exceeds 128 levels.",
+ );
+ if (suppressed.has(name)) {
+ ignored++;
+ return;
+ }
+ if (ignored) return;
+ // HTML permits an omitted <head>. Its first body element ends that head.
+ if (name === "body" || !headTags.has(name)) inHead = false;
+ if (name === "html" && !languageSeen) {
+ fields.language = (attrs.lang ?? "").slice(0, 100);
+ languageSeen = true;
+ }
+ if (name === "title" && inHead && !titleSeen) {
+ titleSeen = true;
+ titleOpen = true;
+ }
+ if (name === "h1") fields.hasH1 = true;
+ if (name === "meta" && inHead) {
+ const key = attrs.name?.toLowerCase();
+ const content = attrs.content ?? "";
+ if (key === "description" && !descriptionSeen) {
+ fields.description = content.trim().slice(0, 2000);
+ descriptionSeen = true;
+ }
+ if (key === "robots") {
+ fields.noindex ||= /\b(noindex|none)\b/i.test(content);
+ fields.nofollow ||= /\b(nofollow|none)\b/i.test(content);
+ }
+ }
+ if (
+ name === "link" &&
+ inHead &&
+ attrs.rel?.toLowerCase().split(/\s+/).includes("canonical")
+ ) {
+ if (fields.canonicalCount === 0)
+ fields.canonical = attrs.href?.slice(0, 2049);
+ fields.canonicalCount++;
+ }
+ if (name === "base" && attrs.href !== undefined && !baseSeen) {
+ fields.base = attrs.href.slice(0, 2049);
+ baseSeen = true;
+ }
+ if (name === "a" && attrs.href !== undefined) {
+ fields.linkCount++;
+ if (fields.linkCount <= 2000)
+ fields.links.push({
+ href: attrs.href.slice(0, 2049),
+ rel: /\bnofollow\b/i.test(attrs.rel ?? "") ? "nofollow" : "",
+ });
+ }
+ },
+ ontext(text) {
+ if (titleOpen && !ignored && fields.title.length < 1000)
+ fields.title += (fields.title ? text : text.trimStart()).slice(
+ 0,
+ 1000 - fields.title.length,
+ );
+ },
+ onclosetag(name) {
+ depth = Math.max(0, depth - 1);
+ if (suppressed.has(name) && ignored > 0) {
+ ignored--;
+ return;
+ }
+ if (ignored) return;
+ if (name === "title") titleOpen = false;
+ if (name === "head") inHead = false;
+ },
+ },
+ { decodeEntities: true },
+ );
+ parser.end(html);
+ return fields;
+}
diff --git a/site-crawl-plugin/src/sitemap.ts b/site-crawl-plugin/src/sitemap.ts
new file mode 100644
index 0000000..9720b4e
--- /dev/null
+++ b/site-crawl-plugin/src/sitemap.ts
@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: 2024-2026 Temps Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+import { Parser } from "htmlparser2";
+import { CrawlError } from "./types";
+/** XML tokenizer: retain only direct loc children and bounded URL strings. */
+export function parseSitemap(
+ xml: string,
+ maxUrls: number,
+): {
+ kind: "urlset" | "sitemapindex" | null;
+ locations: string[];
+ limited: boolean;
+} {
+ let kind: "urlset" | "sitemapindex" | null = null;
+ const locations: string[] = [];
+ const stack: string[] = [];
+ let prefix = "";
+ let text: string | null = null,
+ count = 0;
+ const parser = new Parser(
+ {
+ onopentag(name) {
+ if (stack.length >= 64)
+ throw new CrawlError(
+ "markup_depth",
+ "Sitemap nesting exceeds the 64-level inspection limit.",
+ );
+ const local = name.split(":").pop()!;
+ stack.push(name);
+ if (stack.length === 1)
+ prefix = name.slice(0, name.length - local.length);
+ if (
+ stack.length === 1 &&
+ (local === "urlset" || local === "sitemapindex")
+ )
+ kind = local;
+ if (
+ stack.length === 3 &&
+ name === `${prefix}loc` &&
+ stack[1] === `${prefix}${kind === "urlset" ? "url" : "sitemap"}`
+ )
+ text = "";
+ },
+ ontext(value) {
+ if (text !== null && text.length < 2049)
+ text += (text ? value : value.trimStart()).slice(
+ 0,
+ 2049 - text.length,
+ );
+ },
+ onclosetag() {
+ if (
+ stack.length === 3 &&
+ stack[2] === `${prefix}loc` &&
+ text !== null
+ ) {
+ count++;
+ if (locations.length < (kind === "sitemapindex" ? 5 : maxUrls))
+ locations.push(text.trim());
+ text = null;
+ }
+ stack.pop();
+ },
+ },
+ { xmlMode: true, decodeEntities: true },
+ );
+ parser.end(xml);
+ return {
+ kind,
+ locations,
+ limited: count > (kind === "sitemapindex" ? 5 : maxUrls),
+ };
+}
diff --git a/site-crawl-plugin/src/store.ts b/site-crawl-plugin/src/store.ts
new file mode 100644
index 0000000..606c2fa
--- /dev/null
+++ b/site-crawl-plugin/src/store.ts
@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: 2024-2026 Temps Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import type { Report } from "./types";
+export class Store {
+ private db: Database;
+ constructor(directory: string) {
+ mkdirSync(directory, { recursive: true });
+ this.db = new Database(join(directory, "site-crawl.sqlite"), {
+ create: true,
+ });
+ this.db.exec(
+ "PRAGMA journal_mode=WAL; CREATE TABLE IF NOT EXISTS reports (id TEXT PRIMARY KEY, started_at TEXT NOT NULL, state TEXT NOT NULL, payload TEXT NOT NULL)",
+ );
+ this.db.exec(
+ "CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
+ );
+ for (const row of this.db
+ .query<{ payload: string }, []>(
+ "SELECT payload FROM reports WHERE state='running'",
+ )
+ .all()) {
+ const report = JSON.parse(row.payload) as Report;
+ report.state = "interrupted";
+ report.finishedAt = new Date().toISOString();
+ report.error =
+ "The plugin restarted during this crawl. Start a new crawl to finish checking the site.";
+ this.save(report);
+ }
+ }
+ save(report: Report) {
+ this.db
+ .query(
+ "INSERT INTO reports(id,started_at,state,payload) VALUES(?,?,?,?) ON CONFLICT(id) DO UPDATE SET state=excluded.state,payload=excluded.payload",
+ )
+ .run(report.id, report.startedAt, report.state, JSON.stringify(report));
+ this.db.exec(
+ "DELETE FROM reports WHERE state != 'running' AND id NOT IN (SELECT id FROM reports ORDER BY started_at DESC LIMIT 30)",
+ );
+ }
+ list() {
+ return this.db
+ .query<{ payload: string }, []>(
+ "SELECT payload FROM reports ORDER BY started_at DESC LIMIT 30",
+ )
+ .all()
+ .map((row) => {
+ const report = JSON.parse(row.payload) as Report;
+ return {
+ ...report,
+ pages: undefined,
+ checked: report.pages.length,
+ errors: report.pages.filter((page) =>
+ page.issues.some((issue) => issue.severity === "error"),
+ ).length,
+ issues: report.pages.reduce(
+ (sum, page) => sum + page.issues.length,
+ 0,
+ ),
+ };
+ });
+ }
+ get(id: string): Report | null {
+ const row = this.db
+ .query<{ payload: string }, [string]>(
+ "SELECT payload FROM reports WHERE id=?",
+ )
+ .get(id);
+ return row ? (JSON.parse(row.payload) as Report) : null;
+ }
+ delete(id: string) {
+ this.db.query("DELETE FROM reports WHERE id=?").run(id);
+ }
+ metadata<T>(key: string, fallback: T): T {
+ const row = this.db
+ .query<{ value: string }, [string]>(
+ "SELECT value FROM metadata WHERE key=?",
+ )
+ .get(key);
+ return row ? (JSON.parse(row.value) as T) : fallback;
+ }
+ setMetadata(key: string, value: unknown) {
+ this.db
+ .query(
+ "INSERT INTO metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+ )
+ .run(key, JSON.stringify(value));
+ }
+ transaction(action: () => void) {
+ this.db.transaction(action)();
+ }
+ close() {
+ this.db.close();
+ }
+}
diff --git a/site-crawl-plugin/src/types.ts b/site-crawl-plugin/src/types.ts
new file mode 100644
index 0000000..b47fa27
--- /dev/null
+++ b/site-crawl-plugin/src/types.ts
@@ -0,0 +1,65 @@
+// SPDX-FileCopyrightText: 2024-2026 Temps Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+export type Severity = "error" | "warning" | "info";
+export interface Issue {
+ code: string;
+ severity: Severity;
+ message: string;
+ fix: string;
+}
+export interface Page {
+ url: string;
+ finalUrl: string;
+ status: number | null;
+ durationMs: number;
+ title: string;
+ description: string;
+ canonical: string | null;
+ sources: string[];
+ redirects: string[];
+ issues: Issue[];
+}
+export interface Report {
+ id: string;
+ trigger?: DeploymentTrigger;
+ url: string;
+ maxPages: number;
+ state: "running" | "completed" | "cancelled" | "failed" | "interrupted";
+ startedAt: string;
+ finishedAt: string | null;
+ pages: Page[];
+ notices: string[];
+ discovered: number;
+ limited: boolean;
+ error: string | null;
+}
+export class CrawlError extends Error {
+ constructor(
+ public code: string,
+ message: string,
+ public status?: number,
+ ) {
+ super(message);
+ this.name = "CrawlError";
+ }
+}
+export function message(error: unknown): string {
+ return error instanceof Error ? error.message : "Unexpected crawl failure";
+}
+export interface DeploymentTrigger {
+ projectId: number;
+ environmentId: number;
+ environmentName: string;
+ deploymentId: number;
+}
+export interface AutomationSettings {
+ enabled: boolean;
+ productionOnly: boolean;
+ maxPages: number;
+ excludedProjects: number[];
+}
+export interface HostEventAccess {
+ configured: boolean;
+ reason: string | null;
+ setupPath: string;
+}
diff --git a/site-crawl-plugin/tsconfig.json b/site-crawl-plugin/tsconfig.json
new file mode 100644
index 0000000..06be346
--- /dev/null
+++ b/site-crawl-plugin/tsconfig.json
@@ -0,0 +1,21 @@
+{
+ "compilerOptions": {
+ "target": "ESNext",
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ "strict": true,
+ "noEmit": true,
+ "resolveJsonModule": true,
+ "types": ["bun"],
+ "skipLibCheck": true,
+ "allowImportingTsExtensions": true,
+ "jsx": "react-jsx"
+ },
+ "include": [
+ "src/**/*.ts",
+ "scripts/**/*.ts",
+ "web/**/*.tsx",
+ "web/**/*.ts",
+ "vite.config.ts"
+ ]
+}
diff --git a/site-crawl-plugin/vite.config.ts b/site-crawl-plugin/vite.config.ts
new file mode 100644
index 0000000..dcbf5ca
--- /dev/null
+++ b/site-crawl-plugin/vite.config.ts
@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2024-2026 Temps Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+import { defineConfig } from "vite";
+import tailwindcss from "@tailwindcss/vite";
+export default defineConfig({
+ root: "web",
+ base: "./",
+ plugins: [tailwindcss()],
+ build: {
+ outDir: "dist",
+ emptyOutDir: true,
+ cssCodeSplit: false,
+ rollupOptions: {
+ onwarn(warning, warn) {
+ if (
+ warning.code === "MODULE_LEVEL_DIRECTIVE" &&
+ warning.message.includes("use client")
+ )
+ return;
+ warn(warning);
+ },
+ output: { entryFileNames: "app.js", assetFileNames: "style[extname]" },
+ },
+ },
+});
diff --git a/site-crawl-plugin/web/index.html b/site-crawl-plugin/web/index.html
new file mode 100644
index 0000000..dd11784
--- /dev/null
+++ b/site-crawl-plugin/web/index.html
@@ -0,0 +1,14 @@
+<!doctype html>
+<!-- SPDX-FileCopyrightText: 2024-2026 Temps Contributors -->
+<!-- SPDX-License-Identifier: MIT OR Apache-2.0 -->
+<html lang="en" class="tds">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <title>Site Crawl · Temps
+
+
+
+
+
+