Connect Google and Bing AI reports without file uploads

.changeset/connect-ai-reports.md

@@ -0,0 +1,7 @@
+---
+"@workspace/web": patch
+"@workspace/lib": patch
+"@workspace/config": patch
+---
+
+Connect Google and Bing AI reports through your signed-in browser without file uploads, and separate browser visibility from API answers.

apps/browser-lab/server.mjs

@@ -14,6 +14,7 @@ import {
 	summarize,
 	validateInput,
 } from "./core.mjs";
+import { readWebmasterReport } from "./webmaster-reports.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const directory = defaultRunDirectory();
@@ -134,6 +135,19 @@ async function handle(request, response) {
 	const url = new URL(request.url, `http://127.0.0.1:${port}`);
 	if (request.method === "GET") return handleRead(url, request, response);
 	if (request.method === "POST" && url.pathname === "/api/browser/profile") return selectProfile(request, response);
+	if (request.method === "POST" && url.pathname === "/api/reports") {
+		if (!authorized(request)) return json(response, 403, { error: "Invalid local runner token" });
+		const input = await bodyOf(request);
+		if (!["connect", "sync"].includes(input.action)) return json(response, 400, { error: "Invalid report action" });
+		if (active)
+			return json(response, 409, { error: "The browser is busy. Retry after the current operation finishes." });
+		active = { status: "reading-report" };
+		try {
+			return json(response, 200, await readWebmasterReport(input));
+		} finally {
+			active = null;
+		}
+	}
 	if (request.method !== "POST" || !["/api/run", "/api/collect"].includes(url.pathname))
 		return json(response, 404, { error: "Not found" });
 	return handleCollection(request, response, url);

apps/browser-lab/webmaster-reports.mjs

@@ -0,0 +1,254 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { createBrowser, parseRows } from "./core.mjs";
+
+export function reportUrl(source, property) {
+	if (!["bing", "google"].includes(source)) throw new Error("Unsupported report source.");
+	if (typeof property !== "string" || property.length > 2048) throw new Error("Invalid website.");
+	const domain = property.startsWith("sc-domain:");
+	const parsed = new URL(domain ? `https://${property.slice(10)}` : property);
+	if (!["https:", "http:"].includes(parsed.protocol) || parsed.username || parsed.password || parsed.port)
+		throw new Error("Invalid website.");
+	if (source === "bing" && domain) throw new Error("Bing requires a website URL.");
+	const url = new URL(
+		source === "bing"
+			? "https://www.bing.com/webmasters/aiperformance"
+			: "https://search.google.com/search-console/performance/search-analytics/ai",
+	);
+	url.searchParams.set(source === "bing" ? "siteUrl" : "resource_id", property);
+	return url.href;
+}
+
+// Read only responses generated by the official report page. No cookies,
+// session tokens or undocumented request payloads are replayed or persisted.
+export function responseJson(text) {
+	const start = text.indexOf("\n{");
+	if (start < 0) throw new Error("Report response unavailable.");
+	const body = text
+		.slice(start + 1)
+		.split("\n</page-content>")[0]
+		.trim();
+	return JSON.parse(body);
+}
+
+export function googleFrames(text) {
+	return text
+		.split("\n")
+		.filter((line) => line.startsWith('[["wrb.fr"'))
+		.flatMap((line) => JSON.parse(line))
+		.filter((frame) => frame[0] === "wrb.fr" && typeof frame[2] === "string")
+		.map((frame) => ({ rpc: frame[1], data: JSON.parse(frame[2]) }));
+}
+
+function count(value) {
+	if (!Number.isSafeInteger(value) || value < 0) throw new Error("Invalid report count.");
+	return value;
+}
+function dates(rows) {
+	if (!rows.length) throw new Error("The report has no dated observations yet.");
+	const sorted = rows.map((row) => row.key).sort();
+	return { startDate: sorted[0], endDate: sorted.at(-1) };
+}
+
+export function parseBingHistory(data, property) {
+	if (data.SiteUrl !== property || !Array.isArray(data.CitationStats)) throw new Error("Report website mismatch.");
+	const rows = data.CitationStats.map((row) => ({ key: row.Date.slice(0, 10), value: count(row.Citations) }));
+	return { ...dates(rows), rows, total: count(data.TotalCitations) };
+}
+
+export function parseGoogleReport(text, property) {
+	const frames = googleFrames(text).filter(({ data }) => data?.[0]?.[0] === property);
+	const history = frames.findLast(
+		({ rpc, data }) =>
+			rpc === "OLiH4d" &&
+			data?.[6]?.[2]?.length === 1 &&
+			data[6][2][0][0] === "IMPRESSIONS" &&
+			data?.[1]?.[0]?.length > 1,
+	)?.data;
+	const pages = frames.findLast(
+		({ rpc, data }) => rpc === "nDAfwb" && data?.[1]?.[1]?.[0]?.[9] === "PAGES" && data[1][1][1]?.[9] === "IMPRESSIONS",
+	)?.data;
+	if (!history || !pages) throw new Error("AI report unavailable or its format has changed.");
+	// These observed request descriptors distinguish the AI report and its filters.
+	if (
+		history[0][5]?.[15] !== 4 ||
+		pages[0][5]?.[15] !== 4 ||
+		JSON.stringify(history[0][5][2]) !== "[[null,null,null,3]]" ||
+		JSON.stringify(history[0][5][3]) !== '[[6,["WEB"]]]' ||
+		JSON.stringify(history[0][5].slice(1)) !== JSON.stringify(pages[0][5].slice(1))
+	)
+		throw new Error("AI report filters could not be verified.");
+	const daily = history[1][0].map((row) => {
+		if (!Number.isFinite(row[0]) || row[1]?.length !== 1) throw new Error("Invalid daily report.");
+		return { key: new Date(row[0]).toISOString().slice(0, 10), value: count(row[1][0]) };
+	});
+	const pageRows = pages[1][0].map((row) => {
+		const cells = row?.[0];
+		if (typeof cells?.[0]?.[40] !== "string" || cells?.[1]?.[8] !== 6)
+			throw new Error("AI page report format changed.");
+		return { key: cells[0][40], value: count(cells[1][1]) };
+	});
+	return { ...dates(daily), daily, pages: pageRows, total: count(history[1][1][1][0]) };
+}
+
+async function waitFor(read, delay, accept = () => true) {
+	for (let attempt = 0; attempt < 12; attempt++) {
+		try {
+			const value = await read();
+			if (accept(value)) return value;
+		} catch {
+			/* Page is still loading. */
+		}
+		await delay(1500);
+	}
+	throw new Error(
+		"The AI report could not be read. Check website access, sign-in and report availability, then retry.",
+	);
+}
+
+function appendBingBatch(rows, batch, name, key, total) {
+	if (!Number.isSafeInteger(batch.TotalNumOfRecords) || batch.TotalNumOfRecords < 0)
+		throw new Error("Report pagination could not be verified.");
+	if (total !== undefined && total !== batch.TotalNumOfRecords)
+		throw new Error("Report changed during synchronization. Retry.");
+	for (const row of batch[name]) {
+		if (typeof row[key] !== "string" || rows.has(row[key])) throw new Error("Duplicate report page. Retry.");
+		rows.set(row[key], { key: row[key], value: count(row.Citations) });
+	}
+	return batch.TotalNumOfRecords;
+}
+
+async function bingTable(call, name, property, delay) {
+	const endpoint = `aiperformance/${name === "Pages" ? "pages" : "searchqueries"}/stats`;
+	const key = name === "Pages" ? "PageUrl" : "GroundingQuery";
+	await call("snapshot", {});
+	const tabs = parseRows(await call("query", { selector: '[role="tab"]', extract: { text: ".@text", id: ".@id" } }));
+	const tab = tabs.find((item) => item.text === (name === "Pages" ? "Pages" : "Grounding Queries"));
+	if (!tab?.id) throw new Error("AI report tab unavailable.");
+	await call("click", { selector: `[id=${JSON.stringify(tab.id)}]` });
+	const rows = new Map();
+	let total;
+	for (let page = 0; page < 80; page++) {
+		const batch = await waitFor(
+			async () => responseJson(await call("network", { body: endpoint, maxBody: 2_000_000 })),
+			delay,
+			(data) =>
+				data.SiteUrl === property &&
+				Array.isArray(data[name]) &&
+				(data[name].length === 0 || !rows.has(data[name][0][key])),
+		);
+		total = appendBingBatch(rows, batch, name, key, total);
+		if (rows.size === total) return [...rows.values()];
+		if (rows.size > total || total > 2000 || !batch[name].length)
+			throw new Error("Report exceeds the supported row limit.");
+		await call("snapshot", {});
+		await call("click", { selector: 'button[aria-label="Go to next Page"]' });
+	}
+	throw new Error("Report pagination did not finish. No partial report was saved.");
+}
+
+export async function readWebmasterReport(input, options = {}) {
+	const url = reportUrl(input.source, input.property);
+	const browser = options.browser || createBrowser();
+	const delay = options.delay || sleep;
+	const openingUrl =
+		input.source === "google"
+			? `https://search.google.com/search-console?resource_id=${encodeURIComponent(input.property)}`
+			: url;
+	const space = JSON.parse(
+		await browser("spaces", {
+			action: "open",
+			url: openingUrl,
+			label: `BeatGEO · ${input.source} AI reports`,
+			leaseMs: 600_000,
+			profileId: input.profileId || process.env.BEAT_BROWSER_PROFILE_ID || undefined,
+		}),
+	);
+	if (!space.spaceId || !Number.isInteger(space.tabId) || space.state !== "running")
+		throw new Error("Connect your Chrome first.");
+	const call = (tool, params) => browser(tool, { ...params, spaceId: space.spaceId, tabId: space.tabId });
+	let synced = false;
+	try {
+		let history;
+		if (input.source === "bing") {
+			history = await waitFor(
+				async () =>
+					parseBingHistory(
+						responseJson(
+							await call("network", {
+								body: "aiperformance/citationstats",
+								maxBody: 2_000_000,
+							}),
+						),
+						input.property,
+					),
+				delay,
+			);
+			if (input.action === "connect") return { ready: true, property: input.property, url };
+			const queries = await bingTable(call, "Queries", input.property, delay);
+			const pages = await bingTable(call, "Pages", input.property, delay);
+			synced = true;
+			return {
+				ready: true,
+				property: input.property,
+				url,
+				...dates(history.rows),
+				total: history.total,
+				tables: [
+					{ dimension: "date", rows: history.rows },
+					{ dimension: "page", rows: pages },
+					{ dimension: "query", rows: queries },
+				],
+				coverage: "All rows returned by the official report; platform sampling applies.",
+			};
+		}
+		const sectionSelector = '[aria-label="“效果”报告部分"], [aria-label="Performance report section"]';
+		const sections = await waitFor(
+			async () =>
+				parseRows(await call("query", { selector: sectionSelector, extract: { expanded: ".@aria-expanded" } })),
+			delay,
+			(rows) => rows.length > 0,
+		);
+		if (sections[0].expanded === "false") {
+			await call("snapshot", {});
+			await call("click", { selector: sectionSelector });
+		}
+		await waitFor(
+			async () =>
+				parseRows(
+					await call("query", { selector: 'a[href*="/performance/search-analytics/ai"]', extract: { href: ".@href" } }),
+				),
+			delay,
+			(rows) => rows.length > 0,
+		);
+		await call("snapshot", {});
+		await call("click", { selector: 'a[href*="/performance/search-analytics/ai"]' });
+		history = await waitFor(
+			async () =>
+				parseGoogleReport(
+					await call("network", {
+						body: "rpcids=OLiH4d",
+						maxBody: 2_000_000,
+					}),
+					input.property,
+				),
+			delay,
+		);
+		if (input.action === "connect") return { ready: true, property: input.property, url };
+		synced = true;
+		return {
+			ready: true,
+			property: input.property,
+			url,
+			startDate: history.startDate,
+			endDate: history.endDate,
+			total: history.total,
+			tables: [
+				{ dimension: "date", rows: history.daily },
+				{ dimension: "page", rows: history.pages },
+			],
+			coverage: "Rows returned by the official AI report; page results may be limited by Google.",
+		};
+	} finally {
+		await browser("spaces", { action: "finish", spaceId: space.spaceId, keep: synced ? [] : "all" }).catch(() => {});
+	}
+}