diff --git a/Makefile b/Makefile index e5ba3ff0eee..0333c4375fd 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ include fmt.mk .PHONY: vscode-ext vscode-ext-install .PHONY: docs-server check-docs-links .PHONY: storybook storybook-run storybook-build storybook-flake-check test-storybook storybook-budget -.PHONY: benchmark-terminal bench bench-compare first-load-js page-timing bug-bash mcp-apps-e2e +.PHONY: benchmark-terminal bench bench-compare first-load-js page-timing first-run-view-parity bug-bash mcp-apps-e2e .PHONY: ensure-deps mux .PHONY: check-startup-imports check-startup-imports-runtime check-react-compiler check-test-routing check-test-seam-comments test-bench-scripts @@ -826,6 +826,9 @@ first-load-js: ## Report first-load JS bytes of dist/ (run make build-renderer f page-timing: node_modules/.installed ## Paired mobile UCS/IR timing of two built trees (PAGE_TIMING_ARGS; see scripts/perf/pageTiming.ts) @node --import tsx scripts/perf/pageTiming.ts $(PAGE_TIMING_ARGS) +first-run-view-parity: node_modules/.installed ## Static vs live first-run view boxes and screenshots, 4 widths x 2 themes (FIRST_RUN_VIEW_PARITY_ARGS; see scripts/perf/firstRunViewParity.ts) + @node --import tsx scripts/perf/firstRunViewParity.ts $(FIRST_RUN_VIEW_PARITY_ARGS) + benchmark-terminal: ## Run Terminal-Bench 2.0 with Harbor (use TB_HARBOR_PACKAGE/TB_HARBOR_DAYTONA_PACKAGE/TB_DATASET/TB_CONCURRENCY/TB_TIMEOUT/TB_ENV/TB_MODEL/TB_ARGS to customize) @# Pin Harbor with the Daytona extra so scheduled ingestion does not break on future CLI or adapter API drift. @# Force the Daytona SDK to the cursor-pagination API while keeping Harbor stable. diff --git a/scripts/perf/firstRunViewParity.test.ts b/scripts/perf/firstRunViewParity.test.ts new file mode 100644 index 00000000000..fa6f046f2ba --- /dev/null +++ b/scripts/perf/firstRunViewParity.test.ts @@ -0,0 +1,30 @@ +import { expect, test } from "bun:test"; +import { Window } from "happy-dom"; +import { FIRST_RUN_VIEW_ID } from "@/common/constants/firstRunView"; +import { FIRST_RUN_VIEW_HTML } from "@/node/orpc/firstRunView"; +import { parityFailures, withFirstRunView } from "./firstRunViewParity"; + +// Catches a hand-made test page that puts the view inside #root (React clears #root, so the view +// would vanish before the handoff), adds it twice, or patches an HTML it cannot place it in. +test("the test page gets the constant once, outside #root", async () => { + const served = '
Loading
\n'; + const window = new Window(); + window.document.write(withFirstRunView(served, FIRST_RUN_VIEW_HTML)); + const views = window.document.querySelectorAll(`#${FIRST_RUN_VIEW_ID}`); + expect(views).toHaveLength(1); + expect(views[0].closest("#root") === null).toBe(true); // a compact failure, not an element dump + expect(window.document.getElementById("root")?.textContent).toBe("Loading"); + expect(() => withFirstRunView("", FIRST_RUN_VIEW_HTML)).toThrow(""); + await window.happyDOM.close(); +}); + +// Catches a rule that checks only the position (a paragraph 3 px wider passes), uses a tolerance +// other than 2 px, or lets a second
landmark through. +test("parity allows 2 px of drift in position and size, and exactly one main", () => { + const live = { x: 20, y: 300, width: 351, height: 60 }; + expect(parityFailures({ ...live, x: 22, height: 58 }, live, 1)).toEqual([]); + expect(parityFailures({ ...live, y: 302.5 }, live, 1)).toEqual(["y off by 2.5 px"]); + expect(parityFailures({ ...live, width: 354 }, live, 1)).toEqual(["width off by 3 px"]); + expect(parityFailures(live, live, 2)).toEqual(["2 main landmarks in the static phase"]); + expect(parityFailures(live, live, 0)).toEqual(["0 main landmarks in the static phase"]); +}); diff --git a/scripts/perf/firstRunViewParity.ts b/scripts/perf/firstRunViewParity.ts new file mode 100644 index 00000000000..1a703909baa --- /dev/null +++ b/scripts/perf/firstRunViewParity.ts @@ -0,0 +1,222 @@ +/** + * First-run view parity (T3 #5971, Option C step 2, plan section 5.6: PR 3b, gate 8 of PR 4b). + * make first-run-view-parity FIRST_RUN_VIEW_PARITY_ARGS="--tree [--out ] [--chrome ]" + * Serves the tree's `xum server` on a fresh empty root (the first-run state) and opens "/" in a real + * Chrome at 390, 744, 1200 and 1900 px, in the light and the dark system theme. Each run holds the + * app's main chunk, so the static view (#xum-pre) stays alone on screen, measures it, releases the + * chunk and measures the live wizard's first frame after the handoff. + * Rule: the static "OSS contributors..." paragraph is within 2 px of the live one in x, y, width and + * height, and the static phase has exactly one
. It saves a full-page screenshot of the static + * phase and of the live first frame per run, for PR 4b's shell check (owner decision 10-14). + * Until PR 4b serves the view, the served "/" has no #xum-pre: the tool then inserts #6444's + * FIRST_RUN_VIEW_HTML before of the served "/" by hand (Playwright route), as a test page. + * Exit: 0 every run passes, 1 a run fails the rule, 2 bad input, 3 an invalid run. + */ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { chromium, type Browser, type Page } from "playwright"; +import { FIRST_RUN_VIEW_ID } from "@/common/constants/firstRunView"; +import { FIRST_RUN_VIEW_HTML } from "@/node/orpc/firstRunView"; + +export interface Box { + x: number; + y: number; + width: number; + height: number; +} + +/** The served "/" with the static view inserted once, before : outside #root, which React clears. */ +export function withFirstRunView(html: string, markup: string): string { + const parts = html.split(""); + if (parts.length !== 2) throw new Error("the served / needs exactly one "); + return `${parts[0]}${markup}\n${parts[1]}`; +} + +/** Gate 8's rule for one run: every failure, or none. */ +export function parityFailures(staticBox: Box, liveBox: Box, staticMains: number): string[] { + const failures = (["x", "y", "width", "height"] as const) + .map((key) => ({ key, off: Math.abs(staticBox[key] - liveBox[key]) })) + .filter(({ off }) => off > 2) + .map(({ key, off }) => `${key} off by ${Math.round(off * 10) / 10} px`); + if (staticMains !== 1) failures.push(`${staticMains} main landmarks in the static phase`); + return failures; +} + +const WIDTHS = [ + { width: 390, height: 844, phone: true }, + { width: 744, height: 1133, phone: true }, + { width: 1200, height: 800, phone: false }, + { width: 1900, height: 1000, phone: false }, +]; +const THEMES = ["light", "dark"] as const; +const TEXT = "OSS contributors with GitHub accounts"; + +async function startServer(tree: string, root: string, logFile: string) { + const probePort = net.createServer().listen(0, "127.0.0.1"); + await once(probePort, "listening"); + const { port } = probePort.address() as net.AddressInfo; + probePort.close(); + const log = fs.openSync(logFile, "w"); + const env = { PATH: process.env.PATH, HOME: root, XUM_ROOT: root, XUM_DISABLE_TELEMETRY: "1" }; + const args = `dist/cli/index.js server --no-auth --host 127.0.0.1 --port ${port}`.split(" "); + const server = spawn("node", args, { + cwd: tree, + env: { ...env, GIT_CONFIG_GLOBAL: "/dev/null" }, + detached: true, + stdio: ["ignore", log, log], + }); + const origin = `http://127.0.0.1:${port}`; + const up = () => + fetch(`${origin}/health`) + .then((r) => r.ok) + .catch(() => false); + for (let i = 0; i < 120 && !(await up()); i++) await new Promise((r) => setTimeout(r, 500)); + const stop = async () => { + if (server.exitCode === null && server.signalCode === null) { + process.kill(-server.pid!, "SIGTERM"); + await once(server, "exit"); + } + fs.closeSync(log); + }; + if (!(await up())) { + await stop(); + throw new Error(`the server did not start, see ${logFile}`); + } + return { origin, stop }; +} + +/** The target paragraph's box, in the static view or outside it; null while there is none. */ +function paragraphBox(page: Page, inStatic: boolean) { + return page.evaluate( + ([id, text, wantStatic]) => { + const p = [...document.querySelectorAll("p")].find( + (e) => e.textContent?.startsWith(text) && !!e.closest(`#${id}`) === wantStatic + ); + const r = p?.getBoundingClientRect(); + return r && r.width > 0 ? { x: r.x, y: r.y, width: r.width, height: r.height } : null; + }, + [FIRST_RUN_VIEW_ID, TEXT, inStatic] as const + ); +} + +async function main(): Promise { + const { values: a } = parseArgs({ + options: { + tree: { type: "string" }, + out: { type: "string" }, + chrome: { type: "string", default: "/usr/bin/google-chrome" }, + }, + }); + if (!a.tree || !fs.existsSync(path.join(a.tree, "dist/cli/index.js"))) { + console.error("--tree must be a built checkout (see the header of this file)"); + return 2; + } + const out = path.resolve(a.out ?? fs.mkdtempSync(path.join(os.tmpdir(), "parity-out-"))); + fs.mkdirSync(out, { recursive: true }); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "parity-root-")); + let server: Awaited> | undefined; + let browser: Browser | undefined; + const results = []; + let injected = false; + try { + server = await startServer(path.resolve(a.tree), root, path.join(out, "server.log")); + const { origin } = server; + // A route-fulfilled document has no local address space, so Local Network Access would block the + // app's WebSocket to 127.0.0.1. pageTiming.ts navigates without fulfill and needs no such flag. + browser = await chromium.launch({ + executablePath: a.chrome, + headless: true, + args: ["--disable-features=LocalNetworkAccessChecks"], + }); + for (const size of WIDTHS) { + for (const theme of THEMES) { + const name = `${size.width}-${theme}`; + const context = await browser.newContext({ + viewport: { width: size.width, height: size.height }, + colorScheme: theme, + ...(size.phone ? { isMobile: true, hasTouch: true, deviceScaleFactor: 2 } : {}), + }); + const page = await context.newPage(); + let release!: () => void; + const held = new Promise((resolve) => (release = resolve)); + await page.route(`${origin}/`, async (route) => { + const response = await route.fetch(); + const html = await response.text(); + injected ||= !html.includes(`id="${FIRST_RUN_VIEW_ID}"`); + const body = html.includes(`id="${FIRST_RUN_VIEW_ID}"`) + ? html + : withFirstRunView(html, FIRST_RUN_VIEW_HTML); + await route.fulfill({ status: response.status(), contentType: "text/html", body }); + }); + await page.route(/\/main-[^/]+\.js$/, async (route) => { + await held; + await route.continue(); + }); + await page.goto(`${origin}/`, { waitUntil: "commit" }); + // Measure the painted static view: stylesheets applied (a stylesheet that never loads is an + // invalid run after 60 s), then the fonts that layout asked for. + const sheets = "link[rel=stylesheet]"; + await page.waitForFunction( + (selector) => + [...document.querySelectorAll(selector)].every((l) => l.sheet), + sheets, + { timeout: 60_000 } + ); + // (No named inner functions: tsx's keepNames would wrap them in __name, which the page lacks.) + await page.evaluate(async () => { + void document.body.offsetHeight; // layout now, so the fonts it needs start loading + await document.fonts.ready; + for (let i = 0; i < 2; i++) + await new Promise((resolve) => requestAnimationFrame(resolve)); + }); + const staticBox = await paragraphBox(page, true); + if (!staticBox) throw new Error(`${name}: the static view shows no target paragraph`); + const staticMains = await page.evaluate(() => document.querySelectorAll("main").length); + await page.screenshot({ path: path.join(out, `static-${name}.png`), fullPage: true }); + release(); + const handle = await page.waitForFunction( + ([id, text]) => { + if (document.getElementById(id)) return null; + const p = [...document.querySelectorAll("p")].find((e) => + e.textContent?.startsWith(text) + ); + const r = p?.getBoundingClientRect(); + return r && r.width > 0 ? { x: r.x, y: r.y, width: r.width, height: r.height } : null; + }, + [FIRST_RUN_VIEW_ID, TEXT] as const, + { polling: "raf", timeout: 120_000 } + ); + const liveBox = (await handle.jsonValue())!; + await page.screenshot({ path: path.join(out, `live-${name}.png`), fullPage: true }); + const failures = parityFailures(staticBox, liveBox, staticMains); + results.push({ name, staticBox, liveBox, staticMains, failures }); + console.log(`${name}: ${failures.length ? failures.join(", ") : "pass"}`); + await context.close(); + } + } + } finally { + await browser?.close(); + await server?.stop(); + fs.rmSync(root, { recursive: true, force: true }); + } + const report = { tree: path.resolve(a.tree), injected, results }; + fs.writeFileSync(path.join(out, "parity.json"), JSON.stringify(report, null, 2)); + console.log(`${injected ? "test page (constant inserted by hand)" : "served view"}: ${out}`); + return results.every((r) => r.failures.length === 0) ? 0 : 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().then( + (code) => process.exit(code), + (error: unknown) => { + console.error(error); + process.exit(String((error as { code?: unknown }).code).startsWith("ERR_PARSE_ARGS") ? 2 : 3); + } + ); +}