From f5678a722e39d4b4a94779139b1824a54f67b2c1 Mon Sep 17 00:00:00 2001 From: m-vawter Date: Fri, 2 Oct 2026 20:39:32 -0400 Subject: [PATCH] test: e2e Playwright test of the core loop (#79) Drive the core loop through the window.__dominic hooks: installFixture, reload, updateApp, injectBrokenApp, then Ask Agent to Fix, plus a second test for the runtime-crash path. /api/chat is answered in the browser with an AI SDK v4 data-stream reply (encoded with the SDK's own formatDataStreamPart) carrying an update_app call, so no provider key is needed. Requests off the app's origin are blocked, and the test server starts with blank provider keys. Playwright builds the app and serves the Nitro output on 127.0.0.1:3179. `npm run test:e2e` type-checks tests/e2e first. A new, separate CI job installs Chromium, runs the suite, and uploads the report on failure. lint:md now skips the gitignored e2e artifact folders. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 35 +++++++++ package-lock.json | 50 +++++++++++++ package.json | 4 +- playwright.config.ts | 48 +++++++++++++ tests/e2e/.gitignore | 2 + tests/e2e/README.md | 92 ++++++++++++++++++++++++ tests/e2e/core-loop.spec.ts | 121 ++++++++++++++++++++++++++++++++ tests/e2e/support/agent-stub.ts | 60 ++++++++++++++++ tests/e2e/support/desktop.ts | 59 ++++++++++++++++ tests/e2e/tsconfig.json | 17 +++++ 10 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 playwright.config.ts create mode 100644 tests/e2e/.gitignore create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/core-loop.spec.ts create mode 100644 tests/e2e/support/agent-stub.ts create mode 100644 tests/e2e/support/desktop.ts create mode 100644 tests/e2e/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b96e14b..17f2d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,38 @@ jobs: - name: Run build run: npm run build + + # Playwright end-to-end test of the core loop (NEXT.md step 0.1, #79). + # The chat route is stubbed in the browser, so no provider key is needed. + e2e: + name: E2E (Playwright) + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + NUXT_TELEMETRY_DISABLED: 1 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Run e2e tests + run: npm run test:e2e + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: | + tests/e2e/playwright-report/ + tests/e2e/test-results/ + retention-days: 7 diff --git a/package-lock.json b/package-lock.json index 41cc0a0..75e5662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "dominic", "version": "0.0.0", + "license": "AGPL-3.0-or-later", "dependencies": { "@ai-sdk/anthropic": "^1.2.12", "@ai-sdk/google": "^1.2.22", @@ -20,6 +21,7 @@ "devDependencies": { "@nuxtjs/tailwindcss": "^6.14.0", "@pinia/nuxt": "^0.11.2", + "@playwright/test": "^1.63.0", "@types/canvas-confetti": "^1.9.0", "@types/node": "^22.20.2", "@vueuse/core": "^13.9.0", @@ -31,6 +33,9 @@ "typescript": "^5.9.3", "vue": "^3.5.22", "vue-tsc": "^3.1.3" + }, + "engines": { + "node": ">=24.21.0 <25" } }, "node_modules/@ai-sdk/anthropic": { @@ -2372,6 +2377,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -10073,6 +10094,35 @@ "dev": true, "license": "MIT" }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/portfinder": { "version": "1.0.38", "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.38.tgz", diff --git a/package.json b/package.json index 9f9b635..0f576b6 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "generate": "nuxt generate", "preview": "nuxt preview", "typecheck": "nuxt typecheck", - "lint:md": "markdownlint-cli2 \"**/*.md\" \"#node_modules\" \"#.superpowers\" \"#docs/superpowers\"", + "test:e2e": "tsc -p tests/e2e && playwright test", + "lint:md": "markdownlint-cli2 \"**/*.md\" \"#node_modules\" \"#.superpowers\" \"#docs/superpowers\" \"#tests/e2e/test-results\" \"#tests/e2e/playwright-report\"", "prepare": "husky" }, "engines": { @@ -20,6 +21,7 @@ "devDependencies": { "@nuxtjs/tailwindcss": "^6.14.0", "@pinia/nuxt": "^0.11.2", + "@playwright/test": "^1.63.0", "@types/canvas-confetti": "^1.9.0", "@types/node": "^22.20.2", "@vueuse/core": "^13.9.0", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..507566e --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,48 @@ +import { defineConfig, devices } from '@playwright/test' + +// End-to-end tests run against a production build (`nuxt build`, then the +// Nitro server), not `nuxt dev`: no dependency pre-bundling reloads, and the +// same output that gets deployed. See tests/e2e/README.md. +const PORT = Number(process.env.E2E_PORT ?? 3179) +const baseURL = `http://127.0.0.1:${PORT}` +const CI = !!process.env.CI + +export default defineConfig({ + testDir: './tests/e2e', + outputDir: './tests/e2e/test-results', + fullyParallel: true, + forbidOnly: CI, + retries: CI ? 1 : 0, + timeout: 60_000, + expect: { timeout: 10_000 }, + reporter: [ + [CI ? 'github' : 'list'], + ['html', { open: 'never', outputFolder: './tests/e2e/playwright-report' }], + ], + use: { + baseURL, + actionTimeout: 10_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + webServer: { + command: 'npm run build && node .output/server/index.mjs', + url: baseURL, + reuseExistingServer: !CI, + timeout: 180_000, + env: { + NITRO_HOST: '127.0.0.1', + NITRO_PORT: String(PORT), + NUXT_TELEMETRY_DISABLED: '1', + // The tests answer /api/chat in the browser. Blank provider keys mean a + // request that ever reaches this server fails with 401 instead of + // calling a real model with a key from the shell. + OPENAI_API_KEY: '', + ANTHROPIC_API_KEY: '', + GEMINI_API_KEY: '', + GOOGLE_GENERATIVE_AI_API_KEY: '', + DEEPSEEK_API_KEY: '', + }, + }, +}) diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 0000000..aaa9103 --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,2 @@ +test-results/ +playwright-report/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..8b0b281 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,92 @@ +--- +type: playbook +title: End-to-end tests +description: How to run the Playwright test of the core loop, and how its stubbed agent works. +tags: [tests, e2e, playwright, ci] +generated: { by: claude-code/claude-opus-5.5, at: 2026-10-03T00:34:00Z } +--- + +# End-to-end tests + +A Playwright test of the core loop from [NEXT.md](../../NEXT.md) step +0.1: install a fixture app, reload, update it, break it, and ask the +agent to fix it. NEXT.md makes this suite the definition of "still +works", and CI runs it on pull requests and on pushes to `main`. + +## Run it + +```bash +npm ci +npx playwright install chromium # once per Playwright version +npm run test:e2e +``` + +`test:e2e` type-checks `tests/e2e` and then runs Playwright, which builds +the app (`nuxt build`) and serves the production output on +`http://127.0.0.1:3179`. Set `E2E_PORT` to use another port. Outside CI, +a server that is already listening on that port is reused instead of +rebuilt, so stop any server you started by hand after changing app code. + +Other ways to run it: + +```bash +npm run test:e2e -- --headed # watch it in a browser window +npm run test:e2e -- --ui # Playwright UI mode +npx playwright show-report tests/e2e/playwright-report +``` + +Failure artifacts (traces, screenshots, error context) go to +`tests/e2e/test-results/` and the HTML report to +`tests/e2e/playwright-report/`. Both are gitignored. In CI they are +uploaded as the `playwright-report` artifact when the job fails. + +## What it covers + +| Step | How | Asserts | +| --- | --- | --- | +| Install | `window.__dominic.installFixture('pomodoro-timer')` | the timer renders | +| Persist | reload, then the taskbar's **Open Pomodoro Timer** | the app comes back from the virtual filesystem | +| Update | `window.__dominic.updateApp(...)` | the new source renders in the same window | +| Break | `window.__dominic.injectBrokenApp('compile')` | the error card and **Ask Agent to Fix** appear | +| Recover | click **Ask Agent to Fix** | the request carries the app's id and source; the `update_app` reply mounts a working app in place | + +A second test runs Break and Recover with `injectBrokenApp('runtime')`, +which goes through the runner's `onErrorCaptured` boundary instead of +the compile path. + +## The stubbed agent + +No model provider or API key is involved, locally or in CI. +`support/agent-stub.ts` answers `POST /api/chat` inside the browser with +`page.route`, in the AI SDK v4 data-stream format that the server's +`toDataStreamResponse()` sends: + +```text +f:{"messageId":"msg-e2e"} +0:"Found the bug; sending a fixed version." +9:{"toolCallId":"call-e2e","toolName":"update_app","args":{...}} +e:{"finishReason":"tool-calls","usage":{...},"isContinued":false} +d:{"finishReason":"tool-calls","usage":{...}} +``` + +Every line ends in a newline, because the client parser in +`useAgentChat.ts` ignores a last line that has none. The lines are +built with `formatDataStreamPart` from the `ai` package, so they follow +the SDK version the server uses. If the chat route changes its stream +format, change the stub in the same pull request. + +Two guards keep the suite self-contained: + +- Every request that leaves the app's origin is aborted. Today that is + only the Tailwind Play CDN script; the shell's own styles are built in. +- The test server starts with blank provider keys, so a request that + ever got past the stub fails with 401 instead of calling a real model. + +## Adding a test + +- Start with `openDesktop(page)` and drive the desktop through + `dominicHooks(page)` and visible controls. +- Find windows with `appWindow(page, title)`. +- Use web-first assertions (`await expect(locator)...`), not sleeps. +- Each test gets a fresh browser context, so the virtual filesystem + (`localStorage`) starts empty. diff --git a/tests/e2e/core-loop.spec.ts b/tests/e2e/core-loop.spec.ts new file mode 100644 index 0000000..52b97a3 --- /dev/null +++ b/tests/e2e/core-loop.spec.ts @@ -0,0 +1,121 @@ +import { expect, test, type Locator, type Page } from '@playwright/test' +import { stubAgentReply } from './support/agent-stub' +import { + appWindow, + dominicHooks, + keepRequestsLocal, + openDesktop, + waitForDesktop, +} from './support/desktop' + +// The core loop (NEXT.md step 0.1): installFixture -> reload -> updateApp -> +// injectBrokenApp -> Ask Agent to Fix. The agent is stubbed at the network +// layer, so no provider or API key is needed. + +const UPDATED_POMODORO = ` +` + +// The fix the stubbed agent sends back. The heading text comes from the +// script, so seeing it means the template and the script both compiled and ran. +const REPAIRED_APP = ` + + +` + +test.beforeEach(async ({ page, baseURL }) => { + await keepRequestsLocal(page, baseURL) +}) + +/** + * Click "Ask Agent to Fix" with the agent stubbed to answer with an + * `update_app` call, and expect the broken window to recover in place. + */ +async function askAgentToFix(page: Page, brokenWindow: Locator, expectInRequest: string[]) { + const requests = await stubAgentReply(page, { + text: 'Found the bug; sending a fixed version.', + updateApp: { id: 'broken-app', vueSfcCode: REPAIRED_APP, summary: 'Fix the deliberate error' }, + }) + + await brokenWindow.getByRole('button', { name: 'Ask Agent to Fix' }).click() + + await expect(brokenWindow.getByRole('heading', { name: 'Repaired by the agent' })).toBeVisible() + await expect(brokenWindow.getByRole('button', { name: 'Ask Agent to Fix' })).toHaveCount(0) + await expect( + appWindow(page, 'Agent Chat').getByText('Found the bug; sending a fixed version.'), + ).toBeVisible() + + // One request, carrying what the agent needs: the app id, the error, and + // the app's current source. + expect(requests).toHaveLength(1) + const fixRequest = requests[0]!.messages.at(-1)! + expect(fixRequest.role).toBe('user') + for (const text of expectInRequest) { + expect(fixRequest.content).toContain(text) + } +} + +test('core loop: install, reload, update, break, ask the agent to fix', async ({ page }) => { + const dominic = dominicHooks(page) + const pomodoro = appWindow(page, 'Pomodoro Timer') + const broken = appWindow(page, 'Broken (Syntax Error)') + await openDesktop(page) + + await test.step('installFixture installs the app and it renders', async () => { + expect(await dominic.installFixture('pomodoro-timer')).toBe(true) + await expect(pomodoro.getByText('25:00')).toBeVisible() + }) + + await test.step('after a reload the app is still installed', async () => { + await page.reload() + await waitForDesktop(page) + await page.getByRole('button', { name: 'Open Pomodoro Timer' }).click() + await expect(pomodoro.getByText('25:00')).toBeVisible() + }) + + await test.step('updateApp changes the running app in place', async () => { + await dominic.updateApp({ + id: 'pomodoro-timer', + vueSfcCode: UPDATED_POMODORO, + summary: 'Ten-minute breaks', + }) + await expect(pomodoro.getByText('Breaks are now ten minutes.')).toBeVisible() + await expect(pomodoro.getByText('25:00')).toHaveCount(0) + }) + + await test.step('injectBrokenApp shows the error card', async () => { + await dominic.injectBrokenApp('compile') + await expect(broken.getByText('Component Compilation Error')).toBeVisible() + await expect(broken.getByRole('button', { name: 'Ask Agent to Fix' })).toBeVisible() + }) + + await test.step('Ask Agent to Fix repairs the app through update_app', async () => { + await askAgentToFix(page, broken, ['broken-app', 'Syntax Error Component', 'Unexpected token']) + }) +}) + +test('a runtime crash is caught and repaired the same way', async ({ page }) => { + const dominic = dominicHooks(page) + const broken = appWindow(page, 'Broken (Runtime Crash)') + await openDesktop(page) + + await dominic.injectBrokenApp('runtime') + await expect(broken.getByText('Application Runtime Error')).toBeVisible() + + await askAgentToFix(page, broken, [ + 'broken-app', + 'Runtime Crash Component', + 'Simulated runtime exception inside onMounted()', + ]) +}) diff --git a/tests/e2e/support/agent-stub.ts b/tests/e2e/support/agent-stub.ts new file mode 100644 index 0000000..bb37d32 --- /dev/null +++ b/tests/e2e/support/agent-stub.ts @@ -0,0 +1,60 @@ +import type { Page } from '@playwright/test' +import { formatDataStreamPart } from 'ai' +import type { UpdateAppParams } from '../../../app/features/chat/tools/schemas' + +/** The parts of the `/api/chat` request body the tests check. */ +export interface ChatRequestBody { + messages: { role: string; content: string }[] + installedApps: { id: string }[] +} + +export interface AgentReply { + /** Assistant text, sent as a `0:` part before the tool call. */ + text: string + /** Arguments of the single `update_app` tool call (a `9:` part). */ + updateApp: UpdateAppParams +} + +const USAGE = { promptTokens: 1, completionTokens: 1 } + +/** + * Answer `POST /api/chat` in the browser, so no provider or API key is + * involved. The body is what the server's `toDataStreamResponse()` sends for + * a text-then-tool-call turn, encoded with the same AI SDK the server uses: + * + * f:{"messageId":…} 0:"text" 9:{toolCallId,toolName,args} e:{…} d:{…} + * + * one part per line, each ending in "\n" (the client parser drops a final + * line that has no newline). Returns the request bodies, in order. + */ +export async function stubAgentReply(page: Page, reply: AgentReply): Promise { + const requests: ChatRequestBody[] = [] + + await page.route('**/api/chat', async (route) => { + requests.push(route.request().postDataJSON() as ChatRequestBody) + await route.fulfill({ + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'X-Vercel-AI-Data-Stream': 'v1', + }, + body: [ + formatDataStreamPart('start_step', { messageId: 'msg-e2e' }), + formatDataStreamPart('text', reply.text), + formatDataStreamPart('tool_call', { + toolCallId: 'call-e2e', + toolName: 'update_app', + args: reply.updateApp, + }), + formatDataStreamPart('finish_step', { + finishReason: 'tool-calls', + usage: USAGE, + isContinued: false, + }), + formatDataStreamPart('finish_message', { finishReason: 'tool-calls', usage: USAGE }), + ].join(''), + }) + }) + + return requests +} diff --git a/tests/e2e/support/desktop.ts b/tests/e2e/support/desktop.ts new file mode 100644 index 0000000..4e12544 --- /dev/null +++ b/tests/e2e/support/desktop.ts @@ -0,0 +1,59 @@ +import type { Locator, Page } from '@playwright/test' +import type { UpdateAppParams } from '../../../app/features/chat/tools/schemas' + +/** The test hooks `app/app.vue` puts on `window.__dominic` once it mounts. */ +interface DominicHooks { + installFixture(fixtureId: string): boolean + updateApp(params: UpdateAppParams): void + injectBrokenApp(type?: 'compile' | 'runtime'): void +} + +declare global { + interface Window { + __dominic?: DominicHooks + } +} + +/** + * Keep every request on the app's own origin. The page loads the Tailwind + * Play CDN from its ``; the shell's classes are compiled at build time, + * so the suite runs offline and no third-party host can make it flaky. + */ +export async function keepRequestsLocal(page: Page, baseURL: string | undefined): Promise { + if (!baseURL) throw new Error('Set use.baseURL in playwright.config.ts') + const { origin } = new URL(baseURL) + await page.route( + (url) => url.origin !== origin, + (route) => route.abort('blockedbyclient'), + ) +} + +/** Wait until the desktop has mounted and its test hooks exist. */ +export async function waitForDesktop(page: Page): Promise { + await page.waitForFunction(() => window.__dominic !== undefined) +} + +/** Open the desktop and wait for it to mount. */ +export async function openDesktop(page: Page): Promise { + await page.goto('/') + await waitForDesktop(page) +} + +/** Call the `window.__dominic` hooks from the test. */ +export function dominicHooks(page: Page) { + return { + installFixture: (fixtureId: string) => + page.evaluate((id) => window.__dominic!.installFixture(id), fixtureId), + updateApp: (params: UpdateAppParams) => + page.evaluate((p) => window.__dominic!.updateApp(p), params), + injectBrokenApp: (type: 'compile' | 'runtime') => + page.evaluate((t) => window.__dominic!.injectBrokenApp(t), type), + } +} + +/** A desktop window, found by the exact title in its title bar. */ +export function appWindow(page: Page, title: string): Locator { + return page + .locator('section') + .filter({ has: page.locator('header').getByText(title, { exact: true }) }) +} diff --git a/tests/e2e/tsconfig.json b/tests/e2e/tsconfig.json new file mode 100644 index 0000000..0bfb156 --- /dev/null +++ b/tests/e2e/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2023", "DOM"], + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "forceConsistentCasingInFileNames": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["./**/*.ts", "../../playwright.config.ts"] +}