diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b96e14b..17f2d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,38 @@ jobs: - name: Run build run: npm run build + + # Playwright end-to-end test of the core loop (NEXT.md step 0.1, #79). + # The chat route is stubbed in the browser, so no provider key is needed. + e2e: + name: E2E (Playwright) + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + NUXT_TELEMETRY_DISABLED: 1 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Run e2e tests + run: npm run test:e2e + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: | + tests/e2e/playwright-report/ + tests/e2e/test-results/ + retention-days: 7 diff --git a/package-lock.json b/package-lock.json index 41cc0a0..75e5662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "dominic", "version": "0.0.0", + "license": "AGPL-3.0-or-later", "dependencies": { "@ai-sdk/anthropic": "^1.2.12", "@ai-sdk/google": "^1.2.22", @@ -20,6 +21,7 @@ "devDependencies": { "@nuxtjs/tailwindcss": "^6.14.0", "@pinia/nuxt": "^0.11.2", + "@playwright/test": "^1.63.0", "@types/canvas-confetti": "^1.9.0", "@types/node": "^22.20.2", "@vueuse/core": "^13.9.0", @@ -31,6 +33,9 @@ "typescript": "^5.9.3", "vue": "^3.5.22", "vue-tsc": "^3.1.3" + }, + "engines": { + "node": ">=24.21.0 <25" } }, "node_modules/@ai-sdk/anthropic": { @@ -2372,6 +2377,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -10073,6 +10094,35 @@ "dev": true, "license": "MIT" }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/portfinder": { "version": "1.0.38", "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.38.tgz", diff --git a/package.json b/package.json index 9f9b635..0f576b6 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "generate": "nuxt generate", "preview": "nuxt preview", "typecheck": "nuxt typecheck", - "lint:md": "markdownlint-cli2 \"**/*.md\" \"#node_modules\" \"#.superpowers\" \"#docs/superpowers\"", + "test:e2e": "tsc -p tests/e2e && playwright test", + "lint:md": "markdownlint-cli2 \"**/*.md\" \"#node_modules\" \"#.superpowers\" \"#docs/superpowers\" \"#tests/e2e/test-results\" \"#tests/e2e/playwright-report\"", "prepare": "husky" }, "engines": { @@ -20,6 +21,7 @@ "devDependencies": { "@nuxtjs/tailwindcss": "^6.14.0", "@pinia/nuxt": "^0.11.2", + "@playwright/test": "^1.63.0", "@types/canvas-confetti": "^1.9.0", "@types/node": "^22.20.2", "@vueuse/core": "^13.9.0", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..507566e --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,48 @@ +import { defineConfig, devices } from '@playwright/test' + +// End-to-end tests run against a production build (`nuxt build`, then the +// Nitro server), not `nuxt dev`: no dependency pre-bundling reloads, and the +// same output that gets deployed. See tests/e2e/README.md. +const PORT = Number(process.env.E2E_PORT ?? 3179) +const baseURL = `http://127.0.0.1:${PORT}` +const CI = !!process.env.CI + +export default defineConfig({ + testDir: './tests/e2e', + outputDir: './tests/e2e/test-results', + fullyParallel: true, + forbidOnly: CI, + retries: CI ? 1 : 0, + timeout: 60_000, + expect: { timeout: 10_000 }, + reporter: [ + [CI ? 'github' : 'list'], + ['html', { open: 'never', outputFolder: './tests/e2e/playwright-report' }], + ], + use: { + baseURL, + actionTimeout: 10_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + webServer: { + command: 'npm run build && node .output/server/index.mjs', + url: baseURL, + reuseExistingServer: !CI, + timeout: 180_000, + env: { + NITRO_HOST: '127.0.0.1', + NITRO_PORT: String(PORT), + NUXT_TELEMETRY_DISABLED: '1', + // The tests answer /api/chat in the browser. Blank provider keys mean a + // request that ever reaches this server fails with 401 instead of + // calling a real model with a key from the shell. + OPENAI_API_KEY: '', + ANTHROPIC_API_KEY: '', + GEMINI_API_KEY: '', + GOOGLE_GENERATIVE_AI_API_KEY: '', + DEEPSEEK_API_KEY: '', + }, + }, +}) diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 0000000..aaa9103 --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,2 @@ +test-results/ +playwright-report/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..8b0b281 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,92 @@ +--- +type: playbook +title: End-to-end tests +description: How to run the Playwright test of the core loop, and how its stubbed agent works. +tags: [tests, e2e, playwright, ci] +generated: { by: claude-code/claude-opus-5.5, at: 2026-10-03T00:34:00Z } +--- + +# End-to-end tests + +A Playwright test of the core loop from [NEXT.md](../../NEXT.md) step +0.1: install a fixture app, reload, update it, break it, and ask the +agent to fix it. NEXT.md makes this suite the definition of "still +works", and CI runs it on pull requests and on pushes to `main`. + +## Run it + +```bash +npm ci +npx playwright install chromium # once per Playwright version +npm run test:e2e +``` + +`test:e2e` type-checks `tests/e2e` and then runs Playwright, which builds +the app (`nuxt build`) and serves the production output on +`http://127.0.0.1:3179`. Set `E2E_PORT` to use another port. Outside CI, +a server that is already listening on that port is reused instead of +rebuilt, so stop any server you started by hand after changing app code. + +Other ways to run it: + +```bash +npm run test:e2e -- --headed # watch it in a browser window +npm run test:e2e -- --ui # Playwright UI mode +npx playwright show-report tests/e2e/playwright-report +``` + +Failure artifacts (traces, screenshots, error context) go to +`tests/e2e/test-results/` and the HTML report to +`tests/e2e/playwright-report/`. Both are gitignored. In CI they are +uploaded as the `playwright-report` artifact when the job fails. + +## What it covers + +| Step | How | Asserts | +| --- | --- | --- | +| Install | `window.__dominic.installFixture('pomodoro-timer')` | the timer renders | +| Persist | reload, then the taskbar's **Open Pomodoro Timer** | the app comes back from the virtual filesystem | +| Update | `window.__dominic.updateApp(...)` | the new source renders in the same window | +| Break | `window.__dominic.injectBrokenApp('compile')` | the error card and **Ask Agent to Fix** appear | +| Recover | click **Ask Agent to Fix** | the request carries the app's id and source; the `update_app` reply mounts a working app in place | + +A second test runs Break and Recover with `injectBrokenApp('runtime')`, +which goes through the runner's `onErrorCaptured` boundary instead of +the compile path. + +## The stubbed agent + +No model provider or API key is involved, locally or in CI. +`support/agent-stub.ts` answers `POST /api/chat` inside the browser with +`page.route`, in the AI SDK v4 data-stream format that the server's +`toDataStreamResponse()` sends: + +```text +f:{"messageId":"msg-e2e"} +0:"Found the bug; sending a fixed version." +9:{"toolCallId":"call-e2e","toolName":"update_app","args":{...}} +e:{"finishReason":"tool-calls","usage":{...},"isContinued":false} +d:{"finishReason":"tool-calls","usage":{...}} +``` + +Every line ends in a newline, because the client parser in +`useAgentChat.ts` ignores a last line that has none. The lines are +built with `formatDataStreamPart` from the `ai` package, so they follow +the SDK version the server uses. If the chat route changes its stream +format, change the stub in the same pull request. + +Two guards keep the suite self-contained: + +- Every request that leaves the app's origin is aborted. Today that is + only the Tailwind Play CDN script; the shell's own styles are built in. +- The test server starts with blank provider keys, so a request that + ever got past the stub fails with 401 instead of calling a real model. + +## Adding a test + +- Start with `openDesktop(page)` and drive the desktop through + `dominicHooks(page)` and visible controls. +- Find windows with `appWindow(page, title)`. +- Use web-first assertions (`await expect(locator)...`), not sleeps. +- Each test gets a fresh browser context, so the virtual filesystem + (`localStorage`) starts empty. diff --git a/tests/e2e/core-loop.spec.ts b/tests/e2e/core-loop.spec.ts new file mode 100644 index 0000000..52b97a3 --- /dev/null +++ b/tests/e2e/core-loop.spec.ts @@ -0,0 +1,121 @@ +import { expect, test, type Locator, type Page } from '@playwright/test' +import { stubAgentReply } from './support/agent-stub' +import { + appWindow, + dominicHooks, + keepRequestsLocal, + openDesktop, + waitForDesktop, +} from './support/desktop' + +// The core loop (NEXT.md step 0.1): installFixture -> reload -> updateApp -> +// injectBrokenApp -> Ask Agent to Fix. The agent is stubbed at the network +// layer, so no provider or API key is needed. + +const UPDATED_POMODORO = ` +` + +// The fix the stubbed agent sends back. The heading text comes from the +// script, so seeing it means the template and the script both compiled and ran. +const REPAIRED_APP = ` + + +` + +test.beforeEach(async ({ page, baseURL }) => { + await keepRequestsLocal(page, baseURL) +}) + +/** + * Click "Ask Agent to Fix" with the agent stubbed to answer with an + * `update_app` call, and expect the broken window to recover in place. + */ +async function askAgentToFix(page: Page, brokenWindow: Locator, expectInRequest: string[]) { + const requests = await stubAgentReply(page, { + text: 'Found the bug; sending a fixed version.', + updateApp: { id: 'broken-app', vueSfcCode: REPAIRED_APP, summary: 'Fix the deliberate error' }, + }) + + await brokenWindow.getByRole('button', { name: 'Ask Agent to Fix' }).click() + + await expect(brokenWindow.getByRole('heading', { name: 'Repaired by the agent' })).toBeVisible() + await expect(brokenWindow.getByRole('button', { name: 'Ask Agent to Fix' })).toHaveCount(0) + await expect( + appWindow(page, 'Agent Chat').getByText('Found the bug; sending a fixed version.'), + ).toBeVisible() + + // One request, carrying what the agent needs: the app id, the error, and + // the app's current source. + expect(requests).toHaveLength(1) + const fixRequest = requests[0]!.messages.at(-1)! + expect(fixRequest.role).toBe('user') + for (const text of expectInRequest) { + expect(fixRequest.content).toContain(text) + } +} + +test('core loop: install, reload, update, break, ask the agent to fix', async ({ page }) => { + const dominic = dominicHooks(page) + const pomodoro = appWindow(page, 'Pomodoro Timer') + const broken = appWindow(page, 'Broken (Syntax Error)') + await openDesktop(page) + + await test.step('installFixture installs the app and it renders', async () => { + expect(await dominic.installFixture('pomodoro-timer')).toBe(true) + await expect(pomodoro.getByText('25:00')).toBeVisible() + }) + + await test.step('after a reload the app is still installed', async () => { + await page.reload() + await waitForDesktop(page) + await page.getByRole('button', { name: 'Open Pomodoro Timer' }).click() + await expect(pomodoro.getByText('25:00')).toBeVisible() + }) + + await test.step('updateApp changes the running app in place', async () => { + await dominic.updateApp({ + id: 'pomodoro-timer', + vueSfcCode: UPDATED_POMODORO, + summary: 'Ten-minute breaks', + }) + await expect(pomodoro.getByText('Breaks are now ten minutes.')).toBeVisible() + await expect(pomodoro.getByText('25:00')).toHaveCount(0) + }) + + await test.step('injectBrokenApp shows the error card', async () => { + await dominic.injectBrokenApp('compile') + await expect(broken.getByText('Component Compilation Error')).toBeVisible() + await expect(broken.getByRole('button', { name: 'Ask Agent to Fix' })).toBeVisible() + }) + + await test.step('Ask Agent to Fix repairs the app through update_app', async () => { + await askAgentToFix(page, broken, ['broken-app', 'Syntax Error Component', 'Unexpected token']) + }) +}) + +test('a runtime crash is caught and repaired the same way', async ({ page }) => { + const dominic = dominicHooks(page) + const broken = appWindow(page, 'Broken (Runtime Crash)') + await openDesktop(page) + + await dominic.injectBrokenApp('runtime') + await expect(broken.getByText('Application Runtime Error')).toBeVisible() + + await askAgentToFix(page, broken, [ + 'broken-app', + 'Runtime Crash Component', + 'Simulated runtime exception inside onMounted()', + ]) +}) diff --git a/tests/e2e/support/agent-stub.ts b/tests/e2e/support/agent-stub.ts new file mode 100644 index 0000000..bb37d32 --- /dev/null +++ b/tests/e2e/support/agent-stub.ts @@ -0,0 +1,60 @@ +import type { Page } from '@playwright/test' +import { formatDataStreamPart } from 'ai' +import type { UpdateAppParams } from '../../../app/features/chat/tools/schemas' + +/** The parts of the `/api/chat` request body the tests check. */ +export interface ChatRequestBody { + messages: { role: string; content: string }[] + installedApps: { id: string }[] +} + +export interface AgentReply { + /** Assistant text, sent as a `0:` part before the tool call. */ + text: string + /** Arguments of the single `update_app` tool call (a `9:` part). */ + updateApp: UpdateAppParams +} + +const USAGE = { promptTokens: 1, completionTokens: 1 } + +/** + * Answer `POST /api/chat` in the browser, so no provider or API key is + * involved. The body is what the server's `toDataStreamResponse()` sends for + * a text-then-tool-call turn, encoded with the same AI SDK the server uses: + * + * f:{"messageId":…} 0:"text" 9:{toolCallId,toolName,args} e:{…} d:{…} + * + * one part per line, each ending in "\n" (the client parser drops a final + * line that has no newline). Returns the request bodies, in order. + */ +export async function stubAgentReply(page: Page, reply: AgentReply): Promise { + const requests: ChatRequestBody[] = [] + + await page.route('**/api/chat', async (route) => { + requests.push(route.request().postDataJSON() as ChatRequestBody) + await route.fulfill({ + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'X-Vercel-AI-Data-Stream': 'v1', + }, + body: [ + formatDataStreamPart('start_step', { messageId: 'msg-e2e' }), + formatDataStreamPart('text', reply.text), + formatDataStreamPart('tool_call', { + toolCallId: 'call-e2e', + toolName: 'update_app', + args: reply.updateApp, + }), + formatDataStreamPart('finish_step', { + finishReason: 'tool-calls', + usage: USAGE, + isContinued: false, + }), + formatDataStreamPart('finish_message', { finishReason: 'tool-calls', usage: USAGE }), + ].join(''), + }) + }) + + return requests +} diff --git a/tests/e2e/support/desktop.ts b/tests/e2e/support/desktop.ts new file mode 100644 index 0000000..4e12544 --- /dev/null +++ b/tests/e2e/support/desktop.ts @@ -0,0 +1,59 @@ +import type { Locator, Page } from '@playwright/test' +import type { UpdateAppParams } from '../../../app/features/chat/tools/schemas' + +/** The test hooks `app/app.vue` puts on `window.__dominic` once it mounts. */ +interface DominicHooks { + installFixture(fixtureId: string): boolean + updateApp(params: UpdateAppParams): void + injectBrokenApp(type?: 'compile' | 'runtime'): void +} + +declare global { + interface Window { + __dominic?: DominicHooks + } +} + +/** + * Keep every request on the app's own origin. The page loads the Tailwind + * Play CDN from its ``; the shell's classes are compiled at build time, + * so the suite runs offline and no third-party host can make it flaky. + */ +export async function keepRequestsLocal(page: Page, baseURL: string | undefined): Promise { + if (!baseURL) throw new Error('Set use.baseURL in playwright.config.ts') + const { origin } = new URL(baseURL) + await page.route( + (url) => url.origin !== origin, + (route) => route.abort('blockedbyclient'), + ) +} + +/** Wait until the desktop has mounted and its test hooks exist. */ +export async function waitForDesktop(page: Page): Promise { + await page.waitForFunction(() => window.__dominic !== undefined) +} + +/** Open the desktop and wait for it to mount. */ +export async function openDesktop(page: Page): Promise { + await page.goto('/') + await waitForDesktop(page) +} + +/** Call the `window.__dominic` hooks from the test. */ +export function dominicHooks(page: Page) { + return { + installFixture: (fixtureId: string) => + page.evaluate((id) => window.__dominic!.installFixture(id), fixtureId), + updateApp: (params: UpdateAppParams) => + page.evaluate((p) => window.__dominic!.updateApp(p), params), + injectBrokenApp: (type: 'compile' | 'runtime') => + page.evaluate((t) => window.__dominic!.injectBrokenApp(t), type), + } +} + +/** A desktop window, found by the exact title in its title bar. */ +export function appWindow(page: Page, title: string): Locator { + return page + .locator('section') + .filter({ has: page.locator('header').getByText(title, { exact: true }) }) +} diff --git a/tests/e2e/tsconfig.json b/tests/e2e/tsconfig.json new file mode 100644 index 0000000..0bfb156 --- /dev/null +++ b/tests/e2e/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2023", "DOM"], + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "forceConsistentCasingInFileNames": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["./**/*.ts", "../../playwright.config.ts"] +}