diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b8b363dd2b..d1a4a7d986 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -133,10 +133,8 @@ jobs: with: version: 2026.4.27 cache_key_prefix: mise-ci-${{ github.job }} - - name: Install dependencies - run: mise run playground:install - name: Test - run: pnpm --dir playground exec vitest run --passWithNoTests + run: mise run test:playground playground-check: name: Check playground @@ -635,6 +633,65 @@ jobs: - name: Test coglet-python bindings run: uvx nox -s coglet -p ${{ matrix.python-version }} + test-playground-e2e: + name: Playground Playwright integration tests (Chromium) + needs: + - playground-changes + - fmt-playground + - lint-playground + - test-playground + - playground-check + - build-cog + - build-sdk + - build-rust + if: needs.playground-changes.outputs.changed == 'true' + runs-on: ubuntu-latest-16-cores + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Login to Docker Hub + uses: docker/login-action@v4 + if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' + with: + registry: index.docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Download artifacts + uses: actions/download-artifact@v8 + with: + path: dist + merge-multiple: true + - name: Install cog binary + run: | + cp dist/cog ./cog + chmod +x ./cog + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Install playground dependencies and Chromium + run: | + mise run playground:install + pnpm --dir playground exec playwright install --with-deps chromium + - name: Run browser integration tests + env: + BUILDKIT_PROGRESS: quiet + COG_BINARY: ${{ github.workspace }}/cog + COG_REGISTRY_HOST: ghcr.io/replicate/cog + COG_SDK_WHEEL: ${{ github.workspace }}/dist + COGLET_WHEEL: ${{ github.workspace }}/dist + run: pnpm --dir playground run test:e2e + - name: Upload browser test report + if: failure() + uses: actions/upload-artifact@v6 + with: + name: PlaygroundPlaywrightReport + path: | + playground/playwright-report + playground/test-results + # Compute integration test shards dynamically. # Slow tests (tagged with [short] skip) are distributed round-robin first, # then remaining tests fill in. This ensures slow tests don't pile up on one runner. @@ -824,6 +881,7 @@ jobs: - test-rust - test-python - test-coglet-python + - test-playground-e2e - integration-shards - test-integration if: always() @@ -858,6 +916,7 @@ jobs: echo " test-rust: ${{ needs.test-rust.result }}" echo " test-python: ${{ needs.test-python.result }}" echo " test-coglet-python: ${{ needs.test-coglet-python.result }}" + echo " test-playground-e2e: ${{ needs.test-playground-e2e.result }}" echo " integration-shards: ${{ needs.integration-shards.result }}" echo " test-integration: ${{ needs.test-integration.result }}" @@ -898,6 +957,15 @@ jobs: check_success integration-shards "${{ needs.integration-shards.result }}" check_success test-integration "${{ needs.test-integration.result }}" + if [ "${{ needs.playground-changes.outputs.changed }}" = "true" ]; then + check_success test-playground-e2e "${{ needs.test-playground-e2e.result }}" + else + if [ "${{ needs.test-playground-e2e.result }}" != "skipped" ]; then + echo "::error::Playground integration tests should be skipped when playground sources are unchanged" + FAILED=true + fi + fi + if [ "$FAILED" = "true" ]; then echo "::error::Some jobs failed or were cancelled" exit 1 diff --git a/pkg/cli/playground_test.go b/pkg/cli/playground_test.go index 464fb059c2..2af42ab37f 100644 --- a/pkg/cli/playground_test.go +++ b/pkg/cli/playground_test.go @@ -25,8 +25,6 @@ import ( "github.com/replicate/cog/pkg/global" ) -var _ = newPlaygroundCommand - func newTestPlayground(t *testing.T) *httptest.Server { t.Helper() uiFS, err := fs.Sub(playgroundUI, "playground") diff --git a/playground/e2e/fixture/cog.yaml b/playground/e2e/fixture/cog.yaml new file mode 100644 index 0000000000..e19c4aea46 --- /dev/null +++ b/playground/e2e/fixture/cog.yaml @@ -0,0 +1,3 @@ +build: + python_version: "3.12" +run: "run.py:Runner" diff --git a/playground/e2e/fixture/run.py b/playground/e2e/fixture/run.py new file mode 100644 index 0000000000..fa5761b2cd --- /dev/null +++ b/playground/e2e/fixture/run.py @@ -0,0 +1,20 @@ +import time + +from cog import BaseRunner, ConcatenateIterator, Input, streaming + + +class Runner(BaseRunner): + @streaming + def run( + self, + text: str = Input( + description="Text to prefix with 'hello '", + min_length=2, + max_length=20, + regex=r"^[a-z ]+$", + ), + ) -> ConcatenateIterator[str]: + yield "hello " + if text == "slow": + time.sleep(2) + yield text diff --git a/playground/e2e/playground.spec.ts b/playground/e2e/playground.spec.ts new file mode 100644 index 0000000000..84429a46c2 --- /dev/null +++ b/playground/e2e/playground.spec.ts @@ -0,0 +1,352 @@ +import { expect, test, type Page } from "@playwright/test"; +import { createServer } from "node:http"; + +test.beforeEach(async ({ page }) => { + await page.goto("/"); + await expect(page.getByRole("heading", { name: "Cog Playground" })).toBeVisible(); + await expect(page.getByRole("textbox", { name: "text" })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible({ + timeout: 30_000, + }); +}); + +test("keeps Form and CodeMirror JSON input synchronized", async ({ page }) => { + const run = page.getByRole("button", { name: "Run" }); + await expect(run).toBeEnabled(); + await page.getByRole("textbox", { name: "text" }).fill("from form"); + await page.getByRole("tab", { name: "JSON" }).click(); + + const editor = page.getByRole("textbox", { name: "Prediction input JSON" }); + await expect(editor).toContainText('"text": "from form"'); + await editor.click(); + await page.keyboard.press("Tab"); + await expect(page.getByRole("button", { name: "Copy" })).toBeFocused(); + await replaceEditor(page, "Prediction input JSON", '{"text":"from json"}'); + await page.getByRole("button", { name: "Format" }).click(); + await expect(editor).toContainText('"text": "from json"'); + await expect(run).toBeEnabled(); + + await replaceEditor(page, "Prediction input JSON", "{"); + await expect(run).toBeEnabled(); + await page.getByRole("tab", { name: "Form" }).click(); + await expect(page.getByRole("textbox", { name: "text" })).toHaveValue("from json"); + + await page.getByRole("tab", { name: "JSON" }).click(); + await expect(page.getByRole("textbox", { name: "Prediction input JSON" })).toContainText( + '"text": "from json"', + ); + await expect(run).toBeEnabled(); +}); + +test("folds JSON objects in CodeMirror", async ({ page }) => { + await page.getByRole("tab", { name: "JSON" }).click(); + await replaceEditor( + page, + "Prediction input JSON", + '{\n "text": "folded",\n "metadata": {\n "count": 1\n }\n}', + ); + + const fold = page.locator('.json-input .cm-foldGutter [title="Fold line"]').first(); + await expect(fold).toBeVisible(); + await fold.click(); + await expect(page.locator(".json-input .cm-foldPlaceholder")).toBeVisible(); +}); + +test("validates Form and JSON input against OpenAPI before running", async ({ page }) => { + const run = page.getByRole("button", { name: "Run" }); + const text = page.getByRole("textbox", { name: "text" }); + await text.fill("123"); + await expect(run).toBeEnabled(); + await expect(text).not.toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Does not match the required pattern.")).toHaveCount(0); + + await run.click(); + await expect(text).toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Does not match the required pattern.").first()).toBeVisible(); + + await text.fill("valid input"); + await expect(run).toBeEnabled(); + await expect(text).not.toHaveAttribute("aria-invalid", "true"); + + await page.getByRole("tab", { name: "JSON" }).click(); + const editor = page.getByRole("textbox", { name: "Prediction input JSON" }); + await replaceEditor(page, "Prediction input JSON", '{"text":42}'); + await expect(run).toBeEnabled(); + await expect(editor).not.toHaveAttribute("aria-invalid", "true"); + + await run.click(); + await expect(editor).toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Input does not match the OpenAPI schema.")).toBeVisible(); + + await replaceEditor(page, "Prediction input JSON", '{"text":"valid"}'); + await expect(run).toBeEnabled(); + await expect(editor).not.toHaveAttribute("aria-invalid", "true"); +}); + +test("runs a synchronous prediction and inspects its response", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("sync"); + await page.getByRole("button", { name: "Sync", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello sync"); + + await page.getByRole("tab", { name: "Raw", exact: true }).click(); + const raw = page.getByRole("textbox", { name: "Raw prediction response" }); + await expect(raw).toHaveAttribute("aria-readonly", "true"); + await expect(raw).toHaveAttribute("contenteditable", "false"); + await expect(raw).toContainText('"status": "succeeded"'); + await expect(page.locator(".response-editor .cm-content span").first()).toBeVisible(); + + await page.getByRole("tab", { name: "Timeline" }).click(); + await expect(page.getByText(/POST \/predictions/)).toBeVisible(); + await page.getByRole("tab", { name: "Request" }).click(); + await expect(page.getByText("Prediction time")).toBeVisible(); + await expect(page.getByText("200", { exact: true })).toBeVisible(); + await page.getByText("Response headers", { exact: true }).click(); + const responseHeaders = page.getByLabel("Response headers", { exact: true }); + await expect(responseHeaders).toContainText("content-type"); + await expect(responseHeaders).not.toContainText("content-security-policy"); + await expect(responseHeaders).not.toContainText("x-frame-options"); +}); + +test("keeps model targets isolated across browser workspaces", async ({ context, page }) => { + const firstModel = await startTestModel("first model"); + const secondModel = await startTestModel("second model"); + const secondPage = await context.newPage(); + const thirdPage = await context.newPage(); + + try { + await Promise.all([secondPage.goto("/"), thirdPage.goto("/")]); + await Promise.all([ + connectTarget(page, firstModel.url), + connectTarget(secondPage, secondModel.url), + connectTarget(thirdPage, firstModel.url), + ]); + + await page.getByRole("textbox", { name: "text" }).fill("one"); + await secondPage.getByRole("textbox", { name: "text" }).fill("two"); + await thirdPage.getByRole("textbox", { name: "text" }).fill("three"); + await Promise.all([ + page.getByRole("button", { name: "Run" }).click(), + secondPage.getByRole("button", { name: "Run" }).click(), + thirdPage.getByRole("button", { name: "Run" }).click(), + ]); + + await expect(predictionOutput(page)).toContainText("first model: one"); + await expect(predictionOutput(secondPage)).toContainText("second model: two"); + await expect(predictionOutput(thirdPage)).toContainText("first model: three"); + } finally { + await Promise.all([secondPage.close(), thirdPage.close()]); + await Promise.all([firstModel.close(), secondModel.close()]); + } +}); + +test("shows progressive streaming output before completion", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("slow"); + await page.getByRole("button", { name: "Stream", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + + const output = predictionOutput(page); + await expect(output).toHaveAttribute("aria-busy", "true"); + await expectPredictionStatus(page, "processing"); + await expect(output).toContainText("hello", { timeout: 30_000 }); + await expect(output).not.toContainText("slow"); + + await expectPredictionStatus(page, "succeeded"); + await expect(output).toHaveAttribute("aria-busy", "false"); + await expect(output).toContainText("hello slow"); +}); + +test("stops a running prediction", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("slow"); + await page.getByRole("textbox", { name: "Prediction ID" }).fill("playground-stop-id"); + await page.getByRole("button", { name: "Stream", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + await expect(page.getByRole("button", { name: "Stop" })).toBeEnabled(); + await expect(predictionOutput(page)).toContainText("hello", { timeout: 30_000 }); + + const cancelResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + new URL(response.url()).pathname === "/proxy/predictions/playground-stop-id/cancel", + ); + await page.getByRole("button", { name: "Stop" }).click(); + expect((await cancelResponse).ok()).toBe(true); + + await expectPredictionStatus(page, "canceled"); + await expect(page.getByRole("button", { name: "Run" })).toBeEnabled(); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible({ + timeout: 30_000, + }); +}); + +test("configures and receives an asynchronous webhook prediction", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("webhook"); + await page.getByRole("button", { name: "Async", exact: true }).click(); + await expect(page.getByText(/Webhook: .*\/webhook\/\.\.\./)).toBeVisible(); + await page.getByRole("checkbox", { name: "output" }).click(); + await page.getByRole("checkbox", { name: "logs" }).click(); + await expect(page.getByRole("checkbox", { name: "completed" })).toHaveAttribute( + "aria-disabled", + "true", + ); + await page.getByRole("button", { name: "Run" }).click(); + + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello webhook"); + await page.getByRole("tab", { name: "Request" }).click(); + const requestBody = page.getByLabel("Request body", { exact: true }); + await expect(requestBody).toContainText("start"); + await expect(requestBody).toContainText("completed"); + await expect(requestBody).not.toContainText('"logs"'); +}); + +test("reconnects and opens the loaded schema", async ({ page }) => { + const target = page.getByRole("textbox", { name: "Target" }); + const targetURL = await target.inputValue(); + await target.fill(" "); + await expect(page.getByRole("button", { name: "Connect" })).toBeDisabled(); + await target.fill(`${targetURL}/`); + const schemaResponse = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/proxy/openapi.json", + ); + await target.press("Enter"); + expect((await schemaResponse).ok()).toBe(true); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible(); + + const popupPromise = page.waitForEvent("popup"); + await page.getByRole("button", { name: "Schema" }).click(); + const popup = await popupPromise; + await popup.waitForLoadState(); + const schema = JSON.parse((await popup.locator("body").textContent()) ?? "") as { + openapi?: string; + }; + expect(schema.openapi).toMatch(/^3\./); +}); + +test("toggles the color theme", async ({ page }) => { + const root = page.locator("html"); + const initialTheme = await root.getAttribute("data-mode"); + const nextTheme = initialTheme === "dark" ? "light" : "dark"; + const toggleLabel = initialTheme === "dark" ? "Light" : "Dark"; + await page.getByRole("button", { name: toggleLabel }).click(); + await expect(root).toHaveAttribute("data-mode", nextTheme); +}); + +test("uses a custom prediction ID and resets the playground", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("identified"); + await page.getByRole("textbox", { name: "Prediction ID" }).fill("playground-e2e-id"); + await page.getByRole("button", { name: "Sync", exact: true }).click(); + const predictionRequest = page.waitForRequest( + (request) => + request.method() === "PUT" && + new URL(request.url()).pathname === "/proxy/predictions/playground-e2e-id", + ); + await page.getByRole("button", { name: "Run" }).click(); + await predictionRequest; + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello identified"); + + await page.getByRole("button", { name: "Reset" }).click(); + await expect(page.getByRole("textbox", { name: "text" })).toHaveValue(""); + await expect(page.getByRole("textbox", { name: "Prediction ID" })).toHaveValue(""); + await expect(page.getByText("Run a prediction to see its output.")).toBeVisible(); +}); + +test("fits every primary control on a narrow viewport", async ({ page }) => { + await page.setViewportSize({ width: 320, height: 800 }); + + const widths = await page.evaluate(() => ({ + document: document.documentElement.scrollWidth, + viewport: document.documentElement.clientWidth, + })); + expect(widths.document).toBeLessThanOrEqual(widths.viewport); + await expect(page.getByRole("button", { name: "Run" })).toBeVisible(); + await expect(page.getByRole("tab", { name: "Form" })).toBeVisible(); + await expect(page.getByRole("tab", { name: "Output" })).toBeVisible(); +}); + +async function replaceEditor(page: Page, label: string, value: string): Promise { + const editor = page.getByRole("textbox", { name: label }); + await editor.click(); + await page.keyboard.press("ControlOrMeta+A"); + await page.keyboard.insertText(value); +} + +async function expectPredictionStatus(page: Page, status: string): Promise { + await expect(page.locator("#output-panel .response-title").getByRole("status")).toHaveText( + status, + { timeout: 60_000 }, + ); +} + +function predictionOutput(page: Page) { + return page.getByRole("region", { name: "Prediction output" }); +} + +async function connectTarget(page: Page, target: string): Promise { + const schema = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/proxy/openapi.json" && + response.request().headers()["x-cog-target"] === target, + ); + await page.getByRole("textbox", { name: "Target" }).fill(target); + await page.getByRole("button", { name: "Connect" }).click(); + expect((await schema).ok()).toBe(true); + await expect(page.getByRole("textbox", { name: "text" })).toBeVisible(); +} + +async function startTestModel(name: string): Promise<{ url: string; close: () => Promise }> { + const server = createServer((request, response) => { + response.setHeader("Content-Type", "application/json"); + if (request.url === "/health-check") { + response.end(JSON.stringify({ status: "READY" })); + return; + } + if (request.url === "/openapi.json") { + response.end( + JSON.stringify({ + components: { + schemas: { + Input: { + type: "object", + required: ["text"], + properties: { text: { type: "string" } }, + }, + }, + }, + paths: { "/predictions": { post: {} } }, + }), + ); + return; + } + if (request.url === "/predictions" && request.method === "POST") { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => (body += chunk)); + request.on("end", () => { + const prediction = JSON.parse(body) as { input: { text: string } }; + response.end( + JSON.stringify({ status: "succeeded", output: `${name}: ${prediction.input.text}` }), + ); + }); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Test model did not bind a port"); + return { + url: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} diff --git a/playground/e2e/serve.ts b/playground/e2e/serve.ts new file mode 100644 index 0000000000..f5425f2d21 --- /dev/null +++ b/playground/e2e/serve.ts @@ -0,0 +1,141 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { createServer } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const cog = process.env.COG_BINARY; +if (!cog) throw new Error("COG_BINARY must point to the built Cog CLI"); +const cogBinary = cog; + +const fixture = fileURLToPath(new URL("./fixture/", import.meta.url)); +const modelPort = await availablePort(); +const modelURL = `http://127.0.0.1:${modelPort}`; +const playgroundPort = Number(process.env.PLAYGROUND_E2E_PORT ?? 8400); +const playgroundURL = `http://127.0.0.1:${playgroundPort}`; +const children: ChildProcess[] = []; +const childFailures: Promise[] = []; +let stopping = false; + +function availablePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close((error) => { + if (error) reject(error); + else if (typeof address === "object" && address) resolve(address.port); + else reject(new Error("Could not allocate a model port")); + }); + }); + }); +} + +function start(name: string, args: string[], cwd: string): Promise { + const child = spawn(cogBinary, args, { + cwd, + env: { ...process.env, BUILDKIT_PROGRESS: "quiet" }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout?.on("data", (data: Buffer) => process.stdout.write(`[${name}] ${data}`)); + child.stderr?.on("data", (data: Buffer) => process.stderr.write(`[${name}] ${data}`)); + const failure = new Promise((_, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => { + if (!stopping) reject(new Error(`${name} exited unexpectedly (${code ?? signal})`)); + }); + }); + children.push(child); + childFailures.push(failure); + return failure; +} + +async function waitFor(url: string, timeout: number): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2000) }); + if (response.ok) return; + } catch { + // The process is still starting. + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${url}`); +} + +async function waitForModel(timeout: number): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + try { + const response = await fetch(`${modelURL}/health-check`, { + signal: AbortSignal.timeout(2000), + }); + if (response.ok) { + const health = (await response.json()) as { status?: string }; + if (health.status === "READY" || health.status === "BUSY") return; + if (health.status === "SETUP_FAILED" || health.status === "DEFUNCT") { + throw new Error(`Model startup failed with status ${health.status}`); + } + } + } catch (error) { + if (error instanceof Error && error.message.startsWith("Model startup failed")) throw error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${modelURL}`); +} + +async function stop(): Promise { + if (stopping) return; + stopping = true; + try { + await fetch(`${modelURL}/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(5000), + }); + } catch { + // Fall back to process signals below. + } + for (const child of children) child.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 3000)); + for (const child of children) { + if (child.exitCode === null) child.kill("SIGKILL"); + } +} + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + void stop().finally(() => process.exit(0)); + }); +} + +try { + // Enable Cog's host-gateway mapping so webhooks reach the host on Linux CI. + const modelFailure = start( + "model", + ["serve", "--port", String(modelPort), "--upload-url", "http://unused/"], + fixture, + ); + await Promise.race([waitForModel(12 * 60_000), modelFailure]); + const playgroundFailure = start( + "playground", + [ + "playground", + "--host", + "0.0.0.0", + "--port", + String(playgroundPort), + "--target", + modelURL, + "--no-open", + ], + fixture, + ); + await Promise.race([waitFor(playgroundURL, 30_000), playgroundFailure]); + await Promise.race(childFailures); +} catch (error) { + console.error(error); + await stop(); + process.exit(1); +} diff --git a/playground/playwright.config.ts b/playground/playwright.config.ts new file mode 100644 index 0000000000..e8fb004c00 --- /dev/null +++ b/playground/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from "@playwright/test"; + +const playgroundPort = Number(process.env.PLAYGROUND_E2E_PORT ?? 8400); +const playgroundURL = `http://127.0.0.1:${playgroundPort}`; + +export default defineConfig({ + testDir: "./e2e", + fullyParallel: false, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", + timeout: 60_000, + use: { + baseURL: playgroundURL, + screenshot: "only-on-failure", + trace: "retain-on-failure", + video: "retain-on-failure", + }, + webServer: { + command: "node ./e2e/serve.ts", + url: playgroundURL, + reuseExistingServer: false, + timeout: 15 * 60_000, + gracefulShutdown: { signal: "SIGTERM", timeout: 10_000 }, + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], +}); diff --git a/playground/scripts/merge-licenses.ts b/playground/scripts/merge-licenses.ts index 7299af70eb..e31073da00 100644 --- a/playground/scripts/merge-licenses.ts +++ b/playground/scripts/merge-licenses.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { readFileSync, unlinkSync, writeFileSync } from "node:fs"; // THIRD_PARTY_LICENSES.md is not used at runtime. Playground JS (React, editor // deps, ajv in the validation worker, etc.) is bundled into assets embedded in @@ -8,7 +8,6 @@ const outputDirectory = new URL("../../pkg/cli/playground/", import.meta.url); const mainPath = new URL("THIRD_PARTY_LICENSES.md", outputDirectory); const workerPath = new URL("WORKER_LICENSES.md", outputDirectory); const main = readFileSync(mainPath, "utf8").trimEnd(); -if (!existsSync(workerPath)) process.exit(0); const worker = readFileSync(workerPath, "utf8"); const existing = new Set([...main.matchAll(/^## (.+?) - .+$/gm)].map((match) => match[1])); const workerSections = worker diff --git a/playground/src/app/App.test.tsx b/playground/src/app/App.test.tsx new file mode 100644 index 0000000000..7e97a6980a --- /dev/null +++ b/playground/src/app/App.test.tsx @@ -0,0 +1,303 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { useEffect } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { createInputValidator } from "@/features/inputs/validation/inputValidation"; +import type { ValidationIssue } from "@/features/inputs/validation/inputValidation"; +import { deferred } from "@/test/deferred"; + +const run = vi.fn(); +const reset = vi.fn(); +const { mockValidateInput } = vi.hoisted(() => ({ mockValidateInput: vi.fn() })); + +vi.mock("@/features/connection/hooks/useConnection", () => { + const connection = { + target: "http://localhost:8393", + targetDraft: "http://localhost:8393", + setTargetDraft: vi.fn(), + connect: vi.fn(), + health: { status: "ready", version: { python_sdk: "0.21.0" } }, + schema: { + components: { + schemas: { + Input: { + type: "object", + required: ["prompt"], + properties: { + prompt: { type: "string" }, + }, + }, + Output: { type: "string" }, + }, + }, + paths: { "/predictions": { post: {} } }, + }, + schemaError: "", + webhookBase: "", + cogVersion: "0.21.1-dev+g0db4bffa", + capabilities: { + endpoint: "/predictions", + input: { + type: "object", + required: ["prompt"], + properties: { + prompt: { type: "string" }, + }, + }, + output: { type: "string" }, + streaming: false, + async: false, + }, + }; + return { useConnection: () => connection }; +}); + +vi.mock("@/features/predictions/hooks/usePrediction", () => ({ + usePrediction: () => ({ + running: false, + envelope: undefined, + output: undefined, + rawEvents: [], + error: "", + trace: undefined, + run, + stop: vi.fn(), + reset, + }), +})); + +vi.mock("@/features/inputs/validation/validateInput", () => ({ + validateInput: mockValidateInput, + disposeValidationWorker: vi.fn(), +})); + +vi.mock("@/features/inputs/components/InputForm", () => ({ + InputForm: ({ + onChange, + onValidityChange, + value, + }: { + onChange: (value: Record) => void; + onValidityChange: (valid: boolean) => void; + value: Record; + }) => { + const prompt = String(value.prompt ?? ""); + useEffect(() => onValidityChange(true), [onValidityChange]); + return ( +