From b74f42e1ca89af7f507432ef28a3952d65604e0e Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Tue, 14 Jul 2026 13:21:14 -0500 Subject: [PATCH] feat(playground): compose app and add browser e2e --- .github/workflows/ci.yaml | 243 ++++++++-- pkg/cli/playground_test.go | 633 +++++++++++++++++++++++++++ playground/e2e/fixture/cog.yaml | 3 + playground/e2e/fixture/run.py | 20 + playground/e2e/playground.spec.ts | 352 +++++++++++++++ playground/e2e/serve.ts | 141 ++++++ playground/playwright.config.ts | 26 ++ playground/scripts/merge-licenses.ts | 29 ++ playground/src/app/App.test.tsx | 303 +++++++++++++ playground/src/app/App.tsx | 181 ++++++++ playground/src/main.tsx | 7 + 11 files changed, 1909 insertions(+), 29 deletions(-) create mode 100644 pkg/cli/playground_test.go create mode 100644 playground/e2e/fixture/cog.yaml create mode 100644 playground/e2e/fixture/run.py create mode 100644 playground/e2e/playground.spec.ts create mode 100644 playground/e2e/serve.ts create mode 100644 playground/playwright.config.ts create mode 100644 playground/scripts/merge-licenses.ts create mode 100644 playground/src/app/App.test.tsx create mode 100644 playground/src/app/App.tsx create mode 100644 playground/src/main.tsx diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 142080868b..b618cc131f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -63,6 +63,101 @@ jobs: steps: - run: echo "supported_pythons=$SUPPORTED_PYTHONS" + playground-changes: + name: Detect playground changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + changed: ${{ steps.changes.outputs.changed }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Check playground sources + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" || -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + elif git diff --quiet "$BASE_SHA" "$GITHUB_SHA" -- \ + playground \ + 'pkg/cli/playground*' \ + pkg/cli/root.go \ + cmd/cog \ + .goreleaser.yaml \ + mise.toml \ + mise.lock \ + go.mod \ + go.sum \ + .github/workflows/ci.yaml \ + .github/workflows/release-build.yaml; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + fmt-playground: + name: Format playground + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Check formatting + run: mise run fmt:playground + + lint-playground: + name: Lint playground + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Lint + run: mise run lint:playground + + test-playground: + name: Test playground + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Test + run: mise run test:playground + + playground-check: + name: Check playground + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Type check + run: mise run typecheck:playground + - name: Build embedded assets + run: mise run --force build:playground + - name: Test embedded assets + run: go test -tags playground_assets ./pkg/cli -run '^TestPlaygroundServesUI$' + - name: Upload playground assets + uses: actions/upload-artifact@v6 + with: + name: PlaygroundAssets + path: pkg/cli/playground + # ============================================================================= # Version Check - Validates VERSION.txt format and sync # ============================================================================= @@ -183,6 +278,7 @@ jobs: build-cog: name: Build cog CLI + needs: [playground-changes, playground-check] runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -193,6 +289,11 @@ jobs: with: version: 2026.4.27 cache_key_prefix: mise-ci-${{ github.job }} + - name: Download verified playground assets + uses: actions/download-artifact@v8 + with: + name: PlaygroundAssets + path: pkg/cli/playground - name: Get version from VERSION.txt id: version run: echo "version=$(tr -d '[:space:]' < VERSION.txt)" >> "$GITHUB_OUTPUT" @@ -532,6 +633,65 @@ jobs: - name: Test coglet-python bindings run: uvx nox -s coglet -p ${{ matrix.python-version }} + test-playground-e2e: + name: Playground Playwright integration tests (Chromium) + needs: + - playground-changes + - fmt-playground + - lint-playground + - test-playground + - playground-check + - build-cog + - build-sdk + - build-rust + if: needs.playground-changes.outputs.changed == 'true' + runs-on: ubuntu-latest-16-cores + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Login to Docker Hub + uses: docker/login-action@v4 + if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' + with: + registry: index.docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Download artifacts + uses: actions/download-artifact@v8 + with: + path: dist + merge-multiple: true + - name: Install cog binary + run: | + cp dist/cog ./cog + chmod +x ./cog + - uses: jdx/mise-action@v4 + with: + version: 2026.4.27 + cache_key_prefix: mise-ci-${{ github.job }} + - name: Install playground dependencies and Chromium + run: | + mise run playground:install + pnpm --dir playground exec playwright install --with-deps chromium + - name: Run browser integration tests + env: + BUILDKIT_PROGRESS: quiet + COG_BINARY: ${{ github.workspace }}/cog + COG_REGISTRY_HOST: ghcr.io/replicate/cog + COG_SDK_WHEEL: ${{ github.workspace }}/dist + COGLET_WHEEL: ${{ github.workspace }}/dist + run: pnpm --dir playground run test:e2e + - name: Upload browser test report + if: failure() + uses: actions/upload-artifact@v6 + with: + name: PlaygroundPlaywrightReport + path: | + playground/playwright-report + playground/test-results + # Compute integration test shards dynamically. # Slow tests (tagged with [short] skip) are distributed round-robin first, # then remaining tests fill in. This ensures slow tests don't pile up on one runner. @@ -696,6 +856,11 @@ jobs: name: CI Complete needs: - setup + - playground-changes + - fmt-playground + - lint-playground + - test-playground + - playground-check - version-check - build-cog - build-sdk @@ -716,6 +881,7 @@ jobs: - test-rust - test-python - test-coglet-python + - test-playground-e2e - integration-shards - test-integration if: always() @@ -725,6 +891,11 @@ jobs: - name: Check job results run: | echo "Job results:" + echo " playground-changes: ${{ needs.playground-changes.result }}" + echo " fmt-playground: ${{ needs.fmt-playground.result }}" + echo " lint-playground: ${{ needs.lint-playground.result }}" + echo " test-playground: ${{ needs.test-playground.result }}" + echo " playground-check: ${{ needs.playground-check.result }}" echo " version-check: ${{ needs.version-check.result }}" echo " build-cog: ${{ needs.build-cog.result }}" echo " build-sdk: ${{ needs.build-sdk.result }}" @@ -745,41 +916,55 @@ jobs: echo " test-rust: ${{ needs.test-rust.result }}" echo " test-python: ${{ needs.test-python.result }}" echo " test-coglet-python: ${{ needs.test-coglet-python.result }}" + echo " test-playground-e2e: ${{ needs.test-playground-e2e.result }}" echo " integration-shards: ${{ needs.integration-shards.result }}" echo " test-integration: ${{ needs.test-integration.result }}" - # Fail if any required job failed. - # lint-rust-advisories is excluded: it uses continue-on-error because - # advisory DB updates shouldn't block unrelated PRs. FAILED=false - for result in \ - "${{ needs.setup.result }}" \ - "${{ needs.version-check.result }}" \ - "${{ needs.build-cog.result }}" \ - "${{ needs.build-sdk.result }}" \ - "${{ needs.build-rust.result }}" \ - "${{ needs.fmt-go.result }}" \ - "${{ needs.fmt-rust.result }}" \ - "${{ needs.fmt-python.result }}" \ - "${{ needs.check-llm-docs.result }}" \ - "${{ needs.check-stubs.result }}" \ - "${{ needs.lint-go.result }}" \ - "${{ needs.lint-rust.result }}" \ - "${{ needs.lint-rust-deny.result }}" \ - "${{ needs.lint-python.result }}" \ - "${{ needs.lint-docs.result }}" \ - "${{ needs.test-go.result }}" \ - "${{ needs.fuzz-go.result }}" \ - "${{ needs.test-rust.result }}" \ - "${{ needs.test-python.result }}" \ - "${{ needs.test-coglet-python.result }}" \ - "${{ needs.integration-shards.result }}" \ - "${{ needs.test-integration.result }}" - do - if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + check_success() { + if [ "$2" != "success" ]; then + echo "::error::$1 finished with unexpected result: $2" FAILED=true fi - done + } + + # lint-rust-advisories is informational and intentionally excluded. + check_success setup "${{ needs.setup.result }}" + check_success playground-changes "${{ needs.playground-changes.result }}" + check_success fmt-playground "${{ needs.fmt-playground.result }}" + check_success lint-playground "${{ needs.lint-playground.result }}" + check_success test-playground "${{ needs.test-playground.result }}" + check_success playground-check "${{ needs.playground-check.result }}" + check_success version-check "${{ needs.version-check.result }}" + check_success build-cog "${{ needs.build-cog.result }}" + check_success build-sdk "${{ needs.build-sdk.result }}" + check_success build-rust "${{ needs.build-rust.result }}" + check_success fmt-go "${{ needs.fmt-go.result }}" + check_success fmt-rust "${{ needs.fmt-rust.result }}" + check_success fmt-python "${{ needs.fmt-python.result }}" + check_success check-llm-docs "${{ needs.check-llm-docs.result }}" + check_success check-stubs "${{ needs.check-stubs.result }}" + check_success lint-go "${{ needs.lint-go.result }}" + check_success lint-rust "${{ needs.lint-rust.result }}" + check_success lint-rust-deny "${{ needs.lint-rust-deny.result }}" + check_success lint-python "${{ needs.lint-python.result }}" + check_success lint-docs "${{ needs.lint-docs.result }}" + check_success test-go "${{ needs.test-go.result }}" + check_success fuzz-go "${{ needs.fuzz-go.result }}" + check_success test-rust "${{ needs.test-rust.result }}" + check_success test-python "${{ needs.test-python.result }}" + check_success test-coglet-python "${{ needs.test-coglet-python.result }}" + check_success integration-shards "${{ needs.integration-shards.result }}" + check_success test-integration "${{ needs.test-integration.result }}" + + if [ "${{ needs.playground-changes.outputs.changed }}" = "true" ]; then + check_success test-playground-e2e "${{ needs.test-playground-e2e.result }}" + else + if [ "${{ needs.test-playground-e2e.result }}" != "skipped" ]; then + echo "::error::Playground integration tests should be skipped when playground sources are unchanged" + FAILED=true + fi + fi if [ "$FAILED" = "true" ]; then echo "::error::Some jobs failed or were cancelled" diff --git a/pkg/cli/playground_test.go b/pkg/cli/playground_test.go new file mode 100644 index 0000000000..85ec5325da --- /dev/null +++ b/pkg/cli/playground_test.go @@ -0,0 +1,633 @@ +package cli + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/replicate/cog/pkg/global" +) + +func newTestPlayground(t *testing.T) *httptest.Server { + t.Helper() + uiFS, err := fs.Sub(playgroundUI, "playground") + require.NoError(t, err) + s := newPlaygroundServer("http://wh.example/cb", "http://localhost:8393") + ts := httptest.NewServer(s.routes(uiFS)) + t.Cleanup(ts.Close) + return ts +} + +// echoServer reports the received path and (forwarded) query as JSON. +func echoServer(t *testing.T) *httptest.Server { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"path":%q,"escaped_path":%q,"query":%q}`, r.URL.Path, r.URL.EscapedPath(), r.URL.RawQuery) + })) + t.Cleanup(ts.Close) + return ts +} + +func TestPlaygroundServesUI(t *testing.T) { + ts := newTestPlayground(t) + + resp, err := http.Get(ts.URL + "/") + require.NoError(t, err) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, string(body), "Cog Playground") + + assets := regexp.MustCompile(`(?:src|href)="(/?assets/[^"]+)"`).FindAllStringSubmatch(string(body), -1) + if !playgroundAssetsBuilt { + assert.Empty(t, assets, "stub index should not reference generated assets") + return + } + require.NotEmpty(t, assets, "generated index should reference bundled assets") + for _, match := range assets { + path := "/" + strings.TrimPrefix(match[1], "/") + assetResp, err := http.Get(ts.URL + path) + require.NoError(t, err, "requesting %s", path) + assetResp.Body.Close() + assert.Equal(t, http.StatusOK, assetResp.StatusCode, "%s should be served", path) + if strings.HasSuffix(path, ".css") { + assert.Contains(t, assetResp.Header.Get("Content-Type"), "text/css") + } else if strings.HasSuffix(path, ".js") { + assert.Contains(t, assetResp.Header.Get("Content-Type"), "javascript") + } + } +} + +func TestServePlaygroundWaitsForHandlers(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + started := make(chan struct{}) + canceled := make(chan struct{}) + release := make(chan struct{}) + srv := &http.Server{ + Handler: http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(canceled) + <-release + }), + BaseContext: func(net.Listener) context.Context { return ctx }, + } + serveDone := make(chan error, 1) + go func() { serveDone <- servePlayground(ctx, srv, ln, time.Second) }() + requestDone := make(chan struct{}) + go func() { + defer close(requestDone) + resp, requestErr := http.Get("http://" + ln.Addr().String()) + if requestErr == nil { + resp.Body.Close() + } + }() + + <-started + cancel() + <-canceled + select { + case err := <-serveDone: + require.FailNow(t, "server returned before the active handler finished", "%v", err) + default: + } + close(release) + require.NoError(t, <-serveDone) + <-requestDone +} + +func TestPlaygroundConfig(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Get(ts.URL + "/config") + require.NoError(t, err) + defer resp.Body.Close() + + var config map[string]string + require.NoError(t, json.NewDecoder(resp.Body).Decode(&config)) + assert.Equal(t, "http://wh.example/cb", config["webhookBase"]) + assert.Equal(t, "http://localhost:8393", config["target"]) + assert.Equal(t, global.Version, config["cogVersion"]) +} + +func TestPlaygroundRejectsRemoteUIAndProxyRequests(t *testing.T) { + uiFS, err := fs.Sub(playgroundUI, "playground") + require.NoError(t, err) + s := newPlaygroundServer("http://wh.example/cb", "") + + for _, path := range []string{"/", "/config", "/proxy/health-check"} { + req := httptest.NewRequest(http.MethodGet, "http://playground.example"+path, nil) + req.RemoteAddr = "192.0.2.10:1234" + resp := httptest.NewRecorder() + s.routes(uiFS).ServeHTTP(resp, req) + assert.Equal(t, http.StatusForbidden, resp.Code, path) + } +} + +func TestPlaygroundAllowsRemoteWebhookRequests(t *testing.T) { + uiFS, err := fs.Sub(playgroundUI, "playground") + require.NoError(t, err) + s := newPlaygroundServer("http://wh.example/cb", "") + ch := s.hub.subscribe("token") + defer s.hub.unsubscribe("token", ch) + + req := httptest.NewRequest(http.MethodPost, "http://playground.example/webhook/token", strings.NewReader(`{"status":"succeeded"}`)) + req.RemoteAddr = "192.0.2.10:1234" + resp := httptest.NewRecorder() + s.routes(uiFS).ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) +} + +func TestPlaygroundProxyHeaderTarget(t *testing.T) { + ts := newTestPlayground(t) + stub := echoServer(t) + + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/openapi.json?foo=bar", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", stub.URL) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + var got map[string]string + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + assert.Equal(t, "/openapi.json", got["path"], "/proxy prefix should be stripped") + assert.Equal(t, "foo=bar", got["query"]) +} + +func TestPlaygroundProxyRejectsQueryTarget(t *testing.T) { + ts := newTestPlayground(t) + stub := echoServer(t) + + u := ts.URL + "/proxy/health-check?x=1&cog_target=" + url.QueryEscape(stub.URL) + resp, err := http.Get(u) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestPlaygroundProxyPreservesEscapedPathSegments(t *testing.T) { + ts := newTestPlayground(t) + stub := echoServer(t) + + req, err := http.NewRequest(http.MethodPost, ts.URL+"/proxy/predictions/custom%2Fid/cancel", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", stub.URL+"/api") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + var got map[string]string + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + assert.Equal(t, "/api/predictions/custom/id/cancel", got["path"]) + assert.Equal(t, "/api/predictions/custom%2Fid/cancel", got["escaped_path"]) +} + +func TestPlaygroundProxyForwardsRequestAndResponse(t *testing.T) { + ts := newTestPlayground(t) + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, http.MethodPut, r.Method) + assert.Equal(t, "/api/predictions/p1", r.URL.Path) + assert.Equal(t, `{"input":{"prompt":"hello"}}`, string(body)) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + assert.Empty(t, r.Header.Get("X-Cog-Target")) + assert.Empty(t, r.Header.Get("Authorization")) + assert.Empty(t, r.Header.Get("Cookie")) + w.Header().Set("X-Upstream", "preserved") + w.Header().Set("X-Frame-Options", "SAMEORIGIN") + w.Header().Set("Set-Cookie", "session=upstream") + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(`{"detail":"invalid input"}`)) + })) + t.Cleanup(stub.Close) + + req, err := http.NewRequest(http.MethodPut, ts.URL+"/proxy/predictions/p1", bytes.NewBufferString(`{"input":{"prompt":"hello"}}`)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer local-secret") + req.Header.Set("Cookie", "session=local-secret") + req.Header.Set("X-Cog-Target", stub.URL+"/api") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + assert.Equal(t, "preserved", resp.Header.Get("X-Upstream")) + assert.Empty(t, resp.Header.Get("Set-Cookie")) + encodedHeaders, err := base64.RawURLEncoding.DecodeString(resp.Header.Get(playgroundUpstreamHeaders)) + require.NoError(t, err) + var upstreamHeaders http.Header + require.NoError(t, json.Unmarshal(encodedHeaders, &upstreamHeaders)) + assert.Equal(t, "preserved", upstreamHeaders.Get("X-Upstream")) + assert.Equal(t, "SAMEORIGIN", upstreamHeaders.Get("X-Frame-Options")) + assert.Empty(t, upstreamHeaders.Get("Set-Cookie")) + assert.JSONEq(t, `{"detail":"invalid input"}`, string(body)) +} + +func TestPlaygroundProxyRoutesConcurrentWorkspacesIndependently(t *testing.T) { + ts := newTestPlayground(t) + models := map[string]*httptest.Server{} + for _, name := range []string{"first", "second"} { + models[name] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, name) + })) + t.Cleanup(models[name].Close) + } + + errs := make(chan error, 40) + var requests sync.WaitGroup + for name, model := range models { + for range 20 { + requests.Go(func() { + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/identity", nil) + if err != nil { + errs <- err + return + } + req.Header.Set("X-Cog-Target", model.URL) + resp, err := http.DefaultClient.Do(req) + if err != nil { + errs <- err + return + } + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + errs <- readErr + return + } + if string(body) != name { + errs <- fmt.Errorf("request for %s reached %q", name, body) + } + }) + } + } + requests.Wait() + close(errs) + for err := range errs { + assert.NoError(t, err) + } +} + +func TestPlaygroundProxyOmitsOversizedHeaderMetadata(t *testing.T) { + ts := newTestPlayground(t) + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("X-Large", strings.Repeat("x", maxPlaygroundHeaderMetadata)) + w.Header().Set(playgroundUpstreamHeaders, "spoofed") + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(stub.Close) + + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/health-check", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", stub.URL) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, strings.Repeat("x", maxPlaygroundHeaderMetadata), resp.Header.Get("X-Large")) + assert.Empty(t, resp.Header.Get(playgroundUpstreamHeaders)) +} + +func TestPlaygroundProxyMissingTarget(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Get(ts.URL + "/proxy/health-check") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestPlaygroundProxyInvalidTarget(t *testing.T) { + ts := newTestPlayground(t) + for _, target := range []string{"ftp://example.com", "garbage", "", "http://user@example.com", "http://example.com/#fragment"} { + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/health-check", nil) + require.NoError(t, err) + if target != "" { + req.Header.Set("X-Cog-Target", target) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + resp.Body.Close() + assert.Equal(t, http.StatusBadRequest, resp.StatusCode, "target %q should be rejected", target) + } +} + +func TestPlaygroundWebhookRejectsNonPost(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Get(ts.URL + "/webhook/token") + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode) + assert.Equal(t, http.MethodPost, resp.Header.Get("Allow")) +} + +func TestPlaygroundRecognizesIPv6Loopback(t *testing.T) { + assert.True(t, isLoopbackRemote("[::1]:8080")) + assert.False(t, isLoopbackRemote("not-an-address")) + assert.True(t, isLoopbackHost("[::1]:8080")) + assert.True(t, isLoopbackHost("localhost:8080")) + assert.False(t, isLoopbackHost("playground.example:8080")) +} + +func TestPlaygroundFormatsIPv6Address(t *testing.T) { + assert.Equal(t, "[::1]:8080", playgroundAddress("::1", 8080)) + assert.Equal(t, "[::1]:8080", playgroundAddress("[::1]", 8080)) +} + +func TestPlaygroundUsesLoopbackBrowserHostForWildcard(t *testing.T) { + assert.Equal(t, "127.0.0.1", playgroundBrowserHost("0.0.0.0")) + assert.Equal(t, "::1", playgroundBrowserHost("::")) + assert.Equal(t, "::1", playgroundBrowserHost("[::]")) +} + +func TestPlaygroundRejectsCrossSiteBrowserRequests(t *testing.T) { + ts := newTestPlayground(t) + + for name, mutate := range map[string]func(*http.Request){ + "host": func(req *http.Request) { req.Host = "attacker.example" }, + "origin": func(req *http.Request) { + req.Header.Set("Origin", "https://attacker.example") + }, + "fetch metadata": func(req *http.Request) { + req.Header.Set("Sec-Fetch-Site", "cross-site") + }, + } { + t.Run(name, func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, ts.URL+"/config", nil) + require.NoError(t, err) + mutate(req) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + } +} + +func TestPlaygroundSetsBrowserSecurityHeaders(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Get(ts.URL + "/") + require.NoError(t, err) + defer resp.Body.Close() + + assert.Contains(t, resp.Header.Get("Content-Security-Policy"), "frame-ancestors 'none'") + assert.NotContains(t, resp.Header.Get("Content-Security-Policy"), "'unsafe-eval'") + assert.Equal(t, "no-referrer", resp.Header.Get("Referrer-Policy")) + assert.Equal(t, "nosniff", resp.Header.Get("X-Content-Type-Options")) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/assets/validation.worker-test.js", nil) + request.RemoteAddr = "127.0.0.1:1234" + request.Host = "127.0.0.1" + protectPlayground(http.NotFoundHandler()).ServeHTTP(recorder, request) + workerCSP := recorder.Header().Get("Content-Security-Policy") + assert.Contains(t, workerCSP, "script-src 'self' 'unsafe-eval'") + assert.Contains(t, workerCSP, "connect-src 'none'") +} + +func TestPlaygroundProxyUnreachableTarget(t *testing.T) { + ts := newTestPlayground(t) + dead := httptest.NewServer(http.NotFoundHandler()) + deadURL := dead.URL + dead.Close() // now refuses connections + + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/health-check", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", deadURL) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusBadGateway, resp.StatusCode) +} + +func TestPlaygroundProxyStreamsSSE(t *testing.T) { + ts := newTestPlayground(t) + sse := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + fmt.Fprint(w, "event: start\ndata: {\"a\":1}\n\n") + w.(http.Flusher).Flush() + })) + t.Cleanup(sse.Close) + + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/predictions", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", sse.URL) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, "text/event-stream", resp.Header.Get("Content-Type")) + body, _ := io.ReadAll(resp.Body) + assert.Contains(t, string(body), "event: start") + assert.Contains(t, string(body), `data: {"a":1}`) +} + +func TestPlaygroundWebhookRelay(t *testing.T) { + ts := newTestPlayground(t) + const token = "tok123" + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/events?token="+token, nil) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, "text/event-stream", resp.Header.Get("Content-Type")) + + // The subscription is registered before headers are flushed, so by now the + // hub has our channel; deliver a webhook and expect it relayed. + got := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + if line := scanner.Text(); strings.HasPrefix(line, "data: ") { + got <- strings.TrimPrefix(line, "data: ") + return + } + } + }() + + whResp, err := http.Post(ts.URL+"/webhook/"+token, "application/json", + strings.NewReader(`{"status":"succeeded","id":"p1"}`)) + require.NoError(t, err) + whResp.Body.Close() + assert.Equal(t, http.StatusOK, whResp.StatusCode) + + select { + case data := <-got: + assert.JSONEq(t, `{"status":"succeeded","id":"p1"}`, data) + case <-time.After(2 * time.Second): + require.FailNow(t, "timed out waiting for relayed webhook event") + } +} + +// A payload containing newlines must be framed as one SSE event with a +// "data: " prefix per line, not terminate the event early or inject fields. +func TestPlaygroundWebhookRelayPreservesNewlines(t *testing.T) { + ts := newTestPlayground(t) + const token = "tok-nl" + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+"/events?token="+token, nil) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + got := make(chan []string, 1) + go func() { + scanner := bufio.NewScanner(resp.Body) + var data []string + for scanner.Scan() { + line := scanner.Text() + if after, ok := strings.CutPrefix(line, "data: "); ok { + data = append(data, after) + } else if line == "" && len(data) > 0 { + got <- data + return + } + } + }() + + whResp, err := http.Post(ts.URL+"/webhook/"+token, "application/json", + strings.NewReader("{\"a\":1}\n{\"b\":2}")) + require.NoError(t, err) + whResp.Body.Close() + + select { + case data := <-got: + assert.Equal(t, []string{`{"a":1}`, `{"b":2}`}, data) + case <-time.After(2 * time.Second): + require.FailNow(t, "timed out waiting for relayed webhook event") + } +} + +func TestWriteSSEDataNormalizesLineEndings(t *testing.T) { + for name, input := range map[string]string{ + "LF": "first\nsecond", + "CRLF": "first\r\nsecond", + "CR": "first\rsecond", + } { + t.Run(name, func(t *testing.T) { + var output strings.Builder + writeSSEData(&output, []byte(input)) + assert.Equal(t, "data: first\ndata: second\n\n", output.String()) + }) + } +} + +func TestPlaygroundEventsMissingToken(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Get(ts.URL + "/events") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestPlaygroundWebhookMissingToken(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Post(ts.URL+"/webhook/", "application/json", strings.NewReader("{}")) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) +} + +func TestPlaygroundWebhookRejectsOversizedBody(t *testing.T) { + uiFS, err := fs.Sub(playgroundUI, "playground") + require.NoError(t, err) + // Keep a subscriber active so the handler reaches its body-size check. + s := newPlaygroundServer("http://wh.example/cb", "") + ch := s.hub.subscribe("token") + defer s.hub.unsubscribe("token", ch) + ts := httptest.NewServer(s.routes(uiFS)) + t.Cleanup(ts.Close) + resp, err := http.Post(ts.URL+"/webhook/token", "application/json", + strings.NewReader(strings.Repeat("x", maxWebhookBody+1))) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) +} + +func TestPlaygroundWebhookRejectsMissingSubscriber(t *testing.T) { + ts := newTestPlayground(t) + resp, err := http.Post(ts.URL+"/webhook/token", "application/json", strings.NewReader("{}")) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) +} + +func TestPlaygroundProxyStripsAbsoluteRedirectLocation(t *testing.T) { + ts := newTestPlayground(t) + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "http://169.254.169.254/latest/meta-data/") + w.WriteHeader(http.StatusFound) + })) + t.Cleanup(stub.Close) + + req, err := http.NewRequest(http.MethodGet, ts.URL+"/proxy/health-check", nil) + require.NoError(t, err) + req.Header.Set("X-Cog-Target", stub.URL) + // Do not follow redirects; we want the proxy response as the browser would with redirect:manual. + client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusFound, resp.StatusCode) + assert.Empty(t, resp.Header.Get("Location")) +} + +// A stalled subscriber (full buffer) must not block or deny delivery to a +// healthy one: publish is best-effort and non-blocking. +func TestEventHubPublishDeliversDespiteStalledSubscriber(t *testing.T) { + hub := newEventHub() + stalled := hub.subscribe("token") + healthy := hub.subscribe("token") + // Fill the stalled subscriber's buffer so any send to it would block. + for len(stalled) < cap(stalled) { + stalled <- []byte("fill") + } + + assert.True(t, hub.publish("token", []byte("msg")), "healthy subscriber should receive the message") + + select { + case msg := <-healthy: + assert.Equal(t, []byte("msg"), msg) + case <-time.After(time.Second): + require.FailNow(t, "healthy subscriber did not receive the message") + } +} + +func TestEventHubPublishReturnsFalseWithoutSubscribers(t *testing.T) { + hub := newEventHub() + assert.False(t, hub.publish("token", []byte("msg"))) +} diff --git a/playground/e2e/fixture/cog.yaml b/playground/e2e/fixture/cog.yaml new file mode 100644 index 0000000000..e19c4aea46 --- /dev/null +++ b/playground/e2e/fixture/cog.yaml @@ -0,0 +1,3 @@ +build: + python_version: "3.12" +run: "run.py:Runner" diff --git a/playground/e2e/fixture/run.py b/playground/e2e/fixture/run.py new file mode 100644 index 0000000000..fa5761b2cd --- /dev/null +++ b/playground/e2e/fixture/run.py @@ -0,0 +1,20 @@ +import time + +from cog import BaseRunner, ConcatenateIterator, Input, streaming + + +class Runner(BaseRunner): + @streaming + def run( + self, + text: str = Input( + description="Text to prefix with 'hello '", + min_length=2, + max_length=20, + regex=r"^[a-z ]+$", + ), + ) -> ConcatenateIterator[str]: + yield "hello " + if text == "slow": + time.sleep(2) + yield text diff --git a/playground/e2e/playground.spec.ts b/playground/e2e/playground.spec.ts new file mode 100644 index 0000000000..84429a46c2 --- /dev/null +++ b/playground/e2e/playground.spec.ts @@ -0,0 +1,352 @@ +import { expect, test, type Page } from "@playwright/test"; +import { createServer } from "node:http"; + +test.beforeEach(async ({ page }) => { + await page.goto("/"); + await expect(page.getByRole("heading", { name: "Cog Playground" })).toBeVisible(); + await expect(page.getByRole("textbox", { name: "text" })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible({ + timeout: 30_000, + }); +}); + +test("keeps Form and CodeMirror JSON input synchronized", async ({ page }) => { + const run = page.getByRole("button", { name: "Run" }); + await expect(run).toBeEnabled(); + await page.getByRole("textbox", { name: "text" }).fill("from form"); + await page.getByRole("tab", { name: "JSON" }).click(); + + const editor = page.getByRole("textbox", { name: "Prediction input JSON" }); + await expect(editor).toContainText('"text": "from form"'); + await editor.click(); + await page.keyboard.press("Tab"); + await expect(page.getByRole("button", { name: "Copy" })).toBeFocused(); + await replaceEditor(page, "Prediction input JSON", '{"text":"from json"}'); + await page.getByRole("button", { name: "Format" }).click(); + await expect(editor).toContainText('"text": "from json"'); + await expect(run).toBeEnabled(); + + await replaceEditor(page, "Prediction input JSON", "{"); + await expect(run).toBeEnabled(); + await page.getByRole("tab", { name: "Form" }).click(); + await expect(page.getByRole("textbox", { name: "text" })).toHaveValue("from json"); + + await page.getByRole("tab", { name: "JSON" }).click(); + await expect(page.getByRole("textbox", { name: "Prediction input JSON" })).toContainText( + '"text": "from json"', + ); + await expect(run).toBeEnabled(); +}); + +test("folds JSON objects in CodeMirror", async ({ page }) => { + await page.getByRole("tab", { name: "JSON" }).click(); + await replaceEditor( + page, + "Prediction input JSON", + '{\n "text": "folded",\n "metadata": {\n "count": 1\n }\n}', + ); + + const fold = page.locator('.json-input .cm-foldGutter [title="Fold line"]').first(); + await expect(fold).toBeVisible(); + await fold.click(); + await expect(page.locator(".json-input .cm-foldPlaceholder")).toBeVisible(); +}); + +test("validates Form and JSON input against OpenAPI before running", async ({ page }) => { + const run = page.getByRole("button", { name: "Run" }); + const text = page.getByRole("textbox", { name: "text" }); + await text.fill("123"); + await expect(run).toBeEnabled(); + await expect(text).not.toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Does not match the required pattern.")).toHaveCount(0); + + await run.click(); + await expect(text).toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Does not match the required pattern.").first()).toBeVisible(); + + await text.fill("valid input"); + await expect(run).toBeEnabled(); + await expect(text).not.toHaveAttribute("aria-invalid", "true"); + + await page.getByRole("tab", { name: "JSON" }).click(); + const editor = page.getByRole("textbox", { name: "Prediction input JSON" }); + await replaceEditor(page, "Prediction input JSON", '{"text":42}'); + await expect(run).toBeEnabled(); + await expect(editor).not.toHaveAttribute("aria-invalid", "true"); + + await run.click(); + await expect(editor).toHaveAttribute("aria-invalid", "true"); + await expect(page.getByText("Input does not match the OpenAPI schema.")).toBeVisible(); + + await replaceEditor(page, "Prediction input JSON", '{"text":"valid"}'); + await expect(run).toBeEnabled(); + await expect(editor).not.toHaveAttribute("aria-invalid", "true"); +}); + +test("runs a synchronous prediction and inspects its response", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("sync"); + await page.getByRole("button", { name: "Sync", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello sync"); + + await page.getByRole("tab", { name: "Raw", exact: true }).click(); + const raw = page.getByRole("textbox", { name: "Raw prediction response" }); + await expect(raw).toHaveAttribute("aria-readonly", "true"); + await expect(raw).toHaveAttribute("contenteditable", "false"); + await expect(raw).toContainText('"status": "succeeded"'); + await expect(page.locator(".response-editor .cm-content span").first()).toBeVisible(); + + await page.getByRole("tab", { name: "Timeline" }).click(); + await expect(page.getByText(/POST \/predictions/)).toBeVisible(); + await page.getByRole("tab", { name: "Request" }).click(); + await expect(page.getByText("Prediction time")).toBeVisible(); + await expect(page.getByText("200", { exact: true })).toBeVisible(); + await page.getByText("Response headers", { exact: true }).click(); + const responseHeaders = page.getByLabel("Response headers", { exact: true }); + await expect(responseHeaders).toContainText("content-type"); + await expect(responseHeaders).not.toContainText("content-security-policy"); + await expect(responseHeaders).not.toContainText("x-frame-options"); +}); + +test("keeps model targets isolated across browser workspaces", async ({ context, page }) => { + const firstModel = await startTestModel("first model"); + const secondModel = await startTestModel("second model"); + const secondPage = await context.newPage(); + const thirdPage = await context.newPage(); + + try { + await Promise.all([secondPage.goto("/"), thirdPage.goto("/")]); + await Promise.all([ + connectTarget(page, firstModel.url), + connectTarget(secondPage, secondModel.url), + connectTarget(thirdPage, firstModel.url), + ]); + + await page.getByRole("textbox", { name: "text" }).fill("one"); + await secondPage.getByRole("textbox", { name: "text" }).fill("two"); + await thirdPage.getByRole("textbox", { name: "text" }).fill("three"); + await Promise.all([ + page.getByRole("button", { name: "Run" }).click(), + secondPage.getByRole("button", { name: "Run" }).click(), + thirdPage.getByRole("button", { name: "Run" }).click(), + ]); + + await expect(predictionOutput(page)).toContainText("first model: one"); + await expect(predictionOutput(secondPage)).toContainText("second model: two"); + await expect(predictionOutput(thirdPage)).toContainText("first model: three"); + } finally { + await Promise.all([secondPage.close(), thirdPage.close()]); + await Promise.all([firstModel.close(), secondModel.close()]); + } +}); + +test("shows progressive streaming output before completion", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("slow"); + await page.getByRole("button", { name: "Stream", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + + const output = predictionOutput(page); + await expect(output).toHaveAttribute("aria-busy", "true"); + await expectPredictionStatus(page, "processing"); + await expect(output).toContainText("hello", { timeout: 30_000 }); + await expect(output).not.toContainText("slow"); + + await expectPredictionStatus(page, "succeeded"); + await expect(output).toHaveAttribute("aria-busy", "false"); + await expect(output).toContainText("hello slow"); +}); + +test("stops a running prediction", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("slow"); + await page.getByRole("textbox", { name: "Prediction ID" }).fill("playground-stop-id"); + await page.getByRole("button", { name: "Stream", exact: true }).click(); + await page.getByRole("button", { name: "Run" }).click(); + await expect(page.getByRole("button", { name: "Stop" })).toBeEnabled(); + await expect(predictionOutput(page)).toContainText("hello", { timeout: 30_000 }); + + const cancelResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + new URL(response.url()).pathname === "/proxy/predictions/playground-stop-id/cancel", + ); + await page.getByRole("button", { name: "Stop" }).click(); + expect((await cancelResponse).ok()).toBe(true); + + await expectPredictionStatus(page, "canceled"); + await expect(page.getByRole("button", { name: "Run" })).toBeEnabled(); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible({ + timeout: 30_000, + }); +}); + +test("configures and receives an asynchronous webhook prediction", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("webhook"); + await page.getByRole("button", { name: "Async", exact: true }).click(); + await expect(page.getByText(/Webhook: .*\/webhook\/\.\.\./)).toBeVisible(); + await page.getByRole("checkbox", { name: "output" }).click(); + await page.getByRole("checkbox", { name: "logs" }).click(); + await expect(page.getByRole("checkbox", { name: "completed" })).toHaveAttribute( + "aria-disabled", + "true", + ); + await page.getByRole("button", { name: "Run" }).click(); + + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello webhook"); + await page.getByRole("tab", { name: "Request" }).click(); + const requestBody = page.getByLabel("Request body", { exact: true }); + await expect(requestBody).toContainText("start"); + await expect(requestBody).toContainText("completed"); + await expect(requestBody).not.toContainText('"logs"'); +}); + +test("reconnects and opens the loaded schema", async ({ page }) => { + const target = page.getByRole("textbox", { name: "Target" }); + const targetURL = await target.inputValue(); + await target.fill(" "); + await expect(page.getByRole("button", { name: "Connect" })).toBeDisabled(); + await target.fill(`${targetURL}/`); + const schemaResponse = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/proxy/openapi.json", + ); + await target.press("Enter"); + expect((await schemaResponse).ok()).toBe(true); + await expect(page.getByRole("status").filter({ hasText: "ready" })).toBeVisible(); + + const popupPromise = page.waitForEvent("popup"); + await page.getByRole("button", { name: "Schema" }).click(); + const popup = await popupPromise; + await popup.waitForLoadState(); + const schema = JSON.parse((await popup.locator("body").textContent()) ?? "") as { + openapi?: string; + }; + expect(schema.openapi).toMatch(/^3\./); +}); + +test("toggles the color theme", async ({ page }) => { + const root = page.locator("html"); + const initialTheme = await root.getAttribute("data-mode"); + const nextTheme = initialTheme === "dark" ? "light" : "dark"; + const toggleLabel = initialTheme === "dark" ? "Light" : "Dark"; + await page.getByRole("button", { name: toggleLabel }).click(); + await expect(root).toHaveAttribute("data-mode", nextTheme); +}); + +test("uses a custom prediction ID and resets the playground", async ({ page }) => { + await page.getByRole("textbox", { name: "text" }).fill("identified"); + await page.getByRole("textbox", { name: "Prediction ID" }).fill("playground-e2e-id"); + await page.getByRole("button", { name: "Sync", exact: true }).click(); + const predictionRequest = page.waitForRequest( + (request) => + request.method() === "PUT" && + new URL(request.url()).pathname === "/proxy/predictions/playground-e2e-id", + ); + await page.getByRole("button", { name: "Run" }).click(); + await predictionRequest; + await expectPredictionStatus(page, "succeeded"); + await expect(predictionOutput(page)).toContainText("hello identified"); + + await page.getByRole("button", { name: "Reset" }).click(); + await expect(page.getByRole("textbox", { name: "text" })).toHaveValue(""); + await expect(page.getByRole("textbox", { name: "Prediction ID" })).toHaveValue(""); + await expect(page.getByText("Run a prediction to see its output.")).toBeVisible(); +}); + +test("fits every primary control on a narrow viewport", async ({ page }) => { + await page.setViewportSize({ width: 320, height: 800 }); + + const widths = await page.evaluate(() => ({ + document: document.documentElement.scrollWidth, + viewport: document.documentElement.clientWidth, + })); + expect(widths.document).toBeLessThanOrEqual(widths.viewport); + await expect(page.getByRole("button", { name: "Run" })).toBeVisible(); + await expect(page.getByRole("tab", { name: "Form" })).toBeVisible(); + await expect(page.getByRole("tab", { name: "Output" })).toBeVisible(); +}); + +async function replaceEditor(page: Page, label: string, value: string): Promise { + const editor = page.getByRole("textbox", { name: label }); + await editor.click(); + await page.keyboard.press("ControlOrMeta+A"); + await page.keyboard.insertText(value); +} + +async function expectPredictionStatus(page: Page, status: string): Promise { + await expect(page.locator("#output-panel .response-title").getByRole("status")).toHaveText( + status, + { timeout: 60_000 }, + ); +} + +function predictionOutput(page: Page) { + return page.getByRole("region", { name: "Prediction output" }); +} + +async function connectTarget(page: Page, target: string): Promise { + const schema = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/proxy/openapi.json" && + response.request().headers()["x-cog-target"] === target, + ); + await page.getByRole("textbox", { name: "Target" }).fill(target); + await page.getByRole("button", { name: "Connect" }).click(); + expect((await schema).ok()).toBe(true); + await expect(page.getByRole("textbox", { name: "text" })).toBeVisible(); +} + +async function startTestModel(name: string): Promise<{ url: string; close: () => Promise }> { + const server = createServer((request, response) => { + response.setHeader("Content-Type", "application/json"); + if (request.url === "/health-check") { + response.end(JSON.stringify({ status: "READY" })); + return; + } + if (request.url === "/openapi.json") { + response.end( + JSON.stringify({ + components: { + schemas: { + Input: { + type: "object", + required: ["text"], + properties: { text: { type: "string" } }, + }, + }, + }, + paths: { "/predictions": { post: {} } }, + }), + ); + return; + } + if (request.url === "/predictions" && request.method === "POST") { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => (body += chunk)); + request.on("end", () => { + const prediction = JSON.parse(body) as { input: { text: string } }; + response.end( + JSON.stringify({ status: "succeeded", output: `${name}: ${prediction.input.text}` }), + ); + }); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Test model did not bind a port"); + return { + url: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} diff --git a/playground/e2e/serve.ts b/playground/e2e/serve.ts new file mode 100644 index 0000000000..f5425f2d21 --- /dev/null +++ b/playground/e2e/serve.ts @@ -0,0 +1,141 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { createServer } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const cog = process.env.COG_BINARY; +if (!cog) throw new Error("COG_BINARY must point to the built Cog CLI"); +const cogBinary = cog; + +const fixture = fileURLToPath(new URL("./fixture/", import.meta.url)); +const modelPort = await availablePort(); +const modelURL = `http://127.0.0.1:${modelPort}`; +const playgroundPort = Number(process.env.PLAYGROUND_E2E_PORT ?? 8400); +const playgroundURL = `http://127.0.0.1:${playgroundPort}`; +const children: ChildProcess[] = []; +const childFailures: Promise[] = []; +let stopping = false; + +function availablePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close((error) => { + if (error) reject(error); + else if (typeof address === "object" && address) resolve(address.port); + else reject(new Error("Could not allocate a model port")); + }); + }); + }); +} + +function start(name: string, args: string[], cwd: string): Promise { + const child = spawn(cogBinary, args, { + cwd, + env: { ...process.env, BUILDKIT_PROGRESS: "quiet" }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout?.on("data", (data: Buffer) => process.stdout.write(`[${name}] ${data}`)); + child.stderr?.on("data", (data: Buffer) => process.stderr.write(`[${name}] ${data}`)); + const failure = new Promise((_, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => { + if (!stopping) reject(new Error(`${name} exited unexpectedly (${code ?? signal})`)); + }); + }); + children.push(child); + childFailures.push(failure); + return failure; +} + +async function waitFor(url: string, timeout: number): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2000) }); + if (response.ok) return; + } catch { + // The process is still starting. + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${url}`); +} + +async function waitForModel(timeout: number): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + try { + const response = await fetch(`${modelURL}/health-check`, { + signal: AbortSignal.timeout(2000), + }); + if (response.ok) { + const health = (await response.json()) as { status?: string }; + if (health.status === "READY" || health.status === "BUSY") return; + if (health.status === "SETUP_FAILED" || health.status === "DEFUNCT") { + throw new Error(`Model startup failed with status ${health.status}`); + } + } + } catch (error) { + if (error instanceof Error && error.message.startsWith("Model startup failed")) throw error; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${modelURL}`); +} + +async function stop(): Promise { + if (stopping) return; + stopping = true; + try { + await fetch(`${modelURL}/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(5000), + }); + } catch { + // Fall back to process signals below. + } + for (const child of children) child.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 3000)); + for (const child of children) { + if (child.exitCode === null) child.kill("SIGKILL"); + } +} + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + void stop().finally(() => process.exit(0)); + }); +} + +try { + // Enable Cog's host-gateway mapping so webhooks reach the host on Linux CI. + const modelFailure = start( + "model", + ["serve", "--port", String(modelPort), "--upload-url", "http://unused/"], + fixture, + ); + await Promise.race([waitForModel(12 * 60_000), modelFailure]); + const playgroundFailure = start( + "playground", + [ + "playground", + "--host", + "0.0.0.0", + "--port", + String(playgroundPort), + "--target", + modelURL, + "--no-open", + ], + fixture, + ); + await Promise.race([waitFor(playgroundURL, 30_000), playgroundFailure]); + await Promise.race(childFailures); +} catch (error) { + console.error(error); + await stop(); + process.exit(1); +} diff --git a/playground/playwright.config.ts b/playground/playwright.config.ts new file mode 100644 index 0000000000..e8fb004c00 --- /dev/null +++ b/playground/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from "@playwright/test"; + +const playgroundPort = Number(process.env.PLAYGROUND_E2E_PORT ?? 8400); +const playgroundURL = `http://127.0.0.1:${playgroundPort}`; + +export default defineConfig({ + testDir: "./e2e", + fullyParallel: false, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", + timeout: 60_000, + use: { + baseURL: playgroundURL, + screenshot: "only-on-failure", + trace: "retain-on-failure", + video: "retain-on-failure", + }, + webServer: { + command: "node ./e2e/serve.ts", + url: playgroundURL, + reuseExistingServer: false, + timeout: 15 * 60_000, + gracefulShutdown: { signal: "SIGTERM", timeout: 10_000 }, + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], +}); diff --git a/playground/scripts/merge-licenses.ts b/playground/scripts/merge-licenses.ts new file mode 100644 index 0000000000..e31073da00 --- /dev/null +++ b/playground/scripts/merge-licenses.ts @@ -0,0 +1,29 @@ +import { readFileSync, unlinkSync, writeFileSync } from "node:fs"; + +// THIRD_PARTY_LICENSES.md is not used at runtime. Playground JS (React, editor +// deps, ajv in the validation worker, etc.) is bundled into assets embedded in +// the cog binary and redistributed to users; many OSS licenses require shipping +// those notices with the software. This file is that compliance record. +const outputDirectory = new URL("../../pkg/cli/playground/", import.meta.url); +const mainPath = new URL("THIRD_PARTY_LICENSES.md", outputDirectory); +const workerPath = new URL("WORKER_LICENSES.md", outputDirectory); +const main = readFileSync(mainPath, "utf8").trimEnd(); +const worker = readFileSync(workerPath, "utf8"); +const existing = new Set([...main.matchAll(/^## (.+?) - .+$/gm)].map((match) => match[1])); +const workerSections = worker + .split(/(?=^## )/m) + .slice(1) + .filter((section) => { + const name = /^## (.+?) - /m.exec(section)?.[1]; + return name && !existing.has(name); + }); +const merged = `${main}\n\n${workerSections.join("\n").trim()}\n`; + +for (const dependency of ["ajv", "ajv-draft-04", "ajv-formats", "fast-uri"]) { + if (!merged.includes(`## ${dependency} - `)) { + throw new Error(`Worker dependency ${dependency} is missing from THIRD_PARTY_LICENSES.md`); + } +} + +writeFileSync(mainPath, merged); +unlinkSync(workerPath); diff --git a/playground/src/app/App.test.tsx b/playground/src/app/App.test.tsx new file mode 100644 index 0000000000..7e97a6980a --- /dev/null +++ b/playground/src/app/App.test.tsx @@ -0,0 +1,303 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { useEffect } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { createInputValidator } from "@/features/inputs/validation/inputValidation"; +import type { ValidationIssue } from "@/features/inputs/validation/inputValidation"; +import { deferred } from "@/test/deferred"; + +const run = vi.fn(); +const reset = vi.fn(); +const { mockValidateInput } = vi.hoisted(() => ({ mockValidateInput: vi.fn() })); + +vi.mock("@/features/connection/hooks/useConnection", () => { + const connection = { + target: "http://localhost:8393", + targetDraft: "http://localhost:8393", + setTargetDraft: vi.fn(), + connect: vi.fn(), + health: { status: "ready", version: { python_sdk: "0.21.0" } }, + schema: { + components: { + schemas: { + Input: { + type: "object", + required: ["prompt"], + properties: { + prompt: { type: "string" }, + }, + }, + Output: { type: "string" }, + }, + }, + paths: { "/predictions": { post: {} } }, + }, + schemaError: "", + webhookBase: "", + cogVersion: "0.21.1-dev+g0db4bffa", + capabilities: { + endpoint: "/predictions", + input: { + type: "object", + required: ["prompt"], + properties: { + prompt: { type: "string" }, + }, + }, + output: { type: "string" }, + streaming: false, + async: false, + }, + }; + return { useConnection: () => connection }; +}); + +vi.mock("@/features/predictions/hooks/usePrediction", () => ({ + usePrediction: () => ({ + running: false, + envelope: undefined, + output: undefined, + rawEvents: [], + error: "", + trace: undefined, + run, + stop: vi.fn(), + reset, + }), +})); + +vi.mock("@/features/inputs/validation/validateInput", () => ({ + validateInput: mockValidateInput, + disposeValidationWorker: vi.fn(), +})); + +vi.mock("@/features/inputs/components/InputForm", () => ({ + InputForm: ({ + onChange, + onValidityChange, + value, + }: { + onChange: (value: Record) => void; + onValidityChange: (valid: boolean) => void; + value: Record; + }) => { + const prompt = String(value.prompt ?? ""); + useEffect(() => onValidityChange(true), [onValidityChange]); + return ( +