live-e2e.yml

name: Live E2E

# The real-model browser suite of #204: four task checks driven through the FlyBot web
# UI on a fresh isolated bench, against the real DeepSeek endpoint. Separate from the
# compatibility matrix (nightly.yml) and from every PR check; never added to `tests-ok`
# or branch protection. This is the repository's only workflow that reads a secret: the
# live job runs on main only, the key is mapped step-scoped, and everything uploaded has
# passed the runner's evidence sanitizer (no `token=`, no key material, no cookies) and
# is an explicit allow-list — never a raw bench home.
#
# Without the repository secret DEEPSEEK_API_KEY the suite cannot run at all, and a run
# that is red every night for a reason nobody can act on teaches the team to ignore red
# (#794). So a missing key warns, skips the six steps that need it, and leaves the run
# green while the step summary says plainly that nothing ran and nothing was measured. A
# real failure of the suite still fails the run: the skip is never a pass.
#
# The `mock` job below is the keyless half (#2565): the same bench, adapter and evidence
# path, with DEEPSEEK_BASE_URL pointed at the scripted mock (core/dev/mock-llm.mjs). It
# reads no secret, so it runs on every pull request that touches core/ or plugins/, not only
# on main — and its step summary names each check it ran, so its green is visibly different
# from the live job's skip. The `live` job runs the same published runtime as the mock job;
# it differs in the key, the usage tap and the four real-model checks (it used to clone the
# harness at the pinned tag and never build it, so `dsh web` died at startup with ENOENT).

on:
  schedule:
    - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z
  workflow_dispatch:
  pull_request:
    paths:
      - "core/**"
      - "plugins/**"
      - "dev/live-e2e/**"
      - ".github/workflows/live-e2e.yml"

# Per ref: a pull request's mock run must not sit in the nightly's queue (or start one).
concurrency:
  group: live-e2e-${{ github.ref }}
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  window:
    name: window check
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - name: Check out main with full history
        if: github.event_name == 'schedule'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Decide whether main has new commits
        id: gate
        run: |
          run=false
          reason=""
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            run=true
            reason="manual dispatch always runs"
          elif [ -n "$(git log --since='24 hours ago' --pretty=oneline)" ]; then
            run=true
            reason="main had commits in the last 24 hours"
          else
            reason="main had no commits in the window; no tokens spent"
          fi
          echo "run=$run" >> "$GITHUB_OUTPUT"
          {
            echo "### Live E2E run decision"
            echo ""
            echo "- event: \`${{ github.event_name }}\`"
            echo "- run: \`${run}\`: ${reason}"
          } >> "$GITHUB_STEP_SUMMARY"

  live:
    name: live · ubuntu · Node 24 · real model
    needs: window
    if: needs.window.outputs.run == 'true' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      WEB_PORT: '3196'
      TAP_PORT: '8898'
      BENCH: live-e2e
      MODEL: deepseek-official/deepseek-flash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: core/package-lock.json
      # The key decides whether anything can run, so it is checked once and published as an
      # output that every step below gates on. The decision itself lives in a script with tests
      # (.github/scripts/live-e2e-preflight.test.mjs): a missing key warns and skips, never fails.
      - name: Preflight (a missing key skips every step, and says so)
        id: preflight
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
        run: node .github/scripts/live-e2e-preflight.mjs
      - name: npm ci (core)
        if: steps.preflight.outputs.key == 'true'
        working-directory: core
        run: npm ci --no-fund --no-audit
        timeout-minutes: 5
      # The runtime is the published package, not a source checkout: a checkout has no built
      # client artifacts (dsh web exits with ENOENT on packages/client/*/lib/client.js), and
      # this job must not spend ten minutes building DSH to run four checks. Same runtime,
      # playwright-core and node_modules/.pnpm layout as the mock job below: `pnpm dsh`
      # starts the bench and core/dev/desktop-linux/lib.mjs resolves playwright-core from
      # .pnpm/node_modules.
      - name: Install the pinned DSH runtime
        if: steps.preflight.outputs.key == 'true'
        run: |
          DSH=$(node -p "require('./core/dev/harness/package.json').dependencies['@deepseek-ai/dsh']")
          RUNTIME="$RUNNER_TEMP/dsh-runtime"
          mkdir -p "$RUNTIME"
          cd "$RUNTIME"
          printf '{"name":"dsh-live-e2e-runtime","private":true,"packageManager":"pnpm@12.16.1"}\n' > package.json
          # pnpm 12 refuses dependency build scripts by default; the pinned runtime's native
          # helpers (subprocess spawn, pty, protobuf) are the same packages the mock job runs.
          printf 'dangerouslyAllowAllBuilds: true\n' > pnpm-workspace.yaml
          corepack enable
          corepack pnpm add "@deepseek-ai/dsh@$DSH" playwright-core@1.61.1
          echo "DSH_HARNESS=$RUNTIME" >> "$GITHUB_ENV"
        timeout-minutes: 8
      # The headless shell is what the adapter launches. No --with-deps: the runner image
      # already carries Chrome's system libraries, so no apt-get round; one retry covers the
      # download itself.
      - name: Chromium for the browser
        if: steps.preflight.outputs.key == 'true'
        run: |
          for attempt in 1 2; do
            node "$DSH_HARNESS/node_modules/playwright-core/cli.js" install chromium --only-shell && exit 0
            echo "::warning::chromium install attempt $attempt failed; retrying"
            sleep 5
          done
          exit 1
        timeout-minutes: 8
      - name: Start the usage tap
        if: steps.preflight.outputs.key == 'true'
        run: |
          # The bench home lives outside the checkout (#398): a workspace inside the repository
          # would hand the grader to the Bot under test. bench.mjs and run.mjs read the same
          # variable, and the default is the same on both sides.
          LIVE_E2E_HOME="$HOME/.cache/flybot-live-e2e/$BENCH"
          echo "LIVE_E2E_HOME=$LIVE_E2E_HOME" >> "$GITHUB_ENV"
          mkdir -p "$LIVE_E2E_HOME"
          setsid node core/dev/llm-tap.mjs "$TAP_PORT" https://api.deepseek.com "$LIVE_E2E_HOME/tap.jsonl" \
            </dev/null >"$LIVE_E2E_HOME/tap.out" 2>&1 & echo $! >"$LIVE_E2E_HOME/tap.pid"
          i=0
          until (exec 3<>"/dev/tcp/127.0.0.1/$TAP_PORT") 2>/dev/null; do
            i=$((i + 1))
            [ $i -ge 30 ] && { echo "::error::the usage tap did not start"; cat "$LIVE_E2E_HOME/tap.out"; exit 1; }
            sleep 1
          done
      - name: Start the real-model bench
        if: steps.preflight.outputs.key == 'true'
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
        run: |
          DEEPSEEK_BASE_URL="http://127.0.0.1:$TAP_PORT/anthropic" node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset
        timeout-minutes: 8
      - name: Run the four checks
        if: steps.preflight.outputs.key == 'true'
        id: run
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          DEEPSEEK_BASE_URL: http://127.0.0.1:8898/anthropic
          LIVE_E2E_TAP_UPSTREAM: https://api.deepseek.com
        run: |
          node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" "$WEB_PORT" \
            --model "$MODEL" --harness "$DSH_HARNESS" \
            --tap "$LIVE_E2E_HOME/tap.jsonl" --evidence evidence --tz Asia/Shanghai
      - name: Stop the bench and the tap
        if: always() && steps.preflight.outputs.key == 'true'
        run: |
          node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true
          if [ -f "$LIVE_E2E_HOME/tap.pid" ]; then
            kill -TERM "-$(cat "$LIVE_E2E_HOME/tap.pid")" 2>/dev/null || true
          fi
      - name: Sanitized web log (diagnostics only, never uploaded raw)
        if: failure()
        run: tail -80 "$LIVE_E2E_HOME/web.log" | grep -v 'token=' || true
      # Fail closed twice: the runner writes evidence/sanitized.txt only after a clean
      # scan, and the upload requires that stamp. A bundle the sanitizer flagged (or one
      # from a run that never reached the scan) never becomes a public artifact.
      - name: Evidence gate
        id: evidence-gate
        if: always() && steps.run.outcome != 'skipped'
        run: |
          if [ ! -f evidence/sanitized.txt ]; then
            echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded"
            exit 1
          fi
      # Only the stamped allow-list is uploaded: the report, the screenshots, the
      # transcript — never the bench home, the tap log or any raw log.
      - name: Upload evidence (allow-list, sanitizer-stamped)
        if: always() && steps.evidence-gate.outcome == 'success'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: live-e2e-evidence
          path: evidence/
          if-no-files-found: ignore
          retention-days: 14

  # The keyless browser suite (#2565). Same bench, web adapter and evidence path as the live
  # job, with DEEPSEEK_BASE_URL on the scripted mock and no secret in the environment at all:
  # the two #204 task checks cannot run here (a scripted model answers a task prompt with a
  # canned acknowledgement and writes no files), so this job measures the class of regression
  # the repository had no browser-level check for — a message goes in through the composer, a
  # reply comes out, and both survive a reload.
  mock:
    name: mock · ubuntu · Node 24 · scripted model
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      WEB_PORT: '3197'
      MOCK_PORT: '8765'
      BENCH: live-e2e-mock
      MODEL: deepseek-official/deepseek-flash
      LIVE_E2E_MODE: mock
      DEEPSEEK_BASE_URL: http://127.0.0.1:8765
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: core/package-lock.json
      - name: npm ci (core)
        working-directory: core
        run: npm ci --no-fund --no-audit
        timeout-minutes: 5
      # The runtime is the published package, not a source checkout: a checkout has no built
      # client artifacts (dsh web exits with ENOENT on packages/client/*/lib/client.js), and
      # this job must not spend ten minutes building DSH to prove a reply still renders. pnpm
      # keeps the node_modules/.pnpm layout the adapter resolves playwright-core from.
      - name: Install the pinned DSH runtime
        run: |
          DSH=$(node -p "require('./core/dev/harness/package.json').dependencies['@deepseek-ai/dsh']")
          RUNTIME="$RUNNER_TEMP/dsh-runtime"
          mkdir -p "$RUNTIME"
          cd "$RUNTIME"
          printf '{"name":"dsh-live-e2e-runtime","private":true,"packageManager":"pnpm@12.16.1"}\n' > package.json
          # pnpm 12 refuses dependency build scripts by default; the pinned runtime's native
          # helpers (subprocess spawn, pty, protobuf) are the same packages the live job runs.
          printf 'dangerouslyAllowAllBuilds: true\n' > pnpm-workspace.yaml
          corepack enable
          corepack pnpm add "@deepseek-ai/dsh@$DSH" playwright-core@1.61.1
          echo "DSH_HARNESS=$RUNTIME" >> "$GITHUB_ENV"
        timeout-minutes: 8
      # The headless shell is what the adapter launches. No --with-deps: the runner image
      # already carries Chrome's system libraries, and an apt-get round there is the slow,
      # flaky half (a run timed out on it); one retry covers the download itself.
      - name: Chromium for the browser
        run: |
          for attempt in 1 2; do
            node "$DSH_HARNESS/node_modules/playwright-core/cli.js" install chromium --only-shell && exit 0
            echo "::warning::chromium install attempt $attempt failed; retrying"
            sleep 5
          done
          exit 1
        timeout-minutes: 8
      - name: Start the scripted mock
        run: |
          # The bench home lives outside the checkout (#398), exactly as in the live job.
          LIVE_E2E_HOME="$HOME/.cache/flybot-live-e2e/$BENCH"
          echo "LIVE_E2E_HOME=$LIVE_E2E_HOME" >> "$GITHUB_ENV"
          mkdir -p "$LIVE_E2E_HOME"
          MOCK_LOG="$LIVE_E2E_HOME/mock.jsonl" setsid node core/dev/mock-llm.mjs \
            </dev/null >"$LIVE_E2E_HOME/mock.out" 2>&1 & echo $! >"$LIVE_E2E_HOME/mock.pid"
          i=0
          until (exec 3<>"/dev/tcp/127.0.0.1/$MOCK_PORT") 2>/dev/null; do
            i=$((i + 1))
            [ $i -ge 30 ] && { echo "::error::the scripted mock did not start"; cat "$LIVE_E2E_HOME/mock.out"; exit 1; }
            sleep 1
          done
        timeout-minutes: 2
      - name: Start the mock bench
        run: node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset
        timeout-minutes: 8
      # The summary carries the runner's own lines: one `ok`/`FAIL` per check it ran, the
      # mode, and the outcome — so a green that measured nothing cannot look like this one.
      - name: Run the keyless checks
        id: run
        run: |
          set -o pipefail
          {
            echo "### Live E2E (mock, no key)"
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"
          # The fence closes and the step still fails on the runner's exit code: a summary
          # that ends in a stray `echo` would make a red run look green.
          status=0
          node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" "$WEB_PORT" \
            --model "$MODEL" --harness "$DSH_HARNESS" --evidence evidence --tz Asia/Shanghai \
            | tee -a "$GITHUB_STEP_SUMMARY" || status=$?
          echo '```' >> "$GITHUB_STEP_SUMMARY"
          exit $status
      - name: Stop the bench and the mock
        if: always()
        run: |
          node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true
          if [ -f "$LIVE_E2E_HOME/mock.pid" ]; then
            kill -TERM "-$(cat "$LIVE_E2E_HOME/mock.pid")" 2>/dev/null || true
          fi
      - name: Sanitized web log (diagnostics only, never uploaded raw)
        if: failure()
        run: tail -80 "$LIVE_E2E_HOME/web.log" | grep -v 'token=' || true
      - name: Evidence gate
        id: evidence-gate
        if: always() && steps.run.outcome != 'skipped'
        run: |
          if [ ! -f evidence/sanitized.txt ]; then
            echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded"
            exit 1
          fi
      - name: Upload evidence (allow-list, sanitizer-stamped)
        if: always() && steps.evidence-gate.outcome == 'success'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: live-e2e-mock-evidence
          path: evidence/
          if-no-files-found: ignore
          retention-days: 14