java-playwright-nightly.yml

#  Copyright 2026 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

# Full run of the UI integration suite (*UIIT.java) — lives inside
# openmetadata-integration-tests under the `ui-it` Maven profile. Runs the
# external-mode matrix (ES + OS). Tracks EPIC #3731 / tickets #3767, #3792.
#
# The schedule trigger is intentionally disabled while the suite stabilises. Run it
# on demand via workflow_dispatch (pick the branch to test as the ref); it also runs
# automatically on any PR that touches search-indexing code (see the pull_request
# `paths` filter below), exercising this whole suite (both engines + search-it) against
# the PR head in embedded mode. Re-add `schedule: - cron: '0 2 * * *'` once green on main.
#
# Topology:
#   - build-image: builds the openmetadata-dist tarball and bakes the local
#     openmetadata-server:jpw-snapshot image (BuildKit + GHA layer cache), saves it
#     to a workflow artifact, and primes the Maven cache for downstream jobs.
#   - ui-it-nightly (x2): matrix over [opensearch, elasticsearch]. Both download the
#     pre-built image artifact, restore the Maven cache, then run `mvn verify -P ui-it`
#     with OM_TEST_IMAGE pointing at the pre-loaded tag so ContainerizedServer skips
#     its own build.
#   - search-it-nightly: server-global destructive search ITs (tests/search/*IT.java) via
#     `mvn verify -P search-it`. They run on the embedded bootstrap (testcontainers), not the
#     baked image, so this job is independent of build-image and runs them serially.

name: JavaUIIT Integration Tests (Nightly)

on:
  workflow_dispatch:
    inputs:
      includeSsoBootstrap:
        description: 'Also run @Tag("sso-bootstrap") UIITs (slower; second OM lifecycle)'
        required: false
        type: boolean
        default: false
      simpleReindexTables:
        description: 'SimpleReindexTriggerUIIT table cohort size'
        required: false
        type: string
        default: '5000'
      simpleReindexTopics:
        description: 'SimpleReindexTriggerUIIT topic cohort size'
        required: false
        type: string
        default: '1500'
      simpleReindexDashboards:
        description: 'SimpleReindexTriggerUIIT dashboard cohort size'
        required: false
        type: string
        default: '1500'
      simpleReindexPipelines:
        description: 'SimpleReindexTriggerUIIT pipeline cohort size'
        required: false
        type: string
        default: '2000'
      searchAvailableTables:
        description: 'SearchAvailableDuringReindexUIIT table cohort size'
        required: false
        type: string
        default: '5000'
      reindexTimeoutMin:
        description: 'Per-run reindex completion/acceptance timeout (minutes) for ui-it & search-it'
        required: false
        type: string
        default: '60'
  # Auto-run on PRs that modify search-indexing code (paths below): the full suite runs against
  # the PR head in embedded mode (testcontainers), NOT the external cluster — so a change to the
  # reindex app, search clients, or index mappings is validated before merge without anyone
  # remembering a label. Uses `pull_request` (NOT pull_request_target) on purpose: fork PRs run
  # WITHOUT secrets, so untrusted code never executes with credentials; same-repo branch PRs (the
  # common case here) still get secrets and full check reporting.
  pull_request:
    paths:
      # --- Production search-indexing code (openmetadata-service) ---
      - "openmetadata-service/src/main/java/org/openmetadata/service/search/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/resources/search/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/resources/testsupport/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/events/lifecycle/handlers/SearchIndexHandler.java"
      # --- Index mappings + loader (canonical source lives in the spec module) ---
      - "openmetadata-spec/src/main/resources/elasticsearch/**"
      - "openmetadata-spec/src/main/java/org/openmetadata/search/**"
      # --- The search ITs / UIITs, their test harness, and the Maven profiles that run them ---
      - "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/search/**"
      - "openmetadata-integration-tests/src/test/java/org/openmetadata/it/search/**"
      - "openmetadata-integration-tests/src/test/java/org/openmetadata/playwright/scenarios/search/**"
      - "openmetadata-integration-tests/pom.xml"
      # --- This workflow itself ---
      - ".github/workflows/java-playwright-nightly.yml"

permissions:
  contents: read
  checks: write

concurrency:
  # One run per PR (or per ref for dispatch/cron); a new push cancels the prior in-flight run.
  group: java-playwright-nightly-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  SERVER_IMAGE_TAG: openmetadata-server:jpw-snapshot

jobs:
  build-image:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      ref-sha: ${{ steps.checkout.outputs.commit }}
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: false
          swap-storage: true

      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
        with:
          # On a PR, test the PR head (the changed code). workflow_dispatch honours the
          # dispatched ref so feature branches can validate the nightly matrix before merge.
          # Cron/anything else runs against main (EPIC #3731 / PR #28008).
          ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Install Ubuntu dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          libevent-dev jq
          sudo make install_antlr_cli

      - name: Build dependencies for integration-tests
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am

      - name: Build openmetadata-dist tarball
        # ContainerizedServer.buildLocalImageContainer needs
        # openmetadata-dist/target/openmetadata-*.tar.gz to bake the local image.
        # Dist is NOT a transitive dep of integration-tests, so the previous step
        # doesn't produce it — build it explicitly here.
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: mvn -DskipTests install -pl :openmetadata-dist -am

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build openmetadata-server image (with GHA layer cache)
        uses: docker/build-push-action@v6
        with:
          context: .
          file: docker/development/Dockerfile
          tags: ${{ env.SERVER_IMAGE_TAG }}
          load: true
          # Shared scope across matrix entries so the cache primed by this build
          # is reused by everything that runs this workflow on the same SHA (or
          # later runs whose Dockerfile + dist tarball haven't changed).
          cache-from: type=gha,scope=jpw-server-image
          cache-to: type=gha,scope=jpw-server-image,mode=max

      - name: Export server image as artifact
        run: |
          docker save "${SERVER_IMAGE_TAG}" -o /tmp/jpw-server-image.tar
          ls -lh /tmp/jpw-server-image.tar

      - name: Upload server image artifact
        uses: actions/upload-artifact@v4
        with:
          name: jpw-server-image-${{ github.run_id }}
          path: /tmp/jpw-server-image.tar
          retention-days: 1

  ui-it-nightly:
    needs: build-image
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        searchEngine: [opensearch, elasticsearch]
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Install Ubuntu dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          libevent-dev jq
          sudo make install_antlr_cli

      - name: Add /etc/hosts entry for mock OIDC server
        # The SSO test infrastructure (MockOidcServer) needs `om-mock-idp` to resolve to
        # loopback on the host so the same URL works inside the Docker network and from
        # the host-side Playwright browser, keeping the issued tokens' `iss` claim
        # consistent across actors.
        run: echo "127.0.0.1 om-mock-idp" | sudo tee -a /etc/hosts

      - name: Build dependencies for integration-tests
        # Maven cache primed by build-image makes this a fast restore-only invocation.
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: mvn -DskipTests install -pl :openmetadata-integration-tests -am

      - name: Download server image artifact
        uses: actions/download-artifact@v4
        with:
          name: jpw-server-image-${{ github.run_id }}
          path: /tmp

      - name: Load server image into local Docker
        run: |
          docker load -i /tmp/jpw-server-image.tar
          docker image ls "${SERVER_IMAGE_TAG}"

      - name: Install Playwright browsers
        run: |
          mvn -pl :openmetadata-integration-tests dependency:build-classpath -Dmdep.outputFile=/tmp/cp.txt -q
          java -cp "$(cat /tmp/cp.txt)" com.microsoft.playwright.CLI install --with-deps chromium

      - name: Free build artifacts
        run: |
          rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
          rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
          rm -rf openmetadata-shaded-deps/*/target
          df -h /

      - name: Run UI integration tests (${{ matrix.searchEngine }})
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # OM_TEST_IMAGE makes ContainerizedServer skip its in-test `docker build`
          # and reuse the image artifact loaded above. Cuts ~10-90s per launch.
          OM_TEST_IMAGE: ${{ env.SERVER_IMAGE_TAG }}
          # UiSessionExtension reads PW_VIDEO and records every test's BrowserContext
          # to target/playwright-videos when true. Kept off locally; on in CI for triage.
          PW_VIDEO: 'true'
        run: |
          # Nightly bumps the per-test cohort sizes up to the historical stress numbers.
          # PR runs use the smaller defaults baked into each test so `mvn verify` is fast.
          # Sizes are workflow_dispatch inputs (defaults below) so they're tunable per run.
          # Embedded nightly recreates a fresh stack each run, so data never persists — these
          # small-footprint suites always create on the fly (jpw.data.mode=ingest). Static/ensure
          # reuse is only for the persistent external cluster (see java-playwright-external.yml).
          STRESS_PROPS="-Djpw.simpleReindex.tables=${{ github.event.inputs.simpleReindexTables || '5000' }} \
            -Djpw.simpleReindex.topics=${{ github.event.inputs.simpleReindexTopics || '1500' }} \
            -Djpw.simpleReindex.dashboards=${{ github.event.inputs.simpleReindexDashboards || '1500' }} \
            -Djpw.simpleReindex.pipelines=${{ github.event.inputs.simpleReindexPipelines || '2000' }} \
            -Djpw.searchAvailable.tables=${{ github.event.inputs.searchAvailableTables || '5000' }} \
            -Djpw.data.mode=ingest \
            -Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }}"
          # Opt-in for SSO bootstrap UIITs (second OM lifecycle); default off.
          if [ "${{ github.event.inputs.includeSsoBootstrap }}" = "true" ]; then
            SSO_PROPS="-Dui.it.excludedGroups="
          else
            SSO_PROPS=""
          fi
          if [ "${{ matrix.searchEngine }}" = "elasticsearch" ]; then
            mvn verify -P ui-it -pl :openmetadata-integration-tests $STRESS_PROPS $SSO_PROPS \
              -DsearchType=elasticsearch \
              -DsearchImage=docker.elastic.co/elasticsearch/elasticsearch:9.3.0
          else
            mvn verify -P ui-it -pl :openmetadata-integration-tests $STRESS_PROPS $SSO_PROPS
          fi

      - name: Upload Playwright traces
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: playwright-traces-${{ matrix.searchEngine }}-${{ github.run_id }}
          path: openmetadata-integration-tests/target/playwright-traces
          if-no-files-found: ignore
          retention-days: 14

      - name: Upload Playwright videos
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: playwright-videos-${{ matrix.searchEngine }}-${{ github.run_id }}
          path: openmetadata-integration-tests/target/playwright-videos
          if-no-files-found: ignore
          retention-days: 14

      - name: Upload Failsafe Reports
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: failsafe-reports-${{ matrix.searchEngine }}-${{ github.run_id }}
          path: openmetadata-integration-tests/target/failsafe-reports
          if-no-files-found: ignore
          retention-days: 14

      - name: Publish Test Report
        id: report
        if: ${{ always() }}
        uses: scacap/action-surefire-report@v1
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          # mvn verify already fails the job on test failures. This step just
          # publishes the check run with per-test details; it shouldn't itself
          # gate the job conclusion (otherwise a flake here breaks Slack).
          fail_on_test_failures: false
          report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'

      - name: Slack notification
        if: ${{ always() && github.event_name != 'pull_request' && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
        uses: slackapi/slack-github-action@v2.0.0
        with:
          webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
          webhook-type: incoming-webhook
          payload: |
            text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Playwright Nightly (${{ matrix.searchEngine }}): ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
        env:
          SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}

  search-it-nightly:
    # Server-global destructive search ITs (tests/search/*IT.java): full reindex with
    # recreateIndex, alias swaps, and ES container pause. They mutate cluster-wide state, so
    # the default PR profiles exclude them; here the `search-it` Maven profile runs them
    # serially on their own embedded-bootstrap server (testcontainers). That bootstrap is
    # independent of the baked image, so this job does not need build-image.
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - name: Install Ubuntu dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          libevent-dev jq
          sudo make install_antlr_cli

      - name: Build dependencies for integration-tests
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: mvn -DskipTests install -pl :openmetadata-integration-tests -am

      - name: Run isolated search ITs (postgres + opensearch)
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        # Embedded bootstrap = fresh stack each run; these search ITs create their cohorts on the
        # fly (jpw.data.mode=ingest). Static/ensure reuse is for the external cluster only.
        run: mvn verify -P search-it -pl :openmetadata-integration-tests -DdatabaseType=postgres -DsearchType=opensearch -Djpw.data.mode=ingest -Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }}

      - name: Upload Failsafe Reports
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: failsafe-reports-search-it-${{ github.run_id }}
          path: openmetadata-integration-tests/target/failsafe-reports
          if-no-files-found: ignore
          retention-days: 14

      - name: Publish Test Report
        if: ${{ always() }}
        uses: scacap/action-surefire-report@v1
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          fail_on_test_failures: false
          report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'

      - name: Slack notification
        if: ${{ always() && github.event_name != 'pull_request' && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
        uses: slackapi/slack-github-action@v2.0.0
        with:
          webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
          webhook-type: incoming-webhook
          payload: |
            text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Search ITs Nightly: ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
        env:
          SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}