airflow-apis-tests.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: airflow-apis-tests
on:
  pull_request_target:
    types: [labeled, opened, synchronize, reopened, ready_for_review]
    paths:
      - 'openmetadata-airflow-apis/**'
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: airflow-apis-tests-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}

env:
  SONAR_OPTS: >-
    -Dproject.settings=openmetadata-airflow-apis/sonar-project.properties
    -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
    -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}
    -Dsonar.pullrequest.github.repository=OpenMetadata
    -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
    -Dsonar.pullrequest.provider=github

jobs:
  airflow-apis-tests:
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@main
      with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: false
          swap-storage: true
    - name: Wait for the labeler
      uses: lewagon/wait-on-check-action@v1.7.0
      if: ${{ github.event_name == 'pull_request_target' }}
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        check-name: Team Label
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        wait-interval: 90

    - name: Verify PR labels
      uses: jesusvasquez333/verify-pr-label-action@v1.4.0
      if: ${{ github.event_name == 'pull_request_target' }}
      with:
        github-token: '${{ secrets.GITHUB_TOKEN }}'
        valid-labels: 'safe to test'
        pull-request-number: '${{ github.event.pull_request.number }}'
        disable-reviews: true  # To not auto approve changes

    - name: Checkout
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        fetch-depth: 0
        filter: blob:none

    - name: Set up JDK 21
      uses: actions/setup-java@v4
      with:
        java-version: '21'
        distribution: 'temurin'

    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install Ubuntu dependencies
      run: |
        # stop relying on apt cache of GitHub runners
        sudo apt-get update
        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
        librdkafka-dev unixodbc-dev libevent-dev libkrb5-dev

    - name: Generate models
      run: |
        python3 -m venv env
        source env/bin/activate
        sudo make install_antlr_cli
        make install_dev generate

    - name: Install open-metadata dependencies
      run: |
        source env/bin/activate
        make install_all install_test

    - name: Start Server and Ingest Sample Data
      uses: nick-fields/retry@v3.0.2
      env:
        INGESTION_DEPENDENCY: "mysql,elasticsearch,sample-data"
      with:
        timeout_minutes: 60
        max_attempts: 2
        retry_on: error
        command: ./docker/run_local_docker.sh -m no-ui

    - name: Run Python Tests & Record Coverage
      run: |
        source env/bin/activate
        make install_apis
        make coverage_apis
        rm pom.xml
        # fix coverage xml report for github
        sed -i 's/openmetadata_managed_apis/\/github\/workspace\/openmetadata-airflow-apis\/openmetadata_managed_apis/g' openmetadata-airflow-apis/ci-coverage.xml

    - name: Push Results in PR to Sonar
      id: push-to-sonar
      if: ${{ github.event_name == 'pull_request_target' }}
      continue-on-error: true
      uses: SonarSource/sonarqube-scan-action@v7
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
      with:
        projectBaseDir: openmetadata-airflow-apis/
        args: ${{ env.SONAR_OPTS }}

    # next two steps are for retrying "Push Results in PR to Sonar" step in case it fails
    - name: Wait to retry 'Push Results in PR to Sonar'
      if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
      run: sleep 20s
      shell: bash

    - name: Retry 'Push Results in PR to Sonar'
      uses: SonarSource/sonarqube-scan-action@v7
      if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
      with:
        projectBaseDir: openmetadata-airflow-apis/
        args: ${{ env.SONAR_OPTS }}

    - name: Push Results to Sonar
      uses: SonarSource/sonarqube-scan-action@v7
      if: ${{ github.event_name == 'push' }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
      with:
        projectBaseDir: openmetadata-airflow-apis/
        args: -Dproject.settings=openmetadata-airflow-apis/sonar-project.properties