build.yml

# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

name: Build CI

on:
  push:
    branches:
      - main
      - '*.x'
  pull_request:
  merge_group:
    types: [checks_requested]
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref || github.run_id }}
  cancel-in-progress: true

jobs:
  scheduler:
    runs-on: ubuntu-24.04
    outputs:
      docs: ${{ steps.set-matrix.outputs.docs }}
      zephyr-tests: ${{ steps.set-matrix.outputs.zephyr-tests }}
      ports: ${{ steps.set-matrix.outputs.ports }}
      cp-version: ${{ steps.set-up-submodules.outputs.version }}
    steps:
    - name: Dump GitHub context
      run: echo "$GITHUB_CONTEXT"
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
    - name: Set up repository
      uses: actions/checkout@v6
      with:
        submodules: false
        show-progress: false
        fetch-depth: 1
        persist-credentials: false
    - name: Set up python
      uses: actions/setup-python@v6
      with:
        python-version: 3.x
    - name: Duplicate USB VID/PID check
      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
    - name: Set up submodules
      id: set-up-submodules
      uses: ./.github/actions/deps/submodules
      with:
        action: cache
        version: true
    - name: Set up external
      uses: ./.github/actions/deps/external
      with:
        action: cache
    - name: Set up mpy-cross
      uses: ./.github/actions/mpy_cross
      with:
        cp-version: ${{ steps.set-up-submodules.outputs.version }}
        download: false
    - name: Get last commit with checks
      id: get-last-commit-with-checks
      if: github.event_name == 'pull_request'
      working-directory: tools
      run: python3 -u ci_changes_per_commit.py
      env:
        REPO: ${{ github.repository }}
        PULL: ${{ github.event.number }}
        GITHUB_TOKEN: ${{ github.token }}
        EXCLUDE_COMMIT: ${{ github.event.pull_request.head.sha }}
    - name: Set head sha (pull)
      if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
      env:
        HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      run: echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV
    - name: Set base sha (pull)
      if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
      run: git cat-file -e $SHA && echo "BASE_SHA=$SHA" >> $GITHUB_ENV || true
      env:
        SHA: ${{ steps.get-last-commit-with-checks.outputs.commit_sha || github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
    - name: Set head sha (push)
      if: github.event_name == 'push'
      env:
        SHA: ${{ github.event.after }}
      run: echo "HEAD_SHA=$SHA" >> $GITHUB_ENV
    - name: Set base sha (push)
      if: github.event_name == 'push'
      run: git cat-file -e $SHA && echo "BASE_SHA=$SHA" >> $GITHUB_ENV || true
      env:
        SHA: ${{ github.event.before }}
    - name: Set matrix
      id: set-matrix
      run: python3 -u ci_set_matrix.py
      working-directory: tools
      env:
        LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}

  tests:
    needs: scheduler
    uses: ./.github/workflows/run-tests.yml
    with:
      cp-version: ${{ needs.scheduler.outputs.cp-version }}

  zephyr-tests:
    needs: scheduler
    if: needs.scheduler.outputs.zephyr-tests == 'True'
    uses: ./.github/workflows/run-zephyr-tests.yml
    with:
      cp-version: ${{ needs.scheduler.outputs.cp-version }}

  mpy-cross:
    needs: scheduler
    if: needs.scheduler.outputs.ports != '{}'
    uses: ./.github/workflows/build-mpy-cross.yml
    secrets: inherit
    with:
      cp-version: ${{ needs.scheduler.outputs.cp-version }}

  mpy-cross-mac:
    runs-on: macos-latest
    needs: scheduler
    if: needs.scheduler.outputs.ports != '{}'
    env:
      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
    steps:
    - name: Set up repository
      uses: actions/checkout@v6
      with:
        submodules: false
        show-progress: false
        fetch-depth: 1
        persist-credentials: false
    - name: Set up python
      uses: actions/setup-python@v6
      with:
        python-version: 3.x
    - name: Set up submodules
      uses: ./.github/actions/deps/submodules
    - name: Versions
      run: |
        gcc --version
        python3 --version
        msgfmt --version
    - name: Build mpy-cross (arm64)
      run: make -C mpy-cross -j$(sysctl -n hw.ncpu) -f Makefile.m1 V=2
    - uses: actions/upload-artifact@v7
      with:
        name: mpy-cross-macos-arm64
        path: mpy-cross/build-arm64/mpy-cross-arm64
    - name: Upload to S3
      if: >-
          (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
          (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
      run: |
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/build-arm64/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/macos/mpy-cross-macos-"${CP_VERSION}"-arm64 --no-progress --region us-east-1
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

  docs:
    runs-on: ubuntu-24.04
    needs: scheduler
    if: needs.scheduler.outputs.docs == 'True'
    env:
      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
    steps:
    - name: Set up repository
      uses: actions/checkout@v6
      with:
        submodules: false
        show-progress: false
        fetch-depth: 1
        persist-credentials: false
    - name: Set up python
      uses: actions/setup-python@v6
      with:
        python-version: 3.x
    - name: Set up submodules
      uses: ./.github/actions/deps/submodules
      with:
        version: true
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
        pip install -r requirements-doc.txt
    - name: Build and Validate Stubs
      run: make check-stubs -j$(nproc)
    - uses: actions/upload-artifact@v7
      with:
        name: stubs
        path: circuitpython-stubs/dist/*
    - name: Test Documentation Build (HTML)
      run: sphinx-build -E -W -b html -D version="$CP_VERSION" -D release="$CP_VERSION" . _build/html
    - uses: actions/upload-artifact@v7
      with:
        name: docs-html
        path: _build/html
    - name: Test Documentation Build (LaTeX/PDF)
      run: |
        make latexpdf
    - uses: actions/upload-artifact@v7
      with:
        name: docs-latexpdf
        path: _build/latex
    - name: Upload to S3
      uses: ./.github/actions/upload_aws
      with:
        source: circuitpython-stubs/dist/*.tar.gz
        destination: stubs/circuitpython-stubs-${{ env.CP_VERSION }}.tar.gz
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    - name: Upload stubs to PyPi
      if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
      env:
        TWINE_USERNAME: ${{ secrets.pypi_username }}
        TWINE_PASSWORD: ${{ secrets.pypi_password }}
      run: |
        # python -m build was run by 'make stubs'
        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*

  ports:
    needs: [scheduler, mpy-cross, tests]
    if: needs.scheduler.outputs.ports != '{}'
    uses: ./.github/workflows/build-boards.yml
    secrets: inherit
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      max-parallel: 55
      matrix:
        port: ${{ fromJSON(needs.scheduler.outputs.ports).ports }}
    with:
      boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.ports)[matrix.port]) }}
      cp-version: ${{ needs.scheduler.outputs.cp-version }}
      port: ${{ fromJSON(needs.scheduler.outputs.ports).split_ports[matrix.port] || matrix.port }}

  build-ci:
    # Final gate for ruleset checks: always runs so the required check is
    # reported, and fails if any job (or matrix leg) failed or was cancelled.
    needs: [scheduler, tests, zephyr-tests, mpy-cross, mpy-cross-mac, docs, ports]
    if: ${{ always() }}
    runs-on: ubuntu-24.04
    env:
      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
    steps:
    # Firmware size report. Each board job uploads its sizes/<board>.json as zz-sizes-<board>;
    # merge them into one table and a job summary. This is reporting only, so none of it
    # may fail the gate: every step continues on error.
    - name: Set up repository
      continue-on-error: true
      uses: actions/checkout@v6
      with:
        submodules: false
        show-progress: false
        fetch-depth: 1
        persist-credentials: false

    # actions/download-artifact matches a pattern against a listing of the run's
    # artifacts that stops at 1000, and a full build has more. The script fetches each
    # board's record by exact name instead, from the board list the scheduler produced.
    # It runs from github-script because only action steps get the artifact API token.
    - name: Install the artifact client
      continue-on-error: true
      run: npm install --no-save --no-package-lock --no-audit --no-fund @actions/artifact@6

    - name: Download firmware sizes
      continue-on-error: true
      uses: actions/github-script@v9
      env:
        BOARDS: ${{ needs.scheduler.outputs.ports }}
      with:
        script: |
          const { pathToFileURL } = require("node:url");
          const sizes = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/tools/ci_download_sizes.mjs`));
          await sizes.downloadSizes({
            boards: sizes.boardsFromSchedule(JSON.parse(process.env.BOARDS || "{}")),
            outDir: "sizes",
            info: core.info,
            warning: core.warning,
          });

    - name: Summarize firmware sizes
      continue-on-error: true
      run: python3 tools/ci_firmware_sizes.py sizes sizes-report

    # Uploaded unzipped, one file per artifact, so each opens directly; with
    # `archive: false` the file name is the artifact name.
    - name: Upload firmware size report (HTML)
      continue-on-error: true
      uses: actions/upload-artifact@v7
      with:
        path: sizes-report/0-sizes.html
        archive: false
        if-no-files-found: ignore

    - name: Upload firmware size report (JSON)
      continue-on-error: true
      uses: actions/upload-artifact@v7
      with:
        path: sizes-report/0-sizes.json
        archive: false
        if-no-files-found: ignore

    # The action itself only uploads on pushes to main and release branches and on
    # published releases. The S3 names carry the version, so the directory keeps a history.
    - name: Upload firmware size report to S3 (HTML)
      continue-on-error: true
      uses: ./.github/actions/upload_aws
      with:
        source: sizes-report/0-sizes.html
        destination: 0-sizes/adafruit-circuitpython-sizes-${{ env.CP_VERSION }}.html
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

    - name: Upload firmware size report to S3 (JSON)
      continue-on-error: true
      uses: ./.github/actions/upload_aws
      with:
        source: sizes-report/0-sizes.json
        destination: 0-sizes/adafruit-circuitpython-sizes-${{ env.CP_VERSION }}.json
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

    - name: Check build results
      run: |
        echo "Job results: ${{ toJSON(needs.*.result) }}"
        if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]] || [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
          echo "One or more jobs failed or were cancelled"
          exit 1
        fi
        echo "All build jobs succeeded"