diff --git a/.github/scripts/track-build.sh b/.github/scripts/track-build.sh new file mode 100755 index 0000000..3e45f14 --- /dev/null +++ b/.github/scripts/track-build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Follow the BigLinux builder job started for a package, using the public API +# of https://gitlab.bigib.org/builder/packagebuilder (no token needed). +# +# Environment: +# PACKAGE package (and repository) name, e.g. biglinux-welcome +# ORG GitHub organization the builder clones from, e.g. biglinux +# BRANCH branch the build was requested for, e.g. main +# SINCE ISO 8601 time the request was made (the "Build Package" run end) +# START_TIMEOUT, BUILD_TIMEOUT seconds to wait for the job to appear / end +# +# Exits non-zero when no matching job starts, or when it does not succeed. +set -euo pipefail + +: "${PACKAGE:?}" "${ORG:?}" "${BRANCH:?}" "${SINCE:?}" +START_TIMEOUT=${START_TIMEOUT:-600} +BUILD_TIMEOUT=${BUILD_TIMEOUT:-3600} +api=https://gitlab.bigib.org/api/v4/projects/builder%2Fpackagebuilder +web=https://gitlab.bigib.org/builder/packagebuilder +url="https://github.com/$ORG/$PACKAGE" +summary=${GITHUB_STEP_SUMMARY:-/dev/null} + +trap 'echo "::error::Request to the BigLinux builder failed (line $LINENO); is gitlab.bigib.org reachable?"' ERR + +get() { + curl -fsS --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 "$@" +} + +# One of the header lines the build job prints ("org: biglinux", ...) +header() { + sed -nE "s/^[^ ]+ [0-9]+[OE]\+? ?$1: (.*)$/\1/p" <<<"$2" | head -n1 +} + +# Same rules as package-build in packagebuilder +case "$BRANCH" in +*update-stable*) repo=update-stable ;; +*update-testing*) repo=update-testing ;; +*unstable* | *development*) repo=unstable ;; +*testing*) repo=testing ;; +*stable*) repo=stable ;; +*) repo=unstable ;; +esac + +since=$(date -u -d "$SINCE" +%s) +after=$(date -u -d "@$((since - 120))" +%FT%TZ) +echo "Looking for the $PACKAGE build of $url (branch $BRANCH) requested at $SINCE" + +# 1. Find the job. Every request is a "trigger" pipeline whose child pipeline +# has one job named after the package. +job='' child='' rejected=" " deadline=$((SECONDS + START_TIMEOUT)) +while [ -z "$job" ]; do + while read -r pid created; do + [ "$(date -u -d "$created" +%s)" -ge $((since - 30)) ] || continue + child=$(get "$api/pipelines/$pid/bridges" | jq -r '.[0].downstream_pipeline.id // empty') + [ -n "$child" ] || continue + id=$(get "$api/pipelines/$child/jobs" | jq -r --arg n "$PACKAGE" '.[] | select(.name == $n) | .id') + if [ -z "$id" ] || [[ "$rejected" == *" $id "* ]]; then continue; fi + log=$(get "$web/-/jobs/$id/raw" || true) + [ -n "$(header git_branch "$log")" ] || continue # not started yet + if [ "$(header url "$log")" = "$url" ] && [ "$(header git_branch "$log")" = "$BRANCH" ]; then + job=$id + echo "Builder accepted the request: pipeline $web/-/pipelines/$pid, job $web/-/jobs/$job" + echo " requested by $(header 'user sender' "$log"), created $created" + break + fi + rejected+="$id " + done < <(get "$api/pipelines?source=trigger&updated_after=$after&order_by=id&sort=asc&per_page=100" | + jq -r '.[] | "\(.id) \(.created_at)"') + + if [ -z "$job" ]; then + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::The builder started no $PACKAGE job for $url ($BRANCH) within ${START_TIMEOUT}s. See docs/ci-packaging.md." + printf '### %s: no builder job\nNo job for %s, branch %s, started within %ss of %s.\n' \ + "$PACKAGE" "$url" "$BRANCH" "$START_TIMEOUT" "$SINCE" >>"$summary" + exit 1 + fi + sleep 20 + fi +done + +# 2. Wait for it to finish +deadline=$((SECONDS + BUILD_TIMEOUT)) last= +while :; do + status=$(get "$api/pipelines/$child/jobs" | jq -r --argjson id "$job" '.[] | select(.id == $id) | .status') + [ "$status" = "$last" ] || echo "Job status: $status" + last=$status + case "$status" in + success | failed | canceled | skipped) break ;; + esac + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Builder job $web/-/jobs/$job still '$status' after ${BUILD_TIMEOUT}s" + exit 1 + fi + sleep 30 +done + +# 3. Report what was built and uploaded +log=$(get "$web/-/jobs/$job/raw" | sed -E 's/^[^ ]+ [0-9]+[OE]\+? ?//') +built=$(grep -m1 -oE '^==> Finished making: .*' <<<"$log" | sed 's/^==> Finished making: //' || true) +uploaded=$(grep -A1 '^sending incremental file list' <<<"$log" | grep -E '\.pkg\.tar\.[a-z]+$' || true) +{ + printf '### %s: builder job %s\n\n' "$PACKAGE" "$status" + printf '| Step | Result |\n|---|---|\n' + printf '| Request accepted | [job %s](%s/-/jobs/%s), branch %s |\n' "$job" "$web" "$job" "$BRANCH" + printf '| Package built | %s |\n' "${built:-no}" + printf '| Uploaded for the %s repository | %s |\n' "$repo" "${uploaded:-no}" +} >>"$summary" +echo "Built: ${built:-nothing}" +echo "Uploaded for the $repo repository: ${uploaded:-nothing}" + +if [ "$status" != success ] || [ -z "$built" ] || [ -z "$uploaded" ]; then + echo "::error::Builder job $web/-/jobs/$job ended '$status'" + exit 1 +fi diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 0dfb7cf..865ba7c 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -1,20 +1,53 @@ name: Build Package +# Packages are built by the BigLinux builder on GitLab +# (https://gitlab.bigib.org/builder/packagebuilder). The biglinux organization +# starts a build when a run of the workflow with this name finishes +# successfully, for the branch of that run: main goes to the unstable +# repository, testing-* to testing and stable-* to stable. This workflow sends +# nothing itself and needs no secret: it only has to succeed when, and only +# when, the package may be built. Keep its name and the final job "build". +# See docs/ci-packaging.md. + on: - workflow_dispatch: push: - branches: [main] + branches: [main, "testing-*", "stable-*"] + workflow_dispatch: -# Only the WEBHOOK_TOKEN secret is used; the GITHUB_TOKEN needs no access -permissions: {} +permissions: + contents: read + +# A newer push to the same branch supersedes a running one: the builder always +# clones the head of the branch, so only the last run needs to request a build +concurrency: + group: build-package-${{ github.ref }} + cancel-in-progress: true jobs: + tests: + uses: ./.github/workflows/tests.yml + build: - if: github.ref == 'refs/heads/main' + needs: tests runs-on: ubuntu-latest + timeout-minutes: 5 steps: - - name: Send Hooks BigLinux Build Package - shell: bash + - name: Request the package build + env: + BRANCH: ${{ github.ref_name }} run: | - curl -X POST -H "Accept: application/json" -H "Authorization: token ${{ secrets.WEBHOOK_TOKEN }}" --data '{"event_type": "${{ github.repository }}", "client_payload": { "branch": "${{ github.ref_name }}", "url": "https://github.com/${{ github.repository }}"}}' https://api.github.com/repos/BigLinux-Package-Build/build-package/dispatches - curl -X POST -H "Accept: application/json" -H "Authorization: token ${{ secrets.WEBHOOK_TOKEN }}" --data '{"event_type": "${{ github.repository }}", "client_payload": { "branch": "${{ github.ref_name }}", "url": "https://github.com/${{ github.repository }}"}}' https://api.github.com/repos/BigLinux-Package-Build/build-package-ARM/dispatches + # A failed run requests nothing, so manual runs from other branches fail + case "$BRANCH" in + main) repo=unstable ;; + testing-*) repo=testing ;; + stable-*) repo=stable ;; + *) + echo "::error::Packages are built only from main, testing-* and stable-*, not '$BRANCH'" + exit 1 + ;; + esac + echo "Tests passed for $GITHUB_SHA." + echo "When this run ends, the builder builds branch $BRANCH for the $repo repository;" + echo "the GitLab Status workflow follows that job." + printf '### Build requested\nBranch %s (%s), %s repository.\n' "$BRANCH" "$GITHUB_SHA" "$repo" \ + >>"$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/gitlab-status.yml b/.github/workflows/gitlab-status.yml new file mode 100644 index 0000000..d220071 --- /dev/null +++ b/.github/workflows/gitlab-status.yml @@ -0,0 +1,44 @@ +name: GitLab Status + +# Follows the job the BigLinux builder starts after a successful +# "Build Package" run, and fails when it does not start or does not succeed. +# Read only: it uses the public GitLab API and no secret. The name avoids the +# words "Build" and "Package", so this run never requests a build itself. + +on: + workflow_run: + workflows: [Build Package] + types: [completed] + workflow_dispatch: + inputs: + branch: + description: Branch the build was requested for + default: main + since: + description: Time of the request (ISO 8601, e.g. 2026-10-10T08:36:18Z) + required: true + +permissions: + contents: read + +jobs: + track: + # Only the biglinux organization requests builds + if: >- + github.event_name == 'workflow_dispatch' || + (github.repository_owner == 'biglinux' && github.event.workflow_run.conclusion == 'success') + runs-on: ubuntu-latest + timeout-minutes: 75 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Follow the builder job + env: + PACKAGE: ${{ github.event.repository.name }} + ORG: biglinux + BRANCH: ${{ github.event.workflow_run.head_branch || inputs.branch }} + SINCE: ${{ github.event.workflow_run.updated_at || inputs.since }} + run: .github/scripts/track-build.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 003b719..c28ccf2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,11 @@ name: Tests +# Pushes to main run these jobs through build-package.yml, which only asks +# for a package build after they pass. on: - push: - branches: [main] pull_request: workflow_dispatch: + workflow_call: permissions: contents: read @@ -22,13 +23,15 @@ jobs: # Pinned, so new ruff releases cannot change the rule set unnoticed python -m venv /opt/ruff && /opt/ruff/bin/pip install --quiet ruff==0.17.0 - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Lint run: | /opt/ruff/bin/ruff check usr/share/biglinux/welcome tests generate_strings.py /opt/ruff/bin/ruff format --check usr/share/biglinux/welcome tests generate_strings.py - shellcheck tests/run-headless.sh usr/share/biglinux/welcome/scripts/*.sh update-translations.sh \ + shellcheck tests/run-headless.sh tests/check-pkgbuild.sh usr/share/biglinux/welcome/scripts/*.sh update-translations.sh \ pkgbuild/biglinux-welcome.install usr/bin/biglinux-welcome desktop-file-validate usr/share/applications/org.biglinux.welcome.desktop @@ -43,3 +46,25 @@ jobs: useradd -m tester chown -R tester: "$GITHUB_WORKSPACE" runuser -u tester -- tests/run-headless.sh -v -rs + + package: + runs-on: ubuntu-latest + container: archlinux:latest + steps: + - name: Install dependencies + run: pacman -Syu --noconfirm --needed base-devel git namcap + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build the package + run: | + useradd -m builder + chown -R builder: "$GITHUB_WORKSPACE" + runuser -u builder -- tests/check-pkgbuild.sh build /tmp/pkg + + # Installs the dependencies from the Arch repositories, runs the + # .install scriptlets and checks every installed file + - name: Install, verify and remove + run: tests/check-pkgbuild.sh install /tmp/pkg diff --git a/.gitignore b/.gitignore index c33ad3b..5f88b30 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,8 @@ .ruff_cache/ .pytest_cache/ PLANNING.md +# makepkg output in pkgbuild/ +pkgbuild/src/ +pkgbuild/pkg/ +pkgbuild/biglinux-welcome/ +*.pkg.tar* diff --git a/README.md b/README.md index 66206ef..277949c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@
-
+
@@ -249,10 +249,23 @@ RUN_NETWORK_TESTS=1 python -m pytest tests/test_installer.py -k chatai # pinne
ruff check usr/share/biglinux/welcome tests generate_strings.py
ruff format --check usr/share/biglinux/welcome tests generate_strings.py
-shellcheck usr/share/biglinux/welcome/scripts/*.sh update-translations.sh tests/run-headless.sh
+shellcheck usr/share/biglinux/welcome/scripts/*.sh update-translations.sh tests/*.sh
+
+tests/check-pkgbuild.sh build /tmp/pkg # build this tree as the BigLinux builder would, then namcap
+sudo tests/check-pkgbuild.sh install /tmp/pkg # install, verify, remove: disposable container only
```
-The suite checks the page configuration, translations, version consistency and the installer: it runs the **real privileged helper**, unprivileged, against a fake `pacman` and `pkexec` (success, network failure, locked database, cancellation reaching pacman, Polkit dismissed/denied, verification, pinned repositories), installs ChatAI into a private package root (including tampered and path-traversal archives) and drives the real widgets. GitHub Actions (`.github/workflows/tests.yml`) runs all of this in an `archlinux` container with ruff 0.17.0.
+The suite checks the page configuration, translations, version consistency and the installer: it runs the **real privileged helper**, unprivileged, against a fake `pacman` and `pkexec` (success, network failure, locked database, cancellation reaching pacman, Polkit dismissed/denied, verification, pinned repositories), installs ChatAI into a private package root (including tampered and path-traversal archives) and drives the real widgets. GitHub Actions (`.github/workflows/tests.yml`) runs all of this in an `archlinux` container with ruff 0.17.0, and also builds, installs and removes the package there.
+
+### Continuous integration and packages
+
+| Workflow | Runs on | Does |
+|---|---|---|
+| Tests | pull requests, manual | tests, lint, translations, package build/install/remove |
+| Build Package | pushes to `main`, `testing-*`, `stable-*`, manual | the tests above, then the `build` job |
+| GitLab Status | after Build Package (biglinux only), manual | follows the builder job and fails if it fails |
+
+The BigLinux builder ([packagebuilder](https://gitlab.bigib.org/builder/packagebuilder)) starts by itself when a **Build Package** run in `biglinux/biglinux-welcome` succeeds; no secret is needed. `main` is built for the unstable repository, a `testing-*` branch for testing and a `stable-*` branch for stable. Failed tests mean no build. Details, evidence and the release checklist: [docs/ci-packaging.md](docs/ci-packaging.md).
## Troubleshooting
diff --git a/docs/ci-packaging.md b/docs/ci-packaging.md
new file mode 100644
index 0000000..61213bf
--- /dev/null
+++ b/docs/ci-packaging.md
@@ -0,0 +1,126 @@
+# CI and packaging
+
+How a commit in `biglinux/biglinux-welcome` becomes a package in the BigLinux
+repositories, what was wrong before, and how to release.
+
+## Flow
+
+```
+push to main / testing-* / stable-* (biglinux/biglinux-welcome)
+ └─ Build Package workflow
+ ├─ tests tests.yml: lint, translations, test suite, package build+install+remove
+ └─ build needs: tests — fails for any other branch
+ run ends with success
+ └─ biglinux organization (outside this repository) triggers
+ gitlab.bigib.org/builder/packagebuilder with
+ org, url, user, package_name = repository, git_branch = branch of the run
+ └─ job "biglinux-welcome": clone url, checkout git_branch, rewrite PKGBUILD,
+ chrootbuild (Manjaro stable, x86_64), sign, rsync to the repository server
+ └─ GitLab Status workflow (workflow_run) follows that job through the public
+ GitLab API and fails if it does not start within 10 min or does not succeed
+```
+
+Destination repository, decided by the builder from `git_branch`
+(`package-build` in packagebuilder):
+
+| Branch | Repository |
+|---|---|
+| `main` (anything not below) | unstable |
+| `testing-*` | testing |
+| `stable-*` | stable (also a GitHub release in `biglinux/biglinux-welcome`) |
+
+Only `x86_64` is built: `arch=('any')` is rewritten to `x86_64 aarch64`, but
+the aarch64 step is commented out in the builder.
+
+## How the builder is started (evidence)
+
+No workflow in any BigLinux repository calls GitLab. The trigger was found by
+comparing GitHub run times with GitLab pipeline creation times (2026-10-10):
+
+| GitHub run that ended successfully | Ended (UTC) | GitLab pipeline created | Job |
+|---|---|---|---|
+| biglinux-welcome "Build Package", main | 10-10 08:36:18 | 08:36:19 | 62063 |
+| biglinux-welcome "Build Package", main | 10-03 09:40:19 | 09:40:20 | 60306 |
+| biglinux-settings "Translate and Build Package", main | 09-13 20:06:08 | 20:06:10 | 56153 |
+| biglinux-settings "Translate and Build Package", stable-2026-09-13_17-55 | 09-13 20:57:01 | 20:57:04 | 56156 |
+
+and no pipeline followed: "Tests" runs, a cancelled "Translate and Build
+Package" run, or the creation of branches `testing-2026-10-10`,
+`testing-2026-10-10-1` and `stable-2026-10-03_06-40` (no workflow ran for them).
+Every build job logs `user sender: