diff --git a/.github/workflows/flatpak.yml b/.github/workflows/flatpak.yml index f62aca8..11c9875 100644 --- a/.github/workflows/flatpak.yml +++ b/.github/workflows/flatpak.yml @@ -17,27 +17,29 @@ name: Flatpak # 6. CLI `comline generation` and `generation clean`, through the # launch the extension builds, change the files on disk # -# What it needs, and why it is built this way: +# What it tests, and why it is built this way: # -# - A VM. Flatpak's sandbox (bubblewrap, user namespaces) does not run inside -# our CI container images. `ubuntu-24.04` is the VM the other workflows use. -# - `flatpak` from apt, at run time. EXCEPTION to the org's rule against -# installing packages while a job runs: the sandbox under test IS Flatpak, -# the runner image does not have it, and there is no pinned image that does. -# So the packages are pinned to exact versions (below) and the install is -# retried on network failure. Ubuntu keeps only its current security update -# of a package, so a version here goes stale when Ubuntu ships the next one: -# the weekly run turns that into a failed run that says which pin to move. -# - `com.vscodium.codium` from Flathub, user installation: the real app, with -# the permissions Flathub gives it, not a stand-in. It follows Flathub's -# current build, on purpose: a weekly run is how a change to its permissions -# (--talk-name=org.freedesktop.Flatpak, --filesystem=host) is noticed. The -# installed commit is logged, and the install is cached by that commit. -# - `dbus-run-session`: `flatpak-spawn --host` talks to the session helper -# over the session bus; a runner has none. -# - comline-lsp: the one the latest GitHub release ships (the linux-x64 -# package's bundled binary), not built again here. Which release is logged. -# - comline (CLI): built here from a pinned commit. It has no release yet. +# - The programs are this commit's. comline-lsp is built here from the +# language-server rev package.json pins (`comline.languageServer.rev`), the +# way release.yml builds the one the extension ships: static musl, in the +# tools image. comline (the CLI) is built from a pinned commit (it has no +# release yet). The sha256 of each is logged where it is used. +# - The sandbox is a container: ghcr.io/comlineproject/ci-flatpak (the +# ComlineProject/ci-images repository). It has Flatpak, bubblewrap, +# xdg-dbus-proxy and `dbus-run-session`, and Flathub's real +# `com.vscodium.codium` (the permissions Flathub gives it, not a stand-in), +# installed at an exact Flathub commit. Nothing is installed while the job +# runs. The VSCodium it ran on is logged; a new Flathub build is a new image +# (a digest bump), reviewed like any other change. +# - `--privileged`: bubblewrap makes namespaces and mounts /proc, which a +# container's default seccomp profile and masked paths forbid. +# - The container is the "host" of `flatpak-spawn --host`: the session helper +# runs in it, as `dbus-run-session` starts the session bus the helper needs +# (a runner has none). A path under /usr/local/bin is the host-only /usr: the +# sandbox has the runtime's /usr, so it does not see it. +# - The job runs as root, as every job container does (the workspace is mounted +# from the runner); flatpak needs no helper for that. The image's +# XDG_RUNTIME_DIR (/run/user/0) is the runtime directory flatpak wants. on: pull_request: @@ -56,7 +58,8 @@ on: - ".github/workflows/flatpak.yml" - "scripts/flatpak-e2e.cjs" schedule: - # Mondays: Flathub and Ubuntu move without a change here. + # Mondays: the runner's kernel, Docker and AppArmor move without a change + # here, and a sandbox that stops working on them should show up in a week. - cron: "23 5 * * 1" workflow_dispatch: @@ -77,13 +80,44 @@ env: # ComlineProject/cli, main when this was written. Bump with the CLI's # command set (it has to have `generation`, cli#153). CLI_REV: e79092a3cfec9f2b3d02a45189f51e6a90fb8071 - # Ubuntu 24.04 (noble), noble-updates/-security. - FLATPAK_VERSION: 1.14.6-1ubuntu0.1 - BUBBLEWRAP_VERSION: 0.9.0-1ubuntu0.3 - XDG_DBUS_PROXY_VERSION: 0.1.5-1ubuntu0.2 APP_ID: com.vscodium.codium jobs: + # out/resolve.js: the compiled module the test imports. Compiled on the + # runner (which has yarn), so the sandbox's container needs no Node. + compile: + name: Compile the extension + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Harden the runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit + + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 20.20.2 + cache: yarn + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Compile TypeScript + run: yarn compile + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: extension-out + path: out + if-no-files-found: error + cli: name: Build comline (CLI) runs-on: ubuntu-24.04 @@ -123,6 +157,7 @@ jobs: exit 1 fi "$bin" targets + sha256sum "$bin" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -130,36 +165,86 @@ jobs: path: target/x86_64-unknown-linux-musl/release/comline if-no-files-found: error - sandbox: - name: VSCodium sandbox (Flathub) - needs: cli + # The language server this commit would ship: the pin in package.json, built + # as release.yml's linux-x64 build does (the same image and target). + lsp: + name: Build comline-lsp (pinned) runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 60 + container: ghcr.io/comlineproject/ci-tools@sha256:b9301f9caf3e47f0b839de85efda178f698a55daf7fc387b463adeec93ecd433 steps: - - name: Harden the runner - uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + - name: Checkout extension (for the pin) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - egress-policy: audit + persist-credentials: false - - name: Checkout code + - name: The pinned language-server rev + id: pin + run: | + rev=$(jq -r '.comline.languageServer.rev' package.json) + [[ "$rev" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::no language-server rev in package.json: $rev"; exit 1; } + echo "language-server $rev" + echo "rev=$rev" >> "$GITHUB_OUTPUT" + + - name: Checkout language-server at the pinned rev uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + repository: ComlineProject/language-server + ref: ${{ steps.pin.outputs.rev }} + path: language-server persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + - name: musl C compiler + run: | + # comline-core compiles its tree-sitter grammar from C at build time. + echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV" + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: - node-version: 20.20.2 - cache: yarn + workspaces: language-server + key: x86_64-unknown-linux-musl - - name: Install dependencies - run: yarn install --frozen-lockfile + # The command release.yml runs for linux-x64. + - name: Build + working-directory: language-server + run: cargo build --release --target x86_64-unknown-linux-musl - # out/resolve.js: what the test imports. - - name: Compile TypeScript - run: yarn compile + - name: Static, and it runs + run: | + bin=language-server/target/x86_64-unknown-linux-musl/release/comline-lsp + if readelf -lW "$bin" | grep -q INTERP; then + echo "::error::comline-lsp is dynamically linked" + exit 1 + fi + sha256sum "$bin" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: comline-lsp + path: language-server/target/x86_64-unknown-linux-musl/release/comline-lsp + if-no-files-found: error + + sandbox: + name: VSCodium sandbox (Flathub) + needs: [compile, cli, lsp] + runs-on: ubuntu-24.04 + timeout-minutes: 45 + container: + image: ghcr.io/comlineproject/ci-flatpak@sha256:664637fe0967ad5e01f47e9356fdd941bd2fd03b3e4ab268fcc37aef3916e0a7 + options: --privileged + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # -- what is under test --------------------------------------------------- - # -- the programs the extension will run ------------------------------- + - name: The compiled extension + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: extension-out + path: out - name: The CLI (built above) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -167,106 +252,52 @@ jobs: name: comline-cli path: ${{ runner.temp }}/cli - - name: The language server (the latest release's bundled one) - env: - GH_TOKEN: ${{ github.token }} + - name: The language server (built above, from the pinned rev) + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: comline-lsp + path: ${{ runner.temp }}/bundled + + # Static: checked where each was built (a static binary is what runs in + # the sandbox, whose libraries are the runtime's; scenario 1 runs it). + - name: The programs, as built run: | - tag=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 1 --json tagName --jq '.[0].tagName') - echo "language server from release $tag" - gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ - --pattern '*-linux-x64.vsix' --dir "$RUNNER_TEMP/release" - mkdir -p "$RUNNER_TEMP/bundled" - unzip -p "$RUNNER_TEMP"/release/*-linux-x64.vsix extension/bin/linux-x64/comline-lsp \ - > "$RUNNER_TEMP/bundled/comline-lsp" - chmod +x "$RUNNER_TEMP/bundled/comline-lsp" - sha256sum "$RUNNER_TEMP/bundled/comline-lsp" - # The bundled server is static, which is why it runs in a sandbox. - if readelf -lW "$RUNNER_TEMP/bundled/comline-lsp" | grep -q INTERP; then - echo "::error::the release's comline-lsp is dynamically linked" - exit 1 - fi + chmod +x "$RUNNER_TEMP/bundled/comline-lsp" "$RUNNER_TEMP/cli/comline" + sha256sum "$RUNNER_TEMP/bundled/comline-lsp" "$RUNNER_TEMP/cli/comline" # -- Flatpak ------------------------------------------------------------- - # EXCEPTION to "no packages installed while a job runs": see the header. - # Exact versions, retried on a network failure. - - name: Install Flatpak (apt, pinned) + - name: Which image, which VSCodium run: | - retry() { - for attempt in 1 2 3 4; do - "$@" && return 0 - echo "attempt $attempt failed: $*" >&2 - sleep $((attempt * 10)) - done - return 1 - } - retry sudo apt-get update -q - retry sudo apt-get install -y -q --no-install-recommends \ - "flatpak=$FLATPAK_VERSION" \ - "bubblewrap=$BUBBLEWRAP_VERSION" \ - "xdg-dbus-proxy=$XDG_DBUS_PROXY_VERSION" + echo "VSCodium commit: $(cat /etc/ci-flatpak/vscodium-commit)" + echo "runtime commit: $(cat /etc/ci-flatpak/runtime-commit)" flatpak --version - command -v dbus-run-session - - - name: Flathub, and which build of the app - id: app - run: | - flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo - commit=$(flatpak remote-info --user --show-commit flathub "$APP_ID") - echo "Flathub's current $APP_ID commit: $commit" - echo "commit=$commit" >> "$GITHUB_OUTPUT" - - # Keyed on the app's commit: most runs find it and download nothing. A new - # Flathub build misses, falls back to the latest older cache (the - # runtimes are shared), and updates it. - - name: Flatpak cache - id: cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.local/share/flatpak - key: flatpak-${{ env.FLATPAK_VERSION }}-${{ env.APP_ID }}-${{ steps.app.outputs.commit }} - restore-keys: | - flatpak-${{ env.FLATPAK_VERSION }}-${{ env.APP_ID }}- - - - name: Install the app - run: | - flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo - echo "cache hit: ${{ steps.cache.outputs.cache-hit }}" - if [ "${{ steps.cache.outputs.cache-hit }}" = true ]; then - echo "restored from the cache: nothing to download" - else - flatpak install --user --noninteractive --assumeyes flathub "$APP_ID" - # A restored older build is brought up to date. - flatpak update --user --noninteractive --assumeyes "$APP_ID" - fi - flatpak info --user --show-commit "$APP_ID" - flatpak info --user --show-permissions "$APP_ID" - du -sh ~/.local/share/flatpak + flatpak list --system --columns=ref,active,origin,installation + [ "$(flatpak info --system --show-commit "$APP_ID")" = "$(cat /etc/ci-flatpak/vscodium-commit)" ] # The documented facts the extension relies on, as the installed app has # them: if Flathub drops either permission this is where it shows. - name: The app has the permissions host mode needs run: | - permissions=$(flatpak info --user --show-permissions "$APP_ID") + permissions=$(flatpak info --system --show-permissions "$APP_ID") grep -E '^filesystems=.*\bhost\b' <<<"$permissions" grep -E '^org\.freedesktop\.Flatpak=talk' <<<"$permissions" # -- the session --------------------------------------------------------- - - name: Can this runner run a Flatpak sandbox at all? + - name: Can this container run a Flatpak sandbox at all? run: | sysctl kernel.apparmor_restrict_unprivileged_userns kernel.unprivileged_userns_clone 2>&1 || true - # flatpak wants a runtime dir, which a runner has no login session to make. - sudo mkdir -p "/run/user/$(id -u)" - sudo chown "$(id -u):$(id -g)" "/run/user/$(id -u)" - chmod 700 "/run/user/$(id -u)" + echo "XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" + test -d "$XDG_RUNTIME_DIR" # shellcheck disable=SC2016 # $FLATPAK_ID is for the shell inside the sandbox - XDG_RUNTIME_DIR="/run/user/$(id -u)" flatpak run --command=sh "$APP_ID" -c 'echo "inside the sandbox: $FLATPAK_ID"; cat /.flatpak-info | head -5' + flatpak run --command=sh "$APP_ID" -c 'echo "inside the sandbox: $FLATPAK_ID"; cat /.flatpak-info | head -5' - name: Put the programs where the scenarios expect them run: | - # The host-only /usr: a file the sandbox cannot see at that path. - sudo install -m 755 "$RUNNER_TEMP/bundled/comline-lsp" /usr/local/bin/comline-lsp + # The host-only /usr: a file the sandbox cannot see at that path (its + # /usr is the runtime's). The container is the host here. + install -m 755 "$RUNNER_TEMP/bundled/comline-lsp" /usr/local/bin/comline-lsp # A path under the home directory that is on no PATH. mkdir -p "$HOME/e2e/hostonly" install -m 755 "$RUNNER_TEMP/bundled/comline-lsp" "$HOME/e2e/hostonly/comline-lsp" @@ -291,8 +322,6 @@ jobs: - name: Find the app's Electron binary id: electron run: | - XDG_RUNTIME_DIR="/run/user/$(id -u)" - export XDG_RUNTIME_DIR flatpak run --command=sh "$APP_ID" -c 'ls -l /app/bin /app/share/codium | head -40' flatpak run --command=sh "$APP_ID" -c 'test -x /app/share/codium/com.vscodium.codium' echo "bin=/app/share/codium/com.vscodium.codium" >> "$GITHUB_OUTPUT" @@ -303,8 +332,6 @@ jobs: env: ELECTRON: ${{ steps.electron.outputs.bin }} run: | - XDG_RUNTIME_DIR="/run/user/$(id -u)" - export XDG_RUNTIME_DIR # The host session's PATH, which `flatpak-spawn --host` programs get: # it is the environment the session bus is started in. export PATH="$HOME/.local/bin:$PATH" diff --git a/CHANGELOG.md b/CHANGELOG.md index b6519f2..25173c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,10 @@ All notable changes to the "comline-vscode" extension will be documented in this (`~/.cargo/bin`). - Unit tests for the server and CLI resolution (`yarn test`), run in CI, and a workflow that runs the same code inside Flathub's VSCodium sandbox against a - real `flatpak-spawn --host` (`scripts/flatpak-e2e.cjs`). + real `flatpak-spawn --host` (`scripts/flatpak-e2e.cjs`), with the language + server built from the rev `package.json` pins and the CLI from a pinned + commit, in the `ci-flatpak` image (Flatpak and VSCodium at an exact Flathub + commit, nothing installed while the job runs). - `comline.toml` is its own language, `comline-toml`, highlighted with a bundled TOML grammar (Taplo's, MIT; see `syntaxes/THIRD-PARTY-NOTICES.md`) and served by the Comline language server: diagnostics for its shape and