diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..7c36512 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +# The CI pool runners' labels (.github/workflows/ci.yml), for actionlint. +self-hosted-runner: + labels: [ci-heavy] diff --git a/.github/ci/requirements.txt b/.github/ci/requirements.txt new file mode 100644 index 0000000..bfb54fe --- /dev/null +++ b/.github/ci/requirements.txt @@ -0,0 +1,46 @@ +# Python tools for .github/workflows/ci.yml, installed by .github/ci/setup.sh into the +# CI cache with `pip install --require-hashes --only-binary :all:`. Pinned versions; +# hashes of the PyPI wheels for Linux x86_64 and pure-Python wheels. +diff-cover==10.6.0 \ + --hash=sha256:a701553060f72d59b81f9cb1645b75ef66204427eb9dd99873c91092220e52ce +chardet==7.7.0 \ + --hash=sha256:2a990ca38150936cc8f63d66f360ba8c50310632f159ef482008f033185fb004 \ + --hash=sha256:7f13de78af5da07ddd813ff5a564c3aa8a113f05cba202be649e061c431a0e9a \ + --hash=sha256:feb66c96264e3e6a9ff0c7318ce0028800b6c18ad07c8000d2404a108d1ef767 \ + --hash=sha256:c519c10c315a675bcfd4b55c0e0de1859ca5a39881d8f69750500102c2fc5d2f \ + --hash=sha256:79eec5af9d7715334695254234c8d7c29165841526c11083951d9b9f6458e372 \ + --hash=sha256:d227d2aa329fcf848f4640ed9a4d99e1753840c1cfa19f7ca5a48f30c03982ad \ + --hash=sha256:42c4d8a2dc8abcc067417adc13c2f64f48482e48ce48e3af769b5a44214e5a5c \ + --hash=sha256:091f50d3567b1a066f797835ee5367dbcf3cb77c5e56993b3f61b28264b8e6f8 \ + --hash=sha256:11cf9b719ea3675a2a7d8a3071bbe4d7ea24b26bfe39a51588d515ddd05cacad +Jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +MarkupSafe==3.0.4 \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 +pluggy==1.7.0 \ + --hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec +Pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +coverage==7.17.0 \ + --hash=sha256:8e8113d3852e9377421a61de1917df27b3abb25404a4ecf69f9e64612c4345b5 \ + --hash=sha256:85932227e9dd38e62cd29c17fefdfecfb919eead6af1e3259522d03cf85de983 \ + --hash=sha256:045c1eeb19fb4eae949850a761102fffb3a5f638b97f977912f95843e54f6669 \ + --hash=sha256:acdb5f3c96046efff9dc6c572a6acb2b8d0da09372d7b9a9ed7249cf7d17434f \ + --hash=sha256:7c570293f001ea91598b90e90cb9b98fddee833e90bbede432cbd218331ccd8e \ + --hash=sha256:aed06129a9f530a948928260d7d2d81c47261355b0127e3c782251b38958463c \ + --hash=sha256:0d0fac604a962ff546beb038cba9a5a09d88e6e3f01612fb46f7bd4fe5dc8dca \ + --hash=sha256:bac487dce0d2aff28ddf1a533a916f771416eee0e493960f611e49037b878e9d +Markdown==3.11.2 \ + --hash=sha256:f80390747a2a0ff2ba54da074ce4f862f72a6685f608cd2637b646a6d83993f5 +cmake==4.4.4 \ + --hash=sha256:f773a0544c66370408f451acd487c2c643b0dfebb48c5d64f3638bc1bb820238 +ninja==1.13.2 \ + --hash=sha256:65a24341b5ac09fcadcc37082660be40a94174e51a937fabf6e2cae26225fa2c diff --git a/.github/ci/setup.sh b/.github/ci/setup.sh new file mode 100755 index 0000000..4fd61d0 --- /dev/null +++ b/.github/ci/setup.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Installs a pinned CI toolchain into the CI pool runner's cache and exports it +# to the following steps of the job (through $GITHUB_ENV and $GITHUB_PATH). +# +# .github/ci/setup.sh flutter|python|android|rust +# +# Everything lives under $CI_CACHE_DIR/proxy-app, never another project's +# folder. Downloads are checked against pinned SHA-256 sums, installed into a +# staging directory and kept across runs; nothing needs root. HOME moves into +# the cache too, so the pub cache, the Android debug keystore and the files +# tests write under $HOME stay inside this repository's folder. +set -euo pipefail + +: "${CI_CACHE_DIR:?the CI pool runner exports CI_CACHE_DIR}" +ROOT="$CI_CACHE_DIR/proxy-app" +TOOLS="$ROOT/tools" +REPO="$(cd "$(dirname "$0")/../.." && pwd)" + +FLUTTER_VERSION=3.47.5 +FLUTTER_SHA256=2132e990f236f8d22e7c6314b29a191a95b10d7cbcfec9b4e2e303d996652cbb +JDK_RELEASE="jdk-17.0.21+9" +JDK_FILE=OpenJDK17U-jdk_x64_linux_hotspot_17.0.21_9.tar.gz +JDK_SHA256=8b9b611d917e3dfd8419d3566b5949226f69b052fed7e109c1368add1dfe8711 +ANDROID_TOOLS_BUILD=15859902 # cmdline-tools 22.0 +ANDROID_TOOLS_SHA256=4e4c464f145a7512b57d088ac6c278c03c9eea610886b35a5e0804e74eedf583 +# Flutter 3.47.5 defaults: compileSdk 36, NDK 28.2.13676358. +ANDROID_PACKAGES=("platform-tools" "platforms;android-36" "build-tools;36.0.0" "ndk;28.2.13676358") +RUST_DEFAULT=1.99.0 + +env_set() { export "$1=$2"; echo "$1=$2" >> "$GITHUB_ENV"; } +path_add() { export PATH="$1:$PATH"; echo "$1" >> "$GITHUB_PATH"; } + +# fetch : download and verify. +fetch() { + curl -fsSL --retry 3 -o "$3" "$1" + echo "$2 $3" | sha256sum -c --quiet - +} + +# A fresh staging directory next to ; install() moves it into place. +stage() { rm -rf "$1.part"; mkdir -p "$1.part"; echo "$1.part"; } +install() { rm -rf "$1"; mv "$1.part" "$1"; } + +base() { + mkdir -p "$ROOT/home" "$TOOLS" + # rustup and its toolchains stay where the runner installed them. + env_set RUSTUP_HOME "${RUSTUP_HOME:-$HOME/.rustup}" + env_set HOME "$ROOT/home" + env_set PUB_CACHE "$ROOT/pub-cache" + env_set CARGO_HOME "$ROOT/cargo" +} + +flutter_sdk() { + local dir="$TOOLS/flutter-$FLUTTER_VERSION" tmp + if [ ! -x "$dir/bin/flutter" ]; then + tmp=$(stage "$dir") + fetch "https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_$FLUTTER_VERSION-stable.tar.xz" \ + "$FLUTTER_SHA256" "$tmp/flutter.tar.xz" + tar -xJf "$tmp/flutter.tar.xz" -C "$tmp" --strip-components=1 + rm "$tmp/flutter.tar.xz" + install "$dir" + fi + git config --global --get-all safe.directory | grep -qxF "$dir" || + git config --global --add safe.directory "$dir" + path_add "$dir/bin" + flutter --disable-analytics >/dev/null + dart --disable-analytics >/dev/null + # Toolchain check against the pin. + local actual + actual=$(flutter --version --machine | jq -r .frameworkVersion) + [ "$actual" = "$FLUTTER_VERSION" ] || { echo "Flutter $actual, pinned $FLUTTER_VERSION" >&2; exit 1; } + flutter --version +} + +python_tools() { + local dir + dir="$TOOLS/python-$(sha256sum "$REPO/.github/ci/requirements.txt" | cut -c1-12)" + # A virtual environment cannot move, so it is built in place and marked complete. + if [ ! -f "$dir/.complete" ]; then + rm -rf "$dir" + python3 -m venv "$dir" + "$dir/bin/pip" install --quiet --disable-pip-version-check --require-hashes --only-binary :all: \ + -r "$REPO/.github/ci/requirements.txt" + touch "$dir/.complete" + fi + path_add "$dir/bin" +} + +android_sdk() { + local jdk="$TOOLS/$JDK_RELEASE" sdk="$TOOLS/android-sdk" tmp + if [ ! -x "$jdk/bin/java" ]; then + tmp=$(stage "$jdk") + fetch "https://github.com/adoptium/temurin17-binaries/releases/download/${JDK_RELEASE/+/%2B}/$JDK_FILE" \ + "$JDK_SHA256" "$tmp/jdk.tar.gz" + tar -xzf "$tmp/jdk.tar.gz" -C "$tmp" --strip-components=1 + rm "$tmp/jdk.tar.gz" + install "$jdk" + fi + env_set JAVA_HOME "$jdk" + path_add "$jdk/bin" + local tools="$sdk/cmdline-tools/$ANDROID_TOOLS_BUILD" + if [ ! -x "$tools/bin/sdkmanager" ]; then + tmp=$(stage "$tools") + fetch "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_TOOLS_BUILD}_latest.zip" \ + "$ANDROID_TOOLS_SHA256" "$tmp/tools.zip" + unzip -q "$tmp/tools.zip" -d "$tmp" + mv "$tmp/cmdline-tools"/* "$tmp/" && rmdir "$tmp/cmdline-tools" && rm "$tmp/tools.zip" + install "$tools" + fi + env_set ANDROID_HOME "$sdk" + env_set ANDROID_SDK_ROOT "$sdk" + # sdkmanager verifies each package against the checksum in Google's repository index. + local stamp + stamp="$sdk/.packages-$(printf '%s\n' "${ANDROID_PACKAGES[@]}" | sha256sum | cut -c1-12)" + if [ ! -f "$stamp" ]; then + { yes || true; } | "$tools/bin/sdkmanager" --sdk_root="$sdk" --licenses >/dev/null + "$tools/bin/sdkmanager" --sdk_root="$sdk" "${ANDROID_PACKAGES[@]}" >/dev/null + touch "$stamp" + fi + # Overrides the project's 8 GB Gradle heap: ci-heavy servers have 8 GB in total. + env_set GRADLE_USER_HOME "$ROOT/gradle" + mkdir -p "$ROOT/gradle" + printf '%s\n' 'org.gradle.jvmargs=-Xmx3g -XX:MaxMetaspaceSize=1g' 'org.gradle.daemon=false' \ + 'kotlin.compiler.execution.strategy=in-process' > "$ROOT/gradle/gradle.properties" +} + +rust_toolchain() { + env_set CARGO_TARGET_DIR "$ROOT/target/updater" + env_set CARGO_PROFILE_DEV_DEBUG line-tables-only + if [ -f "$REPO/native/updater/rust-toolchain.toml" ]; then + (cd "$REPO/native/updater" && { rustup show active-toolchain || rustup toolchain install; }) + else + rustup toolchain install "$RUST_DEFAULT" --profile minimal -c rustfmt -c clippy + env_set RUSTUP_TOOLCHAIN "$RUST_DEFAULT" + fi +} + +base +case "${1:?usage: setup.sh flutter|python|android|rust}" in + flutter) flutter_sdk ;; + python) python_tools ;; + android) android_sdk ;; + rust) rust_toolchain ;; + *) echo "unknown tool: $1" >&2; exit 2 ;; +esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7548d91 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,214 @@ +# Tier-1 merge gate (PROJECT checks; README.md and docs/development.md list the +# commands), run on every ready pull request and on every push to main. +# +# Every job runs on the shared CI pool's self-hosted runners (`ci-heavy`: the +# servers are identical and none is pinned). Pinned toolchains (Flutter, JDK, +# Android SDK, Python tools) are installed by .github/ci/setup.sh into +# $CI_CACHE_DIR/proxy-app and reused across runs; nothing else is cached. A job-started hook on the servers refuses every job +# except the owner's own pull requests from branches of this repository, pushes +# to main, schedules and owner dispatches. +# +# What a pull request runs depends on what it changes (the `changes` job): a +# change only to README.md, docs/ or tool/render_docs.py runs only the docs job; +# the Android APKs build when android/ changes; the Linux desktop build and the +# native/updater Rust checks run when linux/ or native/updater/ changes; a change +# to pubspec or to this workflow builds everything. Draft pull requests run +# nothing until they are marked ready. On main every job runs. +# +# Not here (no Mac runners in the pool): macOS and iOS builds, the Swift +# tunnel-proxy tests of test/proxy_test.dart and the Android emulator e2e. They +# stay on the shared pool's Mac class outside GitHub Actions. Live-account tests +# skip themselves without credentials. Known load-sensitive tests are not +# retried: a failure is reported as it is. +name: ci + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # Checkout makes HEAD the pull request merged into its base, so HEAD^1 is + # the base and the diff is the request's; on main HEAD^1 is the previous + # (squash-merged) head. + changes: + name: changes + if: github.event_name != 'pull_request' || !github.event.pull_request.draft + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 5 + outputs: + code: ${{ steps.diff.outputs.code }} + android: ${{ steps.diff.outputs.android }} + linux: ${{ steps.diff.outputs.linux }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + - id: diff + shell: bash + env: + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" != pull_request ]; then + printf 'code=true\nandroid=true\nlinux=true\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + # --no-renames lists a moved file under its old path too. + files=$(git diff --name-only --no-renames HEAD^1 HEAD) + code=false; [ -n "$files" ] || code=true + grep -qvE '^(README\.md|docs/.*|tool/render_docs\.py)$' <<< "$files" && code=true + all='^(pubspec\.(yaml|lock)|\.github/(workflows/ci\.yml|ci/.*))$' + android=false; grep -qE "^android/|$all" <<< "$files" && android=true + linux=false; grep -qE "^(linux|native/updater)/|$all" <<< "$files" && linux=true + printf 'code=%s\nandroid=%s\nlinux=%s\n' "$code" "$android" "$linux" | tee -a "$GITHUB_OUTPUT" + + # Every ready pull request runs this job, a docs-only one nothing else. + docs: + name: docs + needs: changes + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + - run: .github/ci/setup.sh python + - name: Whitespace errors in the change + run: git diff --check HEAD^1 HEAD + - name: Rendered docs are fresh + run: | + python3 tool/render_docs.py + git diff --exit-code --stat || + { echo "Stale: run python3 tool/render_docs.py and commit docs/*.html"; exit 1; } + + lint: + name: format and analyze + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + - run: .github/ci/setup.sh flutter + - id: pub + run: flutter pub get --enforce-lockfile + # Each check below runs even when the one before it failed, so one run + # reports every failure. Format covers changed files only, whole files: + # the tree is not format-clean yet (PROJECT baseline); this becomes the + # whole tree once it is. + - name: dart format (changed files) + run: | + mapfile -t files < <(git diff --name-only --diff-filter=AMR HEAD^1 HEAD -- '*.dart') + if [ ${#files[@]} -eq 0 ]; then echo "No changed Dart files"; exit 0; fi + dart format --output=none --set-exit-if-changed "${files[@]}" + - if: ${{ !cancelled() && steps.pub.outcome == 'success' }} + run: flutter analyze --no-pub --fatal-infos --fatal-warnings + + test: + name: tests and changed-line coverage + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + - run: .github/ci/setup.sh flutter + - run: .github/ci/setup.sh python + - id: pub + run: flutter pub get --enforce-lockfile + # A library no test imports would be missing from lcov.info, and + # diff-cover would not count its lines: one generated (uncommitted) test + # imports every library under lib/. + - name: Load every library for coverage + run: | + pkg=$(sed -n 's/^name: *//p' pubspec.yaml) + { find lib -name '*.dart' | sort | sed -E "s#^lib/(.*)#import 'package:$pkg/\1';#" + echo 'void main() {}'; } > test/ci_all_libraries_test.dart + - run: flutter test --no-pub --coverage + # The checks below run even when the one before failed, so one run + # reports every failure. The publisher tests need OpenSSL 3 (the + # servers' Ubuntu ships it). + - name: tool/update tests + if: ${{ !cancelled() && steps.pub.outcome == 'success' }} + run: | + openssl version + python3 -m coverage run --include='tool/update/*' -m unittest discover -s tool/update -p 'test_*.py' + python3 -m coverage lcov -o coverage/tool-update.lcov + - name: Changed-line coverage (85%; 90% for critical paths) + if: ${{ !cancelled() && steps.pub.outcome == 'success' }} + run: | + git diff -U0 --no-ext-diff HEAD^1 HEAD > "$RUNNER_TEMP/change.diff" + reports=(coverage/lcov.info coverage/tool-update.lcov) + status=0 + diff-cover "${reports[@]}" --diff-file "$RUNNER_TEMP/change.diff" --fail-under 85 \ + --format "markdown:$RUNNER_TEMP/coverage.md" || status=$? + cat "$RUNNER_TEMP/coverage.md" >> "$GITHUB_STEP_SUMMARY" + diff-cover "${reports[@]}" --diff-file "$RUNNER_TEMP/change.diff" --fail-under 90 \ + --include 'lib/update/**' lib/proxy/system_proxy.dart lib/data/store.dart 'tool/update/**' 'native/updater/**' || status=$? + exit "$status" + # test/update_ui_test.dart writes its screenshot outside the checkout. + - name: Remove test output outside the checkout + if: always() + run: | + rm -rf "$GITHUB_WORKSPACE/../reports/app-updates-evidence" + if [ -d "$GITHUB_WORKSPACE/../reports" ]; then rmdir --ignore-fail-on-non-empty "$GITHUB_WORKSPACE/../reports"; fi + + android: + name: android debug APKs + needs: changes + if: needs.changes.outputs.android == 'true' + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: .github/ci/setup.sh flutter + - run: .github/ci/setup.sh android + - run: flutter pub get --enforce-lockfile + - run: flutter build apk --debug --flavor direct + - run: flutter build apk --debug --flavor play + + linux: + name: linux debug build and native/updater + needs: changes + if: needs.changes.outputs.linux == 'true' + runs-on: [self-hosted, ci-heavy] + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: .github/ci/setup.sh flutter + # CMake and Ninja come from the pinned Python tools. + - run: .github/ci/setup.sh python + - id: rust + run: .github/ci/setup.sh rust + # The desktop build below runs even when these checks fail, so one run + # reports both. + - name: cargo fmt, clippy and test + working-directory: native/updater + run: | + cargo fmt --check + cargo clippy --locked --all-targets -- -D warnings + cargo test --locked + # The desktop build links GTK 3 and libsecret, system packages that need + # root to install: the pool's server image must provide them. + - name: System libraries + id: libs + if: ${{ !cancelled() && steps.rust.outcome == 'success' }} + run: | + missing=$(for m in gtk+-3.0 libsecret-1; do pkg-config --exists "$m" || echo "$m"; done) + [ -z "$missing" ] || { echo "Missing on this runner (needs the server image): $missing"; exit 1; } + - if: ${{ !cancelled() && steps.libs.outcome == 'success' }} + run: | + flutter pub get --enforce-lockfile + flutter build linux --debug diff --git a/docs/readme.html b/docs/readme.html index 7571e66..eb26b70 100644 --- a/docs/readme.html +++ b/docs/readme.html @@ -332,4 +332,6 @@

Contributing and support

Report app bugs and propose improvements at shifter-io/proxy-app. Include the platform, operating-system version, reproduction steps, and expected behavior. Run the relevant checks before submitting code changes.

Use Shifter's support resources for account or service questions. Remove API keys, proxy passwords, account details, and browsing history from reports.

License

-

Project source is released under the MIT License. Third-party dependencies, fonts, and assets retain their own licenses. Shifter branding retains its respective rights. See third-party asset notices.

Shifter Proxy App · Standalone documentation · Repository
+

Project source is released under the MIT License. Third-party dependencies, fonts, and assets retain their own licenses. Shifter branding retains its respective rights. See third-party asset notices.

+

Application updates

+

See Application updates for platform integration, signed CDN metadata, Android flavors, publishing commands and production qualification gates. Updates are disabled until production configuration is supplied.