diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a65fcb0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,96 @@ +# CI for linux-kernel-optimization. +# +# Two jobs: +# 1. shellcheck - lint every Bash script in the repository. +# 2. ebpf-compile - build the CO-RE BPF objects with clang to catch type and +# CO-RE errors early. Loading/attaching is intentionally NOT attempted: +# it requires privileges and a matching kernel, which CI does not provide. +name: ci + +on: + push: + branches: [main, 'feat/**', 'fix/**'] + pull_request: + branches: [main, 'feat/**', 'fix/**'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + shellcheck: + name: shellcheck + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Lint all shell scripts + run: | + set -euo pipefail + mapfile -t scripts < <(git ls-files '*.sh') + if [ "${#scripts[@]}" -eq 0 ]; then + echo "No shell scripts found." >&2 + exit 1 + fi + printf 'Linting %d script(s)\n' "${#scripts[@]}" + shellcheck --version + # severity=warning: advisory style/info notes (for example SC1091 for + # dynamically computed source paths) must not fail the build, but real + # warnings and errors do. + shellcheck --severity=warning --external-sources "${scripts[@]}" + + ebpf-compile: + name: eBPF compile check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install build toolchain + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y clang llvm libelf-dev libbpf-dev build-essential + + # Ubuntu ships no standalone 'bpftool' package. The binary comes from + # the per-kernel linux-tools packages, and the 'linux-tools-common' + # wrapper only works when the tools package matching the running kernel + # is installed - which is often not the case on hosted runners. Try the + # plausible sources, then locate the real binary and put it on PATH. + for pkg in \ + "linux-tools-$(uname -r)" \ + linux-tools-azure \ + linux-tools-generic \ + linux-tools-common; do + sudo apt-get install -y --no-install-recommends "${pkg}" \ + || echo "note: ${pkg} is not installable on this runner" + done + + if ! bpftool version >/dev/null 2>&1; then + real="$(find /usr/lib/linux-tools -type f -name bpftool 2>/dev/null | head -n1 || true)" + if [ -n "${real}" ]; then + echo "using ${real} as bpftool" + sudo ln -sf "${real}" /usr/local/bin/bpftool + fi + fi + + command -v bpftool + bpftool version + + - name: Show toolchain versions + run: | + clang --version + + - name: Compile tcp_retrans BPF object + working-directory: ebpf/tcp_retrans + run: make check + + - name: Compile biolatency BPF object + working-directory: ebpf/biolatency + run: make check diff --git a/README.md b/README.md index 5375584..d0cf1dc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,116 @@ # linux-kernel-optimization -Linux kernel optimization and performance tuning experiments with eBPF and system observability. + +Tooling for measuring and tuning Linux kernel behaviour: CO-RE eBPF tracers for +TCP retransmissions and block I/O latency, commented sysctl tuning profiles with +a safe apply/revert script, and a repeatable benchmarking methodology. + +> **No performance numbers live in this repository.** Any claim about a tuning +> change must be produced on your own hardware with the scripts in `bench/` and +> reported using the rules in [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md). + +## Layout + +``` +ebpf/ + tcp_retrans/ Trace TCP retransmits, aggregated per destination address + biolatency/ Block I/O latency histogram (log2 microseconds) +tuning/ + network-latency.conf sysctl profile for latency-sensitive workloads + high-throughput.conf sysctl profile for bulk/throughput workloads + apply-profile.sh Apply a profile, backing up prior values; --revert +bench/ + common.sh Shared helpers (run dirs, metadata, logging) + run-iperf3.sh Capture iperf3 network measurements + run-fio.sh Capture fio block I/O measurements + run-perf-stat.sh Capture perf stat counters for a command +systemd/ + tuning.service Apply the selected profile at boot +docs/ + METHODOLOGY.md Warmup, repeats, and p99 reporting rules +.github/workflows/ci.yml shellcheck + eBPF compile check +``` + +## Requirements + +- Linux with BTF (`/sys/kernel/btf/vmlinux`) for CO-RE. +- Kernel 5.8+ recommended. +- `clang`, `llvm`, `bpftool`, `libbpf`, `libelf`, `zlib` for building the eBPF + programs and their userspace loaders. +- `root` (or `CAP_BPF` + `CAP_PERFMON`) to load and attach BPF programs. +- For benchmarks: `iperf3`, `fio`, `perf` (linux-tools), `sysstat` (optional). + +## Observability + +Two CO-RE tracers are provided. Both use kprobes rather than tracepoints: the +tracepoint context types (`trace_event_raw_*`) are not part of BTF and so are +absent from `vmlinux.h`. + +- `tcp_retrans` attaches to `tcp_retransmit_skb` and counts retransmissions per + destination address. +- `biolatency` attaches to `blk_account_io_start`/`blk_account_io_done` and + builds a log2 histogram of block I/O latency in microseconds. + +If a kprobe cannot be attached on your kernel (the function may be absent or +inlined), the loader reports the error from libbpf; run it as root and check +`/proc/kallsyms` for the symbol. + +## Quick start + +Build a tracer: + +```bash +cd ebpf/tcp_retrans +make # generates vmlinux.h, compiles the .bpf.o and the loader +sudo ./tcp_retrans -i 2 -n 10 +``` + +Block I/O latency histogram, optionally filtered to one device: + +```bash +cd ebpf/biolatency +make +sudo ./biolatency --device 259:0 +``` + +Apply a sysctl profile (records the previous values first): + +```bash +sudo tuning/apply-profile.sh --profile tuning/network-latency.conf +sudo tuning/apply-profile.sh --revert +``` + +Run a before/after comparison: + +```bash +bench/run-iperf3.sh --server 10.0.0.5 --phase before +sudo tuning/apply-profile.sh --profile tuning/network-latency.conf +bench/run-iperf3.sh --server 10.0.0.5 --phase after +``` + +Results are written under `bench/results/--/` +together with a `metadata.txt` describing the host and settings used. + +## Apply at boot + +```bash +sudo install -m 0644 systemd/tuning.service /etc/systemd/system/tuning.service +sudo systemctl daemon-reload +sudo systemctl enable --now tuning.service +``` + +Edit the unit (or use `systemctl edit` for a drop-in) to select the profile and +the checkout path. + +## Continuous integration + +CI runs [shellcheck](https://www.shellcheck.net/) over every `.sh` file and +compiles the BPF objects with `make check`. It does **not** load or attach the +programs: that needs privileges and a matching kernel, so it is left to the +operator. + +## License + +Files carry an SPDX identifier in their header (`GPL-2.0 OR BSD-3-Clause` for +eBPF sources, GPL-2.0 for kernel-loadable objects, and the project default for +scripts and configuration). + diff --git a/bench/common.sh b/bench/common.sh new file mode 100644 index 0000000..d088e33 --- /dev/null +++ b/bench/common.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# common.sh - Shared helpers for the benchmark scripts in this directory. +# +# Sourced (not executed) by run-iperf3.sh, run-fio.sh and run-perf-stat.sh. +# Provides logging, a run-directory factory, and reproducibility metadata. +# Never reports results itself: scripts only capture raw tool output. + +set -euo pipefail + +BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESULTS_ROOT="${BENCH_DIR}/results" + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%SZ)" "$*" >&2; } +die() { log "error: $*"; exit 1; } + +# require_cmd -> fail early if a required tool is missing. +require_cmd() { + local missing=() + local c + for c in "$@"; do + command -v "${c}" >/dev/null 2>&1 || missing+=("${c}") + done + if [[ "${#missing[@]}" -gt 0 ]]; then + die "missing required command(s): ${missing[*]}" + fi +} + +# new_run_dir