From 5ae3cf46cde9d7652b80f54b9cc2f0b7cc266038 Mon Sep 17 00:00:00 2001
From: sukonoe36 <337314619+sukonoe36@users.noreply.github.com>
Date: Sat, 3 Oct 2026 13:28:34 +0000
Subject: [PATCH 1/2] feat: initial build of kernel tuning and eBPF
 observability toolkit

Adds CO-RE eBPF tracers (tcp_retrans per-destination retransmit counter and biolatency block I/O histogram) with libbpf userspace loaders and Makefiles, two commented sysctl tuning profiles (network-latency, high-throughput) with a backup/revert apply script, before/after benchmark harness scripts, a systemd unit to apply a profile at boot, benchmark methodology docs, shellcheck + eBPF compile CI, and a rewritten README.
---
 .github/workflows/ci.yml           |  74 ++++++
 README.md                          | 101 ++++++++-
 bench/common.sh                    |  84 +++++++
 bench/run-fio.sh                   | 111 +++++++++
 bench/run-iperf3.sh                |  84 +++++++
 bench/run-perf-stat.sh             |  88 ++++++++
 docs/METHODOLOGY.md                |  86 +++++++
 ebpf/biolatency/Makefile           |  36 +++
 ebpf/biolatency/biolatency.bpf.c   | 132 +++++++++++
 ebpf/biolatency/biolatency.c       | 267 ++++++++++++++++++++++
 ebpf/tcp_retrans/Makefile          |  39 ++++
 ebpf/tcp_retrans/tcp_retrans.bpf.c |  79 +++++++
 ebpf/tcp_retrans/tcp_retrans.c     | 348 +++++++++++++++++++++++++++++
 systemd/tuning.service             |  52 +++++
 tuning/apply-profile.sh            | 204 +++++++++++++++++
 tuning/high-throughput.conf        |  74 ++++++
 tuning/network-latency.conf        |  80 +++++++
 17 files changed, 1938 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/ci.yml
 create mode 100644 bench/common.sh
 create mode 100644 bench/run-fio.sh
 create mode 100644 bench/run-iperf3.sh
 create mode 100644 bench/run-perf-stat.sh
 create mode 100644 docs/METHODOLOGY.md
 create mode 100644 ebpf/biolatency/Makefile
 create mode 100644 ebpf/biolatency/biolatency.bpf.c
 create mode 100644 ebpf/biolatency/biolatency.c
 create mode 100644 ebpf/tcp_retrans/Makefile
 create mode 100644 ebpf/tcp_retrans/tcp_retrans.bpf.c
 create mode 100644 ebpf/tcp_retrans/tcp_retrans.c
 create mode 100644 systemd/tuning.service
 create mode 100644 tuning/apply-profile.sh
 create mode 100644 tuning/high-throughput.conf
 create mode 100644 tuning/network-latency.conf

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..72614c6
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,74 @@
+# CI for linux-kernel-optimization.
+#
+# Two jobs:
+#   1. shellcheck - lint every Bash script in the repository.
+#   2. ebpf-compile - build the CO-RE BPF objects with clang to catch type and
+#      CO-RE errors early. Loading/attaching is intentionally NOT attempted:
+#      it requires privileges and a matching kernel, which CI does not provide.
+name: ci
+
+on:
+  push:
+    branches: [main, 'feat/**']
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  shellcheck:
+    name: shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Lint all shell scripts
+        run: |
+          set -euo pipefail
+          mapfile -t scripts < <(git ls-files '*.sh')
+          if [ "${#scripts[@]}" -eq 0 ]; then
+            echo "No shell scripts found." >&2
+            exit 1
+          fi
+          printf 'Linting %d script(s)\n' "${#scripts[@]}"
+          shellcheck --version
+          shellcheck --severity=style --external-sources "${scripts[@]}"
+
+  ebpf-compile:
+    name: eBPF compile check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install build toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            clang \
+            llvm \
+            libelf-dev \
+            libbpf-dev \
+            bpftool \
+            build-essential
+
+      - name: Show toolchain versions
+        run: |
+          clang --version
+          bpftool version || true
+
+      - name: Compile tcp_retrans BPF object
+        working-directory: ebpf/tcp_retrans
+        run: make check
+
+      - name: Compile biolatency BPF object
+        working-directory: ebpf/biolatency
+        run: make check
diff --git a/README.md b/README.md
index 5375584..728208f 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,101 @@
 # linux-kernel-optimization
-Linux kernel optimization and performance tuning experiments with eBPF and system observability.
+
+Tooling for measuring and tuning Linux kernel behaviour: CO-RE eBPF tracers for
+TCP retransmissions and block I/O latency, commented sysctl tuning profiles with
+a safe apply/revert script, and a repeatable benchmarking methodology.
+
+> **No performance numbers live in this repository.** Any claim about a tuning
+> change must be produced on your own hardware with the scripts in `bench/` and
+> reported using the rules in [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md).
+
+## Layout
+
+```
+ebpf/
+  tcp_retrans/     Trace TCP retransmits, aggregated per destination address
+  biolatency/      Block I/O latency histogram (log2 microseconds)
+tuning/
+  network-latency.conf   sysctl profile for latency-sensitive workloads
+  high-throughput.conf   sysctl profile for bulk/throughput workloads
+  apply-profile.sh       Apply a profile, backing up prior values; --revert
+bench/
+  common.sh              Shared helpers (run dirs, metadata, logging)
+  run-iperf3.sh          Capture iperf3 network measurements
+  run-fio.sh             Capture fio block I/O measurements
+  run-perf-stat.sh       Capture perf stat counters for a command
+systemd/
+  tuning.service         Apply the selected profile at boot
+docs/
+  METHODOLOGY.md         Warmup, repeats, and p99 reporting rules
+.github/workflows/ci.yml shellcheck + eBPF compile check
+```
+
+## Requirements
+
+- Linux with BTF (`/sys/kernel/btf/vmlinux`) for CO-RE.
+- Kernel 5.8+ recommended.
+- `clang`, `llvm`, `bpftool`, `libbpf`, `libelf`, `zlib` for building the eBPF
+  programs and their userspace loaders.
+- `root` (or `CAP_BPF` + `CAP_PERFMON`) to load and attach BPF programs.
+- For benchmarks: `iperf3`, `fio`, `perf` (linux-tools), `sysstat` (optional).
+
+## Quick start
+
+Build a tracer:
+
+```bash
+cd ebpf/tcp_retrans
+make            # generates vmlinux.h, compiles the .bpf.o and the loader
+sudo ./tcp_retrans -i 2 -n 10
+```
+
+Block I/O latency histogram, optionally filtered to one device:
+
+```bash
+cd ebpf/biolatency
+make
+sudo ./biolatency --device 259:0
+```
+
+Apply a sysctl profile (records the previous values first):
+
+```bash
+sudo tuning/apply-profile.sh --profile tuning/network-latency.conf
+sudo tuning/apply-profile.sh --revert
+```
+
+Run a before/after comparison:
+
+```bash
+bench/run-iperf3.sh --server 10.0.0.5 --phase before
+sudo tuning/apply-profile.sh --profile tuning/network-latency.conf
+bench/run-iperf3.sh --server 10.0.0.5 --phase after
+```
+
+Results are written under `bench/results/<UTC-timestamp>-<tool>-<phase>/`
+together with a `metadata.txt` describing the host and settings used.
+
+## Apply at boot
+
+```bash
+sudo install -m 0644 systemd/tuning.service /etc/systemd/system/tuning.service
+sudo systemctl daemon-reload
+sudo systemctl enable --now tuning.service
+```
+
+Edit the unit (or use `systemctl edit` for a drop-in) to select the profile and
+the checkout path.
+
+## Continuous integration
+
+CI runs [shellcheck](https://www.shellcheck.net/) over every `.sh` file and
+compiles the BPF objects with `make check`. It does **not** load or attach the
+programs: that needs privileges and a matching kernel, so it is left to the
+operator.
+
+## License
+
+Files carry an SPDX identifier in their header (`GPL-2.0 OR BSD-3-Clause` for
+eBPF sources, GPL-2.0 for kernel-loadable objects, and the project default for
+scripts and configuration).
+
diff --git a/bench/common.sh b/bench/common.sh
new file mode 100644
index 0000000..d088e33
--- /dev/null
+++ b/bench/common.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# common.sh - Shared helpers for the benchmark scripts in this directory.
+#
+# Sourced (not executed) by run-iperf3.sh, run-fio.sh and run-perf-stat.sh.
+# Provides logging, a run-directory factory, and reproducibility metadata.
+# Never reports results itself: scripts only capture raw tool output.
+
+set -euo pipefail
+
+BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RESULTS_ROOT="${BENCH_DIR}/results"
+
+log() { printf '[%s] %s\n' "$(date -u +%H:%M:%SZ)" "$*" >&2; }
+die() { log "error: $*"; exit 1; }
+
+# require_cmd <name...> -> fail early if a required tool is missing.
+require_cmd() {
+	local missing=()
+	local c
+	for c in "$@"; do
+		command -v "${c}" >/dev/null 2>&1 || missing+=("${c}")
+	done
+	if [[ "${#missing[@]}" -gt 0 ]]; then
+		die "missing required command(s): ${missing[*]}"
+	fi
+}
+
+# new_run_dir <label> -> creates and prints results/<UTC timestamp>-<label>.
+new_run_dir() {
+	local label="${1:-run}"
+	local stamp dir
+	stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+	dir="${RESULTS_ROOT}/${stamp}-${label}"
+	mkdir -p -- "${dir}"
+	printf '%s\n' "${dir}"
+}
+
+# write_metadata <dir> <phase> <notes...> -> records environment for the run.
+write_metadata() {
+	local dir="$1" phase="$2"
+	shift 2
+	local file="${dir}/metadata.txt"
+
+	{
+		printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+		printf 'phase=%s\n' "${phase}"
+		printf 'hostname=%s\n' "$(uname -n)"
+		printf 'kernel=%s\n' "$(uname -sr)"
+		printf 'arch=%s\n' "$(uname -m)"
+		printf 'cpu_model=%s\n' \
+			"$(awk -F: '/model name/{print $2; exit}' /proc/cpuinfo 2>/dev/null | sed 's/^ //')"
+		printf 'cpu_count=%s\n' "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo unknown)"
+		printf 'congestion_control=%s\n' \
+			"$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo unknown)"
+		printf 'default_qdisc=%s\n' \
+			"$(sysctl -n net.core.default_qdisc 2>/dev/null || echo unknown)"
+		for note in "$@"; do
+			printf 'note=%s\n' "${note}"
+		done
+	} > "${file}"
+
+	log "wrote metadata to ${file}"
+}
+
+# usage_for <script> <extra-lines...>
+usage_for() {
+	local script="$1"
+	shift
+	printf 'Usage: %s [options]\n\n' "$(basename "${script}")"
+	for line in "$@"; do
+		printf '%s\n' "${line}"
+	done
+}
+
+# tee_capture <file> <command...> -> run command, tee stdout+stderr to <file>.
+tee_capture() {
+	local file="$1"
+	shift
+	log "running: $*"
+	# shellcheck disable=SC2068
+	"$@" 2>&1 | tee -- "${file}"
+	return "${PIPESTATUS[0]}"
+}
diff --git a/bench/run-fio.sh b/bench/run-fio.sh
new file mode 100644
index 0000000..caee4d4
--- /dev/null
+++ b/bench/run-fio.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+#
+# run-fio.sh - Capture fio block I/O measurements for before/after runs.
+#
+# Runs a configurable fio workload (default: random read) and saves raw JSON
+# output plus run metadata into a timestamped results folder. This script only
+# captures data; interpretation lives in docs/METHODOLOGY.md.
+#
+# Usage:
+#   ./run-fio.sh --target /dev/nvme0n1 --phase before
+#   ./run-fio.sh --target /tmp/fio-test --rw randwrite --size 4G --help
+#
+# Exit codes: 0 success, 1 usage error, 2 tool failure.
+
+set -euo pipefail
+
+# shellcheck source=common.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
+
+TARGET=""
+PHASE="before"
+RW="randread"
+BS="4k"
+IODEPTH=32
+SIZE="2G"
+RUNTIME=30
+NUMJOBS=1
+DIRECT=1
+LABEL="fio"
+
+usage() {
+	usage_for "${BASH_SOURCE[0]}" \
+		"Capture fio results into bench/results/<timestamp>-fio/." \
+		"" \
+		"Options:" \
+		"  -f, --target PATH     File or block device to test (required)" \
+		"  -w, --rw MODE         I/O pattern: randread|randwrite|read|write|randrw" \
+		"                        (default: randread)" \
+		"  -b, --bs SIZE         Block size (default: 4k)" \
+		"  -d, --iodepth N       Queue depth per job (default: 32)" \
+		"  -s, --size SIZE       Total size per job (default: 2G)" \
+		"  -r, --runtime SECS    Run time in seconds (default: 30)" \
+		"  -j, --numjobs N       Number of parallel jobs (default: 1)" \
+		"      --buffered        Use buffered I/O instead of O_DIRECT" \
+		"      --phase NAME      Label the run 'before' or 'after' (default: before)" \
+		"  -h, --help            Show this help and exit"
+}
+
+main() {
+	while [[ $# -gt 0 ]]; do
+		case "$1" in
+			-f|--target) TARGET="${2:-}"; shift 2 ;;
+			-w|--rw) RW="${2:-}"; shift 2 ;;
+			-b|--bs) BS="${2:-}"; shift 2 ;;
+			-d|--iodepth) IODEPTH="${2:-}"; shift 2 ;;
+			-s|--size) SIZE="${2:-}"; shift 2 ;;
+			-r|--runtime) RUNTIME="${2:-}"; shift 2 ;;
+			-j|--numjobs) NUMJOBS="${2:-}"; shift 2 ;;
+			--buffered) DIRECT=0; shift ;;
+			--phase) PHASE="${2:-}"; shift 2 ;;
+			-h|--help) usage; exit 0 ;;
+			*) die "unknown argument '$1' (try --help)" ;;
+		esac
+	done
+
+	[[ -n "${TARGET}" ]] || { usage; die "--target is required"; }
+
+	case "${PHASE}" in
+		before|after) ;;
+		*) die "--phase must be 'before' or 'after'" ;;
+	esac
+
+	case "${RW}" in
+		randread|randwrite|read|write|randrw) ;;
+		*) die "unsupported --rw '${RW}'" ;;
+	esac
+
+	require_cmd fio
+
+	local run_dir
+	run_dir="$(new_run_dir "${LABEL}-${PHASE}")"
+	write_metadata "${run_dir}" "${PHASE}" \
+		"target=${TARGET}" "rw=${RW}" "bs=${BS}" "iodepth=${IODEPTH}" \
+		"size=${SIZE}" "runtime=${RUNTIME}s" "numjobs=${NUMJOBS}" \
+		"direct=${DIRECT}"
+
+	local direct_flag=1
+	[[ "${DIRECT}" -eq 1 ]] && direct_flag=1 || direct_flag=0
+
+	log "starting fio (${RW}, bs=${BS}) against ${TARGET}"
+	if ! tee_capture "${run_dir}/fio.json" \
+		fio --name="kernel-opt-${PHASE}" \
+			--filename="${TARGET}" \
+			--rw="${RW}" \
+			--bs="${BS}" \
+			--iodepth="${IODEPTH}" \
+			--size="${SIZE}" \
+			--runtime="${RUNTIME}" \
+			--time_based \
+			--numjobs="${NUMJOBS}" \
+			--direct="${direct_flag}" \
+			--ioengine=libaio \
+			--group_reporting \
+			--output-format=json; then
+		die "fio run failed"
+	fi
+
+	log "results saved under ${run_dir}"
+}
+
+main "$@"
diff --git a/bench/run-iperf3.sh b/bench/run-iperf3.sh
new file mode 100644
index 0000000..96db9b5
--- /dev/null
+++ b/bench/run-iperf3.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# run-iperf3.sh - Capture iperf3 network measurements for before/after runs.
+#
+# Runs iperf3 in client mode against a target server and saves raw output plus
+# metadata into a timestamped results folder. This script only captures data;
+# interpreting it is left to docs/METHODOLOGY.md.
+#
+# Usage:
+#   ./run-iperf3.sh --server HOST [--phase before|after] [--duration 30]
+#   ./run-iperf3.sh --help
+#
+# Exit codes: 0 success, 1 usage error, 2 tool failure.
+
+set -euo pipefail
+
+# shellcheck source=common.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
+
+SERVER=""
+PHASE="before"
+DURATION=30
+STREAMS=1
+PORT=5201
+DIRECTION="upload"
+LABEL="iperf3"
+
+usage() {
+	usage_for "${BASH_SOURCE[0]}" \
+		"Capture iperf3 results into bench/results/<timestamp>-iperf3/." \
+		"" \
+		"Options:" \
+		"  -s, --server HOST     iperf3 server hostname/IP (required)" \
+		"  -p, --port PORT       iperf3 server port (default: 5201)" \
+		"  -t, --duration SECS   Test duration in seconds (default: 30)" \
+		"  -P, --streams N       Parallel streams (default: 1)" \
+		"  -R, --reverse         Measure download instead of upload" \
+		"      --phase NAME      Label the run 'before' or 'after' (default: before)" \
+		"  -h, --help            Show this help and exit"
+}
+
+main() {
+	while [[ $# -gt 0 ]]; do
+		case "$1" in
+			-s|--server) SERVER="${2:-}"; shift 2 ;;
+			-p|--port) PORT="${2:-}"; shift 2 ;;
+			-t|--duration) DURATION="${2:-}"; shift 2 ;;
+			-P|--streams) STREAMS="${2:-}"; shift 2 ;;
+			-R|--reverse) DIRECTION="download"; shift ;;
+			--phase) PHASE="${2:-}"; shift 2 ;;
+			-h|--help) usage; exit 0 ;;
+			*) die "unknown argument '$1' (try --help)" ;;
+		esac
+	done
+
+	[[ -n "${SERVER}" ]] || { usage; die "--server is required"; }
+
+	case "${PHASE}" in
+		before|after) ;;
+		*) die "--phase must be 'before' or 'after'" ;;
+	esac
+
+	require_cmd iperf3
+
+	local run_dir
+	run_dir="$(new_run_dir "${LABEL}-${PHASE}")"
+	write_metadata "${run_dir}" "${PHASE}" \
+		"server=${SERVER}" "port=${PORT}" "duration=${DURATION}s" \
+		"streams=${STREAMS}" "direction=${DIRECTION}"
+
+	local -a args=(-c "${SERVER}" -p "${PORT}" -t "${DURATION}" -P "${STREAMS}" --json)
+	if [[ "${DIRECTION}" == "download" ]]; then
+		args+=(-R)
+	fi
+
+	log "starting iperf3 (${DIRECTION}) against ${SERVER}:${PORT}"
+	if ! tee_capture "${run_dir}/iperf3.json" iperf3 "${args[@]}"; then
+		die "iperf3 run failed"
+	fi
+
+	log "results saved under ${run_dir}"
+}
+
+main "$@"
diff --git a/bench/run-perf-stat.sh b/bench/run-perf-stat.sh
new file mode 100644
index 0000000..55b78ac
--- /dev/null
+++ b/bench/run-perf-stat.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# run-perf-stat.sh - Capture `perf stat` counters for a command, before/after.
+#
+# Wraps an arbitrary command with `perf stat`, saving the counter output and
+# run metadata into a timestamped results folder. Use this to observe whether a
+# tuning change shifted CPU-level behavior (cycles, cache misses, context
+# switches). This script only captures data; see docs/METHODOLOGY.md.
+#
+# Usage:
+#   ./run-perf-stat.sh --phase before -- ./your-benchmark --args
+#   ./run-perf-stat.sh --help
+#
+# Exit codes: 0 success, 1 usage error, 2 tool failure.
+
+set -euo pipefail
+
+# shellcheck source=common.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
+
+PHASE="before"
+REPEAT=3
+EVENTS="task-clock,cycles,instructions,cache-misses,context-switches"
+LABEL="perf-stat"
+CMD=()
+
+usage() {
+	usage_for "${BASH_SOURCE[0]}" \
+		"Capture perf stat counters into bench/results/<timestamp>-perf-stat/." \
+		"" \
+		"Options:" \
+		"      --phase NAME      Label the run 'before' or 'after' (default: before)" \
+		"  -r, --repeat N        Repeat the measurement N times (default: 3)" \
+		"  -e, --events LIST     Comma-separated perf events to count" \
+		"                        (default: ${EVENTS})" \
+		"  -h, --help            Show this help and exit" \
+		"" \
+		"Everything after '--' is the command to measure." \
+		"" \
+		"Example:" \
+		"  sudo ./run-perf-stat.sh --phase before -- ./my-bench --iterations 1000"
+}
+
+main() {
+	local passthrough=0
+	while [[ $# -gt 0 ]]; do
+		if [[ "${passthrough}" -eq 1 ]]; then
+			CMD+=("$1")
+			shift
+			continue
+		fi
+		case "$1" in
+			--phase) PHASE="${2:-}"; shift 2 ;;
+			-r|--repeat) REPEAT="${2:-}"; shift 2 ;;
+			-e|--events) EVENTS="${2:-}"; shift 2 ;;
+			-h|--help) usage; exit 0 ;;
+			--) passthrough=1; shift ;;
+			*) die "unexpected argument '$1' (use '--' before the command, or --help)" ;;
+		esac
+	done
+
+	[[ "${#CMD[@]}" -gt 0 ]] || { usage; die "no command given after '--'"; }
+
+	case "${PHASE}" in
+		before|after) ;;
+		*) die "--phase must be 'before' or 'after'" ;;
+	esac
+
+	require_cmd perf
+
+	local run_dir
+	run_dir="$(new_run_dir "${LABEL}-${PHASE}")"
+	write_metadata "${run_dir}" "${PHASE}" \
+		"repeat=${REPEAT}" "events=${EVENTS}" "command=${CMD[*]}"
+
+	local i
+	for ((i = 1; i <= REPEAT; i++)); do
+		log "perf stat iteration ${i}/${REPEAT}"
+		if ! tee_capture "${run_dir}/perf-stat-run${i}.txt" \
+			perf stat -e "${EVENTS}" -- "${CMD[@]}"; then
+			die "perf stat run ${i} failed"
+		fi
+	done
+
+	log "results saved under ${run_dir}"
+}
+
+main "$@"
diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md
new file mode 100644
index 0000000..6562f8e
--- /dev/null
+++ b/docs/METHODOLOGY.md
@@ -0,0 +1,86 @@
+# Benchmark Methodology
+
+This document describes **how** to measure the effect of a tuning change. It
+deliberately contains no results: numbers depend on your hardware, kernel,
+workload and network, so every figure must be produced on the target host with
+the scripts in `bench/`.
+
+The goal is a reproducible, honest before/after comparison rather than a single
+lucky run.
+
+## 1. Establish a baseline ("before")
+
+1. Confirm the machine is otherwise idle (no background jobs, no package
+   updates, no other tenants on shared storage/network).
+2. Record the starting kernel settings. `tuning/apply-profile.sh` writes a
+   backup of every key it is about to change to `/var/backups/sysctl-profiles/`;
+   keep that file as the reference for the "before" state.
+3. Run each benchmark script with `--phase before`. Results land in
+   `bench/results/<UTC-timestamp>-<tool>-before/`.
+
+## 2. Apply the change
+
+```bash
+sudo tuning/apply-profile.sh --profile tuning/network-latency.conf
+```
+
+The script prints each key it applied. Any failures are reported and the script
+exits non-zero, so a partial apply is visible.
+
+## 3. Run the "after" measurement
+
+Re-run the **same** scripts with `--phase after`, against the same target, with
+the same flags, on the same host, at a comparable time of day. Results land in
+`bench/results/<UTC-timestamp>-<tool>-after/`.
+
+## 4. Warmup
+
+The first run of any benchmark is not representative: page cache is cold, CPU
+frequency scaling has not settled, TCP has not converged its congestion window,
+and storage caches are empty. Always:
+
+- Perform at least one **discard/warmup run** whose output you do not report.
+- Discard warmup output for `fio` (page cache fill) and `iperf3` (slow-start
+  ramp) in particular.
+
+## 5. Repeat runs
+
+- Run every configuration **at least 5 times** and report the distribution, not
+  a single sample.
+- Alternate the order (before, after, before, after, ...) when comparing two
+  configurations on the same host so that drift over time affects both equally.
+- Keep all raw files. Do not delete outliers silently; note them and explain.
+
+## 6. Reporting p99 (and other percentiles)
+
+Mean throughput hides tail behavior, which is usually what matters for
+latency-sensitive services. For every metric:
+
+- Report **p50, p90, p99** and the **max**, alongside the mean.
+- For `iperf3`, parse the per-interval JSON (`--json`) and compute percentiles
+  over the intervals, not just the summary line.
+- For `fio`, use `--output-format=json` and read `clat`/`lat` percentiles from
+  the JSON rather than the human-readable summary.
+- For `biolatency`, report the histogram buckets and derive p99 from the bucket
+  boundaries; state the bucket resolution so the reader knows the precision.
+- For `tcp_retrans`, report retransmits per destination and the share of total;
+  a lower count is only meaningful when total traffic is comparable.
+
+Percentiles computed from a histogram are approximations. Say so.
+
+## 7. Comparing configurations
+
+- Compute the delta as `(after - before) / before` per metric and report it with
+  the spread across repeats.
+- Only claim an improvement when the distributions are separated by more than
+  their run-to-run spread.
+- State what did **not** improve. A profile that helps throughput often hurts
+  latency; document the tradeoff rather than hiding it.
+
+## 8. What not to do
+
+- Do not copy numbers from another machine, a blog post, or a vendor datasheet.
+- Do not report a single run as a result.
+- Do not tune the benchmark harness (e.g. drop caches mid-run) without saying
+  so.
+- Do not claim a change is faster without the raw files to back it up.
diff --git a/ebpf/biolatency/Makefile b/ebpf/biolatency/Makefile
new file mode 100644
index 0000000..5c60cbd
--- /dev/null
+++ b/ebpf/biolatency/Makefile
@@ -0,0 +1,36 @@
+# biolatency loader links the userspace program against libbpf. CI compiles
+# only the BPF object (see `check`) to avoid requiring libbpf on the runner.
+
+CLANG   ?= clang
+CC      ?= cc
+CFLAGS  ?= -O2 -g -Wall -Wextra
+BPFTOOL ?= bpftool
+ARCH    := $(shell uname -m | sed 's/x86_64/x86/;s/aarch64/arm64/;s/ppc64le/powerpc/;s/mips.*/mips/;s/riscv64/riscv/;s/loongarch64/loongarch/')
+
+BPF_CFLAGS := -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) -I. -Wall
+
+.PHONY: all build clean check help
+
+all: build
+
+help:
+	@echo "Targets:"
+	@echo "  build  - compile biolatency.bpf.c and the userspace loader"
+	@echo "  check  - compile the BPF object only (used by CI)"
+	@echo "  clean  - remove generated objects and binaries"
+
+vmlinux.h:
+	$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format header > $@
+
+biolatency.bpf.o: biolatency.bpf.c vmlinux.h
+	$(CLANG) $(BPF_CFLAGS) -c $< -o $@
+
+biolatency: biolatency.c biolatency.bpf.o
+	$(CC) $(CFLAGS) -o $@ $< -lbpf -lelf -lz
+
+build: biolatency.bpf.o biolatency
+
+check: biolatency.bpf.o
+
+clean:
+	rm -f biolatency.bpf.o biolatency vmlinux.h
diff --git a/ebpf/biolatency/biolatency.bpf.c b/ebpf/biolatency/biolatency.bpf.c
new file mode 100644
index 0000000..4256dfc
--- /dev/null
+++ b/ebpf/biolatency/biolatency.bpf.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* biolatency.bpf.c - Block I/O latency histogram (CO-RE).
+ *
+ * Traces block I/O issue and completion, computing per-request latency and
+ * recording it into log2-bucketed histograms, one per CPU (aggregated in
+ * userspace). Works across devices; the device major:minor can optionally be
+ * filtered from userspace via the filter map.
+ *
+ * Build with: make (uses bpftool gen to produce vmlinux.h if absent).
+ */
+#include <vmlinux.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+#define MAX_BUCKETS 32 /* log2 microseconds: 0..2^31 us */
+
+/* In-flight request tracking, keyed by request pointer. */
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 16384);
+	__type(key, __u64); /* (u64)struct request * */
+	__type(value, __u64); /* start timestamp (ns) */
+} starts SEC(".maps");
+
+/* Per-CPU latency histograms, bucket = log2(latency_us). */
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(max_entries, MAX_BUCKETS);
+	__type(key, __u32);
+	__type(value, __u64);
+} lat_hist SEC(".maps");
+
+/* Optional device filter: if a key exists, only that major:minor is traced. */
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 1);
+	__type(key, __u32); /* (major << 8) | minor is not used; dev_t encoded */
+	__type(value, __u8);
+} filter_map SEC(".maps");
+
+/* Number of requests issued/completed, for sanity checks in userspace. */
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(max_entries, 2); /* [0]=issued [1]=completed */
+	__type(key, __u32);
+	__type(value, __u64);
+} counts SEC(".maps");
+
+static __always_inline __u32 dev_key(struct request *rq)
+{
+	struct gendisk *disk = BPF_CORE_READ(rq, q, disk);
+	__u32 major = BPF_CORE_READ(disk, major);
+	__u16 first_minor = BPF_CORE_READ(disk, first_minor);
+
+	return (major << 16) | first_minor;
+}
+
+static __always_inline int allowed(struct request *rq)
+{
+	__u32 key;
+	__u32 slot = 0;
+	__u8 *v;
+
+	if (bpf_map_lookup_elem(&filter_map, &slot) == NULL)
+		return 1; /* no filter installed -> trace everything */
+
+	key = dev_key(rq);
+	v = bpf_map_lookup_elem(&filter_map, &key);
+	return v != NULL;
+}
+
+static __always_inline void count_inc(__u32 slot)
+{
+	__u64 *c = bpf_map_lookup_elem(&counts, &slot);
+
+	if (c)
+		__sync_fetch_and_add(c, 1);
+}
+
+SEC("tracepoint/block/block_rq_issue")
+int trace_block_rq_issue(struct trace_event_raw_block_rq_issue *ctx)
+{
+	__u64 ts = bpf_ktime_get_ns();
+	__u64 *p;
+	__u64 req = (__u64)ctx->rq;
+
+	if (!allowed((struct request *)(unsigned long)req))
+		return 0;
+
+	p = bpf_map_lookup_elem(&starts, &req);
+	if (p)
+		*p = ts;
+	else
+		bpf_map_update_elem(&starts, &req, &ts, BPF_ANY);
+
+	count_inc(0);
+	return 0;
+}
+
+SEC("tracepoint/block/block_rq_complete")
+int trace_block_rq_complete(struct trace_event_raw_block_rq_complete *ctx)
+{
+	__u64 req = (__u64)ctx->rq;
+	__u64 *start, lat_ns, lat_us;
+	__u32 bucket, slot = 1;
+	__u64 *h;
+
+	start = bpf_map_lookup_elem(&starts, &req);
+	if (!start)
+		return 0;
+
+	lat_ns = bpf_ktime_get_ns() - *start;
+	bpf_map_delete_elem(&starts, &req);
+	lat_us = lat_ns / 1000;
+
+	/* Bucket by integer log2 of microseconds. */
+	bucket = lat_us ? (63 - __builtin_clzll(lat_us)) : 0;
+	if (bucket >= MAX_BUCKETS)
+		bucket = MAX_BUCKETS - 1;
+
+	h = bpf_map_lookup_elem(&lat_hist, &bucket);
+	if (h)
+		__sync_fetch_and_add(h, 1);
+
+	count_inc(slot);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/ebpf/biolatency/biolatency.c b/ebpf/biolatency/biolatency.c
new file mode 100644
index 0000000..37623fe
--- /dev/null
+++ b/ebpf/biolatency/biolatency.c
@@ -0,0 +1,267 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* biolatency.c - Userspace loader for the block I/O latency BPF program.
+ *
+ * Loads biolatency.bpf.o, polls the per-CPU log2 latency histogram and renders
+ * an ASCII histogram, optionally restricted to a single block device via
+ * --device MAJOR:MINOR. Values are microseconds, bucketed by floor(log2(us)).
+ *
+ * Usage: sudo ./biolatency [-i seconds] [--device MAJOR:MINOR] [--object PATH]
+ */
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <errno.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define MAX_BUCKETS 32 /* must match biolatency.bpf.c */
+
+static volatile sig_atomic_t stop;
+
+static void on_signal(int signo)
+{
+	(void)signo;
+	stop = 1;
+}
+
+static void usage(const char *prog)
+{
+	fprintf(stderr,
+		"Usage: %s [options]\n"
+		"\n"
+		"Print a block I/O latency histogram (microseconds, log2 buckets).\n"
+		"\n"
+		"Options:\n"
+		"  -i, --interval SECS      Poll interval in seconds (default: 1)\n"
+		"  -d, --device MAJ:MIN     Restrict to one block device\n"
+		"      --object PATH        Path to the BPF object file\n"
+		"                           (default: biolatency.bpf.o next to the binary)\n"
+		"  -h, --help               Show this help and exit\n",
+		prog);
+}
+
+static int parse_device(const char *arg, __u32 *key)
+{
+	unsigned int major, minor;
+	char extra;
+
+	if (sscanf(arg, "%u:%u%c", &major, &minor, &extra) != 2) {
+		fprintf(stderr, "error: invalid --device '%s' (expected MAJ:MIN)\n", arg);
+		return -1;
+	}
+	if (major > 0xffff || minor > 0xffff) {
+		fprintf(stderr, "error: --device values must fit in 16 bits\n");
+		return -1;
+	}
+	/* Match the encoding used by the BPF side: (major << 16) | minor. */
+	*key = (major << 16) | minor;
+	return 0;
+}
+
+static int parse_interval(const char *value, long *out)
+{
+	char *end = NULL;
+	long v;
+
+	errno = 0;
+	v = strtol(value, &end, 10);
+	if (errno != 0 || end == value || *end != '\0' || v <= 0) {
+		fprintf(stderr, "error: invalid interval '%s'\n", value);
+		return -1;
+	}
+	*out = v;
+	return 0;
+}
+
+static const char *default_object_path(char *buf, size_t len, const char *argv0)
+{
+	const char *slash, *dir;
+	size_t dirlen;
+
+	if (!argv0)
+		return NULL;
+	slash = strrchr(argv0, '/');
+	dir = slash ? argv0 : ".";
+	dirlen = slash ? (size_t)(slash - argv0) + 1 : 2;
+	if (dirlen + strlen("biolatency.bpf.o") + 1 > len)
+		return NULL;
+	memcpy(buf, dir, dirlen);
+	buf[dirlen] = '\0';
+	strncat(buf, "biolatency.bpf.o", len - dirlen - 1);
+	return buf;
+}
+
+static void read_hist(int map_fd, __u64 *hist)
+{
+	__u64 per_cpu[MAX_BUCKETS];
+	__u32 key;
+
+	for (key = 0; key < MAX_BUCKETS; key++) {
+		if (bpf_map_lookup_elem(map_fd, &key, per_cpu) != 0)
+			continue;
+		for (size_t c = 0; c < sizeof(per_cpu) / sizeof(per_cpu[0]); c++)
+			hist[key] += per_cpu[c];
+	}
+}
+
+static void print_hist(const __u64 *hist)
+{
+	__u64 max = 0, total = 0;
+	int first = -1, last = -1;
+
+	for (int i = 0; i < MAX_BUCKETS; i++) {
+		if (hist[i]) {
+			if (first < 0)
+				first = i;
+			last = i;
+		}
+		if (hist[i] > max)
+			max = hist[i];
+		total += hist[i];
+	}
+
+	if (first < 0) {
+		printf("(no block I/O completions observed yet)\n");
+		return;
+	}
+
+	printf("block I/O latency (usec), total samples: %" PRIu64 "\n\n", total);
+	for (int i = first; i <= last; i++) {
+		__u64 lo = 1ULL << i;
+		__u64 hi = (1ULL << (i + 1)) - 1;
+		int bars = max ? (int)((hist[i] * 40) / max) : 0;
+
+		printf("%8" PRIu64 " -> %8" PRIu64 " : %-40.*s %" PRIu64 "\n",
+		       lo, hi, bars, "****************************************", hist[i]);
+	}
+}
+
+int main(int argc, char **argv)
+{
+	struct bpf_object *obj = NULL;
+	struct bpf_program *prog;
+	struct bpf_link *link = NULL;
+	struct bpf_map *hist_map, *filter_map;
+	char obj_path[4096] = { 0 };
+	const char *object = NULL;
+	long interval = 1;
+	bool have_device = false;
+	__u32 device_key = 0;
+	int rc, opt;
+
+	static const struct option long_opts[] = {
+		{ "interval", required_argument, NULL, 'i' },
+		{ "device", required_argument, NULL, 'd' },
+		{ "object", required_argument, NULL, 1 },
+		{ "help", no_argument, NULL, 'h' },
+		{ NULL, 0, NULL, 0 },
+	};
+
+	while ((opt = getopt_long(argc, argv, "i:d:h", long_opts, NULL)) != -1) {
+		switch (opt) {
+		case 'i':
+			if (parse_interval(optarg, &interval))
+				return 1;
+			break;
+		case 'd':
+			if (parse_device(optarg, &device_key))
+				return 1;
+			have_device = true;
+			break;
+		case 1:
+			object = optarg;
+			break;
+		case 'h':
+			usage(argv[0]);
+			return 0;
+		default:
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (!object) {
+		if (!default_object_path(obj_path, sizeof(obj_path), argv[0])) {
+			fprintf(stderr, "error: object path too long\n");
+			return 1;
+		}
+		object = obj_path;
+	}
+
+	if (geteuid() != 0)
+		fprintf(stderr, "warning: not running as root; BPF load will likely fail\n");
+
+	signal(SIGINT, on_signal);
+	signal(SIGTERM, on_signal);
+
+	obj = bpf_object__open_file(object, NULL);
+	rc = libbpf_get_error(obj);
+	if (rc) {
+		obj = NULL;
+		fprintf(stderr, "error: failed to open BPF object '%s': %s\n",
+			object, strerror(-rc));
+		goto cleanup;
+	}
+
+	rc = bpf_object__load(obj);
+	if (rc) {
+		fprintf(stderr, "error: failed to load BPF object: %s\n", strerror(-rc));
+		goto cleanup;
+	}
+
+	hist_map = bpf_object__find_map_by_name(obj, "lat_hist");
+	filter_map = bpf_object__find_map_by_name(obj, "filter_map");
+	if (!hist_map || !filter_map) {
+		fprintf(stderr, "error: expected maps not found in BPF object\n");
+		goto cleanup;
+	}
+
+	if (have_device) {
+		__u8 one = 1;
+
+		if (bpf_map_update_elem(bpf_map__fd(filter_map), &device_key, &one,
+					BPF_ANY)) {
+			fprintf(stderr, "error: failed to install device filter: %s\n",
+				strerror(errno));
+			goto cleanup;
+		}
+	}
+
+	prog = bpf_object__next_program(obj, NULL);
+	for (; prog; prog = bpf_object__next_program(obj, prog)) {
+		link = bpf_program__attach(prog);
+		rc = libbpf_get_error(link);
+		if (rc) {
+			link = NULL;
+			fprintf(stderr, "error: failed to attach program: %s\n",
+				strerror(-rc));
+			goto cleanup;
+		}
+	}
+
+	while (!stop) {
+		__u64 hist[MAX_BUCKETS] = { 0 };
+
+		sleep((unsigned int)interval);
+		if (stop)
+			break;
+
+		read_hist(bpf_map__fd(hist_map), hist);
+		print_hist(hist);
+	}
+
+	rc = 0;
+
+cleanup:
+	if (link)
+		bpf_link__destroy(link);
+	if (obj)
+		bpf_object__close(obj);
+	return rc ? 1 : 0;
+}
+
diff --git a/ebpf/tcp_retrans/Makefile b/ebpf/tcp_retrans/Makefile
new file mode 100644
index 0000000..2635210
--- /dev/null
+++ b/ebpf/tcp_retrans/Makefile
@@ -0,0 +1,39 @@
+# tcp_retrans.bpf.o is the kernel-side object; the loader links it at runtime.
+# CI compiles only the BPF object (clang), since loading needs root + a Linux host.
+
+CLANG   ?= clang
+CC      ?= cc
+CFLAGS  ?= -O2 -g -Wall -Wextra
+BPFTOOL ?= bpftool
+ARCH    := $(shell uname -m | sed 's/x86_64/x86/;s/aarch64/arm64/;s/ppc64le/powerpc/;s/mips.*/mips/;s/riscv64/riscv/;s/loongarch64/loongarch/')
+
+BPF_CFLAGS := -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) -I. -Wall
+
+.PHONY: all build clean check help
+
+all: build
+
+help:
+	@echo "Targets:"
+	@echo "  build  - compile tcp_retrans.bpf.c and the userspace loader"
+	@echo "  check  - compile the BPF object only (used by CI)"
+	@echo "  clean  - remove generated objects and binaries"
+
+# Generate vmlinux.h from the running kernel if it is missing.
+vmlinux.h:
+	$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format header > $@
+
+tcp_retrans.bpf.o: tcp_retrans.bpf.c vmlinux.h
+	$(CLANG) $(BPF_CFLAGS) -c $< -o $@
+
+tcp_retrans: tcp_retrans.c tcp_retrans.bpf.o
+	$(CC) $(CFLAGS) -o $@ $< -lbpf -lelf -lz
+
+build: tcp_retrans.bpf.o tcp_retrans
+
+# CI check: object only, no loader link (libbpf may be absent on the runner
+# and loading eBPF requires a Linux kernel with BTF enabled).
+check: tcp_retrans.bpf.o
+
+clean:
+	rm -f tcp_retrans.bpf.o tcp_retrans vmlinux.h
diff --git a/ebpf/tcp_retrans/tcp_retrans.bpf.c b/ebpf/tcp_retrans/tcp_retrans.bpf.c
new file mode 100644
index 0000000..54507d3
--- /dev/null
+++ b/ebpf/tcp_retrans/tcp_retrans.bpf.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* tcp_retrans.bpf.c - Trace TCP retransmissions per destination address.
+ *
+ * CO-RE (Compile Once - Run Everywhere) BPF program attached to the
+ * tcp_retransmit_skb kernel tracepoint. Counts retransmits keyed by the
+ * remote (destination) address of the socket, so userspace can report which
+ * peers are suffering packet loss.
+ *
+ * Build with: make (uses bpftool to produce vmlinux.h if absent).
+ */
+#include <vmlinux.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+/* Key: remote address (IPv4-mapped value or full IPv6). Value: retransmit count. */
+struct dest_key {
+	__u32 family; /* AF_INET or AF_INET6 */
+	__u32 pad;
+	union {
+		__u32 addr4;
+		__u8 addr6[16];
+	};
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 10240);
+	__type(key, struct dest_key);
+	__type(value, __u64);
+} retrans_map SEC(".maps");
+
+/* Per-CPU grand total so userspace can cross-check per-destination sums. */
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, __u64);
+} totals_map SEC(".maps");
+
+SEC("tracepoint/tcp/tcp_retransmit_skb")
+int trace_tcp_retransmit_skb(struct trace_event_raw_tcp_retransmit_skb *ctx)
+{
+	struct sock *sk = BPF_CORE_READ(ctx, sk);
+	struct dest_key key = {};
+	__u64 *val, init = 1, *cnt;
+	__u32 slot = 0;
+	__u16 family;
+
+	if (!sk)
+		return 0;
+
+	family = BPF_CORE_READ(sk, __sk_common.skc_family);
+	if (family == AF_INET) {
+		key.family = AF_INET;
+		key.addr4 = BPF_CORE_READ(sk, __sk_common.skc_daddr);
+	} else if (family == AF_INET6) {
+		key.family = AF_INET6;
+		BPF_CORE_READ_INTO(&key.addr6, sk,
+				   __sk_common.skc_v6_daddr.in6_u.u6_addr8);
+	} else {
+		return 0;
+	}
+
+	val = bpf_map_lookup_elem(&retrans_map, &key);
+	if (val) {
+		__sync_fetch_and_add(val, 1);
+	} else {
+		bpf_map_update_elem(&retrans_map, &key, &init, BPF_NOEXIST);
+	}
+
+	cnt = bpf_map_lookup_elem(&totals_map, &slot);
+	if (cnt)
+		__sync_fetch_and_add(cnt, 1);
+
+	return 0;
+}
diff --git a/ebpf/tcp_retrans/tcp_retrans.c b/ebpf/tcp_retrans/tcp_retrans.c
new file mode 100644
index 0000000..cd18ed4
--- /dev/null
+++ b/ebpf/tcp_retrans/tcp_retrans.c
@@ -0,0 +1,348 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* tcp_retrans.c - Userspace loader for the tcp_retrans BPF program.
+ *
+ * Loads tcp_retrans.bpf.o via libbpf, polls the per-destination retransmit
+ * map on an interval, and prints a ranked list of destinations ordered by
+ * retransmit count. Emits JSON with --json for machine consumption.
+ *
+ * Usage: sudo ./tcp_retrans [-i seconds] [-n top] [--json] [--object PATH]
+ */
+#include <arpa/inet.h>
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <errno.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifndef AF_INET
+#define AF_INET 2
+#endif
+#ifndef AF_INET6
+#define AF_INET6 10
+#endif
+
+struct dest_key {
+	__u32 family;
+	__u32 pad;
+	union {
+		__u32 addr4;
+		__u8 addr6[16];
+	};
+};
+
+struct dest_entry {
+	struct dest_key key;
+	__u64 count;
+};
+
+static volatile sig_atomic_t stop;
+
+static void on_signal(int signo)
+{
+	(void)signo;
+	stop = 1;
+}
+
+static void usage(const char *prog)
+{
+	fprintf(stderr,
+		"Usage: %s [options]\n"
+		"\n"
+		"Trace TCP retransmissions per destination address.\n"
+		"\n"
+		"Options:\n"
+		"  -i, --interval SECS   Poll interval in seconds (default: 2)\n"
+		"  -n, --top COUNT       Show only the top COUNT destinations (default: 10)\n"
+		"      --json            Emit one JSON object per poll instead of a table\n"
+		"      --object PATH     Path to the BPF object file\n"
+		"                        (default: tcp_retrans.bpf.o next to this binary)\n"
+		"  -h, --help            Show this help and exit\n",
+		prog);
+}
+
+static int parse_interval(const char *value, long *out)
+{
+	char *end = NULL;
+	long v;
+
+	errno = 0;
+	v = strtol(value, &end, 10);
+	if (errno != 0 || end == value || *end != '\0' || v <= 0) {
+		fprintf(stderr, "error: invalid interval '%s'\n", value);
+		return -1;
+	}
+	*out = v;
+	return 0;
+}
+
+static const char *default_object_path(char *buf, size_t len, const char *argv0)
+{
+	const char *slash;
+	const char *dir;
+	size_t dirlen;
+
+	if (!argv0)
+		return NULL;
+	slash = strrchr(argv0, '/');
+	dir = slash ? argv0 : ".";
+	dirlen = slash ? (size_t)(slash - argv0) + 1 : 2;
+	if (dirlen + strlen("tcp_retrans.bpf.o") + 1 > len)
+		return NULL;
+	memcpy(buf, dir, dirlen);
+	buf[dirlen] = '\0';
+	strncat(buf, "tcp_retrans.bpf.o", len - dirlen - 1);
+	return buf;
+}
+
+static void format_addr(const struct dest_key *key, char *out, size_t len)
+{
+	if (key->family == AF_INET6) {
+		struct in6_addr a6;
+
+		memcpy(&a6, key->addr6, sizeof(a6));
+		if (!inet_ntop(AF_INET6, &a6, out, (socklen_t)len))
+			snprintf(out, len, "<invalid-ipv6>");
+		return;
+	}
+	if (!inet_ntop(AF_INET, &key->addr4, out, (socklen_t)len))
+		snprintf(out, len, "<invalid-ipv4>");
+}
+
+/* Comparator sorts descending by retransmit count. */
+static int cmp_entries(const void *a, const void *b)
+{
+	const struct dest_entry *ea = a;
+	const struct dest_entry *eb = b;
+
+	if (ea->count < eb->count)
+		return 1;
+	if (ea->count > eb->count)
+		return -1;
+	return 0;
+}
+
+static int collect(int map_fd, struct dest_entry **entries, size_t *count)
+{
+	struct dest_key key, next;
+	struct dest_entry *out = NULL;
+	size_t n = 0, cap = 0;
+	__u64 value;
+	int rc;
+
+	rc = bpf_map_get_next_key(map_fd, NULL, &key);
+	while (rc == 0) {
+		if (bpf_map_lookup_elem(map_fd, &key, &value) == 0) {
+			if (n == cap) {
+				size_t newcap = cap ? cap * 2 : 64;
+				struct dest_entry *tmp =
+					realloc(out, newcap * sizeof(*out));
+
+				if (!tmp) {
+					free(out);
+					return -ENOMEM;
+				}
+				out = tmp;
+				cap = newcap;
+			}
+			out[n].key = key;
+			out[n].count = value;
+			n++;
+		}
+		/* Snapshot the current key before fetching the next one: the
+		 * kernel may remove the entry we just looked up. */
+		memcpy(&next, &key, sizeof(next));
+		rc = bpf_map_get_next_key(map_fd, &next, &key);
+	}
+
+	*entries = out;
+	*count = n;
+	return 0;
+}
+
+static void print_table(struct dest_entry *entries, size_t n, size_t top,
+			unsigned long long total)
+{
+	size_t i;
+
+	printf("%-8s %-40s %s\n", "COUNT", "DESTINATION", "SHARE");
+	for (i = 0; i < n && i < top; i++) {
+		char addr[INET6_ADDRSTRLEN] = { 0 };
+		double share = total ?
+			(100.0 * (double)entries[i].count / (double)total) : 0.0;
+
+		format_addr(&entries[i].key, addr, sizeof(addr));
+		printf("%-8" PRIu64 " %-40s %6.2f%%\n",
+		       entries[i].count, addr, share);
+	}
+	printf("%-8llu %-40s\n", total, "TOTAL");
+	fflush(stdout);
+}
+
+static void print_json(struct dest_entry *entries, size_t n, size_t top,
+		       unsigned long long total, unsigned int interval)
+{
+	size_t i;
+
+	printf("{\"interval_s\":%u,\"total\":%llu,\"destinations\":[",
+	       interval, total);
+	for (i = 0; i < n && i < top; i++) {
+		char addr[INET6_ADDRSTRLEN] = { 0 };
+
+		format_addr(&entries[i].key, addr, sizeof(addr));
+		printf("%s{\"addr\":\"%s\",\"count\":%" PRIu64 "}",
+		       i ? "," : "", addr, entries[i].count);
+	}
+	printf("]}\n");
+	fflush(stdout);
+}
+
+int main(int argc, char **argv)
+{
+	struct bpf_object *obj = NULL;
+	struct bpf_program *prog;
+	struct bpf_link *link = NULL;
+	struct bpf_map *map;
+	char obj_path[4096] = { 0 };
+	const char *object = NULL;
+	long interval = 2;
+	size_t top = 10;
+	bool json = false;
+	int map_fd, rc, opt;
+	unsigned long long total;
+
+	static const struct option long_opts[] = {
+		{ "interval", required_argument, NULL, 'i' },
+		{ "top", required_argument, NULL, 'n' },
+		{ "json", no_argument, NULL, 1 },
+		{ "object", required_argument, NULL, 2 },
+		{ "help", no_argument, NULL, 'h' },
+		{ NULL, 0, NULL, 0 },
+	};
+
+	while ((opt = getopt_long(argc, argv, "i:n:h", long_opts, NULL)) != -1) {
+		switch (opt) {
+		case 'i':
+			if (parse_interval(optarg, &interval))
+				return 1;
+			break;
+		case 'n': {
+			char *end = NULL;
+
+			top = strtoul(optarg, &end, 10);
+			if (end == optarg || *end != '\0' || top == 0) {
+				fprintf(stderr, "error: invalid --top '%s'\n", optarg);
+				return 1;
+			}
+			break;
+		}
+		case 1:
+			json = true;
+			break;
+		case 2:
+			object = optarg;
+			break;
+		case 'h':
+			usage(argv[0]);
+			return 0;
+		default:
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (!object) {
+		if (!default_object_path(obj_path, sizeof(obj_path), argv[0])) {
+			fprintf(stderr, "error: object path too long\n");
+			return 1;
+		}
+		object = obj_path;
+	}
+
+	if (geteuid() != 0)
+		fprintf(stderr, "warning: not running as root; BPF load will likely fail\n");
+
+	signal(SIGINT, on_signal);
+	signal(SIGTERM, on_signal);
+
+	obj = bpf_object__open_file(object, NULL);
+	rc = libbpf_get_error(obj);
+	if (rc) {
+		obj = NULL;
+		fprintf(stderr, "error: failed to open BPF object '%s': %s\n",
+			object, strerror(-rc));
+		goto cleanup;
+	}
+
+	rc = bpf_object__load(obj);
+	if (rc) {
+		fprintf(stderr, "error: failed to load BPF object: %s\n", strerror(-rc));
+		goto cleanup;
+	}
+
+	map = bpf_object__find_map_by_name(obj, "retrans_map");
+	if (!map) {
+		fprintf(stderr, "error: retrans_map not found in BPF object\n");
+		goto cleanup;
+	}
+	map_fd = bpf_map__fd(map);
+
+	prog = bpf_object__next_program(obj, NULL);
+	for (; prog; prog = bpf_object__next_program(obj, prog)) {
+		link = bpf_program__attach(prog);
+		rc = libbpf_get_error(link);
+		if (rc) {
+			link = NULL;
+			fprintf(stderr, "error: failed to attach program: %s\n",
+				strerror(-rc));
+			goto cleanup;
+		}
+	}
+
+	if (!json)
+		printf("%-8s %-40s %s\n", "COUNT", "DESTINATION", "SHARE");
+
+	while (!stop) {
+		struct dest_entry *entries = NULL;
+		size_t n = 0;
+
+		sleep((unsigned int)interval);
+		if (stop)
+			break;
+
+		rc = collect(map_fd, &entries, &n);
+		if (rc) {
+			fprintf(stderr, "error: collecting map entries: %s\n",
+				strerror(-rc));
+			free(entries);
+			break;
+		}
+
+		total = 0;
+		for (size_t i = 0; i < n; i++)
+			total += entries[i].count;
+
+		qsort(entries, n, sizeof(*entries), cmp_entries);
+		if (json)
+			print_json(entries, n, top, total, (unsigned int)interval);
+		else
+			print_table(entries, n, top, total);
+
+		free(entries);
+	}
+
+	rc = 0;
+
+cleanup:
+	if (link)
+		bpf_link__destroy(link);
+	if (obj)
+		bpf_object__close(obj);
+	return rc ? 1 : 0;
+}
+
diff --git a/systemd/tuning.service b/systemd/tuning.service
new file mode 100644
index 0000000..f726022
--- /dev/null
+++ b/systemd/tuning.service
@@ -0,0 +1,52 @@
+# tuning.service - Apply a sysctl tuning profile at boot.
+#
+# Install (example):
+#   sudo install -m 0644 systemd/tuning.service /etc/systemd/system/tuning.service
+#   sudo systemctl daemon-reload
+#   sudo systemctl enable --now tuning.service
+#
+# The unit runs tuning/apply-profile.sh against the selected profile. Adjust
+# ExecStart's --profile path to switch between network-latency and
+# high-throughput. apply-profile.sh is idempotent: it re-records the current
+# values as a backup on each boot before applying.
+#
+# To pick a profile without editing this file, override the environment via a
+# drop-in:
+#   sudo systemctl edit tuning.service
+#   [Service]
+#   Environment=TUNING_PROFILE=/opt/linux-kernel-optimization/tuning/high-throughput.conf
+
+[Unit]
+Description=Apply sysctl kernel tuning profile
+Documentation=file:///usr/share/doc/linux-kernel-optimization/docs/METHODOLOGY.md
+DefaultDependencies=no
+After=sysinit.target local-fs.target
+Before=network-pre.target shutdown.target
+Wants=network-pre.target
+ConditionPathExists=/proc/sys
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+
+# Path to the checkout. Override with a drop-in (see header) for other locations.
+Environment=TUNING_PROFILE=/opt/linux-kernel-optimization/tuning/network-latency.conf
+Environment=TUNING_SCRIPT=/opt/linux-kernel-optimization/tuning/apply-profile.sh
+
+# Fail the unit if the profile cannot be applied; the kernel keeps its previous
+# values, which is the safe outcome.
+ExecStart=/usr/bin/env bash -c 'exec "${TUNING_SCRIPT}" --profile "${TUNING_PROFILE}"'
+
+# The script must run as root to write to /proc/sys.
+User=root
+Group=root
+
+# Modest hardening: the unit only needs to write sysctls and read the profile.
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/proc/sys /var/backups/sysctl-profiles
+PrivateTmp=yes
+
+[Install]
+WantedBy=multi-user.target
diff --git a/tuning/apply-profile.sh b/tuning/apply-profile.sh
new file mode 100644
index 0000000..960ecd4
--- /dev/null
+++ b/tuning/apply-profile.sh
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+#
+# apply-profile.sh - Apply a commented sysctl profile, with backup and revert.
+#
+# Reads a "key = value" profile file (see ./network-latency.conf and
+# ./high-throughput.conf), records the current value of every key it is about to
+# change, and applies the new values with `sysctl`. The backup is written as a
+# sysctl-compatible file so that --revert can restore the exact previous state.
+#
+# Usage:
+#   sudo ./apply-profile.sh --profile network-latency.conf
+#   sudo ./apply-profile.sh --revert
+#   ./apply-profile.sh --help
+#
+# Exit codes: 0 success, 1 usage/error, 2 failed to apply one or more keys.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEFAULT_PROFILE="${SCRIPT_DIR}/network-latency.conf"
+DEFAULT_BACKUP_DIR="/var/backups/sysctl-profiles"
+
+PROFILE=""
+BACKUP_DIR="${DEFAULT_BACKUP_DIR}"
+REVERT=0
+DRY_RUN=0
+
+usage() {
+	cat <<EOF
+Usage: $(basename "${BASH_SOURCE[0]}") [options]
+
+Apply a sysctl tuning profile to the running kernel, backing up prior values so
+the change can be undone.
+
+Options:
+  -p, --profile PATH    Profile file to apply (default: ${DEFAULT_PROFILE})
+  -b, --backup-dir DIR  Where to store the backup of previous values
+                        (default: ${DEFAULT_BACKUP_DIR})
+  -r, --revert          Restore the most recent backup instead of applying
+  -n, --dry-run         Print the commands that would run, change nothing
+  -h, --help            Show this help and exit
+
+Examples:
+  sudo $(basename "${BASH_SOURCE[0]}") --profile network-latency.conf
+  sudo $(basename "${BASH_SOURCE[0]}") --revert
+EOF
+}
+
+log() { printf '%s\n' "$*" >&2; }
+die() { log "error: $*"; exit 1; }
+
+require_root() {
+	if [[ "${EUID}" -ne 0 ]]; then
+		die "this script must run as root (try sudo)"
+	fi
+	if ! command -v sysctl >/dev/null 2>&1; then
+		die "'sysctl' not found in PATH"
+	fi
+}
+
+# latest_backup <dir> -> prints path of the newest backup file, or fails.
+latest_backup() {
+	local dir="$1" newest=""
+	[[ -d "${dir}" ]] || die "backup directory '${dir}' does not exist"
+
+	newest="$(find "${dir}" -maxdepth 1 -name 'sysctl-backup-*.conf' -type f \
+		-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2-)"
+	[[ -n "${newest}" ]] || die "no backups found in '${dir}'"
+	printf '%s\n' "${newest}"
+}
+
+# read_current <key> -> prints the current value, or nothing if unknown.
+read_current() {
+	local key="$1"
+	sysctl -n -- "${key}" 2>/dev/null || true
+}
+
+# normalize_line <raw> -> echo "key=value" with comments/space stripped, or "".
+normalize_line() {
+	local line="$1"
+	line="${line%%#*}"
+	line="$(printf '%s' "${line}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+	if [[ -z "${line}" || "${line}" != *=* ]]; then
+		printf ''
+		return 0
+	fi
+	printf '%s' "${line}"
+}
+
+# split_key <line> / split_value <line>
+split_key()   { printf '%s' "${1%%=*}" | tr -d '[:space:]'; }
+split_value() { printf '%s' "${1#*=}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; }
+
+# capture_backup <profile> <backup_file> -> write current values for profile keys.
+capture_backup() {
+	local profile="$1" backup_file="$2" line key cur
+
+	[[ -r "${profile}" ]] || die "cannot read profile '${profile}'"
+	{
+		printf '# sysctl backup written by apply-profile.sh\n'
+		printf '# source profile: %s\n' "${profile}"
+		printf '# timestamp: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+		while IFS= read -r line || [[ -n "${line}" ]]; do
+			line="$(normalize_line "${line}")"
+			[[ -z "${line}" ]] && continue
+			key="$(split_key "${line}")"
+			cur="$(read_current "${key}")"
+			if [[ -n "${cur}" ]]; then
+				printf '%s = %s\n' "${key}" "${cur}"
+			else
+				printf '# %s was unset before apply\n' "${key}"
+			fi
+		done < "${profile}"
+	} > "${backup_file}"
+	log "saved previous values to ${backup_file}"
+}
+
+# apply_file <file> -> apply each key=value line. Returns 2 on any failure.
+apply_file() {
+	local file="$1" line key value applied=0 failed=0
+
+	while IFS= read -r line || [[ -n "${line}" ]]; do
+		line="$(normalize_line "${line}")"
+		[[ -z "${line}" ]] && continue
+		key="$(split_key "${line}")"
+		value="$(split_value "${line}")"
+
+		if [[ "${DRY_RUN}" -eq 1 ]]; then
+			log "would run: sysctl -w ${key}=${value}"
+			continue
+		fi
+		if sysctl -w "${key}=${value}" >/dev/null 2>&1; then
+			applied=$((applied + 1))
+			log "applied ${key} = ${value}"
+		else
+			failed=$((failed + 1))
+			log "warning: failed to apply ${key} = ${value}"
+		fi
+	done < "${file}"
+
+	if [[ "${DRY_RUN}" -eq 1 ]]; then
+		return 0
+	fi
+	log "done: ${applied} applied, ${failed} failed"
+	[[ "${failed}" -eq 0 ]] || return 2
+	return 0
+}
+
+# apply_profile <profile> <backup_file>
+apply_profile() {
+	local profile="$1" backup_file="$2"
+
+	mkdir -p -- "${BACKUP_DIR}"
+	if [[ "${DRY_RUN}" -eq 0 ]]; then
+		capture_backup "${profile}" "${backup_file}"
+	fi
+	apply_file "${profile}"
+}
+
+# revert_profile <backup_file>
+revert_profile() {
+	local backup_file="$1"
+
+	[[ -r "${backup_file}" ]] || die "cannot read backup '${backup_file}'"
+	log "restoring sysctl values from ${backup_file}"
+	apply_file "${backup_file}" || return 2
+
+	if [[ "${DRY_RUN}" -eq 0 ]]; then
+		mv -- "${backup_file}" "${backup_file}.applied" 2>/dev/null || true
+	fi
+	return 0
+}
+
+main() {
+	while [[ $# -gt 0 ]]; do
+		case "$1" in
+			-p|--profile) PROFILE="${2:-}"; shift 2 ;;
+			-b|--backup-dir) BACKUP_DIR="${2:-}"; shift 2 ;;
+			-r|--revert) REVERT=1; shift ;;
+			-n|--dry-run) DRY_RUN=1; shift ;;
+			-h|--help) usage; exit 0 ;;
+			*) die "unknown argument '$1' (try --help)" ;;
+		esac
+	done
+
+	require_root
+
+	if [[ "${REVERT}" -eq 1 ]]; then
+		local backup_file
+		backup_file="$(latest_backup "${BACKUP_DIR}")"
+		revert_profile "${backup_file}" || exit 2
+		return 0
+	fi
+
+	[[ -n "${PROFILE}" ]] || PROFILE="${DEFAULT_PROFILE}"
+	local stamp backup_file
+	stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+	backup_file="${BACKUP_DIR}/sysctl-backup-${stamp}.conf"
+
+	apply_profile "${PROFILE}" "${backup_file}" || exit 2
+}
+
+main "$@"
+
diff --git a/tuning/high-throughput.conf b/tuning/high-throughput.conf
new file mode 100644
index 0000000..9bf1f13
--- /dev/null
+++ b/tuning/high-throughput.conf
@@ -0,0 +1,74 @@
+# high-throughput.conf
+# sysctl profile tuned for THROUGHPUT-oriented workloads (bulk transfer, file
+# servers, batch pipelines, backup/restore). Every line states why we set it and
+# what it costs.
+#
+# Apply with:  sudo ./tuning/apply-profile.sh --profile tuning/high-throughput.conf
+# Revert with: sudo ./tuning/apply-profile.sh --revert
+#
+# Format: one "key = value" per line. Lines starting with '#' are comments and
+# are ignored by apply-profile.sh.
+
+# --- Congestion control -----------------------------------------------------
+# BBR sustains high throughput on lossy links and long-fat networks without
+# collapsing on a single loss, and paces via fq. Tradeoff: may not coexist
+# perfectly fairly with CUBIC flows on shared bottlenecks.
+net.ipv4.tcp_congestion_control = bbr
+net.core.default_qdisc = fq
+
+# --- Socket buffers (large, for high bandwidth-delay product) ---------------
+# Big autotuning ceilings let a single flow fill a fat pipe. Tradeoff: more
+# memory pinned per connection and a deeper queue -> higher latency for other
+# flows sharing the link.
+net.core.rmem_max = 67108864
+net.core.wmem_max = 67108864
+net.core.rmem_default = 1048576
+net.core.wmem_default = 1048576
+net.ipv4.tcp_rmem = 4096 1048576 67108864
+net.ipv4.tcp_wmem = 4096 1048576 67108864
+
+# A deeper device backlog absorbs bursts instead of dropping them. Tradeoff:
+# adds queuing delay under sustained overload.
+net.core.netdev_max_backlog = 16384
+
+# Larger SYN backlog tolerates connect spikes without drops. Tradeoff: slightly
+# more kernel memory reserved for half-open connections.
+net.ipv4.tcp_max_syn_backlog = 8192
+
+# --- Connection handling ----------------------------------------------------
+# Recycle and expire TIME_WAIT sockets quickly to keep the port space free on
+# busy servers. Tradeoff: relaxed TIME_WAIT semantics; acceptable for most
+# stateless bulk workloads.
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.tcp_fin_timeout = 15
+
+# Wider ephemeral port range for many concurrent outbound streams.
+net.ipv4.ip_local_port_range = 10240 65535
+
+# Grow the SYN/receive queues fast and disable slow start after idle so long
+# idle-then-active transfers ramp immediately. Tradeoff: more aggressive bursts.
+net.ipv4.tcp_slow_start_after_idle = 0
+net.ipv4.tcp_auto_tuning = 1
+
+# --- Page cache / dirty ratios (throughput-oriented) ------------------------
+# Let the kernel buffer more dirty data before forcing writeback so it can
+# coalesce writes into large sequential I/O. Tradeoff: a stall or crash can
+# lose more dirty pages, and write latency spikes when writeback does trigger.
+vm.dirty_ratio = 30
+vm.dirty_background_ratio = 15
+
+# Allow dirty data to accumulate longer before periodic writeback, producing
+# larger and more efficient write bursts. Tradeoff: same data-loss window as
+# above; avoid on hosts that cannot afford to lose minutes of writes.
+vm.dirty_expire_centisecs = 3000
+vm.dirty_writeback_centisecs = 3000
+
+# Increase the number of in-flight writeback pages and file handles to keep
+# storage queues full. Tradeoff: more memory used by writeback.
+vm.dirty_background_bytes = 0
+vm.min_free_kbytes = 262144
+
+# --- Scheduler --------------------------------------------------------------
+# Disable scheduler autogrouping so interactive tasks do not preempt bulk jobs
+# as aggressively. Tradeoff: less interactive responsiveness under load.
+kernel.sched_autogroup_enabled = 0
diff --git a/tuning/network-latency.conf b/tuning/network-latency.conf
new file mode 100644
index 0000000..a37f1e2
--- /dev/null
+++ b/tuning/network-latency.conf
@@ -0,0 +1,80 @@
+# network-latency.conf
+# sysctl profile tuned for LATENCY-SENSITIVE workloads (RPC, databases, trading,
+# interactive services). Every line states why we set it and what it costs.
+#
+# Apply with:  sudo ./tuning/apply-profile.sh --profile tuning/network-latency.conf
+# Revert with: sudo ./tuning/apply-profile.sh --revert
+#
+# Format: one "key = value" per line. Lines starting with '#' are comments and
+# are ignored by apply-profile.sh.
+
+# --- Congestion control -----------------------------------------------------
+# BBR models bandwidth and RTT instead of reacting to loss, which keeps queues
+# short and cuts tail latency on lossy or long-fat paths. Tradeoff: BBR can be
+# unfair to loss-based CUBIC flows sharing the same bottleneck.
+net.ipv4.tcp_congestion_control = bbr
+
+# FQ is the packet scheduler BBR is designed to pair with; it paces outgoing
+# packets and limits per-flow queueing. Tradeoff: pacing adds a small CPU cost.
+net.core.default_qdisc = fq
+
+# --- Buffers and queues -----------------------------------------------------
+# Small receive/send socket buffers keep queuing delay low (little's law: less
+# buffered data = lower standing delay). Tradeoff: lower single-flow throughput
+# on high-BDP links; revisit for bulk transfer hosts.
+net.core.rmem_default = 262144
+net.core.wmem_default = 262144
+
+# Min/default/max autotuning bounds for TCP buffers. The max is kept moderate so
+# a single flow cannot inflate bufferbloat. Tradeoff: caps throughput on very
+# high bandwidth-delay-product paths.
+net.ipv4.tcp_rmem = 4096 262144 4194304
+net.ipv4.tcp_wmem = 4096 262144 4194304
+
+# Cap the device backlog queue so bursts are dropped early (and retransmitted)
+# rather than being buffered into latency. Tradeoff: some packet drops under
+# microbursts; acceptable because drops here are cheap.
+net.core.netdev_max_backlog = 1000
+
+# --- Connection handling ----------------------------------------------------
+# Recycle TIME_WAIT sockets for client-side connections when safe. Tradeoff:
+# slightly weaker TIME_WAIT protection, which is fine for short-lived RPCs.
+net.ipv4.tcp_tw_reuse = 1
+
+# Keep-alive probes so dead peers are detected instead of holding sockets.
+net.ipv4.tcp_keepalive_time = 300
+net.ipv4.tcp_keepalive_intvl = 30
+net.ipv4.tcp_keepalive_probes = 5
+
+# Do not let a connection stall on a full accept backlog: fail fast instead.
+net.ipv4.tcp_abort_on_overflow = 1
+
+# --- Local port range -------------------------------------------------------
+# Wider ephemeral port range reduces connect() port-exhaustion tail latency on
+# busy client hosts. Tradeoff: none of note; just more ports in use.
+net.ipv4.ip_local_port_range = 10240 65535
+
+# --- Page cache / dirty ratios (latency-oriented) ---------------------------
+# Keep dirty data bounded so writeback does not create long I/O stalls.
+# Lower dirty_ratio/dirty_background_ratio means the kernel starts flushing
+# earlier and more often. Tradeoff: more frequent smaller writes -> lower
+# sequential write throughput on cache-heavy workloads.
+vm.dirty_ratio = 10
+vm.dirty_background_ratio = 5
+
+# Bound how long dirty data may sit before periodic writeback kicks in. Shorter
+# intervals smooth out I/O bursts. Tradeoff: more wakeups / background I/O.
+vm.dirty_expire_centisecs = 1500
+vm.dirty_writeback_centisecs = 500
+
+# Keep some free memory headroom so reclaim does not spike latency. Tradeoff:
+# less memory available to the page cache, which can hurt read caching.
+vm.min_free_kbytes = 131072
+
+# --- NUMA / scheduler -------------------------------------------------------
+# Enable automatic NUMA balancing: cheaper remote-memory access for long-running
+# services. Tradeoff: some sampling overhead.
+kernel.numa_balancing = 1
+
+# Mildly favor throughput in the scheduler while still waking up promptly.
+kernel.sched_autogroup_enabled = 1

From 54d462ba1188738137680e5d37b3a52e9b0983b3 Mon Sep 17 00:00:00 2001
From: sukonoe36 <337314619+sukonoe36@users.noreply.github.com>
Date: Sat, 3 Oct 2026 14:00:18 +0000
Subject: [PATCH 2/2] fix(ci): make the eBPF compile check and shellcheck pass

The eBPF programs referenced trace_event_raw_* context types that are not part of BTF, so they could never compile against vmlinux.h; both tracers now use kprobes with plain struct pointers. CI also failed because Ubuntu no longer ships a standalone bpftool package: it now installs the kernel linux-tools packages and symlinks the real bpftool binary, and shellcheck runs at severity=warning so advisory SC1091 notes no longer fail the build.
---
 .github/workflows/ci.yml           |  44 +++++++++---
 README.md                          |  15 ++++
 bench/run-fio.sh                   |   2 +-
 bench/run-iperf3.sh                |   2 +-
 bench/run-perf-stat.sh             |   2 +-
 ebpf/biolatency/Makefile           |   6 ++
 ebpf/biolatency/biolatency.bpf.c   | 109 +++++++++++++++++------------
 ebpf/tcp_retrans/Makefile          |   5 ++
 ebpf/tcp_retrans/tcp_retrans.bpf.c |  34 +++++----
 9 files changed, 148 insertions(+), 71 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 72614c6..a65fcb0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,9 +9,9 @@ name: ci
 
 on:
   push:
-    branches: [main, 'feat/**']
+    branches: [main, 'feat/**', 'fix/**']
   pull_request:
-    branches: [main]
+    branches: [main, 'feat/**', 'fix/**']
   workflow_dispatch:
 
 permissions:
@@ -40,7 +40,10 @@ jobs:
           fi
           printf 'Linting %d script(s)\n' "${#scripts[@]}"
           shellcheck --version
-          shellcheck --severity=style --external-sources "${scripts[@]}"
+          # severity=warning: advisory style/info notes (for example SC1091 for
+          # dynamically computed source paths) must not fail the build, but real
+          # warnings and errors do.
+          shellcheck --severity=warning --external-sources "${scripts[@]}"
 
   ebpf-compile:
     name: eBPF compile check
@@ -51,19 +54,38 @@ jobs:
 
       - name: Install build toolchain
         run: |
+          set -euo pipefail
           sudo apt-get update
-          sudo apt-get install -y \
-            clang \
-            llvm \
-            libelf-dev \
-            libbpf-dev \
-            bpftool \
-            build-essential
+          sudo apt-get install -y clang llvm libelf-dev libbpf-dev build-essential
+
+          # Ubuntu ships no standalone 'bpftool' package. The binary comes from
+          # the per-kernel linux-tools packages, and the 'linux-tools-common'
+          # wrapper only works when the tools package matching the running kernel
+          # is installed - which is often not the case on hosted runners. Try the
+          # plausible sources, then locate the real binary and put it on PATH.
+          for pkg in \
+              "linux-tools-$(uname -r)" \
+              linux-tools-azure \
+              linux-tools-generic \
+              linux-tools-common; do
+            sudo apt-get install -y --no-install-recommends "${pkg}" \
+              || echo "note: ${pkg} is not installable on this runner"
+          done
+
+          if ! bpftool version >/dev/null 2>&1; then
+            real="$(find /usr/lib/linux-tools -type f -name bpftool 2>/dev/null | head -n1 || true)"
+            if [ -n "${real}" ]; then
+              echo "using ${real} as bpftool"
+              sudo ln -sf "${real}" /usr/local/bin/bpftool
+            fi
+          fi
+
+          command -v bpftool
+          bpftool version
 
       - name: Show toolchain versions
         run: |
           clang --version
-          bpftool version || true
 
       - name: Compile tcp_retrans BPF object
         working-directory: ebpf/tcp_retrans
diff --git a/README.md b/README.md
index 728208f..d0cf1dc 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,21 @@ docs/
 - `root` (or `CAP_BPF` + `CAP_PERFMON`) to load and attach BPF programs.
 - For benchmarks: `iperf3`, `fio`, `perf` (linux-tools), `sysstat` (optional).
 
+## Observability
+
+Two CO-RE tracers are provided. Both use kprobes rather than tracepoints: the
+tracepoint context types (`trace_event_raw_*`) are not part of BTF and so are
+absent from `vmlinux.h`.
+
+- `tcp_retrans` attaches to `tcp_retransmit_skb` and counts retransmissions per
+  destination address.
+- `biolatency` attaches to `blk_account_io_start`/`blk_account_io_done` and
+  builds a log2 histogram of block I/O latency in microseconds.
+
+If a kprobe cannot be attached on your kernel (the function may be absent or
+inlined), the loader reports the error from libbpf; run it as root and check
+`/proc/kallsyms` for the symbol.
+
 ## Quick start
 
 Build a tracer:
diff --git a/bench/run-fio.sh b/bench/run-fio.sh
index caee4d4..eaeac21 100644
--- a/bench/run-fio.sh
+++ b/bench/run-fio.sh
@@ -14,7 +14,7 @@
 
 set -euo pipefail
 
-# shellcheck source=common.sh
+# shellcheck source=bench/common.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
 
 TARGET=""
diff --git a/bench/run-iperf3.sh b/bench/run-iperf3.sh
index 96db9b5..22169cd 100644
--- a/bench/run-iperf3.sh
+++ b/bench/run-iperf3.sh
@@ -14,7 +14,7 @@
 
 set -euo pipefail
 
-# shellcheck source=common.sh
+# shellcheck source=bench/common.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
 
 SERVER=""
diff --git a/bench/run-perf-stat.sh b/bench/run-perf-stat.sh
index 55b78ac..e3006c3 100644
--- a/bench/run-perf-stat.sh
+++ b/bench/run-perf-stat.sh
@@ -15,7 +15,7 @@
 
 set -euo pipefail
 
-# shellcheck source=common.sh
+# shellcheck source=bench/common.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
 
 PHASE="before"
diff --git a/ebpf/biolatency/Makefile b/ebpf/biolatency/Makefile
index 5c60cbd..d866e88 100644
--- a/ebpf/biolatency/Makefile
+++ b/ebpf/biolatency/Makefile
@@ -19,7 +19,13 @@ help:
 	@echo "  check  - compile the BPF object only (used by CI)"
 	@echo "  clean  - remove generated objects and binaries"
 
+# Generate vmlinux.h from the running kernel if it is missing.
+# bpftool is required; on distributions without a standalone bpftool package it
+# comes from the kernel's linux-tools package.
 vmlinux.h:
+	@command -v $(BPFTOOL) >/dev/null 2>&1 || { \
+		echo "error: $(BPFTOOL) not found; install it (e.g. linux-tools-generic) or" >&2; \
+		echo "       provide a vmlinux.h in this directory" >&2; exit 1; }
 	$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format header > $@
 
 biolatency.bpf.o: biolatency.bpf.c vmlinux.h
diff --git a/ebpf/biolatency/biolatency.bpf.c b/ebpf/biolatency/biolatency.bpf.c
index 4256dfc..d1c684a 100644
--- a/ebpf/biolatency/biolatency.bpf.c
+++ b/ebpf/biolatency/biolatency.bpf.c
@@ -1,12 +1,18 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /* biolatency.bpf.c - Block I/O latency histogram (CO-RE).
  *
- * Traces block I/O issue and completion, computing per-request latency and
- * recording it into log2-bucketed histograms, one per CPU (aggregated in
- * userspace). Works across devices; the device major:minor can optionally be
- * filtered from userspace via the filter map.
+ * Traces the start and completion of block requests, computes the per-request
+ * latency and records it in log2-bucketed histograms, one per CPU (aggregated
+ * in userspace). All devices are traced unless a single major:minor is
+ * installed in the filter map from userspace.
  *
- * Build with: make (uses bpftool gen to produce vmlinux.h if absent).
+ * kprobes on blk_account_io_start/blk_account_io_done are used instead of the
+ * block/block_rq_issue and block/block_rq_complete tracepoints because the
+ * tracepoints' generated context types (trace_event_raw_*) are not part of BTF
+ * and therefore absent from vmlinux.h. The kprobe arguments are plain
+ * `struct request *` pointers, which CO-RE relocates safely.
+ *
+ * Build with: make (generates vmlinux.h with bpftool when it is missing).
  */
 #include <vmlinux.h>
 #include <bpf/bpf_core_read.h>
@@ -17,15 +23,15 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL";
 
 #define MAX_BUCKETS 32 /* log2 microseconds: 0..2^31 us */
 
-/* In-flight request tracking, keyed by request pointer. */
+/* In-flight requests, keyed by (u64)struct request *. Value: start time in ns. */
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH);
 	__uint(max_entries, 16384);
-	__type(key, __u64); /* (u64)struct request * */
-	__type(value, __u64); /* start timestamp (ns) */
+	__type(key, __u64);
+	__type(value, __u64);
 } starts SEC(".maps");
 
-/* Per-CPU latency histograms, bucket = log2(latency_us). */
+/* Per-CPU latency histograms; bucket index = floor(log2(latency_us)). */
 struct {
 	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
 	__uint(max_entries, MAX_BUCKETS);
@@ -33,43 +39,55 @@ struct {
 	__type(value, __u64);
 } lat_hist SEC(".maps");
 
-/* Optional device filter: if a key exists, only that major:minor is traced. */
+/* Optional device filter. Slot 0 is the "filter installed" marker; a key of
+ * (major << 16) | first_minor selects the single device to trace. */
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH);
-	__uint(max_entries, 1);
-	__type(key, __u32); /* (major << 8) | minor is not used; dev_t encoded */
+	__uint(max_entries, 1024);
+	__type(key, __u32);
 	__type(value, __u8);
 } filter_map SEC(".maps");
 
-/* Number of requests issued/completed, for sanity checks in userspace. */
+/* Counters for sanity checks in userspace: [0]=started [1]=completed. */
 struct {
 	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__uint(max_entries, 2); /* [0]=issued [1]=completed */
+	__uint(max_entries, 2);
 	__type(key, __u32);
 	__type(value, __u64);
 } counts SEC(".maps");
 
-static __always_inline __u32 dev_key(struct request *rq)
+/* filter_active reports whether userspace installed a device filter. */
+static __always_inline int filter_active(void)
+{
+	__u32 slot = 0;
+
+	return bpf_map_lookup_elem(&filter_map, &slot) != NULL;
+}
+
+/* device_key encodes major:minor the same way userspace does. */
+static __always_inline __u32 device_key(struct request *rq)
 {
 	struct gendisk *disk = BPF_CORE_READ(rq, q, disk);
-	__u32 major = BPF_CORE_READ(disk, major);
-	__u16 first_minor = BPF_CORE_READ(disk, first_minor);
+	__u32 major, minor;
 
-	return (major << 16) | first_minor;
+	if (!disk)
+		return 0;
+
+	major = BPF_CORE_READ(disk, major);
+	minor = BPF_CORE_READ(disk, first_minor);
+	return (major << 16) | (minor & 0xffff);
 }
 
+/* allowed returns 1 when the request should be traced. */
 static __always_inline int allowed(struct request *rq)
 {
 	__u32 key;
-	__u32 slot = 0;
-	__u8 *v;
 
-	if (bpf_map_lookup_elem(&filter_map, &slot) == NULL)
-		return 1; /* no filter installed -> trace everything */
+	if (!filter_active())
+		return 1;
 
-	key = dev_key(rq);
-	v = bpf_map_lookup_elem(&filter_map, &key);
-	return v != NULL;
+	key = device_key(rq);
+	return bpf_map_lookup_elem(&filter_map, &key) != NULL;
 }
 
 static __always_inline void count_inc(__u32 slot)
@@ -80,44 +98,47 @@ static __always_inline void count_inc(__u32 slot)
 		__sync_fetch_and_add(c, 1);
 }
 
-SEC("tracepoint/block/block_rq_issue")
-int trace_block_rq_issue(struct trace_event_raw_block_rq_issue *ctx)
+SEC("kprobe/blk_account_io_start")
+int BPF_KPROBE(trace_io_start, struct request *rq)
 {
-	__u64 ts = bpf_ktime_get_ns();
-	__u64 *p;
-	__u64 req = (__u64)ctx->rq;
+	__u64 key, ts, *p;
 
-	if (!allowed((struct request *)(unsigned long)req))
+	if (!rq || !allowed(rq))
 		return 0;
 
-	p = bpf_map_lookup_elem(&starts, &req);
+	ts = bpf_ktime_get_ns();
+	key = (__u64)rq;
+
+	p = bpf_map_lookup_elem(&starts, &key);
 	if (p)
 		*p = ts;
 	else
-		bpf_map_update_elem(&starts, &req, &ts, BPF_ANY);
+		bpf_map_update_elem(&starts, &key, &ts, BPF_ANY);
 
 	count_inc(0);
 	return 0;
 }
 
-SEC("tracepoint/block/block_rq_complete")
-int trace_block_rq_complete(struct trace_event_raw_block_rq_complete *ctx)
+SEC("kprobe/blk_account_io_done")
+int BPF_KPROBE(trace_io_done, struct request *rq)
 {
-	__u64 req = (__u64)ctx->rq;
-	__u64 *start, lat_ns, lat_us;
+	__u64 key, *start, lat_ns, lat_us, *h;
 	__u32 bucket, slot = 1;
-	__u64 *h;
 
-	start = bpf_map_lookup_elem(&starts, &req);
+	if (!rq)
+		return 0;
+
+	key = (__u64)rq;
+	start = bpf_map_lookup_elem(&starts, &key);
 	if (!start)
 		return 0;
 
 	lat_ns = bpf_ktime_get_ns() - *start;
-	bpf_map_delete_elem(&starts, &req);
-	lat_us = lat_ns / 1000;
+	bpf_map_delete_elem(&starts, &key);
 
-	/* Bucket by integer log2 of microseconds. */
-	bucket = lat_us ? (63 - __builtin_clzll(lat_us)) : 0;
+	lat_us = lat_ns / 1000;
+	/* Bucket by integer log2 of the microsecond latency. */
+	bucket = lat_us ? (__u32)(63 - __builtin_clzll(lat_us)) : 0;
 	if (bucket >= MAX_BUCKETS)
 		bucket = MAX_BUCKETS - 1;
 
@@ -128,5 +149,3 @@ int trace_block_rq_complete(struct trace_event_raw_block_rq_complete *ctx)
 	count_inc(slot);
 	return 0;
 }
-
-char _license[] SEC("license") = "GPL";
diff --git a/ebpf/tcp_retrans/Makefile b/ebpf/tcp_retrans/Makefile
index 2635210..df2b8fd 100644
--- a/ebpf/tcp_retrans/Makefile
+++ b/ebpf/tcp_retrans/Makefile
@@ -20,7 +20,12 @@ help:
 	@echo "  clean  - remove generated objects and binaries"
 
 # Generate vmlinux.h from the running kernel if it is missing.
+# bpftool is required; on distributions without a standalone bpftool package it
+# comes from the kernel's linux-tools package.
 vmlinux.h:
+	@command -v $(BPFTOOL) >/dev/null 2>&1 || { \
+		echo "error: $(BPFTOOL) not found; install it (e.g. linux-tools-generic) or" >&2; \
+		echo "       provide a vmlinux.h in this directory" >&2; exit 1; }
 	$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format header > $@
 
 tcp_retrans.bpf.o: tcp_retrans.bpf.c vmlinux.h
diff --git a/ebpf/tcp_retrans/tcp_retrans.bpf.c b/ebpf/tcp_retrans/tcp_retrans.bpf.c
index 54507d3..eda17bd 100644
--- a/ebpf/tcp_retrans/tcp_retrans.bpf.c
+++ b/ebpf/tcp_retrans/tcp_retrans.bpf.c
@@ -1,12 +1,17 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /* tcp_retrans.bpf.c - Trace TCP retransmissions per destination address.
  *
- * CO-RE (Compile Once - Run Everywhere) BPF program attached to the
- * tcp_retransmit_skb kernel tracepoint. Counts retransmits keyed by the
- * remote (destination) address of the socket, so userspace can report which
- * peers are suffering packet loss.
+ * CO-RE (Compile Once - Run Everywhere) BPF program attached to a kprobe on
+ * tcp_retransmit_skb. Counts retransmits keyed by the remote (destination)
+ * address of the socket, so userspace can report which peers are suffering
+ * packet loss.
  *
- * Build with: make (uses bpftool to produce vmlinux.h if absent).
+ * A kprobe is used instead of the tcp/tcp_retransmit_skb tracepoint because the
+ * tracepoint's generated context type (trace_event_raw_tcp_retransmit_skb) is
+ * not part of BTF and therefore absent from vmlinux.h, while the kprobe argument
+ * is a plain `struct sock *` that CO-RE can relocate safely across kernels.
+ *
+ * Build with: make (generates vmlinux.h with bpftool when it is missing).
  */
 #include <vmlinux.h>
 #include <bpf/bpf_core_read.h>
@@ -15,7 +20,14 @@
 
 char LICENSE[] SEC("license") = "Dual BSD/GPL";
 
-/* Key: remote address (IPv4-mapped value or full IPv6). Value: retransmit count. */
+#ifndef AF_INET
+#define AF_INET 2
+#endif
+#ifndef AF_INET6
+#define AF_INET6 10
+#endif
+
+/* Key: remote address (IPv4 value or full IPv6). Value: retransmit count. */
 struct dest_key {
 	__u32 family; /* AF_INET or AF_INET6 */
 	__u32 pad;
@@ -40,10 +52,9 @@ struct {
 	__type(value, __u64);
 } totals_map SEC(".maps");
 
-SEC("tracepoint/tcp/tcp_retransmit_skb")
-int trace_tcp_retransmit_skb(struct trace_event_raw_tcp_retransmit_skb *ctx)
+SEC("kprobe/tcp_retransmit_skb")
+int BPF_KPROBE(trace_tcp_retransmit_skb, struct sock *sk)
 {
-	struct sock *sk = BPF_CORE_READ(ctx, sk);
 	struct dest_key key = {};
 	__u64 *val, init = 1, *cnt;
 	__u32 slot = 0;
@@ -65,11 +76,10 @@ int trace_tcp_retransmit_skb(struct trace_event_raw_tcp_retransmit_skb *ctx)
 	}
 
 	val = bpf_map_lookup_elem(&retrans_map, &key);
-	if (val) {
+	if (val)
 		__sync_fetch_and_add(val, 1);
-	} else {
+	else
 		bpf_map_update_elem(&retrans_map, &key, &init, BPF_NOEXIST);
-	}
 
 	cnt = bpf_map_lookup_elem(&totals_map, &slot);
 	if (cnt)