From 71c7422afa51683d485d471867bc4fb14bd92968 Mon Sep 17 00:00:00 2001 From: MDA2AV Date: Fri, 24 Jul 2026 23:54:31 +0100 Subject: [PATCH] Record the cgroup memory breakdown on every result Groundwork for #1015. The issue argues the published memory figures include the Linux page cache and so overstate what applications use. Checking that first changed the picture, so this records the evidence rather than acting on the assumption. What the measurements show: `docker stats`, which the harness already uses, reports memory.current minus inactive_file - so it does subtract page cache. A container holding 512 MiB of cache reports 15 MiB. Re-reading those files continuously does not change it: the kernel keeps sequential reads on the inactive list, so nothing is promoted and nothing extra is counted. The sampler also polls only the framework's own container, so no sidecar or load generator is summed in. That leaves the gigabyte figures unexplained by cache, and two candidates unmeasured: memory the application really does allocate at 4096 concurrent connections, and kernel socket buffers, which cgroup v2 charges to the container and `docker stats` therefore includes. The comparison numbers in the issue also come from a local 8-core wrk run, far below the 512/4096 connections used on the bench box, so part of the gap is load rather than accounting. Rather than redefine "memory used" on a guess - a change that would move every published number and the composite's memory factor - this records what the number is made of: "memory_detail": {"current":264.4,"anon":120.2,"file":56.0,"active_file":1.2, "inactive_file":54.8,"sock":41.0,"slab":7.3,"kernel":7.5} MiB, read from the container's cgroup at the snapshot where peak memory was observed, so it describes the same instant the `memory` field reports rather than a max-per-field mixture of different moments. Nothing consumes it yet. `memory` is unchanged, memory_detail is not in the generator's BASE_FIELDS, and data.js regenerates byte-identical - so the leaderboard and every score are untouched. It rides along into site/data/results/.json because rebuild_site_data.py stores result rows verbatim, which is what makes it available to analyse later. Degrades quietly: cgroup layout varies with driver and version, so the path is probed and skipped if unreadable. Verified the emitted JSON parses both with the field and without it. --- scripts/benchmark-lite.sh | 13 ++++++- scripts/benchmark.sh | 14 ++++++- scripts/lib/stats.sh | 82 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark-lite.sh b/scripts/benchmark-lite.sh index 1e2709673..36c0183b1 100755 --- a/scripts/benchmark-lite.sh +++ b/scripts/benchmark-lite.sh @@ -294,6 +294,7 @@ run_one() { best_output="$output" best_cpu="$STATS_AVG_CPU" best_mem="$STATS_PEAK_MEM" + BEST_MEM_DETAIL="$STATS_MEM_DETAIL" BEST_M=() for k in "${!m[@]}"; do BEST_M[$k]="${m[$k]}"; done fi @@ -321,6 +322,16 @@ save_result() { # Composite-score support — api-4/16 + gateway-64 need per-template splits # or the website scores them as 0. See save_result in benchmark.sh. + # cgroup composition at peak memory, in MiB. Recorded for #1015: the + # `memory` field above is docker's number (memory.current minus + # inactive_file) and cannot show whether it is application memory, page + # cache the kernel kept active, or socket buffers. Nothing reads this yet. + local mem_detail_extra="" + if [ -n "${BEST_MEM_DETAIL:-}" ]; then + mem_detail_extra=", + \"memory_detail\": {${BEST_MEM_DETAIL}}" + fi + local tpl_extra="" if [ "$profile" = "api-4" ] || [ "$profile" = "api-16" ]; then tpl_extra=", @@ -359,7 +370,7 @@ save_result() { "status_2xx": ${BEST_M[status_2xx]:-0}, "status_3xx": ${BEST_M[status_3xx]:-0}, "status_4xx": ${BEST_M[status_4xx]:-0}, - "status_5xx": ${BEST_M[status_5xx]:-0}${tpl_extra} + "status_5xx": ${BEST_M[status_5xx]:-0}${mem_detail_extra}${tpl_extra} } EOF info "saved results/$profile/$CONNS/${FRAMEWORK}.json" diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 5334ac952..94e4f5489 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -281,6 +281,7 @@ run_one() { # even if its rps is 0 (ws-echo, zero-traffic regressions). Without this, # BEST_M would carry stale metrics from a previous profile. local best_rps=-1 best_output="" best_cpu="0%" best_mem="0MiB" best_breakdown="" + BEST_MEM_DETAIL="" BEST_M=() local run @@ -317,6 +318,7 @@ run_one() { best_cpu="$STATS_AVG_CPU" best_mem="$STATS_PEAK_MEM" best_breakdown="$STATS_BREAKDOWN" + BEST_MEM_DETAIL="$STATS_MEM_DETAIL" BEST_M=() for k in "${!m[@]}"; do BEST_M[$k]="${m[$k]}"; done fi @@ -398,6 +400,16 @@ save_result() { \"cpu_breakdown\": \"$best_breakdown\"" fi + # cgroup composition at peak memory, in MiB. Recorded for #1015: the + # `memory` field above is docker's number (memory.current minus + # inactive_file) and cannot show whether it is application memory, page + # cache the kernel kept active, or socket buffers. Nothing reads this yet. + local mem_detail_extra="" + if [ -n "${BEST_MEM_DETAIL:-}" ]; then + mem_detail_extra=", + \"memory_detail\": {${BEST_MEM_DETAIL}}" + fi + local tpl_extra="" if [ "$profile" = "api-4" ] || [ "$profile" = "api-16" ]; then tpl_extra=", @@ -449,7 +461,7 @@ save_result() { "status_2xx": ${BEST_M[status_2xx]:-0}, "status_3xx": ${BEST_M[status_3xx]:-0}, "status_4xx": ${BEST_M[status_4xx]:-0}, - "status_5xx": ${BEST_M[status_5xx]:-0}${tpl_extra}${cpu_extra} + "status_5xx": ${BEST_M[status_5xx]:-0}${mem_detail_extra}${tpl_extra}${cpu_extra} } EOF info "saved results/$profile/$CONNS/${FRAMEWORK}.json" diff --git a/scripts/lib/stats.sh b/scripts/lib/stats.sh index 8be9b7f0c..2b8adcd52 100644 --- a/scripts/lib/stats.sh +++ b/scripts/lib/stats.sh @@ -11,12 +11,43 @@ # stats_start # starts background collector # stats_stop # stops, fills STATS_AVG_CPU / STATS_PEAK_MEM # # and (multi-container) STATS_BREAKDOWN +# # and STATS_MEM_DETAIL +# +# `docker stats` reports memory.current minus inactive_file, so it already +# excludes most page cache - but that single number says nothing about what +# the memory actually is. #1015 asks whether the published figures are really +# the application's memory; answering it needs the composition, not another +# opinion. So alongside the existing number we record the cgroup's own +# accounting (anon, file, sock, slab, kernel) at the moment of peak usage and +# store it on the result row for later analysis. Nothing consumes it yet: the +# leaderboard still reads `memory`. STATS_PID="" STATS_LOG="" +STATS_CG_LOG="" STATS_AVG_CPU="0%" STATS_PEAK_MEM="0MiB" STATS_BREAKDOWN="" +STATS_MEM_DETAIL="" + +# Fields worth keeping from cgroup v2 memory.stat. anon is application memory, +# file is page cache (active_file is the part docker still counts), sock is +# kernel socket buffers - which at 4096 connections is not a rounding error - +# and slab/kernel are kernel structures charged to the container. +STATS_CG_FIELDS="anon file active_file inactive_file sock slab kernel" + +# Resolve a container's cgroup directory. Layout varies with the cgroup driver +# (systemd vs cgroupfs) and version, so try the known shapes and give up +# quietly - this is supplementary data, never a reason to fail a run. +_cg_dir() { + local id="$1" p + for p in "/sys/fs/cgroup/system.slice/docker-$id.scope" \ + "/sys/fs/cgroup/docker/$id" \ + "/sys/fs/cgroup/memory/docker/$id"; do + [ -r "$p/memory.stat" ] && { echo "$p"; return 0; } + done + return 1 +} # Start a background poller. Accepts one or more container names. Each # sample writes one line per container, tagged with a snapshot counter so @@ -26,7 +57,17 @@ STATS_BREAKDOWN="" # Log line format: stats_start() { STATS_LOG=$(mktemp) + STATS_CG_LOG=$(mktemp) local containers=("$@") + + # Resolve cgroup paths once; container ids don't change mid-run. + local _cg_paths=() _c _id _dir + for _c in "${containers[@]}"; do + _id=$(docker inspect -f '{{.Id}}' "$_c" 2>/dev/null) || continue + _dir=$(_cg_dir "$_id") || continue + _cg_paths+=("$_dir") + done + ( local snap=0 while true; do @@ -49,6 +90,16 @@ stats_start() { else if (unit == "B") mem_mib /= (1024 * 1024) printf "%s %s %.2f %.2f\n", snap, name, cpu, mem_mib }' + + # cgroup accounting for the same snapshot, summed across containers + for _d in "${_cg_paths[@]}"; do + awk -v snap="$snap" -v want="$STATS_CG_FIELDS" ' + BEGIN { n = split(want, w, " "); for (i = 1; i <= n; i++) keep[w[i]] = 1 } + keep[$1] { printf "%s %s %s\n", snap, $1, $2 } + ' "$_d/memory.stat" 2>/dev/null + awk -v snap="$snap" '{ printf "%s current %s\n", snap, $1 }' \ + "$_d/memory.current" 2>/dev/null + done >>"$STATS_CG_LOG" done ) >"$STATS_LOG" 2>/dev/null & STATS_PID=$! @@ -90,6 +141,37 @@ stats_stop() { } ' "$STATS_LOG") + # ── cgroup composition at the peak snapshot ───────────────────────── + # Reported as MiB per field. The snapshot is chosen by total memory so + # the breakdown describes the same moment STATS_PEAK_MEM reports, + # rather than being a max-per-field mixture of different instants. + STATS_MEM_DETAIL="" + if [ -s "${STATS_CG_LOG:-/dev/null}" ]; then + STATS_MEM_DETAIL=$(awk ' + { v[$1 SUBSEP $2] += $3; if (!($1 in seen)) { seen[$1]=1 } } + END { + best = ""; bestv = -1 + for (k in v) { + split(k, a, SUBSEP) + if (a[2] == "current" && v[k] > bestv) { bestv = v[k]; best = a[1] } + } + if (best == "") exit + n = split("current anon file active_file inactive_file sock slab kernel", f, " ") + out = "" + for (i = 1; i <= n; i++) { + key = best SUBSEP f[i] + if (key in v) { + if (out != "") out = out "," + out = out sprintf("\"%s\":%.1f", f[i], v[key] / 1048576) + } + } + print out + } + ' "$STATS_CG_LOG") + fi + rm -f "${STATS_CG_LOG:-}" 2>/dev/null || true + STATS_CG_LOG="" + # ── Per-container breakdown — average CPU and peak mem per container, # rendered as "proxy: 4200% 1.2GiB | server: 1200% 512MiB". Skipped # entirely when only one container was sampled (the breakdown would