From 21eadeb08a6705805c202068160336263dc36eea Mon Sep 17 00:00:00 2001 From: Denis_Drobyshev Date: Fri, 18 Sep 2026 22:15:14 +0300 Subject: [PATCH] Correct the figures that drifted, and check them from now on The front page states four numbers under the heading "Numbers with the conditions they were measured under". Two of them were wrong. decisionrl ships 32 algorithms across 24 environments; the page said 31 and 22, the same stale pair its own README badges carried. Line coverage read 86%; the figure CI measures on the fast suite is 84.41%, and the threshold that fails the build is 80, so both halves of the sentence were off. Nothing recomputed any of it, which is the failure this organisation says it does not accept, on the one page where it says so. check_figures.py reads each figure's source -- decisionrl's CITATION.cff, which decisionrl's own tests pin to the package; glia's pyproject; Codecov -- and reports drift into one standing issue. recall@5 is reported as unverified rather than passed: praxis states it in prose with no eval run behind it, which is a gap in praxis to close there. --- .github/workflows/figures.yml | 92 +++++++++++++++++++ index.html | 4 +- ru/index.html | 4 +- scripts/check_figures.py | 162 ++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/figures.yml create mode 100644 scripts/check_figures.py diff --git a/.github/workflows/figures.yml b/.github/workflows/figures.yml new file mode 100644 index 0000000..d72d0f1 --- /dev/null +++ b/.github/workflows/figures.yml @@ -0,0 +1,92 @@ +name: Figures + +# The front page states four numbers under "Numbers with the conditions they +# were measured under". Three were typed by hand, and by September 2026 two had +# gone stale: decisionrl had grown to 32 algorithms across 24 environments while +# the page said 31 and 22, and coverage read 86% where CI measured 84%. The one +# organisation-wide rule this site advertises is that a published number is +# measured on the run that describes it; its own front page was the exception. +# +# This reads each figure's source and reports drift into one standing issue. +# It is a separate workflow rather than a step in CI because it needs the +# network, and a required check that depends on someone else's uptime teaches +# people to re-run it rather than to read it. +on: + schedule: + - cron: "0 7 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + figures: + name: Check the stated figures against their sources + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Compare + id: figures + run: | + python scripts/check_figures.py . | tee figures.txt + echo "drifted=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + continue-on-error: true + + - name: Build the report + if: steps.figures.outputs.drifted != '0' + run: | + { + echo "A figure on the front page no longer matches the repository it describes." + echo + echo '```' + cat figures.txt + echo '```' + echo + echo "Each line names the page, what it says, and what the source says." + echo "Reproduce with \`python scripts/check_figures.py .\`." + } > report.md + + - name: Find the standing figures issue + id: standing + if: steps.figures.outputs.drifted != '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + number=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --label maintenance \ + --search 'in:title "A figure on the site has drifted"' \ + --limit 1 --json number --jq '.[0].number // empty') + echo "number=${number}" >> "$GITHUB_OUTPUT" + + - name: Report into the standing issue + if: steps.figures.outputs.drifted != '0' + uses: peter-evans/create-issue-from-file@v6 + with: + issue-number: ${{ steps.standing.outputs.number }} + title: A figure on the site has drifted + content-filepath: ./report.md + labels: maintenance + + - name: Close the standing issue once every figure matches + if: steps.figures.outputs.drifted == '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + number=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --label maintenance \ + --search 'in:title "A figure on the site has drifted"' \ + --limit 1 --json number --jq '.[0].number // empty') + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" \ + --comment "Every checkable figure matches its source again, as of \`${{ github.sha }}\`." + fi diff --git a/index.html b/index.html index 3bf2165..af7d407 100644 --- a/index.html +++ b/index.html @@ -438,7 +438,7 @@

Numbers with the conditions they were measured under

Algorithms
-
31In decisionrl, across 22 environments — nine of them applied rather than synthetic.
+
32In decisionrl, across 24 environments — nine of them applied rather than synthetic.
Required dependencies
@@ -446,7 +446,7 @@

Numbers with the conditions they were measured under

Line coverage
-
86%decisionrl, measured by pytest-cov in CI. The build fails below the threshold rather than reporting it.
+
84%decisionrl, measured by pytest-cov on the fast suite in CI. The build fails below 80% rather than reporting the number.
Retrieval recall@5
diff --git a/ru/index.html b/ru/index.html index a4cd84b..6e1169d 100644 --- a/ru/index.html +++ b/ru/index.html @@ -436,7 +436,7 @@

Цифры с условиями, при которых они получе
Алгоритмов
-
31В decisionrl, на 22 средах — девять из них прикладные, а не синтетические.
+
32В decisionrl, на 24 средах — девять из них прикладные, а не синтетические.
Обязательных зависимостей
@@ -444,7 +444,7 @@

Цифры с условиями, при которых они получе

Покрытие строк
-
86%decisionrl, замер pytest-cov в CI. Ниже порога сборка падает, а не сообщает цифру.
+
84%decisionrl, замер pytest-cov по быстрому набору в CI. Ниже 80% сборка падает, а не сообщает цифру.
Поиск, recall@5
diff --git a/scripts/check_figures.py b/scripts/check_figures.py new file mode 100644 index 0000000..7a4b747 --- /dev/null +++ b/scripts/check_figures.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Hold the figures on the front page to the repositories they describe. + +The page states four numbers under "Numbers with the conditions they were +measured under". Three of them were typed by hand and nothing recomputed them, +and by September 2026 two had gone stale: decisionrl had grown to 32 algorithms +across 24 environments while the page still said 31 and 22, and line coverage +read 86% when CI measured 84%. An organisation whose stated rule is that a +published number is measured on the run that describes it cannot have its own +front page be the exception. + +Each figure here names its source and is checked against it: + + algorithms / environments decisionrl's CITATION.cff, which decisionrl's own + test suite pins to the package it ships + required dependencies glia's pyproject.toml, read directly + line coverage Codecov, the run that produced it + +recall@5 is not checked yet: praxis states it in prose, with nothing pinning it +to an eval run. That is a gap in praxis, not something to paper over here, so +the figure is reported as unverified rather than quietly passed. + +Standard library only. Needs network, which is why this runs in its own weekly +workflow rather than in the required CI check. + +Usage: + python scripts/check_figures.py [site_root] +""" + +from __future__ import annotations + +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +RAW = "https://raw.githubusercontent.com/DrobyshevDev" +CODECOV = "https://api.codecov.io/api/v2/github/DrobyshevDev/repos/decisionrl/" + +PAGES = ("index.html", "ru/index.html") + + +def fetch(url: str) -> str | None: + try: + request = urllib.request.Request(url, headers={"User-Agent": "drobyshevdev-figures"}) + with urllib.request.urlopen(request, timeout=30) as response: + return response.read().decode("utf-8") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError): + return None + + +def figures_on(page: str) -> dict[str, str]: + """The
/
pairs of the figures list, as {label: value}.""" + block = re.search(r'
(.*?)
', page, re.DOTALL) + if not block: + return {} + pairs = re.findall(r"
(.*?)
\s*
(.*?) tuple[int, int, int] | None: + """Algorithms, environments and applied environments, from decisionrl.""" + citation = fetch(f"{RAW}/decisionrl/main/CITATION.cff") + if citation is None: + return None + match = re.search(r"(\d+) algorithms and (\d+) environments, (\d+) of them applied", citation) + return tuple(int(g) for g in match.groups()) if match else None + + +def source_glia_dependencies() -> int | None: + pyproject = fetch(f"{RAW}/glia/master/pyproject.toml") + if pyproject is None: + return None + match = re.search(r"^dependencies\s*=\s*\[(.*?)\]", pyproject, re.DOTALL | re.MULTILINE) + if not match: + return None + return len([item for item in match.group(1).split(",") if item.strip()]) + + +def source_coverage() -> float | None: + body = fetch(CODECOV) + if body is None: + return None + try: + totals = json.loads(body).get("totals") or {} + except json.JSONDecodeError: + return None + coverage = totals.get("coverage") + return float(coverage) if coverage is not None else None + + +def main(root: Path) -> int: + pages = {} + for name in PAGES: + path = root / name + if not path.exists(): + print(f" {name}: not found", file=sys.stderr) + return 2 + pages[name] = figures_on(path.read_text(encoding="utf-8")) + + problems: list[str] = [] + unchecked: list[str] = [] + + counts = source_counts() + if counts is None: + unchecked.append("algorithms/environments — decisionrl's CITATION.cff could not be read") + else: + algorithms, environments, applied = counts + for name, figures in pages.items(): + stated = next((v for k, v in figures.items() if k in ("Algorithms", "Алгоритмов")), None) + if stated is None: + problems.append(f"{name}: no algorithms figure") + elif stated != str(algorithms): + problems.append(f"{name}: algorithms says {stated}, decisionrl ships {algorithms}") + words = {9: ("nine", "девять")} + for name, page in ((n, (root / n).read_text(encoding="utf-8")) for n in PAGES): + for quoted in re.findall(r"(?:across|на) (\d+) (?:environments|средах)", page): + if int(quoted) != environments: + problems.append(f"{name}: says {quoted} environments, decisionrl ships {environments}") + # The applied count is spelled out in words beside it, which is how + # it escaped every check that looked for digits. + spelled = words.get(applied) + if spelled and not any(word in page for word in spelled): + problems.append( + f"{name}: does not say {spelled[0]}/{spelled[1]} applied environments, " + f"but decisionrl ships {applied}" + ) + + dependencies = source_glia_dependencies() + if dependencies is None: + unchecked.append("required dependencies — glia's pyproject.toml could not be read") + elif dependencies != 0: + problems.append(f"glia now has {dependencies} required dependencies; the page claims none") + + coverage = source_coverage() + if coverage is None: + unchecked.append("line coverage — Codecov has no figure for decisionrl yet") + else: + for name, figures in pages.items(): + stated = next((v for k, v in figures.items() if k in ("Line coverage", "Покрытие строк")), None) + if stated is None: + problems.append(f"{name}: no coverage figure") + elif abs(int(stated.rstrip("%")) - coverage) >= 1: + problems.append( + f"{name}: coverage says {stated}, Codecov measures {coverage:.1f}%" + ) + + unchecked.append("recall@5 — praxis states it in prose, with no eval run pinning it") + + for problem in problems: + print(f" DRIFTED {problem}") + for note in unchecked: + print(f" unchecked {note}") + if not problems: + print("\n Every figure that can be checked matches its source.") + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))