diff --git a/.github/workflows/figures.yml b/.github/workflows/figures.yml new file mode 100644 index 0000000..d72d0f1 --- /dev/null +++ b/.github/workflows/figures.yml @@ -0,0 +1,92 @@ +name: Figures + +# The front page states four numbers under "Numbers with the conditions they +# were measured under". Three were typed by hand, and by September 2026 two had +# gone stale: decisionrl had grown to 32 algorithms across 24 environments while +# the page said 31 and 22, and coverage read 86% where CI measured 84%. The one +# organisation-wide rule this site advertises is that a published number is +# measured on the run that describes it; its own front page was the exception. +# +# This reads each figure's source and reports drift into one standing issue. +# It is a separate workflow rather than a step in CI because it needs the +# network, and a required check that depends on someone else's uptime teaches +# people to re-run it rather than to read it. +on: + schedule: + - cron: "0 7 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + figures: + name: Check the stated figures against their sources + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Compare + id: figures + run: | + python scripts/check_figures.py . | tee figures.txt + echo "drifted=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + continue-on-error: true + + - name: Build the report + if: steps.figures.outputs.drifted != '0' + run: | + { + echo "A figure on the front page no longer matches the repository it describes." + echo + echo '```' + cat figures.txt + echo '```' + echo + echo "Each line names the page, what it says, and what the source says." + echo "Reproduce with \`python scripts/check_figures.py .\`." + } > report.md + + - name: Find the standing figures issue + id: standing + if: steps.figures.outputs.drifted != '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + number=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --label maintenance \ + --search 'in:title "A figure on the site has drifted"' \ + --limit 1 --json number --jq '.[0].number // empty') + echo "number=${number}" >> "$GITHUB_OUTPUT" + + - name: Report into the standing issue + if: steps.figures.outputs.drifted != '0' + uses: peter-evans/create-issue-from-file@v6 + with: + issue-number: ${{ steps.standing.outputs.number }} + title: A figure on the site has drifted + content-filepath: ./report.md + labels: maintenance + + - name: Close the standing issue once every figure matches + if: steps.figures.outputs.drifted == '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + number=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --label maintenance \ + --search 'in:title "A figure on the site has drifted"' \ + --limit 1 --json number --jq '.[0].number // empty') + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" \ + --comment "Every checkable figure matches its source again, as of \`${{ github.sha }}\`." + fi diff --git a/index.html b/index.html index 3bf2165..af7d407 100644 --- a/index.html +++ b/index.html @@ -438,7 +438,7 @@

Numbers with the conditions they were measured under

Algorithms
-
31In decisionrl, across 22 environments — nine of them applied rather than synthetic.
+
32In decisionrl, across 24 environments — nine of them applied rather than synthetic.
Required dependencies
@@ -446,7 +446,7 @@

Numbers with the conditions they were measured under

Line coverage
-
86%decisionrl, measured by pytest-cov in CI. The build fails below the threshold rather than reporting it.
+
84%decisionrl, measured by pytest-cov on the fast suite in CI. The build fails below 80% rather than reporting the number.
Retrieval recall@5
diff --git a/ru/index.html b/ru/index.html index a4cd84b..6e1169d 100644 --- a/ru/index.html +++ b/ru/index.html @@ -436,7 +436,7 @@

Цифры с условиями, при которых они получе
Алгоритмов
-
31В decisionrl, на 22 средах — девять из них прикладные, а не синтетические.
+
32В decisionrl, на 24 средах — девять из них прикладные, а не синтетические.
Обязательных зависимостей
@@ -444,7 +444,7 @@

Цифры с условиями, при которых они получе

Покрытие строк
-
86%decisionrl, замер pytest-cov в CI. Ниже порога сборка падает, а не сообщает цифру.
+
84%decisionrl, замер pytest-cov по быстрому набору в CI. Ниже 80% сборка падает, а не сообщает цифру.
Поиск, recall@5
diff --git a/scripts/check_figures.py b/scripts/check_figures.py new file mode 100644 index 0000000..7a4b747 --- /dev/null +++ b/scripts/check_figures.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Hold the figures on the front page to the repositories they describe. + +The page states four numbers under "Numbers with the conditions they were +measured under". Three of them were typed by hand and nothing recomputed them, +and by September 2026 two had gone stale: decisionrl had grown to 32 algorithms +across 24 environments while the page still said 31 and 22, and line coverage +read 86% when CI measured 84%. An organisation whose stated rule is that a +published number is measured on the run that describes it cannot have its own +front page be the exception. + +Each figure here names its source and is checked against it: + + algorithms / environments decisionrl's CITATION.cff, which decisionrl's own + test suite pins to the package it ships + required dependencies glia's pyproject.toml, read directly + line coverage Codecov, the run that produced it + +recall@5 is not checked yet: praxis states it in prose, with nothing pinning it +to an eval run. That is a gap in praxis, not something to paper over here, so +the figure is reported as unverified rather than quietly passed. + +Standard library only. Needs network, which is why this runs in its own weekly +workflow rather than in the required CI check. + +Usage: + python scripts/check_figures.py [site_root] +""" + +from __future__ import annotations + +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +RAW = "https://raw.githubusercontent.com/DrobyshevDev" +CODECOV = "https://api.codecov.io/api/v2/github/DrobyshevDev/repos/decisionrl/" + +PAGES = ("index.html", "ru/index.html") + + +def fetch(url: str) -> str | None: + try: + request = urllib.request.Request(url, headers={"User-Agent": "drobyshevdev-figures"}) + with urllib.request.urlopen(request, timeout=30) as response: + return response.read().decode("utf-8") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError): + return None + + +def figures_on(page: str) -> dict[str, str]: + """The
/
pairs of the figures list, as {label: value}.""" + block = re.search(r'
(.*?)
', page, re.DOTALL) + if not block: + return {} + pairs = re.findall(r"
(.*?)
\s*
(.*?) tuple[int, int, int] | None: + """Algorithms, environments and applied environments, from decisionrl.""" + citation = fetch(f"{RAW}/decisionrl/main/CITATION.cff") + if citation is None: + return None + match = re.search(r"(\d+) algorithms and (\d+) environments, (\d+) of them applied", citation) + return tuple(int(g) for g in match.groups()) if match else None + + +def source_glia_dependencies() -> int | None: + pyproject = fetch(f"{RAW}/glia/master/pyproject.toml") + if pyproject is None: + return None + match = re.search(r"^dependencies\s*=\s*\[(.*?)\]", pyproject, re.DOTALL | re.MULTILINE) + if not match: + return None + return len([item for item in match.group(1).split(",") if item.strip()]) + + +def source_coverage() -> float | None: + body = fetch(CODECOV) + if body is None: + return None + try: + totals = json.loads(body).get("totals") or {} + except json.JSONDecodeError: + return None + coverage = totals.get("coverage") + return float(coverage) if coverage is not None else None + + +def main(root: Path) -> int: + pages = {} + for name in PAGES: + path = root / name + if not path.exists(): + print(f" {name}: not found", file=sys.stderr) + return 2 + pages[name] = figures_on(path.read_text(encoding="utf-8")) + + problems: list[str] = [] + unchecked: list[str] = [] + + counts = source_counts() + if counts is None: + unchecked.append("algorithms/environments — decisionrl's CITATION.cff could not be read") + else: + algorithms, environments, applied = counts + for name, figures in pages.items(): + stated = next((v for k, v in figures.items() if k in ("Algorithms", "Алгоритмов")), None) + if stated is None: + problems.append(f"{name}: no algorithms figure") + elif stated != str(algorithms): + problems.append(f"{name}: algorithms says {stated}, decisionrl ships {algorithms}") + words = {9: ("nine", "девять")} + for name, page in ((n, (root / n).read_text(encoding="utf-8")) for n in PAGES): + for quoted in re.findall(r"(?:across|на) (\d+) (?:environments|средах)", page): + if int(quoted) != environments: + problems.append(f"{name}: says {quoted} environments, decisionrl ships {environments}") + # The applied count is spelled out in words beside it, which is how + # it escaped every check that looked for digits. + spelled = words.get(applied) + if spelled and not any(word in page for word in spelled): + problems.append( + f"{name}: does not say {spelled[0]}/{spelled[1]} applied environments, " + f"but decisionrl ships {applied}" + ) + + dependencies = source_glia_dependencies() + if dependencies is None: + unchecked.append("required dependencies — glia's pyproject.toml could not be read") + elif dependencies != 0: + problems.append(f"glia now has {dependencies} required dependencies; the page claims none") + + coverage = source_coverage() + if coverage is None: + unchecked.append("line coverage — Codecov has no figure for decisionrl yet") + else: + for name, figures in pages.items(): + stated = next((v for k, v in figures.items() if k in ("Line coverage", "Покрытие строк")), None) + if stated is None: + problems.append(f"{name}: no coverage figure") + elif abs(int(stated.rstrip("%")) - coverage) >= 1: + problems.append( + f"{name}: coverage says {stated}, Codecov measures {coverage:.1f}%" + ) + + unchecked.append("recall@5 — praxis states it in prose, with no eval run pinning it") + + for problem in problems: + print(f" DRIFTED {problem}") + for note in unchecked: + print(f" unchecked {note}") + if not problems: + print("\n Every figure that can be checked matches its source.") + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main(Path(sys.argv[1] if len(sys.argv) > 1 else ".")))