Persist complete dependency-review evidence across paginated compare diffs

scripts/ci/record_dependency_review.py

@@ -151,6 +151,17 @@ def _validate_changes(
             raise ValueError("uv dependency diff does not match lockfile versions and scopes")
 
 
+def _next_page_url(headers) -> str | None:
+    link_header = headers.get("Link")
+    if not isinstance(link_header, str) or not link_header:
+        return None
+    for part in link_header.split(","):
+        match = re.search(r"<([^>]+)>;\s*rel=\"next\"", part)
+        if match:
+            return match.group(1)
+    return None
+
+
 def record_dependency_review(
     environment: dict[str, str],
     destination: Path,
@@ -176,21 +187,39 @@ def record_dependency_review(
             "X-GitHub-Api-Version": "2022-11-28",
         },
     )
-    with opener(request, timeout=90) as response:
-        raw_diff = response.read()
-        snapshot_warning = response.headers.get(
-            "x-github-dependency-graph-snapshot-warnings"
-        )
-    if snapshot_warning:
-        raise ValueError(
-            "Dependency graph reported snapshot warnings; refusing to save "
-            "an incomplete comparison as review evidence"
+    page_url: str | None = request.full_url
+    raw_pages: list[bytes] = []
+    changes: list[dict] = []
+    while page_url:
+        page_request = Request(
+            page_url,
+            headers=request.headers,
         )
-    changes = json.loads(raw_diff)
-    if not isinstance(changes, list) or not all(
-        isinstance(item, dict) for item in changes
-    ):
-        raise ValueError("Dependency diff API did not return a complete change list")
+        with opener(page_request, timeout=90) as response:
+            page_raw = response.read()
+            raw_pages.append(page_raw)
+            snapshot_warning = response.headers.get(
+                "x-github-dependency-graph-snapshot-warnings"
+            )
+            if snapshot_warning:
+                raise ValueError(
+                    "Dependency graph reported snapshot warnings; refusing to save "
+                    "an incomplete comparison as review evidence"
+                )
+            page_changes = json.loads(page_raw)
+            if not isinstance(page_changes, list) or not all(
+                isinstance(item, dict) for item in page_changes
+            ):
+                raise ValueError(
+                    "Dependency diff API did not return a complete change list"
+                )
+            changes.extend(page_changes)
+            page_url = _next_page_url(response.headers)
+    raw_diff = (
+        raw_pages[0]
+        if len(raw_pages) == 1
+        else json.dumps(changes).encode()
+    )
     _validate_changes(
         changes,
         base,

tests/unit/test_dependency_review_workflow.py

@@ -545,6 +545,65 @@ class ApiResponse(io.BytesIO):
     assert (destination / "dependency-diff.json").read_bytes() == b"[]"
 
 
+def test_dependency_review_evidence_fetches_all_compare_pages(
+    tmp_path: Path,
+) -> None:
+    module = runpy.run_path(
+        str(WORKFLOW_PATH.parents[2] / "scripts/ci/record_dependency_review.py")
+    )
+    environment = {
+        "BASE_SHA": "a" * 40,
+        "HEAD_SHA": "b" * 40,
+        "GITHUB_REPOSITORY": "groupthinking/EventRelay",
+        "GITHUB_TOKEN": "dummy-test-token",
+        "GITHUB_RUN_ID": "123",
+        "GITHUB_RUN_ATTEMPT": "1",
+        "REVIEW_OUTCOME": "success",
+    }
+    page_one = [
+        {"manifest": "requirements.txt", "name": "alpha"},
+        {"manifest": "requirements.txt", "name": "beta"},
+    ]
+    page_two = [{"manifest": "requirements.txt", "name": "gamma"}]
+    expected = page_one + page_two
+    api_root = "https://api.github.com"
+    first_url = (
+        f"{api_root}/repos/groupthinking/EventRelay/"
+        f"dependency-graph/compare/{'a' * 40}...{'b' * 40}"
+    )
+    second_url = f"{first_url}?page=2"
+
+    class ApiResponse(io.BytesIO):
+        def __init__(self, payload: bytes, headers: dict[str, str]) -> None:
+            super().__init__(payload)
+            self.headers = headers
+
+    requested_urls: list[str] = []
+
+    def opener(request, timeout):
+        assert timeout == 90
+        requested_urls.append(request.full_url)
+        if request.full_url == first_url:
+            return ApiResponse(
+                json.dumps(page_one).encode(),
+                {"Link": f'<{second_url}>; rel="next"'},
+            )
+        if request.full_url == second_url:
+            return ApiResponse(json.dumps(page_two).encode(), {})
+        raise AssertionError(f"Unexpected URL: {request.full_url}")
+
+    destination = tmp_path / "dependency-review-evidence"
+    saved = module["record_dependency_review"](
+        environment,
+        destination,
+        opener=opener,
+        changed_files={"requirements.txt"},
+    )
+    assert requested_urls == [first_url, second_url]
+    assert saved["dependency_diff_count"] == len(expected)
+    assert json.loads((destination / "dependency-diff.json").read_text()) == expected
+
+
 def test_dependency_review_rejects_warning_free_graph_drift(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None: