One paid solve per page, parser failures exit 5, CSV formula guard, canary fails without its secret

.env.example

@@ -29,8 +29,8 @@ TWOCAPTCHA_KEY=your_2captcha_api_key_here
 
 # --- Scraping Browser API -------------------------------------------------
 # A remote browser you do not run or patch, with a persistent profile and a
-# chosen exit country. Used by --cdp-endpoint. Profiles expire (about a day),
-# so this is the SHAPE of the endpoint, never a working one.
+# chosen exit country. Used by --cdp-endpoint. Profiles can expire (a 401
+# says so), so this is the SHAPE of the endpoint, never a working one.
 #
 #   country-  the exit country. kohls.com is a US shop.
 #   pid-      a profile with persistent cookies. ONE live connection each.

.github/canary_assert.py

@@ -0,0 +1,82 @@
+"""Assertions for the canary run, kept out of the workflow so they can be run
+(and tested) locally against a run's artefacts.
+
+    python .github/canary_assert.py <out-prefix> <exit-code>
+
+Thresholds test the PARSER, not the catalogue: the expected row count and
+page count come from the listing's own arithmetic in the sidecar
+(`total_results`, `pages_available`, `per_page`), so a brand assortment
+growing or shrinking does not turn the canary red. Exit 3 (refused), exit 5
+(never obtained) and zero rows are all failures.
+"""
+import json
+import os
+import sys
+
+PAGES_ASKED = 3
+CORE_FIELDS = ("sku", "url", "title", "price", "currency")
+
+
+def load(path, default):
+    try:
+        with open(path, encoding="utf-8") as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return default
+
+
+def checks(prefix, rc):
+    meta = load(prefix + ".meta.json", {})
+    rows = load(prefix + ".json", [])
+    listing = meta.get("listing") or {}
+    available = listing.get("pages_available") or PAGES_ASKED
+    per_page = listing.get("per_page") or 0
+    total = listing.get("total_results") or 0
+    pages = min(PAGES_ASKED, available)
+    expected = min(total, pages * per_page) if total and per_page else 0
+    out = {
+        "exit 0 (3 = refused, 5 = never obtained, 4 = no rows)": rc == 0,
+        "status complete": meta.get("status") == "complete",
+        "pages completed = min(%d, pages_available)" % PAGES_ASKED:
+            meta.get("pages_completed") == pages,
+        "rows present": bool(rows),
+        "the listing states its size": expected > 0,
+        # 95%: a sorted listing can still lose a few rows to a product going
+        # out of stock between page fetches; a parser regression loses far more.
+        "rows >= 95% of what the listing states for those pages":
+            expected > 0 and len(rows) >= 0.95 * expected,
+    }
+    for name in CORE_FIELDS:
+        out["every row has %s" % name] = bool(rows) and all(
+            r.get(name) not in (None, "") for r in rows)
+    out["page+position unique"] = bool(rows) and len(
+        {(r.get("page"), r.get("position")) for r in rows}) == len(rows)
+    out["no original_price at or below its price"] = all(
+        r.get("original_price") is None or r.get("price") is None
+        or r["original_price"] > r["price"] for r in rows)
+    return out
+
+
+def main(argv):
+    prefix = argv[1]
+    try:
+        rc = int(argv[2])
+    except (IndexError, ValueError):
+        rc = 1
+    results = checks(prefix, rc)
+    for name, ok in results.items():
+        print(("ok     " if ok else "FAILED ") + name)
+    if rc == 3:
+        print("::error::exit 3 — kohls.com refused the run. The endpoint's exit "
+              "may be scored, or the profile is not the US one.")
+    elif rc == 5:
+        print("::error::exit 5 — the page was never obtained, or was served and "
+              "parsed to nothing. A 401 in the log means the profile expired: "
+              "refresh KOHLS_CDP_ENDPOINT.")
+    elif rc == 4:
+        print("::error::exit 4 — zero rows from a listing that should hold products.")
+    return 0 if all(results.values()) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/ci_checks.py

@@ -237,7 +237,7 @@ def tracked_files():
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is

.github/workflows/canary.yml

@@ -2,20 +2,25 @@ name: canary
 
 # A real three-page run against kohls.com, dispatched by hand.
 #
-# Why dispatch-only, and why it SKIPS without its secret:
+# Why it needs a secret, and why it FAILS without one:
 #
 #   * Measured 2026-09-25, kohls.com (Akamai) refused this project's own
 #     Chromium, real Chrome and curl on US residential exits, and served
 #     the same URL through the 2Captcha Scraping Browser API. A GitHub runner
 #     is a datacentre address with a local browser — exactly what was refused
-#     — so without KOHLS_CDP_ENDPOINT this canary cannot pass, and a check
-#     that is always red teaches everyone to ignore checks.
-#   * A Scraping Browser profile's credentials live about a day, so a
-#     scheduled canary's secret would rot between runs and go red for a
-#     reason that says nothing about the site.
+#     — so the only path this canary can test is KOHLS_CDP_ENDPOINT.
+#   * Without that secret the job FAILS with an error naming it. It used to
+#     skip green, and a green canary that never reached the site says
+#     nothing about the site.
+#   * A refusal (exit 3), a page never obtained (exit 5) or zero rows is red.
+#     An expired profile answers 401 and shows up as exit 5: refresh the
+#     secret and dispatch again.
 #
-# Set the secret to a FRESH endpoint, then dispatch. The endpoint reaches the
-# engine through the environment, never through a command line.
+# Dispatch-only: each run spends Scraping Browser usage, so whether to run it
+# on a schedule is the owner's decision. Profiles can expire; how long one
+# lasts has not been measured for this repo, so a 401 is the signal to
+# refresh it, not a fixed lifetime. The endpoint reaches the engine through
+# the environment, never through a command line.
 on:
   workflow_dispatch:
 
@@ -24,56 +29,33 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KOHLS_CDP_ENDPOINT: ${{ secrets.KOHLS_CDP_ENDPOINT }}
-      # A filtered listing with a known size, SORTED: the Cuisinart brand
-      # page, 337 products over 3 pages on 2026-09-25. Sorted because the
-      # default "Featured" order moved mid-run in a live test (22 duplicates
-      # across pages, ~19 products never seen); Price Low-High returned all
-      # 337 with none.
+      # A filtered listing, SORTED: the Cuisinart brand page (337 products
+      # over 3 pages on 2026-09-25). The assertions read the size the page
+      # itself states, so the brand's assortment changing is not a failure.
+      # Sorted because the default "Featured" order moved mid-run in a live
+      # test (22 duplicates across pages, ~19 products never seen); Price
+      # Low-High returned all 337 with none.
       URL: https://www.kohls.com/catalog/cuisinart.jsp?CN=Brand:Cuisinart&S=4
     steps:
       - uses: actions/checkout@v4
-      - name: Skip without a Scraping Browser endpoint
+      - name: Require the Scraping Browser endpoint
         if: env.KOHLS_CDP_ENDPOINT == ''
-        run: echo "::notice::KOHLS_CDP_ENDPOINT is not set — skipping. kohls.com refuses a datacentre runner's own browser; see the comment at the top of this workflow."
+        run: |
+          echo "::error::Missing secret KOHLS_CDP_ENDPOINT — the canary cannot reach kohls.com without it (a datacentre runner's own browser is refused). Set the repo secret to a fresh Scraping Browser endpoint and dispatch again."
+          exit 1
       - uses: actions/setup-python@v5
-        if: env.KOHLS_CDP_ENDPOINT != ''
         with:
           python-version: "3.12"
       - name: Install core + Playwright (no local browser needed over CDP)
-        if: env.KOHLS_CDP_ENDPOINT != ''
         run: pip install -r requirements.txt -r requirements-playwright.txt
       - name: Three-page run
-        if: env.KOHLS_CDP_ENDPOINT != ''
         run: |
           set +e
           mkdir -p live
           python playwright_scraper.py --url "$URL" --pages 3 --out live/canary
           echo "exit_code=$?" >> "$GITHUB_ENV"
       - name: Assert the run
-        if: env.KOHLS_CDP_ENDPOINT != ''
-        run: |
-          python - <<'PY'
-          import json, os, sys
-          rc = int(os.environ.get("exit_code", "1"))
-          meta = json.load(open("live/canary.meta.json")) if os.path.exists("live/canary.meta.json") else {}
-          rows = json.load(open("live/canary.json")) if os.path.exists("live/canary.json") else []
-          priced = sum(1 for r in rows if r["price"] is not None)
-          checks = {
-              "exit 0": rc == 0,
-              "status complete": meta.get("status") == "complete",
-              "3 pages completed": meta.get("pages_completed") == 3,
-              "at least 300 rows": len(rows) >= 300,
-              "every row priced": rows and priced == len(rows),
-              "page+position unique": len({(r["page"], r["position"]) for r in rows}) == len(rows),
-              "no original_price at or below its price": all(
-                  r["original_price"] is None or r["original_price"] > r["price"] for r in rows),
-          }
-          for name, ok in checks.items():
-              print(("ok     " if ok else "FAILED ") + name)
-          if rc == 3:
-              print("::warning::exit 3 — refused. The endpoint's exit may be scored, or the profile expired.")
-          sys.exit(0 if all(checks.values()) else 1)
-          PY
+        run: python .github/canary_assert.py live/canary "$exit_code"
       - uses: actions/upload-artifact@v4
         if: always() && env.KOHLS_CDP_ENDPOINT != ''
         with:

.gitignore

@@ -44,6 +44,10 @@ dump/
 *_debug.html
 *_debug.png
 *.html.page*
+# A bare `--dump-html page.html --pages 1` writes exactly that name, at
+# whatever path was typed: any HTML at the repo root is a capture. The repo
+# ships no HTML of its own.
+/*.html
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -93,7 +97,7 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# and the fix never reached anyone else: when you fix one, check the
+# siblings.
 .env*
 !.env.example

CHANGELOG.md

@@ -6,6 +6,38 @@ All notable changes to this project are documented here. The format follows
 toolkit can: a patch release means "fixes", not that every default is
 frozen — a patch that changes a default says so at the top of its entry.
 
+## [Unreleased]
+
+> **Behaviour changes.** A page that was served with its listing and parsed
+> to zero rows (`served_but_unparsed`) now exits **5**, not 4: exit 4 means
+> the catalogue is empty, and this is a parser failure. A page now buys at
+> most **one** paid captcha solve across all its retries and exit rotations
+> (it could buy up to `--proxy-block-retries + 1`). A rewritten output file
+> keeps its existing permissions instead of being reset to the umask default.
+
+### Fixed
+
+- **CSV formula cells are neutralised.** A string cell starting with `=`,
+  `+`, `-`, `@`, a tab, CR or LF gets a leading apostrophe in the CSV only
+  (after list cells are joined); numbers and the JSON are unchanged. The
+  sidecar records `csv_cells_escaped`.
+- **One paid solve per page.** The allowance is created once per page and
+  checked before the solver is called, so neither a block retry nor a
+  rotation to another exit buys another solve.
+- **Existing output modes are kept.** A 0640 output stayed 0640 only until
+  the next run rewrote it as 0644; new files still follow the umask.
+- **pyppeteer proxy authentication on a current Chromium.** `page.authenticate`
+  needs a CDP method current Chromium no longer has; that one error now
+  falls back to CDP `Fetch.authRequired`, answering only proxy challenges.
+  Not yet run through an authenticated proxy against kohls.com.
+- **A bare `--dump-html page.html` at the repo root is ignored by git.**
+- **The canary fails, rather than skips, without `KOHLS_CDP_ENDPOINT`**, and
+  its thresholds now come from the listing's own stated size instead of a
+  fixed 300-row floor. Exit 3, exit 5 and zero rows are red.
+- Removed references to private notes from comments and docs, and the
+  "credentials live about a day" claim from `.env.example` and the canary,
+  which was never measured for this repo.
+
 ## [1.0.1] - 2026-09-28
 
 Fixes from a third-party audit (2026-09-27) and its re-check. Each is

README.md

@@ -202,7 +202,8 @@ command line, where `ps` and your shell history would keep them.
 
 `0` ok · `1` crash · `2` bad usage · `3` refused or challenged · `4` the site
 answered with no products · `5` the page was never obtained (a dead proxy, a
-blank document, a timeout, a CDP connection refused, a remote API failing) ·
+blank document, a timeout, a CDP connection refused, a remote API failing),
+or it was served and this parser read no rows from it ·
 `6` partial. A run that gathers nothing writes nothing, so last night's good
 file is never replaced by an empty one — and every file is written to a
 temporary name and renamed into place, so a run killed mid-write cannot
@@ -215,7 +216,7 @@ reason `internal_error`, and the log carries the masked traceback.
 | Script | Engine | Scraping Browser API | Authenticated proxy |
 |---|---|---|---|
 | `playwright_scraper.py` | Playwright (recommended) | yes | yes |
-| `puppeteer_scraper.py` | pyppeteer | yes | yes (`page.authenticate`) |
+| `puppeteer_scraper.py` | pyppeteer | yes | yes (`page.authenticate`, falling back to CDP `Fetch.authRequired` on a Chromium that lacks it; not yet run through an authenticated proxy here) |
 | `selenium_scraper.py` | Selenium | **no** — chromedriver's `debuggerAddress` has nowhere to put a password; refused with exit 2 | **no** — credentials are stripped, with a warning |
 | `scraper_api_client.py` | none (2Captcha Scraper API) | via `--use-cdp` | — |
 

TROUBLESHOOTING.md

@@ -10,7 +10,7 @@ exit code: it says which of these you are in.
 | 2 | bad usage (the message says what) | [Usage errors](#usage-errors) |
 | 3 | refused or challenged | [Exit 3](#exit-3-refused) |
 | 4 | the site answered, and the answer was "no products" | [Exit 4](#exit-4-no-products) |
-| 5 | the page was never obtained | [Exit 5](#exit-5-never-obtained) |
+| 5 | the page was never obtained, or was served and parsed to nothing | [Exit 5](#exit-5-never-obtained) |
 | 6 | some pages gathered, then the run stopped | [Exit 6](#exit-6-partial) |
 | 1 | a crash — please report it | [Crashes](#crashes) |
 
@@ -63,6 +63,12 @@ the empty files if your pipeline wants them.
 
 ## Exit 5: never obtained
 
+**`The page was served and parsed to 0 rows (served_but_unparsed)`** — the
+page arrived and held the listing, and this parser read nothing from it.
+That is a parser problem, not an empty catalogue: re-run with `--dump-html`
+and report it with the file. (A listing that genuinely holds no products is
+exit 4.) Until 2026-10-15 this case exited 4.
+
 **`could not connect to --cdp-endpoint … 500`** — a Scraping Browser profile
 allows ONE live connection at a time, and the previous run's may not have
 been released yet. Measured: a run started seconds after another got 500,

browser_bridge.py

@@ -180,7 +180,7 @@ class PageOutcome:
     # The server answered with a different page than the URL asked for.
     served_other: bool = False
     # Served (the island or product links were there) and parsed to 0 rows:
-    # this parser's problem, not the end of the listing (§20).
+    # this parser's problem, not the end of the listing.
     parsed_nothing: bool = False
     # An unexpected exception while fetching this page — recorded, masked,
     # so the pages already gathered survive it.
@@ -209,14 +209,15 @@ def counted(self) -> bool:
 # ---------------------------------------------------------------------------
 # Captcha
 # ---------------------------------------------------------------------------
-def handle_captcha_if_present(driver, args) -> bool:
+def handle_captcha_if_present(driver, args, budget=None) -> bool:
     """Detect and solve a reCAPTCHA. True if something was solved.
 
-    Called ONCE per fetch attempt, and a fetch has at most
-    `block_retries + 1` attempts — so a page can buy at most that many
-    solves, and that is the whole budget. (A sibling repo called its solver
-    twice per attempt and counted only one call; one page bought three
-    Turnstile solves.)
+    `budget` is the PAGE's allowance (page_flow.SolveBudget), created once
+    per page by fetch_one_page and shared by every attempt of that page, so
+    a block retry or an exit rotation never buys another solve. It is
+    checked before the purchase, and spent when the solver is called (a
+    failed solve can still be billed). No budget given means a fresh
+    one-solve allowance for this call.
 
     Kohl's renders reCAPTCHA v3 on its account pages (/signin); none was
     found on any listing, search or product page captured. This path exists
@@ -249,6 +250,13 @@ def handle_captcha_if_present(driver, args) -> bool:
         logger.warning("No 2captcha API key is configured, so no solve is "
                        "attempted — continuing with whatever the page already holds.")
         return False
+    if budget is None:
+        budget = page_flow.SolveBudget()
+    if not budget.allows():
+        logger.warning("This page has already used its %d paid solve(s) — not "
+                       "buying another on a retry or a new exit.", budget.limit)
+        return False
+    budget.spend()
     try:
         token = solve_recaptcha(challenge, args.twocaptcha_key,
                                api_version=args.captcha_api,
@@ -309,6 +317,9 @@ def fetch_one_page(driver, args, pool, page_num: int, url: str) -> PageOutcome:
     # jar, same address) — see page_flow for why that number is small.
     block_retries = args.proxy_block_retries if rotating else BLOCK_RETRIES_WITHOUT_POOL
     html, state, status, load_failed = "", "blocked", None, False
+    # One allowance for the whole page: outside the loop, so neither a block
+    # retry nor a rotation to another exit resets it.
+    solve_budget = page_flow.SolveBudget()
 
     for block_attempt in range(block_retries + 1):
         logger.info("Fetching page %d: %s", page_num, url)
@@ -343,7 +354,7 @@ def fetch_one_page(driver, args, pool, page_num: int, url: str) -> PageOutcome:
         if load_failed:
             break
 
-        if handle_captcha_if_present(driver, args):
+        if handle_captcha_if_present(driver, args, solve_budget):
             driver.sleep(1000)
             status = None  # the solve navigated; the old status is stale
 

captcha_solver.py

@@ -577,8 +577,8 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st
                      api_version: str = "v2", min_score: float = 0.7) -> str:
     """Public entry point: solve `challenge` through 2captcha and return the token.
 
-    There used to be a `use_antidetect` branch here, behind a CLI flag of the
-    same name, that POSTed to a hardcoded local "solve-captcha" endpoint.
+    There used to be a second branch here, behind a CLI flag, that POSTed to
+    a hardcoded local "solve-captcha" endpoint.
     It was removed before publishing: that endpoint was a placeholder for a
     product that does not exist under that name, so the flag could not work for
     anyone who set it, and the real "the browser solves it for you" path is