diff --git a/.env.example b/.env.example index 250ad91..d1c81ef 100644 --- a/.env.example +++ b/.env.example @@ -29,8 +29,8 @@ TWOCAPTCHA_KEY=your_2captcha_api_key_here # --- Scraping Browser API ------------------------------------------------- # A remote browser you do not run or patch, with a persistent profile and a -# chosen exit country. Used by --cdp-endpoint. Profiles expire (about a day), -# so this is the SHAPE of the endpoint, never a working one. +# chosen exit country. Used by --cdp-endpoint. Profiles can expire (a 401 +# says so), so this is the SHAPE of the endpoint, never a working one. # # country- the exit country. kohls.com is a US shop. # pid- a profile with persistent cookies. ONE live connection each. diff --git a/.github/canary_assert.py b/.github/canary_assert.py new file mode 100644 index 0000000..bb698e9 --- /dev/null +++ b/.github/canary_assert.py @@ -0,0 +1,82 @@ +"""Assertions for the canary run, kept out of the workflow so they can be run +(and tested) locally against a run's artefacts. + + python .github/canary_assert.py + +Thresholds test the PARSER, not the catalogue: the expected row count and +page count come from the listing's own arithmetic in the sidecar +(`total_results`, `pages_available`, `per_page`), so a brand assortment +growing or shrinking does not turn the canary red. Exit 3 (refused), exit 5 +(never obtained) and zero rows are all failures. +""" +import json +import os +import sys + +PAGES_ASKED = 3 +CORE_FIELDS = ("sku", "url", "title", "price", "currency") + + +def load(path, default): + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return default + + +def checks(prefix, rc): + meta = load(prefix + ".meta.json", {}) + rows = load(prefix + ".json", []) + listing = meta.get("listing") or {} + available = listing.get("pages_available") or PAGES_ASKED + per_page = listing.get("per_page") or 0 + total = listing.get("total_results") or 0 + pages = min(PAGES_ASKED, available) + expected = min(total, pages * per_page) if total and per_page else 0 + out = { + "exit 0 (3 = refused, 5 = never obtained, 4 = no rows)": rc == 0, + "status complete": meta.get("status") == "complete", + "pages completed = min(%d, pages_available)" % PAGES_ASKED: + meta.get("pages_completed") == pages, + "rows present": bool(rows), + "the listing states its size": expected > 0, + # 95%: a sorted listing can still lose a few rows to a product going + # out of stock between page fetches; a parser regression loses far more. + "rows >= 95% of what the listing states for those pages": + expected > 0 and len(rows) >= 0.95 * expected, + } + for name in CORE_FIELDS: + out["every row has %s" % name] = bool(rows) and all( + r.get(name) not in (None, "") for r in rows) + out["page+position unique"] = bool(rows) and len( + {(r.get("page"), r.get("position")) for r in rows}) == len(rows) + out["no original_price at or below its price"] = all( + r.get("original_price") is None or r.get("price") is None + or r["original_price"] > r["price"] for r in rows) + return out + + +def main(argv): + prefix = argv[1] + try: + rc = int(argv[2]) + except (IndexError, ValueError): + rc = 1 + results = checks(prefix, rc) + for name, ok in results.items(): + print(("ok " if ok else "FAILED ") + name) + if rc == 3: + print("::error::exit 3 — kohls.com refused the run. The endpoint's exit " + "may be scored, or the profile is not the US one.") + elif rc == 5: + print("::error::exit 5 — the page was never obtained, or was served and " + "parsed to nothing. A 401 in the log means the profile expired: " + "refresh KOHLS_CDP_ENDPOINT.") + elif rc == 4: + print("::error::exit 4 — zero rows from a listing that should hold products.") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 1669284..02983bb 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -237,7 +237,7 @@ def tracked_files(): # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index c71dedc..50a7d8d 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -2,20 +2,25 @@ name: canary # A real three-page run against kohls.com, dispatched by hand. # -# Why dispatch-only, and why it SKIPS without its secret: +# Why it needs a secret, and why it FAILS without one: # # * Measured 2026-09-25, kohls.com (Akamai) refused this project's own # Chromium, real Chrome and curl on US residential exits, and served # the same URL through the 2Captcha Scraping Browser API. A GitHub runner # is a datacentre address with a local browser — exactly what was refused -# — so without KOHLS_CDP_ENDPOINT this canary cannot pass, and a check -# that is always red teaches everyone to ignore checks. -# * A Scraping Browser profile's credentials live about a day, so a -# scheduled canary's secret would rot between runs and go red for a -# reason that says nothing about the site. +# — so the only path this canary can test is KOHLS_CDP_ENDPOINT. +# * Without that secret the job FAILS with an error naming it. It used to +# skip green, and a green canary that never reached the site says +# nothing about the site. +# * A refusal (exit 3), a page never obtained (exit 5) or zero rows is red. +# An expired profile answers 401 and shows up as exit 5: refresh the +# secret and dispatch again. # -# Set the secret to a FRESH endpoint, then dispatch. The endpoint reaches the -# engine through the environment, never through a command line. +# Dispatch-only: each run spends Scraping Browser usage, so whether to run it +# on a schedule is the owner's decision. Profiles can expire; how long one +# lasts has not been measured for this repo, so a 401 is the signal to +# refresh it, not a fixed lifetime. The endpoint reaches the engine through +# the environment, never through a command line. on: workflow_dispatch: @@ -24,56 +29,33 @@ jobs: runs-on: ubuntu-latest env: KOHLS_CDP_ENDPOINT: ${{ secrets.KOHLS_CDP_ENDPOINT }} - # A filtered listing with a known size, SORTED: the Cuisinart brand - # page, 337 products over 3 pages on 2026-09-25. Sorted because the - # default "Featured" order moved mid-run in a live test (22 duplicates - # across pages, ~19 products never seen); Price Low-High returned all - # 337 with none. + # A filtered listing, SORTED: the Cuisinart brand page (337 products + # over 3 pages on 2026-09-25). The assertions read the size the page + # itself states, so the brand's assortment changing is not a failure. + # Sorted because the default "Featured" order moved mid-run in a live + # test (22 duplicates across pages, ~19 products never seen); Price + # Low-High returned all 337 with none. URL: https://www.kohls.com/catalog/cuisinart.jsp?CN=Brand:Cuisinart&S=4 steps: - uses: actions/checkout@v4 - - name: Skip without a Scraping Browser endpoint + - name: Require the Scraping Browser endpoint if: env.KOHLS_CDP_ENDPOINT == '' - run: echo "::notice::KOHLS_CDP_ENDPOINT is not set — skipping. kohls.com refuses a datacentre runner's own browser; see the comment at the top of this workflow." + run: | + echo "::error::Missing secret KOHLS_CDP_ENDPOINT — the canary cannot reach kohls.com without it (a datacentre runner's own browser is refused). Set the repo secret to a fresh Scraping Browser endpoint and dispatch again." + exit 1 - uses: actions/setup-python@v5 - if: env.KOHLS_CDP_ENDPOINT != '' with: python-version: "3.12" - name: Install core + Playwright (no local browser needed over CDP) - if: env.KOHLS_CDP_ENDPOINT != '' run: pip install -r requirements.txt -r requirements-playwright.txt - name: Three-page run - if: env.KOHLS_CDP_ENDPOINT != '' run: | set +e mkdir -p live python playwright_scraper.py --url "$URL" --pages 3 --out live/canary echo "exit_code=$?" >> "$GITHUB_ENV" - name: Assert the run - if: env.KOHLS_CDP_ENDPOINT != '' - run: | - python - <<'PY' - import json, os, sys - rc = int(os.environ.get("exit_code", "1")) - meta = json.load(open("live/canary.meta.json")) if os.path.exists("live/canary.meta.json") else {} - rows = json.load(open("live/canary.json")) if os.path.exists("live/canary.json") else [] - priced = sum(1 for r in rows if r["price"] is not None) - checks = { - "exit 0": rc == 0, - "status complete": meta.get("status") == "complete", - "3 pages completed": meta.get("pages_completed") == 3, - "at least 300 rows": len(rows) >= 300, - "every row priced": rows and priced == len(rows), - "page+position unique": len({(r["page"], r["position"]) for r in rows}) == len(rows), - "no original_price at or below its price": all( - r["original_price"] is None or r["original_price"] > r["price"] for r in rows), - } - for name, ok in checks.items(): - print(("ok " if ok else "FAILED ") + name) - if rc == 3: - print("::warning::exit 3 — refused. The endpoint's exit may be scored, or the profile expired.") - sys.exit(0 if all(checks.values()) else 1) - PY + run: python .github/canary_assert.py live/canary "$exit_code" - uses: actions/upload-artifact@v4 if: always() && env.KOHLS_CDP_ENDPOINT != '' with: diff --git a/.gitignore b/.gitignore index 89a86e6..400733a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,10 @@ dump/ *_debug.html *_debug.png *.html.page* +# A bare `--dump-html page.html --pages 1` writes exactly that name, at +# whatever path was typed: any HTML at the repo root is a capture. The repo +# ships no HTML of its own. +/*.html # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -93,7 +97,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else: when you fix one, check the +# siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d0722..9b55354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,38 @@ All notable changes to this project are documented here. The format follows toolkit can: a patch release means "fixes", not that every default is frozen — a patch that changes a default says so at the top of its entry. +## [Unreleased] + +> **Behaviour changes.** A page that was served with its listing and parsed +> to zero rows (`served_but_unparsed`) now exits **5**, not 4: exit 4 means +> the catalogue is empty, and this is a parser failure. A page now buys at +> most **one** paid captcha solve across all its retries and exit rotations +> (it could buy up to `--proxy-block-retries + 1`). A rewritten output file +> keeps its existing permissions instead of being reset to the umask default. + +### Fixed + +- **CSV formula cells are neutralised.** A string cell starting with `=`, + `+`, `-`, `@`, a tab, CR or LF gets a leading apostrophe in the CSV only + (after list cells are joined); numbers and the JSON are unchanged. The + sidecar records `csv_cells_escaped`. +- **One paid solve per page.** The allowance is created once per page and + checked before the solver is called, so neither a block retry nor a + rotation to another exit buys another solve. +- **Existing output modes are kept.** A 0640 output stayed 0640 only until + the next run rewrote it as 0644; new files still follow the umask. +- **pyppeteer proxy authentication on a current Chromium.** `page.authenticate` + needs a CDP method current Chromium no longer has; that one error now + falls back to CDP `Fetch.authRequired`, answering only proxy challenges. + Not yet run through an authenticated proxy against kohls.com. +- **A bare `--dump-html page.html` at the repo root is ignored by git.** +- **The canary fails, rather than skips, without `KOHLS_CDP_ENDPOINT`**, and + its thresholds now come from the listing's own stated size instead of a + fixed 300-row floor. Exit 3, exit 5 and zero rows are red. +- Removed references to private notes from comments and docs, and the + "credentials live about a day" claim from `.env.example` and the canary, + which was never measured for this repo. + ## [1.0.1] - 2026-09-28 Fixes from a third-party audit (2026-09-27) and its re-check. Each is diff --git a/README.md b/README.md index 5eb3e63..60852d6 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,8 @@ command line, where `ps` and your shell history would keep them. `0` ok · `1` crash · `2` bad usage · `3` refused or challenged · `4` the site answered with no products · `5` the page was never obtained (a dead proxy, a -blank document, a timeout, a CDP connection refused, a remote API failing) · +blank document, a timeout, a CDP connection refused, a remote API failing), +or it was served and this parser read no rows from it · `6` partial. A run that gathers nothing writes nothing, so last night's good file is never replaced by an empty one — and every file is written to a temporary name and renamed into place, so a run killed mid-write cannot @@ -215,7 +216,7 @@ reason `internal_error`, and the log carries the masked traceback. | Script | Engine | Scraping Browser API | Authenticated proxy | |---|---|---|---| | `playwright_scraper.py` | Playwright (recommended) | yes | yes | -| `puppeteer_scraper.py` | pyppeteer | yes | yes (`page.authenticate`) | +| `puppeteer_scraper.py` | pyppeteer | yes | yes (`page.authenticate`, falling back to CDP `Fetch.authRequired` on a Chromium that lacks it; not yet run through an authenticated proxy here) | | `selenium_scraper.py` | Selenium | **no** — chromedriver's `debuggerAddress` has nowhere to put a password; refused with exit 2 | **no** — credentials are stripped, with a warning | | `scraper_api_client.py` | none (2Captcha Scraper API) | via `--use-cdp` | — | diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 4977005..5f82b61 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -10,7 +10,7 @@ exit code: it says which of these you are in. | 2 | bad usage (the message says what) | [Usage errors](#usage-errors) | | 3 | refused or challenged | [Exit 3](#exit-3-refused) | | 4 | the site answered, and the answer was "no products" | [Exit 4](#exit-4-no-products) | -| 5 | the page was never obtained | [Exit 5](#exit-5-never-obtained) | +| 5 | the page was never obtained, or was served and parsed to nothing | [Exit 5](#exit-5-never-obtained) | | 6 | some pages gathered, then the run stopped | [Exit 6](#exit-6-partial) | | 1 | a crash — please report it | [Crashes](#crashes) | @@ -63,6 +63,12 @@ the empty files if your pipeline wants them. ## Exit 5: never obtained +**`The page was served and parsed to 0 rows (served_but_unparsed)`** — the +page arrived and held the listing, and this parser read nothing from it. +That is a parser problem, not an empty catalogue: re-run with `--dump-html` +and report it with the file. (A listing that genuinely holds no products is +exit 4.) Until 2026-10-15 this case exited 4. + **`could not connect to --cdp-endpoint … 500`** — a Scraping Browser profile allows ONE live connection at a time, and the previous run's may not have been released yet. Measured: a run started seconds after another got 500, diff --git a/browser_bridge.py b/browser_bridge.py index ca70917..c06bad3 100644 --- a/browser_bridge.py +++ b/browser_bridge.py @@ -180,7 +180,7 @@ class PageOutcome: # The server answered with a different page than the URL asked for. served_other: bool = False # Served (the island or product links were there) and parsed to 0 rows: - # this parser's problem, not the end of the listing (§20). + # this parser's problem, not the end of the listing. parsed_nothing: bool = False # An unexpected exception while fetching this page — recorded, masked, # so the pages already gathered survive it. @@ -209,14 +209,15 @@ def counted(self) -> bool: # --------------------------------------------------------------------------- # Captcha # --------------------------------------------------------------------------- -def handle_captcha_if_present(driver, args) -> bool: +def handle_captcha_if_present(driver, args, budget=None) -> bool: """Detect and solve a reCAPTCHA. True if something was solved. - Called ONCE per fetch attempt, and a fetch has at most - `block_retries + 1` attempts — so a page can buy at most that many - solves, and that is the whole budget. (A sibling repo called its solver - twice per attempt and counted only one call; one page bought three - Turnstile solves.) + `budget` is the PAGE's allowance (page_flow.SolveBudget), created once + per page by fetch_one_page and shared by every attempt of that page, so + a block retry or an exit rotation never buys another solve. It is + checked before the purchase, and spent when the solver is called (a + failed solve can still be billed). No budget given means a fresh + one-solve allowance for this call. Kohl's renders reCAPTCHA v3 on its account pages (/signin); none was found on any listing, search or product page captured. This path exists @@ -249,6 +250,13 @@ def handle_captcha_if_present(driver, args) -> bool: logger.warning("No 2captcha API key is configured, so no solve is " "attempted — continuing with whatever the page already holds.") return False + if budget is None: + budget = page_flow.SolveBudget() + if not budget.allows(): + logger.warning("This page has already used its %d paid solve(s) — not " + "buying another on a retry or a new exit.", budget.limit) + return False + budget.spend() try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -309,6 +317,9 @@ def fetch_one_page(driver, args, pool, page_num: int, url: str) -> PageOutcome: # jar, same address) — see page_flow for why that number is small. block_retries = args.proxy_block_retries if rotating else BLOCK_RETRIES_WITHOUT_POOL html, state, status, load_failed = "", "blocked", None, False + # One allowance for the whole page: outside the loop, so neither a block + # retry nor a rotation to another exit resets it. + solve_budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d: %s", page_num, url) @@ -343,7 +354,7 @@ def fetch_one_page(driver, args, pool, page_num: int, url: str) -> PageOutcome: if load_failed: break - if handle_captcha_if_present(driver, args): + if handle_captcha_if_present(driver, args, solve_budget): driver.sleep(1000) status = None # the solve navigated; the old status is stale diff --git a/captcha_solver.py b/captcha_solver.py index bedff75..9bad4f1 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -577,8 +577,8 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st api_version: str = "v2", min_score: float = 0.7) -> str: """Public entry point: solve `challenge` through 2captcha and return the token. - There used to be a `use_antidetect` branch here, behind a CLI flag of the - same name, that POSTed to a hardcoded local "solve-captcha" endpoint. + There used to be a second branch here, behind a CLI flag, that POSTed to + a hardcoded local "solve-captcha" endpoint. It was removed before publishing: that endpoint was a placeholder for a product that does not exist under that name, so the flag could not work for anyone who set it, and the real "the browser solves it for you" path is diff --git a/fingerprint_client.py b/fingerprint_client.py index a687c11..181aa92 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -257,7 +257,7 @@ def playwright_context_kwargs(fp: dict) -> dict: # 2026-09-11: a fingerprint stating `deviceScaleFactor: 1.25` produced a # browser reporting `window.devicePixelRatio === 1`, so the identity # contradicted itself on an axis any fingerprinter reads for free. Same - # shape as the three defects §16 lists -- a key the API returns that + # shape as other defects found in copied code -- a key the API returns that # nothing applies -- and found the same way, by comparing the live # browser against the fingerprint rather than by reading the code. scale = (fp.get("screen") or {}).get("deviceScaleFactor") @@ -398,7 +398,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as README and .env.example instruct worked for every # engine and failed here with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, a defect shape that copied code keeps producing. # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, which `load_env` is what fills from the file. Calling it # here rather than relying on an engine having called it is the whole diff --git a/output_writer.py b/output_writer.py index a60bb5b..f9da2d3 100644 --- a/output_writer.py +++ b/output_writer.py @@ -29,6 +29,7 @@ import csv import json import os +import stat import tempfile from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone @@ -194,6 +195,24 @@ def _csv_value(v: Any) -> Any: return v +# A spreadsheet opening the CSV executes a cell that begins with one of these +# as a formula, and every string in these rows was written by whoever wrote +# the listing, not by us. Such a cell gets a leading apostrophe in the CSV +# only; the JSON keeps the site's exact bytes. Only STRINGS are touched (a +# numeric -5 stays a number, so sums over the column still work), and the +# check runs AFTER list serialisation, because a joined list is text too and +# its first element can be formula-shaped. +CSV_FORMULA_LEADS = ("=", "+", "-", "@", "\t", "\r", "\n") +CSV_FORMULA_ESCAPE = "'" + + +def _csv_escape(v: Any) -> Tuple[Any, bool]: + """Neutralise a formula-shaped cell. Returns (value, was_escaped).""" + if isinstance(v, str) and v.startswith(CSV_FORMULA_LEADS): + return CSV_FORMULA_ESCAPE + v, True + return v, False + + # --------------------------------------------------------------------------- # Atomic publication # --------------------------------------------------------------------------- @@ -216,6 +235,20 @@ def _csv_value(v: Any) -> Any: os.umask(_UMASK) +def _target_mode(path: str) -> int: + """The mode the published file should have. + + An existing file keeps its own mode (a 0640 output stays 0640 across + runs); a new one gets what a plain open() would give it under the + current umask. Never a flat value: a fixed 0644 would narrow a 0664 + file under umask 002, and mkstemp's 0600 would hide it from the group. + """ + try: + return stat.S_IMODE(os.stat(path).st_mode) + except OSError: + return 0o666 & ~_UMASK + + def _stage(path: str, write: Callable[[Any], None], newline: Optional[str] = None) -> str: """Write via `write(fileobj)` to a temp file beside `path`; return its path.""" directory = os.path.dirname(os.path.abspath(path)) @@ -227,7 +260,7 @@ def _stage(path: str, write: Callable[[Any], None], newline: Optional[str] = Non write(f) f.flush() os.fsync(f.fileno()) - os.chmod(tmp, 0o666 & ~_UMASK) + os.chmod(tmp, _target_mode(path)) except BaseException: _discard([(tmp, path)]) raise @@ -252,7 +285,8 @@ def _json_writer(rows: Sequence[Any]) -> Callable[[Any], None]: return lambda f: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def _csv_writer(rows: Sequence[Any], row_cls: Type) -> Callable[[Any], None]: +def _csv_writer(rows: Sequence[Any], row_cls: Type, + counter: Optional[List[int]] = None) -> Callable[[Any], None]: # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -262,11 +296,20 @@ def _csv_writer(rows: Sequence[Any], row_cls: Type) -> Callable[[Any], None]: # run still writes the columns of the mode that produced it. fieldnames = [f.name for f in fields(row_cls)] + # `counter`, when given, receives how many cells were formula-escaped, + # so the sidecar can declare that the CSV and the JSON differ there. def write(f): writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() + escaped = 0 for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + out = {} + for k, v in asdict(r).items(): + out[k], was = _csv_escape(_csv_value(v)) + escaped += was + writer.writerow(out) + if counter is not None: + counter.append(escaped) return write @@ -494,8 +537,12 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, def _stage_rows(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool, row_cls: Type) -> Tuple[int, List[Tuple[str, str]]]: - """Stage the row files `save` would write; return (exit code, staged pairs).""" + allow_empty: bool, row_cls: Type, + csv_escaped: Optional[List[int]] = None) -> Tuple[int, List[Tuple[str, str]]]: + """Stage the row files `save` would write; return (exit code, staged pairs). + + `csv_escaped`, when given, receives the CSV's formula-escaped cell count. + """ if not rows and not allow_empty: print(f"[!] 0 products — refusing to write {out_prefix}.json/.csv, so an " f"earlier good result isn't overwritten with an empty one. " @@ -509,7 +556,8 @@ def _stage_rows(rows: Sequence[Any], out_prefix: str, fmt: str, staged.append((_stage(path, _json_writer(rows)), path)) if fmt in ("csv", "both"): path = f"{out_prefix}.csv" - staged.append((_stage(path, _csv_writer(rows, row_cls), newline=""), path)) + staged.append((_stage(path, _csv_writer(rows, row_cls, csv_escaped), + newline=""), path)) except BaseException: _discard(staged) raise @@ -543,10 +591,13 @@ def _report_saved(rows: Sequence[Any], staged: Sequence[Tuple[str, str]]) -> Non # A page the site SERVED — the listing island was there, or product links # were — that parsed to zero rows. That is this parser's fault, not the -# catalogue's (§20), so it has its own stop reason and is NOT complete: -# mid-run it makes the run partial (exit 6). With nothing gathered at all it -# keeps the "no products" exit (4) rather than 5, because the page was -# obtained; the stop reason is what names whose problem it is. +# catalogue's, so it has its own stop reason and is NOT complete: mid-run it +# makes the run partial (exit 6), and with nothing gathered at all it is +# exit 5, not 4. Exit 4 states a fact about the catalogue ("the answer was +# nothing"), and here the page itself said there were products: a genuinely +# empty listing is classified "empty" before parsing and never lands here. +# Until 2026-10-15 this exited 4, which told a pipeline the listing was +# empty when the rows were there and unread. PARSER_STOP_REASONS = ("served_but_unparsed",) @@ -590,7 +641,8 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, complete = (stop_reason in COMPLETE_STOP_REASONS and not pages_failed and not pages_unattempted) row_cls = ROW_CLASS_BY_MODE.get(mode, Product) - rc, staged = _stage_rows(rows, out_prefix, fmt, allow_empty, row_cls) + csv_escaped: List[int] = [] + rc, staged = _stage_rows(rows, out_prefix, fmt, allow_empty, row_cls, csv_escaped) wrote_output = bool(rows) or allow_empty if wrote_output: @@ -603,6 +655,15 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, outputs=[os.path.basename(final) for _, final in staged], mode=mode, source=source, listing=listing, start_url=start_url, final_url=final_url, products=len(rows)) + if csv_escaped: + # Merged UNDER the run's own keys: housekeeping never overwrites + # a field the run recorded. + meta = {"csv_cells_escaped": csv_escaped[0], **meta} + if csv_escaped[0]: + print(f"[!] {csv_escaped[0]} CSV cell(s) began with a formula " + f"character and were prefixed with " + f"{CSV_FORMULA_ESCAPE!r} in the CSV only; the JSON is " + f"unchanged — see csv_cells_escaped in the sidecar.") meta_path = _meta_path(out_prefix) try: staged.append((_stage(meta_path, _meta_writer(meta)), meta_path)) @@ -631,10 +692,10 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, return EXIT_BLOCKED if stop_reason in PARSER_STOP_REASONS: print(f"[!] The page was served and parsed to 0 rows " - f"({stop_reason}) — exit {EXIT_NO_PRODUCTS}, and this is the " - f"PARSER's problem, not an empty catalogue. Re-run with " - f"--dump-html and report it.") - return EXIT_NO_PRODUCTS + f"({stop_reason}) — exit {EXIT_FETCH_FAILED}, NOT an empty " + f"catalogue (exit {EXIT_NO_PRODUCTS}): this is the PARSER's " + f"problem. Re-run with --dump-html and report it.") + return EXIT_FETCH_FAILED if not complete: print(f"[!] Nothing was gathered and the run did not finish " f"({stop_reason}) — exit {EXIT_FETCH_FAILED}, NOT an empty " diff --git a/page_flow.py b/page_flow.py index a866b03..2a01cd8 100644 --- a/page_flow.py +++ b/page_flow.py @@ -126,7 +126,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # # There is no "solve" column: the captcha check runs on every fetch before # the state is decided (a solve can change it), and a column no engine -# consulted would be policy that reads like enforcement and is not (§17). +# consulted would be policy that reads like enforcement and is not. STATE_POLICY = { "content": {"retry": False, "blocked": False}, # A correct answer. Not retried: retrying spends the user's budget on @@ -169,6 +169,31 @@ def counts_as_blocked(state: str) -> bool: BLOCK_RETRIES_WITHOUT_POOL = 1 +# Paid captcha solves one page may buy, across ALL of its attempts. A block +# retry or an exit rotation is a reason to fetch the page again, not a fresh +# allowance: before this was enforced, one page served a challenge on every +# attempt bought 2 solves with no pool and 6 with --proxy-block-retries 5. +SOLVES_PER_PAGE = 1 + + +class SolveBudget: + """One page's allowance of paid solves, checked BEFORE each purchase. + + Created once per page, outside the retry and rotation loops, and never + reset by either. + """ + + def __init__(self, limit: int = SOLVES_PER_PAGE): + self.limit = limit + self.spent = 0 + + def allows(self) -> bool: + return self.spent < self.limit + + def spend(self) -> None: + self.spent += 1 + + def served_other_page(asked_url: str, html: str) -> bool: """True when the server answered with a DIFFERENT page than `asked_url` asked for. diff --git a/product_parser.py b/product_parser.py index 9d8643b..033d809 100644 --- a/product_parser.py +++ b/product_parser.py @@ -20,8 +20,9 @@ the SERVER actually answered with. That island is the primary source here. JSON-LD is present and is NOT the primary source, and the reason is the -trap §24 of the family template describes. Every listing page carries exactly -one `application/ld+json` block, a `CollectionPage` whose `ItemList` holds +trap of a structured block that is valid and about the wrong products. +Every listing page carries exactly one +`application/ld+json` block, a `CollectionPage` whose `ItemList` holds FIFTEEN items against the page's 120 — measured on eleven live listing captures, all fifteen, every time, and always the first fifteen payload entries INCLUDING sponsored ones. A JSON-LD-first parser would return 15 rows @@ -124,7 +125,7 @@ def host_currency(url: str) -> Optional[str]: # The numeric id, and ONLY the numeric one. The same page links to "outfit" # collections as /product/prd-c2435951/… — the same URL shape with a letter in # front of the number, matched by the selector above and describing no single -# product. They are the "junk link" §4 of the family template warns about: in +# product. They are "junk links": in # the fallback path they would become rows with a plausible title and a # neighbour's price. _SKU_IN_URL_RE = re.compile(r"/product/prd-(\d+)(?=[/.?#]|$)") @@ -338,7 +339,7 @@ def detect_page_state(html: str, status: Optional[int] = None, """One of "content", "empty", "end", "blocked", "challenge", "captcha", "unloaded". - Ordered by how much each signal PROVES, not by how cheap it is (§17): + Ordered by how much each signal PROVES, not by how cheap it is: 1. the site's structured data — content, or an honestly empty listing (only a served page carries the island or the product model); @@ -357,8 +358,8 @@ def detect_page_state(html: str, status: Optional[int] = None, return "unloaded" # The site's own structured data first: a page carrying the listing # island or the product model was SERVED, whatever strings it also holds - # — a marker may only refine the reason for a page already given up on - # (§18), never overrule the data. No refusal capture carries either. + # — a marker may only refine the reason for a page already given up on, + # never overrule the data. No refusal capture carries either. data = catalog_data(html) if data is not None: products = data.get("products") or [] @@ -780,7 +781,7 @@ def _parse_url_fallback(html: str, base_url: str, label: Optional[str]) -> List[ soup = BeautifulSoup(html or "", "html.parser") # The page's JSON-LD first. Failing that, the host's currency — and that # is the "$" the tile itself prints (every amount this path reads is - # matched with its "$"), read as USD on a US-only shop: §4's bare-symbol + # matched with its "$"), read as USD on a US-only shop: the bare-symbol # tier, a guess the site's own symbol supports, not a compiled default. currency = page_currency(html) or host_currency(base_url) page = page_number_from_url(base_url) diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 6b0649b..1f9a31d 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -185,7 +185,7 @@ def _launch_local(self): if self.pool: scrubbed, credentials = split_credentials(self.pool.current) # The address on the command line, the credentials through - # page.authenticate(): argv is readable by anything that runs `ps`. + # _install_proxy_auth(): argv is readable by anything that runs `ps`. launch_args.append(f"--proxy-server={scrubbed}") logger.info("Using proxy exit %s", mask(self.pool.current)) # handleSIGINT/TERM/HUP off: pyppeteer installs signal handlers inside @@ -211,11 +211,63 @@ def _launch_local(self): ) from None self.page = self.bridge.run(self.browser.newPage()) if credentials: - self.bridge.run(self.page.authenticate( - {"username": credentials[0], "password": credentials[1]})) + self._install_proxy_auth(*credentials) if self.args.fingerprint: self._apply_fingerprint() + def _install_proxy_auth(self, username: str, password: str): + """Answer the proxy's auth challenge. + + `page.authenticate` works on the old Chromium pyppeteer bundles but + relies on Network.setRequestInterception, which current Chromium no + longer has: a sibling repo measured it on 2026-10-05 dying with + "Protocol error (Network.setRequestInterception): ... wasn't found" + before the first navigation, an uncaught traceback, under + --chromium-path at Chromium 153. So it is tried first, and only that + one error falls back to CDP's Fetch domain with `handleAuthRequests`, + which is what Playwright uses. Anything else propagates. Not yet run + through an authenticated proxy on kohls.com. + + The fallback pauses every request and continues it unchanged; only an + auth challenge whose source is the PROXY is answered, with + credentials that never leave this process's memory. + """ + try: + self.bridge.run(self.page.authenticate( + {"username": username, "password": password})) + return + except Exception as e: # noqa: BLE001 - pyppeteer raises several types + if "setRequestInterception" not in str(e): + raise + logger.info("page.authenticate is not available on this Chromium " + "-- answering the proxy through CDP Fetch instead " + "(slower: every request is paused and continued).") + + async def _enable(): + session = await self.page.target.createCDPSession() + + def _paused(event): + asyncio.ensure_future(session.send( + "Fetch.continueRequest", {"requestId": event["requestId"]})) + + def _auth(event): + source = (event.get("authChallenge") or {}).get("source") + response = ({"response": "ProvideCredentials", + "username": username, "password": password} + if source == "Proxy" else {"response": "Default"}) + asyncio.ensure_future(session.send( + "Fetch.continueWithAuth", + {"requestId": event["requestId"], + "authChallengeResponse": response})) + + session.on("Fetch.requestPaused", _paused) + session.on("Fetch.authRequired", _auth) + await session.send("Fetch.enable", {"handleAuthRequests": True, + "patterns": [{"urlPattern": "*"}]}) + return session + + self._auth_session = self.bridge.run(_enable(), timeout=30) + def _apply_fingerprint(self): from fingerprint_client import get_fingerprint, cdp_identity_commands fp = get_fingerprint(self.args.twocaptcha_key, tags=self.args.fp_tags, diff --git a/smoke_test.py b/smoke_test.py index 756a981..8f22b2c 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -430,7 +430,7 @@ def boom(_): raise RuntimeError("navigated") ok &= check("a count that raises mid-navigation is polled, not fatal", page_flow.wait_for_count(boom, lambda ms: None, "x", 1, 0) == 0) - # A policy constant nothing reads is dead code (§17): grep the consumer. + # A policy constant nothing reads is dead code: grep the consumer. tree = ast.parse(open(os.path.join(REPO_ROOT, "browser_bridge.py"), encoding="utf-8").read()) loads = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)} for const in ("BLOCK_RETRIES_WITHOUT_POOL", "MIN_CARD_MATCHES"): @@ -671,9 +671,13 @@ def test_whole_run(): rc, meta, rows = _run(["--url", url4, "--pages", "3"], {1: (200, s4), 2: (200, unparsed)}) ok &= check("a SERVED page 2 that parses to nothing: partial (exit 6), named, not 'complete'", rc == EXIT_PARTIAL and meta["stop_reason"] == "served_but_unparsed") + # This pinned exit 4 until 2026-10-15. Exit 4 says the CATALOGUE is + # empty, and a page whose listing was served and parsed to nothing says + # the opposite; a genuinely empty listing is classified "empty" before + # parsing and keeps exit 4 (checked above). rc, meta, rows = _run(["--url", url4], {1: (200, unparsed)}) - ok &= check("...on page 1: exit 4 (the page was obtained), with its own reason in the log", - rc == EXIT_NO_PRODUCTS) + ok &= check("...on page 1: exit 5 (a parser failure, NOT an empty catalogue)", + rc == EXIT_FETCH_FAILED) rc, meta, rows = _run(["--url", url4, "--pages", "2"], {1: (200, s4), 2: (None, end)}) ok &= check("a past-the-end page with NO status (Selenium) and the asked URL: end_of_listing", rc == 0 and meta["stop_reason"] == "end_of_listing" and meta["pages_completed"] == 1 @@ -791,7 +795,7 @@ def test_output_contract(): before = {ext: open(out + ext, "rb").read() for ext in (".json", ".csv", ".meta.json")} import output_writer real_csv = getattr(output_writer, "_csv_writer", None) - def half_then_fail(rows, row_cls): + def half_then_fail(rows, row_cls, *rest): def write(f): f.write("source,scraped_at") raise OSError(28, "No space left on device") @@ -871,7 +875,7 @@ def test_captcha(): for name in ("listing_sort4_p1", "detail_croft_tee", "search_crocs_p1"): ok &= check("%s: the static detector stays quiet despite the injected " "recaptcha hunter scripts" % name, detect_recaptcha_v3(*fx(name)) is None) - # §23: a cap nothing enforces is a bill. One call site per attempt. + # A cap nothing enforces is a bill. One call site per attempt. src = open(os.path.join(REPO_ROOT, "browser_bridge.py"), encoding="utf-8").read() calls = [n for n in ast.walk(ast.parse(src)) if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "handle_captcha_if_present"] @@ -1111,7 +1115,7 @@ def test_wording(): offenders = [os.path.relpath(f, REPO_ROOT) for f in files if phrase.lower() in open(f, encoding="utf-8", errors="replace").read().lower()] ok &= check("no shipped file says %r %s" % (phrase, offenders or ""), not offenders) - # §19: never claim a captcha is beyond solving — only that this repo + # Never claim a captcha is beyond solving — only that this repo # does not implement a solver for it. claim = re.compile(r"\b(?:un" + r"solvable|can" + r"not\s+be\s+solved|can't\s+be\s+solved)\b", re.I) offenders = [os.path.relpath(f, REPO_ROOT) for f in files @@ -1170,7 +1174,7 @@ def _undefined_names(path): def _unreachable(path): - """Statements after return/raise/break/continue in the same block (§22).""" + """Statements after return/raise/break/continue in the same block.""" tree = ast.parse(open(path, encoding="utf-8").read()) found = [] for node in ast.walk(tree): @@ -1334,7 +1338,7 @@ def test_ci_checks_wired(): gh = os.path.join(REPO_ROOT, ".github") if not os.path.isdir(gh): # The Docker image deliberately carries no .github; the escape is the - # WHOLE directory being absent, never one file inside it (§22). + # WHOLE directory being absent, never one file inside it. print(" (no .github directory here — skipping, as the image build does)") return ok script = os.path.join(gh, "ci_checks.py") @@ -1371,7 +1375,7 @@ def ignored(path): ok &= check("%s is committed, not ignored" % path, not ignored(path)) for path in (".env", ".env.bak", ".env.local", "live/page.html", "out/run.json", "runs/x_page1_debug.html", "dump.page3", "kohls_products.json", - "foo_page1_debug.png"): + "foo_page1_debug.png", "page.html"): ok &= check("%s is ignored" % path, ignored(path)) return ok @@ -1495,6 +1499,283 @@ class A: return ok +# --------------------------------------------------------------------------- +# Family rules added 2026-10-15 +# --------------------------------------------------------------------------- +def test_csv_formula_guard(): + group("CSV formula cells are neutralised in the CSV only") + ok = True + import csv as _csv + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "o") + rows = [Product(sku="1", title='=HYPERLINK("http://x","y")', price=-5.0, + url="https://www.kohls.com/product/prd-1/x.jsp"), + Product(sku="2", title="@cmd", price=3.0), + Product(sku="3", title="plain", price=4.0)] + list_field = next((f.name for f in fields(Product) + if f.name not in ("title",) and isinstance( + getattr(Product(), f.name), list)), None) + if list_field: + setattr(rows[2], list_field, ["+1 formula", "b"]) + rc = quiet(finish_run, rows, out, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, pages_completed=1, + start_url="u", final_url="u") + got = list(_csv.DictReader(open(out + ".csv", newline="", encoding="utf-8"))) + meta = json.load(open(out + ".meta.json")) + js = json.load(open(out + ".json")) + ok &= check("a formula-shaped title gets a leading apostrophe in the CSV", + rc == 0 and got[0]["title"] == "'" + rows[0].title + and got[1]["title"] == "'@cmd" and got[2]["title"] == "plain") + ok &= check("a negative NUMBER is left a number", got[0]["price"] == "-5.0") + if list_field: + ok &= check("a joined list cell is escaped after serialisation (%s)" % list_field, + got[2][list_field].startswith("'+1 formula")) + import output_writer as _ow + ok &= check("a list is escaped AFTER it is joined into one cell", + _ow._csv_escape(_ow._csv_value(["+1 formula", "b"]))[0] + == "'+1 formula | b") + ok &= check("the JSON keeps the site's exact text", + js[0]["title"] == rows[0].title and js[1]["title"] == "@cmd") + expected = 2 + (1 if list_field else 0) + ok &= check("the sidecar declares csv_cells_escaped (%s)" + % meta.get("csv_cells_escaped"), meta.get("csv_cells_escaped") == expected) + ok &= check("csv_cells_escaped sits UNDER the run's own keys (status etc. intact)", + meta.get("status") == "complete" and meta.get("products") == 3) + return ok + + +def test_output_modes(): + group("published files keep an existing mode, and new ones follow the umask") + ok = True + if os.name != "posix": + return ok + import output_writer + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "o") + good = [Product(sku="1", price=1.0)] + quiet(save, good, out, "both") + os.chmod(out + ".json", 0o640) + quiet(save, good, out, "both") + ok &= check("an existing 0640 output stays 0640 after a rewrite", + (os.stat(out + ".json").st_mode & 0o777) == 0o640) + # The writer reads the umask once at import; set both the process + # umask and that cached value to a NON-default 002, the case where a + # flat 0644 and the right rule disagree. + old, cached = os.umask(0o002), output_writer._UMASK + output_writer._UMASK = 0o002 + try: + quiet(save, good, os.path.join(d, "n"), "json") + mode = os.stat(os.path.join(d, "n.json")).st_mode & 0o777 + finally: + os.umask(old) + output_writer._UMASK = cached + ok &= check("a NEW file under umask 002 is 0664, not a flat 0644 or 0600 (%o)" % mode, + mode == 0o664) + return ok + + +def test_solve_budget(): + group("paid solves: one per page, whatever the retries and rotations") + ok = True + html, _ = fx("signin_recaptcha") + _, url = fx("listing_sort4_p1") + ok &= check("(the reCAPTCHA fixture classifies as a retryable challenge)", + page_flow.should_retry(page_flow.classify(html, status=200, url=url))) + calls = [] + real = browser_bridge.solve_recaptcha + + def fake_solve(*a, **kw): + calls.append(1) + return "token" + browser_bridge.solve_recaptcha = fake_solve + try: + with tempfile.TemporaryDirectory() as pd: + plist = os.path.join(pd, "proxies.txt") + with open(plist, "w") as f: + f.write("".join("http://10.0.0.%d:8000\n" % i for i in range(1, 7))) + for label, extra in (("no pool", []), + ("6 exits, --proxy-block-retries 5", + ["--proxy-file", plist, "--proxy-block-retries", "5"])): + del calls[:] + rc, _, _ = _run(["--url", url, "--twocaptcha-key", "k" * 8] + extra, + {1: (200, html)}) + ok &= check("%s: exactly %d solve bought for one page (got %d), exit 3" + % (label, page_flow.SOLVES_PER_PAGE, len(calls)), + len(calls) == page_flow.SOLVES_PER_PAGE and rc == EXIT_BLOCKED) + finally: + browser_bridge.solve_recaptcha = real + src = open(os.path.join(REPO_ROOT, "browser_bridge.py"), encoding="utf-8").read() + ok &= check("the budget is checked BEFORE the solver is called", + 0 < src.find("budget.allows()") < src.find("token = solve_recaptcha(")) + return ok + + +# Assembled from pieces so this file does not match itself. +_PRIVATE_CITATIONS = re.compile( + "|".join([re.escape("CLAUDE" + ".md"), "claude_" + "2scraper", + "2scraper_" + "repo_rules", "play" + "book", + "family " + "template", "\u00a7"]), re.I) + + +def test_no_private_citations(): + group("no citations of private instruction files") + ok = True + if os.path.isdir(os.path.join(REPO_ROOT, ".git")): + listed = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, capture_output=True, + text=True).stdout.split() + files = [os.path.join(REPO_ROOT, f) for f in listed] + else: + files = _shipped_text_files() + offenders = [] + for path in files: + try: + text = open(path, encoding="utf-8").read() + except (OSError, UnicodeDecodeError): + continue + if _PRIVATE_CITATIONS.search(text): + offenders.append(os.path.relpath(path, REPO_ROOT)) + ok &= check("no tracked file cites a private file or section number %s" + % (offenders or ""), files and not offenders) + return ok + + +def test_canary_assert(): + group("the canary's assertions fail on a refusal, a never-obtained page and zero rows") + ok = True + path = os.path.join(REPO_ROOT, ".github", "canary_assert.py") + if not os.path.isdir(os.path.join(REPO_ROOT, ".github")): + print(" (no .github directory here — skipping, as the image build does)") + return ok + import importlib.util + spec = importlib.util.spec_from_file_location("canary_assert", path) + ca = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ca) + with tempfile.TemporaryDirectory() as d: + prefix = os.path.join(d, "canary") + rows = [dict(asdict(Product(sku=str(i), title="t", price=1.0, currency="USD", + url="https://www.kohls.com/product/prd-%d/x.jsp" % i)), + page=1 + i // 120, position=1 + i % 120) for i in range(337)] + json.dump(rows, open(prefix + ".json", "w")) + meta = {"status": "complete", "pages_completed": 3, + "listing": {"total_results": 337, "pages_available": 3, "per_page": 120}} + json.dump(meta, open(prefix + ".meta.json", "w")) + ok &= check("a good run passes", all(ca.checks(prefix, 0).values())) + for rc in (3, 4, 5): + ok &= check("exit %d is red" % rc, not all(ca.checks(prefix, rc).values())) + json.dump(rows[:150], open(prefix + ".json", "w")) + ok &= check("a parser that loses half the rows is red", + not all(ca.checks(prefix, 0).values())) + json.dump(rows[:240], open(prefix + ".json", "w")) + json.dump(dict(meta, pages_completed=2, listing={"total_results": 240, + "pages_available": 2, "per_page": 120}), open(prefix + ".meta.json", "w")) + ok &= check("a SMALLER brand (2 pages, 240 rows) is not a failure", + all(ca.checks(prefix, 0).values())) + os.remove(prefix + ".json") + ok &= check("no rows at all is red", not all(ca.checks(prefix, 0).values())) + wf = open(os.path.join(REPO_ROOT, ".github", "workflows", "canary.yml"), encoding="utf-8").read() + ok &= check("the canary fails, not skips, without KOHLS_CDP_ENDPOINT", + "::error::Missing secret KOHLS_CDP_ENDPOINT" in wf and "::notice::" not in wf) + ok &= check("the canary runs these assertions", "canary_assert.py" in wf) + return ok + + +def test_pyppeteer_proxy_auth(): + """page.authenticate needs Network.setRequestInterception, which current + Chromium no longer has (measured in a sibling repo on 2026-10-05: exit 1 + and a traceback before the first navigation). Only that error falls back + to CDP Fetch; only a PROXY challenge gets the credentials.""" + group("pyppeteer: proxy auth falls back to CDP Fetch") + ok = True + try: + module = importlib.import_module("puppeteer_scraper") + except ImportError: + print(" (pyppeteer absent — covered in its engine-smoke venv)") + return ok + import asyncio + + class FakeSession: + def __init__(self): + self.sent, self.handlers = [], {} + + def on(self, name, fn): + self.handlers[name] = fn + + async def send(self, method, params=None): + self.sent.append((method, params or {})) + + def make(authenticate_error): + sess = FakeSession() + + class Target: + async def createCDPSession(self_inner): + return sess + + class Page: + target = Target() + calls = 0 + + async def authenticate(self_inner, creds): + Page.calls += 1 + if authenticate_error is not None: + raise authenticate_error + + class Bridge: + def run(self_inner, coro, timeout=None): + return asyncio.new_event_loop().run_until_complete(coro) + + obj = object.__new__(module.PuppeteerDriver) + obj.page, obj.bridge = Page(), Bridge() + return obj, Page, sess + + def attempt(fn): + try: + fn() + return None + except Exception as exc: # noqa: BLE001 + return exc + + obj, Page, sess = make(None) + err = attempt(lambda: obj._install_proxy_auth("u", "p")) + ok &= check("a Chromium with page.authenticate uses it and nothing else", + err is None and Page.calls == 1 and not sess.sent) + missing = Exception("Protocol error (Network.setRequestInterception): " + "'Network.setRequestInterception' wasn't found") + obj, Page, sess = make(missing) + err = attempt(lambda: obj._install_proxy_auth("u", "p")) + ok &= check("setRequestInterception missing falls back instead of raising", err is None) + enable = [p for m, p in sess.sent if m == "Fetch.enable"] + ok &= check("and enables Fetch with handleAuthRequests", + len(enable) == 1 and enable[0].get("handleAuthRequests") is True) + + async def drive(): + def fire(name, event): + handler = sess.handlers.get(name) + if handler is not None: + handler(event) + fire("Fetch.authRequired", {"requestId": "r1", "authChallenge": {"source": "Proxy"}}) + fire("Fetch.authRequired", {"requestId": "r2", "authChallenge": {"source": "Server"}}) + fire("Fetch.requestPaused", {"requestId": "r3"}) + await asyncio.sleep(0) + await asyncio.sleep(0) + asyncio.new_event_loop().run_until_complete(drive()) + auth = {p["requestId"]: p["authChallengeResponse"] + for m, p in sess.sent if m == "Fetch.continueWithAuth"} + ok &= check("a PROXY challenge is answered with the credentials", + auth.get("r1") == {"response": "ProvideCredentials", + "username": "u", "password": "p"}) + ok &= check("a SERVER challenge is not answered with them", + auth.get("r2") == {"response": "Default"}) + ok &= check("every other request is continued unchanged", + ("Fetch.continueRequest", {"requestId": "r3"}) in sess.sent) + obj, Page, sess = make(Exception("Target closed")) + err = attempt(lambda: obj._install_proxy_auth("u", "p")) + ok &= check("any other authenticate error still propagates", + err is not None and "Target closed" in str(err) and not sess.sent) + ok &= check("the driver installs auth through _install_proxy_auth only", + inspect.getsource(module).count("self.page.authenticate(") == 1) + return ok + + def main() -> int: import logging # The engines log every fetch at INFO; here that would bury the PASS/FAIL @@ -1509,7 +1790,9 @@ def main() -> int: test_fingerprint_helpers, test_wording, test_fixture_corpus_is_scrubbed, test_static_analysis, test_shared_calls_bind, test_dockerfile, test_sample_output, test_ci_checks_wired, test_gitignore, test_scraper_api_payload, - test_no_env_read_on_help): + test_no_env_read_on_help, test_csv_formula_guard, test_output_modes, + test_solve_budget, test_no_private_citations, test_canary_assert, + test_pyppeteer_proxy_auth): ok &= t() ok &= test_engines(skips) print()