Report markup drift as a parser failure; typed query flags win over a .env URL

.env.example

@@ -54,8 +54,8 @@ TWOCAPTCHA_KEY=your_2captcha_api_key_here
 # Selenium cannot use this: chromedriver's `debuggerAddress` takes a bare
 # host:port with nowhere to put a password. Playwright and pyppeteer can.
 #
-# The credentials in a profile expire in about a day, so never paste a
-# working one anywhere that is committed.
+# The credentials in a profile expire (a 401 on connect means fetch fresh
+# ones), so never paste a working one anywhere that is committed.
 HACKERNEWS_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-de-pid-{profileId}:{password}@cb.2captcha.com:9222
 
 # --- Proxy ----------------------------------------------------------------
@@ -80,8 +80,10 @@ HACKERNEWS_PROXY=http://{user}:{password}@eu.proxy.2captcha.com:2334
 #   https://news.ycombinator.com/item?id=49901736           a comment tree
 #   https://news.ycombinator.com/user?id=dang               a profile
 #
-# The flags (--feed, --day, --item, --user, ...) do the same without a URL.
-HACKERNEWS_URL=https://news.ycombinator.com/news
+# The flags (--feed, --day, --item, --user, ...) do the same without a URL,
+# and a flag you type wins over this default. Commented out so a copied
+# example changes nothing; uncomment it to set a default target.
+# HACKERNEWS_URL=https://news.ycombinator.com/news
 
 # Deliberately NOT here: an output prefix. `--out` already carries a
 # non-empty default, so the loader, which only fills UNSET values, would

.github/ci_checks.py

@@ -88,7 +88,7 @@
 # A 2captcha API key is a 32-character hex string.
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 
-# EMPTY, and that is the stricter arrangement rather than an omission (§24).
+# EMPTY, and that is the stricter arrangement rather than an omission.
 #
 # Nothing news.ycombinator.com publishes is a 32-hex string: its ids are
 # decimal integers (`item?id=49901736`), its usernames are words, and the two
@@ -210,8 +210,8 @@ def _without_site_ids(line):
 #
 # A suffix allowlist is a scanner that cannot see the thing most likely to
 # leak. `--dump-html live_results` writes `live_results.page1` — no suffix
-# the list knew — and a merge committed two of them, 1.5 MB each, carrying 26
-# per-seller UUIDs, 12 copies of the site's Algolia key and its Sentry keys.
+# the list knew — and in a sibling repo a merge committed two of them, 1.5 MB
+# each, carrying that site's front-end keys.
 # The check that exists to stop exactly that ran, passed, and never opened
 # them.
 #
@@ -273,7 +273,7 @@ def tracked_files():
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is

.github/workflows/canary.yml

@@ -3,7 +3,7 @@
 #
 # No secrets are needed, and that is the point of running it: the README's
 # central claim is "no key, no proxy, no account", and an ungated scheduled
-# canary is that sentence under test every morning (CLAUDE.md §21, §24). Every
+# canary is that sentence under test every morning. Every
 # page kind answered a datacentre address normally when this repo was written
 # (2026-09-30); a GitHub runner is a datacentre address too. So this is NOT
 # gated on a secret: a canary that CAN pass without a credential must not be
@@ -83,7 +83,7 @@ jobs:
           playwright install --with-deps chromium
 
       # >= 3 pages on the listings, because with one page pagination is never
-      # exercised (§11).
+      # exercised.
       - name: Run
         id: run
         run: |
@@ -100,8 +100,8 @@ jobs:
           case "$code" in
             0) ;;
             3) echo "::error::Blocked (HTTP 403). HN has never refused an address this repo used, so this is a new state: the runner's address was refused. Set the HACKERNEWS_PROXY secret (see the header)."; exit 1 ;;
-            4) echo "::error::Zero rows from a page that is never empty. Either the markup has moved (see canary_dump.html*) or the site answered with its own 'nothing there' sentence."; exit 1 ;;
-            5) echo "::error::The page never arrived. This job uses no credential, so this is not an expired key: a timeout or a throttle that did not lift. Check the log."; exit 1 ;;
+            4) echo "::error::Zero rows from a page that is never empty: the site answered with its own 'nothing there' sentence, or a page past the end (see canary_dump.html*)."; exit 1 ;;
+            5) echo "::error::No rows were obtained. Either the page never arrived (this job uses no credential, so not an expired key: a timeout or a throttle that did not lift) or it arrived and parsed to none (stop_reason parser_found_nothing: the markup moved, see canary_page*_debug.html). Check the log."; exit 1 ;;
             6) echo "::error::Partial run: a page failed after others succeeded. The sidecar names it."; exit 1 ;;
             *) echo "::error::Unexpected exit $code."; exit 1 ;;
           esac

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         # 3.9 is the floor the README claims. Claiming it without testing it is
         # how a walrus operator or a `X | None` annotation ships and breaks it.
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9", "3.14"]
 
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -23,7 +23,7 @@ hackernews_rows_*.*
 live_results/
 live_results.zip
 # ANY working directory a live run writes into, by KIND rather than by the one
-# name an older repo used (CLAUDE.md §24). Measured here: `live/` was not
+# name an older repo used. Measured here: `live/` was not
 # listed, and a `--dump-html` of a 3-page run wrote `canary_dump.json.page3`,
 # which `*.json` does not cover. It sat untracked and NOT ignored, one
 # `git add -A` from a commit, until the credential scan flagged the 32-hex
@@ -101,7 +101,7 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# and the fix never reached anyone else, which is the lesson: when you fix one,
+# check the siblings.
 .env*
 !.env.example

CHANGELOG.md

@@ -9,8 +9,36 @@ said so at the top of its release notes.
 
 ## [Unreleased]
 
+> **A page that says it has rows and parses to none now exits 5 (or 6), not
+> 4 or 0.** When the markup moves, a page can still carry its own More link or
+> a comment count while the parser reads nothing from it. That used to be
+> reported as an empty answer: exit 4 on page 1, and on a later page the end
+> of the list, so a three-page run ended `complete`, exit 0, with a third of
+> the data. Such a page is now a failed page with `stop_reason:
+> parser_found_nothing`: exit 5 when nothing was gathered, exit 6 (partial)
+> when earlier pages had rows. The site's own "nothing here" sentences and a
+> real page past the end of a list are still exit 4 / end of list.
+
 ### Fixed
 
+- **`cp .env.example .env` no longer breaks every query flag.** The example
+  set `HACKERNEWS_URL`, and a URL from the environment was treated as typed,
+  so `--feed show`, `--item 1` and `--mode user --user dang` were all refused
+  with exit 2. The example now ships it commented out, and in every entry
+  point a URL from the environment is a default: a typed query flag or
+  `--mode` wins over it. A typed `--url` with a query flag is still refused.
+- **More key shapes are masked in logged errors and the Scraper API's
+  `x-debug` header**: `clientKey: …` and JSON `"key": "…"` as well as
+  `key=…`.
+- Documentation: a Scraping Browser profile's credentials expire, but not on
+  a fixed one-day schedule; a 401 on connect is the signal to fetch fresh
+  ones. The exit-code table now says Selenium, which sees no HTTP status,
+  reports an unrecognised 403 page as exit 5 rather than 3.
+- Comments and docstrings state their reasons in their own words instead of
+  pointing at an unpublished document, and a check keeps it that way. Stale
+  comments describing other sites' columns were reworded.
+- CI tests the newest Python (3.14) beside the oldest (3.9).
+
 - **`--retries 0` sent no request and reported an unreadable page.** The flag
   counts attempts and the loop is `while attempt < retries`, so zero attempts
   fetched nothing and the run exited 5 with an empty debug file, which reads

README.md

@@ -217,10 +217,10 @@ long link. `status` is `ok`, `flagged`, `dead` or `deleted`.
 | 0 | ok |
 | 1 | crash |
 | 2 | bad usage (a flag combination that would scrape something nobody asked for) |
-| 3 | blocked: HTTP 403 (not observed here) |
-| 4 | zero rows: nothing there, a list past its end, or a served page that parsed to none (that last one is a bug in this repo and is logged as one) |
-| 5 | the content was never obtained: a timeout, a dead proxy, a throttle that never lifted, a remote API error |
-| 6 | partial: some pages arrived and then the run stopped; `pages_failed` in the sidecar names them |
+| 3 | blocked: HTTP 403 (not observed here). Selenium sees no HTTP status, so a 403 whose body is not HN's own page reaches it as an unreadable page and ends exit 5 instead |
+| 4 | zero rows: nothing there, or a list that starts past its end |
+| 5 | the content was never obtained: a timeout, a dead proxy, a throttle that never lifted, a remote API error, or a served page that says it has rows (a More link, a comment count) and parsed to none (`stop_reason: parser_found_nothing`, a bug in this repo) |
+| 6 | partial: some pages arrived and then the run stopped, including on a later page that parsed to none; `pages_failed` in the sidecar names them |
 
 `<out>.meta.json` records `status` (`complete`, `partial`, `failed`),
 `stop_reason`, `pages_requested`, `pages_completed`, `pages_failed`, the query,
@@ -265,8 +265,8 @@ secret in `argv` is readable by anything that can run `ps`). Copy
 Precedence: explicit flag, then exported variable, then `.env`, then default.
 `python3 env_config.py` prints what was picked up without printing a secret.
 
-A Scraping Browser profile's credentials last about a day, so none is written
-down anywhere in this repo; the endpoint's shape is in `.env.example`.
+A Scraping Browser profile's credentials expire (a 401 on connect means it is
+time to fetch fresh ones), so none is written down anywhere in this repo; the endpoint's shape is in `.env.example`.
 
 ## Tests, CI and the container
 

TROUBLESHOOTING.md

@@ -24,7 +24,7 @@ report with the sidecar's `stop_reason` is welcome.
 
 ## Exit 4 — zero rows
 
-One of three things, and only the last is a bug:
+One of two things:
 
 * **There is nothing there.** `No such item.`, `No such user.`,
   `We don't have that data yet.` and `HN didn't exist yet.` are plain-text
@@ -34,23 +34,26 @@ One of three things, and only the last is a bug:
   and `?p=27` is empty. A run that starts inside the list and reaches the end
   is `complete`, exit 0; exit 4 is only for a FIRST page that is already empty,
   such as `--url ".../news?p=30"`.
-* **A served page parsed to no rows.** That is a parser bug or a markup
-  change. Re-run with `--dump-html page.html` (it writes the exact bytes the
-  parser saw, on success too) and open a "Site changed" issue.
-
 Nothing is written on exit 4, so an earlier good file is left alone;
 `--allow-empty` writes the empty file.
 
 ## Exit 5 — the content was never obtained
 
+* **A served page parsed to no rows** (`stop_reason: parser_found_nothing`).
+  The page still says it has content (rows in its markup, a More link, a
+  comment count above zero, a profile page) and the parser read none of it:
+  a parser bug or a markup change, not an empty list. The page is saved as
+  `<out>_pageN_debug.html`; re-run with `--dump-html page.html` if you want
+  the bytes on success too, and open a "Site changed" issue. On a later page
+  of a run that already has rows, the same thing is exit 6.
 * **A timeout or a dead proxy.** The log names which. A dead proxy wants a
   different exit; a timeout wants another try at the same one.
 * **A throttle that did not lift.** HN is documented to throttle by address
   with HTTP 503; that was NOT OBSERVED in testing (about fifty requests
   spaced 4 s apart drew none, one measurement). Raise `--delay`, lower
   `--concurrency`, or spread the run over `--proxy-file`.
 * **HTTP 401 from a Scraping Browser endpoint.** The profile's credentials
-  have expired; they last about a day. Get fresh ones.
+  have expired. Get fresh ones.
 * **HTTP 500 `profile_locked` from a Scraping Browser endpoint.** The profile
   has one live connection and something still holds it, including a run that
   disconnected a second ago. Wait, or use another `pid`.

diff_runs.py

@@ -62,8 +62,7 @@
 # ARE the story's figures.) `depth` and `parent_id` never change for a
 # comment, and stay tracked so a re-parenting would show.
 UNTRACKED_FIELDS = frozenset({
-    "source", "scraped_at", "sku", "page", "position", "rank", "mode", "sort",
-    "data_source", "url",
+    "source", "scraped_at", "sku", "page", "position", "rank", "url",
 })
 
 

env_config.py

@@ -67,8 +67,8 @@
 # A literal-only check reported both of those as CONFIGURED, so `cp
 # .env.example .env` followed by a run connected to cb.2captcha.com with the
 # string `{login}-zone-…` as its username and got a 401 — a confusing failure
-# a long way from its cause, which is precisely what §3's rule exists to
-# prevent. Any `{…}` left in a value means the line was never filled in.
+# a long way from its cause, which is precisely what the placeholder rule
+# exists to prevent. Any `{…}` left in a value means the line was never filled in.
 _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}")
 
 _loaded_from = None
@@ -204,7 +204,7 @@ def _dests_given_on_argv(dests, argv=None):
     first-class, as they are here: the site needs neither, so switching
     between them is routine rather than exceptional.
 
-    Scanning argv is the smallest fix that keeps §3's precedence intact --
+    Scanning argv is the smallest fix that keeps the documented precedence intact --
     explicit flag, then exported variable, then `.env`, then default -- with
     "explicit" now meaning what the caller typed rather than what happens to
     be truthy.

fingerprint_client.py

@@ -88,12 +88,16 @@
 # Everything raised or logged from this module goes through here first. The
 # endpoint and the status survive, because which call failed is the useful
 # half and is not the secret.
+# `key=value` in a URL, and `clientKey: value` / `"key": "value"` in a
+# header or a JSON body: the separator and any quotes are kept, the value is
+# masked.
 _KEY_IN_TEXT_RE = re.compile(
-    r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE)
+    r"((?:client)?key|token|api[_-]?key)(\"?\s*[:=]\s*\"?)([^&\s'\",}]{6,})",
+    re.IGNORECASE)
 
 
 def _redact(text) -> str:
-    return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text))
+    return _KEY_IN_TEXT_RE.sub(r"\1\2***", str(text))
 
 API_BASE = "https://api.2captcha.com"
 RANDOM_URL = f"{API_BASE}/fingerprint/random"
@@ -247,8 +251,8 @@ def playwright_context_kwargs(fp: dict) -> dict:
     # option and which was previously dropped on the floor. Measured
     # 2026-09-11: a fingerprint stating `deviceScaleFactor: 1.25` produced a
     # browser reporting `window.devicePixelRatio === 1`, so the identity
-    # contradicted itself on an axis any fingerprinter reads for free. Same
-    # shape as the three defects §16 lists -- a key the API returns that
+    # contradicted itself on an axis any fingerprinter reads for free. A
+    # familiar shape -- a key the API returns that
     # nothing applies -- and found the same way, by comparing the live
     # browser against the fingerprint rather than by reading the code.
     scale = (fp.get("screen") or {}).get("deviceScaleFactor")
@@ -350,7 +354,7 @@ def main() -> int:
     # loader. It used to read `os.environ` alone, which meant a key put in
     # `.env` exactly as README and .env.example instruct worked for every
     # engine and failed here with "No API key" — a documented mechanism not
-    # applied on one path, which is the shape of half the defects §16 lists.
+    # applied on one path, a familiar shape of defect.
     # `.env` has to be LOADED before it can be read: `env_value` looks at
     # os.environ, which `load_env` is what fills from the file. Calling it
     # here rather than relying on an engine having called it is the whole

make_fixtures.py

@@ -7,7 +7,7 @@
 `captures/` holds raw pages fetched from news.ycombinator.com on 2026-09-30
 (plain `curl`, and one browser-serialised copy per page kind, see --render).
 It is NOT committed: a raw page dump carries the session that fetched it and
-strangers' words (CLAUDE.md §10). This script turns each capture into a
+strangers' words. This script turns each capture into a
 smaller, scrubbed fixture and PROVES the trimmed copy parses the way the
 original did, so the suite tests the markup of the real site and not a
 hand-written imitation of it.
@@ -249,7 +249,7 @@ def build(captures, render):
             # Chromium's own network-error page, produced for real: a proxy
             # that refuses the connection. It is NOT the site's page, and
             # nothing in it names a vendor, so only "is it built out of the
-            # site's own markup" tells it from a served page (§18).
+            # site's own markup" tells it from a served page.
             browser = pw.chromium.launch(proxy={"server": "http://127.0.0.1:9"})
             page = browser.new_page()
             try: