CSV formula escaping, JUSTJOIN_URL as a default, facets diff, scan from any directory

.env.example

@@ -60,7 +60,7 @@
 #
 #
 # WORTH KNOWING ABOUT CAPTCHAS HERE, because "none rendered" is not "none
-# configured" (CLAUDE.md §18).
+# configured".
 #
 # justjoin.it publishes its own reCAPTCHA **v2** key in the page config on
 # every page:
@@ -148,7 +148,12 @@ JUSTJOIN_PROXY=http://{user}:{password}@pl.proxy.2captcha.com:2334
 # `x-api-version` header its own bundle sends. The route that works is the
 # same-origin proxy at justjoin.it/api/candidate-api/, and the scraper
 # refuses an api.justjoin.it URL with that reason rather than a generic one.
-JUSTJOIN_URL=https://justjoin.it/job-offers/all-locations
+#
+# Shipped commented out: every mode already has its own default, and a
+# listings URL set here would otherwise stand in for --url on every run.
+# A URL set here that names a different mode than --mode is ignored with a
+# log line; a URL you type with --url is still checked against --mode.
+# JUSTJOIN_URL=https://justjoin.it/job-offers/all-locations
 
 # Deliberately NOT here: an output prefix. `--out` already carries a
 # non-empty default, so the loader — which only fills UNSET values — would

.github/ci_checks.py

@@ -81,7 +81,7 @@
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 # Contexts in which a 32-hex string is plainly not a key. Site-specific
 # entries belong here only once a committed fixture actually carries one —
-# CLAUDE.md §16: a family-wide entry copied without checking is how dead
+# A family-wide entry copied without checking is how dead
 # code spreads. Kept as a CONTEXT allowlist rather than by loosening the
 # pattern: a bare 32-hex string anywhere else still fails, which is the
 # point of the check.
@@ -107,7 +107,7 @@ def _is_virtualenv(path):
     walked into pip's vendored code and flagged a 32-hex string in
     `_elffile.py` as key-shaped. The run was correct about the string and
     wrong about the file, and a guard people have to argue with is one they
-    learn to suppress (CLAUDE.md §22).
+    learn to suppress.
 
     Structural rather than by name, for the same reason a parser anchors on a
     URL pattern instead of a CSS class: `venv`, `.venv`, `.v`, `env39` and
@@ -128,7 +128,10 @@ def scanned_files():
             continue
         if path.suffix not in SCANNED_SUFFIXES:
             continue
-        if any(part in _SKIP_NAMES for part in path.parts):
+        # RELATIVE parts: the absolute path includes every ancestor, so a
+        # clone under ~/build, /srv/env or a directory called venv matched
+        # here on every file, scanned nothing and reported clean.
+        if any(part in _SKIP_NAMES for part in path.relative_to(REPO).parts):
             continue
         if any(root in path.parents for root in skip_roots):
             continue

.github/workflows/canary.yml

@@ -5,9 +5,9 @@
 #
 # Most canaries in this family skip when a proxy secret is absent, because
 # their site refuses a datacenter address and a GitHub runner is one — and a
-# check that is always red teaches everyone to ignore checks (CLAUDE.md §11).
+# check that is always red teaches everyone to ignore checks.
 #
-# justjoin.it is the other case, and CLAUDE.md §21 states the rule for it: a
+# justjoin.it is the other case, and the rule for it is: a
 # canary that CAN pass without a credential should never be gated on one.
 # Measured 2026-09-18 from a bare netcup datacentre address, with no key and
 # no proxy: the offers endpoint 200 with 100 offers, an offer's own record
@@ -23,9 +23,8 @@
 # limit, the badge goes red the next morning and the claim is retested
 # without anyone having to remember to.
 #
-# There is no secret anywhere in this file, which also means there is
-# nothing here that can rot: a Scraping Browser profile's credentials live
-# about a day, so a scheduled job holding one is stale between runs.
+# There is no secret anywhere in this file, so there is nothing here that
+# can expire between runs.
 
 name: canary
 
@@ -58,7 +57,7 @@ jobs:
 
       # THREE pages, not one. With one page pagination is never exercised,
       # which is exactly how a silent single-page bug survived in a sibling
-      # repo (CLAUDE.md §11). A filtered category is used rather than the
+      # repo. A filtered category is used rather than the
       # whole board so the result set is under the site's 10,000 cap and two
       # runs are comparable as a census.
       - name: Run --mode listings (3 pages, filtered)
@@ -204,7 +203,7 @@ jobs:
                    "a monthly figure came out below its hourly rate")
 
           # No row may carry a salary of zero: the site's own "Undisclosed
-          # Salary" must reach the output as null (CLAUDE.md §21).
+          # Salary" must reach the output as null.
           want("an undisclosed salary is null, never zero",
                not any(r.get("salary_min") == 0 or r.get("salary_max") == 0
                        for r in rows),
@@ -232,7 +231,7 @@ jobs:
                "an id came back in an unexpected shape")
           # page+position must be unique ACROSS pages — position restarts at
           # 1 on each, so without the page number beside it the column is
-          # worthless (CLAUDE.md §18).
+          # worthless.
           want("page+position unique",
                len({(r["page"], r["position"]) for r in rows}) == len(rows),
                "collisions — the page number is not reaching the parser")

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         # 3.9 is the floor the README claims. Claiming it without testing it is
         # how a walrus operator or a `X | None` annotation ships and breaks it.
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v4
@@ -59,7 +59,7 @@ jobs:
         # check still imported the row dataclass by a name this repo renamed,
         # and it failed on this repo's FIRST push while the script it
         # duplicates passed — two sources of truth, one of them stale, which
-        # is exactly the defect CLAUDE.md §17 describes.
+        # is exactly the dead-check defect this guards against.
         #
         # So: one implementation, and `smoke_test.py` now asserts the workflow
         # does not grow another one.
@@ -150,7 +150,7 @@ jobs:
         # version of this step looked for "<engine>_scraper could not be
         # imported", a string no suite in this family emits, so the job could
         # never fail and passed for the wrong reason — which is the whole
-        # thing §10 says this job exists to prevent ("skipped, engine absent"
+        # thing this job exists to prevent ("skipped, engine absent"
         # reading identically to a real import error). Checked 2026-09-18:
         # seven sibling repos carry the same dead grep.
         #

.gitignore

@@ -24,6 +24,13 @@ live_results.zip
 *_debug.html
 *_debug.png
 *.html.page*
+# --dump-html X --pages 3 writes X.page2, X.page3 whatever X ends in, so a
+# .json or .txt dump name is not covered by the patterns above.
+*.page[0-9]*
+# Scratch captures by KIND rather than by one chosen name, and a bare HTML
+# dump at the repo root.
+live/
+/*.html
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -73,7 +80,7 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
+# and the fix never reached anyone else, which is the rule unapplied: when
 # you fix one, check the siblings.
 .env*
 !.env.example

CHANGELOG.md

@@ -11,8 +11,48 @@ burying it.
 
 ## [Unreleased]
 
+> **Behaviour changes.** CSV output now prefixes a cell that starts with
+> `=`, `+`, `-`, `@`, a tab, CR or LF with an apostrophe (JSON is
+> unchanged), and the sidecar gains `csv_cells_escaped`. `.env.example`
+> no longer sets `JUSTJOIN_URL`; a `JUSTJOIN_URL` from the environment that
+> names a different mode than `--mode` is now ignored with a log line
+> instead of exiting 2.
+
 ### Fixed
 
+- **CSV cells are formula-escaped.** Every string in a row is written by an
+  employer, and a spreadsheet runs a cell starting with `=`, `+`, `-` or
+  `@` as a formula. Escaped in the CSV only, after list cells are joined;
+  numbers stay numbers and JSON keeps the site's bytes. The count goes in
+  the sidecar as `csv_cells_escaped`, under any caller-provided field of
+  the same name.
+- **`JUSTJOIN_URL` no longer breaks `--mode offer` and `--mode facets`.**
+  The listings URL that `.env.example` shipped was read as if `--url` had
+  been typed, so a copied example made both modes exit 2. The engines now
+  record what was typed before reading the environment; only a typed URL is
+  checked against `--mode`. The example now shows the variable commented
+  out.
+- **`diff_runs.py` sees changes in a facets run.** It compared the job
+  columns on facet rows, which have none of them, so an `offer_count`
+  going from 100 to 250 reported "0 changed". Facets runs are now compared
+  on `title`, `offer_count`, `is_cumulative` and `overlaps_other_keys`.
+- **The credential scan works from any parent directory.** Its skip list
+  was matched against the absolute path, so a clone under a directory
+  named `build`, `env` or `venv` scanned 0 files and reported clean. It now
+  matches the path inside the repo, and the suite runs the scan from such a
+  copy with a planted key.
+- `--dump-html` help pointed at a `TROUBLESHOOTING.md` this repo does not
+  have; it now points at the README section on traps that look like bugs.
+- Engine comments said the paid captcha path runs only for a `challenge`
+  state that does not exist here; they now describe what
+  `page_flow.STATE_POLICY` does.
+- `.gitignore` covers `--dump-html` page files of any extension
+  (`*.page[0-9]*`), a `live/` scratch directory and HTML dumps at the root.
+- CI tests Python 3.13, which the README badge already claimed.
+- Comments, docstrings and changelog entries no longer cite internal notes
+  that are not public; each rule is stated in place. A suite check keeps it
+  that way.
+
 - **Scraper API: `waitFor` is now sent as a JSON object.** Measured
   2026-09-23 against `scraper.2captcha.com/tasks/sync`: the JSON-encoded
   string this client sent (the form older docs described) is refused with
@@ -63,7 +103,7 @@ reading.
   "Parsed 0 job(s)" and exit 4 — "the query matched nothing". Measured: the
   Scraper API rendering justjoin.it's 1.8 MB listing page returned upstream
   HTTP 500 with **zero bytes**, and that path called it an empty result. It
-  now names the state, saves the body and exits 5. CLAUDE.md §8: blocked,
+  now names the state, saves the body and exits 5. Blocked,
   empty and failed are three different answers.
 
 - **A latent `NameError` in the same client** — `EXIT_NO_PRODUCTS` was used
@@ -89,21 +129,21 @@ reading.
   The opposite of what was written: point it at the endpoint.
 
 - **An inherited figure replaced with a measured one.** A comment repeated
-  CLAUDE.md's "17 of 18 repos" for the fingerprint flags. Re-counted across
-  the 26 sibling repos: 25 of 26, with the command beside it (§13).
+  an inherited "17 of 18 repos" for the fingerprint flags. Re-counted across
+  the 26 sibling repos: 25 of 26, with the command beside it.
 
 ### Removed
 
 - `SITE_TOTAL_HINT` and `EXPIRED_SITEMAP_INDEX`: two constants nothing
   read. A sweep for public names with no consumer now finds exactly two,
   both in the shared captcha client and both dead in every sibling repo
   checked — those are pinned by a check rather than removed, so the list
-  cannot grow silently (§17).
+  cannot grow silently.
 
 ### Verified live, against a real 2Captcha key
 
-- **`--fingerprint` applies what it fetches.** CLAUDE.md §16 lists four
-  defects that shipped on this path in four repos at once; none is present
+- **`--fingerprint` applies what it fetches.** Four
+  defects shipped on this path in four repos at once; none is present
   here. A PL fingerprint reached the browser intact: user agent
   `Windows NT 10.0 … Chrome/151`, locale `pl-PL` (not `en-PL`), timezone
   `Europe/Warsaw`, and `--fp-tags Windows` is accepted by the API.
@@ -237,7 +277,7 @@ Browser API over CDP.
 ### Fixed in the family core
 
 Both were inherited from the repo this one was ported from, and both are
-the shape CLAUDE.md §16 describes — copied code with no evidence behind it.
+the same shape — copied code with no evidence behind it.
 
 - **The two parity engines crashed instead of retrying when no proxy pool
   was configured.** `mask(pool.current)` was called unguarded in the

captcha_solver.py

@@ -21,7 +21,7 @@
 from `curl` with no headers at all, from `python-requests` and from `wget`.
 No challenge has ever been rendered to this scraper.
 
-But "no challenge rendered" is NOT "no captcha configured" (CLAUDE.md §18),
+But "no challenge rendered" is NOT "no captcha configured",
 and this site has one:
 
   * **justjoin.it runs Google reCAPTCHA v2, on its APPLICATION FORM.** It
@@ -52,7 +52,7 @@
     This is the INVERSE of the shape a sibling repo found, and both are
     worth carrying. There, the sitekey appeared in no served HTML and only
     a live browser revealed it. Here it is in every served page and a live
-    browser reveals nothing. So §18's "grep a good page for the site's own
+    browser reveals nothing. So the "grep a good page for the site's own
     captcha config" works on this site — and a reader who stopped at the
     grep would wrongly conclude a widget is waiting on the listing.
     `product_parser.site_recaptcha_sitekey` reads that key, and this
@@ -77,8 +77,7 @@
 configures — as well as enterprise reCAPTCHA
 (`RecaptchaV2EnterpriseTaskProxyless`), v3 and Cloudflare Turnstile
 (`TurnstileTaskProxyless`), and this module builds all of them. The only
-sentence this repo is entitled to is which task types it implements
-(CLAUDE.md §19).
+sentence this repo is entitled to is which task types it implements.
 
 PROVENANCE OF THE FIGURES BELOW
 -------------------------------

diff_runs.py

@@ -52,7 +52,7 @@
 different SAMPLES of the same board: measured 2026-09-18, the first 100 ids
 under the two orderings overlapped on 5. So two runs that disagree on
 `sort` or on their filters are refused outright rather than diffed — every
-line of that diff would be an artefact of the query (CLAUDE.md §21).
+line of that diff would be an artefact of the query.
 
 **`position` and `page` are deliberately not tracked.** justjoin.it's order
 was measured STABLE on 2026-09-18: two fetches 25 seconds apart returned
@@ -91,7 +91,7 @@
 # reports success. `smoke_test.py` pins every name against the dataclass.
 #
 # A shop's fields are absent because a job offer has no stock or discount —
-# porting them would be dead code that looks load-bearing (CLAUDE.md §4).
+# porting them would be dead code that looks load-bearing.
 # What changes on a justjoin.it offer is its PAY, its SENIORITY, its
 # SKILLS, where it can be done from and how you apply to it.
 #
@@ -146,6 +146,34 @@
     "expires_at",
 )
 
+# What is worth watching on a `--mode facets` row: the board's own count
+# per filter key. The JobPosting tuple above names none of these, so a
+# facets diff used to compare nothing and report "0 changed" while a key
+# went from 100 offers to 250.
+#
+# DELIBERATELY NOT TRACKED: `site_total`, the whole board's count. It is
+# the same figure on every row of a run, so a change in it would be
+# reported once per key and say nothing about any of them.
+FACET_TRACKED_FIELDS = (
+    "title",
+    "offer_count",
+    "is_cumulative",
+    "overlaps_other_keys",
+)
+
+
+def tracked_fields_for(mode: Optional[str], rows: List[dict]) -> Tuple[str, ...]:
+    """The fields to compare for a run of `mode`.
+
+    The mode comes from the sidecar. Without one (--force on bare files) it
+    is read off the rows: a facets row carries `facet_group`.
+    """
+    if mode == "facets" or (mode is None and any("facet_group" in r
+                                                 for r in rows)):
+        return FACET_TRACKED_FIELDS
+    return TRACKED_FIELDS
+
+
 # The subset that only SOME of the three routes populate.
 #
 # This repo reads the same offer three ways and they publish different
@@ -169,7 +197,7 @@
 #                     that payload carries no normalisation.
 #
 # When two rows disagree on `data_source`, a difference on any of these is
-# reported as `source_changed` rather than as a change (§8: a difference
+# reported as `source_changed` rather than as a change (a difference
 # that comes with a provenance difference says something about our own two
 # snapshots, not about the site).
 #
@@ -211,7 +239,8 @@ def _by_sku(products: List[dict]) -> Tuple[Dict[str, dict], int]:
     return indexed, unmatchable
 
 
-def diff_products(old: List[dict], new: List[dict]) -> dict:
+def diff_products(old: List[dict], new: List[dict],
+                  tracked: Tuple[str, ...] = TRACKED_FIELDS) -> dict:
     old_by_sku, old_unmatchable = _by_sku(old)
     new_by_sku, new_unmatchable = _by_sku(new)
 
@@ -223,7 +252,7 @@ def diff_products(old: List[dict], new: List[dict]) -> dict:
         before, after = old_by_sku[sku], new_by_sku[sku]
         field_changes = {
             field: {"old": before.get(field), "new": after.get(field)}
-            for field in TRACKED_FIELDS
+            for field in tracked
             if before.get(field) != after.get(field)
         }
         if not field_changes:
@@ -379,7 +408,7 @@ def _check_comparable(args) -> bool:
     # under `sortBy=publishedAt` and under `sortBy=salary` returned first
     # pages whose 100 ids overlapped on FIVE. Diffing one against the other
     # would report ~95 added and ~95 removed, every line an artefact of the
-    # query rather than a fact about the board (CLAUDE.md §21).
+    # query rather than a fact about the board.
     #
     # The same argument applies to the FILTERS, and more obviously: a
     # `--category python` run and a `--category java` run are different
@@ -429,7 +458,7 @@ def _check_comparable(args) -> bool:
                   f"rather than that an offer was taken down. Use --pages "
                   f"{enumerated} on both runs to compare a full census.")
         if meta.get("capped_by_site"):
-            # CLAUDE.md §21: the third meaning of `removed`. A capped run
+            # The third meaning of `removed`. A capped run
             # holds a moving window over a board that is published to
             # continuously, so rows leave the window without anything
             # happening to the job.
@@ -489,7 +518,9 @@ def main() -> int:
         print(f"[!] Could not read one of the input files: {e}")
         return 2
 
-    result = diff_products(old, new)
+    _, meta = _run_status(args.new)
+    result = diff_products(old, new, tracked_fields_for(
+        (meta or {}).get("mode"), old + new))
     _print_summary(result)
 
     if args.out:

env_config.py

@@ -66,7 +66,7 @@
 # A literal-only check reported both of those as CONFIGURED, so `cp
 # .env.example .env` followed by a run connected to cb.2captcha.com with the
 # string `{login}-zone-…` as its username and got a 401 — a confusing failure
-# a long way from its cause, which is precisely what §3's rule exists to
+# a long way from its cause, which is precisely what the rule exists to
 # prevent. Any `{…}` left in a value means the line was never filled in.
 _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}")