From 4f71c86c6b9b6ffb79ac199388a6e8ee08e75916 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:57:01 +0000 Subject: [PATCH 1/2] Escape spreadsheet formulas in CSV, read JUSTJOIN_URL as a default, diff facets runs - CSV: a string cell starting with = + - @ TAB CR LF gets a leading apostrophe (CSV only; JSON and numbers untouched); the count is recorded in the sidecar as csv_cells_escaped, under caller extra. - JUSTJOIN_URL from .env no longer makes --mode offer/facets exit 2: the engines capture the typed --url before env_config.apply and only check a typed URL against --mode. .env.example shows the variable commented. - diff_runs compares facets runs on their own columns (offer_count, ...) instead of the job columns, which a facet row does not have. - The credential scan matches its skip list against the path inside the repo, so a clone under build/, env/ or venv/ is scanned. - Comments, docstrings and changelog entries state their rules in place instead of citing internal notes; a suite check guards it. - Small: --dump-html help link, engine comments about the solve state, .gitignore for page dumps of any extension, CI on Python 3.13, canary comment no longer repeats a retracted credential-lifetime claim. Co-Authored-By: Claude Opus 5.5 --- .env.example | 9 +- .github/ci_checks.py | 9 +- .github/workflows/canary.yml | 15 +- .github/workflows/tests.yml | 6 +- .gitignore | 9 +- CHANGELOG.md | 54 +++++- captcha_solver.py | 7 +- diff_runs.py | 47 ++++- env_config.py | 2 +- fingerprint_client.py | 2 +- output_writer.py | 84 ++++++--- page_flow.py | 35 ++-- playwright_scraper.py | 77 ++++---- product_parser.py | 85 +++++---- puppeteer_scraper.py | 54 +++--- scraper_api_client.py | 6 +- selenium_scraper.py | 48 +++-- smoke_test.py | 328 +++++++++++++++++++++++++++-------- 18 files changed, 603 insertions(+), 274 deletions(-) diff --git a/.env.example b/.env.example index 5414267..9002d74 100644 --- a/.env.example +++ b/.env.example @@ -60,7 +60,7 @@ # # # WORTH KNOWING ABOUT CAPTCHAS HERE, because "none rendered" is not "none -# configured" (CLAUDE.md §18). +# configured". # # justjoin.it publishes its own reCAPTCHA **v2** key in the page config on # every page: @@ -148,7 +148,12 @@ JUSTJOIN_PROXY=http://{user}:{password}@pl.proxy.2captcha.com:2334 # `x-api-version` header its own bundle sends. The route that works is the # same-origin proxy at justjoin.it/api/candidate-api/, and the scraper # refuses an api.justjoin.it URL with that reason rather than a generic one. -JUSTJOIN_URL=https://justjoin.it/job-offers/all-locations +# +# Shipped commented out: every mode already has its own default, and a +# listings URL set here would otherwise stand in for --url on every run. +# A URL set here that names a different mode than --mode is ignored with a +# log line; a URL you type with --url is still checked against --mode. +# JUSTJOIN_URL=https://justjoin.it/job-offers/all-locations # Deliberately NOT here: an output prefix. `--out` already carries a # non-empty default, so the loader — which only fills UNSET values — would diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 87d00b0..62e8ced 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -81,7 +81,7 @@ HEX32 = re.compile(r"\b[0-9a-f]{32}\b") # Contexts in which a 32-hex string is plainly not a key. Site-specific # entries belong here only once a committed fixture actually carries one — -# CLAUDE.md §16: a family-wide entry copied without checking is how dead +# A family-wide entry copied without checking is how dead # code spreads. Kept as a CONTEXT allowlist rather than by loosening the # pattern: a bare 32-hex string anywhere else still fails, which is the # point of the check. @@ -107,7 +107,7 @@ def _is_virtualenv(path): walked into pip's vendored code and flagged a 32-hex string in `_elffile.py` as key-shaped. The run was correct about the string and wrong about the file, and a guard people have to argue with is one they - learn to suppress (CLAUDE.md §22). + learn to suppress. Structural rather than by name, for the same reason a parser anchors on a URL pattern instead of a CSS class: `venv`, `.venv`, `.v`, `env39` and @@ -128,7 +128,10 @@ def scanned_files(): continue if path.suffix not in SCANNED_SUFFIXES: continue - if any(part in _SKIP_NAMES for part in path.parts): + # RELATIVE parts: the absolute path includes every ancestor, so a + # clone under ~/build, /srv/env or a directory called venv matched + # here on every file, scanned nothing and reported clean. + if any(part in _SKIP_NAMES for part in path.relative_to(REPO).parts): continue if any(root in path.parents for root in skip_roots): continue diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index bff4b92..4da6d01 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -5,9 +5,9 @@ # # Most canaries in this family skip when a proxy secret is absent, because # their site refuses a datacenter address and a GitHub runner is one — and a -# check that is always red teaches everyone to ignore checks (CLAUDE.md §11). +# check that is always red teaches everyone to ignore checks. # -# justjoin.it is the other case, and CLAUDE.md §21 states the rule for it: a +# justjoin.it is the other case, and the rule for it is: a # canary that CAN pass without a credential should never be gated on one. # Measured 2026-09-18 from a bare netcup datacentre address, with no key and # no proxy: the offers endpoint 200 with 100 offers, an offer's own record @@ -23,9 +23,8 @@ # limit, the badge goes red the next morning and the claim is retested # without anyone having to remember to. # -# There is no secret anywhere in this file, which also means there is -# nothing here that can rot: a Scraping Browser profile's credentials live -# about a day, so a scheduled job holding one is stale between runs. +# There is no secret anywhere in this file, so there is nothing here that +# can expire between runs. name: canary @@ -58,7 +57,7 @@ jobs: # THREE pages, not one. With one page pagination is never exercised, # which is exactly how a silent single-page bug survived in a sibling - # repo (CLAUDE.md §11). A filtered category is used rather than the + # repo. A filtered category is used rather than the # whole board so the result set is under the site's 10,000 cap and two # runs are comparable as a census. - name: Run --mode listings (3 pages, filtered) @@ -204,7 +203,7 @@ jobs: "a monthly figure came out below its hourly rate") # No row may carry a salary of zero: the site's own "Undisclosed - # Salary" must reach the output as null (CLAUDE.md §21). + # Salary" must reach the output as null. want("an undisclosed salary is null, never zero", not any(r.get("salary_min") == 0 or r.get("salary_max") == 0 for r in rows), @@ -232,7 +231,7 @@ jobs: "an id came back in an unexpected shape") # page+position must be unique ACROSS pages — position restarts at # 1 on each, so without the page number beside it the column is - # worthless (CLAUDE.md §18). + # worthless. want("page+position unique", len({(r["page"], r["position"]) for r in rows}) == len(rows), "collisions — the page number is not reaching the parser") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c63ec64..3a0ad7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: matrix: # 3.9 is the floor the README claims. Claiming it without testing it is # how a walrus operator or a `X | None` annotation ships and breaks it. - python-version: ["3.9", "3.12"] + python-version: ["3.9", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -59,7 +59,7 @@ jobs: # check still imported the row dataclass by a name this repo renamed, # and it failed on this repo's FIRST push while the script it # duplicates passed — two sources of truth, one of them stale, which - # is exactly the defect CLAUDE.md §17 describes. + # is exactly the dead-check defect this guards against. # # So: one implementation, and `smoke_test.py` now asserts the workflow # does not grow another one. @@ -150,7 +150,7 @@ jobs: # version of this step looked for "_scraper could not be # imported", a string no suite in this family emits, so the job could # never fail and passed for the wrong reason — which is the whole - # thing §10 says this job exists to prevent ("skipped, engine absent" + # thing this job exists to prevent ("skipped, engine absent" # reading identically to a real import error). Checked 2026-09-18: # seven sibling repos carry the same dead grep. # diff --git a/.gitignore b/.gitignore index 686aa3e..e853c47 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,13 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# --dump-html X --pages 3 writes X.page2, X.page3 whatever X ends in, so a +# .json or .txt dump name is not covered by the patterns above. +*.page[0-9]* +# Scratch captures by KIND rather than by one chosen name, and a bare HTML +# dump at the repo root. +live/ +/*.html # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -73,7 +80,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when +# and the fix never reached anyone else, which is the rule unapplied: when # you fix one, check the siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec3f1f..88ad899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,48 @@ burying it. ## [Unreleased] +> **Behaviour changes.** CSV output now prefixes a cell that starts with +> `=`, `+`, `-`, `@`, a tab, CR or LF with an apostrophe (JSON is +> unchanged), and the sidecar gains `csv_cells_escaped`. `.env.example` +> no longer sets `JUSTJOIN_URL`; a `JUSTJOIN_URL` from the environment that +> names a different mode than `--mode` is now ignored with a log line +> instead of exiting 2. + ### Fixed +- **CSV cells are formula-escaped.** Every string in a row is written by an + employer, and a spreadsheet runs a cell starting with `=`, `+`, `-` or + `@` as a formula. Escaped in the CSV only, after list cells are joined; + numbers stay numbers and JSON keeps the site's bytes. The count goes in + the sidecar as `csv_cells_escaped`, under any caller-provided field of + the same name. +- **`JUSTJOIN_URL` no longer breaks `--mode offer` and `--mode facets`.** + The listings URL that `.env.example` shipped was read as if `--url` had + been typed, so a copied example made both modes exit 2. The engines now + record what was typed before reading the environment; only a typed URL is + checked against `--mode`. The example now shows the variable commented + out. +- **`diff_runs.py` sees changes in a facets run.** It compared the job + columns on facet rows, which have none of them, so an `offer_count` + going from 100 to 250 reported "0 changed". Facets runs are now compared + on `title`, `offer_count`, `is_cumulative` and `overlaps_other_keys`. +- **The credential scan works from any parent directory.** Its skip list + was matched against the absolute path, so a clone under a directory + named `build`, `env` or `venv` scanned 0 files and reported clean. It now + matches the path inside the repo, and the suite runs the scan from such a + copy with a planted key. +- `--dump-html` help pointed at a `TROUBLESHOOTING.md` this repo does not + have; it now points at the README section on traps that look like bugs. +- Engine comments said the paid captcha path runs only for a `challenge` + state that does not exist here; they now describe what + `page_flow.STATE_POLICY` does. +- `.gitignore` covers `--dump-html` page files of any extension + (`*.page[0-9]*`), a `live/` scratch directory and HTML dumps at the root. +- CI tests Python 3.13, which the README badge already claimed. +- Comments, docstrings and changelog entries no longer cite internal notes + that are not public; each rule is stated in place. A suite check keeps it + that way. + - **Scraper API: `waitFor` is now sent as a JSON object.** Measured 2026-09-23 against `scraper.2captcha.com/tasks/sync`: the JSON-encoded string this client sent (the form older docs described) is refused with @@ -63,7 +103,7 @@ reading. "Parsed 0 job(s)" and exit 4 — "the query matched nothing". Measured: the Scraper API rendering justjoin.it's 1.8 MB listing page returned upstream HTTP 500 with **zero bytes**, and that path called it an empty result. It - now names the state, saves the body and exits 5. CLAUDE.md §8: blocked, + now names the state, saves the body and exits 5. Blocked, empty and failed are three different answers. - **A latent `NameError` in the same client** — `EXIT_NO_PRODUCTS` was used @@ -89,8 +129,8 @@ reading. The opposite of what was written: point it at the endpoint. - **An inherited figure replaced with a measured one.** A comment repeated - CLAUDE.md's "17 of 18 repos" for the fingerprint flags. Re-counted across - the 26 sibling repos: 25 of 26, with the command beside it (§13). + an inherited "17 of 18 repos" for the fingerprint flags. Re-counted across + the 26 sibling repos: 25 of 26, with the command beside it. ### Removed @@ -98,12 +138,12 @@ reading. read. A sweep for public names with no consumer now finds exactly two, both in the shared captcha client and both dead in every sibling repo checked — those are pinned by a check rather than removed, so the list - cannot grow silently (§17). + cannot grow silently. ### Verified live, against a real 2Captcha key -- **`--fingerprint` applies what it fetches.** CLAUDE.md §16 lists four - defects that shipped on this path in four repos at once; none is present +- **`--fingerprint` applies what it fetches.** Four + defects shipped on this path in four repos at once; none is present here. A PL fingerprint reached the browser intact: user agent `Windows NT 10.0 … Chrome/151`, locale `pl-PL` (not `en-PL`), timezone `Europe/Warsaw`, and `--fp-tags Windows` is accepted by the API. @@ -237,7 +277,7 @@ Browser API over CDP. ### Fixed in the family core Both were inherited from the repo this one was ported from, and both are -the shape CLAUDE.md §16 describes — copied code with no evidence behind it. +the same shape — copied code with no evidence behind it. - **The two parity engines crashed instead of retrying when no proxy pool was configured.** `mask(pool.current)` was called unguarded in the diff --git a/captcha_solver.py b/captcha_solver.py index 42154eb..6d5eea4 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -21,7 +21,7 @@ from `curl` with no headers at all, from `python-requests` and from `wget`. No challenge has ever been rendered to this scraper. -But "no challenge rendered" is NOT "no captcha configured" (CLAUDE.md §18), +But "no challenge rendered" is NOT "no captcha configured", and this site has one: * **justjoin.it runs Google reCAPTCHA v2, on its APPLICATION FORM.** It @@ -52,7 +52,7 @@ This is the INVERSE of the shape a sibling repo found, and both are worth carrying. There, the sitekey appeared in no served HTML and only a live browser revealed it. Here it is in every served page and a live - browser reveals nothing. So §18's "grep a good page for the site's own + browser reveals nothing. So the "grep a good page for the site's own captcha config" works on this site — and a reader who stopped at the grep would wrongly conclude a widget is waiting on the listing. `product_parser.site_recaptcha_sitekey` reads that key, and this @@ -77,8 +77,7 @@ configures — as well as enterprise reCAPTCHA (`RecaptchaV2EnterpriseTaskProxyless`), v3 and Cloudflare Turnstile (`TurnstileTaskProxyless`), and this module builds all of them. The only -sentence this repo is entitled to is which task types it implements -(CLAUDE.md §19). +sentence this repo is entitled to is which task types it implements. PROVENANCE OF THE FIGURES BELOW ------------------------------- diff --git a/diff_runs.py b/diff_runs.py index 9e36a2c..61e2936 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -52,7 +52,7 @@ different SAMPLES of the same board: measured 2026-09-18, the first 100 ids under the two orderings overlapped on 5. So two runs that disagree on `sort` or on their filters are refused outright rather than diffed — every -line of that diff would be an artefact of the query (CLAUDE.md §21). +line of that diff would be an artefact of the query. **`position` and `page` are deliberately not tracked.** justjoin.it's order was measured STABLE on 2026-09-18: two fetches 25 seconds apart returned @@ -91,7 +91,7 @@ # reports success. `smoke_test.py` pins every name against the dataclass. # # A shop's fields are absent because a job offer has no stock or discount — -# porting them would be dead code that looks load-bearing (CLAUDE.md §4). +# porting them would be dead code that looks load-bearing. # What changes on a justjoin.it offer is its PAY, its SENIORITY, its # SKILLS, where it can be done from and how you apply to it. # @@ -146,6 +146,34 @@ "expires_at", ) +# What is worth watching on a `--mode facets` row: the board's own count +# per filter key. The JobPosting tuple above names none of these, so a +# facets diff used to compare nothing and report "0 changed" while a key +# went from 100 offers to 250. +# +# DELIBERATELY NOT TRACKED: `site_total`, the whole board's count. It is +# the same figure on every row of a run, so a change in it would be +# reported once per key and say nothing about any of them. +FACET_TRACKED_FIELDS = ( + "title", + "offer_count", + "is_cumulative", + "overlaps_other_keys", +) + + +def tracked_fields_for(mode: Optional[str], rows: List[dict]) -> Tuple[str, ...]: + """The fields to compare for a run of `mode`. + + The mode comes from the sidecar. Without one (--force on bare files) it + is read off the rows: a facets row carries `facet_group`. + """ + if mode == "facets" or (mode is None and any("facet_group" in r + for r in rows)): + return FACET_TRACKED_FIELDS + return TRACKED_FIELDS + + # The subset that only SOME of the three routes populate. # # This repo reads the same offer three ways and they publish different @@ -169,7 +197,7 @@ # that payload carries no normalisation. # # When two rows disagree on `data_source`, a difference on any of these is -# reported as `source_changed` rather than as a change (§8: a difference +# reported as `source_changed` rather than as a change (a difference # that comes with a provenance difference says something about our own two # snapshots, not about the site). # @@ -211,7 +239,8 @@ def _by_sku(products: List[dict]) -> Tuple[Dict[str, dict], int]: return indexed, unmatchable -def diff_products(old: List[dict], new: List[dict]) -> dict: +def diff_products(old: List[dict], new: List[dict], + tracked: Tuple[str, ...] = TRACKED_FIELDS) -> dict: old_by_sku, old_unmatchable = _by_sku(old) new_by_sku, new_unmatchable = _by_sku(new) @@ -223,7 +252,7 @@ def diff_products(old: List[dict], new: List[dict]) -> dict: before, after = old_by_sku[sku], new_by_sku[sku] field_changes = { field: {"old": before.get(field), "new": after.get(field)} - for field in TRACKED_FIELDS + for field in tracked if before.get(field) != after.get(field) } if not field_changes: @@ -379,7 +408,7 @@ def _check_comparable(args) -> bool: # under `sortBy=publishedAt` and under `sortBy=salary` returned first # pages whose 100 ids overlapped on FIVE. Diffing one against the other # would report ~95 added and ~95 removed, every line an artefact of the - # query rather than a fact about the board (CLAUDE.md §21). + # query rather than a fact about the board. # # The same argument applies to the FILTERS, and more obviously: a # `--category python` run and a `--category java` run are different @@ -429,7 +458,7 @@ def _check_comparable(args) -> bool: f"rather than that an offer was taken down. Use --pages " f"{enumerated} on both runs to compare a full census.") if meta.get("capped_by_site"): - # CLAUDE.md §21: the third meaning of `removed`. A capped run + # The third meaning of `removed`. A capped run # holds a moving window over a board that is published to # continuously, so rows leave the window without anything # happening to the job. @@ -489,7 +518,9 @@ def main() -> int: print(f"[!] Could not read one of the input files: {e}") return 2 - result = diff_products(old, new) + _, meta = _run_status(args.new) + result = diff_products(old, new, tracked_fields_for( + (meta or {}).get("mode"), old + new)) _print_summary(result) if args.out: diff --git a/env_config.py b/env_config.py index 9eb62b3..b11f8cb 100644 --- a/env_config.py +++ b/env_config.py @@ -66,7 +66,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what the rule exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") diff --git a/fingerprint_client.py b/fingerprint_client.py index e84ce1e..459b9f4 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -331,7 +331,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is the shape of many copied-code defects. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/output_writer.py b/output_writer.py index ca79c7f..3a00521 100644 --- a/output_writer.py +++ b/output_writer.py @@ -44,7 +44,7 @@ unfiltered run cannot. `mode` goes in the sidecar and `diff_runs.py` REFUSES a cross-shape -comparison, which is CLAUDE.md §9's rule for a repo that reads more than +comparison, which is the rule for a repo that reads more than one kind of thing. The row is `JobPosting` and not `Product` @@ -53,7 +53,7 @@ commerce columns even where they are null, because on a shop a null price is a fact worth recording. justjoin.it sells no products: there is no price, no discount, no stock and no brand, and those columns null on every -row of every run of every mode is exactly what §9 says must not exist. +row of every run of every mode is exactly what must not exist. `wellfound-scraper`, `bbb-scraper` and `mercor-scraper` are the precedent. What a job board has where a shop has a price is a RANGE, which is two @@ -66,11 +66,11 @@ What IS kept, byte-identical and in order, is the family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — so one column name works across the family and a consumer reading several of these repos reads the same first -five columns in the same order (§9). +five columns in the same order. Columns that are absent, and the measurement for each ------------------------------------------------------ -§9: a column that is null on every row of every run should not exist, and +A column that is null on every row of every run should not exist, and removing one needs the measurement written down so someone can add it back with a better one. Measured over 1,000 listing records and 40 detail records on 2026-09-18: @@ -110,7 +110,7 @@ import json from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone -from typing import Optional, List, Set, Sequence, Any, Type +from typing import Optional, List, Set, Sequence, Any, Tuple, Type # The site a row came from. One value, because justjoin.it serves @@ -137,13 +137,12 @@ class JobPosting: The family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — is byte-identical and in this order across every repo in the family, so a - consumer reading several of them reads the same first five columns - (CLAUDE.md §9). Everything after it is Just Join IT's. + consumer reading several of them reads the same first five columns. Everything after it is Just Join IT's. Every coverage figure quoted below was measured on 2026-09-18 over 1,000 consecutive listing records and 40 randomly sampled offer records. They are a reading taken on one day, not a property of the - site: re-measure before quoting them (CLAUDE.md §13). + site: re-measure before quoting them. """ source: str = SOURCE_DEFAULT @@ -193,7 +192,7 @@ class JobPosting: # of 1,040 — every one of 1,000 listing records and 40 detail records # measured 2026-09-18. justjoin.it's 25 categories are a flat list, not # a tree. A field that looks available and never is costs more than a - # missing one (CLAUDE.md §9), and this note is the measurement someone + # missing one, and this note is the measurement someone # needs to add it back if the site grows subcategories. # intern / junior / mid / senior / manager / c_level. Measured over # 1,000: senior 519, mid 380, manager 58, junior 38, c_level 4, intern 1. @@ -223,7 +222,7 @@ class JobPosting: # Null on 461 of 1,000 rows, where the employer disclosed no salary. # Null, not zero — the site's own UI says "Undisclosed Salary" for # these, and a zero would drag every average a reader computes - # (CLAUDE.md §21). + #. salary_min: Optional[float] = None salary_max: Optional[float] = None # An ISO 4217 code the site states — never defaulted, never inferred @@ -287,7 +286,7 @@ class JobPosting: # The 1-5 level the employer set for each skill, positionally aligned # with `required_skills`. Null on a `--route ssr` row: the rendered # payload publishes skills as bare strings with no levels at all, and - # inventing a level would be a guess in a column (CLAUDE.md §8). + # inventing a level would be a guess in a column. required_skill_levels: Optional[List[int]] = None # Sparse and real: 46 of 1,000 listing rows, 3 of 40 detail records. nice_to_have_skills: Optional[List[str]] = None @@ -317,18 +316,18 @@ class JobPosting: # through the result rather than banked at the top: 33, 32, 10, 34 and # 9 per hundred at offsets 0, 500, 2,000, 5,000 and 9,000. So the # site's default ordering is NOT a paid ranking — measured, because - # CLAUDE.md §21 found that it was on BBB, and this scraper therefore + # A sibling directory found that it was on BBB, and this scraper therefore # keeps the site's own default sort rather than overriding it. # # `isPromoted` is deliberately NOT a column beside it: false on # 1,000 of 1,000 rows and at every offset sampled. A field that looks - # available and never is costs more than a missing one (CLAUDE.md §9), + # available and never is costs more than a missing one, # and this note is the measurement someone needs to add it back. is_super_offer: Optional[bool] = None # Detail route only. True on 40 of 40 sampled — the endpoint serves # active offers, and no expired one was reachable to check the other # value with. So a False here is UNPROVEN and a True says only what the - # site said (CLAUDE.md §20). Carried because this is the one column a + # site said. Carried because this is the one column a # withdrawn offer could ever announce itself in. is_active: Optional[bool] = None # external 800 / form 200 of 1,000. @@ -356,7 +355,7 @@ class JobPosting: # ---- provenance ----------------------------------------------------- # Which route this row was read out of: `api` (the offers endpoint), # `ssr` (a rendered listing page's flight payload) or `detail` (one - # offer's endpoint). CLAUDE.md §8 — provenance of a value goes in a + # offer's endpoint). Provenance of a value goes in a # column, and `diff_runs.py` reports a difference that comes with a # `data_source` difference as `source_changed` rather than as a real # change. That matters more here than in most repos in this family, @@ -375,7 +374,7 @@ class Facet: A different KIND of thing from a job posting — it describes the board rather than a job — so it gets its own row class rather than being - bent into `JobPosting` (CLAUDE.md §9). The family prefix is kept + bent into `JobPosting`. The family prefix is kept byte-identical and in order, and `sku` is reused as the id column so one column name works across the family. @@ -498,12 +497,31 @@ def _csv_value(v: Any) -> Any: return v +# A spreadsheet opening the CSV runs a cell that starts with one of these as +# a formula, and every string in a row was written by an employer, not by +# this project. So a string cell starting with one gets a leading apostrophe +# in the CSV ONLY: JSON keeps the site's exact bytes, and numbers are never +# touched (turning -5 into text would break every sum over the column). The +# check runs AFTER `_csv_value`, because a list joined into one cell is text +# too and its first element can be formula-shaped. The count goes into the +# sidecar as `csv_cells_escaped`, so the difference between the two outputs +# is declared rather than discovered. +CSV_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r", "\n") + + +def _csv_cell(v: Any) -> Tuple[Any, bool]: + v = _csv_value(v) + if isinstance(v, str) and v.startswith(CSV_FORMULA_PREFIXES): + return "'" + v, True + return v, False + + def write_json(rows: Sequence[Any], path: str) -> None: with open(path, "w", encoding="utf-8") as f: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> None: +def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> int: # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -515,8 +533,14 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> Non with open(path, "w", encoding="utf-8", newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() + escaped = 0 for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + row = {} + for k, v in asdict(r).items(): + row[k], hit = _csv_cell(v) + escaped += hit + writer.writerow(row) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 @@ -639,7 +663,7 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, results, the ordering and the filters decide WHICH offers are in the file and not merely their order. Two orderings of one query overlapped on 5 of their first 100 ids when this was measured, so `diff_runs.py` - refuses a pair that disagrees on either (CLAUDE.md §21). + refuses a pair that disagrees on either. `source` is `justjoin.it` on every row of every run. The site is served from one host and this column @@ -650,7 +674,7 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, `extra` carries facts about the run that are not about any single row. justjoin.it states `meta.totalItems` on every response, so unlike a site that states nothing there IS a figure to record — and recording it alone - would be the lie §21 warns about, because the site caps that figure at + would be a lie, because the site caps that figure at 10,000 however many the query matched. So a listings run writes the whole arithmetic: `total_results`, `site_total`, `reachable_max`, `capped_by_site`, `share_of_query_pct` and `share_of_board_pct`, plus @@ -685,14 +709,15 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, } if extra: # Merged rather than nested under a key, so a consumer reads - # `shop_rating` at the top level beside `products`. Run fields win a + # `total_results` at the top level beside `products`. Run fields win a # name collision: a caller cannot accidentally overwrite `status`. meta.update({k: v for k, v in extra.items() if k not in meta}) return meta def save(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool = False, row_cls: Type = JobPosting) -> int: + allow_empty: bool = False, row_cls: Type = JobPosting, + stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. Returns 0 when rows were written, EXIT_NO_PRODUCTS when there were none. @@ -708,6 +733,8 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, `allow_empty=True` is for the legitimate case: a filter that genuinely matches nothing, where an empty file is the answer. + + `stats`, when given, receives `csv_cells_escaped` if a CSV was written. """ if not rows and not allow_empty: print(f"[!] 0 jobs — refusing to write {out_prefix}.json/.csv, so an " @@ -719,7 +746,9 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, write_json(rows, f"{out_prefix}.json") print(f"[+] Saved {len(rows)} jobs -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + escaped = write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + if stats is not None: + stats["csv_cells_escaped"] = escaped print(f"[+] Saved {len(rows)} jobs -> {out_prefix}.csv") return 0 if rows else EXIT_NO_PRODUCTS @@ -754,7 +783,7 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # Note what "complete" does NOT mean here. justjoin.it caps every query at # 10,000 results however many it matched — the board held 19,381 when this # was measured — so a complete run can still be about half the board. -# CLAUDE.md §21: complete and exhaustive are different words, and the +# Complete and exhaustive are different words, and the # sidecar records both figures so a consumer is not left inferring one from # the other. COMPLETE_STOP_REASONS = ("completed", "pagination_exhausted", "no_new_products", @@ -795,7 +824,12 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, # names has a hole for every name nobody added to it. complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed row_cls = ROW_CLASS_BY_MODE.get(mode, JobPosting) - rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls) + stats: dict = {} + rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls, + stats=stats) + # Housekeeping goes UNDER the caller's extra: a name collision must never + # silently drop a value the site stated. + extra = {**stats, **(extra or {})} wrote_output = bool(rows) or allow_empty if wrote_output: diff --git a/page_flow.py b/page_flow.py index 0b58c6a..4a8930a 100644 --- a/page_flow.py +++ b/page_flow.py @@ -2,7 +2,7 @@ page_flow.py ------------ The retry / solve / blocked decision, as DATA rather than as three copies of -an if-chain (CLAUDE.md §1). +an if-chain. justjoin.it answers a request six ways, and five of them want a different response: @@ -30,7 +30,7 @@ Selenium's `execute_script` takes a function BODY with an explicit `return` while Playwright and pyppeteer take `() => expr`, so a shared snippet would quietly acquire one driver's dialect. The callbacks below are named for the -OPERATION instead, and each engine spells it in its own dialect (§1). +OPERATION instead, and each engine spells it in its own dialect. """ import logging @@ -54,7 +54,7 @@ # How many offer links mean "this response is the board". # -# `> 1` on purpose, per CLAUDE.md §5: waiting for ONE match resolves on +# `> 1` on purpose: waiting for ONE match resolves on # something unrelated — a nav link, a footer — long before the grid is # really there. # @@ -82,7 +82,7 @@ # Anchored on a URL pattern, never a CSS class: justjoin.it's classes are # build-generated (`mui-1vaayb6`) and churn on every deploy, while -# `/job-offer/{slug}` is a contract with search engines (§4). +# `/job-offer/{slug}` is a contract with search engines. # # One spelling is enough here, and that is worth stating because a sibling # repo needed two: that site rewrites every job anchor during hydration, so @@ -120,7 +120,7 @@ def needs_readiness_wait(mode: str, route: str = DEFAULT_ROUTE) -> bool: complete when the navigation resolves, there is no grid to paint, and polling a selector against Chromium's JSON viewer would simply burn the full 45-second budget on every page before timing out. A readiness wait - that can never succeed is worse than none — CLAUDE.md §8 records what + that can never succeed is worse than none — a sibling repo recorded what that cost in a sibling repo, where it turned a background captcha into a paid solve on a page that already held every row. """ @@ -131,7 +131,7 @@ def needs_readiness_wait(mode: str, route: str = DEFAULT_ROUTE) -> bool: # # Polled through `count(selector)` — a callback each engine implements with # its own `querySelectorAll` call — and NEVER by handing the browser a -# string to evaluate. CLAUDE.md §18: a site whose Content-Security-Policy +# string to evaluate. A site whose Content-Security-Policy # omits `unsafe-eval` kills `wait_for_function` with an `EvalError` and # takes the run down with exit 1, on the site's most obvious URL. # @@ -178,7 +178,7 @@ def classify(body: Optional[str], status: Optional[int] = None, `classify(html, url=...)` in two of three engines against a callee that took `status` second, and both crashed on their first fetch — invisible to import, `--help`, `compileall` and 400+ green offline assertions, - because none of those calls a function the way a live run does (§17). + because none of those calls a function the way a live run does. `smoke_test.py` binds every engine's call against this signature for exactly that reason. """ @@ -214,12 +214,12 @@ def classify(body: Optional[str], status: Optional[int] = None, # identical impossible question again; reporting it as blocked would # send a reader after a proxy. The planner caps offsets so this should # be unreachable, and it is in the table because "unreachable" is a - # claim about today's code (CLAUDE.md §21). + # claim about today's code. "cap_exceeded": {"retry": False, "solve": False, "blocked": False, "parse": False}, # A substantial body this parser could not read. NOT "zero offers": a # response the site plainly served that parses to nothing is OUR bug, # and reporting it as an empty board sends the reader to check their - # filters instead of the parser (CLAUDE.md §20). Worth one retry in + # filters instead of the parser. Worth one retry in # case a document was caught mid-swap, and always worth a dump. "parse_error": {"retry": True, "solve": False, "blocked": False, "parse": False}, # Something that is neither the site nor a recognised refusal — an @@ -255,7 +255,7 @@ def is_terminal(state: str) -> bool: # True, and CONSULTED rather than merely documented — the engines read it, # so setting it False really does stop the retry loop. (A sibling repo # carried this constant with a paragraph of justification and no reader, -# which is the same defect as dead code that looks load-bearing: §17.) +# which is the same defect as dead code that looks load-bearing.) # # True is the cautious setting rather than a measured one here, and the # honest statement of the evidence is that there is none: justjoin.it has @@ -285,7 +285,7 @@ def is_terminal(state: str) -> bool: def pagination_is_addressable(url: str, route: str = DEFAULT_ROUTE) -> bool: """Whether page N of this run can be fetched without walking to it. - CLAUDE.md §18 says to ask this PER URL rather than per site, and on + Ask this PER URL rather than per site, and on justjoin.it the two listing routes give opposite answers: * the offers endpoint IS addressable. It paginates on a numeric @@ -313,7 +313,7 @@ def pages_to_plan(pages_requested: int, pages_available: Optional[int]) -> int: """How many pages a run may ask for, given what the site will serve. justjoin.it states `meta.totalItems` on page 1, so unlike a site that - states nothing there is real arithmetic to do — and CLAUDE.md §7's + states nothing there is real arithmetic to do — and pagination's second layer says to use it. `pages_available` is computed from that total AND from the site's own 10,000 cap, because asking for the page after the cap is an HTTP 500 rather than an empty page. @@ -334,7 +334,7 @@ def concurrency_limit(cdp_endpoint: Optional[str]) -> Optional[int]: The Scraping Browser API allows ONE live connection per profile, so N workers sharing a `pid` collide with `profile_locked`. Several `pid`s, - one run each, is the way to parallelise that path (§7). + one run each, is the way to parallelise that path. """ return 1 if cdp_endpoint else None @@ -349,8 +349,7 @@ def concurrency_for_mode(mode: str, concurrency: int, honest about what it did. `listings --route api` and `offer` both address every page - independently and may use every worker they are given — with CLAUDE.md - §7's warning attached, that concurrency without a proxy pool is a + independently and may use every worker they are given — with the warning attached, that concurrency without a proxy pool is a faster way to get an address scored than to gather data. """ if mode == "facets": @@ -377,7 +376,7 @@ def concurrency_for_mode(mode: str, concurrency: int, def mask_secrets(text: str) -> str: """Credentials out of an exception message before it reaches a log. - CLAUDE.md §8: an EXCEPTION MESSAGE is a log. A driver that fails while + An EXCEPTION MESSAGE is a log. A driver that fails while connecting through an authenticated proxy puts the whole URL into the text of the error, credentials included — so anything this module prints from an exception goes through here first. Host and port are @@ -394,7 +393,7 @@ def plan_page_urls(args, page_one_url: str, offer_slugs: Optional[List[str]] = None) -> List[str]: """URLs for pages 2..N, decided once before any of them is fetched. - CLAUDE.md §7 says to plan page URLs up front but to VERIFY first, and + Plan page URLs up front, but VERIFY first, and on justjoin.it the three routes give three different answers: * `--mode listings --route api` IS addressable. The endpoint @@ -463,7 +462,7 @@ def plan_page_urls(args, page_one_url: str, # Past the 10,000 ceiling. Stopping here is the whole point: # the next address would answer HTTP 500, an error that reads # like a bug in this scraper rather than like the site refusing - # an impossible request (CLAUDE.md §21). + # an impossible request. log.info("Stopping the plan at page %d: offset %d is at " "justjoin.it's own 10,000 ceiling, and asking past " "it is an HTTP 500 rather than an empty page.", diff --git a/playwright_scraper.py b/playwright_scraper.py index d9650cb..b8a62d5 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -35,7 +35,7 @@ names.** `api.justjoin.it` — the host the site's own `baseApiUrl` points at — answered HTTP 503 from nginx to every request tried. The route that works is the same-origin proxy at `justjoin.it/api/candidate-api/`, which - is what this engine uses. Gating is per-ROUTE, not per-site (§21). + is what this engine uses. Gating is per-ROUTE, not per-site. * **`?page=2` on a rendered listing silently returns page 1.** Not an error, not an empty page: HTTP 200, `meta.from` still 0, a @@ -182,12 +182,12 @@ class PageOutcome: # `urls_in_itemlist` is how many offer URLs a rendered page's own # `CollectionPage` JSON-LD names, which is only meaningful on # `--route ssr`. Carrying both means every run records the two views - # rather than a reader inheriting a figure from a comment (§13). + # rather than a reader inheriting a figure from a comment. total_available: Optional[int] = None # What the BOARD holds, against what this query can reach. justjoin.it # caps every query at 10,000 however many it matched, so these two are # different numbers and keeping only the first would let a run report - # "complete" while holding half the board (CLAUDE.md §21). + # "complete" while holding half the board. site_total: Optional[int] = None # The `meta.from` the server echoed. The endpoint states the offset it # actually served, so a page that came back as a repeat of an earlier @@ -291,7 +291,7 @@ def _driver(page): # stating twice: measured 2026-09-18, `a[href*="/job-offer/"]` was 200 # in the served markup and 18 in the live DOM after hydration. So # scrolling would REPLACE DOM nodes rather than add rows, and no row is - # read from the DOM anyway (CLAUDE.md §4). + # read from the DOM anyway. return { "count": lambda selector: len(page.query_selector_all(selector)), "sleep": page.wait_for_timeout, @@ -387,7 +387,7 @@ def _fetch_body(session, args, url: str) -> str: # timeout, and the two want opposite responses: a timeout deserves another # try at the same exit, a dead proxy deserves a different one. Catching # only the timeout type let this escape as a traceback in a sibling repo -# (CLAUDE.md §8). +#. _PROXY_ERROR_MARKERS = ( "ERR_PROXY_CONNECTION_FAILED", # nothing listening / refused "ERR_TUNNEL_CONNECTION_FAILED", # CONNECT rejected by the proxy @@ -460,7 +460,7 @@ def _launch_local(pw, args, pool): # and keeps nothing, so `sitekey`, `action`, `cData` and `chlPageData` # exist only inside that call. A static read of a challenge page — any # static read, however careful — cannot produce a solvable task - # (CLAUDE.md §19). It costs nothing on a page that never renders one. + #. It costs nothing on a page that never renders one. context.add_init_script(TURNSTILE_INTERCEPT_JS) return browser, context, context.new_page() @@ -685,7 +685,7 @@ def handle_captcha_if_present(page, args) -> bool: about the variant and the parameters for one are rejected for the other. WHAT THIS SITE ACTUALLY PUTS IN FRONT OF A RUN, measured 2026-09-18, - because CLAUDE.md §19 is explicit that a sentence about what a solver + because a sentence about what a solver can do is the most expensive thing this family can get wrong: * Cloudflare's **managed challenge**. justjoin.it sits behind @@ -712,7 +712,7 @@ def handle_captcha_if_present(page, args) -> bool: guards submitting an application — sending a CV — which this scraper never does. - That was measured rather than reasoned, because CLAUDE.md §18 is + That was measured rather than reasoned, because it is explicit that a configured captcha and a rendered one are different questions. In a live headless Chromium on 2026-09-18, seven seconds after load, on a rendered listing, on an offer page and on the @@ -730,7 +730,7 @@ def handle_captcha_if_present(page, args) -> bool: is why a grep of a capture finds one and a browser finds none. Neither is "unsolvable" — that word belongs to a PAGE with no widget - on it, not to a vendor (CLAUDE.md §19). If justjoin.it ever puts its v2 + on it, not to a vendor. If justjoin.it ever puts its v2 widget in front of a listing, 2Captcha solves exactly that with `RecaptchaV2TaskProxyless`, and the sitekey is published in the page source where this code already looks for it. What is true today is @@ -791,7 +791,7 @@ def handle_captcha_if_present(page, args) -> bool: # the served markup — a threshold of 100 would report "no content" # a few seconds after load on a perfectly good page. # - # CLAUDE.md §18 states the rule for the PARSER's marker set; this + # The marker rule is stated for the PARSER's marker set; this # is the same rule applied to the SOLVER's gate — and the reminder # that a gate is only as good as the selector it counts with. if challenge is None and already_rendered <= MIN_CARD_MATCHES: @@ -813,7 +813,7 @@ def handle_captcha_if_present(page, args) -> bool: "publishes one in its own config — using it.") else: # Raising a task without a sitekey buys a rejected token and - # an ERROR_CAPTCHA_UNSOLVABLE bill (CLAUDE.md §19). Say so + # an ERROR_CAPTCHA_UNSOLVABLE bill. Say so # instead. logger.warning( "A Cloudflare Turnstile is on this page but no sitekey " @@ -886,7 +886,7 @@ def _solve_budget(args, spent: int): gated — and only the SECOND call was counted. INHERITED EVIDENCE, from a SIBLING repo and not from justjoin.it — - labelled because §13 says a number you inherited is not a number you + labelled because a number you inherited is not a number you measured, and this one cannot be reproduced here: justjoin.it has never rendered a challenge to this scraper, so no page of this site has ever bought a solve at all. @@ -895,7 +895,7 @@ def _solve_budget(args, spent: int): challenge on every fetch (2026-09-17): ONE page bought THREE Turnstile solves — two from the uncounted call across two block attempts, one from the counted one — and every token was refused. The cap read as - enforced and was not (CLAUDE.md §17: a policy constant nothing reads is + enforced and was not (a policy constant nothing reads is the same defect as dead code). The fix is carried here anyway. A budget that is never exercised is @@ -1049,12 +1049,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = _classify(session.page, body, http_status, args.mode, args.route) - # The paid path is reached only for state "challenge" — Cloudflare's - # managed challenge, which IS a test and can be solved once - # `TURNSTILE_INTERCEPT_JS` has captured its parameters. It is bounded - # by SOLVES_PER_PAGE so a rotation loop cannot become a bill, and a + # The paid path is reached for any state page_flow.STATE_POLICY marks + # solvable — on this site that is `blocked`; there is no separate + # `challenge` state (see page_flow's docstring). Bounded by + # SOLVES_PER_PAGE so a rotation loop cannot become a bill, and a # sitekey-less detection refuses to build a task at all rather than - # paying for one the API will reject (CLAUDE.md §19). + # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): solves_bought += 1 @@ -1112,7 +1112,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -1183,7 +1183,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # endpoint answers with the whole payload, and a rendered listing # server-renders all 100 offers into its flight payload. Porting the # sibling repos' scroll loop here would be dead code that looks - # load-bearing (CLAUDE.md §4) — and worse than usual on this site, + # load-bearing — and worse than usual on this site, # because the grid is virtualised: scrolling REPLACES DOM nodes, and no # row is read from the DOM in the first place. @@ -1263,7 +1263,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # offers on it and publishes not one fact about any of them — # no title, no company, no salary, no id. Reported so the two # views are compared every run rather than a reader inheriting - # the figure from a comment (§13): the flight payload should + # the figure from a comment: the flight payload should # hold at least as many records as the JSON-LD names URLs, and # an inversion would mean the primary source had started # dropping rows. @@ -1487,7 +1487,7 @@ def scrape(args) -> int: # reporting path would lose a run that had already done all its work. total_enumerated = 0 # What the board says it holds, against what this query can reach. Both - # are read from the run rather than inherited from a comment (§13). + # are read from the run rather than inherited from a comment. site_total: Optional[int] = None pool = proxy_pool_from_args(args) @@ -1555,7 +1555,7 @@ def scrape(args) -> int: if not offer_slugs: # Nothing to fetch, so nothing is written — `save` # leaves the previous run's good output in place and - # no sidecar contradicts it (CLAUDE.md §9). + # no sidecar contradicts it. return finish_run( [], args.out, args.format, args.allow_empty, blocked=False, stop_reason="enumeration_empty", @@ -1575,7 +1575,7 @@ def scrape(args) -> int: # The board-wide total, for the cap arithmetic. One extra # request, and it buys the difference between a sidecar that # says "complete" and one that says "complete, and a 52% - # sample" (CLAUDE.md §21). Never fatal: a run without it is + # sample". Never fatal: a run without it is # still a good run, it just cannot state the share. if args.mode == "listings": try: @@ -1690,7 +1690,7 @@ def scrape(args) -> int: # nothing further to fetch, and this is the honest # terminating condition: a property of the DATA, not # of a CSS selector that may have been renamed - # (CLAUDE.md §7). + #. if not fresh_count: logger.info("Page %d added no rows not already seen " "— treating that as the end of the " @@ -1729,7 +1729,7 @@ def scrape(args) -> int: # Completeness, checked over the MERGED result rather than per page — a # per-page check cannot see a gap BETWEEN two pages, which is exactly - # where a short page hides (CLAUDE.md §8). + # where a short page hides. total_available = next((o.total_available for o in outcomes if o.total_available is not None), None) pages_available = next((o.pages_available for o in outcomes @@ -1739,10 +1739,10 @@ def scrape(args) -> int: if args.mode == "listings" and all_rows: if cap["capped_by_site"]: - # CLAUDE.md §21: "complete" and "exhaustive" are different + # "Complete" and "exhaustive" are different # words, and a run that says only "complete" is lying by # omission. Reported every run rather than inherited from a - # README measured on some other day (§13). + # README measured on some other day. logger.info("This query was CAPPED: justjoin.it reports at most " "%s results for any one query, and this one hit that " "ceiling. The board holds %s offer(s). Collected %d " @@ -1796,7 +1796,7 @@ def scrape(args) -> int: # One-per-run context, in the sidecar rather than repeated down a # column. What goes here is this run's OWN arithmetic about how much of - # the board it covered — which is what a bare "complete" hides (§21), + # the board it covered — which is what a bare "complete" hides, # and what stops a reader of the file inheriting a number from a README # that was measured on some other day. extra = {"mode": args.mode, "route": args.route, @@ -1813,7 +1813,7 @@ def scrape(args) -> int: # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and # `diff_runs.py` refuses to compare two runs that disagree on it - # (CLAUDE.md §21). + #. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters @@ -2061,12 +2061,16 @@ def parse_args(): p.add_argument("--dump-html", default=None, metavar="PATH", help="Save the exact HTML the parser is given, on success as " "well as failure. Useful when the row count is right but " - "a column comes back empty — see TROUBLESHOOTING.md.") + "a column comes back empty — see the README section on traps that look like bugs.") p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the # environment or .env when the flag was not given. An explicit flag wins. + # What the user TYPED, captured before the environment fills the gap: + # a JUSTJOIN_URL from .env is a default, not an instruction, and must + # not make `--mode offer` or `--mode facets` a usage error. + typed_url = args.url env_config.apply(args) # Every mode has a correct default URL, so `--url` is optional @@ -2076,17 +2080,22 @@ def parse_args(): # A listing URL and the filter flags do NOT conflict: a browsable # address is read back into the same filters the site's own front end # would send for it, and an explicit flag wins over anything read off - # the URL (CLAUDE.md §3's precedence rule, applied to filters). + # the URL (the precedence rule, applied to filters). if args.url: ok, why = is_supported_url(args.url) if not ok: p.error(why) implied = mode_for_url(args.url) - if implied and implied != args.mode: + if implied and implied != args.mode and not typed_url: + logger.info("JUSTJOIN_URL (%s) is a %s URL and --mode is %s; " + "it is ignored and --mode %s uses its own default.", + args.url, implied, args.mode, args.mode) + args.url = None + elif implied and implied != args.mode: # Refused rather than silently corrected. Reading an offer URL # under --mode listings would parse it with the wrong reader # and report zero rows for a perfectly good response — the - # failure CLAUDE.md §20 describes, where a served page that + # failure where a served page that # parses to nothing sends the reader to check the URL instead # of the code. p.error("--url is a %s URL but --mode is %s. %s is read by " diff --git a/product_parser.py b/product_parser.py index 43b047f..e60cbe7 100644 --- a/product_parser.py +++ b/product_parser.py @@ -3,7 +3,7 @@ Everything this project knows about Just Join IT lives here. The engines know how to drive a browser and nothing about job offers; `output_writer` knows the row schema and nothing about where a field came from. If you are -editing site knowledge anywhere else, it belongs in this file (CLAUDE.md §1). +editing site knowledge anywhere else, it belongs in this file. WHAT THIS SITE SERVES --------------------- @@ -15,7 +15,7 @@ the one its front end calls for every page after the first. HTTP 200 with NO headers, NO cookies and NO key, 100 complete records in 253 KB. Offset pagination (`from`/`itemsCount`), so pages are addressable and - `--concurrency` is meaningful (CLAUDE.md §7). + `--concurrency` is meaningful. 2. The rendered listing page — `/job-offers/all-locations` and friends. Server-renders the first 100 offers into the React Server Component @@ -31,7 +31,7 @@ request** from this address — bare, with browser headers, with the `x-api-version` header its own bundle sends, and on every path tried. The `/api/candidate-api/` prefix on the main host is a same-origin proxy in -front of it, and that proxy is what works. This is CLAUDE.md §21's lesson +front of it, and that proxy is what works. This is the lesson stated as code: gating is per-ROUTE, not per-site, and the route the front end actually calls is the one to ask for. @@ -92,7 +92,7 @@ disclose a salary whose `original` is not at index 0 — 40% — and one of them reads 39.15 CHF where the employer offers 180 PLN. -This is CLAUDE.md §4's "structured data is a fact" rule: `currencySource` +This is the "structured data is a fact" rule: `currencySource` is the site telling you which number is real, so select on it and never on position. The all-null conversion entries that lead an undisclosed row are also why the selection cannot be "the first entry with a number in it" — @@ -110,7 +110,7 @@ here, or a consumer grouping by unit gets two buckets for one unit. * 461 of 1000 rows disclose no salary at all — `from`/`to` are null on the `original` entry. Those stay null. They are NOT zero: a zero would - drag every average a reader computes (CLAUDE.md §21). + drag every average a reader computes. WHAT THE SITE CAPS ------------------ @@ -132,7 +132,7 @@ count of every other vendor marker was zero. This is a statement about what was MEASURED, not about what 2Captcha can -solve (CLAUDE.md §19). Should this site ever put a reCAPTCHA v2 in front of +solve. Should this site ever put a reCAPTCHA v2 in front of a listing, `captcha_solver.py` solves exactly that with `RecaptchaV2TaskProxyless`, and the sitekey is published in the page source. """ @@ -159,7 +159,7 @@ # Both answer. `www.` 301s to the bare host for HTML and for the endpoint # alike, so a URL rebuilt with either prefix reaches the same page — unlike -# mediamarkt, where two hosts of ten refuse `www.` outright (CLAUDE.md §5). +# mediamarkt, where two hosts of ten refuse `www.` outright. HOSTS = (HOST, WWW_HOST) # The host the site's own config calls `baseApiUrl`. Named here ONLY so @@ -219,7 +219,7 @@ # Every value below was checked against the live endpoint on 2026-09-18 by # asking for it and comparing `meta.totalItems` against the unfiltered # 10,000. A parameter that answered 10,000 with an identical first row is -# NOT a filter, whatever its name suggests — CLAUDE.md §21 found the same +# NOT a filter, whatever its name suggests — the same was found # thing on BBB, where `showOnlyAccredited` does nothing at all. # # These filter: measured totalItems @@ -284,7 +284,7 @@ # `from=10000` and `from=10100` are both HTTP 500 — walking off the end # manufactures an error that reads like a bug in this scraper. Plan against -# the cap instead (CLAUDE.md §21). +# the cap instead. MAX_FROM = 10_000 # --------------------------------------------------------------------------- @@ -305,7 +305,7 @@ # library sends if you do not set one. Nothing else was refused — this is # not a general bot gate, and sending a browser UA buys nothing over # sending an honest one. The engines take theirs from the browser -# (CLAUDE.md §8); this constant is for the HTTP paths. +# instead; this constant is for the HTTP paths. HTTP_USER_AGENT = "Mozilla/5.0 (compatible; justjoin-scraper; +https://github.com/2scraper/justjoin-scraper)" _REFUSED_USER_AGENT_RE = re.compile(r"^Python-urllib/", re.I) @@ -317,7 +317,7 @@ def user_agent_is_refused(value: Optional[str]) -> bool: Exists so the HTTP paths can assert they are not about to send it, and so the suite can pin the measurement. A stdlib default that produces a 403 a long way from its cause is exactly the confusing failure - CLAUDE.md §3 wants prevented. + the configuration rules exist to prevent. """ return bool(value) and bool(_REFUSED_USER_AGENT_RE.match(value.strip())) @@ -464,7 +464,7 @@ def _bare_host(netloc: str) -> str: # It is part of the slug and is NOT the id — the id is the `guid`, which # appears nowhere in the URL. So `sku` cannot be recovered from a URL on # this site, and `sku_from_url` says so by returning None rather than -# handing back a slug that would not join against anything (CLAUDE.md §5). +# handing back a slug that would not join against anything. _SKU_IN_URL_RE = None @@ -769,7 +769,7 @@ def filters_from_listing_url(url: str) -> Filters: def page_url(url: str, page: int) -> Optional[str]: """None for a browsable listing URL, and that is the finding. - CLAUDE.md §7 asks for a `page_url` that reconstructs page N when no + Pagination needs a `page_url` that reconstructs page N when no next-link matches. On justjoin.it's rendered pages there is no such address, and the failure mode is the dangerous one: `?page=2` on a listing URL does not error and does not return an empty page — it @@ -792,8 +792,7 @@ def api_page_url(url: str, page: int, *, items: int = DEFAULT_ITEMS_COUNT """Page N of an offers-endpoint URL, by rewriting `from`. Offset pagination, so page N's address is knowable without fetching - page N-1 — which is what makes `--concurrency` meaningful here - (CLAUDE.md §7). Returns None past the site's cap rather than building a + page N-1 — which is what makes `--concurrency` meaningful here. Returns None past the site's cap rather than building a URL that answers HTTP 500. """ if page < 1: @@ -847,7 +846,7 @@ def category_from_url(url: str) -> Optional[str]: # --------------------------------------------------------------------------- # Every candidate marker below was COUNTED on pages this site plainly -# served before being kept, which is CLAUDE.md §18's rule and which threw +# served before being kept, which is the rule and which threw # two of them out: # # `akamai` 4 occurrences on a served offer page — in the job @@ -855,7 +854,7 @@ def category_from_url(url: str) -> Optional[str]: # GCP, AWS, Azure lub Akamai Linode" among the clouds a # candidate should know. A marker that fires on a real # page is worse than no marker. -# `cf-turnstile` not carried, for the reason CLAUDE.md §8 gives: the +# `cf-turnstile` not carried, for this reason: the # 2Captcha Scraping Browser's auto-solve extension # injects it into every page it loads, so it fires on # good pages and has been measured ABSENT from real @@ -888,15 +887,15 @@ def category_from_url(url: str) -> Optional[str]: ) # The site's own asset host. Every page justjoin.it actually serves is -# built out of it; an interstitial would not be (CLAUDE.md §8's -# `assets.mmsrg.com` trick, now on its third site). Counted 2026-09-18: +# built out of it; an interstitial would not be (the own-asset +# test, proven on sibling sites). Counted 2026-09-18: # 5 references on the homepage, 4 on an offer page, 5 on a filtered # listing. SITE_ASSET_MARKERS = ("public.justjoin.it", "imgproxy.justjoinit.tech", "_next/static") # The threshold is 1 rather than the 2-3 this trick usually takes, and -# deliberately: CLAUDE.md §17's classification-order trap is that a +# deliberately: the classification-order trap is that a # minimal-but-real page with fewer references than the measured average # comes back "blocked". The asset check is a WEAK secondary signal here, # consulted only after the unambiguous ones. @@ -908,7 +907,7 @@ def category_from_url(url: str) -> Optional[str]: def _normalised_head(html: str) -> str: """Lower-cased, entity-unescaped first few KB, for marker matching. - CLAUDE.md §20: an edge's refusal page reaches a parser spelled two ways + An edge's refusal page reaches a parser spelled two ways — entity-escaped over a raw HTTP client, plain out of a browser DOM. So unescape before matching, and bound it: a refusal page is a few hundred bytes, and unescaping 1.8 MB of listing would let a job title deep in @@ -945,7 +944,7 @@ def count_site_assets(html: Optional[str]) -> int: # # "publicRuntimeConfig":{...,"googleRecaptchaV2Key":"6LeTZ-Er...",...} # -# CLAUDE.md §18 says to grep a GOOD page for the site's own captcha config +# Grep a GOOD page for the site's own captcha config # rather than building a marker set from a vendor list, and this is that # grep. What it found, and what the engines' docstrings state at length: # the key is v2 by the site's own field name, the only string referencing @@ -955,7 +954,7 @@ def count_site_assets(html: Optional[str]) -> int: # # So this function has never returned a key that mattered. It is here for # the day one does: a solvable challenge whose key the page publishes is -# better answered with that key than refused (§19 — never pay for a task +# better answered with that key than refused (never pay for a task # the API will reject, but do not decline a task it would accept). _RECAPTCHA_KEY_RE = re.compile( r'googleRecaptchaV2Key\\?"\s*:\s*\\?"([0-9A-Za-z_-]{20,})') @@ -1164,7 +1163,7 @@ def _salary_option(entry: Dict[str, Any]) -> SalaryOption: minimum=quoted_min, maximum=quoted_max, # An ISO 4217 code the site states — the most trustworthy rung of - # CLAUDE.md §4's ladder. Never defaulted: a null means the entry + # the ladder. Never defaulted: a null means the entry # named no currency. currency=_upper_or_none(entry.get("currency")), # Lower-cased: the site spells the same unit "month" and "Month". @@ -1320,7 +1319,7 @@ def _category_key(record: Dict[str, Any]) -> Optional[str]: The flight payload states `categoryId`, an integer, and publishes no table mapping it to a key. So a `--route ssr` row's category is null rather than a number nobody can read — a guess presented as a fact is - the one thing CLAUDE.md §8 will not allow. + a guess presented as a fact, which this project never writes. `parentKey` is read and discarded: it was null on all 1,040 records measured, so it gets no column. See the note in `output_writer`. @@ -1352,7 +1351,7 @@ def _hybrid_days(record: Dict[str, Any]) -> Tuple[Optional[int], Optional[int]]: # `applyMethod` is "external" on 800 of 1,000 rows and "form" on 200, and # `applyUrl` is non-null on exactly the 800 externals. So the null is the # site saying "applied to on our own page", not a missing field — the same -# shape as CLAUDE.md §21's "use the site's own should-I-show-this flag". +# shape as the "use the site's own should-I-show-this flag". APPLY_METHODS = ("external", "form") @@ -1364,8 +1363,7 @@ def _row_from_record(record: Dict[str, Any], *, data_source: str, Returns None for a record with no id, which is what a payload shape this parser does not understand looks like. Failing loudly beats - emitting a row of nulls that a consumer cannot tell from a real offer - (CLAUDE.md §8). + emitting a row of nulls that a consumer cannot tell from a real offer. """ if not isinstance(record, dict): return None @@ -1492,7 +1490,7 @@ def flight_payload(html: Optional[str]) -> str: # An offer record in the flight payload always opens with `applyUrl`, which # is the first key in the site's own serialisation order. Anchoring on a -# KEY rather than on a CSS class is the same reasoning CLAUDE.md §4 gives +# KEY rather than on a CSS class is the same reasoning as # for anchoring on a URL pattern: the class names here are build hashes and # the key names are the site's API contract. _FLIGHT_RECORD_RE = re.compile(r'\{"applyUrl":') @@ -1590,8 +1588,8 @@ def site_total_from_flight(payload: str) -> Optional[int]: def jsonld_blocks(html: Optional[str]) -> List[Any]: """Every JSON-LD block on a page. - Counted before a line of the parser was written, which CLAUDE.md §15 - asks for and which decided this repo's whole shape: a rendered listing + Counted before a line of the parser was written, which + is the right order and which decided this repo's whole shape: a rendered listing page publishes exactly TWO blocks, a `CollectionPage` and a `BreadcrumbList`. The `CollectionPage` carries `hasPart` — a list of 100 bare URLs with `@type: CreativeWork` and NOTHING else: no title, no @@ -1638,7 +1636,7 @@ def itemlist_urls(html: Optional[str]) -> List[str]: # The one this scraper walks. justjoin.it advertises eight sitemaps in its # robots.txt, including an `expired-jobs.xml`; that one is deliberately NOT # carried here, because a constant nothing reads is the same defect as dead -# code (CLAUDE.md §17). Anyone who wants the expired board can point +# code. Anyone who wants the expired board can point # `--slugs-file` at it. SITEMAP_INDEX = BASE + "/sitemaps/active-jobs.xml" @@ -1731,8 +1729,7 @@ class ListingPage: The shape every engine in this family expects from a listing parser. `total_items` is the site's own capped figure and `site_total` is what the board actually holds; keeping both is what lets a run report - `complete` honestly while still saying it holds half the board - (CLAUDE.md §21). + `complete` honestly while still saying it holds half the board. """ rows: List[JobPosting] = field(default_factory=list) @@ -1767,7 +1764,7 @@ def parse_offers_response(text: Optional[str], *, url: str = "", scraped_at: Optional[str] = None) -> ListingPage: """Rows from one offers-endpoint response. - `page` is threaded in rather than defaulted. CLAUDE.md §18 names the + `page` is threaded in rather than defaulted. A sibling repo hit the arithmetic bug that prevents: `position` restarts at 1 on every page, so a `page` column that is 1 on every row makes `page`+`position` collide and the position column worthless. @@ -1915,7 +1912,7 @@ def parse_facets_response(text: Optional[str], *, """The board's own taxonomy with its own counts. A different KIND of object from a job posting, so it gets its own row - class rather than being bent into `JobPosting` (CLAUDE.md §9). What + class rather than being bent into `JobPosting`. What makes it worth a mode: these counts are NOT capped at 10,000. They describe all 19,381 offers, which makes them the tool for planning a run that reaches past the cap — 25 category runs of at most 1,991 rows @@ -1977,7 +1974,7 @@ def cap_report(total_items: Optional[int], site_total: Optional[int], rows_collected: int) -> Dict[str, Any]: """The arithmetic a sidecar records beside `status: complete`. - CLAUDE.md §21: "complete" and "exhaustive" are different words. A run + "Complete" and "exhaustive" are different words. A run that fetches all 10,000 reachable rows really did fetch everything the site will serve for that query, and it is also a 52% sample of a board holding 19,381. Recording only "complete" is lying by omission, and the @@ -2029,12 +2026,12 @@ def cap_report(total_items: Optional[int], site_total: Optional[int], # # These five are pure functions of the parsed arguments, with no driver and # no JavaScript in them, and they live here rather than in each engine for -# the reason CLAUDE.md §1 gives for `page_flow`: three copies of "which URL +# the reason `page_flow` exists: three copies of "which URL # does this run fetch" would drift, and the drift would be silent — one # engine reading the endpoint where its twin reads the rendered page, on # the same command line. # -# It is the same argument §17 makes about a shared module's signature, one +# It is the same argument as for a shared module's signature, one # step earlier: an engine cannot disagree with its twins about a decision # it does not make. @@ -2050,7 +2047,7 @@ def filters_from_args(args) -> Filters: A `--url` naming a browsable listing is translated into the same filters the site's own front end would send for it, so a user can paste an address out of their browser and the flags stay optional. Explicit - flags win over anything read from the URL — CLAUDE.md §3's precedence + flags win over anything read from the URL — the precedence rule, applied to filters rather than to credentials. """ from_url = (filters_from_listing_url(args.url) @@ -2284,7 +2281,7 @@ def parse_for_mode(body: str, url: str, args, page_num: int = 1): # classify a delisted offer as `parse_error` there: a RETRY, a debug dump # and a wasted fetch, for an address that will never exist again. # -# Found by the canary on its first run, which is CLAUDE.md §15 exactly — +# Found by the canary on its first run, which is why live runs come first — # the first slug in the sitemap had been taken down between the sitemap # being generated and the run reading it, and every engine called it a # parse_error because the navigation status was being discarded. @@ -2305,7 +2302,7 @@ def detect_page_state(body: Optional[str], status: Optional[int] = None, """Name what justjoin.it answered with. The signals are ordered by how much each one PROVES, not by how cheap - it is to test — CLAUDE.md §17's classification-order trap, where a + it is to test — the classification-order trap, where a threshold heuristic ran ahead of an unambiguous marker and reported exit 3 for a correct answer. So: @@ -2324,7 +2321,7 @@ def detect_page_state(body: Optional[str], status: Optional[int] = None, 6. a substantial body this parser could not read is `parse_error` — NOT "zero offers". A served response that parses to nothing is this repo's bug, and reporting it as an empty board sends the - reader to check their filters instead of the parser (§20); + reader to check their filters instead of the parser; 7. anything else is `unknown`: wait and retry, do not spend. """ text = body or "" @@ -2372,7 +2369,7 @@ def detect_page_state(body: Optional[str], status: Optional[int] = None, # (4) Walking off the end of a capped result set. `from=10000` and # `from=10100` are both HTTP 500 — the site manufactures an error that - # reads like a bug in this scraper, so name it for what it is (§21). + # reads like a bug in this scraper, so name it for what it is. if status == 500 and _asks_past_cap(url): return "cap_exceeded" diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index b41ec85..dba93a1 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -4,7 +4,7 @@ Behaviour is identical to `playwright_scraper.py`: same flags, same modes, same exit codes, same output. Playwright is the primary engine; this one exists so the family's claim of three interchangeable engines is testable -rather than asserted (CLAUDE.md §6). +rather than asserted. Everything this file knows about justjoin.it it gets from `product_parser.py`, and every retry / solve / blocked decision from @@ -238,7 +238,7 @@ class PageOutcome: # How many entries the same response's ItemList JSON-LD indexes. 326 # against 390 records on 2026-09-18 — the gap is why the parser reads # the React Query cache, and recording it per run means nobody inherits - # the figure from a comment (§13). + # the figure from a comment. urls_in_itemlist: Optional[int] = None pages_available: Optional[int] = None # The page number the SERVER answered with, read back out of the Apollo @@ -585,7 +585,7 @@ def _solve_budget(args, spent: int): page is classified and once after — and only the SECOND call was counted. INHERITED EVIDENCE, from a SIBLING repo and not from justjoin.it — labelled - because §13 says a number you inherited is not a number you measured, + because a number you inherited is not a number you measured, and this one cannot be reproduced here: justjoin.it has never rendered a challenge to this scraper, so no page of this site has ever bought a solve at all. @@ -593,7 +593,7 @@ def _solve_budget(args, spent: int): On that sibling, from a datacenter address that meets a real Cloudflare challenge on every fetch (2026-09-17): ONE page bought THREE Turnstile solves, and every token was refused. The cap read as enforced and was - not (CLAUDE.md §17). Both call sites now go through here. + not. Both call sites now go through here. The fix is carried here anyway. A budget that is never exercised is still the difference between a bill and no bill on the day it is. @@ -636,7 +636,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # `exit_failed` holds Chromium's own name for a proxy fault, or None. # Named and shaped exactly as in the other two engines: the give-up # branch below reports it, and three engines that structure this - # differently drift (CLAUDE.md §6). + # differently drift. load_failed, exit_failed = False, None # The navigation's HTTP status, kept rather than discarded — see # the Playwright engine for what discarding it cost. @@ -714,15 +714,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = page_flow.classify(html, http_status, page.url, args.mode, args.route) - # The paid path is reached only for state "challenge" — Cloudflare's - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": that page carries no widget, so a solve there would be a - # charge for nothing. Mirrors the other two engines. - # - # The paid path is reached only for state "challenge", which no - # capture of this site has ever produced. Wired up because a bot - # manager can be switched on between deploys, and bounded by - # SOLVES_PER_PAGE so a speculative path cannot become a bill. + # The paid path is reached for any state page_flow.STATE_POLICY marks + # solvable — on this site that is `blocked`; there is no separate + # `challenge` state (see page_flow's docstring). Bounded by + # SOLVES_PER_PAGE so a rotation loop cannot become a bill, and a + # sitekey-less detection refuses to build a task at all rather than + # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): solves_bought += 1 @@ -775,7 +772,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # 'current'` instead of retrying. It survived because the # donor's site never produced a retryable state without a # pool; this one does, on its own `--mode facets` path, and - # a live run found it in a minute (CLAUDE.md §15, §16). + # a live run found it in a minute. pause = args.retry_delay * (block_attempt + 1) logger.warning("Page %d came back as %s — re-fetching " "through the same access path in %.1fs " @@ -784,7 +781,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -1043,7 +1040,7 @@ def scrape(args) -> int: # The board-wide total, for the cap arithmetic. One extra # request, and it buys the difference between a sidecar that # says "complete" and one that says "complete, and a 52% - # sample" (CLAUDE.md §21). Never fatal. + # sample". Never fatal. if args.mode == "listings": try: site_total = parse_count_response( @@ -1122,7 +1119,7 @@ def scrape(args) -> int: # Completeness, checked over the MERGED result rather than per page — a # per-page check cannot see a gap BETWEEN two pages, which is exactly - # where a short page hides (CLAUDE.md §8). + # where a short page hides. total_available = next((o.total_available for o in outcomes if o.total_available is not None), None) pages_available = next((o.pages_available for o in outcomes @@ -1132,10 +1129,10 @@ def scrape(args) -> int: if args.mode == "listings" and all_rows: if cap["capped_by_site"]: - # CLAUDE.md §21: "complete" and "exhaustive" are different + # "Complete" and "exhaustive" are different # words, and a run that says only "complete" is lying by # omission. Reported every run rather than inherited from a - # README measured on some other day (§13). + # README measured on some other day. logger.info("This query was CAPPED: justjoin.it reports at most " "%s results for any one query, and this one hit that " "ceiling. The board holds %s offer(s). Collected %d " @@ -1189,7 +1186,7 @@ def scrape(args) -> int: # One-per-run context, in the sidecar rather than repeated down a # column. What goes here is this run's OWN arithmetic about how much of - # the board it covered — which is what a bare "complete" hides (§21), + # the board it covered — which is what a bare "complete" hides, # and what stops a reader of the file inheriting a number from a README # that was measured on some other day. extra = {"mode": args.mode, "route": args.route, @@ -1206,7 +1203,7 @@ def scrape(args) -> int: # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and # `diff_runs.py` refuses to compare two runs that disagree on it - # (CLAUDE.md §21). + #. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters @@ -1426,6 +1423,10 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What the user TYPED, captured before the environment fills the gap: + # a JUSTJOIN_URL from .env is a default, not an instruction, and must + # not make `--mode offer` or `--mode facets` a usage error. + typed_url = args.url env_config.apply(args) # Spelled identically to the other two engines. @@ -1439,10 +1440,15 @@ def parse_args(): if not ok: p.error(why) implied = mode_for_url(args.url) - if implied and implied != args.mode: + if implied and implied != args.mode and not typed_url: + logger.info("JUSTJOIN_URL (%s) is a %s URL and --mode is %s; " + "it is ignored and --mode %s uses its own default.", + args.url, implied, args.mode, args.mode) + args.url = None + elif implied and implied != args.mode: # Refused rather than silently corrected: reading a careers URL # under --mode listings would parse it with the wrong reader and - # report zero rows for a perfectly good page (CLAUDE.md §20). + # report zero rows for a perfectly good page. p.error("--url is a %s URL but --mode is %s. %s is read by " "--mode %s; pass that, or drop --url and let --mode %s " "use its own default." diff --git a/scraper_api_client.py b/scraper_api_client.py index 2d57e1b..e595234 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -41,7 +41,7 @@ # TWOCAPTCHA_KEY and the CDP endpoint are read from .env as # $JUSTJOIN_CDP_ENDPOINT, so neither needs to be typed — a secret in - # argv is readable by anything that can run `ps` (CLAUDE.md §3). + # argv is readable by anything that can run `ps`. """ import argparse @@ -303,7 +303,7 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: # nothing" — for a body the site never served. Measured 2026-09-18: the # Scraper API rendering justjoin.it's 1.8 MB listing page came back with # upstream HTTP 500 and ZERO bytes, and this path called it an empty - # result. CLAUDE.md §8: blocked, empty and failed are three different + # result. Blocked, empty and failed are three different # answers, and a consumer that cannot tell them apart reads an upstream # failure as a board with no jobs on it. if not page_flow.should_parse(state): @@ -337,7 +337,7 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: # from the repo it was copied from, where the row class has no # `category` field either — so it set an attribute that `asdict()` never # reads and that reached neither the JSON nor the CSV. It looked like it - # was labelling rows and was doing nothing at all (CLAUDE.md §16: a + # was labelling rows and was doing nothing at all (a # copied file is the one with no evidence behind it). # # `--category` labels the RUN, and the sidecar is where a run-level fact diff --git a/selenium_scraper.py b/selenium_scraper.py index 6edb9c8..c6e8ab2 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -4,7 +4,7 @@ Behaviour is identical to `playwright_scraper.py`: same flags, same modes, same exit codes, same output. Playwright is the primary engine; this one exists so the family's claim of three interchangeable engines is testable -rather than asserted (CLAUDE.md §6). +rather than asserted. Everything this file knows about justjoin.it it gets from `product_parser.py`, and every retry / solve / blocked decision from @@ -138,7 +138,7 @@ class PageOutcome: # How many entries the same response's ItemList JSON-LD indexes. 326 # against 390 records on 2026-09-18 — the gap is why the parser reads # the React Query cache, and recording it per run means nobody inherits - # the figure from a comment (§13). + # the figure from a comment. urls_in_itemlist: Optional[int] = None pages_available: Optional[int] = None # The page number the SERVER answered with, read back out of the Apollo @@ -488,7 +488,7 @@ def _solve_budget(args, spent: int): page is classified and once after — and only the SECOND call was counted. INHERITED EVIDENCE, from a SIBLING repo and not from justjoin.it — labelled - because §13 says a number you inherited is not a number you measured, + because a number you inherited is not a number you measured, and this one cannot be reproduced here: justjoin.it has never rendered a challenge to this scraper, so no page of this site has ever bought a solve at all. @@ -496,7 +496,7 @@ def _solve_budget(args, spent: int): On that sibling, from a datacenter address that meets a real Cloudflare challenge on every fetch (2026-09-17): ONE page bought THREE Turnstile solves, and every token was refused. The cap read as enforced and was - not (CLAUDE.md §17). Both call sites now go through here. + not. Both call sites now go through here. The fix is carried here anyway. A budget that is never exercised is still the difference between a bill and no bill on the day it is. @@ -611,11 +611,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = page_flow.classify(html, http_status, d["current_url"](), args.mode, args.route) - # The paid path is reached only for state "challenge" — Cloudflare's - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": the hard "You have been blocked" page carries no widget - # and no sitekey, so a solve there would be a charge for nothing. - # Bounded by SOLVES_PER_PAGE. Mirrors playwright_scraper. + # The paid path is reached for any state page_flow.STATE_POLICY marks + # solvable — on this site that is `blocked`; there is no separate + # `challenge` state (see page_flow's docstring). Bounded by + # SOLVES_PER_PAGE so a rotation loop cannot become a bill, and a + # sitekey-less detection refuses to build a task at all rather than + # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): solves_bought += 1 @@ -660,7 +661,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # unguarded while the Playwright engine guarded it, so any # retryable state on a run with no proxy died with # `AttributeError: 'NoneType' object has no attribute - # 'current'` instead of retrying (CLAUDE.md §16: copied core + # 'current'` instead of retrying (copied core # is untested core). pause = args.retry_delay * (block_attempt + 1) logger.warning("Page %d came back as %s — re-fetching " @@ -670,7 +671,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -928,7 +929,7 @@ def scrape(args) -> int: # The board-wide total, for the cap arithmetic. One extra # request, and it buys the difference between a sidecar that # says "complete" and one that says "complete, and a 52% - # sample" (CLAUDE.md §21). Never fatal. + # sample". Never fatal. if args.mode == "listings": try: site_total = parse_count_response( @@ -1006,7 +1007,7 @@ def scrape(args) -> int: # Completeness, checked over the MERGED result rather than per page — a # per-page check cannot see a gap BETWEEN two pages, which is exactly - # where a short page hides (CLAUDE.md §8). + # where a short page hides. total_available = next((o.total_available for o in outcomes if o.total_available is not None), None) pages_available = next((o.pages_available for o in outcomes @@ -1016,10 +1017,10 @@ def scrape(args) -> int: if args.mode == "listings" and all_rows: if cap["capped_by_site"]: - # CLAUDE.md §21: "complete" and "exhaustive" are different + # "Complete" and "exhaustive" are different # words, and a run that says only "complete" is lying by # omission. Reported every run rather than inherited from a - # README measured on some other day (§13). + # README measured on some other day. logger.info("This query was CAPPED: justjoin.it reports at most " "%s results for any one query, and this one hit that " "ceiling. The board holds %s offer(s). Collected %d " @@ -1073,7 +1074,7 @@ def scrape(args) -> int: # One-per-run context, in the sidecar rather than repeated down a # column. What goes here is this run's OWN arithmetic about how much of - # the board it covered — which is what a bare "complete" hides (§21), + # the board it covered — which is what a bare "complete" hides, # and what stops a reader of the file inheriting a number from a README # that was measured on some other day. extra = {"mode": args.mode, "route": args.route, @@ -1090,7 +1091,7 @@ def scrape(args) -> int: # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and # `diff_runs.py` refuses to compare two runs that disagree on it - # (CLAUDE.md §21). + #. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters @@ -1306,6 +1307,10 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What the user TYPED, captured before the environment fills the gap: + # a JUSTJOIN_URL from .env is a default, not an instruction, and must + # not make `--mode offer` or `--mode facets` a usage error. + typed_url = args.url env_config.apply(args) # Spelled identically to the other two engines. @@ -1319,10 +1324,15 @@ def parse_args(): if not ok: p.error(why) implied = mode_for_url(args.url) - if implied and implied != args.mode: + if implied and implied != args.mode and not typed_url: + logger.info("JUSTJOIN_URL (%s) is a %s URL and --mode is %s; " + "it is ignored and --mode %s uses its own default.", + args.url, implied, args.mode, args.mode) + args.url = None + elif implied and implied != args.mode: # Refused rather than silently corrected: reading a careers URL # under --mode listings would parse it with the wrong reader and - # report zero rows for a perfectly good page (CLAUDE.md §20). + # report zero rows for a perfectly good page. p.error("--url is a %s URL but --mode is %s. %s is read by " "--mode %s; pass that, or drop --url and let --mode %s " "use its own default." diff --git a/smoke_test.py b/smoke_test.py index 99c72b3..058a421 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2,7 +2,7 @@ smoke_test.py — the offline suite for justjoin-scraper. One file of plain functions with inline fixtures. No pytest, no conftest, no -fixtures directory (CLAUDE.md §10); `tests/test_smoke.py` wraps this as a +fixtures directory; `tests/test_smoke.py` wraps this as a single pytest test so `pytest` works as an entry point without a second copy of the checks. @@ -30,7 +30,7 @@ Three values ARE scrubbed, and the fixtures say so rather than leaving a reader to wonder why a captured page does not match the live one -(CLAUDE.md §10): two `applyUrl` form ids on an external ATS and the +: two `applyUrl` form ids on an external ATS and the `traceId` of the request that produced the 404. None of them granted anything — the form ids are public and the trace expired — but all three are 32-hex strings, which is the shape this repo's own credential scan @@ -128,7 +128,7 @@ def skip(group, reason): # fill: `companyUrl` and `companySize` were non-null on 5 of 40 sampled # records. Without it, `check_no_column_is_null_on_every_row_of_every_route` # would report them as columns nothing ever populates — which is what -# CLAUDE.md §9 says to delete, and would be the wrong conclusion drawn from +# the null-column rule says to delete, and would be the wrong conclusion drawn from # a fixture rather than from the site. DETAIL_WITH_COMPANY_JSON = "{\"id\": \"8aaed32d-6ded-4a51-bf05-32d0af529d6f\", \"slug\": \"astek-polska-senior-angular-developer-warszawa-mazowieckie--javascript-b28b4fcc\", \"title\": \"Senior Angular Developer\", \"experienceLevel\": \"senior\", \"category\": {\"key\": \"javascript\", \"parentKey\": null}, \"companyName\": \"ASTEK Polska\", \"companyUrl\": \"http://astek.pl/\", \"body\": \"

Astek

Za\u0142o\u017cona w 1988 roku we Francji Grupa Astek jest \u015bwiatowym partnerem w obszarze doradztwa in\u017cynieryjnego oraz IT. Dzi\u0119ki swojej wiedzy specjalistycznej w wielu sektorach przemys\u0142owych i technologicznych, Astek \", \"locationId\": \"0e4697c2-ed6c-47d2-8f60-68db61d925d2\", \"city\": \"Warszawa (Mazowieckie)\", \"street\": \"Centrum\", \"countryCode\": \"PL\", \"latitude\": 52.32121919999999, \"longitude\": 21.1036813, \"companySize\": \"501+\", \"informationClause\": \"Informujemy, \u017ce administratorem danych jest ASTEK Polska sp. z o.o. z siedzib\u0105 w Warszawie, Al. Jana Paw\u0142a II 22, 00-133 Warszawa (dalej jako \\\"administrator\\\"). Masz prawo do \u017c\u0105dania dost\u0119pu do swoich danych osobowych, ich sprostowania, usuni\u0119cia lub ograniczenia przetwarzania, prawo do wniesienia sprzeciwu wobec przetwarzania, a tak\u017ce prawo do przenoszenia danych oraz wniesienia skargi do organu nadzorczego. Dane osobowe przetwarzane b\u0119d\u0105 w celu realizacji procesu rekrutacji. Podanie danych w zakresie wynikaj\u0105cym z ustawy z dnia 26 czerwca 1974 r. Kodeks pracy jest obowi\u0105zkowe. W pozosta\u0142ym zakresie podanie danych jest dobrowolne. Odmowa podania danych obowi\u0105zkowych mo\u017ce skutkowa\u0107 brakiem mo\u017cliwo\u015bci przeprowadzenia procesu rekrutacji. Administrator przetwarza dane obowi\u0105zkowe na podstawie ci\u0105\u017c\u0105cego na nim obowi\u0105zku prawnego, za\u015b w zakresie danych dodatkowych podstaw\u0105 przetwarzania jest zgoda. Dane osobowe b\u0119d\u0105 przetwarzane do czasu zako\u0144czenia post\u0119powania rekrutacyjnego i przez okres mo\u017cliwo\u015bci dochodzenia ewentualnych roszcze\u0144, a w przypadku wyra\u017cenia zgody na udzia\u0142 w przysz\u0142ych post\u0119powaniach rekrutacyjnych - do czasu wycofania tej zgody. Zgoda na przetwarzanie danych osobowych mo\u017ce zosta\u0107 wycofana w dowolnym momencie. W razie wycofania zg\u00f3d prosimy o kontakt e-mailowy na adres privacy@astek.net lub pisemnie na adres administratora. Odbiorc\u0105 danych jest serwis Just Join IT oraz inne podmioty, kt\u00f3rym powierzyli\u015bmy przetwarzanie danych w zwi\u0105zku z rekrutacj\u0105.\\n\\nProcedura dokonywania zg\u0142osze\u0144 przez sygnalist\u00f3w w ASTEK Polska sp. z o. o.: https://astek.pl/sygnalisci/\\nSzczeg\u00f3\u0142owe informacje dotycz\u0105ce przetwarzania znajdzie Pan/Pani: http://astek.pl/polityka-prywatnosci\", \"futureConsent\": \"Zgadzam si\u0119 na przetwarzanie moich danych osobowych przez ASTEK Polska sp. z o.o. w celu realizacji przysz\u0142ych proces\u00f3w rekrutacji.\", \"customConsent\": \"Zgadzam si\u0119 na przetwarzanie moich danych osobowych przez ASTEK Polska sp. z o.o. w celu realizacji procesu rekrutacji. Zapozna\u0142am/em si\u0119 z informacj\u0105 o przetwarzaniu danych osobowych dost\u0119pn\u0105 pod linkiemhttp://astek.pl/polityka-prywatnosci\", \"companyLogoUrl\": \"https://public.justjoin.it/companies/logos/original/3de9aa63e2331cebd70d66e2b523fa1560bdd100.png?1670599242\", \"employmentTypes\": [{\"from\": 4558.0, \"fromPerUnit\": 217.032728535463, \"to\": 5697.0, \"toPerUnit\": 271.29091066932875, \"currency\": \"CHF\", \"currencySource\": \"conversion\", \"type\": \"b2b\", \"unit\": \"Day\", \"gross\": false}, {\"from\": 4122.0, \"fromPerUnit\": 196.301676416317, \"to\": 5153.0, \"toPerUnit\": 245.37709552039624, \"currency\": \"GBP\", \"currencySource\": \"conversion\", \"type\": \"b2b\", \"unit\": \"Day\", \"gross\": false}, {\"from\": 5522.0, \"fromPerUnit\": 262.950302392848, \"to\": 6902.0, \"toPerUnit\": 328.68787799106, \"currency\": \"USD\", \"currencySource\": \"conversion\", \"type\": \"b2b\", \"unit\": \"Day\", \"gross\": false}, {\"from\": 4813.0, \"fromPerUnit\": 229.189585625229, \"to\": 6016.0, \"toPerUnit\": 286.48698203153623, \"currency\": \"EUR\", \"currencySource\": \"conversion\", \"type\": \"b2b\", \"unit\": \"Day\", \"gross\": false}, {\"from\": 21000.0, \"fromPerUnit\": 1000.0, \"to\": 26250.0, \"toPerUnit\": 1250.0, \"currency\": \"PLN\", \"currencySource\": \"original\", \"type\": \"b2b\", \"unit\": \"Day\", \"gross\": false}], \"workplaceType\": \"hybrid\", \"requiredSkills\": [{\"name\": \"Angular\", \"level\": 5}, {\"name\": \"TypeScript\", \"level\": 5}, {\"name\": \"JavaScript\", \"level\": 5}, {\"name\": \"RxJS\", \"level\": 4}, {\"name\": \"Node.js\", \"level\": 4}, {\"name\": \"HTML5\", \"level\": 4}, {\"name\": \"CSS\", \"level\": 4}], \"niceToHaveSkills\": [{\"name\": \"Git\", \"level\": 4}, {\"name\": \"REST API\", \"level\": 4}, {\"name\": \"React\", \"level\": 3}], \"workingTime\": \"full_time\", \"applyUrl\": null, \"publishedAt\": \"2026-09-17T22:50:16.50239Z\", \"companyProfileSlug\": null, \"companyProfileCoverPhotoUrl\": null, \"companyProfileShortDescription\": null, \"isOpenToHireUkrainians\": false, \"locations\": [{\"city\": \"Warszawa (Mazowieckie)\", \"street\": \"Centrum\", \"latitude\": 52.32121919999999, \"longitude\": 21.1036813, \"slug\": \"astek-polska-senior-angular-developer-warszawa-mazowieckie--javascript-b28b4fcc\"}], \"videoUrl\": null, \"bannerUrl\": null, \"isActive\": true, \"expiredAt\": \"2026-10-17T21:59:59.999Z\", \"languages\": [], \"coverImage\": \"https://og-image.justjoin.it/ogimage/justjoinit/astek-polska-senior-angular-developer-warszawa-mazowieckie--javascript-b28b4fcc\", \"appliedAt\": null, \"hybridWorkSchedule\": null, \"url\": \"https://justjoin.it/job-offer/astek-polska-senior-angular-developer-warszawa-mazowieckie--javascript-b28b4fcc\"}" @@ -152,7 +152,7 @@ def skip(group, reason): # NOT from this site. A hand-built page carrying a Cloudflare managed # challenge, so the marker set has something to bite on — justjoin.it has # never served this scraper one, and a check with no positive case proves -# nothing (CLAUDE.md §19: a measurement with no negative case is an +# nothing (a measurement with no negative case is an # anecdote, and the inverse holds too). SYNTHETIC_CHALLENGE_HTML = ( "Just a moment..." @@ -235,7 +235,7 @@ def check_the_salary_is_never_read_from_index_zero(): — PLN 129.37 where the employer offers EUR 30 — and the check asserts both that the parser gets the right one AND that the wrong one is genuinely there to be got. A check whose trap is absent from its own - fixture passes for the wrong reason (CLAUDE.md §21). + fixture passes for the wrong reason. """ records = json.loads(LISTING_JSON)["data"] rows = {r.sku: r for r in _listing().rows} @@ -278,7 +278,7 @@ def check_no_conversion_currency_ever_reaches_a_row(): Asked through `product_parser.converted_currencies`, which exists for exactly this and whose docstring says so — a helper that promises the - suite reads it and is then read by nothing is the defect CLAUDE.md §17 + suite reads it and is then read by nothing is the dead-code defect names. """ records = {r["guid"]: r for r in json.loads(LISTING_JSON)["data"]} @@ -296,7 +296,7 @@ def check_no_conversion_currency_ever_reaches_a_row(): check("no row carries a conversion-only currency", not offenders, str(offenders)) # Without this the check could pass on a fixture that holds no - # conversions at all — green for the wrong reason (CLAUDE.md §22). + # conversions at all — green for the wrong reason. check("the fixture actually contains conversions to avoid", saw_conversions >= 5, "only %d of 7 records carry a computed currency" % saw_conversions) @@ -337,7 +337,7 @@ def check_the_monthly_figure_is_not_read_as_the_quoted_rate(): def check_an_undisclosed_salary_is_null_and_never_zero(): """461 of 1,000 offers disclose nothing. A zero would drag every - average a consumer computes (CLAUDE.md §21).""" + average a consumer computes.""" page = _listing() undisclosed = [r for r in page.rows if r.salary_min is None] check("the fixture holds undisclosed offers", undisclosed) @@ -413,7 +413,7 @@ def check_the_unit_and_country_are_case_normalised(): def check_a_form_apply_has_no_url_and_that_is_the_site_saying_so(): """`applyUrl` is non-null on exactly the offers whose `applyMethod` is "external" — 800 of 1,000. The null is the site saying "applied to on - our own page", not a missing field (CLAUDE.md §21).""" + our own page", not a missing field.""" for row in _listing().rows: if row.apply_method == "form": equal("%s form apply has no url" % row.sku[:8], row.apply_url, None) @@ -449,7 +449,7 @@ def check_multi_city_offers_keep_every_city(): def check_the_sparse_company_columns_are_real_on_some_records(): """`companyUrl` and `companySize` are non-null on 5 of 40 sampled detail records — sparse, and real. A column that no fixture ever - populates reads as one to delete (§9), so one that does is kept.""" + populates reads as one to delete, so one that does is kept.""" row = pp.parse_detail_response(DETAIL_WITH_COMPANY_JSON) check("the record names a company website", row.company_url) check("and a company size", row.company_size) @@ -568,7 +568,7 @@ def check_the_404_is_not_a_block(): def check_walking_past_the_cap_is_named_for_what_it_is(): """`from=10000` answers HTTP 500 — the site refusing an impossible - request, not failing. Retrying would ask it again (CLAUDE.md §21).""" + request, not failing. Retrying would ask it again.""" past = pp.api_url(offset=pp.MAX_FROM) equal("state", pp.detect_page_state("500", 500, past), "cap_exceeded") @@ -598,7 +598,7 @@ def check_the_planner_never_builds_an_address_past_the_cap(): def check_a_rendered_listing_is_never_paginated(): """`?page=2` answers HTTP 200 with page 1 again — a complete-looking - run holding a hundred rows of ten thousand (CLAUDE.md §7, §18).""" + run holding a hundred rows of ten thousand.""" equal("page_url refuses to invent a second page", pp.page_url(pp.DEFAULT_LISTING_URL, 2), None) equal("and says so for any page", pp.page_url(pp.DEFAULT_LISTING_URL, 7), @@ -658,7 +658,7 @@ def check_the_ssr_page_states_the_cap_without_a_second_request(): def check_the_jsonld_names_offers_and_says_nothing_about_them(): - """Counted before a line of the parser was written (CLAUDE.md §15): a + """Counted before a line of the parser was written: a listing page publishes two blocks, and the one that lists offers carries only their URLs. That is why the flight payload is primary.""" blocks = pp.jsonld_blocks(SSR_HTML) @@ -761,7 +761,7 @@ def check_the_facet_counts_are_not_capped(): def check_facets_and_offers_are_refused_against_each_other(): """Different row classes and no shared id — every row would be - reported as both added and removed (CLAUDE.md §9).""" + reported as both added and removed.""" equal("facets get their own class", ROW_CLASS_BY_MODE["facets"], Facet) equal("both job modes share one", ROW_CLASS_BY_MODE["listings"], ROW_CLASS_BY_MODE["offer"]) @@ -828,7 +828,7 @@ def check_the_sku_is_not_recoverable_from_a_url_and_says_so(): def check_only_filters_that_measurably_filter_are_offered(): """Several plausible parameter names are ACCEPTED AND IGNORED by the - endpoint — the same defect CLAUDE.md §21 found on BBB, where + endpoint — the same defect found on BBB, where `showOnlyAccredited` does nothing. Only the measured ones are built.""" params = pp.Filters(category="python", city="Kraków", remote="remote", keyword="rust", experience=["senior"], @@ -935,7 +935,7 @@ def check_page_states_on_real_captures(): def check_content_wins_over_every_heuristic(): - """CLAUDE.md §17's classification-order trap: a threshold heuristic + """the classification-order trap: a threshold heuristic that runs ahead of an unambiguous signal reports exit 3 for a correct answer. A payload holding records is `content` whatever else is in it — including a job description that happens to say "access denied".""" @@ -949,7 +949,7 @@ def check_content_wins_over_every_heuristic(): def check_no_marker_matches_a_page_the_site_serves(): - """CLAUDE.md §18: count every candidate on a page you know is good + """Count every candidate on a page you know is good BEFORE adding it. Two candidates were thrown out this way — `cf-turnstile`, which 2Captcha's own extension injects into every page it loads, and `akamai`, which appears four times on a served offer page @@ -979,7 +979,7 @@ def check_no_marker_matches_a_page_the_site_serves(): def check_a_marker_survives_both_encodings_of_the_same_page(): - """CLAUDE.md §20: an edge's refusal reaches a parser entity-escaped + """An edge's refusal reaches a parser entity-escaped over a raw HTTP client and plain out of a browser DOM.""" escaped = SYNTHETIC_CHALLENGE_HTML.replace( "challenges.cloudflare.com", @@ -992,7 +992,7 @@ def check_a_marker_survives_both_encodings_of_the_same_page(): def check_positive_asset_detection_reads_the_sites_own_hosts(): - """CLAUDE.md §8's trick, on its third site: a served page is built out + """the trick, on its third site: a served page is built out of the site's own assets; an interstitial is not.""" check("a rendered page references them", pp.count_site_assets(SSR_HTML) >= pp.MIN_ASSET_MARKERS, pp.count_site_assets(SSR_HTML)) @@ -1000,11 +1000,11 @@ def check_positive_asset_detection_reads_the_sites_own_hosts(): pp.count_site_assets(SYNTHETIC_CHALLENGE_HTML), 0) check("the threshold is 1, not the measured average", pp.MIN_ASSET_MARKERS == 1, - "a higher threshold calls a minimal-but-real page blocked (§17)") + "a higher threshold calls a minimal-but-real page blocked") def check_the_sites_own_captcha_is_recorded_even_though_it_never_renders(): - """CLAUDE.md §18: grep a GOOD page for the site's own captcha config + """Grep a GOOD page for the site's own captcha config rather than building a marker set from a vendor list. justjoin.it publishes a reCAPTCHA **v2** key on every page, and it @@ -1170,7 +1170,7 @@ def check_row_schema(): def check_no_column_is_null_on_every_row_of_every_route(): - """CLAUDE.md §9: a column that is null on every row of every run should + """A column that is null on every row of every run should not exist. Checked across all three routes together, because several columns are legitimately filled by only one of them.""" rows = (_listing().rows + _ssr().rows + [_detail()] @@ -1185,7 +1185,7 @@ def check_no_column_is_null_on_every_row_of_every_route(): def check_page_and_position_are_unique_across_pages(): - """CLAUDE.md §18: `position` restarts at 1 on every page, so without + """`position` restarts at 1 on every page, so without the page number beside it a row from page 2 claims a position another row already has. One line, and the column is worthless without it.""" first = pp.parse_offers_response(LISTING_JSON, page=1, items=7).rows @@ -1264,7 +1264,7 @@ def check_the_page_size_ceiling_is_enforced_before_the_request(): def check_fixtures_carry_no_personal_names(): """justjoin.it's offer records name COMPANIES and job titles, never individuals. This guards the SHAPE so a future capture from a route - that does carry a person is caught (CLAUDE.md §10).""" + that does carry a person is caught.""" person_fields = ("firstName", "lastName", "authorName", "recruiterName", "displayName", "userName", "profileUrl", "avatarUrl", "email", "phone") @@ -1296,7 +1296,7 @@ def check_fixtures_carry_no_personal_names(): "puppeteer_scraper": "pyppeteer", } -# The family contract (CLAUDE.md §9), re-derived across the repos rather +# The family contract, re-derived across the repos rather # than copied from that list — which was itself wrong for months. CONTRACT_FLAGS = { "--url", "--pages", "--category", "--format", "--out", "--delay", @@ -1304,9 +1304,9 @@ def check_fixtures_carry_no_personal_names(): "--proxy-rotate", "--proxy-shuffle", "--proxy-block-retries", "--twocaptcha-key", "--captcha-api", "--solve-captcha", "--min-score", "--cdp-endpoint", "--allow-empty", "--dump-html", - # The five CLAUDE.md §9 admits its own list omitted for months while + # The five the family's flag list omitted for months while # nearly every repo shipped them, plus the headless pair every engine - # carries. RE-MEASURED here rather than inherited (§13 — a number you + # carries. RE-MEASURED here rather than inherited (a number you # inherited is not a number you measured): counted 2026-09-18 across # the 26 sibling repos in ~/2scraper, `--fingerprint`, `--fp-tags` and # `--fp-country` are in 25 of 26 and `--locale` in 23, against 23 flags @@ -1336,10 +1336,10 @@ def check_fixtures_carry_no_personal_names(): "--slugs-file", } -# §12: wording the build fails on. Assembled from pieces rather than written +# Wording the build fails on. Assembled from pieces rather than written # out, so this file can scan ITSELF — three sibling repos exempted # `smoke_test.py` wholesale, which made the file most likely to acquire a -# stray phrase the one file nobody scanned (CLAUDE.md §22). +# stray phrase the one file nobody scanned. BANNED_WORDING = ( "cloud " + "browser", "anti" + "detect browser", "2scraper Anti" + "detect Browser", @@ -1371,7 +1371,7 @@ def check_state_policy(): page_flow.should_retry("blocked") and page_flow.should_solve("blocked") and page_flow.counts_as_blocked("blocked")) - # A served page we could not read is OUR bug, never "0 jobs" (§20), so + # A served page we could not read is OUR bug, never "0 jobs", so # it neither counts as blocked nor buys a solve. check("parse_error: retried once, never solved, NOT blocked", page_flow.should_retry("parse_error") @@ -1386,7 +1386,7 @@ def check_state_policy(): # Asking past the site's 10,000 ceiling is an HTTP 500 — the site # refusing an impossible request rather than failing. A retry would ask # the identical impossible question again; calling it blocked would - # send a reader after a proxy (§21). + # send a reader after a proxy. check("cap_exceeded: not retried, not blocked, not solved", not page_flow.should_retry("cap_exceeded") and not page_flow.counts_as_blocked("cap_exceeded") @@ -1407,7 +1407,7 @@ def check_state_policy(): def check_the_two_known_dead_names_in_the_shared_solver_stay_pinned(): - """§17 says a public name nothing reads is the same defect as dead code, + """A public name nothing reads is the same defect as dead code, and a sweep of this repo finds exactly two — both in `captcha_solver.py`, which is family core copied verbatim. @@ -1419,7 +1419,7 @@ def check_the_two_known_dead_names_in_the_shared_solver_stay_pinned(): What the check buys is that the list cannot GROW silently. A third dead name appearing in the shared solver is a real finding; these two are a - decision (CLAUDE.md §10: pin a known limitation rather than + decision (pin a known limitation rather than half-guarding it). """ import ast as _ast @@ -1459,7 +1459,7 @@ def check_the_two_known_dead_names_in_the_shared_solver_stay_pinned(): def check_policy_constants_have_a_consumer(): - """§17: a policy constant nothing reads is the same defect as dead code. + """A policy constant nothing reads is the same defect as dead code. `RETRY_ON_BLOCKED` carried a paragraph of justification in a sibling repo and no engine consulted it, so setting it False changed nothing. @@ -1597,7 +1597,7 @@ def check_a_run_that_finds_nothing_writes_nothing(): code = save([], prefix, "json", allow_empty=True) equal("--allow-empty WRITES the empty file...", json.load(open(prefix + ".json", encoding="utf-8")), []) - # ...and still reports exit 4. Pinned deliberately (§10: pin a known + # ...and still reports exit 4. Pinned deliberately (pin a known # behaviour rather than half-guarding it): "zero offers" is true # whether or not the file was written, and a caller that wanted the # file still wants to know the result was empty. @@ -1622,7 +1622,7 @@ def check_sidecar_shape(): meta["total_results"], 10000) # Both figures, because every query is capped at 10,000 while the board # holds more. Without the second number a reader cannot tell that a - # "complete" run is about half the board (CLAUDE.md §21: complete and + # "complete" run is about half the board (complete and # exhaustive are different words). equal("...and how many offers the whole board held", meta["site_total"], 19388) @@ -1670,7 +1670,7 @@ def check_engines_import_their_driver_at_module_level(): def check_shared_calls_bind_against_the_real_signature(): - """§17's check #1, and the one that earns its keep. + """the check #1, and the one that earns its keep. A sibling repo shipped `classify(html, url=…)` in two of three engines against a callee taking `status` second, and BOTH crashed on their first @@ -1787,7 +1787,7 @@ def _argparse_flags(module_name): def check_engine_flag_sets(): - """§17's check #2: against the contract AND against each other, both ways. + """the check #2: against the contract AND against each other, both ways. A missing flag fails; so does closing a difference the README documents. """ @@ -1804,7 +1804,7 @@ def check_engine_flag_sets(): # and could not launch it on the development machine, so it needs a way # to point at another one. Its twins have no equivalent because they do # not ship a browser. Listed here so that closing the difference — or - # growing a second one — fails the build (§17). + # growing a second one — fails the build. DOCUMENTED_DIFFERENCES = {"puppeteer_scraper": {"--chromium-path"}} names = sorted(sets) for i in range(len(names) - 1): @@ -1818,7 +1818,7 @@ def check_engine_flag_sets(): def check_the_concurrency_difference_is_documented_in_both_directions(): - """§20: the exception list IS the documentation. + """The exception list IS the documentation. All three engines take `--concurrency` because it is in the family's CLI contract, but only the Playwright engine implements it — the other two @@ -1877,15 +1877,14 @@ def check_banned_and_removed_flags(): identically from every exit tried, so such a flag could only contradict what the URL already says. On `fingerprint_client.py` the same name is legitimate — there it picks a fingerprint locale, not a target — which - is why this check is scoped to the engines rather than to the tree - (CLAUDE.md §10). + is why this check is scoped to the engines rather than to the tree. What the rule is really about is a flag that can disagree with the URL or with the mode. Two refusals enforce that here: * `--url` and `--mode` are refused when they name different routes, because reading an offer URL with the listings reader returns zero - rows from a perfectly good response (§20). + rows from a perfectly good response. * the filter flags are refused outside `--mode listings`, because `--mode offer` reads a sitemap and `--mode facets` reads one endpoint — neither takes a query. A `--category python --mode @@ -1911,7 +1910,7 @@ def check_banned_and_removed_flags(): def check_undefined_names_in_every_module(): - """§10: compileall proves a file PARSES, not that its names RESOLVE. + """Compileall proves a file PARSES, not that its names RESOLVE. A live run of a sibling repo's pyppeteer engine died with NameError on a line reached only while fetching, after an import had been removed — the @@ -1969,7 +1968,7 @@ def _import_graph(entrypoint): def check_dockerfile_copies_everything_the_entrypoint_imports(): - """§10: all three repos in this family shipped an image that died with + """All three repos in this family shipped an image that died with ModuleNotFoundError on every invocation, --help included, because proxy_pool.py was missing from the COPY list. CI never built the image; this check needs no Docker.""" @@ -2017,7 +2016,7 @@ def check_env_example_documents_exactly_what_the_loader_reads(): def check_a_copied_env_example_reads_as_UNSET(): - """§17: `cp .env.example .env` followed by a run must not connect. + """`cp .env.example .env` followed by a run must not connect. The placeholder check was a literal set in a sibling repo, and the two credentialled URLs are documented the way the vendor documents them — @@ -2032,13 +2031,21 @@ def check_a_copied_env_example_reads_as_UNSET(): return text = open(example, encoding="utf-8").read() values = dict(re.findall(r"^([A-Z][A-Z0-9_]+)=(.*)$", text, re.M)) - check("the example actually sets every variable", - set(values) == set(env_config.ENV_KEYS), - "example has %s, loader reads %s" - % (sorted(values), sorted(env_config.ENV_KEYS))) + commented = dict(re.findall(r"^#\s*([A-Z][A-Z0-9_]+)=(\S.*)$", text, re.M)) + check("the example sets or shows every variable", + set(values) | set(commented) == set(env_config.ENV_KEYS), + "example has %s + commented %s, loader reads %s" + % (sorted(values), sorted(commented), sorted(env_config.ENV_KEYS))) + # The URL is offered COMMENTED OUT. Active, it stood in for --url on + # every run, and a listings URL made `--mode offer` and `--mode facets` + # a usage error for anyone who copied the file. + check("the example ships JUSTJOIN_URL commented out", + "JUSTJOIN_URL" in commented and "JUSTJOIN_URL" not in values, + "active: %s" % sorted(values)) + values = {**commented, **values} # Every CREDENTIAL must read as unset. The default TARGET must not: it is # a real, usable URL, and blanking it would remove the one setting this - # file exists to make convenient (§17's check #3 says exactly this — the + # file exists to make convenient (the check #3 says exactly this — the # credentials unset, the non-credential default still usable). CREDENTIALS = {"TWOCAPTCHA_KEY", "JUSTJOIN_CDP_ENDPOINT", "JUSTJOIN_PROXY"} before = dict(os.environ) @@ -2072,7 +2079,7 @@ def check_a_copied_env_example_reads_as_UNSET(): def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): """The `engine-smoke` job must be ABLE to fail. - That job exists for one reason (CLAUDE.md §10): "skipped, engine absent" + That job exists for one reason: "skipped, engine absent" reads identically to a real import error, so CI installs each engine and fails if THAT engine's group still reports a skip. It works by grepping the suite's own output for a sentinel. @@ -2081,7 +2088,7 @@ def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): imported"` — a string no suite in this family emits. Checked 2026-09-18, seven sibling repos carry the same dead grep, so in none of them could the job ever have failed. It passed for the wrong reason, which is - §22's "a check that swallows the loudest failure it could report". + the "a check that swallows the loudest failure it could report". This check is the guard against that coming back: whatever sentinel the workflow looks for, this suite has to be capable of printing it. It @@ -2091,7 +2098,7 @@ def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): """ workflow = os.path.join(HERE, ".github", "workflows", "tests.yml") if not os.path.isdir(os.path.join(HERE, ".github")): - # §22: trigger on the WHOLE .github directory being absent — which + # Trigger on the WHOLE .github directory being absent — which # is the Docker image, where it is deliberately not COPYed — and # never on a file inside it going missing, because a check that # quietly starts passing once its input disappears is the failure @@ -2134,7 +2141,7 @@ def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): def check_credential_scan_is_one_implementation_invoked_from_both(): - """§17: two sources of truth, one dead and one holed. + """Two sources of truth, one dead and one holed. `.github/ci_checks.py` sat in three repos invoked by NOTHING, while tests.yml carried an inline grep doing a narrower version of the same job @@ -2151,7 +2158,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): # file being missing: two suites in this family run INSIDE the Docker # image, which deliberately COPYs no .github/, and a check that quietly # starts passing once its input disappears is the same failure this - # function is about (CLAUDE.md §22). + # function is about. if not os.path.isdir(workflow_dir): skip("ci-wiring", "no .github/ in this tree (the Docker image)") elif os.path.exists(workflow): @@ -2180,7 +2187,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): "tests.yml's offline job mentions %r — one implementation, " "in ci_checks.py, invoked from both" % marker) # And the guard must have had something to read, or it passed for the - # wrong reason (CLAUDE.md §22). + # wrong reason. check("...and the offline job was actually found to scan", "ci_checks.py" in offline, "no offline job in tests.yml") result = subprocess.run([sys.executable, script, "--all"], cwd=HERE, @@ -2202,7 +2209,7 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): The fix is structural rather than a longer list of names — a directory holding `pyvenv.cfg` is a virtualenv whatever it is called — and this pins BOTH halves, because narrowing a credential scan is exactly how one - stops catching things. CLAUDE.md §22 records a sibling repo whose scan + stops catching things. A sibling repo recorded whose scan caught an UNTRACKED `.env.bak` holding a live key, so scanning must not be reduced to tracked files. """ @@ -2240,8 +2247,41 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): "git ls-files" not in open(script, encoding="utf-8").read()) +def check_the_credential_scan_works_from_any_parent_directory(): + """A clone under ~/build, /srv/env or …/venv must still be scanned. + + The skip list was matched against the ABSOLUTE path, so every file of a + clone whose ancestor was called `build`, `env` or `venv` was skipped: + "0 files scanned, nothing credential-shaped", exit 0, with a key in the + tree. Run here exactly the way CI runs it, from such a copy, with a + key-shaped string planted (assembled, so this file does not carry one). + """ + script = os.path.join(HERE, ".github", "ci_checks.py") + if not os.path.isdir(os.path.join(HERE, ".github")): + skip("credential-scan-parent", "no .github/ in this tree (the Docker image)") + return + planted = "ab" * 16 + for parent in ("build", "venv", "env"): + with tempfile.TemporaryDirectory() as tmp: + repo = os.path.join(tmp, parent, "clone") + os.makedirs(os.path.join(repo, ".github")) + with open(script, encoding="utf-8") as src, open( + os.path.join(repo, ".github", "ci_checks.py"), "w", + encoding="utf-8") as dst: + dst.write(src.read()) + with open(os.path.join(repo, "planted.py"), "w", + encoding="utf-8") as f: + f.write("KEY = '%s'\n" % planted) + run = subprocess.run( + [sys.executable, os.path.join(repo, ".github", "ci_checks.py"), + "--secret-check"], capture_output=True, text=True, cwd=repo) + check("the scan finds a planted key in a clone under %s/" % parent, + run.returncode != 0 and "planted.py" in run.stdout + run.stderr, + "exit %s: %s" % (run.returncode, (run.stdout + run.stderr)[-200:])) + + def check_banned_wording(): - """§12: enforced by this test rather than by review.""" + """Enforced by this test rather than by review.""" for root, dirs, files in os.walk(HERE): dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", ".pytest_cache", "node_modules")] @@ -2258,8 +2298,44 @@ def check_banned_wording(): check("banned-wording scan ran", True) +def check_no_private_document_is_cited(): + """Shipped files explain their rules in their own words. + + The project's internal notes are not public, so a comment that says + "see section N" is a pointer nobody outside can follow. The + tokens are assembled from pieces so this file can scan itself. + """ + section = chr(167) + tokens = ("CLAUDE" + ".md", "claude_" + "2scraper", + "2scraper_" + "repo_rules", "play" + "book " + section, + "template " + section) + numbered = re.compile(section + r"\s?\d") + scanned = 0 + for root, dirs, files in os.walk(HERE): + dirs[:] = [d for d in dirs if d not in + (".git", "__pycache__", ".pytest_cache", "node_modules") + and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))] + for filename in files: + if not filename.endswith((".py", ".md", ".yml", ".yaml", ".txt", + ".toml", ".html", ".example", ".cfg", + ".gitignore", ".dockerignore")) \ + and filename != "Dockerfile": + continue + path = os.path.join(root, filename) + text = open(path, encoding="utf-8", errors="replace").read() + scanned += 1 + rel = os.path.relpath(path, HERE) + for token in tokens: + check("%s cites no private %r" % (rel, token), token not in text) + hit = numbered.search(text) + check("%s carries no bare section reference" % rel, not hit, + hit.group(0) if hit else "") + check("the citation scan read the repo's files", scanned > 20, + "%d file(s)" % scanned) + + def check_concurrency_with_the_browser_stubbed(): - """§10: a live run cannot always reach this machinery. + """A live run cannot always reach this machinery. Page 1 is fetched alone and decides how many pages there are, so a blocked page 1 means the workers never start. Driven directly instead, @@ -2405,7 +2481,7 @@ def check_worker_pools_start_on_different_exits(): def check_fingerprint_kwargs_are_ones_the_driver_accepts(): - """§10: an unknown key in new_context(**kwargs) is a TypeError at launch, + """An unknown key in new_context(**kwargs) is a TypeError at launch, on the PAID path, at runtime.""" engine = _import_engine("playwright_scraper") if engine is None: @@ -2437,7 +2513,7 @@ def check_every_engine_exposes_the_same_public_surface(): # `target_url` and `parse_for_mode` are deliberately NOT here: # they moved into product_parser so the three engines cannot # disagree about which URL a run fetches or how a body is read - # (CLAUDE.md §1, the argument page_flow already makes for policy). + # (the argument page_flow already makes for policy). for name in ("scrape", "parse_args", "PageOutcome", "_fetch_one_page", "_is_endpoint", "_snapshot"): check("%s.%s exists" % (module, name), hasattr(engine, name)) @@ -2470,7 +2546,7 @@ def check_every_engine_exposes_the_same_public_surface(): def check_every_solve_is_counted_against_the_budget(): """`SOLVES_PER_PAGE` is a MONEY limit, so every call that can buy must be - counted — CLAUDE.md §17's "a policy constant nothing reads". + counted — the "a policy constant nothing reads". `handle_captcha_if_present` is called TWICE per attempt in every engine: once before the page is classified (so a challenge is cleared before @@ -2513,7 +2589,7 @@ def check_every_solve_is_counted_against_the_budget(): def check_a_dead_proxy_is_reported_as_a_proxy_failure(): - """CLAUDE.md §8: a proxy failure is not a timeout, and the two want + """A proxy failure is not a timeout, and the two want opposite responses — another try at the same exit versus a different one. The engines all compute the reason (`_proxy_failure`) and, WITH a pool, @@ -2536,7 +2612,7 @@ def check_a_dead_proxy_is_reported_as_a_proxy_failure(): # Anchored on the GIVE-UP branch — the one that reports and returns — # not on the `if load_failed: break` inside the retry loop, which # comes first in the file and would make this check read the wrong - # block and pass for the wrong reason (CLAUDE.md §22). + # block and pass for the wrong reason. marker = " outcome.load_failed = True" if marker not in source: check("%s has a give-up branch to check" % module, False) @@ -2569,7 +2645,7 @@ class _E(Exception): def check_engines_do_not_evaluate_a_string_in_the_browser(): - """§18: a site whose CSP omits `unsafe-eval` kills wait_for_function with + """A site whose CSP omits `unsafe-eval` kills wait_for_function with an EvalError and takes the run down with exit 1, on the site's most obvious URL. justjoin.it has not been measured for that, and the cheap habit costs nothing on a site that would have allowed it.""" @@ -2586,8 +2662,122 @@ def check_engines_do_not_evaluate_a_string_in_the_browser(): "poll through page_flow.wait_for_count instead") +def check_an_env_url_never_turns_a_mode_into_a_usage_error(): + """JUSTJOIN_URL from the environment is a default, not a typed --url. + + `env_config.apply` fills only unset values, so once it has run nothing + tells an environment URL from a typed one. With the listings URL that + `.env.example` used to ship, `--mode offer` and `--mode facets` exited 2 + with "--url is a listings URL" although nobody had typed --url. + """ + listing = "https://justjoin.it/job-offers/all-locations" + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + saved_argv, saved_env = sys.argv[:], dict(os.environ) + try: + os.environ["JUSTJOIN_URL"] = listing + for mode in ("facets", "offer"): + sys.argv = [module, "--mode", mode] + try: + args = engine.parse_args() + got = args.url + except SystemExit as e: + got = "exit %s" % e.code + equal("%s --mode %s ignores a listings JUSTJOIN_URL" + % (module, mode), got, None) + # A URL the user TYPED is still checked against --mode. + sys.argv = [module, "--mode", "facets", "--url", listing] + try: + engine.parse_args() + code = 0 + except SystemExit as e: + code = e.code + equal("%s still refuses a TYPED --url of another mode" % module, + code, 2) + finally: + sys.argv = saved_argv + os.environ.clear() + os.environ.update(saved_env) + + +def check_diff_runs_tracks_real_columns_of_each_mode(): + """A tracked name the row class lacks compares None with None forever. + + The facets mode used to be diffed with the JobPosting list, 24 of whose + 25 names a Facet row does not have, so a key going from 100 offers to + 250 reported "0 changed" and exit 0. + """ + import diff_runs + from output_writer import JobPosting, Facet + for name, tracked, cls in ( + ("TRACKED_FIELDS", diff_runs.TRACKED_FIELDS, JobPosting), + ("DETAIL_ONLY_FIELDS", diff_runs.DETAIL_ONLY_FIELDS, JobPosting), + ("FACET_TRACKED_FIELDS", diff_runs.FACET_TRACKED_FIELDS, Facet)): + have = {f.name for f in fields(cls)} + check("every name in diff_runs.%s is a %s column" + % (name, cls.__name__), set(tracked) <= have, + "missing: %s" % sorted(set(tracked) - have)) + old = [{"sku": "skill:python", "facet_group": "skill", + "facet_key": "python", "title": "Python", "offer_count": 100}] + new = [dict(old[0], offer_count=250)] + for mode in ("facets", None): + result = diff_runs.diff_products( + old, new, diff_runs.tracked_fields_for(mode, old + new)) + equal("a facets diff (mode %r) sees an offer_count change" % mode, + [c["changes"] for c in result["changed"]], + [{"offer_count": {"old": 100, "new": 250}}]) + equal("a listings run is still diffed on the JobPosting columns", + diff_runs.tracked_fields_for("listings", []), + diff_runs.TRACKED_FIELDS) + + +def check_csv_cells_are_formula_escaped_in_csv_only(): + """An employer-written string starting = + - @ TAB CR LF is a formula. + + Escaped with a leading apostrophe in the CSV only; JSON keeps the + site's bytes, numbers stay numbers, a joined list is checked too, and + the count reaches the sidecar UNDER the caller's extra. + """ + import csv as _csv + from output_writer import finish_run, JobPosting + rows = [JobPosting(sku="a", title='=HYPERLINK("http://x")', + salary_min=-5, required_skills=["@admin", "python"], + company_name="plain")] + with tempfile.TemporaryDirectory() as d: + prefix = os.path.join(d, "out") + code = finish_run(rows, prefix, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + extra={"total_results": 1}) + equal("the run itself succeeds", code, 0) + with open(prefix + ".csv", encoding="utf-8", newline="") as f: + got = list(_csv.DictReader(f))[0] + equal("a formula title is escaped in the CSV", got["title"], + '\'=HYPERLINK("http://x")') + equal("a joined list whose first item is formula-shaped is escaped", + got["required_skills"], "'@admin | python") + equal("a negative NUMBER is left alone", got["salary_min"], "-5") + equal("a plain string is left alone", got["company_name"], "plain") + data = json.load(open(prefix + ".json", encoding="utf-8")) + equal("JSON keeps the site's bytes", data[0]["title"], + '=HYPERLINK("http://x")') + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + equal("the sidecar counts the escaped cells", + meta.get("csv_cells_escaped"), 2) + equal("...beside the caller's own extra", meta.get("total_results"), 1) + finish_run(rows, prefix, "csv", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + extra={"csv_cells_escaped": "caller"}) + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + equal("a caller's value wins a name collision with the count", + meta.get("csv_cells_escaped"), "caller") + + def check_credentials_never_reach_a_log(): - """§8: an EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. + """An EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. A Playwright connection error repeats the endpoint five times (the message plus a four-line call log), so a masker handling only the first @@ -2657,7 +2847,7 @@ def _tree_state(): def check_no_test_mutates_the_working_tree(): - """§10: one suite used its own file as a fake chromedriver and chmod'd it + """One suite used its own file as a fake chromedriver and chmod'd it to 755, leaving a mode change in git status. Compares the tree against how it looked when the suite STARTED, not @@ -2675,7 +2865,7 @@ def check_no_test_mutates_the_working_tree(): def check_captcha_capability_claims_match_the_code(): - """§19: the most expensive bug this family can ship is a SENTENCE. + """The most expensive bug this family can ship is a SENTENCE. It fails in both directions and this family has shipped both: From eb379ecec82d9d7ad6d754cc61f84039098fa0be Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:57:28 +0000 Subject: [PATCH 2/2] Comments: rejoin sentences split where a citation was removed Co-Authored-By: Claude Opus 5.5 --- output_writer.py | 3 +-- playwright_scraper.py | 12 ++++-------- puppeteer_scraper.py | 3 +-- selenium_scraper.py | 3 +-- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/output_writer.py b/output_writer.py index 3a00521..eb0fa7f 100644 --- a/output_writer.py +++ b/output_writer.py @@ -221,8 +221,7 @@ class JobPosting: # # Null on 461 of 1,000 rows, where the employer disclosed no salary. # Null, not zero — the site's own UI says "Undisclosed Salary" for - # these, and a zero would drag every average a reader computes - #. + # these, and a zero would drag every average a reader computes. salary_min: Optional[float] = None salary_max: Optional[float] = None # An ISO 4217 code the site states — never defaulted, never inferred diff --git a/playwright_scraper.py b/playwright_scraper.py index b8a62d5..ef225e2 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -386,8 +386,7 @@ def _fetch_body(session, args, url: str) -> str: # Chromium reports a DEAD PROXY as a generic error rather than as a # timeout, and the two want opposite responses: a timeout deserves another # try at the same exit, a dead proxy deserves a different one. Catching -# only the timeout type let this escape as a traceback in a sibling repo -#. +# only the timeout type let this escape as a traceback in a sibling repo. _PROXY_ERROR_MARKERS = ( "ERR_PROXY_CONNECTION_FAILED", # nothing listening / refused "ERR_TUNNEL_CONNECTION_FAILED", # CONNECT rejected by the proxy @@ -459,8 +458,7 @@ def _launch_local(pw, args, pool): # navigation: Cloudflare calls `turnstile.render(container, params)` once # and keeps nothing, so `sitekey`, `action`, `cData` and `chlPageData` # exist only inside that call. A static read of a challenge page — any - # static read, however careful — cannot produce a solvable task - #. It costs nothing on a page that never renders one. + # static read, however careful — cannot produce a solvable task. It costs nothing on a page that never renders one. context.add_init_script(TURNSTILE_INTERCEPT_JS) return browser, context, context.new_page() @@ -1689,8 +1687,7 @@ def scrape(args) -> int: # is looping back on itself. Either way there is # nothing further to fetch, and this is the honest # terminating condition: a property of the DATA, not - # of a CSS selector that may have been renamed - #. + # of a CSS selector that may have been renamed. if not fresh_count: logger.info("Page %d added no rows not already seen " "— treating that as the end of the " @@ -1812,8 +1809,7 @@ def scrape(args) -> int: # order: the same query under `publishedAt` and under `salary` # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and - # `diff_runs.py` refuses to compare two runs that disagree on it - #. + # `diff_runs.py` refuses to compare two runs that disagree on it. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index dba93a1..8eae366 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -1202,8 +1202,7 @@ def scrape(args) -> int: # order: the same query under `publishedAt` and under `salary` # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and - # `diff_runs.py` refuses to compare two runs that disagree on it - #. + # `diff_runs.py` refuses to compare two runs that disagree on it. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters diff --git a/selenium_scraper.py b/selenium_scraper.py index c6e8ab2..ba5a99d 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -1090,8 +1090,7 @@ def scrape(args) -> int: # order: the same query under `publishedAt` and under `salary` # overlapped on 5 of 100 ids (measured 2026-09-18), because each is # a different slice of a capped result set. So it is recorded, and - # `diff_runs.py` refuses to compare two runs that disagree on it - #. + # `diff_runs.py` refuses to compare two runs that disagree on it. filters = filters_from_args(args).as_params() if filters: extra["filters"] = filters