diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 439785d..5112320 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -68,7 +68,7 @@ jobs: 'If it was refused (exit 3), set the INSTAGRAM_PROXY repo secret (a residential ' 'http://login:password@host:port proxy) so the canary can reach the site.' % code) else: - print('::error::canary exit %d through the INSTAGRAM_PROXY secret (1 crash, 3 blocked, 4 zero rows, 5 content never obtained, 6 partial). A proxy that rejects its credential (HTTP 407) reaches Chromium as a navigation timeout, so on exit 5 check the INSTAGRAM_PROXY secret first.' % code) + print('::error::canary exit %d through the INSTAGRAM_PROXY secret (1 crash, 3 blocked, 4 zero rows, 5 content never obtained, 6 partial). A proxy that rejects its credential (HTTP 407) or has no traffic left (HTTP 402) stops the run at its proxy pre-check with exit 5 and says so in the log above, so on exit 5 check the INSTAGRAM_PROXY secret first.' % code) raise SystemExit(f'canary requires EXIT_OK=0, got {code}') meta = json.load(open('canary.json.meta.json')) print(json.dumps(meta, indent=2)) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc856e..b9a01c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,27 @@ that changes a default is called out at the top of its entry. ## [Unreleased] +> **Behaviour changes.** +> * Behind `--proxy` / `--proxy-file`, all three browser engines send one +> `HEAD https://www.instagram.com/robots.txt` through the first exit before +> launching a browser. A gateway `407` (credential refused, or traffic +> exhausted) or `402` (traffic/balance exhausted) now ends the run with exit +> 5 and `stop_reason: proxy_auth_failed` in the new +> `.latest_attempt.meta.json`. No browser is started, and `` and +> `.meta.json` are left as they were. Any other answer, or a transport +> error, leaves the run as it was. Not sent with `--cdp-endpoint`, +> `--scraper-api` or without a proxy. +> * Behind a local proxy, images, video/audio and web fonts are no longer +> downloaded. Playwright aborts them by resource type. Puppeteer fails them +> in the `Fetch` handler that answers the proxy's password prompt. Selenium +> blocks them by URL extension. `--load-media` restores the old behaviour. +> Nothing changes with `--cdp-endpoint`, `--scraper-api` or with no proxy. +> `.meta.json` records `media_blocked`. + +### Added + +- `--load-media` in all three engines. + ### Changed - CI only: a failed canary check now always ends with an `::error::` line. A run @@ -15,6 +36,8 @@ that changes a default is called out at the top of its entry. annotation, and a missing or wrong output file showed only a traceback. The message notes that a proxy rejecting its credential (HTTP 407) shows up in Chromium as a navigation timeout (exit 5), not as a 407. +- CI only: that message now points at the proxy pre-check, which reports a + `407` or `402` by name before any browser is started. - CI only: the canary now runs weekly (Mondays 06:41 UTC) and on a manual dispatch, with its optional secret `INSTAGRAM_PROXY` passed on every run; the daily schedule is removed. ## [0.3.0] - 2026-10-15 diff --git a/README.md b/README.md index 989b4fa..8845408 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,55 @@ getting blocked is dropped; once every exit is dropped the run stops, and never falls back to your own address), keep `--delay-between-pages` at 2s or more, or spread a big list over time. +## Traffic through a paid proxy + +A residential proxy bills per byte, so two things happen only while a +LOCAL browser goes through `--proxy` / `--proxy-file` (or +`INSTAGRAM_PROXY`). Neither applies with `--cdp-endpoint` (that browser +brings its own exit and is billed on its own side) or with `--scraper-api` +(no local proxy is used), or with no proxy, where nothing is metered. + +**Pre-check before the browser starts.** All three engines send ONE +`HEAD https://www.instagram.com/robots.txt` through the exit the first URL +would use, before launching a browser: + +- `407`: the gateway refused the credential, or the traffic balance is + exhausted. Exit `5`. +- `402`: the proxy's traffic or balance is exhausted. Exit `5`. + +The message names `INSTAGRAM_PROXY` and never prints its value. The run +writes `.latest_attempt.meta.json` with `status: failed` and +`stop_reason: proxy_auth_failed`, and leaves `` and `.meta.json` +from an earlier run exactly as they were. Any other answer (a `200`, a +`403` from the site, a timeout, a DNS error) is not a verdict on the proxy, +and the run carries on exactly as before. It is one request, never retried +and never rotated to another exit. + +**Media is not downloaded.** Every value comes from the JSON the page +embeds, image and video URLs included, so images, video/audio and web fonts +are aborted: + +| Engine | How | What still loads | +|---|---|---| +| Playwright | `context.route` by resource type | nothing of those three types | +| Puppeteer | the CDP `Fetch` handler that already answers the proxy's password prompt fails them by resource type; with no password, a `Fetch` interception paused on those three types only | nothing of those three types | +| Selenium | CDP `Network.setBlockedURLs` by file extension | an image or font served from a URL **without** an extension | + +`--load-media` keeps them. A `--dump-html` capture is unaffected, because +it is the page's HTML. Playwright switches its HTTP cache off while a +route is installed, so a script or stylesheet repeated across URLs is +fetched again. Each browser context is new per URL anyway, so this costs +little here. `.meta.json` records `media_blocked`. + +Live comparison, 2026-10-20: `playwright_scraper.py --url natgeo` with no +proxy, once as it is and once with blocking forced on, to compare the rows. +It was not possible from this datacentre address: both runs got Instagram's +login page (exit 3), the same as the datacentre measurement under +[Known limitations](#known-limitations). The verification is therefore +offline: the fixtures (the parser reads embedded JSON only), and each real +browser driven through a local proxy to a test page. With blocking, only the +HTML and the script went through the proxy, and the script still ran. + ## Scraper API mode With `--scraper-api` no browser is driven: each page is one 2Captcha @@ -228,7 +277,9 @@ solves spent, and a hash of the output file). With the options below it also lists the posts left out (`excluded_urls`), every post seen so far (`seen_post_skus`) and whether the run was incremental. A run that collects nothing writes neither, so it never replaces your previous good -file. +file. The one exception is a proxy refused at the pre-check (see +[Traffic through a paid proxy](#traffic-through-a-paid-proxy)): that run +records itself in `.latest_attempt.meta.json`, a separate file. | Exit | Meaning | |---|---| @@ -236,7 +287,7 @@ file. | `6` | partial: rows were written, but the run did not finish cleanly — `stop_reason` says why (`blocked`, `rate_limited`, `remote_api_error`, `failed_pages`, `parse_error`, `proxy_pool_exhausted`) | | `3` | blocked or throttled, no rows | | `4` | no rows: every input was not found, or every post was older than `--since` or already collected by `--incremental-from` | -| `5` | nothing could be read due to fetch, parse, or remote-service failure; no rows | +| `5` | nothing could be read due to fetch, parse, or remote-service failure; no rows. Also a proxy that answered `407`/`402` at the pre-check, before any browser was started | | `2` | bad usage, including input where every line was skipped | | `1` | crash (a bug; please report it) | @@ -358,6 +409,7 @@ the command line. | `--format` / `--out` | json / `instagram_results.` | output format and path | | `--proxy` / `--proxy-file` / `--proxy-shuffle` | `INSTAGRAM_PROXY` | one proxy or a rotating pool, for a local browser | | `--proxy-block-retries` | 3 | blocked answers before that proxy is dropped from the pool | +| `--load-media` | off | load images, video and fonts behind a proxy too (they are aborted by default there; see [Traffic through a paid proxy](#traffic-through-a-paid-proxy)) | | `--scraper-api` | off | no browser: fetch each page through 2Captcha's Scraper API (needs `TWOCAPTCHA_KEY`; routed through `INSTAGRAM_CDP_ENDPOINT` when set) — posts only, see [Scraper API mode](#scraper-api-mode) | | `--cdp-endpoint` | `INSTAGRAM_CDP_ENDPOINT` | connect to a Scraping Browser API profile instead of launching a browser | | `--solve-captcha` | when-blocked | `off` disables the Browser API's own captcha auto-solve; there is no local solver | diff --git a/output_writer.py b/output_writer.py index 341fd2c..8705d72 100644 --- a/output_writer.py +++ b/output_writer.py @@ -329,6 +329,41 @@ def write_meta( json.dump(meta, f, ensure_ascii=False, indent=2) +# A run that stopped before it fetched anything — the proxy refused its +# credential at the pre-check — has no dataset to describe, and +# `.meta.json` must keep describing the output still beside it. What +# happened to THIS attempt goes into its own file instead. +ATTEMPT_META_SUFFIX = ".latest_attempt.meta.json" + + +def write_failed_attempt( + out_path: str, *, stop_reason: str, engine: str, url: str, pages_requested: int, + started_at: float, transport_error: str, +) -> str: + """Record an attempt that ended before `finish_run` could run. Only + `.latest_attempt.meta.json` is written; the output file and + `.meta.json` are left exactly as they were.""" + meta = { + "status": "failed", + "stop_reason": stop_reason, + "engine": engine, + "url": url, + "pages_requested": pages_requested, + "pages_completed": 0, + "failed_pages": [], + "product_count": 0, + "started_at": started_at, + "finished_at": time.time(), + "blocked": False, + "transport_error": transport_error, + "data_updated": False, + } + path = f"{out_path}{ATTEMPT_META_SUFFIX}" + with _atomic(path) as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + return path + + # --------------------------------------------------------------------------- # # finish_run — the single place every engine calls to decide exit code, # whether to write output at all, and whether to write a sidecar. Keeping diff --git a/page_flow.py b/page_flow.py index 515e79f..da6f9a6 100644 --- a/page_flow.py +++ b/page_flow.py @@ -425,13 +425,14 @@ async def run(engine: Engine, args, *, urls: List[str], proxy_pool, client, star engine_name=engine.name, urls=urls, fetched=queue[:args.max_results], started_at=started_at, pages_completed=completed, failed_pages=failed_pages, failures=failures, rate_limited=rate_limited, not_found=not_found, - capped=len(queue) > args.max_results or posts_per_profile > 0, exclusions=exclusions) + capped=len(queue) > args.max_results or posts_per_profile > 0, exclusions=exclusions, + media_blocked=bool(getattr(engine, "media_blocked", False))) def finish(args, *, products: List[Product], blocked: bool, remote_api_error: bool, engine_name: str, urls: List[str], started_at: float, pages_completed: int, failed_pages: List[int], failures=None, rate_limited=False, fetched: Optional[List[str]] = None, not_found=None, - capped: Optional[bool] = None, exclusions=None) -> int: + capped: Optional[bool] = None, exclusions=None, media_blocked: bool = False) -> int: budget = getattr(args, "_solve_budget", None) fetched = list(urls if fetched is None else fetched) selection = run_state.selection(args, urls) @@ -439,6 +440,7 @@ def finish(args, *, products: List[Product], blocked: bool, remote_api_error: bo "failed_urls": failures or [], "not_found_urls": not_found or [], "excluded_urls": exclusions or [], "incremental": bool(getattr(args, "incremental_from", None)), "post_window": "embedded_pinned_first" if getattr(args, "posts", 0) else None, + "media_blocked": bool(media_blocked), # A post's date never changes, so one before --since stays excluded: # remembering it spares the next delta a fetch that only drops it. "seen_post_skus": sorted(set(getattr(args, "_known_posts", set())) | @@ -479,3 +481,63 @@ def validate_common(args, *, urls, skipped, print_err) -> Optional[int]: print_err(f"Error: {_redact(str(exc))}") return EXIT_BAD_USAGE return None + + +# --------------------------------------------------------------------------- # +# Traffic savers behind a metered proxy (the helpers are in proxy_pool.py) +# --------------------------------------------------------------------------- # +# One HEAD for www.instagram.com/robots.txt, through the proxy, before a local +# browser is launched. Short on purpose: a slow answer is not a verdict, and +# the run then goes ahead exactly as it did before. +PROXY_PRECHECK_TIMEOUT_S = 20.0 + + +def proxy_precheck(args, proxy_pool, *, urls: List[str], engine_name: str, started_at: float, + print_err=None) -> Optional[int]: + """Pre-check the exit the first URL will use; return the exit code to + stop with, or None to go on. + + A gateway 407 or 402 prints why (naming INSTAGRAM_PROXY, never its + value), writes `.latest_attempt.meta.json` (status failed, + stop_reason proxy_auth_failed), leaves the output and `.meta.json` + untouched, and returns exit 5. Never sent over --cdp-endpoint or + --scraper-api (no local proxy there) or without a proxy. Anything else — + a 200, a 403 from the site, a timeout — returns None: the run carries on + unchanged. One request, never retried and never rotated to another exit. + """ + import sys + import output_writer + import proxy_pool as _savers + print_err = print_err or (lambda m: print(m, file=sys.stderr)) + if proxy_pool is None or getattr(args, "cdp_endpoint", None) or getattr(args, "scraper_api", False): + return None + proxy = proxy_pool.peek() + if proxy is None or not urls: + return None + try: + _savers.precheck_proxy(proxy.url(), _savers.robots_url(urls[0]), timeout=PROXY_PRECHECK_TIMEOUT_S) + except _savers.ProxyPrecheckError as exc: + print_err(_savers.precheck_message(exc, "INSTAGRAM_PROXY", "www.instagram.com")) + try: + output_writer.write_failed_attempt( + args.out, stop_reason=_savers.PROXY_PRECHECK_STOP_REASON, engine=engine_name, + url=urls[0], pages_requested=len(urls), started_at=started_at, + transport_error="proxy pre-check: %s" % exc) + except OSError as write_exc: + # The verdict stands without its record: still exit 5, not a crash. + print_err("[!] could not write the attempt metadata: %s" % write_exc) + return output_writer.EXIT_REMOTE_API_ERROR + return None + + +def add_media_arguments(parser) -> None: + """`--load-media` (documented) and the hidden `--force-block-media`.""" + import argparse + parser.add_argument( + "--load-media", action="store_true", + help="Load images, video and web fonts. By default they are aborted while a local browser goes " + "through --proxy / --proxy-file (the parser reads the page's embedded JSON, and a proxy bills " + "per byte); nothing is blocked with --cdp-endpoint, --scraper-api or no proxy.") + # Turns blocking on without a proxy, so its effect on parsing can be + # measured for free. Not a user feature. + parser.add_argument("--force-block-media", action="store_true", help=argparse.SUPPRESS) diff --git a/playwright_scraper.py b/playwright_scraper.py index ae7b9d1..4122428 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -46,7 +46,8 @@ import page_parser as pp from fingerprint_client import fetch_fingerprint, refuse_if_cdp, user_agent_from from output_writer import EXIT_BAD_USAGE, EXIT_CRASH -from proxy_pool import Proxy, ProxyPool, ProxyParseError, load_proxies, redact_credentials +from proxy_pool import (BLOCKED_RESOURCE_TYPES, Proxy, ProxyPool, ProxyParseError, load_proxies, redact_credentials, + should_block_media) from scraper_api_client import TwoCaptchaClient ENGINE_NAME = "playwright" @@ -99,6 +100,7 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--dump-html", action="store_true", help="Save each fetched page's HTML next to --out, on success too") p.add_argument("--headless", dest="headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") + page_flow.add_media_arguments(p) page_flow.add_run_arguments(p) return p @@ -112,6 +114,7 @@ def _default_out(fmt: str) -> str: async def _new_context( browser: Browser, proxy: Optional[Proxy], user_agent: Optional[str], *, reuse_default: bool = False, + block_media: bool = False, ) -> BrowserContext: """Over --cdp-endpoint, reuse the profile's own default context: it holds the cookies that make the profile worth reusing. A local browser @@ -124,7 +127,22 @@ async def _new_context( kwargs["proxy"] = proxy.playwright_proxy_dict() if user_agent: kwargs["user_agent"] = user_agent - return await browser.new_context(**kwargs) + context = await browser.new_context(**kwargs) + if block_media: + # Images, video and fonts through a metered proxy: the parser reads + # the page's embedded JSON, never these. + await context.route("**/*", _async_media_route) + return context + + +async def _async_media_route(route) -> None: + """proxy_pool.playwright_media_route for the ASYNC API, whose + route.abort()/continue_() are coroutines that must be awaited (the sync + handler would create them and drop them, so nothing would be decided).""" + if route.request.resource_type in BLOCKED_RESOURCE_TYPES: + await route.abort("blockedbyclient") + else: + await route.continue_() async def _close(page: Page, context: BrowserContext, *, reuse_default: bool) -> None: @@ -199,10 +217,14 @@ class _PlaywrightEngine: def __init__(self, browser: Browser, args: argparse.Namespace, *, autosolve: bool, user_agent: Optional[str], client): self.browser, self.args, self.autosolve, self.user_agent, self.client = browser, args, autosolve, user_agent, client + self.media_blocked = False async def open(self, proxy) -> _PlaywrightSession: reuse_default = bool(self.args.cdp_endpoint) - context = await _new_context(self.browser, proxy, self.user_agent, reuse_default=reuse_default) + block = should_block_media(self.args, proxy_in_use=proxy is not None, over_cdp=reuse_default) + self.media_blocked = self.media_blocked or block + context = await _new_context(self.browser, proxy, self.user_agent, reuse_default=reuse_default, + block_media=block) page = await context.new_page() if self.autosolve: await _enable_scraping_browser_auto_solve(context, page) @@ -242,6 +264,11 @@ async def run(args: argparse.Namespace) -> int: print(f"Error: {exc}", file=sys.stderr) return EXIT_BAD_USAGE proxy_pool = ProxyPool(proxies, shuffle=args.proxy_shuffle, block_retries=args.proxy_block_retries) if proxies else None + # Before any browser exists: a proxy that refuses its credential (407) or + # has no traffic left (402) would refuse every request the browser makes. + stop = page_flow.proxy_precheck(args, proxy_pool, urls=urls, engine_name=ENGINE_NAME, started_at=started_at) + if stop is not None: + return stop client = None if args.twocaptcha_key and (args.solve_captcha != "off" or args.fingerprint): diff --git a/proxy_pool.py b/proxy_pool.py index 992d9e6..1dd5bba 100644 --- a/proxy_pool.py +++ b/proxy_pool.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """proxy_pool.py — proxy pool, rotation policy, credential masking, argv -safety. No site knowledge lives here. +safety, and the traffic savers for a metered proxy (a pre-check before a +browser starts, media blocking — see the end of this file). No site +knowledge lives here. Accepted proxy formats (one per line in --proxy-file, or a single value on --proxy): @@ -21,6 +23,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterable, List, Optional, Sequence +from urllib.parse import quote, urlparse # Chromium reports a dead upstream proxy as one of these — NOT as a timeout. # A caller must retry a *timeout* at the same exit, and a *dead proxy* error @@ -72,6 +75,15 @@ def server_only(self) -> str: authenticate that way (Selenium's --proxy-server takes no auth).""" return f"{self.scheme}://{self.host}:{self.port}" + def url(self) -> str: + """The proxy as one URL, credentials included — ONLY for an HTTP + client's `proxies=` (the pre-check below). Never log it or put it + on a command line; use `masked()` / `server_only()` for those.""" + if not self.login: + return self.server_only() + auth = "%s:%s@" % (quote(self.login, safe=""), quote(self.password or "", safe="")) + return f"{self.scheme}://{auth}{self.host}:{self.port}" + def playwright_proxy_dict(self) -> dict: """Playwright's native `proxy=` field — credentials never touch argv.""" d = {"server": self.server_only()} @@ -158,6 +170,11 @@ def worker_view(self, worker_index: int, total_workers: int) -> "ProxyPool": def _live_proxies(self) -> List[Proxy]: return [p for p in self._proxies if not self._dead[id(p)]] + def peek(self) -> Optional[Proxy]: + """The exit `next()` would return, without moving the cursor.""" + live = self._live_proxies() + return live[self._cursor % len(live)] if live else None + def next(self) -> Optional[Proxy]: live = self._live_proxies() if not live: @@ -221,3 +238,159 @@ def redact_credentials(text: str) -> str: text = _KEY_PARAM_RE.sub(lambda m: f"{m.group(1)}=***", text) text = _BEARER_RE.sub(lambda m: f"{m.group(1)} ***", text) return text + + +# --------------------------------------------------------------------------- +# Traffic savers for runs through a paid proxy. +# +# 1. Pre-check. ONE small request (a HEAD for the target site's robots.txt) +# goes through the proxy BEFORE a browser is started. A gateway that +# answers 407 (credential refused, or the traffic balance is exhausted) or +# 402 (traffic/balance exhausted) ends the run there, with exit 5, instead +# of launching a browser whose every request would be refused the same way. +# Any other outcome — a 200, a 403 from the site, a timeout, a DNS error — +# is not a verdict on the proxy and the run continues exactly as before. +# 2. Media blocking. Images, video/audio and web fonts are aborted in the +# browser while its traffic goes through a LOCAL proxy (`--proxy` / +# `--proxy-file`): no parser here reads them, and they are most of a page's +# bytes. Not applied with `--cdp-endpoint` (the remote browser is billed +# and configured on its own side) or with no proxy (nothing is metered). +# `--load-media` keeps them. +# --------------------------------------------------------------------------- + +PROXY_PRECHECK_STATUSES = { + 407: "the gateway refused the credential (or the traffic balance is exhausted)", + 402: "proxy traffic/balance exhausted", +} +PROXY_PRECHECK_STOP_REASON = "proxy_auth_failed" + +# Playwright `request.resource_type` values that are aborted. +BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font"}) +# The same three as CDP `Network.ResourceType` names (pyppeteer, via Fetch). +BLOCKED_CDP_RESOURCE_TYPES = ("Image", "Media", "Font") +# Selenium has no request events, only `Network.setBlockedURLs`, which +# matches URLs, not resource types: an image served without an extension +# still loads there. +BLOCKED_URL_PATTERNS = tuple( + "*.%s%s" % (ext, tail) + for ext in ("jpg", "jpeg", "png", "gif", "webp", "avif", "bmp", "ico", + "mp4", "webm", "m4v", "mov", "mp3", "m4a", "ogg", "wav", + "woff", "woff2", "ttf", "otf", "eot") + for tail in ("", "?*")) + + +class ProxyPrecheckError(RuntimeError): + """The proxy gateway itself refused the pre-check (407 or 402).""" + + def __init__(self, status: int): + self.status = status + super().__init__("HTTP %d: %s" % (status, PROXY_PRECHECK_STATUSES[status])) + + +_STATUS_IN_ERROR_RE = re.compile(r"Tunnel connection failed:\s*(40[27])\b") + + +def robots_url(url: Optional[str]) -> Optional[str]: + """`scheme://host/robots.txt` for a page URL — the pre-check's target.""" + try: + parts = urlparse(url or "") + except ValueError: + return None + if parts.scheme not in ("http", "https") or not parts.hostname: + return None + return "%s://%s/robots.txt" % (parts.scheme, parts.netloc.rsplit("@", 1)[-1]) + + +def precheck_proxy(proxy_url: Optional[str], probe_url: Optional[str], + timeout: float = 20.0) -> Optional[int]: + """One HEAD for `probe_url` through `proxy_url`. + + Raises ProxyPrecheckError on a gateway 407/402. Returns the status the + probe got otherwise, or None when there was nothing to check (no proxy, a + non-HTTP proxy scheme, no probe URL) or the request failed in transport — + a transport failure keeps the run's existing behaviour. The proxy URL is + never printed or put in the exception. + """ + if not proxy_url or not probe_url: + return None + try: + scheme = (urlparse(proxy_url).scheme or "http").lower() + except ValueError: + return None + if scheme not in ("http", "https"): + return None + try: + import requests + except ImportError: + return None + try: + response = requests.head(probe_url, proxies={"http": proxy_url, "https": proxy_url}, + timeout=timeout, allow_redirects=False) + except requests.exceptions.ProxyError as exc: + # An https target: the gateway answers the CONNECT, and urllib3 + # reports its status line as "Tunnel connection failed: 407 ...". + match = _STATUS_IN_ERROR_RE.search(str(exc)) + if match: + raise ProxyPrecheckError(int(match.group(1))) from None + return None + except Exception: # noqa: BLE001 - transport errors are not a verdict + return None + if response.status_code in PROXY_PRECHECK_STATUSES: + raise ProxyPrecheckError(response.status_code) + return response.status_code + + +def precheck_message(exc: ProxyPrecheckError, env_var: str, site: str, + browser: bool = True) -> str: + """What to print. Names the variable, never its value.""" + return ("[!] proxy pre-check failed — %s.\n" + " %s%s was never reached. Check %s (or the --proxy / --proxy-file " + "value)." % (exc, "No browser was started and " if browser else "", + site, env_var)) + + +def should_block_media(args, proxy_in_use: bool, over_cdp: bool = False) -> bool: + """Abort images/media/fonts? Only for a LOCAL browser behind a proxy. + + `--load-media` opts out. The hidden `--force-block-media` turns it on + without a proxy, so the effect on parsing can be measured for free. + """ + if getattr(args, "force_block_media", False): + return True + if getattr(args, "load_media", False) or over_cdp: + return False + return bool(proxy_in_use) + + +def playwright_media_route(route) -> None: + """`context.route("**/*", ...)` handler: abort the heavy resource types.""" + if route.request.resource_type in BLOCKED_RESOURCE_TYPES: + route.abort("blockedbyclient") + else: + route.continue_() + + +async def pyppeteer_block_media(page) -> None: + """Abort images/media/fonts in a pyppeteer page through the CDP Fetch domain. + + `page.setRequestInterception` sends Network.setRequestInterception, which + current Chromium no longer implements; Fetch.enable with resource-type + patterns pauses only the requests to be aborted. + """ + import asyncio + client = await page.target.createCDPSession() + + def on_paused(event): + asyncio.ensure_future(client.send("Fetch.failRequest", { + "requestId": event["requestId"], "errorReason": "BlockedByClient"})) + + client.on("Fetch.requestPaused", on_paused) + await client.send("Fetch.enable", {"patterns": [ + {"resourceType": kind, "requestStage": "Request"} + for kind in BLOCKED_CDP_RESOURCE_TYPES]}) + + +def selenium_block_media(driver) -> None: + """Block media URLs in a Chrome WebDriver through `Network.setBlockedURLs`.""" + driver.execute_cdp_cmd("Network.enable", {}) + driver.execute_cdp_cmd("Network.setBlockedURLs", {"urls": list(BLOCKED_URL_PATTERNS)}) diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index b1c5abc..2b027ef 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -55,7 +55,8 @@ import page_parser as pp from output_writer import EXIT_BAD_USAGE, EXIT_CRASH from fingerprint_client import fetch_fingerprint, refuse_if_cdp, user_agent_from -from proxy_pool import Proxy, ProxyPool, ProxyParseError, load_proxies, redact_credentials +from proxy_pool import (BLOCKED_CDP_RESOURCE_TYPES, Proxy, ProxyPool, ProxyParseError, load_proxies, pyppeteer_block_media, + redact_credentials, should_block_media) from scraper_api_client import TwoCaptchaClient ENGINE_NAME = "puppeteer" @@ -105,6 +106,7 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--dump-html", action="store_true") p.add_argument("--headless", dest="headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") + page_flow.add_media_arguments(p) page_flow.add_run_arguments(p) return p @@ -134,7 +136,7 @@ async def _launch(*, headless: bool, proxy: Optional[Proxy], cdp_endpoint: Optio return await pyppeteer_launch(**kwargs) -async def _authenticate_if_needed(page, proxy: Optional[Proxy]) -> None: +async def _authenticate_if_needed(page, proxy: Optional[Proxy], *, block_media: bool = False) -> None: """Answer the proxy's auth challenge over the CDP `Fetch` domain. pyppeteer's own `page.authenticate()` turns on @@ -142,14 +144,24 @@ async def _authenticate_if_needed(page, proxy: Optional[Proxy]) -> None: measured 2026-10-04 with Chrome for Testing, every proxied URL failed with "'Network.setRequestInterception' wasn't found" before a byte was fetched. `Fetch` is that method's replacement and exists in every - Chromium since 74. Credentials travel in the CDP message, never argv.""" + Chromium since 74. Credentials travel in the CDP message, never argv. + + With `block_media`, the same requestPaused handler fails images, media + and fonts (one Fetch interception per page, not two); without a + credential to answer, proxy_pool.pyppeteer_block_media does it alone.""" auth = proxy.pyppeteer_auth_dict() if proxy is not None else None if not auth: + if block_media: + await pyppeteer_block_media(page) return client = page._client async def _continue(event): try: + if block_media and event.get("resourceType") in BLOCKED_CDP_RESOURCE_TYPES: + await client.send("Fetch.failRequest", {"requestId": event["requestId"], + "errorReason": "BlockedByClient"}) + return await client.send("Fetch.continueRequest", {"requestId": event["requestId"]}) except Exception as exc: # noqa: BLE001 — a request cancelled meanwhile is not an error log.debug("Fetch.continueRequest failed: %s", redact_credentials(str(exc))) @@ -198,11 +210,12 @@ async def _release(browser, *, remote: bool) -> None: log.debug("browser release failed: %s", exc) -async def _open_page(browser, *, proxy: Optional[Proxy], user_agent: Optional[str], autosolve: bool): +async def _open_page(browser, *, proxy: Optional[Proxy], user_agent: Optional[str], autosolve: bool, + block_media: bool = False): page = await browser.newPage() if user_agent: await page.setUserAgent(user_agent) - await _authenticate_if_needed(page, proxy) + await _authenticate_if_needed(page, proxy, block_media=block_media) if autosolve: await _enable_scraping_browser_auto_solve(page) return page @@ -244,10 +257,14 @@ class _PyppeteerEngine: def __init__(self, args: argparse.Namespace, *, remote_browser, autosolve: bool, user_agent: Optional[str], client): self.args, self.remote_browser, self.autosolve, self.user_agent, self.client = args, remote_browser, autosolve, user_agent, client + self.media_blocked = False async def open(self, proxy) -> _PyppeteerSession: browser = self.remote_browser or await _launch(headless=self.args.headless, proxy=proxy, cdp_endpoint=None) - page = await _open_page(browser, proxy=proxy, user_agent=self.user_agent, autosolve=self.autosolve) + block = should_block_media(self.args, proxy_in_use=proxy is not None, over_cdp=self.remote_browser is not None) + self.media_blocked = self.media_blocked or block + page = await _open_page(browser, proxy=proxy, user_agent=self.user_agent, autosolve=self.autosolve, + block_media=block) return _PyppeteerSession(page, browser, owns_browser=self.remote_browser is None) async def sleep(self, seconds: float) -> None: @@ -306,6 +323,11 @@ async def run(args: argparse.Namespace) -> int: if args.cdp_endpoint and proxy_pool is not None: log.warning("Ignoring --proxy: a --cdp-endpoint session already carries its own exit IP.") proxy_pool = None + # Before any browser exists: a proxy that refuses its credential (407) or + # has no traffic left (402) would refuse every request the browser makes. + stop = page_flow.proxy_precheck(args, proxy_pool, urls=urls, engine_name=ENGINE_NAME, started_at=started_at) + if stop is not None: + return stop client = None if args.twocaptcha_key and (args.solve_captcha != "off" or args.fingerprint): diff --git a/selenium_scraper.py b/selenium_scraper.py index ed01d11..be2aca7 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -55,7 +55,7 @@ import scraper_api_engine import page_parser as pp from output_writer import EXIT_BAD_USAGE, EXIT_CRASH -from proxy_pool import Proxy, ProxyPool, ProxyParseError, load_proxies +from proxy_pool import Proxy, ProxyPool, ProxyParseError, load_proxies, selenium_block_media, should_block_media from fingerprint_client import fetch_fingerprint, refuse_if_cdp, user_agent_from from scraper_api_client import TwoCaptchaClient @@ -113,6 +113,7 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--dump-html", action="store_true") p.add_argument("--headless", dest="headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") + page_flow.add_media_arguments(p) page_flow.add_run_arguments(p) return p @@ -231,10 +232,20 @@ class _SeleniumEngine: def __init__(self, args: argparse.Namespace, *, user_agent: Optional[str], client): self.args, self.user_agent, self.client = args, user_agent, client + self.media_blocked = False async def open(self, proxy) -> _SeleniumSession: - return _SeleniumSession(_build_driver(headless=self.args.headless, proxy=proxy, - cdp_endpoint=self.args.cdp_endpoint, user_agent=self.user_agent)) + driver = _build_driver(headless=self.args.headless, proxy=proxy, + cdp_endpoint=self.args.cdp_endpoint, user_agent=self.user_agent) + if should_block_media(self.args, proxy_in_use=proxy is not None, over_cdp=bool(self.args.cdp_endpoint)): + # By URL pattern (Selenium has no request events): an image + # served without a file extension still loads. + try: + selenium_block_media(driver) + self.media_blocked = True + except WebDriverException as exc: + log.warning("Could not block media over CDP (continuing with it): %s", str(exc).splitlines()[0][:200]) + return _SeleniumSession(driver) async def sleep(self, seconds: float) -> None: time.sleep(seconds) @@ -281,6 +292,13 @@ def run(args: argparse.Namespace) -> int: if args.cdp_endpoint and proxy_pool is not None: log.warning("Ignoring --proxy: a --cdp-endpoint session already carries its own exit IP.") proxy_pool = None + # Before any browser exists: a proxy that refuses its credential (407) or + # has no traffic left (402) would refuse every request the browser makes. + # Checked with the proxy as configured, credentials included: the + # question is whether the proxy account itself is refused or exhausted. + stop = page_flow.proxy_precheck(args, proxy_pool, urls=urls, engine_name=ENGINE_NAME, started_at=started_at) + if stop is not None: + return stop client = None if args.twocaptcha_key and (args.solve_captcha != "off" or args.fingerprint): diff --git a/smoke_test.py b/smoke_test.py index 3496412..f4ba7b0 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1357,6 +1357,316 @@ def _(): assert "::error::" in proc.stdout, (has_proxy, code, proc.stdout[-300:], proc.stderr[-300:]) +# --------------------------------------------------------------------------- # +# Traffic savers: the proxy pre-check and media blocking +# --------------------------------------------------------------------------- # +def _status_proxy(status): + """A local stand-in for a proxy gateway that answers every request, + CONNECT included, with `status`. Returns (server, port, hits).""" + import http.server + import socketserver + import threading + hits = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _answer(self): + hits.append((self.command, self.path)) + self.send_response(status) + self.send_header("Content-Length", "0") + self.send_header("Proxy-Authenticate", 'Basic realm="stub"') + self.end_headers() + do_CONNECT = do_GET = do_HEAD = _answer + + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, server.server_address[1], hits + + +_STUB_LOGIN = "user" +_STUB_SECRET = "S3cret" + "Pw" + + +def _stub_proxy_url(port): + return "http://" + "%s:%s" % (_STUB_LOGIN, _STUB_SECRET) + "@127.0.0.1:%d" % port + + +def _patched_browser_start(mod): + """Patch the engine's browser launch to record the call and fail. The + driver need not be installed: the pre-check runs before it is used.""" + from unittest import mock + started = [] + + def launch(*a, **k): + started.append(True) + raise RuntimeError("stub: the browser would start here") + if mod is playwright_scraper: + return started, [mock.patch.object(mod, "async_playwright", launch)] + if mod is puppeteer_scraper: + return started, [mock.patch.object(mod, "pyppeteer_launch", launch), + mock.patch.object(mod, "_launch", lambda **k: launch())] + return started, [mock.patch.object(mod, "webdriver", object()), + mock.patch.object(mod, "_build_driver", launch)] + + +def _run_engine_quietly(mod, argv): + import contextlib + import io + import logging + args = mod.build_arg_parser().parse_args(argv) + out = io.StringIO() + logging.disable(logging.CRITICAL) + try: + with contextlib.redirect_stderr(out), contextlib.redirect_stdout(out): + rc = asyncio_run_maybe(mod, args) + finally: + logging.disable(logging.NOTSET) + return rc, out.getvalue() + + +@check("proxy pre-check: a gateway 407 or 402 exits 5 after ONE request, before any browser, naming INSTAGRAM_PROXY and never the credential; only the attempt sidecar is written and earlier output is untouched (every engine)") +def _(): + import contextlib + for mod in (playwright_scraper, puppeteer_scraper, selenium_scraper): + for status, words in ((407, "refused the credential"), (402, "balance exhausted")): + server, port, hits = _status_proxy(status) + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "o.json" + out.write_text("[1]") + Path(f"{out}.meta.json").write_text('{"status": "complete"}') + started, patches = _patched_browser_start(mod) + try: + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + rc, text = _run_engine_quietly(mod, ["--url", "natgeo", "--out", str(out), + "--proxy", _stub_proxy_url(port)]) + finally: + server.shutdown() + server.server_close() + tag = f"{mod.__name__} {status}" + assert rc == output_writer.EXIT_REMOTE_API_ERROR, (tag, rc, text[-300:]) + assert not started, f"{tag}: a browser was started" + assert hits == [("CONNECT", "www.instagram.com:443")], (tag, hits) + assert words in text and "INSTAGRAM_PROXY" in text, (tag, text[-300:]) + assert _STUB_SECRET not in text, f"{tag}: the credential was printed" + attempt = Path(f"{out}{output_writer.ATTEMPT_META_SUFFIX}") + meta = json.loads(attempt.read_text()) + assert (meta["status"], meta["stop_reason"], meta["data_updated"], meta["engine"]) == \ + ("failed", "proxy_auth_failed", False, mod.ENGINE_NAME), (tag, meta) + assert _STUB_SECRET not in attempt.read_text(), tag + assert out.read_text() == "[1]" and Path(f"{out}.meta.json").read_text() == '{"status": "complete"}', tag + assert sorted(p.name for p in Path(td).iterdir()) == sorted( + ["o.json", "o.json.meta.json", "o.json" + output_writer.ATTEMPT_META_SUFFIX]), tag + + +@check("proxy pre-check: a site 403 through the tunnel, or an unreachable proxy, is not a verdict — the browser still starts (every engine)") +def _(): + import contextlib + for mod in (playwright_scraper, puppeteer_scraper, selenium_scraper): + server, port, hits = _status_proxy(403) + with tempfile.TemporaryDirectory() as td: + started, patches = _patched_browser_start(mod) + try: + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + _run_engine_quietly(mod, ["--url", "natgeo", "--out", str(Path(td) / "o.json"), + "--proxy", _stub_proxy_url(port), "--retries", "0"]) + finally: + server.shutdown() + server.server_close() + assert started, f"{mod.__name__}: a site 403 at the pre-check stopped the run" + assert len(hits) == 1, (mod.__name__, hits) + assert proxy_pool.precheck_proxy("http://user:pass@127.0.0.1:1", "https://x.invalid/robots.txt", 3) is None + + +@check("proxy pre-check: never sent over --cdp-endpoint, --scraper-api or without a proxy; probes the first URL's robots.txt with the exit the first fetch uses") +def _(): + import argparse + from unittest import mock + pool = proxy_pool.ProxyPool(proxy_pool.load_proxies("http://user:pass@127.0.0.1:1", None)) + with mock.patch.object(proxy_pool, "precheck_proxy", side_effect=AssertionError("pre-check sent")): + for extra in ({"cdp_endpoint": "ws://127.0.0.1:1"}, {"scraper_api": True}): + args = argparse.Namespace(**{"out": "x.json", "cdp_endpoint": None, "scraper_api": False, **extra}) + assert page_flow.proxy_precheck(args, pool, urls=["https://www.instagram.com/natgeo/"], + engine_name="t", started_at=0.0) is None, extra + args = argparse.Namespace(out="x.json", cdp_endpoint=None, scraper_api=False) + assert page_flow.proxy_precheck(args, None, urls=["https://www.instagram.com/natgeo/"], + engine_name="t", started_at=0.0) is None + seen = [] + with mock.patch.object(proxy_pool, "precheck_proxy", lambda proxy, probe, timeout: seen.append((proxy, probe))): + page_flow.proxy_precheck(args, pool, urls=["https://www.instagram.com/p/DeAZ0uAgNuA/"], + engine_name="t", started_at=0.0) + assert seen == [("http://user:pass@127.0.0.1:1", "https://www.instagram.com/robots.txt")], seen + assert pool.next().host == "127.0.0.1", "peek() must not have moved the cursor" + odd = proxy_pool.Proxy(host="h.example", port=8080, login="user@x", password="pass:s/s") + assert odd.url() == "http://" + "user%40x:pass%3As%2Fs" + "@h.example:8080", odd.url() + assert proxy_pool.parse_proxy_line("h.example:8080").url() == "http://h.example:8080" + + +@check("media: blocked only for a LOCAL browser behind a proxy; --load-media keeps it; never over CDP; the hidden switch forces it") +def _(): + import argparse + ns = argparse.Namespace + sbm = proxy_pool.should_block_media + assert sbm(ns(load_media=False), proxy_in_use=True) + assert not sbm(ns(load_media=True), proxy_in_use=True) + assert not sbm(ns(load_media=False), proxy_in_use=False) + assert not sbm(ns(load_media=False), proxy_in_use=True, over_cdp=True) + assert sbm(ns(load_media=False, force_block_media=True), proxy_in_use=False) + for mod in (playwright_scraper, puppeteer_scraper, selenium_scraper): + text = mod.build_arg_parser().format_help() + assert "--load-media" in text and "--force-block-media" not in text, mod.__name__ + args = mod.build_arg_parser().parse_args(["--url", "natgeo"]) + assert args.load_media is False and args.force_block_media is False, mod.__name__ + + +@check("media: playwright aborts image/media/font and continues everything else, on the context, behind a proxy only") +def _(): + import argparse + + class Route: + def __init__(self, kind): + self.request = argparse.Namespace(resource_type=kind) + self.done = None + + async def abort(self, reason=None): + self.done = "abort" + + async def continue_(self): + self.done = "continue" + for kind, want in (("image", "abort"), ("media", "abort"), ("font", "abort"), ("document", "continue"), + ("script", "continue"), ("xhr", "continue"), ("fetch", "continue"), ("stylesheet", "continue")): + route = Route(kind) + asyncio.run(playwright_scraper._async_media_route(route)) + assert route.done == want, (kind, route.done) + + class Context: + def __init__(self): + self.routes = [] + + async def route(self, pattern, handler): + self.routes.append((pattern, handler)) + + async def new_page(self): + return object() + + class Browser: + contexts = [] + + def __init__(self): + self.made = [] + + async def new_context(self, **kwargs): + context = Context() + self.made.append(context) + return context + proxy = proxy_pool.parse_proxy_line("http://user:pass@127.0.0.1:1") + for argv, use_proxy, cdp, want in ((["--url", "x"], True, False, True), (["--url", "x"], False, False, False), + (["--url", "x", "--load-media"], True, False, False), + (["--url", "x", "--force-block-media"], False, False, True)): + args = playwright_scraper.build_arg_parser().parse_args(argv) + browser = Browser() + engine = playwright_scraper._PlaywrightEngine(browser, args, autosolve=False, user_agent=None, client=None) + asyncio.run(engine.open(proxy if use_proxy else None)) + routes = browser.made[0].routes + assert (routes == [("**/*", playwright_scraper._async_media_route)]) == want, (argv, use_proxy, routes) + assert engine.media_blocked == want, argv + args = playwright_scraper.build_arg_parser().parse_args(["--url", "x", "--cdp-endpoint", "ws://127.0.0.1:1"]) + browser = Browser() + engine = playwright_scraper._PlaywrightEngine(browser, args, autosolve=False, user_agent=None, client=None) + asyncio.run(engine.open(proxy)) + assert browser.made[0].routes == [] and not engine.media_blocked, "media blocked over --cdp-endpoint" + + +@check("media: pyppeteer fails images/media/fonts in its EXISTING proxy-auth Fetch handler (one Fetch.enable), and alone via Fetch resource-type patterns without a credential") +def _(): + class Client: + def __init__(self): + self.sent, self.handlers = [], {} + + def on(self, name, fn): + self.handlers[name] = fn + + async def send(self, cmd, params=None): + self.sent.append((cmd, params)) + + async def drive(proxy, block): + client = Client() + + async def create(): + return client + import argparse + page = argparse.Namespace(_client=client, target=argparse.Namespace(createCDPSession=create)) + await puppeteer_scraper._authenticate_if_needed(page, proxy, block_media=block) + if "Fetch.requestPaused" in client.handlers: + for rid, kind in (("i", "Image"), ("f", "Font"), ("m", "Media"), ("d", "Document"), ("s", "Script")): + client.handlers["Fetch.requestPaused"]({"requestId": rid, "resourceType": kind}) + for _ in range(5): + await asyncio.sleep(0) + return client.sent + + authed = proxy_pool.parse_proxy_line("http://user:pass@127.0.0.1:1") + sent = asyncio.run(drive(authed, True)) + assert [c for c, _ in sent].count("Fetch.enable") == 1, sent + failed = {p["requestId"] for c, p in sent if c == "Fetch.failRequest"} + continued = {p["requestId"] for c, p in sent if c == "Fetch.continueRequest"} + assert failed == {"i", "f", "m"} and continued == {"d", "s"}, sent + sent = asyncio.run(drive(authed, False)) + assert not [c for c, _ in sent if c == "Fetch.failRequest"], "media failed with --load-media" + sent = asyncio.run(drive(None, True)) + enable = [p for c, p in sent if c == "Fetch.enable"] + assert len(enable) == 1 and sorted(p["resourceType"] for p in enable[0]["patterns"]) == ["Font", "Image", "Media"], sent + assert {p["requestId"] for c, p in sent if c == "Fetch.failRequest"} >= {"i", "f", "m"}, sent + assert asyncio.run(drive(None, False)) == [], "Fetch opened with no credential and no blocking" + + +@check("media: selenium blocks images, video and fonts by URL extension (not js/css) right after the driver is built, behind a proxy only") +def _(): + from unittest import mock + + class Driver: + def __init__(self): + self.sent = [] + + def execute_cdp_cmd(self, cmd, params): + self.sent.append((cmd, params)) + proxy = proxy_pool.parse_proxy_line("http://user:pass@127.0.0.1:1") + for argv, use_proxy, want in ((["--url", "x"], True, True), (["--url", "x"], False, False), + (["--url", "x", "--load-media"], True, False)): + driver = Driver() + args = selenium_scraper.build_arg_parser().parse_args(argv) + engine = selenium_scraper._SeleniumEngine(args, user_agent=None, client=None) + with mock.patch.object(selenium_scraper, "_build_driver", lambda **k: driver): + asyncio.run(engine.open(proxy if use_proxy else None)) + blocked = dict(driver.sent).get("Network.setBlockedURLs", {}).get("urls", []) + assert bool(blocked) == want and engine.media_blocked == want, (argv, use_proxy, driver.sent) + if want: + assert all(p in blocked for p in ("*.jpg", "*.webp?*", "*.mp4", "*.woff2")), blocked + assert not any(p.startswith(("*.js", "*.css")) for p in blocked), blocked + + +@check("media: every local-browser engine consults the policy and applies its blocker, and the sidecar records media_blocked") +def _(): + for path, blocker in (("playwright_scraper.py", "_async_media_route"), ("puppeteer_scraper.py", "BLOCKED_CDP_RESOURCE_TYPES"), + ("selenium_scraper.py", "selenium_block_media(driver)")): + src = (ROOT / path).read_text(encoding="utf-8") + assert "should_block_media(" in src and blocker in src, path + assert "page_flow.proxy_precheck(args, proxy_pool" in src, f"{path}: no pre-check" + assert "page_flow.add_media_arguments(p)" in src, path + with tempfile.TemporaryDirectory() as td: + args = playwright_scraper.build_arg_parser().parse_args(["--url", "natgeo", "--out", str(Path(td) / "o.json"), + "--allow-empty"]) + args._since = None + page_flow.finish(args, products=[], blocked=False, remote_api_error=False, engine_name="t", urls=["u"], + started_at=0.0, pages_completed=1, failed_pages=[], media_blocked=True) + assert json.loads(Path(td, "o.json.meta.json").read_text())["media_blocked"] is True + + def run() -> int: """All @check-decorated functions above already ran at import time (that's the point — see the `check()` docstring) and self-registered