Count proxy traffic per run and stop at --proxy-budget-mb (default 10 MB)

.github/proxy_traffic_summary.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Canary step: one line about a run's proxy traffic for the job summary.
+
+    python3 .github/proxy_traffic_summary.py <out> <label> [budget_mb]
+
+Reads `<out>.meta.json` and `<out>.latest_attempt.meta.json` (the newer of
+the two when both exist), appends `**<label>**: proxy traffic X.X MB of
+<budget> MB` to $GITHUB_STEP_SUMMARY, and exits 1 with an `::error::` line
+when the run stopped at its proxy budget. A missing or unreadable sidecar,
+or a run that used no local proxy, is reported and is not a failure here:
+the canary's own check step judges the run itself.
+"""
+import json
+import os
+import sys
+
+
+def _load(path):
+    try:
+        with open(path, encoding="utf-8") as stream:
+            data = json.load(stream)
+        return data if isinstance(data, dict) else None
+    except FileNotFoundError:
+        return None
+    except (OSError, ValueError) as exc:
+        print("::warning::%s is unreadable: %s" % (path, exc))
+        return None
+
+
+def summarise(out, label, budget_mb=10.0):
+    """Return (line, whether the run stopped at its proxy budget)."""
+    metas = [m for m in (_load(out + ".meta.json"), _load(out + ".latest_attempt.meta.json")) if m]
+    meta = max(metas, key=lambda m: m.get("finished_at") or 0) if metas else None
+    budget = "%g MB" % budget_mb if budget_mb else "no"
+    if meta is None:
+        line = "**%s**: proxy traffic not recorded (no sidecar for %s)" % (label, out)
+    elif "proxy_bytes" not in meta:
+        line = "**%s**: proxy traffic not recorded (the sidecar has no proxy_bytes)" % label
+    elif meta.get("proxy_bytes") is None:
+        line = "**%s**: proxy traffic: none (no local proxy on this run)" % label
+    else:
+        line = "**%s**: proxy traffic %.1f MB of %s" % (label, (meta.get("proxy_bytes") or 0) / 1e6, budget)
+    exceeded = bool(meta and meta.get("proxy_budget_exceeded"))
+    return line, exceeded
+
+
+def main(argv):
+    if len(argv) < 3:
+        print("usage: proxy_traffic_summary.py <out> <label> [budget_mb]", file=sys.stderr)
+        return 2
+    try:
+        budget_mb = float(argv[3]) if len(argv) > 3 else 10.0
+    except ValueError:
+        budget_mb = 10.0
+    line, exceeded = summarise(argv[1], argv[2], budget_mb)
+    print(line)
+    summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary:
+        try:
+            with open(summary, "a", encoding="utf-8") as stream:
+                stream.write(line + "\n")
+        except OSError as exc:
+            print("::warning::could not write the job summary: %s" % exc)
+    if exceeded:
+        print("::error::stopped: proxy budget %g MB exceeded" % budget_mb)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/workflows/canary.yml

@@ -49,9 +49,15 @@ jobs:
         run: |
           set +e
           python3 playwright_scraper.py --url "$CANARY_ACCOUNT" --posts 3 \
-            --format json --out canary.json --dump-html
+            --format json --out canary.json --dump-html --proxy-budget-mb 10
           echo "canary_exit=$?" >> "$GITHUB_ENV"
 
+      # What the run sent through INSTAGRAM_PROXY, read from its sidecar; red
+      # when it stopped at the 10 MB budget (stop_reason proxy_budget_exceeded).
+      - name: Proxy traffic
+        if: always()
+        run: python3 .github/proxy_traffic_summary.py canary.json canary 10
+
       - name: Require a complete run with full rows
         run: |
           python3 -c "

.github/workflows/tests.yml

@@ -123,6 +123,10 @@ jobs:
         run: |
           source .venv-${{ matrix.engine }}/bin/activate
           python3 smoke_test.py
+          # The proxy meter against a stub proxy in a real browser: a missing
+          # browser fails this job instead of skipping.
+          PROXY_METER_REQUIRE=${{ matrix.engine == 'puppeteer' && 'pyppeteer' || matrix.engine }} \
+            python3 -m unittest -v tests.test_proxy_meter
           python3 -c "
           import ${{ matrix.engine }}_scraper as m
           assert getattr(m, {'playwright': 'async_playwright', 'selenium': 'webdriver', 'puppeteer': 'pyppeteer_launch'}['${{ matrix.engine }}']) is not None, \

CHANGELOG.md

@@ -24,10 +24,30 @@ that changes a default is called out at the top of its entry.
 >   blocks them by URL extension. `--load-media` restores the old behaviour.
 >   Nothing changes with `--cdp-endpoint`, `--scraper-api` or with no proxy.
 >   `<out>.meta.json` records `media_blocked`.
+> * Behind a local proxy, a run now stops once about 10 MB have gone through
+>   it (`--proxy-budget-mb`, `0` = unlimited): the rows read so far are kept
+>   and the run exits 6 (5 with none), `stop_reason: proxy_budget_exceeded`.
+>   A proxied run that collects nothing now writes
+>   `<out>.latest_attempt.meta.json` with its outcome and byte count. Not
+>   applied with `--cdp-endpoint`, `--scraper-api` or without a proxy.
 
 ### Added
 
 - `--load-media` in all three engines.
+- Proxy traffic meter (`proxy_meter.py`) in all three browser engines, active
+  only behind a local proxy: Playwright sums `request.sizes()`, Puppeteer
+  and Selenium read CDP `Network.loadingFinished.encodedDataLength` plus the
+  request sizes (Selenium turns Chrome's performance log on for this). It is
+  an estimate of the tunnelled payload (TLS/`CONNECT` overhead not
+  included). The sidecar records `proxy_bytes`, `proxy_budget_bytes` and
+  `proxy_budget_exceeded`, and a proxied run ends with one `Proxy traffic:
+  X.X MB of Y MB budget` line. New flag `--proxy-budget-mb` (default 10).
+  The proxy pre-check's own `HEAD` is counted too.
+- CI only: the canary passes `--proxy-budget-mb 10`, writes `**canary**:
+  proxy traffic X.X MB of 10 MB` to the job summary, and fails with
+  `::error::stopped: proxy budget 10 MB exceeded` when the run hit the
+  budget. The engine jobs run the meter against a stub proxy in a real
+  browser.
 
 ### Changed
 

Dockerfile

@@ -19,7 +19,7 @@ RUN pip install --no-cache-dir -r requirements-playwright.txt \
 # installed in this image, so their own try/except ImportError guard is
 # exactly what fires, the same as smoke_test.py's no-engine-installed path
 # in CI.
-COPY env_config.py proxy_pool.py output_writer.py captcha_solver.py \
+COPY env_config.py proxy_pool.py proxy_meter.py output_writer.py captcha_solver.py \
      fingerprint_client.py scraper_api_client.py scraper_api_engine.py diff_runs.py \
      page_parser.py page_flow.py run_state.py playwright_scraper.py puppeteer_scraper.py \
      selenium_scraper.py smoke_test.py ./

README.md

@@ -210,7 +210,7 @@ or more, or spread a big list over time.
 
 ## Traffic through a paid proxy
 
-A residential proxy bills per byte, so two things happen only while a
+A residential proxy bills per byte, so three things happen only while a
 LOCAL browser goes through `--proxy` / `--proxy-file` (or
 `INSTAGRAM_PROXY`). Neither applies with `--cdp-endpoint` (that browser
 brings its own exit and is billed on its own side) or with `--scraper-api`
@@ -257,6 +257,33 @@ offline: the fixtures (the parser reads embedded JSON only), and each real
 browser driven through a local proxy to a test page. With blocking, only the
 HTML and the script went through the proxy, and the script still ran.
 
+**A traffic budget.** Every byte a local browser sends or receives through
+the proxy is counted, and `--proxy-budget-mb` (default `10`; `0` =
+unlimited) caps a run. Once the running total passes the budget, no
+further URL (and no further `--posts` post) is fetched, the browser aborts
+any request it still tries, the rows already read are kept, and the run
+ends with `stop_reason: proxy_budget_exceeded`: exit 6 with rows, 5
+without. The URLs not fetched are listed as `not_attempted`, so a `--checkpoint` run can
+`--resume` them. It never goes on silently. At the end of a proxied run one
+line is printed, e.g. `Proxy traffic: 3.2 MB of 10 MB budget`, and the
+sidecar records `proxy_bytes`, `proxy_budget_bytes` and
+`proxy_budget_exceeded` (`null` / `null` / `false` when no local proxy was
+used). A proxied run that writes no rows records them in
+`<out>.latest_attempt.meta.json`, leaving `<out>` and `<out>.meta.json` as
+they were.
+
+The count is an estimate of the tunnelled payload — request line, headers
+and body sent, response headers and body as received (compressed) — not
+the proxy's invoice: TLS records, the `CONNECT` exchange, WebSocket frames
+and Chrome's own background requests are not visible to it, so the bill is
+a few per cent higher. 1 MB = 1,000,000 bytes.
+
+| Engine | Counted from |
+|---|---|
+| Playwright | `request.sizes()` of every finished or failed request (request headers + body, response headers + body) |
+| Puppeteer | CDP `Network.loadingFinished.encodedDataLength` of every response that came over the network, plus each request's line, headers and body (`Network.requestWillBeSent`) |
+| Selenium | the same CDP events, from Chrome's performance log (turned on behind a local proxy only) |
+
 ## Scraper API mode
 
 With `--scraper-api` no browser is driven: each page is one 2Captcha
@@ -277,17 +304,18 @@ solves spent, and a hash of the output file). With the options below it
 also lists the posts left out (`excluded_urls`), every post seen so far
 (`seen_post_skus`) and whether the run was incremental. A run that
 collects nothing writes neither, so it never replaces your previous good
-file. The one exception is a proxy refused at the pre-check (see
-[Traffic through a paid proxy](#traffic-through-a-paid-proxy)): that run
-records itself in `<out>.latest_attempt.meta.json`, a separate file.
+file. The exceptions are a proxy refused at the pre-check, and any run
+through a local proxy that collects nothing (see
+[Traffic through a paid proxy](#traffic-through-a-paid-proxy)): those runs
+record themselves in `<out>.latest_attempt.meta.json`, a separate file.
 
 | Exit | Meaning |
 |---|---|
 | `0` | complete |
-| `6` | partial: rows were written, but the run did not finish cleanly — `stop_reason` says why (`blocked`, `rate_limited`, `remote_api_error`, `failed_pages`, `parse_error`, `proxy_pool_exhausted`) |
+| `6` | partial: rows were written, but the run did not finish cleanly — `stop_reason` says why (`blocked`, `rate_limited`, `remote_api_error`, `failed_pages`, `parse_error`, `proxy_pool_exhausted`, `proxy_budget_exceeded`) |
 | `3` | blocked or throttled, no rows |
 | `4` | no rows: every input was not found, or every post was older than `--since` or already collected by `--incremental-from` |
-| `5` | nothing could be read due to fetch, parse, or remote-service failure; no rows. Also a proxy that answered `407`/`402` at the pre-check, before any browser was started |
+| `5` | nothing could be read due to fetch, parse, or remote-service failure; no rows. Also a proxy that answered `407`/`402` at the pre-check, before any browser was started, and a proxy budget that ran out before any row was read |
 | `2` | bad usage, including input where every line was skipped |
 | `1` | crash (a bug; please report it) |
 
@@ -410,6 +438,7 @@ the command line.
 | `--proxy` / `--proxy-file` / `--proxy-shuffle` | `INSTAGRAM_PROXY` | one proxy or a rotating pool, for a local browser |
 | `--proxy-block-retries` | 3 | blocked answers before that proxy is dropped from the pool |
 | `--load-media` | off | load images, video and fonts behind a proxy too (they are aborted by default there; see [Traffic through a paid proxy](#traffic-through-a-paid-proxy)) |
+| `--proxy-budget-mb` | 10 | stop once this many MB went through a local proxy; `0` = unlimited (see [Traffic through a paid proxy](#traffic-through-a-paid-proxy)) |
 | `--scraper-api` | off | no browser: fetch each page through 2Captcha's Scraper API (needs `TWOCAPTCHA_KEY`; routed through `INSTAGRAM_CDP_ENDPOINT` when set) — posts only, see [Scraper API mode](#scraper-api-mode) |
 | `--cdp-endpoint` | `INSTAGRAM_CDP_ENDPOINT` | connect to a Scraping Browser API profile instead of launching a browser |
 | `--solve-captcha` | when-blocked | `off` disables the Browser API's own captcha auto-solve; there is no local solver |

output_writer.py

@@ -29,6 +29,8 @@
 # caller (CI, a cron job, another program) must be able to tell these apart
 # without parsing stdout.
 # --------------------------------------------------------------------------- #
+import proxy_meter
+
 EXIT_OK = 0
 EXIT_CRASH = 1
 EXIT_BAD_USAGE = 2
@@ -358,6 +360,34 @@ def write_failed_attempt(
         "transport_error": transport_error,
         "data_updated": False,
     }
+    meta.update(proxy_meter.sidecar())  # proxy_bytes / proxy_budget_bytes / proxy_budget_exceeded
+    path = f"{out_path}{ATTEMPT_META_SUFFIX}"
+    with _atomic(path) as f:
+        json.dump(meta, f, ensure_ascii=False, indent=2)
+    proxy_meter.report()
+    return path
+
+
+def _write_attempt(out_path: str, *, status: str, stop_reason: str, engine: str, url: str, pages_requested: int,
+                   pages_completed: int, failed_pages: List[int], blocked: bool, started_at: float,
+                   extra: Optional[dict] = None) -> str:
+    """`<out>.latest_attempt.meta.json` for a proxied run that wrote no data."""
+    meta = {
+        "status": status,
+        "stop_reason": stop_reason,
+        "engine": engine,
+        "url": url,
+        "pages_requested": pages_requested,
+        "pages_completed": pages_completed,
+        "failed_pages": failed_pages or [],
+        "product_count": 0,
+        "started_at": started_at,
+        "finished_at": time.time(),
+        "blocked": bool(blocked),
+        "data_updated": False,
+    }
+    meta.update(proxy_meter.sidecar())
+    meta.update(extra or {})
     path = f"{out_path}{ATTEMPT_META_SUFFIX}"
     with _atomic(path) as f:
         json.dump(meta, f, ensure_ascii=False, indent=2)
@@ -398,6 +428,9 @@ def finish_run(
     NEVER overwrites a previous good output with an empty one unless the
     caller explicitly passed --allow-empty."""
     failed_pages = failed_pages or []
+    meter = proxy_meter.current()
+    if meter.exceeded:
+        incomplete_reason = proxy_meter.STOP_REASON
     partial = bool(failed_pages) or bool(incomplete_reason)
     zero_products = len(products) == 0
 
@@ -435,6 +468,11 @@ def finish_run(
     else:
         status, exit_code = "complete", EXIT_OK
         stop_reason = status
+    if meter.exceeded:
+        # The run stopped itself at --proxy-budget-mb: rows kept -> 6, none -> 5,
+        # whatever else the pages it did fetch said.
+        status, exit_code = ("remote_api_error", EXIT_REMOTE_API_ERROR) if zero_products else ("partial", EXIT_PARTIAL)
+        stop_reason = proxy_meter.STOP_REASON
 
     # The "never overwrite good output with empty" rule: a zero-product
     # outcome (whatever its status above — blocked/remote_api_error/empty
@@ -446,6 +484,13 @@ def finish_run(
     # engine's own logs carry the diagnostic detail — that's what a
     # failed run's output is FOR, not this file.
     if zero_products and not allow_empty:
+        if meter.active:
+            # Nothing is written, but what the proxy carried is still recorded
+            # (`<out>.latest_attempt.meta.json`; data and .meta.json untouched).
+            _write_attempt(out_path, status=status, stop_reason=stop_reason, engine=engine, url=url,
+                           pages_requested=pages_requested, pages_completed=pages_completed,
+                           failed_pages=failed_pages, blocked=blocked, started_at=started_at, extra=extra_meta)
+        proxy_meter.report()
         return exit_code
 
     escaped = write_output(products, out_path, fmt)
@@ -454,6 +499,7 @@ def finish_run(
         # Under the caller's extra, never over it: housekeeping must not
         # silently replace a value the engine chose to record.
         extra["csv_cells_escaped"] = escaped
+    extra.update(proxy_meter.sidecar())
     extra.update(extra_meta or {})
     if rejected_rows:
         extra["rejected_rows"] = rejected_rows
@@ -474,4 +520,5 @@ def finish_run(
         price_confirmed_pct=price_confirmed_pct, started_at=started_at,
         extra=extra or None,
     )
+    proxy_meter.report()
     return exit_code

page_flow.py

@@ -38,6 +38,7 @@
 from typing import List, Optional
 
 import page_parser as pp
+import proxy_meter as _meter
 import run_state
 from run_state import add_arguments as add_run_arguments
 from captcha_solver import detect_from_html
@@ -260,6 +261,9 @@ async def fetch_item(engine: Engine, args, url: str, index: int, proxy_pool, cli
     try:
         last_error, status = None, None
         for attempt in range(args.retries + 1):
+            if _meter.exceeded():
+                last_error = _meter.STOP_REASON
+                break
             try:
                 status = await session.goto(url)
                 last_error = None
@@ -275,6 +279,8 @@ async def fetch_item(engine: Engine, args, url: str, index: int, proxy_pool, cli
                     break  # a refused key or an empty balance: every retry would fail the same way
                 if attempt < args.retries:
                     await engine.sleep(args.retry_delay)
+        if last_error == _meter.STOP_REASON:
+            return Outcome(None, False, False, "none", failure=_meter.STOP_REASON)
         if last_error is not None:
             _log.error("%s permanently failed to load: %s", url, last_error)
             return Outcome(None, False, False, "none",
@@ -388,7 +394,7 @@ async def run(engine: Engine, args, *, urls: List[str], proxy_pool, client, star
         records[url] = record
         run_state.save(args, urls, queue, records)
         consecutive_blocks = consecutive_blocks + 1 if outcome.blocked else 0
-        stop = (outcome.failure == "proxy_pool_exhausted" or getattr(engine, "fatal", None)
+        stop = (outcome.failure == "proxy_pool_exhausted" or getattr(engine, "fatal", None) or _meter.exceeded()
                 or (proxy_pool is None and consecutive_blocks >= STOP_AFTER_CONSECUTIVE_BLOCKS))
         if stop:
             for pending in queue[i:args.max_results]:
@@ -453,7 +459,7 @@ def finish(args, *, products: List[Product], blocked: bool, remote_api_error: bo
         failed_pages=failed_pages or None, blocked=blocked, remote_api_error=remote_api_error,
         allow_empty=args.allow_empty, started_at=started_at, price_confirmed_pct=None,
         max_results=args.max_results, rate_limited=rate_limited,
-        incomplete_reason=next((f["reason"] for f in (failures or []) if f["reason"] in ("parse_error", "proxy_pool_exhausted")), None),
+        incomplete_reason=next((f["reason"] for f in (failures or []) if f["reason"] in ("parse_error", "proxy_pool_exhausted", _meter.STOP_REASON)), None),
         capped=bool(capped) if capped is not None else len(urls) > args.max_results or bool(getattr(args, "posts", 0)),
         extra_meta=extra,
     )
@@ -492,6 +498,14 @@ def validate_common(args, *, urls, skipped, print_err) -> Optional[int]:
 PROXY_PRECHECK_TIMEOUT_S = 20.0
 
 
+def start_proxy_meter(args, proxy_pool) -> "_meter.ProxyMeter":
+    """Meter this run's traffic when it goes through a LOCAL proxy (not
+    over --cdp-endpoint, not the Scraper API, not without a proxy)."""
+    active = (proxy_pool is not None and not getattr(args, "cdp_endpoint", None)
+              and not getattr(args, "scraper_api", False))
+    return _meter.start(getattr(args, "proxy_budget_mb", _meter.DEFAULT_BUDGET_MB), active=active)
+
+
 def proxy_precheck(args, proxy_pool, *, urls: List[str], engine_name: str, started_at: float,
                    print_err=None) -> Optional[int]:
     """Pre-check the exit the first URL will use; return the exit code to
@@ -541,3 +555,4 @@ def add_media_arguments(parser) -> None:
     # Turns blocking on without a proxy, so its effect on parsing can be
     # measured for free. Not a user feature.
     parser.add_argument("--force-block-media", action="store_true", help=argparse.SUPPRESS)
+    _meter.add_budget_argument(parser)