From 41a25bf278c2214c96e1398172913c53768c5cfc Mon Sep 17 00:00:00 2001 From: flowcool <19570361+flowcool@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:47:13 +0100 Subject: [PATCH 1/2] feat: integrate the reviewed catalog adapter without changing legacy BD tasks --- CHANGELOG.md | 5 + Dockerfile | 4 + README.md | 1 + apps/BillCollectorServices_pw.py | 4 +- apps/billcollector/bourse_direct_boundary.py | 281 ++++++++++++ apps/billcollector/collection.py | 5 + apps/billcollector/methods.py | 35 +- apps/billcollector/notice.py | 2 +- apps/billcollector/runner.py | 22 +- apps/recipes_playwright/recipe-pw-schema.yaml | 51 ++- doc/BOURSE_DIRECT_ADAPTER.md | 157 +++++++ doc/RECIPE_METHODS.md | 15 +- tests/qualify_bourse_direct_image.py | 7 +- tests/test_bc_bourse_direct_boundary.py | 399 ++++++++++++++++++ 14 files changed, 976 insertions(+), 12 deletions(-) create mode 100644 apps/billcollector/bourse_direct_boundary.py create mode 100644 doc/BOURSE_DIRECT_ADAPTER.md create mode 100644 tests/test_bc_bourse_direct_boundary.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3abc4a0..252fced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- Explicit `bourse_direct_adapter` catalog boundary with native login, fixed run + dates, C3 load/save/run-start validation, frozen trusted source and bounded + authenticated original reads. Existing `bourse_direct` catalogs keep their + current contract. Provider qualification and deployment remain separate. + - One Infra runtime contract for consume-folder delivery, exact vault lookup, schema promotion, DB/configuration handoffs and daemon-path qualification. Provider docs and harness boundaries now refer to it. No runtime, recipe-schema diff --git a/Dockerfile b/Dockerfile index c83936a..b0ebe77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,10 @@ COPY --chown=${APP_UID}:${APP_GID} apps/. . # non-root app user RUN mkdir -p /apps/Downloads "$HOME" && chown -R ${APP_UID}:${APP_GID} /apps "$HOME" +# Trusted catalog application code must not be group/world writable. +RUN find /apps/recipes_playwright -type f -name 'bourse_direct_adapter.py' \ + -exec chmod go-w {} + + # Keep the schema available when recipes are mounted as mutable configuration. COPY apps/recipes_playwright/recipe-pw-schema.yaml /opt/billcollector/recipe-pw-schema.yaml diff --git a/README.md b/README.md index 105a4bd..3ef4b90 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ The BillCollector configuration for each web service from which you want to retr - Each recipe is structured in numbered `steps`, executed in ascending `step` order. - Each step is a chain of `methods`: Playwright Page/Locator calls executed in order. Locator methods (`locator` for CSS selectors, `get_by_role`, `get_by_label`, `get_by_text`, `get_by_title`, `get_by_placeholder`, `get_by_test_id`) yield a locator that is chained into the following action (`click`, `fill`, `press`, `first`). Page-level methods include `goto` (open the start URL), `content_frame` (enter an iframe) and `close`. - Downloads: a step with the `expect_download` method wraps the nested `steps` that trigger the download; the file is saved into the Downloads folder. A re-downloaded file that is byte-identical to one already tracked for the same task (same sha256) is recorded as a duplicate and the consumer hook is not re-run for it. An opt-in `notice_identity` step option switches a download step from byte-hash deduplication to source-scoped identity plus a conservative PDF revision digest; see [notice identity](doc/NOTICE_IDENTITY.md). + - Catalog adapter: use the standalone daemon-only [`bourse_direct_adapter` action](doc/BOURSE_DIRECT_ADAPTER.md) after native login. The existing locator-prefixed `bourse_direct` catalog contract remains separate. - Bounded pagination: use one page locator followed by `click_until_absent`. Its single options object requires `progress_selector`, `max_clicks`, and `timeout_ms`. `timeout_ms` is the total browser-action budget. Every click must increase the visible progress count or make the pager absent or hidden. This action supports cumulative "load more" controls. It rejects page-replacement pagination and virtualized lists because their visible item count does not increase. For monthly replacement calendars, use the daemon's [replacement_calendar action](doc/REPLACEMENT_CALENDARS.md). - Bounded collections: use one page locator followed by `download_all`. Its single options object requires `key_attribute`, `max_items`, `timeout_ms`, and `event_window_ms`. `timeout_ms` bounds the browser actions and observations for the whole collection. Persistence (`save_as`, hashing, consumers, and SQLite) remains synchronous and cannot be preempted by that browser deadline. `event_window_ms` is the full attribution window observed after each click, so `timeout_ms` must exceed the actual item count multiplied by `event_window_ms`, with additional time for browser actions. The engine reports the required and remaining budgets before its first click when the minimum attribution-window budget cannot fit. The key must be non-empty and unique for every item. The ordered key set must remain stable while each control produces exactly one download. `download_all` also rejects a suggested filename already saved earlier in the run instead of overwriting it. - Incremental collections: `"incremental": true` in the `download_all` options records each item key (`=`) with its download. Later runs of the same task skip known keys without clicking them, so only the first run collects everything. Use it only with a key that is stable per document across sessions (for example an invoice number); a constant or session-bound key misses or repeats documents. `"unpack": "pdf"` publishes the single PDF member of a ZIP download unchanged under its member name and deduplicates on the PDF; any other archive content fails without publication. Both options require the daemon runner. diff --git a/apps/BillCollectorServices_pw.py b/apps/BillCollectorServices_pw.py index f1c0c53..d6d9a9a 100644 --- a/apps/BillCollectorServices_pw.py +++ b/apps/BillCollectorServices_pw.py @@ -285,7 +285,7 @@ def run_step(step, *, nested=False): if isinstance(entry, dict) } if nested and methods & { - "click_until_absent", "download_all", "replacement_calendar" + "click_until_absent", "download_all", "replacement_calendar", "bourse_direct_adapter" }: raise StepAbortError( "bounded collection actions cannot be nested " @@ -459,7 +459,7 @@ def transform_step_to_json(step): if graceful and any( isinstance(entry, dict) - and entry.get("method") in {"click_until_absent", "download_all", "replacement_calendar"} + and entry.get("method") in {"click_until_absent", "download_all", "replacement_calendar", "bourse_direct_adapter"} for entry in methods ): message = "bounded collection actions cannot be graceful" diff --git a/apps/billcollector/bourse_direct_boundary.py b/apps/billcollector/bourse_direct_boundary.py new file mode 100644 index 0000000..a73e6cf --- /dev/null +++ b/apps/billcollector/bourse_direct_boundary.py @@ -0,0 +1,281 @@ +"""One agreed catalog adapter, with native login/date/download ownership. + +The catalog module is trusted application code, not a sandboxed plugin. Only +its literal CHAINS declaration is read during preflight. No import, discovery, +registration, recipe-selected path or alternate credential engine is provided. +""" + +import ast +import copy +import inspect +import os +import stat +import time +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + +from . import config +from .collection import CollectionError +from .interrupt import RunInterrupted + +ACTION = 'bourse_direct_adapter' +IDENTITY = {'query_parameters': ['nc', 'type', 'year', 'month', 'day']} + + +def _document(chains): + return {'services': [{'serviceName': 'BourseDirect adapter', 'steps': [ + {'step': index, 'methods': chain} + for index, chain in enumerate(chains.values(), 1)]}]} + + +def _validate_chains(chains): + from .methods import _RESERVED, validate_method_recipe + + if not isinstance(chains, dict) or not chains or len(chains) > 64: + raise ValueError('adapter requires 1..64 named chains') + for name, chain in chains.items(): + if (not isinstance(name, str) or not name.isidentifier() + or not isinstance(chain, list) or not 1 <= len(chain) <= 16): + raise ValueError('adapter chain declaration is invalid') + for entry in chain: + if (not isinstance(entry, dict) or set(entry) - {'method', 'arguments'} + or not isinstance(entry.get('method'), str) + or entry['method'] in _RESERVED): + raise ValueError('adapter chains require public Playwright methods only') + validate_method_recipe(_document(chains)) + + +@dataclass(frozen=True) +class Candidate: + source: bytes + chains: dict + + +def candidate(): + """Freeze source without executing catalog code at load/save/run start.""" + try: + path = config.RECIPE_DIR / 'bourse_direct_adapter.py' + metadata = path.stat() + if metadata.st_uid != os.getuid() or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise ValueError('adapter catalog must be runtime-owned and not group/world-writable') + source = path.read_bytes() + if len(source) > 256 * 1024: + raise ValueError + tree = ast.parse(source) + declarations = [node for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == 'CHAINS' + for target in node.targets)] + if len(declarations) != 1: + raise ValueError + chains = ast.literal_eval(declarations[0].value) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == 'collect'] + if (len(functions) != 1 or functions[0].decorator_list + or [arg.arg for arg in functions[0].args.args] != ['api', 'options'] + or functions[0].args.defaults or functions[0].args.posonlyargs + or functions[0].args.kwonlyargs or functions[0].args.vararg + or functions[0].args.kwarg): + raise ValueError + except ValueError as error: + if str(error).startswith('adapter catalog must be'): + raise + raise ValueError('fixed catalog adapter source or literal CHAINS is invalid') from None + except (OSError, SyntaxError, TypeError, RecursionError): + raise ValueError('fixed catalog adapter source or literal CHAINS is invalid') from None + _validate_chains(chains) + # These are also installed Playwright calls. Check them before resources. + _validate_chains({'read': [ + {'method': 'context'}, {'method': 'request'}, + {'method': 'get', 'arguments': [{'url': 'https://example.invalid', + 'max_redirects': 0, 'timeout': 1}]}, + {'method': 'body'}]}) + return Candidate(source, chains) + + +def validate_options(options): + if (not isinstance(options, dict) + or set(options) != {'account_type', 'max_months', 'max_items', 'timeout_ms'} + or options['account_type'] not in ('PEA', 'PEA-PME')): + raise ValueError('adapter requires one supported account type and fixed bounds') + for name, maximum in [('max_months', 120), ('max_items', 200), ('timeout_ms', 7_200_000)]: + if type(options[name]) is not int or not 1 <= options[name] <= maximum: + raise ValueError('adapter bound is invalid') + + +def prepare(recipe): + """One standalone collection after native final-session proof, one service.""" + found = [] + + def visit(step, top=False): + if any(entry.get('method') == 'bourse_direct' for entry in step.get('methods', [])): + legacy.append(step) + if any(entry.get('method') == ACTION for entry in step.get('methods', [])): + if (not top or len(step['methods']) != 1 or step.get('graceful', False) + or any(key in step for key in ('steps', 'submit', 'phases')) + or step.get('notice_identity') != IDENTITY): + raise ValueError('adapter requires one standalone notice collection step') + arguments = step['methods'][0].get('arguments', []) + if len(arguments) != 1: + raise ValueError('adapter requires one options object') + validate_options(arguments[0]) + found.append(step) + for child in step.get('steps', []): + visit(child) + if 'submit' in step: + visit(step['submit']) + for phase in step.get('phases', []): + visit(phase) + + legacy = [] + services = recipe.get('services', []) + for service in services: + for step in service.get('steps', []): + visit(step, True) + if not found: + return None + if legacy: + raise ValueError('catalog adapter cannot share a task with the legacy source action') + if len(found) != 1 or len(services) != 1: + raise ValueError('adapter requires one service and one collection per task') + from .login import is_login + + ordered = sorted(services[0]['steps'], key=lambda step: step['step']) + before = ordered[:ordered.index(found[0])] + if sum(is_login(step) for step in ordered) != 1 or not any(is_login(step) for step in before): + raise ValueError('adapter requires a preceding native login with final-session proof') + return candidate() + + +class Original: + def __init__(self, body, url, filename): + self.body, self.url, self.suggested_filename = body, url, filename + + def save_as(self, path): + Path(path).write_bytes(self.body) + + +def _data(value): + """Return data only; do not hand the adapter an unchecked browser object.""" + if value is None or type(value) in (str, int, float, bool, bytes): + return value + if isinstance(value, (list, tuple)): + return [_data(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _data(item) for key, item in value.items()} + raise CollectionError('adapter chain must return data, not a browser object') + + +class Acquisition: + """Capabilities for the fixed adapter. No vault, DB or login callback.""" + + def __init__(self, ctx, spec, options, save_download): + from .calendar import _months + + validate_options(options) + self.dates = ctx.bcs.date_context + self.months = tuple(_months(self.dates, options['max_months'])) + self._page, self._context = ctx.page, ctx.context + self._chains, self._save = copy.deepcopy(spec.chains), save_download + self._deadline = time.monotonic() + options['timeout_ms'] / 1000 + self._maximum, self._offered, self._saved, self._reads = options['max_items'], 0, 0, 0 + self._origin = self._origin_of(ctx.page.url.split('#', 1)[0]) + + @staticmethod + def _origin_of(url): + try: + parts = urlsplit(url) + if (parts.scheme not in ('http', 'https') or not parts.hostname + or parts.username is not None or parts.password is not None + or parts.fragment): + raise ValueError + return parts.scheme, parts.hostname, parts.port or (443 if parts.scheme == 'https' else 80) + except (ValueError, TypeError): + raise CollectionError('adapter source origin is invalid') from None + + def remaining_ms(self): + remaining = int((self._deadline - time.monotonic()) * 1000) + if remaining < 1: + raise CollectionError('adapter total deadline expired') + return remaining + + def offered(self, count): + """Count every offered item, including repeats, before reading a month.""" + self.remaining_ms() + if type(count) is not int or count < 0 or self._offered + count > self._maximum: + raise CollectionError('adapter offered item limit exceeded') + self._offered += count + + def call(self, name, **arguments): + """Use one declared C3 chain; dynamic kwargs replace its last arguments.""" + if name not in self._chains: + raise CollectionError('adapter chain is not declared') + chain = copy.deepcopy(self._chains[name]) + if arguments: + chain[-1]['arguments'] = [arguments] + _validate_chains({name: chain}) + previous, result = self._page, None + for entry in chain: + remaining = self.remaining_ms() + method = getattr(previous, entry['method']) + args = entry.get('arguments', []) + positional = ['Enter'] if args == ['Enter'] else [] + kwargs = {key: value for item in ([] if positional else args) + for key, value in item.items()} + # A zero timeout means unbounded in Playwright. Replace it here. + if callable(method) and 'timeout' in inspect.signature(method).parameters: + kwargs['timeout'] = min(kwargs.get('timeout') or remaining, remaining) + result = method(*positional, **kwargs) if callable(method) else method + previous = result if result is not None else previous + self.remaining_ms() + return _data(result) + + def read(self, url, *, pdf=False): + """Read unchanged bytes through the current authenticated request context.""" + if self._origin_of(url) != self._origin or len(url) > 16384: + raise CollectionError('adapter request origin changed') + if self._reads >= self._maximum * 2: + raise CollectionError('adapter request limit exceeded') + self._reads += 1 + response = self._context.request.get( + url, max_redirects=0, timeout=min(30_000, self.remaining_ms())) + try: + if response.status != 200 or response.url != url: + raise CollectionError('adapter response is not proven') + body = response.body() + self.remaining_ms() + if (not body or len(body) > 20 * 1024 * 1024 + or (pdf and not body.startswith(b'%PDF-'))): + raise CollectionError('adapter original response is invalid') + return body + finally: + response.dispose() + + def save_original(self, url, filename): + if self._saved >= self._offered: + raise CollectionError('adapter original exceeds offered item count') + if (not isinstance(filename, str) or not filename.endswith('.pdf') + or Path(filename).name != filename or '\\' in filename): + raise CollectionError('adapter original filename is invalid') + body = self.read(url, pdf=True) + self._saved += 1 + self._save(Original(body, url, filename)) + self.remaining_ms() + + +def collect(ctx, spec, options, save_download): + """Execute the frozen trusted catalog candidate once; redact its failures.""" + try: + if spec is None: + raise CollectionError('adapter was not prepared before resources') + api = Acquisition(ctx, spec, options, save_download) + namespace = {'__name__': 'billcollector_bourse_direct_adapter'} + # Fixed operator-installed application code; never a YAML-selected path. + code = compile(spec.source, '', 'exec') + exec(code, namespace) # nosec B102 + namespace['collect'](api, copy.deepcopy(options)) + api.remaining_ms() + except RunInterrupted: + raise + except Exception: + raise CollectionError('adapter acquisition failed; inspect before resuming') from None diff --git a/apps/billcollector/collection.py b/apps/billcollector/collection.py index 2b2333b..1b4ae74 100644 --- a/apps/billcollector/collection.py +++ b/apps/billcollector/collection.py @@ -24,6 +24,7 @@ }) COLLECTION_ACTION_METHODS = frozenset({ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct", + "bourse_direct_adapter", }) logger = logging.getLogger(__name__) @@ -53,6 +54,10 @@ class PaginationResult: def validate_terminal_action(methods, index, name): """Require exactly one page locator followed by one collection action.""" + if name == 'bourse_direct_adapter': + if len(methods) != 1 or index != 0: + raise CollectionError('bourse_direct_adapter requires one standalone method') + return if len(methods) != 2 or index != 1: raise CollectionError( f"{name} requires exactly one locator method followed by the action" diff --git a/apps/billcollector/methods.py b/apps/billcollector/methods.py index 3116a37..d789bb5 100644 --- a/apps/billcollector/methods.py +++ b/apps/billcollector/methods.py @@ -16,7 +16,7 @@ from .collection import CollectionError _RESERVED = frozenset({'login', 'wait_user', 'expect_download', - 'click_until_absent', 'download_all', 'replacement_calendar', 'bourse_direct'}) + 'click_until_absent', 'download_all', 'replacement_calendar', 'bourse_direct', 'bourse_direct_adapter'}) _UNTIL = (('role', 'get_by_role'), ('test_id', 'get_by_test_id'), ('placeholder', 'get_by_placeholder'), ('label', 'get_by_label'), ('text', 'get_by_text'), ('selector', 'locator')) @@ -148,6 +148,39 @@ def locator_arguments(until): def validate_method_recipe(recipe): """Shared UI/run-start gate; diagnostics never contain argument values.""" + from .bourse_direct_boundary import prepare + + try: + prepare(recipe) + except ValueError as error: + def locate(service, step): + if any(entry.get('method') == 'bourse_direct_adapter' + for entry in step.get('methods', [])): + return service, step + for child in step.get('steps', []): + found = locate(service, child) + if found: + return found + if step.get('submit'): + found = locate(service, step['submit']) + if found: + return found + for phase in step.get('phases', []): + found = locate(service, phase) + if found: + return found + return None + + for service in recipe.get('services', []): + for step in service.get('steps', []): + found = locate(service, step) + if found: + owner, action = found + raise ValueError( + f"service {owner.get('serviceName')}, step {action.get('step')}, " + f"method bourse_direct_adapter: {error}") from None + raise + def visit(step, service): owners = {Page} for entry in step.get('methods', []): diff --git a/apps/billcollector/notice.py b/apps/billcollector/notice.py index d0c2180..d6bb485 100644 --- a/apps/billcollector/notice.py +++ b/apps/billcollector/notice.py @@ -218,7 +218,7 @@ def visit(step): if "notice_identity" in step: validate_identity(step["notice_identity"]) names = {entry.get("method") for entry in step.get("methods", [])} - if not names & {"expect_download", "download_all", "replacement_calendar", "bourse_direct"}: + if not names & {"expect_download", "download_all", "replacement_calendar", "bourse_direct", "bourse_direct_adapter"}: raise ValueError("notice_identity requires a download step") for nested in step.get("steps", []): visit(nested) diff --git a/apps/billcollector/runner.py b/apps/billcollector/runner.py index bfed3b9..3ab530e 100644 --- a/apps/billcollector/runner.py +++ b/apps/billcollector/runner.py @@ -78,6 +78,7 @@ # operator-driven resume starts a new, explicit invocation after inspection. _COLLECTION_METHODS = frozenset({ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct", + "bourse_direct_adapter", }) @@ -355,7 +356,7 @@ def transform_step_to_json(step): if method_name in [ "click", "fill", "expect_download", "goto", "close", "content_frame", "first", "click_until_absent", "download_all", - "replacement_calendar", "bourse_direct", + "replacement_calendar", "bourse_direct", "bourse_direct_adapter", ]: transformed_step["actions"] = transformed_step.get("actions", []) transformed_step["actions"].append({ @@ -447,7 +448,7 @@ def fail_step(error_entry): methods, method_index, method_name ) processed_args = [process_argument(arg, bcs) for arg in arguments] - if previous_result is bcs.page: + if previous_result is bcs.page and method_name != 'bourse_direct_adapter': raise ValueError( f"{method_name} requires a locator prefix" ) @@ -533,6 +534,13 @@ def bourse_direct(root, options): lambda download: save_download(download, delivery_policy="bourse-direct"), lambda token: process_argument(token, ctx.bcs)) + def bourse_direct_adapter(_root, options): + from .bourse_direct_boundary import collect + + if not getattr(ctx, 'native_login_proven', False): + raise collection.CollectionError('adapter requires native final-session proof') + return collect(ctx, getattr(ctx, 'bourse_direct_candidate', None), options, save_download) + def save_download(download, *, source_key=None, unpack=None, delivery_policy=None): filename = str(download.suggested_filename) # An unpacked archive name is a transport name, not document identity. @@ -594,6 +602,7 @@ def replacement_calendar(root, options): "download_all": download, "replacement_calendar": replacement_calendar, "bourse_direct": bourse_direct, + "bourse_direct_adapter": bourse_direct_adapter, } @@ -837,6 +846,7 @@ def proven(timeout): return False if not ctx.db.login_succeeded(ctx.task["id"], ctx.run_id): raise StepAbortError("Another run owns the unresolved login attempt") + ctx.native_login_proven = True return True if proven(options["session_timeout_ms"]): @@ -1096,6 +1106,13 @@ def _run_task(task_id: int, debug: bool = False, *, date_from=None, collection_m print(f"runner: invalid variable contract: {e}", file=sys.stderr) return 1 + try: + from .bourse_direct_boundary import prepare + adapter_candidate = prepare(recipe) + except ValueError as e: + print(f"runner: invalid method contract: {e}", file=sys.stderr) + return 1 + # C6: the runner owns the per-run display: claim a display from the # configured range before the Run row exists (the row records it), # start the Xvfb/x11vnc/websockify stack, and point DISPLAY at it @@ -1135,6 +1152,7 @@ def _run_task(task_id: int, debug: bool = False, *, date_from=None, collection_m credentials = _vault_preflight(task, db) ctx = _RunContext(task, run_id, db, debug, credentials, api_url) ctx.bcs.date_context = dates + ctx.bourse_direct_candidate = adapter_candidate files, aborted, pause_outcome = _run_recipe(ctx, recipe) result = run_result(aborted, files, pause_outcome, known_source_count=ctx.incremental_known_count) diff --git a/apps/recipes_playwright/recipe-pw-schema.yaml b/apps/recipes_playwright/recipe-pw-schema.yaml index 4c04e63..3cf77c9 100644 --- a/apps/recipes_playwright/recipe-pw-schema.yaml +++ b/apps/recipes_playwright/recipe-pw-schema.yaml @@ -105,6 +105,22 @@ definitions: additionalItems: false not: required: [ steps ] + - if: + required: [ methods ] + properties: + methods: + contains: + type: object + properties: + method: + const: bourse_direct_adapter + required: [ method ] + then: + properties: + graceful: + const: false + not: + required: [ steps ] nestedStep: type: object required: [ step ] @@ -136,7 +152,7 @@ definitions: required: [ method ] properties: method: - enum: [ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct" ] + enum: [ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct", "bourse_direct_adapter" ] method: oneOf: - $ref: "#/definitions/playwrightMethod" @@ -144,6 +160,7 @@ definitions: - $ref: "#/definitions/downloadAllMethod" - $ref: "#/definitions/replacementCalendarMethod" - $ref: "#/definitions/bourseDirectMethod" + - $ref: "#/definitions/bourseDirectAdapterMethod" collectionLocatorMethod: type: object required: [ method ] @@ -178,7 +195,7 @@ definitions: method: type: string not: - enum: [ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct" ] + enum: [ "click_until_absent", "download_all", "replacement_calendar", "bourse_direct", "bourse_direct_adapter" ] arguments: oneOf: - type: array @@ -341,6 +358,36 @@ definitions: type: string minLength: 1 + bourseDirectAdapterMethod: + type: object + required: [method, arguments] + additionalProperties: false + properties: + method: + const: bourse_direct_adapter + arguments: + type: array + minItems: 1 + maxItems: 1 + items: + type: object + required: [account_type, max_months, max_items, timeout_ms] + additionalProperties: false + properties: + account_type: + enum: [PEA, PEA-PME] + max_months: + type: integer + minimum: 1 + maximum: 120 + max_items: + type: integer + minimum: 1 + maximum: 200 + timeout_ms: + type: integer + minimum: 1 + maximum: 7200000 bourseDirectMethod: type: object required: [method, arguments] diff --git a/doc/BOURSE_DIRECT_ADAPTER.md b/doc/BOURSE_DIRECT_ADAPTER.md new file mode 100644 index 0000000..b365f51 --- /dev/null +++ b/doc/BOURSE_DIRECT_ADAPTER.md @@ -0,0 +1,157 @@ +# Bourse Direct engine boundary + +This boundary implements the [issue #51 agreement](https://github.com/s-t-e-f-a-n/BillCollector/issues/51#issuecomment-6560326064). +It does not contain or qualify a provider collector. The recipes project owns +the portal selectors, HTML parser and source identity proofs. + +## Fork compatibility + +This fork names the new action `bourse_direct_adapter`. The public-baseline +candidate in [PR #89](https://github.com/flowcool/BillCollector/pull/89) names +it `bourse_direct`. This explicit difference preserves the installed catalog +v1.0.0: its locator-prefixed `bourse_direct` action still uses the existing fork +implementation, login, date modes and delivery policy. There is no automatic +fallback between contracts. A task cannot combine the two source actions. + +The new catalog action requires native `login`. It uses the task's fixed date +window and ordinary notice tracking. It has no invocation mode override or +strict delivery policy. Full and incremental recipes must reuse the same task +and history. A provider change or production switch needs separate recipes +qualification and Infra adoption; this engine integration changes neither. + +## Install the candidate + +Install the reviewed catalog Python file as +`apps/recipes_playwright/bourse_direct_adapter.py`, next to the recipes. +`config.RECIPE_DIR` is the fixed installation directory. YAML cannot select a +module, path, entry point or another provider. There is no plugin discovery or +registration API. This is trusted application code. Review it with its recipe; +the capabilities below are an integration interface, not a Python sandbox. +Do not install untrusted Python through the recipe editor. +The file must be owned by the runtime user and must not be group- or +world-writable. A normal `0644` file passes this check. The image build removes +group/world write bits from the installed adapter file. + +The file declares a literal `CHAINS` dictionary and `collect(api, options)`: + +```python +CHAINS = { + 'calendar_html': [ + {'method': 'frame_locator', 'arguments': [{'selector': 'iframe#calendar'}]}, + {'method': 'locator', 'arguments': [{'selector': 'body'}]}, + {'method': 'inner_html'}, + ], + 'select_account': [ + {'method': 'frame_locator', 'arguments': [{'selector': 'iframe#calendar'}]}, + {'method': 'locator', 'arguments': [{'selector': 'select#nc'}]}, + {'method': 'select_option', 'arguments': [{'value': 'synthetic'}]}, + ], +} + +def collect(api, options): + # Implement the provider checks described below in the recipes project. + raise NotImplementedError('A qualified provider adapter is required') +``` + +These selectors are examples, not portal evidence. The Python module may use +standard-library parsers. Keep Playwright access in its declared chains and the +engine's authenticated read methods. Do not import another browser, collector, +vault client or variable engine. Do not hide collection in injected JavaScript. + +Load, save and run start parse the literal declaration without importing or +executing the module. Every chain passes the installed Playwright C3 signature +and argument gate. Reserved engine actions are prohibited inside adapter chains. +The engine also validates its authenticated-request chain. Run start freezes the +source and manifest before display, vault or browser access. A concurrent catalog +change cannot replace the code of an admitted run. + +## Recipe + +Use one service and one task for one account. Before collection, navigate through +ordinary C3 methods and execute one native `login` block. Follow +[SAFE_LOGIN.md](SAFE_LOGIN.md) for stored sessions, password submission and optional +fresh TOTP phases. Its final success locator must prove the authenticated session. +The adapter receives no credential resolver, vault client, task database or login +callback. The runner admits collection only after the native login proof succeeds. + +Add this standalone top-level step after that login block: + +```yaml +- step: 3 + notice_identity: + query_parameters: [nc, type, year, month, day] + methods: + - method: bourse_direct_adapter + arguments: + - account_type: PEA + max_months: 120 + max_items: 200 + timeout_ms: 7200000 +``` + +The exact options are required. `account_type` is `PEA` or `PEA-PME`. +Bounds are positive integers: at most 120 months, 200 offered items and +7,200,000 milliseconds. No locator prefix, graceful mode, nesting or second +collection is allowed. To migrate the historical locator -> `bourse_direct` +recipe, rewrite it for this explicit catalog action and native login. Keeping +the historical recipe selects the existing fork implementation. No invocation +options are introduced for the catalog action. + +## Acquisition interface + +| Member | Contract | +| --- | --- | +| `api.dates` | Native immutable `DateContext`: `date_from` and `today`, fixed at run start. | +| `api.months` | Descending `(year, month)` tuples for every complete month intersecting the inclusive date window. Rejects an insufficient month budget before adapter execution. | +| `api.call(name, **arguments)` | Execute a declared Page-rooted chain. Optional keyword arguments replace the final method's arguments. Validate the resulting chain with C3 before dispatch. No variable expansion occurs here. Return data only, never a raw browser object. | +| `api.remaining_ms()` | Remaining total acquisition deadline. Raises after expiry. All engine calls check it. Playwright calls with a timeout parameter receive a positive timeout capped to this deadline. | +| `api.offered(count)` | Count all offered items before reading that month, including repeats. Reject a total above `max_items`. An empty month counts zero and still requires source proof. | +| `api.read(url)` | Read unchanged notice HTML bytes through the current authenticated context. Require the entry page's origin, no redirects, exact response URL, HTTP 200 and 1 byte..20 MiB. Dispose each response. Total reads, including originals, are at most `2 * max_items`. | +| `api.save_original(url, filename)` | Read the offered PDF through the same transport. Require `%PDF-` and a plain `.pdf` basename. Pass unchanged bytes and source URL to native `track_download`, with the step's notice identity. Save at most the counted offered items. | + +The module must not retain a browser object or implement blocking work outside +these calls. The time budget is enforced at capability boundaries, not by a +Python process sandbox. Calls that return API objects must end in a data read. +Use `read` and `save_original` for source documents so their transport and bounds +apply. Do not fetch originals through a browser chain or a separate HTTP client. + +Before each month, prove the offered account and month. Keep PEA and PEA-PME +distinct. Follow offered controls. Prove an explicit empty listing. Reject +ambiguous identity or source drift. Count every offered item. Follow notice HTML +to its unique offered `Format PDF` URL and verify account, type and date before +`save_original`. Read every original before notice-revision comparison. Do not +use a daily source key to skip corrections. + +The step runs once automatically. A failure enters the existing runner pause; +an operator resume is an explicit new attempt after inspection. Errors contain +no provider values. No new pending state or receipts are added. Native Run +results remain unchanged, including failure for a completed empty run. Native +at-least-once delivery and `task_id:sha256` keys remain unchanged. Full and +incremental actions use the same task, recipe and history. + +## Isolated engine proof + +Install `apps/requirements.txt` and pytest in an isolated environment. From the +checkout root, use the browser cache that matches Playwright 1.64.0: + +```sh +PLAYWRIGHT_BROWSERS_PATH=/path/to/browser-cache \ +BC_BROWSER_DIR=/path/to/browser-cache \ + -m pytest tests/test_bc_bourse_direct_boundary.py \ + tests/test_bc_method_validation.py tests/test_notice_identity.py \ + tests/test_bc_login.py tests/test_bc_calendar.py tests/test_bc_collection.py \ + tests/test_bc_runner.py -q +``` + +The boundary suite installs a synthetic catalog file in a temporary directory. +It proves all C3 gates, rejection before resources, no preflight execution, +frozen source, the actual adapter dispatch, native session proof, authenticated +original reads, unchanged stored bytes, regenerated repeats, material corrections, +source drift, empty proof, bounds and single automatic execution. Existing suites +cover native password/TOTP, dates, collection and notice compatibility. +No provider login, vault, production or private original is used. The recipes +project must still qualify its actual provider file and YAML on this exact +engine commit. The retained negative probe remains a historical-baseline proof. + +Recovery: revert this focused engine commit and remove this action from any +unpublished recipe candidate. No database migration or live-state repair is needed. diff --git a/doc/RECIPE_METHODS.md b/doc/RECIPE_METHODS.md index f17fe6f..bc25870 100644 --- a/doc/RECIPE_METHODS.md +++ b/doc/RECIPE_METHODS.md @@ -243,6 +243,17 @@ publish its exact bytes, retaining any UTF-8 BOM. Unsafe paths/special entries, multiple files and size/count violations fail closed. Both options are rejected by the legacy batch adapter. See [bounds and storage](../README.md#incremental-document-collection). +### Fixed catalog adapter (fork) + +`bourse_direct_adapter` is a standalone top-level daemon action after one native +`login`. Its options are exactly `account_type`, `max_months`, `max_items` and +`timeout_ms`. It requires the notice identity policy and a reviewed fixed catalog +file with literal C3-validated `CHAINS`. It cannot be graceful, nested, preceded +by a locator, or combined with the legacy `bourse_direct` source action. +See [BOURSE_DIRECT_ADAPTER.md](BOURSE_DIRECT_ADAPTER.md) for the supported +installation, limits and the naming difference from public-baseline PR #89. +The existing `bourse_direct` recipe and delivery contracts are unchanged. + ## 4. Variables The daemon expands original string argument values recursively in dictionaries @@ -287,8 +298,8 @@ browser access. Already enforced by the existing gates (keep): the schema rejects unknown top-level keys, missing `serviceName`/`steps`/`step`, non- integer step numbers, wrong argument value types, the collection -action chain shape (exactly `[page-locator, action]`, `graceful: -false`, no nested `steps`), the collection action argument names, +action chain shape (exactly `[page-locator, action]`, or a standalone +`bourse_direct_adapter`, `graceful: false`, no nested `steps`), the collection action argument names, types and ranges (section 3 tables), and the `phases` block shape; `validate_login_recipe` rejects a non-standalone, nested or multi-per-recipe `login`, `submit`/`phases` outside a login block, the diff --git a/tests/qualify_bourse_direct_image.py b/tests/qualify_bourse_direct_image.py index a67302e..f336162 100644 --- a/tests/qualify_bourse_direct_image.py +++ b/tests/qualify_bourse_direct_image.py @@ -31,7 +31,8 @@ shutil.copytree('/checks/tests', root/'tests') shutil.copytree('/apps/recipes_playwright', root/'apps/recipes_playwright') hashes = {name: hashlib.sha256((Path('/apps')/name).read_bytes()).hexdigest() - for name in ('billcollector/bourse_direct.py', 'billcollector/throttle.py', 'billcollector/notice.py', + for name in ('billcollector/bourse_direct.py', 'billcollector/bourse_direct_boundary.py', + 'billcollector/throttle.py', 'billcollector/notice.py', 'billcollector/downloads.py', 'billcollector/runner.py', 'recipes_playwright/recipe-pw-schema.yaml')} print(json.dumps({'packaged_source_sha256':hashes, 'uid':os.getuid(), @@ -41,6 +42,7 @@ sys.path.insert(0, '/test-deps') import pytest code = pytest.main([str(root/'tests/test_bourse_direct.py'), + str(root/'tests/test_bc_bourse_direct_boundary.py'), str(root/'tests/test_notice_identity.py'), str(root/'tests/test_throttle.py'), '-q', '--basetemp', str(root/'pytest-proof')]) @@ -104,7 +106,8 @@ def main(): state = json.loads(subprocess.run(['docker', 'inspect', container], check=True, capture_output=True, text=True).stdout)[0] hashes = {name: hashlib.sha256((engine/'apps'/name).read_bytes()).hexdigest() - for name in ('billcollector/bourse_direct.py', 'billcollector/throttle.py', 'billcollector/notice.py', + for name in ('billcollector/bourse_direct.py', 'billcollector/bourse_direct_boundary.py', + 'billcollector/throttle.py', 'billcollector/notice.py', 'billcollector/downloads.py', 'billcollector/runner.py', 'recipes_playwright/recipe-pw-schema.yaml')} log_text = (root/'container.log').read_text() diff --git a/tests/test_bc_bourse_direct_boundary.py b/tests/test_bc_bourse_direct_boundary.py new file mode 100644 index 0000000..46acad9 --- /dev/null +++ b/tests/test_bc_bourse_direct_boundary.py @@ -0,0 +1,399 @@ +"""Offline proof of the engine boundary, not a qualified provider adapter.""" + +import copy +import io +import json +import threading +from datetime import date +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from unittest.mock import Mock + +import pytest +import yaml +from playwright.sync_api import sync_playwright +from pypdf import PdfWriter + +from billcollector import bourse_direct_boundary as boundary +from billcollector import config, runner +from billcollector.db import Database +from billcollector.recipes import load_recipe, save_recipe, validate_recipe +from billcollector.variables import DateContext + +OPTIONS = {'account_type': 'PEA', 'max_months': 3, 'max_items': 4, 'timeout_ms': 30000} +CHAINS = {'listing': [ + {'method': 'locator', 'arguments': [{'selector': '#listing'}]}, + {'method': 'inner_text'}]} +SOURCE = '''import json +CHAINS = %r +def collect(api, options): + listing = json.loads(api.call('listing')) + if listing['account_type'] != options['account_type']: + raise ValueError('private-account-drift') + if tuple(listing['month']) != api.months[0]: + raise ValueError('private-month-drift') + api.offered(len(listing['items'])) + if not listing['items'] and not listing['empty']: + raise ValueError('private-missing-empty-proof') + for item in listing['items']: + notice = json.loads(api.read(item['notice'])) + if len(notice['pdfs']) != 1: + raise ValueError('private-ambiguous-original') + api.save_original(notice['pdfs'][0], item['filename']) +''' % CHAINS + + +def recipe(url='http://127.0.0.1:1'): + return {'services': [{'serviceName': 'synthetic', 'steps': [ + {'step': 1, 'methods': [{'method': 'goto', 'arguments': [{'url': url}]}]}, + {'step': 2, 'methods': [{'method': 'login', 'arguments': [ + {'success': {'selector': '#session'}, 'session_timeout_ms': 1000}]}], + 'submit': {'step': 21, 'methods': [ + {'method': 'locator', 'arguments': [{'selector': '#submit'}]}, + {'method': 'click'}]}}, + {'step': 3, 'notice_identity': copy.deepcopy(boundary.IDENTITY), + 'methods': [{'method': boundary.ACTION, 'arguments': [copy.deepcopy(OPTIONS)]}]}]}]} + + +@pytest.fixture +def catalog(tmp_path, monkeypatch): + monkeypatch.setattr(config, 'RECIPE_DIR', tmp_path) + source = tmp_path / 'bourse_direct_adapter.py' + source.write_text(SOURCE) + source.chmod(0o644) + return source + + +def test_load_save_and_run_start_do_not_execute_adapter(catalog, tmp_path, monkeypatch): + marker = tmp_path / 'should-not-exist' + catalog.write_text(SOURCE + f'\nopen({str(marker)!r}, "w").write("executed")\n') + path = tmp_path / 'recipe.yaml' + text = yaml.safe_dump(recipe()) + assert validate_recipe(text)[0] + assert save_recipe(path, text) == (True, None) + assert load_recipe(path)[0] + monkeypatch.setattr(config, 'DB_FILE', tmp_path / 'isolated.db') + db = Database() + task = db.create_task('synthetic', 'test', recipe_file=str(path)) + db.close() + resource = Mock(side_effect=runner.display.DisplayError('synthetic stop')) + monkeypatch.setattr(runner.display, 'allocate_display', resource) + denied = Mock(side_effect=AssertionError('must not reach vault/browser')) + monkeypatch.setattr(runner, '_vault_preflight', denied) + monkeypatch.setattr(runner, 'sync_playwright', denied) + assert runner.run_task(task) == 1 + resource.assert_called_once() + denied.assert_not_called() + assert not marker.exists() + + +def test_catalog_rejects_group_writable_source(catalog): + catalog.chmod(0o664) + with pytest.raises(ValueError, match='runtime-owned'): + boundary.candidate() + + +@pytest.mark.parametrize('corrupt', ['method', 'arguments', 'literal', 'missing']) +def test_bad_adapter_rejected_at_all_gates(catalog, tmp_path, monkeypatch, corrupt): + path = tmp_path / 'recipe.yaml' + text = yaml.safe_dump(recipe()) + assert save_recipe(path, text)[0] + if corrupt == 'missing': + catalog.rename(catalog.with_suffix('.retained')) + elif corrupt == 'literal': + catalog.write_text(SOURCE.replace(repr(CHAINS), 'dict()')) + else: + chains = copy.deepcopy(CHAINS) + if corrupt == 'method': + chains['listing'][-1]['method'] = 'private_unknown' + else: + chains['listing'][-1]['arguments'] = [{'private_unknown': 'secret-value'}] + catalog.write_text(SOURCE.replace(repr(CHAINS), repr(chains))) + ok, error, _ = validate_recipe(text) + assert not ok and 'secret-value' not in error + assert 'service synthetic, step 3, method bourse_direct' in error + assert not load_recipe(path)[0] + assert not save_recipe(path, text + '\n# rejected')[0] + assert path.read_text() == text + monkeypatch.setattr(config, 'DB_FILE', tmp_path / 'isolated.db') + db = Database() + task = db.create_task('synthetic', 'test', recipe_file=str(path)) + db.close() + denied = Mock(side_effect=AssertionError('must not acquire resources')) + monkeypatch.setattr(runner.display, 'allocate_display', denied) + monkeypatch.setattr(runner, '_vault_preflight', denied) + monkeypatch.setattr(runner, 'sync_playwright', denied) + assert runner.run_task(task) == 1 + denied.assert_not_called() + + +@pytest.mark.parametrize('change', ['prefix', 'graceful', 'nested', 'login', 'identity', + 'duplicate', 'services', 'bounds', 'override']) +def test_adapter_scope_rejected(catalog, change): + data = recipe() + steps = data['services'][0]['steps'] + step = steps[-1] + if change == 'prefix': + step['methods'].insert(0, {'method': 'locator', 'arguments': [{'selector': 'body'}]}) + elif change == 'graceful': + step['graceful'] = True + elif change == 'nested': + steps[-1] = {'step': 4, 'methods': [{'method': 'expect_download'}], 'steps': [step]} + elif change == 'login': + steps.pop(1) + elif change == 'identity': + step.pop('notice_identity') + elif change == 'duplicate': + steps.append(copy.deepcopy(step)) + elif change == 'services': + data['services'].append(copy.deepcopy(data['services'][0])) + elif change == 'bounds': + step['methods'][0]['arguments'][0]['max_items'] = True + else: + step['methods'][0]['arguments'][0]['date_from'] = '2020-01-01' + assert not validate_recipe(yaml.safe_dump(data))[0] + + +def pdf(metadata='one', width=100): + writer = PdfWriter() + writer.add_blank_page(width, 100) + writer.add_metadata({'/CreationDate': metadata}) + stream = io.BytesIO() + writer.write(stream) + return stream.getvalue() + + +@pytest.fixture +def portal(): + state = {'body': pdf(), 'requests': [], 'account_type': 'PEA', 'month': [2026, 1], + 'empty': True, 'items': 2, 'pdfs': 1, 'status': 200} + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_GET(self): + state['requests'].append((self.path, self.headers.get('Cookie'))) + base = f'http://127.0.0.1:{self.server.server_port}' + query = '?nc=123&type=RO&year=2026&month=1&day=' + if self.path.startswith('/original'): + body = state['body'] + elif self.path.startswith('/notice'): + target = base + self.path.replace('/notice', '/original') + '&pdf=1' + body = json.dumps({'pdfs': [target] * state['pdfs']}).encode() + else: + listing = {key: state[key] for key in ('account_type', 'month', 'empty')} + listing['items'] = [ + {'notice': base + '/notice' + query + str(day), + 'filename': f'original-{day}.pdf'} + for day in range(1, state['items'] + 1)] + body = ('
session
'
+                        + json.dumps(listing) + '
').encode() + self.send_response(state['status']) + self.send_header('Set-Cookie', 'fixture=session; Path=/') + self.end_headers() + self.wfile.write(body) + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f'http://127.0.0.1:{server.server_port}', state + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.fixture +def browser_ctx(catalog, portal, tmp_path, monkeypatch): + url, state = portal + monkeypatch.setattr(config, 'DB_FILE', tmp_path / 'isolated.db') + monkeypatch.setattr(config, 'DOWNLOAD_DIR', tmp_path / 'downloads') + monkeypatch.setattr(config, 'RUN_DIR', tmp_path / 'runs') + db = Database() + db.set_setting('download_dir', str(tmp_path / 'downloads')) + task_id = db.create_task('synthetic', 'test') + task = db.get_task(task_id) + run_id = db.create_run(task_id) + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + context = browser.new_context() + page = context.new_page() + page.goto(url) + ctx = runner._RunContext(task, run_id, db, False, + {'user': 'synthetic', 'password': 'synthetic', 'otp': None}, '') + ctx.page, ctx.context, ctx.bcs.page = page, context, page + ctx.bcs.date_context = DateContext(date(2026, 1, 15), date(2025, 11, 15)) + ctx.bourse_direct_candidate = boundary.prepare(recipe(url)) + monkeypatch.setattr(runner, 'pause_loop', Mock(side_effect=AssertionError('unexpected pause'))) + # Exercise the actual native stored-session proof before collection. + runner._execute_login(ctx, recipe(url)['services'][0]['steps'][1]) + monkeypatch.setattr(runner, '_record_step', Mock()) + monkeypatch.setattr(runner.PageState, 'set_interactive_elements', lambda *_args: []) + try: + yield ctx, state + finally: + context.close() + browser.close() + db.close() + + +def test_actual_adapter_path_original_bytes_and_notice_revisions(browser_ctx, catalog): + ctx, state = browser_ctx + step = recipe()['services'][0]['steps'][-1] + assert ctx.native_login_proven + # Execution uses frozen source, even if a catalog update races with the run. + catalog.write_text('raise AssertionError("must not execute changed source")') + runner._execute_step(ctx, step) + rows = ctx.db.conn.execute('SELECT * FROM Download').fetchall() + assert len(rows) == 2 + for row in rows: + assert Path(row['saved_path']).read_bytes() == state['body'] + assert all(cookie == 'fixture=session' for path, cookie in state['requests'] + if path.startswith(('/notice', '/original'))) + # Regenerated PDF with changed Info timestamp remains one notice revision. + state['body'] = pdf('two') + runner._execute_step(ctx, step) + assert ctx.db.conn.execute('SELECT count(*) FROM Download WHERE duplicate_of_id IS NULL').fetchone()[0] == 2 + # A material correction is fetched and kept, despite the same daily source. + state['body'] = pdf('three', width=200) + runner._execute_step(ctx, step) + assert ctx.db.conn.execute('SELECT count(*) FROM Download WHERE duplicate_of_id IS NULL').fetchone()[0] == 4 + assert ctx.db.conn.execute('SELECT count(*) FROM Download').fetchone()[0] == 6 + assert len([path for path, _ in state['requests'] if path.startswith('/original')]) == 6 + assert ctx.bcs.date_context.today == date(2026, 1, 15) + + +@pytest.mark.parametrize('field,value', [('account_type', 'PEA-PME'), ('month', [2025, 12]), + ('items', 5), ('pdfs', 2), ('status', 302)]) +def test_synthetic_failure_is_single_attempt_and_redacted(browser_ctx, field, value): + ctx, state = browser_ctx + state[field] = value + # Update rendered listing after changing source identity/item count. + ctx.page.reload() + calls = [] + original = boundary.collect + + def count(*args): + calls.append(1) + return original(*args) + + from unittest.mock import patch + with patch.object(boundary, 'collect', count): + with pytest.raises(runner.StepAbortError) as error: + runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) + assert len(calls) == 1 + assert 'private-' not in str(error.value) + + +def test_explicit_empty_retains_native_empty_run_result(browser_ctx): + ctx, state = browser_ctx + state['items'] = 0 + ctx.page.reload() + runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) + assert ctx.files_downloaded == [] + assert runner.run_result(False, ctx.files_downloaded) == 'failure' + state['empty'] = False + ctx.page.reload() + with pytest.raises(runner.StepAbortError): + runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) + + +def test_dates_bounds_and_runtime_call_validation(browser_ctx): + ctx, state = browser_ctx + save = Mock() + api = boundary.Acquisition(ctx, ctx.bourse_direct_candidate, OPTIONS, save) + assert api.months == ((2026, 1), (2025, 12), (2025, 11)) + with pytest.raises(ValueError): + api.call('listing', unsupported='private') + with pytest.raises(boundary.CollectionError): + api.read('http://other.invalid/original') + with pytest.raises(boundary.CollectionError): + api.save_original(ctx.page.url, '../unsafe.pdf') + with pytest.raises(boundary.CollectionError): + boundary.Acquisition(ctx, ctx.bourse_direct_candidate, dict(OPTIONS, max_months=2), save) + api._deadline = 0 + with pytest.raises(boundary.CollectionError): + api.call('listing') + save.assert_not_called() + + +def test_full_run_start_reaches_frozen_adapter(catalog, portal, tmp_path, monkeypatch): + url, state = portal + path = tmp_path / 'recipe.yaml' + assert save_recipe(path, yaml.safe_dump(recipe(url)))[0] + monkeypatch.setattr(config, 'DB_FILE', tmp_path / 'isolated.db') + db = Database() + db.set_setting('download_dir', str(tmp_path / 'originals')) + task_id = db.create_task('synthetic', 'test', recipe_file=str(path)) + db.close() + monkeypatch.setattr(runner.DateContext, 'capture', Mock(return_value=DateContext( + date(2026, 1, 15), date(2025, 11, 15)))) + monkeypatch.setattr(runner.display, 'allocate_display', Mock(return_value=(99, 5999))) + monkeypatch.setattr(runner.display, 'release_display', Mock()) + monkeypatch.setattr(runner, '_start_display_stack', Mock(return_value=[])) + monkeypatch.setattr(runner.display, 'stop_display_stack', Mock()) + monkeypatch.setattr(runner, 'pause_loop', Mock(side_effect=AssertionError('unexpected pause'))) + monkeypatch.setattr(runner, '_record_step', Mock()) + monkeypatch.setattr(runner.PageState, 'set_interactive_elements', lambda *_args: []) + + def synthetic_credentials(*_args): + catalog.write_text('raise AssertionError("catalog changed after admission")') + return {'user': 'synthetic', 'password': 'synthetic', 'otp': None} + + def isolated_browser(ctx, p): + ctx.context = p.chromium.launch_persistent_context(str(tmp_path / 'profile'), headless=True) + ctx.page = ctx.context.new_page() + ctx.bcs.page = ctx.page + + monkeypatch.setattr(runner, '_vault_preflight', synthetic_credentials) + monkeypatch.setattr(runner, '_launch_browser', isolated_browser) + assert runner.run_task(task_id) == 0 + db = Database() + try: + assert db.conn.execute('SELECT result FROM Run').fetchone()[0] == 'success' + rows = db.conn.execute('SELECT saved_path FROM Download').fetchall() + assert len(rows) == 2 + assert all(Path(row['saved_path']).read_bytes() == state['body'] for row in rows) + finally: + db.close() + + +def test_fork_catalog_contracts_remain_explicit(catalog, monkeypatch): + from billcollector import bourse_direct as legacy + + legacy_recipe = yaml.safe_load((Path(__file__).parent / 'fixtures/bourse_direct.yaml').read_text()) + # Existing catalog validation never requires a new catalog module. + catalog.rename(catalog.with_suffix('.retained')) + assert validate_recipe(yaml.safe_dump(legacy_recipe))[0] + assert boundary.prepare(legacy_recipe) is None + dates = DateContext(date(2026, 1, 15), date(2025, 11, 15)) + assert legacy.collection_dates(recipe(), dates) is dates + with pytest.raises(ValueError, match='Collection mode requires'): + legacy.select_mode(recipe(), 'all') + mixed = recipe() + mixed['services'][0]['steps'].append(legacy_recipe['services'][0]['steps'][-1]) + with pytest.raises(ValueError, match='cannot share a task'): + boundary.prepare(mixed) + + +def test_adapter_dispatch_never_falls_back_to_legacy(browser_ctx, monkeypatch): + from billcollector import bourse_direct as legacy + + denied = Mock(side_effect=AssertionError('legacy provider must not run')) + monkeypatch.setattr(legacy, 'collect', denied) + ctx, _state = browser_ctx + runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) + denied.assert_not_called() + assert len(ctx.files_downloaded) == 2 + assert ctx.db.get_setting(f'bourse_direct_pending_{ctx.task["id"]}') in (None, '') + + +def test_catalog_dispatch_requires_native_final_session(browser_ctx): + ctx, state = browser_ctx + ctx.native_login_proven = False + with pytest.raises(runner.StepAbortError): + runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) + assert ctx.files_downloaded == [] + assert not any(path.startswith('/original') for path, _ in state['requests']) From ecec703ed94bc4e0955e399887d0847eec893422 Mon Sep 17 00:00:00 2001 From: flowcool <19570361+flowcool@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:01:59 +0100 Subject: [PATCH 2/2] fix: preflight adapter month budgets and address Pullfrog contract findings --- CHANGELOG.md | 4 +- apps/billcollector/bourse_direct_boundary.py | 13 ++++-- apps/billcollector/runner.py | 2 +- doc/BOURSE_DIRECT_ADAPTER.md | 6 ++- doc/INFRA_RUNTIME_CONTRACT.md | 2 +- doc/RECIPE_AUTHORING.md | 2 +- doc/RECIPE_METHODS.md | 6 ++- tests/test_bc_bourse_direct_boundary.py | 49 ++++++++++++++++++++ 8 files changed, 73 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 252fced..27485d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Explicit `bourse_direct_adapter` catalog boundary with native login, fixed run dates, C3 load/save/run-start validation, frozen trusted source and bounded authenticated original reads. Existing `bourse_direct` catalogs keep their - current contract. Provider qualification and deployment remain separate. + current contract. The additive recipe schema must be compared and explicitly + promoted by Infra before a new catalog action is adopted. Provider + qualification and deployment remain separate. - One Infra runtime contract for consume-folder delivery, exact vault lookup, schema promotion, DB/configuration handoffs and daemon-path qualification. diff --git a/apps/billcollector/bourse_direct_boundary.py b/apps/billcollector/bourse_direct_boundary.py index a73e6cf..bbd1052 100644 --- a/apps/billcollector/bourse_direct_boundary.py +++ b/apps/billcollector/bourse_direct_boundary.py @@ -57,8 +57,8 @@ def candidate(): try: path = config.RECIPE_DIR / 'bourse_direct_adapter.py' metadata = path.stat() - if metadata.st_uid != os.getuid() or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): - raise ValueError('adapter catalog must be runtime-owned and not group/world-writable') + if metadata.st_uid not in (0, os.getuid()) or metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise ValueError('adapter catalog must be root/runtime-owned and not group/world-writable') source = path.read_bytes() if len(source) > 256 * 1024: raise ValueError @@ -103,7 +103,7 @@ def validate_options(options): raise ValueError('adapter bound is invalid') -def prepare(recipe): +def prepare(recipe, *, dates=None): """One standalone collection after native final-session proof, one service.""" found = [] @@ -144,6 +144,13 @@ def visit(step, top=False): before = ordered[:ordered.index(found[0])] if sum(is_login(step) for step in ordered) != 1 or not any(is_login(step) for step in before): raise ValueError('adapter requires a preceding native login with final-session proof') + if dates is not None: + from .calendar import _months + + try: + tuple(_months(dates, found[0]['methods'][0]['arguments'][0]['max_months'])) + except CollectionError as error: + raise ValueError(str(error)) from None return candidate() diff --git a/apps/billcollector/runner.py b/apps/billcollector/runner.py index 3ab530e..ef4bdeb 100644 --- a/apps/billcollector/runner.py +++ b/apps/billcollector/runner.py @@ -1108,7 +1108,7 @@ def _run_task(task_id: int, debug: bool = False, *, date_from=None, collection_m try: from .bourse_direct_boundary import prepare - adapter_candidate = prepare(recipe) + adapter_candidate = prepare(recipe, dates=dates) except ValueError as e: print(f"runner: invalid method contract: {e}", file=sys.stderr) return 1 diff --git a/doc/BOURSE_DIRECT_ADAPTER.md b/doc/BOURSE_DIRECT_ADAPTER.md index b365f51..74fbce7 100644 --- a/doc/BOURSE_DIRECT_ADAPTER.md +++ b/doc/BOURSE_DIRECT_ADAPTER.md @@ -28,7 +28,7 @@ module, path, entry point or another provider. There is no plugin discovery or registration API. This is trusted application code. Review it with its recipe; the capabilities below are an integration interface, not a Python sandbox. Do not install untrusted Python through the recipe editor. -The file must be owned by the runtime user and must not be group- or +The file must be owned by root or the runtime user and must not be group- or world-writable. A normal `0644` file passes this check. The image build removes group/world write bits from the installed adapter file. @@ -62,7 +62,9 @@ Load, save and run start parse the literal declaration without importing or executing the module. Every chain passes the installed Playwright C3 signature and argument gate. Reserved engine actions are prohibited inside adapter chains. The engine also validates its authenticated-request chain. Run start freezes the -source and manifest before display, vault or browser access. A concurrent catalog +source and manifest before display, vault or browser access. It also rejects +a month budget that cannot cover the fixed run-start date window before any +resources or native login. A concurrent catalog change cannot replace the code of an admitted run. ## Recipe diff --git a/doc/INFRA_RUNTIME_CONTRACT.md b/doc/INFRA_RUNTIME_CONTRACT.md index f117222..1f2c4c0 100644 --- a/doc/INFRA_RUNTIME_CONTRACT.md +++ b/doc/INFRA_RUNTIME_CONTRACT.md @@ -96,7 +96,7 @@ Every release and Infra handoff states the SHA-256 of `apps/recipes_playwright/recipe-pw-schema.yaml`. This candidate's schema is: ```text -1d3b111472c6fdfaab9860fe1ee28d6c255105c69de8281902eefcef631e70f2 +023dd141a90db23d1d6d9f182d40fe9347f7901ef0fb582f9cfa8de6a6c50e75 ``` Calculate it from the exact candidate with diff --git a/doc/RECIPE_AUTHORING.md b/doc/RECIPE_AUTHORING.md index c8af7eb..4714928 100644 --- a/doc/RECIPE_AUTHORING.md +++ b/doc/RECIPE_AUTHORING.md @@ -51,7 +51,7 @@ copied from another version. For example, `get_by_label` and `get_by_placeholder` take a `text` argument. `press` takes `key`, not `value`. A Page-level `click` requires `selector`; a locator's `click` does not. The [C3 reference](RECIPE_METHODS.md#2-dispatch-model-the-playwright-surface) -describes the general surface and the five reserved names. +describes the general surface and the reserved engine actions. `arguments` is a list of option objects merged into keyword arguments. Prefer one key per object for ordinary Playwright methods. A collection action uses diff --git a/doc/RECIPE_METHODS.md b/doc/RECIPE_METHODS.md index bc25870..242da3a 100644 --- a/doc/RECIPE_METHODS.md +++ b/doc/RECIPE_METHODS.md @@ -83,10 +83,12 @@ design: a locator chain such as `get_by_role` -> `first` -> `filter` -> `get_by_label` -> `click` is valid, and new Playwright API added by a version bump stays supported automatically. -Six method names are **reserved**: the runner intercepts them before +Eight method names are **reserved**: the runner intercepts them before Playwright dispatch and they are never passed to the browser API: `login`, `wait_user`, `expect_download`, `click_until_absent`, -`download_all`, `replacement_calendar`. Their exact contracts are in section 3 +`download_all`, `replacement_calendar`, `bourse_direct`, `bourse_direct_adapter`. +The catalog adapter contract and legacy compatibility are in +[BOURSE_DIRECT_ADAPTER.md](BOURSE_DIRECT_ADAPTER.md). Other contracts are in section 3 and the [replacement-calendar contract](REPLACEMENT_CALENDARS.md). C3 validates calendar options with the same offline validator used by calendar execution. diff --git a/tests/test_bc_bourse_direct_boundary.py b/tests/test_bc_bourse_direct_boundary.py index 46acad9..d1206cb 100644 --- a/tests/test_bc_bourse_direct_boundary.py +++ b/tests/test_bc_bourse_direct_boundary.py @@ -76,6 +76,8 @@ def test_load_save_and_run_start_do_not_execute_adapter(catalog, tmp_path, monke db = Database() task = db.create_task('synthetic', 'test', recipe_file=str(path)) db.close() + monkeypatch.setattr(runner.DateContext, 'capture', Mock(return_value=DateContext( + date(2026, 1, 15), date(2025, 11, 15)))) resource = Mock(side_effect=runner.display.DisplayError('synthetic stop')) monkeypatch.setattr(runner.display, 'allocate_display', resource) denied = Mock(side_effect=AssertionError('must not reach vault/browser')) @@ -397,3 +399,50 @@ def test_catalog_dispatch_requires_native_final_session(browser_ctx): runner._execute_step(ctx, recipe()['services'][0]['steps'][-1]) assert ctx.files_downloaded == [] assert not any(path.startswith('/original') for path, _ in state['requests']) + + +@pytest.mark.parametrize('uid,accepted', [(0, True), (987654, False)]) +def test_catalog_owner_must_be_root_or_runtime(catalog, monkeypatch, uid, accepted): + import os + + original = Path.stat + + def metadata(path, **kwargs): + value = original(path, **kwargs) + if path == catalog: + fields = list(value) + fields[4] = uid + return os.stat_result(fields) + return value + + monkeypatch.setattr(Path, 'stat', metadata) + if accepted: + assert boundary.candidate().source == catalog.read_bytes() + else: + with pytest.raises(ValueError, match='root/runtime-owned'): + boundary.candidate() + + +@pytest.mark.parametrize('start', [None, '2025-01-01']) +def test_month_budget_rejected_before_any_resources(catalog, tmp_path, monkeypatch, start): + path = tmp_path / 'recipe.yaml' + assert save_recipe(path, yaml.safe_dump(recipe()))[0] + monkeypatch.setattr(config, 'DB_FILE', tmp_path / 'isolated.db') + db = Database() + task_id = db.create_task('synthetic', 'test', recipe_file=str(path)) + db.close() + dates = DateContext(date(2026, 1, 15), date(2025, 9, 15)) + capture = Mock(return_value=dates) + monkeypatch.setattr(runner.DateContext, 'capture', capture) + denied = Mock(side_effect=AssertionError('must not acquire resources')) + monkeypatch.setattr(runner.display, 'allocate_display', denied) + monkeypatch.setattr(runner, '_vault_preflight', denied) + monkeypatch.setattr(runner, 'sync_playwright', denied) + assert runner.run_task(task_id, date_from=start) == 1 + capture.assert_called_once_with(90, start) + denied.assert_not_called() + db = Database() + try: + assert not db.list_runs() + finally: + db.close()