From 35947361d359beb78cec782823f04f538d6e851c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 18:29:39 -0700 Subject: [PATCH 01/37] feat: lease in-flight kitchen operations --- scripts/generate_core_stub.py | 1 + src/autoskillit/cli/app.py | 43 +- src/autoskillit/core/__init__.pyi | 14 + src/autoskillit/core/pipeline/__init__.py | 18 + .../core/pipeline/_operation_lease.py | 281 +++++++++++ .../types/constants/_type_constants_env.py | 4 + .../protocols/_type_protocols_execution.py | 12 + src/autoskillit/execution/__init__.py | 1 - src/autoskillit/pipeline/context.py | 8 + src/autoskillit/server/__init__.py | 4 + src/autoskillit/server/_factory.py | 3 + .../server/_operation_lease_middleware.py | 46 ++ .../_execution_helpers/_session_deadline.py | 22 + .../tools_execution/_run_skill_dispatch.py | 11 +- tests/_ambient_env_surface.py | 9 + tests/arch/_core_gateway_snapshot.py | 10 + .../test_codex_mcp_server_env_contract.py | 5 + tests/arch/test_watcher_signal_consistency.py | 14 +- tests/cli/test_app_main.py | 63 ++- tests/cli/test_serve_guard_deferral.py | 45 +- tests/conftest.py | 2 + tests/core/test_operation_lease.py | 467 ++++++++++++++++++ .../test_type_constants_private_env_vars.py | 1 + tests/execution/backends/test_codex_config.py | 54 ++ .../server/test_operation_lease_middleware.py | 200 ++++++++ tests/server/test_tools_execution_command.py | 47 ++ 26 files changed, 1320 insertions(+), 65 deletions(-) create mode 100644 src/autoskillit/core/pipeline/_operation_lease.py create mode 100644 src/autoskillit/server/_operation_lease_middleware.py create mode 100644 src/autoskillit/server/tools/_execution_helpers/_session_deadline.py create mode 100644 tests/core/test_operation_lease.py create mode 100644 tests/server/test_operation_lease_middleware.py diff --git a/scripts/generate_core_stub.py b/scripts/generate_core_stub.py index 795b5e214d..6ce26e686b 100644 --- a/scripts/generate_core_stub.py +++ b/scripts/generate_core_stub.py @@ -54,6 +54,7 @@ ".io", ".logging", ".pipeline._execution_marker", + ".pipeline._operation_lease", ".pipeline._step_context", ".pipeline.pipeline_tracker", ".planset", diff --git a/src/autoskillit/cli/app.py b/src/autoskillit/cli/app.py index f2a81f5241..8cc2ad42c2 100644 --- a/src/autoskillit/cli/app.py +++ b/src/autoskillit/cli/app.py @@ -31,22 +31,20 @@ from autoskillit.cli.session._session_order import _recipes_dir_for from autoskillit.core import ( AUDIT_ADMISSION_AUTHORITY_PATH_ENV_VAR, + OPERATION_LEASE_DIR_ENV_VAR, AuditAdmissionStoreAuthority, RecipeSource, atomic_write, get_logger, ) -from autoskillit.execution import _has_active_execution_marker if TYPE_CHECKING: - from autoskillit.core import ManagedWorkerCapacity, SkillResult + from autoskillit.core import InFlightOperationsProtocol, ManagedWorkerCapacity, SkillResult from autoskillit.migration import MigrationEngine, MigrationFile from autoskillit.workspace import SkillInfo logger = get_logger(__name__) -logger = get_logger(__name__) - app = App( name="autoskillit", help="MCP server for executing recipes with Claude Code.", @@ -69,20 +67,33 @@ def is_server_active( - marker_dir: Path | None, worker_capacity: ManagedWorkerCapacity | None, + in_flight_operations: InFlightOperationsProtocol, ) -> bool: - if worker_capacity is not None and worker_capacity.active_count > 0: - return True - if marker_dir is not None and _has_active_execution_marker(marker_dir): - return True - return False + return bool( + in_flight_operations.active_count > 0 + or (worker_capacity is not None and worker_capacity.active_count > 0) + ) class CliError(Exception): """Raised by CLI helpers to signal a user-facing error that should abort the command.""" +def _operation_lease_channel_from_env() -> Path | None: + value = os.environ.get(OPERATION_LEASE_DIR_ENV_VAR) + if value is None: + return None + try: + candidate = Path(value) + if candidate.is_absolute() and candidate.is_dir(): + return candidate + except (OSError, ValueError): + pass + logger.warning("operation_lease_channel_rejected", value=value) + return None + + @app.default def serve(*, verbose: Annotated[bool, Parameter(name=["--verbose", "-v"])] = False): """Start the MCP server (default command).""" @@ -165,6 +176,7 @@ def serve(*, verbose: Annotated[bool, Parameter(name=["--verbose", "-v"])] = Fal cfg, project_dir=_explicit_project_dir, audit_admission_store_authority=audit_admission_store_authority, + operation_lease_channel=_operation_lease_channel_from_env(), plugin_retirement_coordinator=default_plugin_retirement_coordinator(), ) _initialize(ctx) @@ -172,16 +184,13 @@ def serve(*, verbose: Annotated[bool, Parameter(name=["--verbose", "-v"])] = Fal try: backend_options = {"use_uvloop": sys.platform != "win32"} - _marker_dir: Path | None = ( - ctx.backend.session_locator().project_log_dir(str(ctx.project_dir)) - if ctx.backend is not None - else None - ) - async def _guarded_serve() -> None: await serve_with_signal_guard( mcp, - activity_check=lambda: is_server_active(_marker_dir, ctx.worker_capacity), + activity_check=lambda: is_server_active( + ctx.worker_capacity, + ctx.in_flight_operations, + ), ) anyio.run( diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index 199740536f..d0754441fa 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -188,6 +188,18 @@ from .logging import configure_logging as configure_logging from .logging import get_logger as get_logger from .logging import log_plugin_artifact_lifecycle as log_plugin_artifact_lifecycle from .pipeline._execution_marker import execution_marker as execution_marker +from .pipeline._operation_lease import ( + OPERATION_LEASE_FRESHNESS_SECONDS as OPERATION_LEASE_FRESHNESS_SECONDS, +) +from .pipeline._operation_lease import ( + OPERATION_LEASE_HEARTBEAT_SECONDS as OPERATION_LEASE_HEARTBEAT_SECONDS, +) +from .pipeline._operation_lease import InFlightOperations as InFlightOperations +from .pipeline._operation_lease import OperationLeaseHandle as OperationLeaseHandle +from .pipeline._operation_lease import OperationLeaseRecord as OperationLeaseRecord +from .pipeline._operation_lease import current_operation_lease as current_operation_lease +from .pipeline._operation_lease import operation_lease as operation_lease +from .pipeline._operation_lease import read_active_operation_leases as read_active_operation_leases from .pipeline._step_context import current_order_id as current_order_id from .pipeline._step_context import current_step_name as current_step_name from .pipeline.pipeline_tracker import TrackerAuthorityReadResult as TrackerAuthorityReadResult @@ -613,6 +625,7 @@ from .types import MAXIMUM_FIELDS as MAXIMUM_FIELDS from .types import MCP_CLIENT_BACKEND_ENV_VAR as MCP_CLIENT_BACKEND_ENV_VAR from .types import NATIVE_SHELL_CAPTURE_MODE_ENV_VAR as NATIVE_SHELL_CAPTURE_MODE_ENV_VAR from .types import NON_VARIADIC_CLAUDE_FLAGS as NON_VARIADIC_CLAUDE_FLAGS +from .types import OPERATION_LEASE_DIR_ENV_VAR as OPERATION_LEASE_DIR_ENV_VAR from .types import ( ORCHESTRATOR_FACING_INSTRUCTION_SURFACES as ORCHESTRATOR_FACING_INSTRUCTION_SURFACES, ) @@ -1018,6 +1031,7 @@ from .types import HostClientAttestation as HostClientAttestation from .types import IdempotencyExpiredEffect as IdempotencyExpiredEffect from .types import IdempotencyNamespace as IdempotencyNamespace from .types import IdempotencyRecord as IdempotencyRecord +from .types import InFlightOperationsProtocol as InFlightOperationsProtocol from .types import InfraExitCategory as InfraExitCategory from .types import InfraOutcome as InfraOutcome from .types import InfrastructureFaultError as InfrastructureFaultError diff --git a/src/autoskillit/core/pipeline/__init__.py b/src/autoskillit/core/pipeline/__init__.py index be963b8e40..be12bb3077 100644 --- a/src/autoskillit/core/pipeline/__init__.py +++ b/src/autoskillit/core/pipeline/__init__.py @@ -12,6 +12,16 @@ from __future__ import annotations from autoskillit.core.pipeline._execution_marker import execution_marker +from autoskillit.core.pipeline._operation_lease import ( + OPERATION_LEASE_FRESHNESS_SECONDS, + OPERATION_LEASE_HEARTBEAT_SECONDS, + InFlightOperations, + OperationLeaseHandle, + OperationLeaseRecord, + current_operation_lease, + operation_lease, + read_active_operation_leases, +) from autoskillit.core.pipeline._step_context import current_order_id, current_step_name from autoskillit.core.pipeline.pipeline_tracker import ( TrackerAuthorityReadResult, @@ -54,6 +64,11 @@ "AssistantTurn", "DFG", "GapStats", + "InFlightOperations", + "OPERATION_LEASE_FRESHNESS_SECONDS", + "OPERATION_LEASE_HEARTBEAT_SECONDS", + "OperationLeaseHandle", + "OperationLeaseRecord", "TrackerAuthorityReadResult", "TrackerAuthorityTarget", "TrackerParticipantKey", @@ -62,6 +77,7 @@ "build_dfg_by_recipe", "compute_analysis", "compute_gap_stats", + "current_operation_lease", "current_order_id", "current_step_name", "execution_marker", @@ -72,11 +88,13 @@ "initialize_manual_tracker", "iter_merged_assistant_turns", "mutate_tracker", + "operation_lease", "parse_raw_cc_jsonl", "parse_sessions_from_summary_dir", "pipeline_tracker_directory", "pipeline_tracker_path", "read_tracker_authority", + "read_active_operation_leases", "release_tracker_lease", "render_adjacency_table", "render_dot", diff --git a/src/autoskillit/core/pipeline/_operation_lease.py b/src/autoskillit/core/pipeline/_operation_lease.py new file mode 100644 index 0000000000..06d31501bf --- /dev/null +++ b/src/autoskillit/core/pipeline/_operation_lease.py @@ -0,0 +1,281 @@ +"""Deadline-bounded evidence for an in-flight MCP tool call. + +Operation leases are not locks and do not prove that a child's work is making +progress. A live producer renews the lease file's mtime; readers accept it only +while that mtime is fresh and its recorded deadline has not passed. This assumes +the producer and supervisor share a filesystem with coherent timestamps. The +supervisor is responsible for supplying a channel on such a filesystem. +""" + +from __future__ import annotations + +import asyncio +import math +import os +import threading +import time +import uuid +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from contextvars import ContextVar, Token +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Final + +import anyio + +from ..io.io import read_versioned_json, write_versioned_json +from ..logging import get_logger +from ..types.protocols._type_protocols_execution import InFlightOperationsProtocol + +__all__ = [ + "InFlightOperations", + "OPERATION_LEASE_FRESHNESS_SECONDS", + "OPERATION_LEASE_HEARTBEAT_SECONDS", + "OperationLeaseHandle", + "OperationLeaseRecord", + "current_operation_lease", + "operation_lease", + "read_active_operation_leases", +] + +logger = get_logger(__name__) + +OPERATION_LEASE_HEARTBEAT_SECONDS: Final = 30.0 +OPERATION_LEASE_FRESHNESS_SECONDS: Final = 90.0 +_LEASE_SUFFIX: Final = ".lease.json" +_SCHEMA_VERSION: Final = 1 + + +@dataclass(frozen=True, slots=True) +class OperationLeaseRecord: + operation_id: str + operation: str + started_at_epoch: float + not_after_epoch: float + + def __post_init__(self) -> None: + for name, value in ( + ("started_at_epoch", self.started_at_epoch), + ("not_after_epoch", self.not_after_epoch), + ): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a finite epoch timestamp") + if not math.isfinite(value): + raise ValueError(f"{name} must be a finite epoch timestamp") + if self.not_after_epoch <= self.started_at_epoch: + raise ValueError("not_after_epoch must be later than started_at_epoch") + + def to_payload(self) -> dict[str, object]: + return { + "operation_id": self.operation_id, + "operation": self.operation, + "started_at_epoch": self.started_at_epoch, + "not_after_epoch": self.not_after_epoch, + } + + @classmethod + def from_payload(cls, payload: dict[str, object]) -> OperationLeaseRecord: + schema_version = payload.get("schema_version") + if isinstance(schema_version, bool) or schema_version != _SCHEMA_VERSION: + raise ValueError("operation lease schema version is unsupported") + operation_id = payload.get("operation_id") + operation = payload.get("operation") + started_at = payload.get("started_at_epoch") + not_after = payload.get("not_after_epoch") + if not isinstance(operation_id, str) or not isinstance(operation, str): + raise ValueError("operation lease identity fields must be strings") + if ( + isinstance(started_at, bool) + or not isinstance(started_at, (int, float)) + or isinstance(not_after, bool) + or not isinstance(not_after, (int, float)) + ): + raise ValueError("operation lease epochs must be numbers") + return cls(operation_id, operation, float(started_at), float(not_after)) + + +class InFlightOperations: + """Thread-safe count of tool operations currently inside the kitchen.""" + + __slots__ = ("_active_count", "_lock") + + def __init__(self) -> None: + self._active_count = 0 + self._lock = threading.Lock() + + @property + def active_count(self) -> int: + with self._lock: + return self._active_count + + def _enter(self) -> None: + with self._lock: + self._active_count += 1 + + def _exit(self) -> None: + with self._lock: + self._active_count -= 1 + + +class OperationLeaseHandle: + """Mutable deadline and current channel path for one in-flight operation.""" + + __slots__ = ("record", "path") + + def __init__(self, record: OperationLeaseRecord, path: Path | None) -> None: + self.record = record + self.path = path + + def _retire_file(self) -> None: + path = self.path + self.path = None + if path is None: + return + try: + path.unlink(missing_ok=True) + except OSError: + logger.warning("operation_lease_unlink_failed", path=str(path), exc_info=True) + + def narrow(self, not_after_epoch: float) -> None: + if isinstance(not_after_epoch, bool) or not isinstance(not_after_epoch, (int, float)): + raise ValueError("not_after_epoch must be a finite epoch timestamp") + if not math.isfinite(not_after_epoch): + raise ValueError("not_after_epoch must be a finite epoch timestamp") + if not_after_epoch >= self.record.not_after_epoch: + return + if not_after_epoch <= self.record.started_at_epoch: + self._retire_file() + raise ValueError("operation lease deadline has already expired") + + self.record = replace(self.record, not_after_epoch=float(not_after_epoch)) + path = self.path + if path is None: + return + if not path.exists(): + self.path = None + return + try: + write_versioned_json(path, self.record.to_payload(), schema_version=_SCHEMA_VERSION) + except OSError: + self.path = None + logger.warning("operation_lease_narrow_write_failed", path=str(path), exc_info=True) + try: + path.unlink(missing_ok=True) + except OSError: + logger.warning("operation_lease_unlink_failed", path=str(path), exc_info=True) + raise + + +_CURRENT_LEASE: ContextVar[OperationLeaseHandle | None] = ContextVar( + "autoskillit_current_operation_lease", default=None +) + + +def current_operation_lease() -> OperationLeaseHandle | None: + return _CURRENT_LEASE.get() + + +async def _heartbeat(handle: OperationLeaseHandle, interval: float) -> None: + while True: + await anyio.sleep(interval) + path = handle.path + if path is None: + return + try: + os.utime(path, None) + except FileNotFoundError: + if handle.path == path: + handle.path = None + return + + +@asynccontextmanager +async def operation_lease( + channel_dir: Path | None, + *, + operation: str, + not_after_epoch: float, + registry: InFlightOperationsProtocol, + heartbeat_interval: float = OPERATION_LEASE_HEARTBEAT_SECONDS, +) -> AsyncGenerator[OperationLeaseHandle, None]: + """Register, publish, and clean up one deadline-bounded tool operation.""" + record = OperationLeaseRecord( + operation_id=uuid.uuid4().hex, + operation=operation, + started_at_epoch=time.time(), + not_after_epoch=not_after_epoch, + ) + handle = OperationLeaseHandle(record, None) + registry._enter() + heartbeat_task: asyncio.Task[None] | None = None + token: Token[OperationLeaseHandle | None] | None = None + try: + if channel_dir is not None: + path = channel_dir / f"{record.operation_id}{_LEASE_SUFFIX}" + try: + write_versioned_json(path, record.to_payload(), schema_version=_SCHEMA_VERSION) + except OSError: + logger.warning("operation_lease_write_failed", path=str(path), exc_info=True) + else: + handle.path = path + heartbeat_task = asyncio.get_running_loop().create_task( + _heartbeat(handle, heartbeat_interval) + ) + token = _CURRENT_LEASE.set(handle) + yield handle + finally: + try: + if token is not None: + _CURRENT_LEASE.reset(token) + finally: + if heartbeat_task is not None: + heartbeat_task.cancel() + try: + handle._retire_file() + finally: + registry._exit() + if heartbeat_task is not None: + with anyio.CancelScope(shield=True): + try: + await heartbeat_task + except asyncio.CancelledError: + pass + except Exception: + logger.warning("operation_lease_heartbeat_failed", exc_info=True) + + +def read_active_operation_leases( + channel_dir: Path, + *, + now_epoch: float, + freshness_seconds: float = OPERATION_LEASE_FRESHNESS_SECONDS, +) -> tuple[OperationLeaseRecord, ...]: + """Return valid, fresh leases whose deadlines have not passed.""" + records: list[OperationLeaseRecord] = [] + try: + paths = tuple(channel_dir.glob(f"*{_LEASE_SUFFIX}")) + except OSError: + logger.debug("operation_lease_channel_unreadable", path=str(channel_dir), exc_info=True) + return () + for path in paths: + try: + stat_result = path.stat() + except OSError: + continue + if now_epoch - stat_result.st_mtime > freshness_seconds: + continue + payload = read_versioned_json(path, _SCHEMA_VERSION) + if payload is None: + logger.debug("operation_lease_read_skipped", path=str(path)) + continue + try: + record = OperationLeaseRecord.from_payload(payload) + except (TypeError, ValueError): + logger.debug("operation_lease_read_skipped", path=str(path), exc_info=True) + continue + if now_epoch >= record.not_after_epoch: + continue + records.append(record) + records.sort(key=lambda record: record.not_after_epoch) + return tuple(records) diff --git a/src/autoskillit/core/types/constants/_type_constants_env.py b/src/autoskillit/core/types/constants/_type_constants_env.py index b0a0e0a81b..e9b6778c10 100644 --- a/src/autoskillit/core/types/constants/_type_constants_env.py +++ b/src/autoskillit/core/types/constants/_type_constants_env.py @@ -38,6 +38,7 @@ "DISPATCH_ID_ENV_VAR", "OWNER_SCOPE_ENV_VAR", "OWNER_SCOPE_DIR_ENV_VAR", + "OPERATION_LEASE_DIR_ENV_VAR", "FLEET_INSPECTOR_MODEL_ENV_VAR", "KITCHEN_SESSION_ID_ENV_VAR", "LAUNCH_ID_ENV_VAR", @@ -100,6 +101,8 @@ OWNER_SCOPE_DIR_ENV_VAR: str = "AUTOSKILLIT_OWNER_SCOPE_DIR" FLEET_INSPECTOR_MODEL_ENV_VAR: str = "AUTOSKILLIT_FLEET_INSPECTOR_MODEL" KITCHEN_SESSION_ID_ENV_VAR: str = "AUTOSKILLIT_KITCHEN_SESSION_ID" +# Producer: supervisor attempt. Consumer: child's kitchen MCP server. Lifetime: one attempt. +OPERATION_LEASE_DIR_ENV_VAR: str = "AUTOSKILLIT_OPERATION_LEASE_DIR" LAUNCH_ID_ENV_VAR: str = "AUTOSKILLIT_LAUNCH_ID" # One-attempt handoff from run_cook_attempt to the live session's hooks. SESSION_LIFETIME_NOTICE_ENV_VAR: str = "AUTOSKILLIT_SESSION_LIFETIME_NOTICE" @@ -189,6 +192,7 @@ def managed_join_parent_id() -> str: OWNER_SCOPE_DIR_ENV_VAR, FLEET_INSPECTOR_MODEL_ENV_VAR, KITCHEN_SESSION_ID_ENV_VAR, + OPERATION_LEASE_DIR_ENV_VAR, "AUTOSKILLIT_CAMPAIGN_STATE_PATH", "AUTOSKILLIT_PROJECT_DIR", FOOD_TRUCK_TOOL_TAGS_ENV_VAR, diff --git a/src/autoskillit/core/types/protocols/_type_protocols_execution.py b/src/autoskillit/core/types/protocols/_type_protocols_execution.py index 086e5ab77a..be74ed89ba 100644 --- a/src/autoskillit/core/types/protocols/_type_protocols_execution.py +++ b/src/autoskillit/core/types/protocols/_type_protocols_execution.py @@ -40,6 +40,7 @@ __all__ = [ "CompletionRequiredResolver", + "InFlightOperationsProtocol", "InputContractResolver", "LaunchAdapter", "LaunchResolver", @@ -54,6 +55,17 @@ ] +class InFlightOperationsProtocol(Protocol): + """Minimal counter contract for active kitchen tool operations.""" + + @property + def active_count(self) -> int: ... + + def _enter(self) -> None: ... + + def _exit(self) -> None: ... + + @runtime_checkable class RunSkillCompletionAuthority(Protocol): """Server-owned lifecycle for launched ``run_skill`` invocations.""" diff --git a/src/autoskillit/execution/__init__.py b/src/autoskillit/execution/__init__.py index a7193cc090..5f90eb483b 100644 --- a/src/autoskillit/execution/__init__.py +++ b/src/autoskillit/execution/__init__.py @@ -165,7 +165,6 @@ TetherSweepOutcome, TetherSweepReport, _active_liveness_signals, - _has_active_execution_marker, # noqa: F401 — re-exported for cli/app.py signal guard async_kill_process_tree, default_tether_dir, dispatch_scope_tokens, diff --git a/src/autoskillit/pipeline/context.py b/src/autoskillit/pipeline/context.py index 87a5404456..f8d6d5540f 100644 --- a/src/autoskillit/pipeline/context.py +++ b/src/autoskillit/pipeline/context.py @@ -40,6 +40,8 @@ GitHubReviewPosterProtocol, HeadlessExecutor, HostClientAttestation, + InFlightOperations, + InFlightOperationsProtocol, InputContractResolver, KitchenProcessIdentity, KitchenTransitionLock, @@ -176,6 +178,10 @@ class ToolContext: quota refresh background task. worker_capacity: ManagedWorkerCapacity — shared owner-bound guard for fleet dispatch and managed fixed-batch leaves. + in_flight_operations: InFlightOperationsProtocol — counts active MCP tool calls for + kitchen shutdown and operation-lease production. + operation_lease_channel: Path | None — per-attempt directory where active tool-call + leases are published for the supervising process. build_protected_campaign_ids: CampaignProtector — resolves campaign IDs exempt from log retention purge. session_skill_manager: SessionSkillManager — manages per-session ephemeral skill dirs @@ -317,6 +323,8 @@ class ToolContext: quota_refresh_task: QuotaRefreshTask | None = field(default=None) token_factory: TokenFactory | None = field(default=None) worker_capacity: ManagedWorkerCapacity | None = field(default=None) + in_flight_operations: InFlightOperationsProtocol = field(default_factory=InFlightOperations) + operation_lease_channel: Path | None = field(default=None) managed_fixed_batch_supervisor: ManagedFixedBatchSupervisor | None = field( default=None, repr=False ) diff --git a/src/autoskillit/server/__init__.py b/src/autoskillit/server/__init__.py index e4fd64621a..d7515d8e21 100644 --- a/src/autoskillit/server/__init__.py +++ b/src/autoskillit/server/__init__.py @@ -180,10 +180,14 @@ # Wire-format sanitization: strip fields that trigger Claude Code #25081 # (silent full-tool-list rejection when outputSchema/annotations are present). +from autoskillit.server._operation_lease_middleware import ( # noqa: E402 + OperationLeaseMiddleware, +) from autoskillit.server.response._run_skill_completion import ( # noqa: E402 RunSkillCompletionMiddleware, ) +mcp.add_middleware(OperationLeaseMiddleware()) mcp.add_middleware(RunSkillCompletionMiddleware(mcp)) from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware # noqa: E402 diff --git a/src/autoskillit/server/_factory.py b/src/autoskillit/server/_factory.py index 9821743e95..097a9ce30c 100644 --- a/src/autoskillit/server/_factory.py +++ b/src/autoskillit/server/_factory.py @@ -207,6 +207,7 @@ def make_context( plugin_authority: PluginArtifactAuthority = _UNSET, plugin_retirement_coordinator: PluginRetirementCoordinator | None = None, worker_capacity: ManagedWorkerCapacity | None = None, + operation_lease_channel: Path | None = None, project_dir: Path | None = None, audit_admission_store_authority: AuditAdmissionStoreAuthority | None = None, ) -> ToolContext: @@ -231,6 +232,7 @@ def make_context( worker_capacity: ManagedWorkerCapacity implementation to inject. Defaults to DefaultManagedWorkerCapacity(max_concurrent_dispatches) when None. Pass a custom implementation in tests to substitute without monkey-patching. + operation_lease_channel: Per-attempt directory for publishing in-flight tool leases. project_dir: Explicit project root path. When supplied, used directly. When None, resolve_project_dir() is called (git toplevel → cwd) — the same helper `autoskillit cook` uses. @@ -470,6 +472,7 @@ def make_context( ci_watcher=DefaultCIWatcher(token=token_factory, tracker=github_api_log), merge_queue_watcher=DefaultMergeQueueWatcher(token=token_factory, tracker=github_api_log), github_api_log=github_api_log, + operation_lease_channel=operation_lease_channel, session_skill_manager=session_mgr, skill_resolver=provider.resolver, skill_session_contract_store=DefaultSkillSessionContractStore(), diff --git a/src/autoskillit/server/_operation_lease_middleware.py b/src/autoskillit/server/_operation_lease_middleware.py new file mode 100644 index 0000000000..c23d4c0c5f --- /dev/null +++ b/src/autoskillit/server/_operation_lease_middleware.py @@ -0,0 +1,46 @@ +"""Publish a deadline-bounded lease around each kitchen tool call.""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +from fastmcp.server.middleware import Middleware +from fastmcp.tools.base import ToolResult + +from autoskillit.core import operation_lease +from autoskillit.server.tools._execution_helpers._session_deadline import ( + inherited_session_deadline_epoch, +) + +if TYPE_CHECKING: + import mcp.types as mt + from fastmcp.server.middleware import CallNext, MiddlewareContext + + +class OperationLeaseMiddleware(Middleware): + async def on_call_tool( + self, + context: MiddlewareContext[mt.CallToolRequestParams], + call_next: CallNext[mt.CallToolRequestParams, ToolResult], + ) -> ToolResult: + from autoskillit.server.lifecycle._state import ( # circular-break + _get_ctx_or_none, + ) + + tool_ctx = _get_ctx_or_none() + if tool_ctx is None: + return await call_next(context) + + not_after = time.time() + tool_ctx.config.run_skill.mcp_tool_timeout_sec + inherited_deadline = inherited_session_deadline_epoch() + if inherited_deadline > 0: + not_after = min(not_after, inherited_deadline) + + async with operation_lease( + tool_ctx.operation_lease_channel, + operation=context.message.name, + not_after_epoch=not_after, + registry=tool_ctx.in_flight_operations, + ): + return await call_next(context) diff --git a/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py b/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py new file mode 100644 index 0000000000..b49bc6aa58 --- /dev/null +++ b/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py @@ -0,0 +1,22 @@ +"""Read the session deadline inherited by this process.""" + +from __future__ import annotations + +import math +import os + +__all__ = ["inherited_session_deadline_epoch"] + + +def inherited_session_deadline_epoch() -> float: + """Return this process's inherited session deadline, or zero when invalid.""" + value = os.environ.get("AUTOSKILLIT_SESSION_DEADLINE") + if value is None: + return 0.0 + try: + deadline = float(value) + except ValueError: + return 0.0 + if not math.isfinite(deadline) or deadline <= 0: + return 0.0 + return deadline diff --git a/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py b/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py index 34d197e2be..9a3213dcca 100644 --- a/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py +++ b/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py @@ -24,6 +24,7 @@ RetryReason, SkillContractError, SkillResult, + current_operation_lease, get_logger, read_tracker_authority, ) @@ -56,6 +57,9 @@ from autoskillit.server.tools._execution_helpers import ( validate_resumed_skill_contract as _validate_resumed_skill_contract, ) +from autoskillit.server.tools._execution_helpers._session_deadline import ( + inherited_session_deadline_epoch, +) from autoskillit.server.tools._types import deny_envelope from autoskillit.server.tools.tools_execution._gates import _authority_blocks_dependency_check from autoskillit.server.tools.tools_execution._managed_leaf import ( # noqa: F401 @@ -419,10 +423,7 @@ async def run_skill( started_epoch + run_config.timeout, started_epoch + run_config.mcp_tool_timeout_sec, ) - try: - inherited_deadline = float(os.environ.get("AUTOSKILLIT_SESSION_DEADLINE", "")) - except ValueError: - inherited_deadline = 0.0 + inherited_deadline = inherited_session_deadline_epoch() if 0 < inherited_deadline < deadline_epoch: deadline_epoch = inherited_deadline if resume_session_id and state.execution_selection is not None: @@ -431,6 +432,8 @@ async def run_skill( raise SkillContractError("Resume continuation deadline is unavailable or expired") deadline_epoch = min(deadline_epoch, float(original_deadline)) state._invocation_deadline_epoch = deadline_epoch + if (lease := current_operation_lease()) is not None: + lease.narrow(state._invocation_deadline_epoch) state._invocation_deadline_monotonic = started_monotonic + (deadline_epoch - started_epoch) state._completion_invocation_id = _te_pkg._begin_run_skill_completion( state.tool_ctx, diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 5181bcd620..6dad168f51 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -1532,6 +1532,15 @@ class AmbientEnvDisposition: "of an unrelated lookup collection; never set as a real OS environment variable." ), ), + "AUTOSKILLIT_OPERATION_LEASE_DIR": AmbientEnvDisposition( + var="AUTOSKILLIT_OPERATION_LEASE_DIR", + disposition="scrub", + owner="autoskillit", + justification=( + "Attempt-scoped operation lease channel supplied to the child's kitchen server;" + " scrubbed so an unrelated child cannot inherit another attempt's liveness channel." + ), + ), "AUTOSKILLIT_KITCHEN_SESSION_ID": AmbientEnvDisposition( var="AUTOSKILLIT_KITCHEN_SESSION_ID", disposition="scrub", diff --git a/tests/arch/_core_gateway_snapshot.py b/tests/arch/_core_gateway_snapshot.py index 79efd5b9df..8ec74d5d3c 100644 --- a/tests/arch/_core_gateway_snapshot.py +++ b/tests/arch/_core_gateway_snapshot.py @@ -426,6 +426,8 @@ "IdempotencyExpiredEffect", "IdempotencyNamespace", "IdempotencyRecord", + "InFlightOperations", + "InFlightOperationsProtocol", "InfraExitCategory", "InfraOutcome", "InfrastructureFaultError", @@ -533,6 +535,9 @@ "NdjsonDriftOutcome", "NoResume", "NodeKey", + "OPERATION_LEASE_DIR_ENV_VAR", + "OPERATION_LEASE_FRESHNESS_SECONDS", + "OPERATION_LEASE_HEARTBEAT_SECONDS", "ORCHESTRATOR_FACING_INSTRUCTION_SURFACES", "ORCHESTRATOR_SESSION_REQUIRED_ENV", "ORDER_INTERACTIVE_REQUIRED_ENV", @@ -547,6 +552,8 @@ "ObserverStatus", "OccurrenceStateChangedEffect", "OpenEpochEvent", + "OperationLeaseHandle", + "OperationLeaseRecord", "OrchestratorSurfaceDef", "OutputFormat", "OutputPatternResolver", @@ -962,6 +969,7 @@ "contains_test_gate_command", "context_admission_envelope_header", "context_admission_reducer_for_protocol", + "current_operation_lease", "current_order_id", "current_step_name", "decode_stored_context_admission_envelope", @@ -1067,6 +1075,7 @@ "normalize_owner_repo", "normalize_parent_sandbox_mode", "observe_path_mode", + "operation_lease", "parse_canonical_json_bytes", "parse_direct_url", "parse_github_remote_url", @@ -1100,6 +1109,7 @@ "provenance_path", "publish_private_file", "read_active_kitchens_registry", + "read_active_operation_leases", "read_boot_id", "read_claude_plugin_tool_prefix", "read_installed_plugin_artifact_identity", diff --git a/tests/arch/test_codex_mcp_server_env_contract.py b/tests/arch/test_codex_mcp_server_env_contract.py index d2f6e06191..1d29c64ed6 100644 --- a/tests/arch/test_codex_mcp_server_env_contract.py +++ b/tests/arch/test_codex_mcp_server_env_contract.py @@ -14,6 +14,7 @@ DISPATCH_ID_ENV_VAR, FLEET_MODE_ENV_VAR, KITCHEN_SESSION_ID_ENV_VAR, + OPERATION_LEASE_DIR_ENV_VAR, ) from tests._ambient_env_surface import ( _PRODUCTION_SRC_ROOT, @@ -99,3 +100,7 @@ def test_dispatch_identity_channel_regression() -> None: } assert dispatch_identity_vars <= CODEX_MCP_ENV_FORWARD_VARS + + +def test_operation_lease_channel_reaches_codex_kitchens() -> None: + assert OPERATION_LEASE_DIR_ENV_VAR in CODEX_MCP_ENV_FORWARD_VARS diff --git a/tests/arch/test_watcher_signal_consistency.py b/tests/arch/test_watcher_signal_consistency.py index 350a93aee4..c5863f8e8c 100644 --- a/tests/arch/test_watcher_signal_consistency.py +++ b/tests/arch/test_watcher_signal_consistency.py @@ -100,16 +100,12 @@ def test_watcher_calls_has_active_execution_marker(watcher: str) -> None: ) -_SIGNAL_GUARD_ACTIVITY_MUST_CHECK_MARKER = frozenset({"is_server_active"}) +def test_signal_guard_activity_check_uses_in_flight_registry() -> None: + function = _function(_CLI_APP, "is_server_active") + source = ast.unparse(function) - -@pytest.mark.parametrize("fn_name", sorted(_SIGNAL_GUARD_ACTIVITY_MUST_CHECK_MARKER)) -def test_signal_guard_activity_check_calls_has_active_execution_marker(fn_name: str) -> None: - callers = _functions_calling_predicate(_CLI_APP, "_has_active_execution_marker") - assert fn_name in callers, ( - f"{fn_name} in cli/app.py does not call _has_active_execution_marker. " - f"Functions that do: {sorted(callers)}" - ) + assert "in_flight_operations.active_count" in source + assert "_has_active_execution_marker" not in source _KILL_EXECUTORS_THAT_MUST_CHECK_CHILD_LIVENESS = frozenset( diff --git a/tests/cli/test_app_main.py b/tests/cli/test_app_main.py index 340de17982..1354455786 100644 --- a/tests/cli/test_app_main.py +++ b/tests/cli/test_app_main.py @@ -279,24 +279,38 @@ def test_main_repair_classifies_missing_expected_version_as_incomplete( ] -def test_serve_activity_check_uses_backend_derived_marker_dir( +@pytest.mark.parametrize("channel_kind", ["existing", "relative", "missing"]) +def test_serve_activity_check_uses_in_flight_registry_and_validates_channel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + channel_kind: str, ) -> None: - """serve() must derive _marker_dir via backend.session_locator().project_log_dir().""" + """serve() reads the channel env at the CLI boundary and guards on ToolContext state.""" app_module = importlib.import_module("autoskillit.cli.app") + from autoskillit.core import InFlightOperations monkeypatch.chdir(tmp_path) - expected_dir = tmp_path / "backend-marker" + if channel_kind == "existing": + configured_path = tmp_path / "operation-leases" + configured_path.mkdir() + expected_channel = configured_path + elif channel_kind == "relative": + configured_path = "relative-channel" + expected_channel = None + else: + configured_path = str(tmp_path / "missing-channel") + expected_channel = None + monkeypatch.setenv("AUTOSKILLIT_OPERATION_LEASE_DIR", str(configured_path)) mock_backend = MagicMock() - mock_backend.session_locator.return_value.project_log_dir.return_value = expected_dir mock_ctx = MagicMock() mock_ctx.project_dir = tmp_path mock_ctx.backend = mock_backend mock_ctx.worker_capacity = None + registry = InFlightOperations() + mock_ctx.in_flight_operations = registry mock_cfg = MagicMock() mock_cfg.logging.level = "INFO" @@ -305,9 +319,22 @@ def test_serve_activity_check_uses_backend_derived_marker_dir( monkeypatch.setattr("autoskillit.config.load_config", lambda _: mock_cfg) monkeypatch.setattr("autoskillit.core.configure_logging", lambda **kw: None) - monkeypatch.setattr("autoskillit.server.make_context", lambda *a, **kw: mock_ctx) + make_context_kwargs: list[dict[str, object]] = [] + + def fake_make_context(*args, **kwargs): + make_context_kwargs.append(kwargs) + return mock_ctx + + monkeypatch.setattr("autoskillit.server.make_context", fake_make_context) monkeypatch.setattr("autoskillit.server._initialize", lambda ctx: None) + warnings: list[tuple[str, dict[str, object]]] = [] + monkeypatch.setattr( + app_module.logger, + "warning", + lambda event, **kwargs: warnings.append((event, kwargs)), + ) + captured_activity_check: list = [] async def fake_serve_guard(mcp_instance, *, activity_check=None, **kw): @@ -320,21 +347,29 @@ async def fake_serve_guard(mcp_instance, *, activity_check=None, **kw): serve() - mock_backend.session_locator.return_value.project_log_dir.assert_called_once_with( - str(tmp_path) - ) + mock_backend.session_locator.assert_not_called() + assert make_context_kwargs[0]["operation_lease_channel"] == expected_channel + if expected_channel is None: + assert warnings == [ + ( + "operation_lease_channel_rejected", + {"value": str(configured_path)}, + ) + ] + else: + assert warnings == [] assert len(captured_activity_check) == 1, "serve_with_signal_guard was not called exactly once" - seen_marker_dirs: list = [] + seen_registries: list = [] monkeypatch.setattr( app_module, "is_server_active", - lambda md, fl: (seen_marker_dirs.append(md), False)[-1], + lambda worker_capacity, in_flight_operations: ( + seen_registries.append((worker_capacity, in_flight_operations)), + False, + )[-1], ) captured_activity_check[0]() - assert seen_marker_dirs == [expected_dir], ( - f"activity_check should pass backend-derived marker_dir ({expected_dir}), " - f"got {seen_marker_dirs}" - ) + assert seen_registries == [(None, registry)] diff --git a/tests/cli/test_serve_guard_deferral.py b/tests/cli/test_serve_guard_deferral.py index 69e97d191b..bb6542ff66 100644 --- a/tests/cli/test_serve_guard_deferral.py +++ b/tests/cli/test_serve_guard_deferral.py @@ -5,8 +5,6 @@ import inspect import os import signal -import time -import uuid from pathlib import Path import anyio @@ -14,6 +12,7 @@ from autoskillit.cli._serve_guard import serve_with_signal_guard from autoskillit.cli.app import is_server_active +from autoskillit.core import InFlightOperations pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] @@ -109,10 +108,13 @@ async def _send_signal() -> None: class TestActivityCheckCompleteness: - def test_activity_check_true_when_execution_marker_exists(self, tmp_path: Path) -> None: - marker_name = f"run-skill-in-progress-sess-{uuid.uuid4()}.marker" - (tmp_path / marker_name).touch() - assert is_server_active(marker_dir=tmp_path, worker_capacity=None) is True + def test_activity_check_true_when_operation_is_in_flight(self) -> None: + operations = InFlightOperations() + operations._enter() + try: + assert is_server_active(worker_capacity=None, in_flight_operations=operations) is True + finally: + operations._exit() @pytest.mark.anyio async def test_activity_check_true_when_worker_capacity_active(self) -> None: @@ -121,29 +123,32 @@ async def test_activity_check_true_when_worker_capacity_active(self) -> None: capacity = DefaultManagedWorkerCapacity() permit = await capacity.acquire("active-dispatch") try: - assert is_server_active(marker_dir=None, worker_capacity=capacity) is True + assert is_server_active( + worker_capacity=capacity, in_flight_operations=InFlightOperations() + ) finally: capacity.release(permit) - def test_activity_check_false_when_both_idle(self, tmp_path: Path) -> None: + def test_activity_check_false_when_both_idle(self) -> None: from autoskillit.core import DefaultManagedWorkerCapacity assert ( - is_server_active(marker_dir=tmp_path, worker_capacity=DefaultManagedWorkerCapacity()) + is_server_active( + worker_capacity=DefaultManagedWorkerCapacity(), + in_flight_operations=InFlightOperations(), + ) is False ) - def test_activity_check_true_when_marker_within_age(self, tmp_path: Path) -> None: - marker = tmp_path / f"run-skill-in-progress-sess-{uuid.uuid4()}.marker" - marker.touch() - os.utime(marker, (time.time() - 50, time.time() - 50)) - assert is_server_active(marker_dir=tmp_path, worker_capacity=None) is True - - def test_activity_check_false_when_marker_expired(self, tmp_path: Path) -> None: - marker = tmp_path / f"run-skill-in-progress-sess-{uuid.uuid4()}.marker" - marker.touch() - os.utime(marker, (time.time() - 120, time.time() - 120)) - assert is_server_active(marker_dir=tmp_path, worker_capacity=None) is False + def test_stray_backend_marker_does_not_make_server_active(self, tmp_path: Path) -> None: + (tmp_path / "dispatch-in-progress-other-session-marker.marker").touch() + assert ( + is_server_active( + worker_capacity=None, + in_flight_operations=InFlightOperations(), + ) + is False + ) class TestParameterNaming: diff --git a/tests/conftest.py b/tests/conftest.py index 3d35b357f7..400c42d8ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -794,6 +794,7 @@ def _factory( config: AutomationConfig | None = None, *, runner: SubprocessRunner | None = None, + operation_lease_channel=None, ): subprocess_runner = runner if runner is not None else MockSubprocessRunner() plugin_authority = FakePluginArtifactAuthority(fake_projected_plugin_root(tmp_path)) @@ -803,6 +804,7 @@ def _factory( runner=subprocess_runner, plugin_authority=plugin_authority, project_dir=tmp_path, + operation_lease_channel=operation_lease_channel, ) if config is None: # Generic server fixtures exercise dispatch without live OAuth authority. diff --git a/tests/core/test_operation_lease.py b/tests/core/test_operation_lease.py new file mode 100644 index 0000000000..2876559d36 --- /dev/null +++ b/tests/core/test_operation_lease.py @@ -0,0 +1,467 @@ +"""Tests for deadline-bound in-flight operation leases.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import anyio +import pytest + +import autoskillit.core.pipeline._operation_lease as lease_module +from autoskillit.core import ( + OPERATION_LEASE_FRESHNESS_SECONDS, + OPERATION_LEASE_HEARTBEAT_SECONDS, + InFlightOperations, + OperationLeaseRecord, + current_operation_lease, + operation_lease, + read_active_operation_leases, +) +from autoskillit.core.io.io import write_versioned_json + +pytestmark = [pytest.mark.layer("core"), pytest.mark.medium] + + +@pytest.mark.anyio +async def test_operation_lease_round_trip_and_cleanup(tmp_path: Path) -> None: + registry = InFlightOperations() + deadline = time.time() + 60 + + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=deadline, + registry=registry, + ) as handle: + records = read_active_operation_leases(tmp_path, now_epoch=time.time()) + lease_path = handle.path + assert len(records) == 1 + assert records[0] == handle.record + assert records[0].operation == "run_skill" + assert records[0].not_after_epoch == deadline + assert registry.active_count == 1 + assert current_operation_lease() is handle + assert lease_path is not None and lease_path.exists() + + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + assert lease_path is not None and not lease_path.exists() + assert registry.active_count == 0 + assert current_operation_lease() is None + + +def test_reader_ignores_expired_and_stale_leases(tmp_path: Path) -> None: + now = time.time() + expired = OperationLeaseRecord("expired", "tool", now - 20, now - 1) + stale = OperationLeaseRecord("stale", "tool", now - 20, now + 60) + expired_path = tmp_path / "expired.lease.json" + stale_path = tmp_path / "stale.lease.json" + write_versioned_json(expired_path, expired.to_payload(), schema_version=1) + write_versioned_json(stale_path, stale.to_payload(), schema_version=1) + old_mtime = now - OPERATION_LEASE_FRESHNESS_SECONDS - 1 + os.utime(stale_path, (old_mtime, old_mtime)) + + assert read_active_operation_leases(tmp_path, now_epoch=now) == () + + +def test_operation_lease_record_rejects_invalid_epochs() -> None: + for started_at, not_after in ( + (float("nan"), 10.0), + (10.0, float("inf")), + (10.0, 10.0), + ): + with pytest.raises(ValueError): + OperationLeaseRecord("invalid", "tool", started_at, not_after) + + +def test_reader_sorts_active_leases_by_deadline(tmp_path: Path) -> None: + now = time.time() + later = OperationLeaseRecord("later", "tool", now - 1, now + 20) + sooner = OperationLeaseRecord("sooner", "tool", now - 1, now + 10) + write_versioned_json(tmp_path / "later.lease.json", later.to_payload(), schema_version=1) + write_versioned_json(tmp_path / "sooner.lease.json", sooner.to_payload(), schema_version=1) + + assert read_active_operation_leases(tmp_path, now_epoch=now) == (sooner, later) + + +def test_reader_ignores_malformed_and_unknown_schema_files(tmp_path: Path) -> None: + (tmp_path / "garbage.lease.json").write_bytes(b"not json") + write_versioned_json( + tmp_path / "unknown.lease.json", + {"operation_id": "unknown"}, + schema_version=999, + ) + + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + +@pytest.mark.anyio +async def test_heartbeat_restores_freshness_while_body_is_active(tmp_path: Path) -> None: + registry = InFlightOperations() + + async with operation_lease( + tmp_path, + operation="long_tool", + not_after_epoch=time.time() + 60, + registry=registry, + heartbeat_interval=0.01, + ) as handle: + assert handle.path is not None + old_mtime = time.time() - OPERATION_LEASE_FRESHNESS_SECONDS - 1 + os.utime(handle.path, (old_mtime, old_mtime)) + + with anyio.fail_after(0.5): + while not read_active_operation_leases(tmp_path, now_epoch=time.time()): + await anyio.sleep(0.005) + assert registry.active_count == 1 + + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_heartbeat_never_recreates_a_missing_lease_file(tmp_path: Path) -> None: + registry = InFlightOperations() + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + heartbeat_interval=0.005, + ) as handle: + path = handle.path + assert path is not None + path.unlink() + await anyio.sleep(0.02) + assert not path.exists() + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_narrow_is_min_only_and_reader_honors_expiry(tmp_path: Path) -> None: + registry = InFlightOperations() + now = time.time() + original_deadline = now + 60 + narrowed_deadline = now + 30 + + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=original_deadline, + registry=registry, + ) as handle: + handle.narrow(now + 90) + assert handle.record.not_after_epoch == original_deadline + handle.narrow(narrowed_deadline) + assert handle.record.not_after_epoch == narrowed_deadline + assert read_active_operation_leases(tmp_path, now_epoch=now)[0].not_after_epoch == ( + narrowed_deadline + ) + + handle.narrow(now + 120) + assert handle.record.not_after_epoch == narrowed_deadline + assert read_active_operation_leases(tmp_path, now_epoch=original_deadline + 1) == () + + assert registry.active_count == 0 + + +@pytest.mark.parametrize("deadline", [float("nan"), float("inf"), float("-inf")]) +@pytest.mark.anyio +async def test_non_finite_narrow_leaves_record_and_file_unchanged( + tmp_path: Path, deadline: float +) -> None: + registry = InFlightOperations() + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ) as handle: + assert handle.path is not None + original_record = handle.record + original_bytes = handle.path.read_bytes() + with pytest.raises(ValueError): + handle.narrow(deadline) + assert handle.record is original_record + assert handle.path.read_bytes() == original_bytes + assert len(read_active_operation_leases(tmp_path, now_epoch=time.time())) == 1 + + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_narrowing_to_expired_deadline_retires_file_and_fails(tmp_path: Path) -> None: + registry = InFlightOperations() + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ) as handle: + path = handle.path + assert path is not None + with pytest.raises(ValueError): + handle.narrow(handle.record.started_at_epoch) + assert not path.exists() + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_no_channel_keeps_in_process_count_and_contextvar(tmp_path: Path) -> None: + registry = InFlightOperations() + + async with operation_lease( + None, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ) as handle: + assert handle.path is None + assert tuple(tmp_path.iterdir()) == () + assert registry.active_count == 1 + assert current_operation_lease() is handle + + assert registry.active_count == 0 + assert current_operation_lease() is None + + +@pytest.mark.anyio +async def test_concurrent_leases_are_task_local_and_counted(tmp_path: Path) -> None: + registry = InFlightOperations() + + async with operation_lease( + None, + operation="parent", + not_after_epoch=time.time() + 60, + registry=registry, + ) as parent: + entered_a = anyio.Event() + entered_b = anyio.Event() + exit_a = anyio.Event() + exit_b = anyio.Event() + exited_a = anyio.Event() + handles: list[object] = [] + + async def run_child(entered: anyio.Event, exit_event: anyio.Event, exited: anyio.Event): + async with operation_lease( + None, + operation="child", + not_after_epoch=time.time() + 60, + registry=registry, + ) as child: + handles.append(current_operation_lease()) + assert current_operation_lease() is child + entered.set() + await exit_event.wait() + exited.set() + + with anyio.fail_after(1): + async with anyio.create_task_group() as group: + group.start_soon(run_child, entered_a, exit_a, exited_a) + group.start_soon(run_child, entered_b, exit_b, anyio.Event()) + await entered_a.wait() + await entered_b.wait() + assert registry.active_count == 3 + assert current_operation_lease() is parent + exit_a.set() + await exited_a.wait() + assert registry.active_count == 2 + assert current_operation_lease() is parent + exit_b.set() + + assert len(handles) == 2 + assert registry.active_count == 1 + + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_body_exception_cleans_file_and_registry(tmp_path: Path) -> None: + registry = InFlightOperations() + + with pytest.raises(RuntimeError, match="body failure"): + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ): + raise RuntimeError("body failure") + + assert tuple(tmp_path.iterdir()) == () + assert registry.active_count == 0 + assert current_operation_lease() is None + + +@pytest.mark.anyio +async def test_cancellation_cleans_file_and_registry(tmp_path: Path) -> None: + registry = InFlightOperations() + + with anyio.move_on_after(0.02): + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + heartbeat_interval=0.001, + ): + await anyio.sleep_forever() + + assert tuple(tmp_path.iterdir()) == () + assert registry.active_count == 0 + assert current_operation_lease() is None + + +@pytest.mark.parametrize("deadline", [float("nan"), 0.0]) +@pytest.mark.anyio +async def test_invalid_or_expired_deadline_fails_before_admission( + tmp_path: Path, deadline: float +) -> None: + registry = InFlightOperations() + + with pytest.raises(ValueError): + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=deadline, + registry=registry, + ): + pytest.fail("invalid lease unexpectedly entered") + + assert registry.active_count == 0 + assert tuple(tmp_path.iterdir()) == () + + +@pytest.mark.anyio +async def test_narrow_write_failure_retires_lease_and_balances_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry = InFlightOperations() + original_writer = lease_module.write_versioned_json + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ) as handle: + path = handle.path + assert path is not None + narrowed_deadline = handle.record.not_after_epoch - 1 + + def fail_write(*args: object, **kwargs: object) -> None: + raise OSError("rewrite failed") + + monkeypatch.setattr(lease_module, "write_versioned_json", fail_write) + with pytest.raises(OSError, match="rewrite failed"): + handle.narrow(narrowed_deadline) + assert handle.record.not_after_epoch == narrowed_deadline + assert handle.path is None + assert not path.exists() + + monkeypatch.setattr(lease_module, "write_versioned_json", original_writer) + assert registry.active_count == 0 + + +@pytest.mark.anyio +async def test_write_failure_yields_fileless_lease_and_logs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry = InFlightOperations() + events: list[str] = [] + + def fail_write(*args: object, **kwargs: object) -> None: + raise PermissionError("read-only channel") + + monkeypatch.setattr(lease_module, "write_versioned_json", fail_write) + monkeypatch.setattr(lease_module.logger, "warning", lambda event, **_: events.append(event)) + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ) as handle: + assert handle.path is None + assert registry.active_count == 1 + assert tuple(tmp_path.iterdir()) == () + + assert registry.active_count == 0 + assert "operation_lease_write_failed" in events + + +@pytest.mark.anyio +async def test_secondary_cleanup_failures_preserve_body_error_and_balance_registry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry = InFlightOperations() + events: list[str] = [] + original_unlink = Path.unlink + + def fail_lease_unlink(path: Path, *args: object, **kwargs: object) -> None: + if path.name.endswith(".lease.json"): + raise OSError("unlink failed") + original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_lease_unlink) + monkeypatch.setattr(lease_module.logger, "warning", lambda event, **_: events.append(event)) + + with pytest.raises(RuntimeError, match="primary body error"): + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + heartbeat_interval=0.005, + ): + raise RuntimeError("primary body error") + + lease_file = next(tmp_path.glob("*.lease.json")) + mtime = lease_file.stat().st_mtime_ns + await anyio.sleep(0.02) + assert lease_file.stat().st_mtime_ns == mtime + assert registry.active_count == 0 + assert current_operation_lease() is None + assert "operation_lease_unlink_failed" in events + + +@pytest.mark.anyio +async def test_unexpected_heartbeat_failure_does_not_replace_body_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry = InFlightOperations() + started = anyio.Event() + events: list[str] = [] + + async def fail_heartbeat(_handle: object, _interval: float) -> None: + started.set() + raise RuntimeError("heartbeat failed") + + monkeypatch.setattr(lease_module, "_heartbeat", fail_heartbeat) + monkeypatch.setattr(lease_module.logger, "warning", lambda event, **_: events.append(event)) + + with pytest.raises(RuntimeError, match="primary body error"): + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=registry, + ): + await started.wait() + raise RuntimeError("primary body error") + + assert registry.active_count == 0 + assert current_operation_lease() is None + assert "operation_lease_heartbeat_failed" in events + + +def test_lease_freshness_allows_three_heartbeat_intervals() -> None: + assert OPERATION_LEASE_FRESHNESS_SECONDS >= 3 * OPERATION_LEASE_HEARTBEAT_SECONDS diff --git a/tests/core/test_type_constants_private_env_vars.py b/tests/core/test_type_constants_private_env_vars.py index a7b7a1bb7f..7ef1568083 100644 --- a/tests/core/test_type_constants_private_env_vars.py +++ b/tests/core/test_type_constants_private_env_vars.py @@ -17,6 +17,7 @@ def test_private_env_vars_includes_franchise_tier_vars() -> None: "AUTOSKILLIT_CAMPAIGN_STATE_PATH", "AUTOSKILLIT_PROJECT_DIR", "AUTOSKILLIT_FOOD_TRUCK_TOOL_TAGS", + "AUTOSKILLIT_OPERATION_LEASE_DIR", } assert expected <= AUTOSKILLIT_PRIVATE_ENV_VARS assert "AUTOSKILLIT_L3_TOOL_TAGS" not in AUTOSKILLIT_PRIVATE_ENV_VARS diff --git a/tests/execution/backends/test_codex_config.py b/tests/execution/backends/test_codex_config.py index fd2bff51de..d769ddbbab 100644 --- a/tests/execution/backends/test_codex_config.py +++ b/tests/execution/backends/test_codex_config.py @@ -1061,3 +1061,57 @@ def fake_runtime(*, config_path: Path, **_: object) -> None: "yield", "lock-exit", ] + + +def test_prelaunch_transaction_forwards_lease_channel_and_preserves_user_config( + tmp_path: Path, +) -> None: + import autoskillit.execution.backends._codex_prelaunch as prelaunch + from autoskillit.core import OPERATION_LEASE_DIR_ENV_VAR + + source_home = tmp_path / "source-home" + source_home.mkdir() + source_config = source_home / "config.toml" + env_vars = json.dumps(sorted(CODEX_MCP_ENV_FORWARD_VARS - {OPERATION_LEASE_DIR_ENV_VAR})) + original_source = ( + 'model = "user-selected-model"\n' + 'approval_policy = "never"\n' + "[mcp_servers.autoskillit]\n" + 'command = "autoskillit"\n' + 'args = ["serve"]\n' + f"env_vars = {env_vars}\n" + "[mcp_servers.user_owned]\n" + 'command = "user-tool"\n' + 'args = ["--keep"]\n' + ).encode() + source_config.write_bytes(original_source) + destination_home = tmp_path / "generated-home" + root = projection_shaped_hook_root(tmp_path) + + with prelaunch.codex_prelaunch_transaction( + source_codex_home=source_home, + destination_home=destination_home, + runtime_spec=CodexRuntimeSpec(), + plugin_dir=root.plugin_dir, + ) as target: + first_config = target.read_bytes() + + assert source_config.read_bytes() == original_source + parsed = tomllib.loads(first_config.decode("utf-8")) + assert OPERATION_LEASE_DIR_ENV_VAR in parsed["mcp_servers"]["autoskillit"]["env_vars"] + assert parsed["mcp_servers"]["user_owned"] == { + "command": "user-tool", + "args": ["--keep"], + } + assert parsed["model"] == "user-selected-model" + assert parsed["approval_policy"] == "never" + + with prelaunch.codex_prelaunch_transaction( + source_codex_home=source_home, + destination_home=destination_home, + runtime_spec=CodexRuntimeSpec(), + plugin_dir=root.plugin_dir, + ) as target: + assert target.read_bytes() == first_config + + assert source_config.read_bytes() == original_source diff --git a/tests/server/test_operation_lease_middleware.py b/tests/server/test_operation_lease_middleware.py new file mode 100644 index 0000000000..38f054b891 --- /dev/null +++ b/tests/server/test_operation_lease_middleware.py @@ -0,0 +1,200 @@ +"""Tests for the per-tool operation lease middleware.""" + +from __future__ import annotations + +import time +from pathlib import Path + +import pytest +from fastmcp import FastMCP +from fastmcp.client import Client +from fastmcp.exceptions import ToolError + +from autoskillit.core import read_active_operation_leases +from autoskillit.server._operation_lease_middleware import OperationLeaseMiddleware +from autoskillit.server.lifecycle import _state + +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] + +_TOOL_NAME = "operation_lease_probe" + + +def test_make_context_stores_operation_lease_channel(make_tool_ctx, tmp_path: Path) -> None: + ctx = make_tool_ctx(operation_lease_channel=tmp_path) + + assert ctx.operation_lease_channel == tmp_path + + +def _app(handler, *, completion_middleware: bool = False) -> FastMCP: + app = FastMCP("operation-lease-test") + app.add_middleware(OperationLeaseMiddleware()) + if completion_middleware: + from autoskillit.server.response._run_skill_completion import ( + RunSkillCompletionMiddleware, + ) + + app.add_middleware(RunSkillCompletionMiddleware(app)) + + @app.tool(name=_TOOL_NAME) + async def operation_lease_probe() -> str: + return await handler() + + return app + + +async def _call(app: FastMCP): + async with Client(app) as client: + return await client.call_tool(_TOOL_NAME, {}) + + +@pytest.mark.parametrize("inherited", [None, "future"]) +@pytest.mark.anyio +async def test_tool_call_writes_deadline_bounded_lease( + tool_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, inherited: str | None +) -> None: + tool_ctx.operation_lease_channel = tmp_path + monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) + inherited_epoch = time.time() + 10 + if inherited is None: + monkeypatch.delenv("AUTOSKILLIT_SESSION_DEADLINE", raising=False) + else: + monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", str(inherited_epoch)) + + observed: list[tuple[float, int]] = [] + + async def inspect_lease() -> str: + records = read_active_operation_leases(tmp_path, now_epoch=time.time()) + assert len(records) == 1 + assert records[0].operation == _TOOL_NAME + observed.append((records[0].not_after_epoch, tool_ctx.in_flight_operations.active_count)) + return "ok" + + before = time.time() + result = await _call(_app(inspect_lease)) + assert len(result.content) == 1 + assert tool_ctx.in_flight_operations.active_count == 0 + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + expected = min( + before + tool_ctx.config.run_skill.mcp_tool_timeout_sec, + inherited_epoch if inherited is not None else float("inf"), + ) + assert observed[0][1] == 1 + assert observed[0][0] == pytest.approx(expected, abs=0.1) + + +@pytest.mark.anyio +async def test_handler_failure_still_removes_lease( + tool_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tool_ctx.operation_lease_channel = tmp_path + monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) + + async def fail() -> str: + assert len(read_active_operation_leases(tmp_path, now_epoch=time.time())) == 1 + raise RuntimeError("handler failed") + + with pytest.raises(ToolError): + await _call(_app(fail)) + + assert tool_ctx.in_flight_operations.active_count == 0 + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + +@pytest.mark.anyio +async def test_no_channel_keeps_registry_active_without_file( + tool_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tool_ctx.operation_lease_channel = None + monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) + counts: list[int] = [] + + async def inspect_registry() -> str: + counts.append(tool_ctx.in_flight_operations.active_count) + return "ok" + + await _call(_app(inspect_registry)) + + assert counts == [1] + assert tool_ctx.in_flight_operations.active_count == 0 + assert tuple(tmp_path.iterdir()) == () + + +def test_registered_lease_middleware_is_outermost() -> None: + from autoskillit.server import mcp + + assert isinstance(mcp.middleware[0], OperationLeaseMiddleware) + + +@pytest.mark.anyio +async def test_completion_denial_releases_outer_lease( + tool_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.pipeline import DefaultRunSkillCompletionAuthority + + authority = DefaultRunSkillCompletionAuthority() + invocation = authority.begin( + kitchen_id="kitchen", + request_session_id="request", + tracker_order_id="order", + tracker_path="/tracker.json", + tracker_kitchen_id="kitchen", + tracker_incarnation_id="incarnation", + step_name="step", + ) + authority.draft( + invocation, + classification="success", + success=True, + result_digest="sha256:digest", + ) + tool_ctx.run_skill_completion = authority + tool_ctx.operation_lease_channel = tmp_path + monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) + called = False + + async def handler() -> str: + nonlocal called + called = True + return "unexpected" + + result = await _call(_app(handler, completion_middleware=True)) + + assert not called + assert result.isError + assert tool_ctx.in_flight_operations.active_count == 0 + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + +@pytest.mark.anyio +async def test_expired_inherited_deadline_fails_before_lease_admission( + tool_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tool_ctx.operation_lease_channel = tmp_path + monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) + monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", str(time.time() - 1)) + called = False + + async def handler() -> str: + nonlocal called + called = True + return "unexpected" + + with pytest.raises(ToolError): + await _call(_app(handler)) + + assert not called + assert tool_ctx.in_flight_operations.active_count == 0 + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () + + +@pytest.mark.parametrize("value", ["", "invalid", "nan", "inf", "-1"]) +def test_inherited_session_deadline_parser_rejects_invalid_values( + monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + from autoskillit.server.tools._execution_helpers._session_deadline import ( + inherited_session_deadline_epoch, + ) + + monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", value) + + assert inherited_session_deadline_epoch() == 0.0 diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py index ab63eadd78..5431b8e992 100644 --- a/tests/server/test_tools_execution_command.py +++ b/tests/server/test_tools_execution_command.py @@ -516,6 +516,53 @@ async def _capture_marker(marker_dir, *args, **kwargs): class TestRunSkillMcpTimeout: """run_skill wraps executor.run with anyio.fail_after(mcp_tool_timeout_sec).""" + @pytest.mark.anyio + async def test_run_skill_narrows_current_operation_lease_to_invocation_deadline( + self, tool_ctx_kitchen_open, monkeypatch + ): + import time + + from autoskillit.core import current_operation_lease, operation_lease + from autoskillit.server.tools.tools_execution import _run_skill_dispatch + + cfg = _command_config() + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + monkeypatch.delenv("AUTOSKILLIT_SESSION_DEADLINE", raising=False) + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard + + states = [] + state_type = _run_skill_dispatch._RunSkillDispatchState + + def capture_state(*args, **kwargs): + state = state_type(*args, **kwargs) + states.append(state) + return state + + monkeypatch.setattr(_run_skill_dispatch, "_RunSkillDispatchState", capture_state) + observed = {} + + async def capture_run(*args, **kwargs): + lease = current_operation_lease() + assert lease is not None + observed["lease_deadline"] = lease.record.not_after_epoch + observed["invocation_deadline"] = states[-1]._invocation_deadline_epoch + raise TimeoutError("stop after lease deadline inspection") + + monkeypatch.setattr(tool_ctx_kitchen_open.executor, "run", capture_run) + + async with operation_lease( + None, + operation="run_skill", + not_after_epoch=time.time() + 100_000, + registry=tool_ctx_kitchen_open.in_flight_operations, + ): + result_json = await run_skill("/investigate lease deadline", "/tmp") + + assert observed["lease_deadline"] == observed["invocation_deadline"] + assert tool_ctx_kitchen_open.in_flight_operations.active_count == 0 + assert json.loads(result_json)["subtype"] == "crashed" + @pytest.mark.anyio async def test_run_skill_returns_crashed_on_mcp_timeout( self, tool_ctx_kitchen_open, monkeypatch From af2ff488f95075e03bd5f7f640af0b0103627b8b Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 19:19:45 -0700 Subject: [PATCH 02/37] fix: integrate operation leases with architecture and isolated fixtures --- scripts/generate_core_stub.py | 2 +- src/autoskillit/core/__init__.pyi | 24 +++++++++---------- src/autoskillit/core/pipeline/__init__.py | 20 ++++++++-------- .../core/pipeline/_step_context.py | 10 +++++++- .../{pipeline => plugins}/_operation_lease.py | 11 ++++----- src/autoskillit/server/__init__.py | 4 ++-- .../server/_operation_lease_middleware.py | 2 +- .../_execution_helpers/_dispatch_metadata.py | 15 ++++++++++++ .../_execution_helpers/_session_deadline.py | 22 ----------------- .../tools_execution/_run_skill_dispatch.py | 2 +- tests/_ambient_env_surface.py | 11 ++++++++- tests/_test_filter.py | 1 + .../test_subpackage_isolation_tool_context.py | 1 + tests/arch/test_subpackage_structure.py | 4 ++-- .../core/test_grab_bag_split_completeness.py | 1 + tests/core/test_operation_lease.py | 2 +- tests/pipeline/test_context.py | 6 +++-- .../server/test_operation_lease_middleware.py | 6 ++--- tests/server/test_tools_execution_command.py | 1 - tests/test_test_filter_core_cascade.py | 1 + 20 files changed, 79 insertions(+), 67 deletions(-) rename src/autoskillit/core/{pipeline => plugins}/_operation_lease.py (98%) delete mode 100644 src/autoskillit/server/tools/_execution_helpers/_session_deadline.py diff --git a/scripts/generate_core_stub.py b/scripts/generate_core_stub.py index 6ce26e686b..beb73af5ab 100644 --- a/scripts/generate_core_stub.py +++ b/scripts/generate_core_stub.py @@ -54,7 +54,7 @@ ".io", ".logging", ".pipeline._execution_marker", - ".pipeline._operation_lease", + ".plugins._operation_lease", ".pipeline._step_context", ".pipeline.pipeline_tracker", ".planset", diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index d0754441fa..0749eaeff4 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -188,18 +188,6 @@ from .logging import configure_logging as configure_logging from .logging import get_logger as get_logger from .logging import log_plugin_artifact_lifecycle as log_plugin_artifact_lifecycle from .pipeline._execution_marker import execution_marker as execution_marker -from .pipeline._operation_lease import ( - OPERATION_LEASE_FRESHNESS_SECONDS as OPERATION_LEASE_FRESHNESS_SECONDS, -) -from .pipeline._operation_lease import ( - OPERATION_LEASE_HEARTBEAT_SECONDS as OPERATION_LEASE_HEARTBEAT_SECONDS, -) -from .pipeline._operation_lease import InFlightOperations as InFlightOperations -from .pipeline._operation_lease import OperationLeaseHandle as OperationLeaseHandle -from .pipeline._operation_lease import OperationLeaseRecord as OperationLeaseRecord -from .pipeline._operation_lease import current_operation_lease as current_operation_lease -from .pipeline._operation_lease import operation_lease as operation_lease -from .pipeline._operation_lease import read_active_operation_leases as read_active_operation_leases from .pipeline._step_context import current_order_id as current_order_id from .pipeline._step_context import current_step_name as current_step_name from .pipeline.pipeline_tracker import TrackerAuthorityReadResult as TrackerAuthorityReadResult @@ -238,6 +226,18 @@ from .plugins._active_kitchens import ( sample_kitchen_process_identity as sample_kitchen_process_identity, ) from .plugins._active_kitchens import unregister_active_kitchen as unregister_active_kitchen +from .plugins._operation_lease import ( + OPERATION_LEASE_FRESHNESS_SECONDS as OPERATION_LEASE_FRESHNESS_SECONDS, +) +from .plugins._operation_lease import ( + OPERATION_LEASE_HEARTBEAT_SECONDS as OPERATION_LEASE_HEARTBEAT_SECONDS, +) +from .plugins._operation_lease import InFlightOperations as InFlightOperations +from .plugins._operation_lease import OperationLeaseHandle as OperationLeaseHandle +from .plugins._operation_lease import OperationLeaseRecord as OperationLeaseRecord +from .plugins._operation_lease import current_operation_lease as current_operation_lease +from .plugins._operation_lease import operation_lease as operation_lease +from .plugins._operation_lease import read_active_operation_leases as read_active_operation_leases from .plugins._plugin_artifact_identity import ( INSTALLED_PLUGIN_ARTIFACT_MANIFEST_FIELDS as INSTALLED_PLUGIN_ARTIFACT_MANIFEST_FIELDS, ) diff --git a/src/autoskillit/core/pipeline/__init__.py b/src/autoskillit/core/pipeline/__init__.py index be12bb3077..77ce17d62b 100644 --- a/src/autoskillit/core/pipeline/__init__.py +++ b/src/autoskillit/core/pipeline/__init__.py @@ -12,16 +12,6 @@ from __future__ import annotations from autoskillit.core.pipeline._execution_marker import execution_marker -from autoskillit.core.pipeline._operation_lease import ( - OPERATION_LEASE_FRESHNESS_SECONDS, - OPERATION_LEASE_HEARTBEAT_SECONDS, - InFlightOperations, - OperationLeaseHandle, - OperationLeaseRecord, - current_operation_lease, - operation_lease, - read_active_operation_leases, -) from autoskillit.core.pipeline._step_context import current_order_id, current_step_name from autoskillit.core.pipeline.pipeline_tracker import ( TrackerAuthorityReadResult, @@ -58,6 +48,16 @@ render_dot, render_mermaid, ) +from autoskillit.core.plugins._operation_lease import ( + OPERATION_LEASE_FRESHNESS_SECONDS, + OPERATION_LEASE_HEARTBEAT_SECONDS, + InFlightOperations, + OperationLeaseHandle, + OperationLeaseRecord, + current_operation_lease, + operation_lease, + read_active_operation_leases, +) __all__ = [ "AnalysisResult", diff --git a/src/autoskillit/core/pipeline/_step_context.py b/src/autoskillit/core/pipeline/_step_context.py index 8b420eb87b..b50a148436 100644 --- a/src/autoskillit/core/pipeline/_step_context.py +++ b/src/autoskillit/core/pipeline/_step_context.py @@ -1,4 +1,4 @@ -"""ContextVars for current pipeline step attribution. +"""ContextVars for pipeline attribution and the current MCP operation. IL-0 module — stdlib only. Read by execution/ and server/ layers at GitHub API recording time; set/reset by tools_execution.run_skill(). @@ -7,8 +7,16 @@ from __future__ import annotations from contextvars import ContextVar +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..plugins._operation_lease import OperationLeaseHandle as _OperationLeaseHandle __all__ = ["current_order_id", "current_step_name"] current_step_name: ContextVar[str] = ContextVar("current_step_name", default="") current_order_id: ContextVar[str] = ContextVar("current_order_id", default="") + +_CURRENT_LEASE: ContextVar[_OperationLeaseHandle | None] = ContextVar( + "autoskillit_current_operation_lease", default=None +) diff --git a/src/autoskillit/core/pipeline/_operation_lease.py b/src/autoskillit/core/plugins/_operation_lease.py similarity index 98% rename from src/autoskillit/core/pipeline/_operation_lease.py rename to src/autoskillit/core/plugins/_operation_lease.py index 06d31501bf..ec895b0590 100644 --- a/src/autoskillit/core/pipeline/_operation_lease.py +++ b/src/autoskillit/core/plugins/_operation_lease.py @@ -17,7 +17,7 @@ import uuid from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -from contextvars import ContextVar, Token +from contextvars import Token from dataclasses import dataclass, replace from pathlib import Path from typing import Final @@ -167,12 +167,9 @@ def narrow(self, not_after_epoch: float) -> None: raise -_CURRENT_LEASE: ContextVar[OperationLeaseHandle | None] = ContextVar( - "autoskillit_current_operation_lease", default=None -) - - def current_operation_lease() -> OperationLeaseHandle | None: + from ..pipeline._step_context import _CURRENT_LEASE # circular-break + return _CURRENT_LEASE.get() @@ -206,6 +203,8 @@ async def operation_lease( started_at_epoch=time.time(), not_after_epoch=not_after_epoch, ) + from ..pipeline._step_context import _CURRENT_LEASE # circular-break + handle = OperationLeaseHandle(record, None) registry._enter() heartbeat_task: asyncio.Task[None] | None = None diff --git a/src/autoskillit/server/__init__.py b/src/autoskillit/server/__init__.py index d7515d8e21..294ad685f0 100644 --- a/src/autoskillit/server/__init__.py +++ b/src/autoskillit/server/__init__.py @@ -181,13 +181,13 @@ # Wire-format sanitization: strip fields that trigger Claude Code #25081 # (silent full-tool-list rejection when outputSchema/annotations are present). from autoskillit.server._operation_lease_middleware import ( # noqa: E402 - OperationLeaseMiddleware, + OperationLeaseMiddleware as _OperationLeaseMiddleware, ) from autoskillit.server.response._run_skill_completion import ( # noqa: E402 RunSkillCompletionMiddleware, ) -mcp.add_middleware(OperationLeaseMiddleware()) +mcp.add_middleware(_OperationLeaseMiddleware()) mcp.add_middleware(RunSkillCompletionMiddleware(mcp)) from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware # noqa: E402 diff --git a/src/autoskillit/server/_operation_lease_middleware.py b/src/autoskillit/server/_operation_lease_middleware.py index c23d4c0c5f..1f7944bd38 100644 --- a/src/autoskillit/server/_operation_lease_middleware.py +++ b/src/autoskillit/server/_operation_lease_middleware.py @@ -9,7 +9,7 @@ from fastmcp.tools.base import ToolResult from autoskillit.core import operation_lease -from autoskillit.server.tools._execution_helpers._session_deadline import ( +from autoskillit.server.tools._execution_helpers._dispatch_metadata import ( inherited_session_deadline_epoch, ) diff --git a/src/autoskillit/server/tools/_execution_helpers/_dispatch_metadata.py b/src/autoskillit/server/tools/_execution_helpers/_dispatch_metadata.py index 6833f15999..b726a58a4f 100644 --- a/src/autoskillit/server/tools/_execution_helpers/_dispatch_metadata.py +++ b/src/autoskillit/server/tools/_execution_helpers/_dispatch_metadata.py @@ -4,6 +4,7 @@ import dataclasses import json +import math import os from collections.abc import Mapping from pathlib import Path @@ -71,6 +72,20 @@ def check_review_approach_plan_path(step_name: str, skill_command: str) -> str | ) +def inherited_session_deadline_epoch() -> float: + """Return this process's inherited session deadline, or zero when invalid.""" + value = os.environ.get("AUTOSKILLIT_SESSION_DEADLINE") + if value is None: + return 0.0 + try: + deadline = float(value) + except ValueError: + return 0.0 + if not math.isfinite(deadline) or deadline <= 0: + return 0.0 + return deadline + + def derive_run_cmd_write_prefixes() -> tuple[str, ...]: """Read allowed write prefixes from the canonical environment variables.""" multi = os.environ.get("AUTOSKILLIT_ALLOWED_WRITE_PREFIXES", "") diff --git a/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py b/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py deleted file mode 100644 index b49bc6aa58..0000000000 --- a/src/autoskillit/server/tools/_execution_helpers/_session_deadline.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Read the session deadline inherited by this process.""" - -from __future__ import annotations - -import math -import os - -__all__ = ["inherited_session_deadline_epoch"] - - -def inherited_session_deadline_epoch() -> float: - """Return this process's inherited session deadline, or zero when invalid.""" - value = os.environ.get("AUTOSKILLIT_SESSION_DEADLINE") - if value is None: - return 0.0 - try: - deadline = float(value) - except ValueError: - return 0.0 - if not math.isfinite(deadline) or deadline <= 0: - return 0.0 - return deadline diff --git a/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py b/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py index 9a3213dcca..eda77c382e 100644 --- a/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py +++ b/src/autoskillit/server/tools/tools_execution/_run_skill_dispatch.py @@ -57,7 +57,7 @@ from autoskillit.server.tools._execution_helpers import ( validate_resumed_skill_contract as _validate_resumed_skill_contract, ) -from autoskillit.server.tools._execution_helpers._session_deadline import ( +from autoskillit.server.tools._execution_helpers._dispatch_metadata import ( inherited_session_deadline_epoch, ) from autoskillit.server.tools._types import deny_envelope diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 6dad168f51..26a632a008 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -997,7 +997,7 @@ class AmbientEnvDisposition: "install/publish subprocess env; intentional wholesale forward for a maintenance-style" "operation." ), - "cli/app.py:307": ( + "cli/app.py:316": ( "Bare os.environ passed as `child_env` to the maintenance installer, which itself applies" "an explicit allowlist (build_maintenance_env) before spawning; unfiltered by design here." ), @@ -1204,6 +1204,15 @@ class AmbientEnvDisposition: "of an unrelated lookup collection; never set as a real OS environment variable." ), ), + "OPERATION_LEASE_DIR_ENV_VAR": AmbientEnvDisposition( + var="OPERATION_LEASE_DIR_ENV_VAR", + disposition="scrub", + owner="autoskillit", + justification=( + "R4 false positive: an exported constant name in __all__, " + "not an OS environment variable." + ), + ), "AUDIT_ADMISSION_AUTHORITY_PATH_ENV_VAR": AmbientEnvDisposition( var="AUDIT_ADMISSION_AUTHORITY_PATH_ENV_VAR", disposition="scrub", diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 32bbe61997..16736fb36d 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -658,6 +658,7 @@ class SourceMapRejection(enum.StrEnum): "_type_truth": frozenset({"core", "recipe", "server"}), "_step_context": frozenset({"core", "execution", "pipeline", "server"}), "_execution_marker": frozenset({"core", "execution", "fleet", "server"}), + "_operation_lease": frozenset({"core", "pipeline", "execution", "fleet", "server", "cli"}), "test_gate_command": frozenset({"core", "execution", "recipe", "server"}), "_delivery_bounds": frozenset({"core", "execution", "server"}), "_type_audit_artifact_ref": frozenset({"core", "pipeline", "recipe", "server"}), diff --git a/tests/arch/test_subpackage_isolation_tool_context.py b/tests/arch/test_subpackage_isolation_tool_context.py index 9d4efd5187..d6b591f468 100644 --- a/tests/arch/test_subpackage_isolation_tool_context.py +++ b/tests/arch/test_subpackage_isolation_tool_context.py @@ -75,6 +75,7 @@ def test_tool_context_service_fields_use_protocol_types() -> None: "temp_dir", "project_dir", "ephemeral_root", + "operation_lease_channel", "_baseline_config", "_session_config_overrides", } diff --git a/tests/arch/test_subpackage_structure.py b/tests/arch/test_subpackage_structure.py index 0a0ed06aca..71804c66ef 100644 --- a/tests/arch/test_subpackage_structure.py +++ b/tests/arch/test_subpackage_structure.py @@ -257,8 +257,8 @@ def test_type_constants_split_completeness(self): + len(retirements) + len(skill_contract) ), "Duplicate symbols across split modules" - assert len(combined) == 176, ( - f"Expected 176 symbols total, got {len(combined)} " + assert len(combined) == 177, ( + f"Expected 177 symbols total, got {len(combined)} " f"(remaining={len(remaining)}, durable_writers={len(durable_writers)}, " f"env={len(env)}, features={len(features)}, " f"registries={len(registries)}, retirements={len(retirements)}, " diff --git a/tests/core/test_grab_bag_split_completeness.py b/tests/core/test_grab_bag_split_completeness.py index dd76b50feb..86aa4c36e6 100644 --- a/tests/core/test_grab_bag_split_completeness.py +++ b/tests/core/test_grab_bag_split_completeness.py @@ -525,6 +525,7 @@ def test_no_unintended_new_test_files_under_tests_core() -> None: "test_label_lifecycle.py", "test_logging.py", "test_measure_aggregation.py", + "test_operation_lease.py", "test_parse_plan_paths.py", "test_path_containment.py", "test_paths.py", diff --git a/tests/core/test_operation_lease.py b/tests/core/test_operation_lease.py index 2876559d36..40a3504f2c 100644 --- a/tests/core/test_operation_lease.py +++ b/tests/core/test_operation_lease.py @@ -9,7 +9,7 @@ import anyio import pytest -import autoskillit.core.pipeline._operation_lease as lease_module +import autoskillit.core.plugins._operation_lease as lease_module from autoskillit.core import ( OPERATION_LEASE_FRESHNESS_SECONDS, OPERATION_LEASE_HEARTBEAT_SECONDS, diff --git a/tests/pipeline/test_context.py b/tests/pipeline/test_context.py index bf5e019b75..4c02ec05b7 100644 --- a/tests/pipeline/test_context.py +++ b/tests/pipeline/test_context.py @@ -252,9 +252,11 @@ def test_toolcontext_optional_fields_all_have_protocol_annotations() -> None: and name != "Protocol" } - # All optional service fields — exactly those declared with field(default=None) + # The supervision-channel path is data; the remaining optional fields are services. optional_fields = { - name: f for name, f in ToolContext.__dataclass_fields__.items() if f.default is None + name: f + for name, f in ToolContext.__dataclass_fields__.items() + if f.default is None and name != "operation_lease_channel" } violations: list[str] = [] diff --git a/tests/server/test_operation_lease_middleware.py b/tests/server/test_operation_lease_middleware.py index 38f054b891..6522378df2 100644 --- a/tests/server/test_operation_lease_middleware.py +++ b/tests/server/test_operation_lease_middleware.py @@ -55,9 +55,7 @@ async def test_tool_call_writes_deadline_bounded_lease( tool_ctx.operation_lease_channel = tmp_path monkeypatch.setattr(_state, "_get_ctx_or_none", lambda: tool_ctx) inherited_epoch = time.time() + 10 - if inherited is None: - monkeypatch.delenv("AUTOSKILLIT_SESSION_DEADLINE", raising=False) - else: + if inherited is not None: monkeypatch.setenv("AUTOSKILLIT_SESSION_DEADLINE", str(inherited_epoch)) observed: list[tuple[float, int]] = [] @@ -191,7 +189,7 @@ async def handler() -> str: def test_inherited_session_deadline_parser_rejects_invalid_values( monkeypatch: pytest.MonkeyPatch, value: str ) -> None: - from autoskillit.server.tools._execution_helpers._session_deadline import ( + from autoskillit.server.tools._execution_helpers._dispatch_metadata import ( inherited_session_deadline_epoch, ) diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py index 5431b8e992..09c2f3d156 100644 --- a/tests/server/test_tools_execution_command.py +++ b/tests/server/test_tools_execution_command.py @@ -528,7 +528,6 @@ async def test_run_skill_narrows_current_operation_lease_to_invocation_deadline( cfg = _command_config() cfg.safety.require_dry_walkthrough = False tool_ctx_kitchen_open.config = cfg - monkeypatch.delenv("AUTOSKILLIT_SESSION_DEADLINE", raising=False) tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) # clone guard states = [] diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index 9714f4e5ba..71886eb818 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -199,6 +199,7 @@ def test_all_entries_present(self) -> None: "_type_session_invariant_admission", "_step_context", "_execution_marker", + "_operation_lease", "git_refs", "git_remote", "fs_observation", From c653f3f86e7fe58c8b5215025a22dbaa7bbaee69 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 19:40:40 -0700 Subject: [PATCH 03/37] fix: place operation leases outside default MCP middleware --- src/autoskillit/server/__init__.py | 5 ++++- tests/core/test_type_protocol_shards.py | 1 + tests/server/test_operation_lease_middleware.py | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/server/__init__.py b/src/autoskillit/server/__init__.py index 294ad685f0..7670857208 100644 --- a/src/autoskillit/server/__init__.py +++ b/src/autoskillit/server/__init__.py @@ -187,7 +187,10 @@ RunSkillCompletionMiddleware, ) -mcp.add_middleware(_OperationLeaseMiddleware()) +_lease_middleware = _OperationLeaseMiddleware() +mcp.add_middleware(_lease_middleware) +mcp.middleware.remove(_lease_middleware) +mcp.middleware.insert(0, _lease_middleware) mcp.add_middleware(RunSkillCompletionMiddleware(mcp)) from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware # noqa: E402 diff --git a/tests/core/test_type_protocol_shards.py b/tests/core/test_type_protocol_shards.py index 4e28622b07..a996ce0996 100644 --- a/tests/core/test_type_protocol_shards.py +++ b/tests/core/test_type_protocol_shards.py @@ -25,6 +25,7 @@ def test_execution_shard_all(): "CompletionRequiredResolver", "SkillProjectionPreparation", "InputContractResolver", + "InFlightOperationsProtocol", "TestRunner", "HeadlessExecutor", "LaunchAdapter", diff --git a/tests/server/test_operation_lease_middleware.py b/tests/server/test_operation_lease_middleware.py index 6522378df2..e8f2cf0418 100644 --- a/tests/server/test_operation_lease_middleware.py +++ b/tests/server/test_operation_lease_middleware.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import time from pathlib import Path @@ -114,7 +115,7 @@ async def inspect_registry() -> str: assert counts == [1] assert tool_ctx.in_flight_operations.active_count == 0 - assert tuple(tmp_path.iterdir()) == () + assert not tuple(tmp_path.rglob("*.lease.json")) def test_registered_lease_middleware_is_outermost() -> None: @@ -158,7 +159,7 @@ async def handler() -> str: result = await _call(_app(handler, completion_middleware=True)) assert not called - assert result.isError + assert json.loads(result.content[0].text)["is_error"] is True assert tool_ctx.in_flight_operations.active_count == 0 assert read_active_operation_leases(tmp_path, now_epoch=time.time()) == () From 7262d848e91089335677d43b34012c4d3c4eed54 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 22:04:37 -0700 Subject: [PATCH 04/37] feat: supervise headless attempts with operation leases --- src/autoskillit/config/_automation_config.py | 2 - src/autoskillit/config/_coherence.py | 45 +- src/autoskillit/config/settings.py | 12 - .../types/constants/_type_constants_env.py | 6 + .../core/types/execution/_type_backend.py | 1 - .../core/types/execution/_type_subprocess.py | 4 + .../core/types/launch/_type_launch.py | 7 +- .../types/launch/_type_launch_projection.py | 1 - .../backends/_codex/session_commands.py | 25 - .../execution/headless/_headless_launch.py | 104 ++-- .../execution/headless/_managed/_attempt.py | 26 +- .../headless/_managed/_food_truck_executor.py | 15 +- .../headless/_managed/_launch_adapter.py | 1 - src/autoskillit/execution/process/__init__.py | 4 + .../execution/process/_lifecycle/runner.py | 2 + .../execution/process/_process_monitor.py | 84 +++- .../execution/process/_process_race.py | 3 + .../execution/process/_race_watchers.py | 57 ++- .../execution/process/_termination.py | 12 +- .../execution/recording/recording.py | 8 + tests/_ambient_env_surface.py | 13 +- .../test_codex_mcp_server_env_contract.py | 74 +++ tests/config/test_timeout_coherence.py | 52 +- .../test_config_decomposition_identity.py | 14 +- .../test_config_decomposition_surface.py | 4 - tests/contracts/test_protocol_satisfaction.py | 16 + tests/core/test_backend_dataclasses.py | 8 - .../test_type_constants_private_env_vars.py | 1 + tests/core/test_types_protocols.py | 27 + .../execution/backends/test_codex_backend.py | 42 +- tests/execution/test_headless_execute.py | 147 +++--- .../test_headless_provider_fallback.py | 4 +- tests/execution/test_launch_resolution.py | 13 +- .../test_operation_lease_watchers.py | 468 ++++++++++++++++++ tests/execution/test_process_run.py | 156 ++++++ tests/execution/test_recording.py | 41 ++ tests/execution/test_termination_executor.py | 12 +- tests/fakes.py | 3 + tests/fleet/test_fleet_e2e.py | 43 ++ tests/fleet/test_fleet_e2e_codex.py | 40 ++ tests/infra/test_plugin_source_ratchets.py | 5 + 41 files changed, 1249 insertions(+), 353 deletions(-) create mode 100644 tests/execution/test_operation_lease_watchers.py diff --git a/src/autoskillit/config/_automation_config.py b/src/autoskillit/config/_automation_config.py index 1a0477450d..149e067eae 100644 --- a/src/autoskillit/config/_automation_config.py +++ b/src/autoskillit/config/_automation_config.py @@ -31,7 +31,6 @@ _claude_mcp_timeout_coherence_gate, _codex_mcp_timeout_coherence_gate, _process_tether_coherence_gate, - _timeout_coherence_gate, ) from autoskillit.config._dataclasses_diagnostics import ( DiagnosticsConfig, @@ -307,7 +306,6 @@ def sec(name: str) -> dict[str, Any]: raise ValueError( "providers.execution_candidates requires the providers feature to be enabled" ) - _timeout_coherence_gate(result.run_skill) _codex_mcp_timeout_coherence_gate( result.run_skill, result.fleet, tool_timeout=result.run_skill.mcp_tool_timeout_sec ) diff --git a/src/autoskillit/config/_coherence.py b/src/autoskillit/config/_coherence.py index d073916eeb..f67464fceb 100644 --- a/src/autoskillit/config/_coherence.py +++ b/src/autoskillit/config/_coherence.py @@ -1,11 +1,7 @@ """Timeout-coherence gates for cross-cutting runtime configuration. -Owns: ``_timeout_coherence_gate`` (idle_output_timeout vs known long-poll tools), -``_codex_mcp_timeout_coherence_gate`` and ``_claude_mcp_timeout_coherence_gate`` -(MCP tool-timeout vs session-max), ``_process_tether_coherence_gate`` (orphan -ceiling vs session-max), the helper ``compute_codex_mcp_tool_timeout``, and the -shared time-limit constants ``_MERGE_QUEUE_DEFAULT``, ``_MERGE_QUEUE_RECIPE_MAX``, -``_CI_WATCH_DEFAULT``. +Owns the MCP tool-timeout and process-tether coherence gates plus the helper +``compute_codex_mcp_tool_timeout``. The gates are warning-only: existing configs continue to work even when the gate fires. The implementation simply emits a structlog ``logger.warning`` so on-call @@ -22,43 +18,6 @@ logger = get_logger(__name__) -# Known tool timeouts for coherence validation. -# These are the maximum observed blocking durations for tools that may produce -# zero stdout during execution — used to validate idle_output_timeout coherence. -_MERGE_QUEUE_DEFAULT = 600 -_MERGE_QUEUE_RECIPE_MAX = 900 -_CI_WATCH_DEFAULT = 300 - - -def _timeout_coherence_gate(run_skill: RunSkillConfig) -> None: - """Warn when idle_output_timeout is too low relative to known long-polling tool durations. - - The idle stall watchdog monitors raw stdout byte growth with no awareness of MCP tool - execution state. When idle_output_timeout <= a known tool's max duration, the watchdog - can fire and kill legitimate sessions that are simply waiting on a long poll. - - This is a WARNING-only gate — existing configs continue working. - """ - idle = run_skill.idle_output_timeout - if idle == 0: - return - if idle <= _MERGE_QUEUE_RECIPE_MAX: - logger.warning( - "idle_output_timeout_coherence", - idle_output_timeout=idle, - merge_queue_recipe_max=_MERGE_QUEUE_RECIPE_MAX, - merge_queue_default=_MERGE_QUEUE_DEFAULT, - ci_watch_default=_CI_WATCH_DEFAULT, - message=( - f"idle_output_timeout={idle}s is at or below the maximum known blocking tool " - f"duration ({_MERGE_QUEUE_RECIPE_MAX}s for wait_for_merge_queue recipe override). " - f"This creates a race condition where the idle stall watchdog fires before the " - f"long-polling tool returns. Consider raising idle_output_timeout to at least " - f"{_MERGE_QUEUE_RECIPE_MAX + 100}s, or set it to 0 to disable the watchdog " - f"for L2 food truck sessions." - ), - ) - def compute_codex_mcp_tool_timeout( run_skill: RunSkillConfig | None = None, diff --git a/src/autoskillit/config/settings.py b/src/autoskillit/config/settings.py index 06954a27fd..4ae954cb80 100644 --- a/src/autoskillit/config/settings.py +++ b/src/autoskillit/config/settings.py @@ -41,15 +41,6 @@ from autoskillit.config._coercion import ( _preprocess_agent_backend as _preprocess_agent_backend, ) -from autoskillit.config._coherence import ( - _CI_WATCH_DEFAULT as _CI_WATCH_DEFAULT, -) -from autoskillit.config._coherence import ( - _MERGE_QUEUE_DEFAULT as _MERGE_QUEUE_DEFAULT, -) -from autoskillit.config._coherence import ( - _MERGE_QUEUE_RECIPE_MAX as _MERGE_QUEUE_RECIPE_MAX, -) from autoskillit.config._coherence import ( _claude_mcp_timeout_coherence_gate as _claude_mcp_timeout_coherence_gate, ) @@ -59,9 +50,6 @@ from autoskillit.config._coherence import ( _process_tether_coherence_gate as _process_tether_coherence_gate, ) -from autoskillit.config._coherence import ( - _timeout_coherence_gate as _timeout_coherence_gate, -) from autoskillit.config._coherence import ( compute_codex_mcp_tool_timeout as compute_codex_mcp_tool_timeout, ) diff --git a/src/autoskillit/core/types/constants/_type_constants_env.py b/src/autoskillit/core/types/constants/_type_constants_env.py index e9b6778c10..c9408f0759 100644 --- a/src/autoskillit/core/types/constants/_type_constants_env.py +++ b/src/autoskillit/core/types/constants/_type_constants_env.py @@ -211,6 +211,12 @@ def managed_join_parent_id() -> str: "AUTOSKILLIT_CWD", "MAX_MCP_OUTPUT_TOKENS", "AUTOSKILLIT_SESSION_DEADLINE", + # L2 producer: _merge_food_truck_extras; L1 consumer: _attempt; lifetime: one L2 session. + AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, + # Backend launch producer; kitchen factory consumer; lifetime: one headless session. + "AUTOSKILLIT_CHILD_OUTCOME_LOG_DIR", + # Campaign session shape producer; fleet tool consumer; lifetime: one campaign session. + "AUTOSKILLIT_CONTINUE_ON_FAILURE", # Hook-script-only escape hatch (src/autoskillit/hooks/_runtime/_hook_settings.py) — # read directly via os.environ, bypassing Dynaconf/_CONFIG_SCHEMA entirely. # No quota_guard.disabled field exists; adding this name here (rather than diff --git a/src/autoskillit/core/types/execution/_type_backend.py b/src/autoskillit/core/types/execution/_type_backend.py index 295ceed5ff..f296410c05 100644 --- a/src/autoskillit/core/types/execution/_type_backend.py +++ b/src/autoskillit/core/types/execution/_type_backend.py @@ -632,7 +632,6 @@ class CmdSpec: cwd: str = "" origin: CmdOrigin | None = None is_resume: bool = False - process_idle_timeout_ms: int = 0 inherited_fds: tuple[int, ...] = () managed_skill_catalog: ValidatedAddDir | None = None projected_skill_entries: tuple[tuple[str, str], ...] = () diff --git a/src/autoskillit/core/types/execution/_type_subprocess.py b/src/autoskillit/core/types/execution/_type_subprocess.py index 8508e904a0..619f277a13 100644 --- a/src/autoskillit/core/types/execution/_type_subprocess.py +++ b/src/autoskillit/core/types/execution/_type_subprocess.py @@ -241,6 +241,9 @@ class SubprocessRunner(Protocol): the originating session. Threaded from fleet dispatch / run_skill through headless execution to ``_session_log_monitor``'s ``caller_session_id`` parameter. Default ``None`` (match any marker). + operation_lease_dir : Path | None + Per-attempt directory containing the supervised child's in-flight tool leases. + The channel is shared with the child through its environment. max_combined_output_bytes : int | None Combined stdout/stderr byte ceiling. When set, managed capture terminates the owned process as soon as the aggregate output exceeds this value. @@ -271,6 +274,7 @@ def __call__( max_extension_seconds: float = 7200, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, stream_parser: Any | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), session_record_types: frozenset[str] = frozenset({"assistant"}), diff --git a/src/autoskillit/core/types/launch/_type_launch.py b/src/autoskillit/core/types/launch/_type_launch.py index cae39b9312..2e452bca41 100644 --- a/src/autoskillit/core/types/launch/_type_launch.py +++ b/src/autoskillit/core/types/launch/_type_launch.py @@ -52,7 +52,7 @@ ] -LAUNCH_CONTRACT_SCHEMA_VERSION = 4 +LAUNCH_CONTRACT_SCHEMA_VERSION = 5 # Top-level field order is part of the persisted stable-digest schema. Runtime # observations, attempt counters, retry state, and resume state do not belong here. @@ -424,7 +424,6 @@ def __post_init__(self) -> None: "command": { "argv": self.cmd_spec.cmd, "origin": origin_payload, - "process_idle_timeout_ms": self.cmd_spec.process_idle_timeout_ms, "force_inactive_agent_teams": self.cmd_spec.force_inactive_agent_teams, "sandbox_mode": self.sandbox_mode, "network_access": self.network_access, @@ -648,10 +647,6 @@ def from_payload( env=nonsecret_env, cwd=cwd, origin=origin, - process_idle_timeout_ms=_require_payload_int( - command["process_idle_timeout_ms"], - "command process idle timeout", - ), force_inactive_agent_teams=bool(command["force_inactive_agent_teams"]), ), sandbox_mode=str(command["sandbox_mode"]), diff --git a/src/autoskillit/core/types/launch/_type_launch_projection.py b/src/autoskillit/core/types/launch/_type_launch_projection.py index e032e58c11..0a3c0a710e 100644 --- a/src/autoskillit/core/types/launch/_type_launch_projection.py +++ b/src/autoskillit/core/types/launch/_type_launch_projection.py @@ -346,7 +346,6 @@ def bind_launch( else None ), "nonsecret_env": cmd_spec.env, - "process_idle_timeout_ms": cmd_spec.process_idle_timeout_ms, } command_digest = sha256( json.dumps( diff --git a/src/autoskillit/execution/backends/_codex/session_commands.py b/src/autoskillit/execution/backends/_codex/session_commands.py index 217fb7a020..4804c8305d 100644 --- a/src/autoskillit/execution/backends/_codex/session_commands.py +++ b/src/autoskillit/execution/backends/_codex/session_commands.py @@ -12,7 +12,6 @@ AGENT_BACKEND_CODEX, AGENT_BACKEND_DYNACONF_ENV_VAR, AGENT_BACKEND_ENV_VAR, - AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, AUTOSKILLIT_INSTALLED_VERSION, AUTOSKILLIT_STATE_ROOT_ENV_VAR, CODEX_INTERACTIVE_REQUIRED_ENV, @@ -132,8 +131,6 @@ def _prepare_skill_session_environment( cwd: str, completion_marker: str, add_dirs: Sequence[ValidatedAddDir], - exit_after_stop_delay_ms: int, - stream_idle_timeout_ms: int, scenario_step_name: str, child_outcome_log_dir: str, allowed_write_prefix: str, @@ -176,14 +173,6 @@ def _prepare_skill_session_environment( managed_catalog = add_dirs[0] for reserved_key in CODEX_RESERVED_HOME_ENV_VARS: extras[reserved_key] = managed_catalog.session_home - if exit_after_stop_delay_ms: - extras.setdefault( - AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, str(exit_after_stop_delay_ms / 1000) - ) - if stream_idle_timeout_ms: - extras.setdefault( - AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, str(stream_idle_timeout_ms / 1000) - ) filtered_base = {k: v for k, v in os.environ.items() if k not in _HEADLESS_EXCLUSIVE_VARS} env = CodexEnvPolicy().build_env( filtered_base, @@ -276,8 +265,6 @@ def build_skill_session_cmd( plugin_binding = cfg["plugin_binding"] output_format = cfg["output_format"] add_dirs = cfg["add_dirs"] - exit_after_stop_delay_ms = cfg["exit_after_stop_delay_ms"] - stream_idle_timeout_ms = cfg["stream_idle_timeout_ms"] scenario_step_name = cfg["scenario_step_name"] child_outcome_log_dir = cfg["child_outcome_log_dir"] temp_dir_relpath = cfg["temp_dir_relpath"] @@ -342,8 +329,6 @@ def build_skill_session_cmd( cwd=cwd, completion_marker=completion_marker, add_dirs=add_dirs, - exit_after_stop_delay_ms=exit_after_stop_delay_ms, - stream_idle_timeout_ms=stream_idle_timeout_ms, scenario_step_name=scenario_step_name, child_outcome_log_dir=child_outcome_log_dir, allowed_write_prefix=allowed_write_prefix, @@ -402,7 +387,6 @@ def build_skill_session_cmd( env=env, cwd=cwd, is_resume=bool(resume_session_id), - process_idle_timeout_ms=stream_idle_timeout_ms, inherited_fds=plugin_binding.inherited_fds if plugin_binding is not None else (), managed_skill_catalog=managed_catalog, app_server_plan=app_server_plan, @@ -507,14 +491,6 @@ def build_food_truck_cmd( ) for reserved_key in CODEX_RESERVED_HOME_ENV_VARS: extras[reserved_key] = session_home - if exit_after_stop_delay_ms: - extras.setdefault( - AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, str(exit_after_stop_delay_ms / 1000) - ) - if stream_idle_timeout_ms: - extras.setdefault( - AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, str(stream_idle_timeout_ms / 1000) - ) filtered_base = {k: v for k, v in os.environ.items() if k not in _HEADLESS_EXCLUSIVE_VARS} env = CodexEnvPolicy().build_env( filtered_base, @@ -569,7 +545,6 @@ def build_food_truck_cmd( env=env, cwd=cwd, is_resume=bool(resume_session_id), - process_idle_timeout_ms=stream_idle_timeout_ms, inherited_fds=plugin_binding.inherited_fds if plugin_binding is not None else (), managed_skill_catalog=managed_skill_catalog, app_server_plan=app_server_plan, diff --git a/src/autoskillit/execution/headless/_headless_launch.py b/src/autoskillit/execution/headless/_headless_launch.py index 042026eaa0..06d4ebe736 100644 --- a/src/autoskillit/execution/headless/_headless_launch.py +++ b/src/autoskillit/execution/headless/_headless_launch.py @@ -42,7 +42,10 @@ _BuildSpec, _ManagedLineageObserver, ) -from autoskillit.execution.headless._managed._attempt import _generated_home_attempt +from autoskillit.execution.headless._managed._attempt import ( + _generated_home_attempt, + _operation_lease_channel, +) from autoskillit.execution.headless._managed._launch_adapter import ( _binding_identity, _HeadlessLaunchAdapter, @@ -166,11 +169,6 @@ async def _run_headless_attempt( adapter.secret_environment, inherited_fds=adapter.inherited_fds, ) - effective_idle = idle_output_timeout - if spec.process_idle_timeout_ms > 0: - spec_idle = spec.process_idle_timeout_ms / 1000.0 - if effective_idle is None or spec_idle < effective_idle: - effective_idle = spec_idle with _generated_home_attempt( backend, spec, @@ -179,52 +177,56 @@ async def _run_headless_attempt( attempt=attempt, resume_session_id=backend_resume_session_id, ) as handle: - if handle is not None: - spec = dataclasses.replace( - spec, - inherited_fds=tuple(dict.fromkeys((*spec.inherited_fds, *handle.pass_fds))), + with _operation_lease_channel(spec) as (spec, channel): + if handle is not None: + spec = dataclasses.replace( + spec, + inherited_fds=tuple( + dict.fromkeys((*spec.inherited_fds, *handle.pass_fds)) + ), + ) + if on_spec_built is not None: + on_spec_built(spec) + if mark_execution_started is not None: + mark_execution_started() + result = await runner( + list(spec.cmd), + cwd=Path(spec.cwd), + timeout=timeout, + env=spec.env, + pty_mode=( + pty_override if pty_override is not None else _resolve_pty_mode(backend) + ), + session_log_dir=_resolve_session_log_dir(spec.cwd, backend), + completion_marker=completion_marker, + stale_threshold=stale_threshold, + completion_drain_timeout=completion_drain_timeout, + natural_exit_grace_seconds=natural_exit_grace_seconds, + linux_tracing_config=linux_tracing_config, + idle_output_timeout=idle_output_timeout, + max_suppression_seconds=max_suppression_seconds, + child_deferral_ceiling=child_deferral_ceiling, + on_pid_resolved=on_spawn, + on_process_spawned=handle.record_spawn if handle is not None else None, + on_process_reaped=handle.record_reaped if handle is not None else None, + enable_deadline_extension=enable_deadline_extension, + max_extension_seconds=max_extension_seconds, + ceiling_seconds=ceiling_seconds, + systemd_scope_enabled=systemd_scope_enabled, + marker_dir=marker_dir, + session_id=session_id, + operation_lease_dir=channel, + on_session_id_resolved=on_session_id_resolved, + stream_parser=stream_parser, + completion_record_types=backend.capabilities.completion_record_types, + session_record_types=backend.capabilities.session_record_types, + inspector_callback=None, + workload_basenames=backend.capabilities.process_name_aliases or None, + pass_fds=spec.inherited_fds, + backend_resume_session_id=backend_resume_session_id, + line_driver=backend.line_driver(spec), + lifecycle_observation_enabled=lifecycle_observation_enabled, ) - if on_spec_built is not None: - on_spec_built(spec) - if mark_execution_started is not None: - mark_execution_started() - result = await runner( - list(spec.cmd), - cwd=Path(spec.cwd), - timeout=timeout, - env=spec.env, - pty_mode=( - pty_override if pty_override is not None else _resolve_pty_mode(backend) - ), - session_log_dir=_resolve_session_log_dir(spec.cwd, backend), - completion_marker=completion_marker, - stale_threshold=stale_threshold, - completion_drain_timeout=completion_drain_timeout, - natural_exit_grace_seconds=natural_exit_grace_seconds, - linux_tracing_config=linux_tracing_config, - idle_output_timeout=effective_idle, - max_suppression_seconds=max_suppression_seconds, - child_deferral_ceiling=child_deferral_ceiling, - on_pid_resolved=on_spawn, - on_process_spawned=handle.record_spawn if handle is not None else None, - on_process_reaped=handle.record_reaped if handle is not None else None, - enable_deadline_extension=enable_deadline_extension, - max_extension_seconds=max_extension_seconds, - ceiling_seconds=ceiling_seconds, - systemd_scope_enabled=systemd_scope_enabled, - marker_dir=marker_dir, - session_id=session_id, - on_session_id_resolved=on_session_id_resolved, - stream_parser=stream_parser, - completion_record_types=backend.capabilities.completion_record_types, - session_record_types=backend.capabilities.session_record_types, - inspector_callback=None, - workload_basenames=backend.capabilities.process_name_aliases or None, - pass_fds=spec.inherited_fds, - backend_resume_session_id=backend_resume_session_id, - line_driver=backend.line_driver(spec), - lifecycle_observation_enabled=lifecycle_observation_enabled, - ) return result, spec diff --git a/src/autoskillit/execution/headless/_managed/_attempt.py b/src/autoskillit/execution/headless/_managed/_attempt.py index ce55c18add..90edff319a 100644 --- a/src/autoskillit/execution/headless/_managed/_attempt.py +++ b/src/autoskillit/execution/headless/_managed/_attempt.py @@ -4,14 +4,16 @@ import dataclasses import os -from collections.abc import Callable, Mapping, Sequence -from contextlib import AbstractContextManager, nullcontext +import shutil +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager, contextmanager, nullcontext from pathlib import Path from typing import Any, cast from uuid import uuid4 from autoskillit.core import ( AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, + OPERATION_LEASE_DIR_ENV_VAR, SESSION_TYPE_ENV_VAR, BackendCapabilities, CmdSpec, @@ -35,6 +37,7 @@ SkillResult, SubprocessResult, ValidatedAddDir, + default_log_dir, get_logger, new_managed_attempt_id, ) @@ -72,6 +75,25 @@ def _bind_effective_execution_identity( return dataclasses.replace(skill_result, execution_identity=effective) +@contextmanager +def _operation_lease_channel(spec: CmdSpec) -> Iterator[tuple[CmdSpec, Path]]: + """Bind one spec to its own absolute supervision channel for this attempt. + + A killed supervisor can leave an orphan directory. Later attempts neither + reuse that directory nor scan it for liveness. + """ + path = default_log_dir().resolve() / "operation-leases" / uuid4().hex + path.mkdir(parents=True, exist_ok=False) + try: + values = {**spec.env, OPERATION_LEASE_DIR_ENV_VAR: str(path)} + env: Mapping[str, str] = values + if not isinstance(spec.env, dict): + env = cast(Any, type(spec.env))(values) + yield dataclasses.replace(spec, env=env), path + finally: + shutil.rmtree(path, ignore_errors=True) + + def _generated_home_attempt( backend: CodingAgentBackend, spec: CmdSpec, diff --git a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py index 7c2107c88a..0c9989c86e 100644 --- a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py +++ b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py @@ -72,7 +72,6 @@ def _merge_food_truck_extras( *, env_extras: Mapping[str, str] | None, requires_packs: Sequence[str], - idle_output_timeout: float | None, fleet_idle_output_timeout: float, run_skill_idle_output_timeout: float, ) -> dict[str, str]: @@ -85,9 +84,7 @@ def _merge_food_truck_extras( f"{FOOD_TRUCK_TOOL_TAGS_ENV_VAR} — use requires_packs exclusively" ) merged_extras[FOOD_TRUCK_TOOL_TAGS_ENV_VAR] = ",".join(sorted(requires_packs)) - if idle_output_timeout is not None: - merged_extras[AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR] = str(idle_output_timeout) - elif fleet_idle_output_timeout > 0: + if fleet_idle_output_timeout > 0: merged_extras.setdefault( AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT_ENV_VAR, str(fleet_idle_output_timeout) ) @@ -266,7 +263,6 @@ async def dispatch_food_truck( merged_extras = _merge_food_truck_extras( env_extras=env_extras, requires_packs=requires_packs, - idle_output_timeout=idle_output_timeout, fleet_idle_output_timeout=fleet_cfg.idle_output_timeout, run_skill_idle_output_timeout=cfg.run_skill.idle_output_timeout, ) @@ -407,13 +403,10 @@ async def dispatch_food_truck( effective_max_ext = float(fleet_cfg.max_extension_seconds) effective_ceiling_seconds = float(cfg.process_tether.orphan_ceiling_seconds) effective_systemd_scope_enabled = cfg.process_tether.systemd_scope_enabled - effective_idle_out: float | None = ( - idle_output_timeout - if idle_output_timeout is not None - else float(fleet_idle) - if fleet_idle > 0 - else None + selected_idle = ( + idle_output_timeout if idle_output_timeout is not None else float(fleet_idle) ) + effective_idle_out = max(float(selected_idle), float(cfg.run_skill.timeout)) effective_natural_exit_grace_seconds: float = cfg.run_skill.natural_exit_grace_seconds effective_marker_dir: Path | None = marker_dir or ( headless_facade._resolve_session_log_dir( diff --git a/src/autoskillit/execution/headless/_managed/_launch_adapter.py b/src/autoskillit/execution/headless/_managed/_launch_adapter.py index 7d625c27d7..a553aa4be3 100644 --- a/src/autoskillit/execution/headless/_managed/_launch_adapter.py +++ b/src/autoskillit/execution/headless/_managed/_launch_adapter.py @@ -121,7 +121,6 @@ def build(self, preparation: LaunchPreparation) -> LaunchAdapterResult: "origin": repr(spec.origin), "nonsecret_env": nonsecret_environment, "secret_keys": secret_keys, - "process_idle_timeout_ms": spec.process_idle_timeout_ms, "inherited_fd_count": len(spec.inherited_fds), "force_inactive_agent_teams": self._force_inactive_agent_teams, "app_server_plan": _app_server_plan_digest_payload(spec.app_server_plan), diff --git a/src/autoskillit/execution/process/__init__.py b/src/autoskillit/execution/process/__init__.py index 46a5730980..64e42228aa 100644 --- a/src/autoskillit/execution/process/__init__.py +++ b/src/autoskillit/execution/process/__init__.py @@ -267,6 +267,7 @@ async def run_managed_async( _session_id_timeout: float = 1.0, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, inspector_callback: InspectorCallback | None = None, workload_basenames: frozenset[str] | None = None, @@ -530,6 +531,7 @@ async def run_managed_async( max_suppression_seconds=max_suppression_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, on_session_id_resolved=on_session_id_resolved, backend_resume_session_id=backend_resume_session_id, channel_b_selected=channel_b_selected, @@ -560,6 +562,7 @@ async def run_managed_async( max_extension_seconds=max_extension_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, lifecycle_observation_enabled=lifecycle_observation_enabled, acc=acc, session_log_dir=session_log_dir, @@ -657,6 +660,7 @@ def _observe_final_channel_b(event: SessionEvent) -> None: pid=_observed_pid, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, child_deferral_ceiling=child_deferral_ceiling, process_observation_snapshot=signals.process_observation_snapshot, ) diff --git a/src/autoskillit/execution/process/_lifecycle/runner.py b/src/autoskillit/execution/process/_lifecycle/runner.py index 6d0530577b..c2c9ba484b 100644 --- a/src/autoskillit/execution/process/_lifecycle/runner.py +++ b/src/autoskillit/execution/process/_lifecycle/runner.py @@ -158,6 +158,7 @@ async def __call__( max_extension_seconds: float = 7200, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), session_record_types: frozenset[str] = frozenset({"assistant"}), @@ -199,6 +200,7 @@ async def __call__( child_deferral_ceiling=child_deferral_ceiling, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, session_record_types=session_record_types, diff --git a/src/autoskillit/execution/process/_process_monitor.py b/src/autoskillit/execution/process/_process_monitor.py index 0c52c9ef2e..57ecaecd4f 100644 --- a/src/autoskillit/execution/process/_process_monitor.py +++ b/src/autoskillit/execution/process/_process_monitor.py @@ -10,7 +10,13 @@ import anyio import psutil -from autoskillit.core import VANISHED_ERRORS, ChannelBStatus, get_logger, scan_observed +from autoskillit.core import ( + VANISHED_ERRORS, + ChannelBStatus, + get_logger, + read_active_operation_leases, + scan_observed, +) from autoskillit.execution.process._process_jsonl import ( EventCursor, _jsonl_contains_marker, @@ -24,6 +30,49 @@ logger = get_logger(__name__) +class _OperationLeaseActivity: + """Sample operation evidence, refresh its clock, and throttle deferred logs.""" + + def __init__( + self, + channel: Path | None, + event: str, + log: Callable[..., None], + ) -> None: + self._channel = channel + self._event = event + self._log = log + self._signature: tuple[tuple[str, float], ...] = () + self._logged_at: float | None = None + + def refresh( + self, + now: float, + reference: float, + suppression_start: float | None, + *, + silence_seconds: float, + ) -> tuple[float, float | None]: + leases = ( + read_active_operation_leases(self._channel, now_epoch=time.time()) + if self._channel is not None + else () + ) + if not leases: + return reference, suppression_start + signature = tuple((lease.operation_id, lease.not_after_epoch) for lease in leases) + if signature != self._signature or self._logged_at is None or now - self._logged_at >= 300: + self._log( + self._event, + operation=leases[0].operation, + not_after_epoch=leases[0].not_after_epoch, + silence_seconds=silence_seconds, + ) + self._signature = signature + self._logged_at = now + return now, None + + class SessionMonitorResult(NamedTuple): """Result from _session_log_monitor with discovered session identity.""" @@ -240,12 +289,20 @@ async def _discover_session_log( phase1_timeout: float, expected_session_id: str | None, resume_cursor: EventCursor | None, + operation_lease_dir: Path | None = None, ) -> Path | SessionMonitorResult: """Find the session JSONL file or report why discovery cannot continue.""" session_file = None os_error_count = 0 phase1_start = time.monotonic() while session_file is None: + operation_leases = ( + read_active_operation_leases(operation_lease_dir, now_epoch=time.time()) + if operation_lease_dir is not None + else () + ) + if operation_leases: + phase1_start = time.monotonic() if time.monotonic() - phase1_start >= phase1_timeout: logger.warning( "Session log file not found within phase1_timeout (%.1fs); treating as stale", @@ -297,10 +354,12 @@ def _active_liveness_signals( pid: int | None, marker_dir: Path | None, session_id: str | None, + *, + operation_lease_dir: Path | None = None, ) -> frozenset[str]: - """Return the subset of {'api_connection', 'child_processes', 'dispatch_marker'} - currently active — the three liveness predicates shared with _termination.py's - post-completion drain deferral. + """Return active network, child, marker, and operation-lease signals. + + The same evidence set is used by the termination drain and the session monitor. """ signals: set[str] = set() if pid is not None and _has_active_api_connection(pid): @@ -309,6 +368,10 @@ def _active_liveness_signals( signals.add("child_processes") if marker_dir is not None and _has_active_execution_marker(marker_dir, session_id=session_id): signals.add("dispatch_marker") + if operation_lease_dir is not None and read_active_operation_leases( + operation_lease_dir, now_epoch=time.time() + ): + signals.add("operation_lease") return frozenset(signals) @@ -407,6 +470,7 @@ async def _session_log_monitor( resume_cursor: EventCursor | None = None, on_session_file_selected: Callable[[Path, EventCursor], None] | None = None, has_pending_tasks: Callable[[], bool] | None = None, + operation_lease_dir: Path | None = None, ) -> SessionMonitorResult: """Watch Claude Code session log for completion or staleness. @@ -433,6 +497,7 @@ async def _session_log_monitor( _phase1_timeout, expected_session_id, resume_cursor, + operation_lease_dir, ) if isinstance(discovered, SessionMonitorResult): return discovered @@ -473,10 +538,17 @@ async def _session_log_monitor( os_error_count = 0 suppression_start: float | None = None _last_record_type: str | None = None + operation_activity = _OperationLeaseActivity( + operation_lease_dir, "stale_deferred_to_operation", logger.info + ) while True: await anyio.sleep(_phase2_poll) if _on_poll is not None: _on_poll() + now = time.monotonic() + last_change, suppression_start = operation_activity.refresh( + now, last_change, suppression_start, silence_seconds=now - last_change + ) try: current_size = session_file.stat().st_size os_error_count = 0 @@ -487,7 +559,7 @@ async def _session_log_monitor( continue if current_size > last_size: last_size = current_size - last_change = time.monotonic() + last_change = now suppression_start = None try: content = session_file.read_bytes() @@ -511,7 +583,7 @@ async def _session_log_monitor( except OSError: pass else: - elapsed = time.monotonic() - last_change + elapsed = now - last_change if elapsed >= stale_threshold: reason = _stale_suppression_reason( has_pending_tasks, pid, marker_dir, caller_session_id diff --git a/src/autoskillit/execution/process/_process_race.py b/src/autoskillit/execution/process/_process_race.py index fc445a70b9..32274c4847 100644 --- a/src/autoskillit/execution/process/_process_race.py +++ b/src/autoskillit/execution/process/_process_race.py @@ -381,6 +381,7 @@ async def _watch_session_log( on_session_id_resolved: Callable[[str], None] | None = None, backend_resume_session_id: str = "", channel_b_selected: anyio.Event | None = None, + operation_lease_dir: Path | None = None, ) -> None: """Monitor the session JSONL log and deposit the Channel B signal. @@ -416,6 +417,8 @@ def _selected(_path: Path, cursor: EventCursor) -> None: _monitor_kwargs["max_suppression_seconds"] = max_suppression_seconds if marker_dir is not None: _monitor_kwargs["marker_dir"] = marker_dir + if operation_lease_dir is not None: + _monitor_kwargs["operation_lease_dir"] = operation_lease_dir if session_id is not None: _monitor_kwargs["caller_session_id"] = session_id monitor_result = await _session_log_monitor( diff --git a/src/autoskillit/execution/process/_race_watchers.py b/src/autoskillit/execution/process/_race_watchers.py index d882bbd634..c9dee0438a 100644 --- a/src/autoskillit/execution/process/_race_watchers.py +++ b/src/autoskillit/execution/process/_race_watchers.py @@ -4,6 +4,7 @@ import functools import subprocess +import time from collections.abc import Callable from pathlib import Path from typing import IO, TYPE_CHECKING, Any @@ -11,12 +12,13 @@ import anyio import anyio.abc -from autoskillit.core import InspectorEvidence, get_logger +from autoskillit.core import InspectorEvidence, get_logger, read_active_operation_leases from autoskillit.execution.process._process_jsonl import fold_event_cursor from autoskillit.execution.process._process_monitor import ( _has_active_api_connection, _has_active_child_processes, _has_active_execution_marker, + _OperationLeaseActivity, ) from autoskillit.execution.process._process_race import RaceAccumulator @@ -53,6 +55,7 @@ async def _watch_stdout_idle( *, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, max_suppression_seconds: float = 1800.0, inspector_callback: InspectorCallback | None = None, timeout_scope_ref: list[anyio.CancelScope | None] | None = None, @@ -62,12 +65,23 @@ async def _watch_stdout_idle( import time as _time last_size: int = 0 - last_growth_time: float = _time.monotonic() + last_liveness_evidence = _time.monotonic() + last_stdout_growth = last_liveness_evidence suppression_start_marker: float | None = None + operation_activity = _OperationLeaseActivity( + operation_lease_dir, "stdout_idle_deferred_to_operation", logger.info + ) while True: await anyio.sleep(_poll_interval) if trigger.is_set(): return + now = _time.monotonic() + last_liveness_evidence, suppression_start_marker = operation_activity.refresh( + now, + last_liveness_evidence, + suppression_start_marker, + silence_seconds=now - last_stdout_growth, + ) try: current_size = stdout_path.stat().st_size except OSError: @@ -75,9 +89,10 @@ async def _watch_stdout_idle( continue if current_size > last_size: last_size = current_size - last_growth_time = _time.monotonic() + last_liveness_evidence = now + last_stdout_growth = now suppression_start_marker = None - elif _time.monotonic() - last_growth_time >= idle_output_timeout: + elif now - last_liveness_evidence >= idle_output_timeout: authoritative_task_active = has_pending_tasks is not None and has_pending_tasks() marker_active = marker_dir is not None and _has_active_execution_marker( marker_dir, session_id=session_id @@ -110,12 +125,14 @@ async def _watch_stdout_idle( ), ) logger.warning( - "stdout idle for %ss — firing IDLE_STALL", - idle_output_timeout, + "stdout_idle_stall_firing", + silence_seconds=now - last_stdout_growth, + idle_threshold=idle_output_timeout, + operation_lease_dir=str(operation_lease_dir), ) spared_at = await _inspect_stdout_idle( stdout_path, - last_growth_time, + last_liveness_evidence, acc, inspector_callback, timeout_scope_ref, @@ -123,7 +140,7 @@ async def _watch_stdout_idle( session_id=session_id, ) if spared_at is not None: - last_growth_time = spared_at + last_liveness_evidence = spared_at continue acc.idle_stall = True @@ -133,7 +150,7 @@ async def _watch_stdout_idle( async def _inspect_stdout_idle( stdout_path: Path, - last_growth_time: float, + last_liveness_evidence: float, acc: RaceAccumulator, inspector_callback: InspectorCallback | None, timeout_scope_ref: list[anyio.CancelScope | None] | None, @@ -165,7 +182,7 @@ async def _inspect_stdout_idle( ) evidence = _package_evidence( stdout_path, - idle_seconds=_time.monotonic() - last_growth_time, + idle_seconds=_time.monotonic() - last_liveness_evidence, execution_marker_present=marker_present, ) try: @@ -197,6 +214,7 @@ async def _watch_child_activity( *, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, has_pending_tasks: Callable[[], bool] | None = None, ) -> None: """Extend the wall-clock deadline while managed child activity remains active.""" @@ -214,7 +232,13 @@ async def _watch_child_activity( if _first_observed_deadline is None: _first_observed_deadline = scope.deadline - active = ( + operation_leases = ( + read_active_operation_leases(operation_lease_dir, now_epoch=time.time()) + if operation_lease_dir is not None + else () + ) + lease_active = bool(operation_leases) + active = lease_active or ( (has_pending_tasks is not None and has_pending_tasks()) or _has_active_child_processes(pid) or _has_active_api_connection(pid) @@ -312,6 +336,7 @@ def _enroll_race_watchers( max_suppression_seconds: float | None, marker_dir: Path | None, session_id: str | None, + operation_lease_dir: Path | None = None, on_session_id_resolved: Callable[[str], None] | None, backend_resume_session_id: str, channel_b_selected: anyio.Event, @@ -350,7 +375,10 @@ def _enroll_race_watchers( stdout_session_id_ready, ) tg.start_soon( - _watch_session_log, + functools.partial( + _watch_session_log, + operation_lease_dir=operation_lease_dir, + ), session_log_dir, completion_marker, stale_threshold, @@ -393,6 +421,7 @@ def _enroll_race_watchers( trigger, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, max_suppression_seconds=max_suppression_seconds or 1800.0, inspector_callback=inspector_callback, timeout_scope_ref=timeout_scope_ref, @@ -414,6 +443,7 @@ def _enroll_child_activity_watcher( trigger: anyio.Event, marker_dir: Path | None, session_id: str | None, + operation_lease_dir: Path | None = None, lifecycle_observation_enabled: bool, acc: RaceAccumulator, ) -> None: @@ -427,6 +457,7 @@ def _enroll_child_activity_watcher( trigger, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, has_pending_tasks=( acc.has_unresolved_obligations if lifecycle_observation_enabled else None ), @@ -446,6 +477,7 @@ async def _await_race_and_drain( max_extension_seconds: float, marker_dir: Path | None, session_id: str | None, + operation_lease_dir: Path | None = None, lifecycle_observation_enabled: bool, acc: RaceAccumulator, session_log_dir: Path | None, @@ -464,6 +496,7 @@ async def _await_race_and_drain( trigger=trigger, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, lifecycle_observation_enabled=lifecycle_observation_enabled, acc=acc, ) diff --git a/src/autoskillit/execution/process/_termination.py b/src/autoskillit/execution/process/_termination.py index cfa713c9e4..0d486faaca 100644 --- a/src/autoskillit/execution/process/_termination.py +++ b/src/autoskillit/execution/process/_termination.py @@ -88,6 +88,7 @@ async def _drain_before_escalation( pid: int | None, marker_dir: Path | None, session_id: str | None, + operation_lease_dir: Path | None = None, child_deferral_ceiling: float, ) -> tuple[int | None, ProcessCleanupResult] | None: with anyio.move_on_after(grace_seconds): @@ -105,7 +106,14 @@ async def _drain_before_escalation( return await anyio.to_thread.run_sync( owner.settle_evidence, abandon_on_cancel=False ) - active = bool(_active_liveness_signals(pid, marker_dir, session_id)) + active = bool( + _active_liveness_signals( + pid, + marker_dir, + session_id, + operation_lease_dir=operation_lease_dir, + ) + ) if not active: proc_log.debug("no_active_children_proceeding_to_kill") break @@ -129,6 +137,7 @@ async def execute_termination_action( pid: int | None = None, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, child_deferral_ceiling: float = 0.0, process_observation_snapshot: ProcessObservationSnapshot | None = None, ) -> tuple[KillReason, int | None, ProcessCleanupResult]: @@ -162,6 +171,7 @@ async def execute_termination_action( pid=pid, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, child_deferral_ceiling=child_deferral_ceiling, ) if settled is not None: diff --git a/src/autoskillit/execution/recording/recording.py b/src/autoskillit/execution/recording/recording.py index 1a546d7a49..8ce74fe692 100644 --- a/src/autoskillit/execution/recording/recording.py +++ b/src/autoskillit/execution/recording/recording.py @@ -145,6 +145,7 @@ async def __call__( max_extension_seconds: float = 7200, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), session_record_types: frozenset[str] = frozenset({"assistant"}), @@ -203,6 +204,7 @@ async def __call__( max_extension_seconds=max_extension_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, step_name=step_name, completion_record_types=completion_record_types, @@ -242,6 +244,7 @@ async def __call__( max_extension_seconds=max_extension_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, session_record_types=session_record_types, @@ -290,6 +293,7 @@ async def __call__( max_extension_seconds=max_extension_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, session_record_types=session_record_types, @@ -400,6 +404,7 @@ async def _record_non_pty_session( max_extension_seconds: float, marker_dir: Path | None, session_id: str | None, + operation_lease_dir: Path | None, stream_parser: StreamParser | None, step_name: str, completion_record_types: frozenset[str] = frozenset({"result"}), @@ -438,6 +443,7 @@ async def _record_non_pty_session( max_extension_seconds=max_extension_seconds, marker_dir=marker_dir, session_id=session_id, + operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, session_record_types=session_record_types, @@ -557,6 +563,7 @@ async def __call__( max_extension_seconds: float = 7200, marker_dir: Path | None = None, session_id: str | None = None, + operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), session_record_types: frozenset[str] = frozenset({"assistant"}), @@ -583,6 +590,7 @@ async def __call__( backend_resume_session_id, max_combined_output_bytes, line_driver, + operation_lease_dir, ceiling_seconds, systemd_scope_enabled, natural_exit_grace_seconds, diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 26a632a008..73ebb35898 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -1033,19 +1033,19 @@ class AmbientEnvDisposition: "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" " Codex generic-prompt launch; the finalized generated home is layered by the caller." ), - "execution/backends/_codex/session_commands.py:187": ( + "execution/backends/_codex/session_commands.py:176": ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex skill-session launch." ), - "execution/backends/_codex/session_commands.py:518": ( + "execution/backends/_codex/session_commands.py:494": ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex food-truck orchestrator-session launch." ), - "execution/backends/_codex/session_commands.py:205": ( + "execution/backends/_codex/session_commands.py:194": ( "Excludes _HEADLESS_EXCLUSIVE_VARS before extras merge for a Codex interactive-session" "launch base env." ), - "execution/backends/_codex/session_commands.py:732": ( + "execution/backends/_codex/session_commands.py:707": ( "Excludes _HEADLESS_EXCLUSIVE_VARS before build_env layers extras back in for a headless" "Codex resume-session launch." ), @@ -1527,9 +1527,8 @@ class AmbientEnvDisposition: disposition="scrub", owner="autoskillit", justification=( - "Real AutoSkillit orchestration/session-control environment variable read by" - "production code; scrubbed as internal state that must not leak across test" - "boundaries." + "Produced by the L2 food-truck attempt and consumed by its kitchen for the" + "one-session L1 idle policy; private so nested launches cannot inherit it." ), ), "AUTOSKILLIT_INSTALLED_VERSION": AmbientEnvDisposition( diff --git a/tests/arch/test_codex_mcp_server_env_contract.py b/tests/arch/test_codex_mcp_server_env_contract.py index 1d29c64ed6..e008599a8e 100644 --- a/tests/arch/test_codex_mcp_server_env_contract.py +++ b/tests/arch/test_codex_mcp_server_env_contract.py @@ -38,6 +38,57 @@ ) +_HOST_ONLY_READS: tuple[tuple[str, str, str], ...] = ( + ("cli/app.py", "AUTOSKILLIT_PROTECTED_BRANCHES", "Host CLI seeds repository hook policy."), + ( + "cli/update/_update_checks_fetch.py", + "AUTOSKILLIT_FETCH_CACHE_TTL_SECONDS", + "Operator control of the host update-check cache.", + ), + ( + "cli/update/_update_checks_source.py", + "AUTOSKILLIT_SOURCE_REPO", + "Operator selection of the local update source.", + ), + ("core/io/paths.py", "AUTOSKILLIT_LOG_DIR", "Operator override of the host log root."), + ( + "core/runtime/kitchen_state.py", + "AUTOSKILLIT_STATE_DIR", + "Override of this process's persistent state location.", + ), + ( + "core/runtime/session_provenance.py", + "AUTOSKILLIT_STATE_DIR", + "Override of this process's persistent state location.", + ), + ( + "execution/backends/_codex_recipe_delivery.py", + "AUTOSKILLIT_STATE_DIR", + "Override of the local recipe-delivery state location.", + ), + ( + "pipeline/exploration_context_durable.py", + "AUTOSKILLIT_EXPLORATION_CAPABILITY", + "Sealed native terminal authority; not an ordinary MCP server input.", + ), + ( + "pipeline/exploration_context_durable.py", + "AUTOSKILLIT_EXPLORATION_ROLE", + "Sealed native terminal authority; not an ordinary MCP server input.", + ), + ( + "pipeline/exploration_context_durable.py", + "AUTOSKILLIT_EXPLORATION_SESSION_ID", + "Sealed native terminal authority; not an ordinary MCP server input.", + ), + ( + "pipeline/exploration_context_durable.py", + "AUTOSKILLIT_EXPLORATION_AUTHORITY_PATH", + "Sealed native terminal authority; not an ordinary MCP server input.", + ), +) + + def _server_process_private_reads() -> tuple[EnvRead, ...]: return tuple( read @@ -88,6 +139,29 @@ def test_every_server_process_private_read_is_forwarded() -> None: ) +def test_every_server_autoskillit_read_is_private() -> None: + reads = tuple( + read + for read in production_env_read_surface(_PRODUCTION_SRC_ROOT).reads + if read.rule in {"R1", "R2"} + and not read.file.startswith("hooks/") + and read.var.startswith("AUTOSKILLIT_") + ) + host_only_pairs = {(file, var) for file, var, _reason in _HOST_ONLY_READS} + missing = tuple( + read + for read in reads + if read.var not in AUTOSKILLIT_PRIVATE_ENV_VARS + and (read.file, read.var) not in host_only_pairs + ) + + assert not missing, ( + "Server-process AUTOSKILLIT environment reads must be private and therefore scrubbed/" + "forwarded deliberately:\n" + f"{_format_reads(missing)}" + ) + + def test_dispatch_identity_channel_regression() -> None: dispatch_identity_vars = { DISPATCH_ID_ENV_VAR, diff --git a/tests/config/test_timeout_coherence.py b/tests/config/test_timeout_coherence.py index bb3f606349..2d18a6da48 100644 --- a/tests/config/test_timeout_coherence.py +++ b/tests/config/test_timeout_coherence.py @@ -3,7 +3,6 @@ import pytest import structlog.testing -from autoskillit.config import load_config from autoskillit.config.settings import ( _claude_mcp_timeout_coherence_gate, _codex_mcp_timeout_coherence_gate, @@ -12,55 +11,10 @@ pytestmark = [pytest.mark.layer("config"), pytest.mark.small] -class TestTimeoutCoherenceGate: - """Tests for _timeout_coherence_gate warning behavior.""" +def test_general_idle_timeout_coherence_gate_is_removed() -> None: + from autoskillit.config import _coherence - def test_idle_output_timeout_less_than_tool_max_emits_warning(self, tmp_path): - """idle_output_timeout < known tool max triggers coherence warning.""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - # idle_output_timeout=600, but wait_for_merge_queue recipe override is 900s - (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 600\n") - with structlog.testing.capture_logs() as cap_logs: - cfg = load_config(tmp_path) - assert cfg.run_skill.idle_output_timeout == 600 - assert any("idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs) - - def test_idle_output_timeout_zero_skips_coherence_check(self, tmp_path): - """Disabled watchdog (0) passes coherence check unconditionally.""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 0\n") - with structlog.testing.capture_logs() as cap_logs: - cfg = load_config(tmp_path) - assert cfg.run_skill.idle_output_timeout == 0 - assert not any( - "idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs - ) - - def test_coherence_gate_warns_on_matched_defaults(self, tmp_path): - """idle_output_timeout == tool_timeout is incoherent (race condition).""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - # idle_output_timeout=600, wait_for_merge_queue default=600 — exact match - (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 600\n") - with structlog.testing.capture_logs() as cap_logs: - cfg = load_config(tmp_path) - assert cfg.run_skill.idle_output_timeout == 600 - assert any("idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs) - - def test_coherence_gate_passes_when_idle_exceeds_tool_max(self, tmp_path): - """idle_output_timeout > all known tool timeouts passes cleanly.""" - config_dir = tmp_path / ".autoskillit" - config_dir.mkdir() - # idle_output_timeout=1000 > _MERGE_QUEUE_RECIPE_MAX (900) - (config_dir / "config.yaml").write_text("run_skill:\n idle_output_timeout: 1000\n") - with structlog.testing.capture_logs() as cap_logs: - cfg = load_config(tmp_path) - assert cfg.run_skill.idle_output_timeout == 1000 - assert not any( - "idle_output_timeout_coherence" in entry.get("event", "") for entry in cap_logs - ) + assert not hasattr(_coherence, "_timeout_coherence_gate") class TestCodexMcpTimeoutCoherenceGate: diff --git a/tests/contracts/test_config_decomposition_identity.py b/tests/contracts/test_config_decomposition_identity.py index 87a0a9268c..abd7b4661f 100644 --- a/tests/contracts/test_config_decomposition_identity.py +++ b/tests/contracts/test_config_decomposition_identity.py @@ -64,8 +64,18 @@ def test_automation_config_imports_coherence_gate_with_identity() -> None: import autoskillit.config._coherence as coh_mod import autoskillit.config.settings as settings_mod - assert auto_mod._timeout_coherence_gate is coh_mod._timeout_coherence_gate - assert auto_mod._timeout_coherence_gate is settings_mod._timeout_coherence_gate + assert ( + auto_mod._claude_mcp_timeout_coherence_gate is coh_mod._claude_mcp_timeout_coherence_gate + ) + assert ( + auto_mod._claude_mcp_timeout_coherence_gate + is settings_mod._claude_mcp_timeout_coherence_gate + ) + assert auto_mod._codex_mcp_timeout_coherence_gate is coh_mod._codex_mcp_timeout_coherence_gate + assert ( + auto_mod._codex_mcp_timeout_coherence_gate + is settings_mod._codex_mcp_timeout_coherence_gate + ) assert auto_mod._process_tether_coherence_gate is coh_mod._process_tether_coherence_gate assert auto_mod._process_tether_coherence_gate is settings_mod._process_tether_coherence_gate diff --git a/tests/contracts/test_config_decomposition_surface.py b/tests/contracts/test_config_decomposition_surface.py index 79edab22b1..e4839cad44 100644 --- a/tests/contracts/test_config_decomposition_surface.py +++ b/tests/contracts/test_config_decomposition_surface.py @@ -42,16 +42,12 @@ def test_public_symbol_importable_from_package(symbol: str) -> None: "_process_tether_coherence_gate", "_claude_mcp_timeout_coherence_gate", "_codex_mcp_timeout_coherence_gate", - "_timeout_coherence_gate", "_coerce_value", "_field_defaults", "_build_subconfig", "_preprocess_agent_backend", "_build_config_schema", "_UNSET", - "_CI_WATCH_DEFAULT", - "_MERGE_QUEUE_DEFAULT", - "_MERGE_QUEUE_RECIPE_MAX", # Transitively-imported autoskillit.core re-exports. "FEATURE_REGISTRY", "FeatureLifecycle", diff --git a/tests/contracts/test_protocol_satisfaction.py b/tests/contracts/test_protocol_satisfaction.py index abb470b0ac..3aae98d30d 100644 --- a/tests/contracts/test_protocol_satisfaction.py +++ b/tests/contracts/test_protocol_satisfaction.py @@ -507,6 +507,7 @@ def test_req_api_001_public_params_present(self): "max_extension_seconds", "marker_dir", "session_id", + "operation_lease_dir", "stream_parser", "inspector_callback", "workload_basenames", @@ -589,6 +590,7 @@ def test_req_api_002_call_params_match_protocol(self): "max_extension_seconds", "marker_dir", "session_id", + "operation_lease_dir", "stream_parser", "completion_record_types", "session_record_types", @@ -626,6 +628,13 @@ def test_run_managed_async_marker_params_after_session_id_timeout(self): assert marker_dir_idx == session_id_timeout_idx + 1 assert session_id_idx == marker_dir_idx + 1 + def test_run_managed_async_operation_lease_dir_defaults_and_order(self): + sig = inspect.signature(run_managed_async) + assert sig.parameters["operation_lease_dir"].default is None + assert list(sig.parameters).index("operation_lease_dir") == ( + list(sig.parameters).index("session_id") + 1 + ) + def test_default_subprocess_runner_marker_dir_session_id_defaults(self): """DefaultSubprocessRunner.__call__ marker_dir and session_id default to None.""" sig = inspect.signature(DefaultSubprocessRunner.__call__) @@ -641,6 +650,13 @@ def test_default_subprocess_runner_marker_params_after_max_extension(self): assert marker_dir_idx == max_extension_idx + 1 assert session_id_idx == marker_dir_idx + 1 + def test_default_subprocess_runner_operation_lease_dir_defaults_and_order(self): + sig = inspect.signature(DefaultSubprocessRunner.__call__) + assert sig.parameters["operation_lease_dir"].default is None + assert list(sig.parameters).index("operation_lease_dir") == ( + list(sig.parameters).index("session_id") + 1 + ) + def test_default_subprocess_runner_satisfies_protocol_with_marker_params(self): """DefaultSubprocessRunner() satisfies SubprocessRunner with marker_dir/session_id.""" from autoskillit.core.types import SubprocessRunner diff --git a/tests/core/test_backend_dataclasses.py b/tests/core/test_backend_dataclasses.py index dd43627c87..9fda28b868 100644 --- a/tests/core/test_backend_dataclasses.py +++ b/tests/core/test_backend_dataclasses.py @@ -45,7 +45,6 @@ def test_cmd_spec_fields(): "cwd", "origin", "is_resume", - "process_idle_timeout_ms", "inherited_fds", "managed_skill_catalog", "projected_skill_entries", @@ -69,13 +68,6 @@ def test_cmd_spec_is_resume_default(): assert spec.is_resume is False -def test_cmd_spec_process_idle_timeout_default(): - from autoskillit.core import CmdSpec - - spec = CmdSpec(cmd=(), env={}) - assert spec.process_idle_timeout_ms == 0 - - def test_cmd_spec_inherited_fds_default(): from autoskillit.core import CmdSpec diff --git a/tests/core/test_type_constants_private_env_vars.py b/tests/core/test_type_constants_private_env_vars.py index 7ef1568083..6a86970d08 100644 --- a/tests/core/test_type_constants_private_env_vars.py +++ b/tests/core/test_type_constants_private_env_vars.py @@ -32,6 +32,7 @@ def test_private_env_vars_includes_execution_control_vars() -> None: "AUTOSKILLIT_ALLOWED_WRITE_PREFIX", "AUTOSKILLIT_ALLOWED_WRITE_PREFIXES", "MAX_MCP_OUTPUT_TOKENS", + "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", } assert expected <= AUTOSKILLIT_PRIVATE_ENV_VARS diff --git a/tests/core/test_types_protocols.py b/tests/core/test_types_protocols.py index c477a5a8f4..c958a0b28b 100644 --- a/tests/core/test_types_protocols.py +++ b/tests/core/test_types_protocols.py @@ -193,6 +193,33 @@ def test_subprocess_runner_protocol_marker_params_are_keyword_only() -> None: ) +def test_subprocess_runner_protocol_operation_lease_dir_default_none() -> None: + import inspect + + from autoskillit.core import SubprocessRunner + + sig = inspect.signature(SubprocessRunner.__call__) + assert sig.parameters["operation_lease_dir"].default is None + + +def test_subprocess_runner_protocol_operation_lease_dir_follows_session_id() -> None: + import inspect + + from autoskillit.core import SubprocessRunner + + params = list(inspect.signature(SubprocessRunner.__call__).parameters) + assert params.index("operation_lease_dir") == params.index("session_id") + 1 + + +def test_subprocess_runner_protocol_operation_lease_dir_is_keyword_only() -> None: + import inspect + + from autoskillit.core import SubprocessRunner + + param = inspect.signature(SubprocessRunner.__call__).parameters["operation_lease_dir"] + assert param.kind == inspect.Parameter.KEYWORD_ONLY + + # --------------------------------------------------------------------------- # CIRunScope event field # --------------------------------------------------------------------------- diff --git a/tests/execution/backends/test_codex_backend.py b/tests/execution/backends/test_codex_backend.py index cb4ae67476..623c33f5ce 100644 --- a/tests/execution/backends/test_codex_backend.py +++ b/tests/execution/backends/test_codex_backend.py @@ -1014,11 +1014,11 @@ def test_default_params_emit_no_warnings(self) -> None: assert "CLAUDE_CODE_EXIT_AFTER_STOP_DELAY" not in spec.env assert "CLAUDE_STREAM_IDLE_TIMEOUT_MS" not in spec.env - def test_stream_idle_timeout_routed_to_cmdspec(self) -> None: + def test_stream_idle_timeout_is_not_supervisor_policy(self) -> None: spec = CodexBackend().build_skill_session_cmd( **{**self.BASE, "stream_idle_timeout_ms": 30000} ) - assert spec.process_idle_timeout_ms == 30000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env def test_cwd_set(self) -> None: spec = CodexBackend().build_skill_session_cmd( @@ -1798,11 +1798,11 @@ def test_bypass_hook_trust_absent_from_argv_present_in_plan(self) -> None: assert spec.app_server_plan.bypass_hook_trust is True assert spec.app_server_plan.config_overrides["bypass_hook_trust"] is True - def test_stream_idle_timeout_routed_to_cmdspec(self) -> None: + def test_food_truck_stream_idle_is_not_supervisor_policy(self) -> None: spec = CodexBackend().build_food_truck_cmd( **{**self.BASE, "stream_idle_timeout_ms": 60000} ) - assert spec.process_idle_timeout_ms == 60000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env class TestCodexBackendVersion: @@ -2103,18 +2103,6 @@ def test_conventions_persistent_session_root(self) -> None: assert CodexBackend().conventions.persistent_session_root_subdir == Path("codex-sessions") -class TestClaudeCodeBackendProcessIdleDefault: - def test_claude_code_backend_process_idle_default_zero(self) -> None: - from autoskillit.execution.backends.claude import ClaudeCodeBackend - - spec = ClaudeCodeBackend().build_skill_session_cmd( - "/test-skill", - cwd="/work", - completion_marker="%%DONE%%", - ) - assert spec.process_idle_timeout_ms == 0 - - class TestCodexDiscardDispositions: """Codex builder parameter disposition contracts. @@ -2124,8 +2112,8 @@ class TestCodexDiscardDispositions: catalog's session_home (the mandatory add-dir for skill sessions, the required managed_skill_catalog for food trucks). output_format -> logged warning when != JSON. - exit_after_stop_delay_ms -> AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT env injection via setdefault. - stream_idle_timeout_ms -> routed to CmdSpec.process_idle_timeout_ms + env injection. + exit_after_stop_delay_ms / stream_idle_timeout_ms -> no supervisor idle policy + or kitchen env write. """ SKILL_BASE: dict[str, object] = { @@ -2197,44 +2185,42 @@ def test_exit_delay_injects_idle_timeout_skill_builder(self) -> None: **self.SKILL_BASE, exit_after_stop_delay_ms=5000, ) - assert spec.env["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "5.0" + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env def test_exit_delay_injects_idle_timeout_food_truck_builder(self) -> None: spec = CodexBackend().build_food_truck_cmd( **self.FOOD_TRUCK_BASE, exit_after_stop_delay_ms=5000, ) - assert spec.env["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "5.0" + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env def test_stream_idle_injects_idle_timeout_skill_builder(self) -> None: spec = CodexBackend().build_skill_session_cmd( **self.SKILL_BASE, stream_idle_timeout_ms=3000, ) - assert spec.env["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "3.0" - assert spec.process_idle_timeout_ms == 3000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env def test_stream_idle_injects_idle_timeout_food_truck_builder(self) -> None: spec = CodexBackend().build_food_truck_cmd( **self.FOOD_TRUCK_BASE, stream_idle_timeout_ms=3000, ) - assert spec.env["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "3.0" - assert spec.process_idle_timeout_ms == 3000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env - def test_stream_idle_routed_to_process_idle_skill_builder(self) -> None: + def test_stream_idle_is_not_routed_to_skill_supervisor_policy(self) -> None: spec = CodexBackend().build_skill_session_cmd( **self.SKILL_BASE, stream_idle_timeout_ms=10000, ) - assert spec.process_idle_timeout_ms == 10000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env - def test_stream_idle_routed_to_process_idle_food_truck_builder(self) -> None: + def test_stream_idle_is_not_routed_to_food_truck_supervisor_policy(self) -> None: spec = CodexBackend().build_food_truck_cmd( **self.FOOD_TRUCK_BASE, stream_idle_timeout_ms=10000, ) - assert spec.process_idle_timeout_ms == 10000 + assert "AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT" not in spec.env def test_zero_ms_no_idle_timeout_skill_builder(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", raising=False) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index ab9dff0bb0..0d6cd34cfc 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -578,86 +578,95 @@ async def test_managed_session_type_reaches_runner_and_session_index( assert entry["session_type"] == expected_session_type -class TestProcessIdleTimeoutOverride: - """Tests for CmdSpec.process_idle_timeout_ms overriding effective_idle.""" - - @pytest.mark.anyio - async def test_spec_idle_used_when_caller_supplies_none( - self, minimal_ctx, tmp_path: Path, monkeypatch - ) -> None: - from autoskillit.execution.headless import run_headless_core - from tests.execution.conftest import _mock_backend - from tests.fakes import MockSubprocessRunner +@pytest.mark.anyio +@pytest.mark.parametrize("backend_name", ["claude-code", "codex"]) +async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_lease_channel( + minimal_ctx, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, backend_name: str +) -> None: + import autoskillit.execution.headless._managed._attempt as managed_attempt + from autoskillit.execution.backends import CodexBackend + from autoskillit.execution.backends.claude import ClaudeCodeBackend + from autoskillit.execution.headless import run_headless_core + from tests.execution.conftest import _mock_backend + from tests.fakes import MockSubprocessRunner - monkeypatch.delenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", raising=False) - runner = MockSubprocessRunner() - runner.set_default(_success_result()) - minimal_ctx.runner = runner - backend = _mock_backend(pty_required=True, channel_b_capable=True) - backend.build_skill_session_cmd.return_value = CmdSpec( - cmd=("claude", "-p", "test"), - env={}, - process_idle_timeout_ms=30000, - ) - minimal_ctx.backend = backend + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(managed_attempt, "default_log_dir", lambda: Path("relative-logs")) + monkeypatch.setenv("AUTOSKILLIT_OPERATION_LEASE_DIR", "/parent") + minimal_ctx.config.features.update({"fleet": True, "codex_backend": backend_name == "codex"}) + minimal_ctx.config.agent_backend.backend = backend_name + minimal_ctx.config.linux_tracing.log_dir = str(tmp_path) + real_backend = CodexBackend() if backend_name == "codex" else ClaudeCodeBackend() + backend = _mock_backend(pty_required=True, channel_b_capable=True) + backend.name = backend_name + backend.capabilities = real_backend.capabilities + built_specs = [] - await run_headless_core("/test foo", str(tmp_path), minimal_ctx) + def build_skill_session_spec(*args, **kwargs): + spec = real_backend.build_skill_session_cmd(*args, **kwargs) + built_specs.append(spec) + return spec - assert runner.call_args_list, "runner was never called" - _cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] - assert kwargs.get("idle_output_timeout") == 30.0 + backend.build_skill_session_cmd.side_effect = build_skill_session_spec + minimal_ctx.backend = backend - @pytest.mark.anyio - async def test_spec_idle_overrides_when_smaller( - self, minimal_ctx, tmp_path: Path, monkeypatch - ) -> None: - from autoskillit.execution.headless import run_headless_core - from tests.execution.conftest import _mock_backend - from tests.fakes import MockSubprocessRunner + runner_calls: list[tuple[Path, bool]] = [] - monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "45") - runner = MockSubprocessRunner() - runner.set_default(_success_result()) - minimal_ctx.runner = runner - backend = _mock_backend(pty_required=True, channel_b_capable=True) - backend.build_skill_session_cmd.return_value = CmdSpec( - cmd=("claude", "-p", "test"), - env={}, - process_idle_timeout_ms=15000, - ) - minimal_ctx.backend = backend + class CapturingRunner(MockSubprocessRunner): + async def __call__(self, *args, **kwargs): + channel = kwargs["operation_lease_dir"] + runner_calls.append((channel, channel.is_dir())) + return await super().__call__(*args, **kwargs) - await run_headless_core("/test foo", str(tmp_path), minimal_ctx) + runner = CapturingRunner() + runner.set_default(_success_result()) + minimal_ctx.runner = runner + attempt_specs = [] + original_attempt = _patch_headless__headless_execute._run_headless_attempt - assert runner.call_args_list, "runner was never called" - _cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] - assert kwargs.get("idle_output_timeout") == 15.0 + async def capture_attempt(*args, **kwargs): + result, spec = await original_attempt(*args, **kwargs) + attempt_specs.append(spec) + return result, spec - @pytest.mark.anyio - async def test_zero_spec_idle_leaves_effective_unaffected( - self, minimal_ctx, tmp_path: Path, monkeypatch - ) -> None: - from autoskillit.execution.headless import run_headless_core - from tests.execution.conftest import _mock_backend - from tests.fakes import MockSubprocessRunner + monkeypatch.setattr( + _patch_headless__headless_execute, "_run_headless_attempt", capture_attempt + ) - monkeypatch.setenv("AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT", "30") - runner = MockSubprocessRunner() - runner.set_default(_success_result()) - minimal_ctx.runner = runner - backend = _mock_backend(pty_required=True, channel_b_capable=True) - backend.build_skill_session_cmd.return_value = CmdSpec( - cmd=("claude", "-p", "test"), - env={}, - process_idle_timeout_ms=0, + await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45) + await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45) + + assert len(runner.call_args_list) == 2 + assert len(built_specs) == 2 + assert len(attempt_specs) == 2 + channels = [] + for index, final_spec in enumerate(attempt_specs): + channel = runner.call_args_list[index][3]["operation_lease_dir"] + assert channel.is_absolute() + assert channel == Path(final_spec.env["AUTOSKILLIT_OPERATION_LEASE_DIR"]) + assert ( + runner.call_args_list[index][3]["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] != "/parent" ) - minimal_ctx.backend = backend + assert runner_calls[index] == (channel, True) + assert runner.call_args_list[index][3]["idle_output_timeout"] == 45 + assert not channel.exists() + channels.append(channel) + assert channels[0] != channels[1] + - await run_headless_core("/test foo", str(tmp_path), minimal_ctx) +def test_food_truck_env_extras_override_remains_authoritative() -> None: + from autoskillit.execution.headless._managed._food_truck_executor import ( + _merge_food_truck_extras, + ) + + merged = _merge_food_truck_extras( + env_extras={"AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT": "77"}, + requires_packs=(), + fleet_idle_output_timeout=1800, + run_skill_idle_output_timeout=600, + ) - assert runner.call_args_list, "runner was never called" - _cmd, _cwd, _timeout, kwargs = runner.call_args_list[0] - assert kwargs.get("idle_output_timeout") == 30.0 + assert merged["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "77" class TestPreSessionIndexSignaling: @@ -681,7 +690,6 @@ async def test_dirty_state_logged_with_structured_metadata( backend.build_skill_session_cmd.return_value = CmdSpec( cmd=("claude", "-p", "test"), env={}, - process_idle_timeout_ms=30000, ) minimal_ctx.backend = backend @@ -723,7 +731,6 @@ async def test_clean_state_does_not_log_dirty_warning( backend.build_skill_session_cmd.return_value = CmdSpec( cmd=("claude", "-p", "test"), env={}, - process_idle_timeout_ms=30000, ) minimal_ctx.backend = backend diff --git a/tests/execution/test_headless_provider_fallback.py b/tests/execution/test_headless_provider_fallback.py index 2a3ac54a0b..7b2afd9fa1 100644 --- a/tests/execution/test_headless_provider_fallback.py +++ b/tests/execution/test_headless_provider_fallback.py @@ -275,7 +275,9 @@ def build_spec(binding, provider_extras, managed_attempt_id): ) assert call_count[0] == 1 - assert runner_envs == [sink_env] + assert len(runner_envs) == 1 + assert {key: runner_envs[0][key] for key in sink_env} == sink_env + assert "AUTOSKILLIT_OPERATION_LEASE_DIR" in runner_envs[0] assert runner_pass_fds == [(77,)] assert len(authority.bindings) == 1 assert all(binding.closed for binding in authority.bindings) diff --git a/tests/execution/test_launch_resolution.py b/tests/execution/test_launch_resolution.py index 0a815f47ba..9a5b4f6444 100644 --- a/tests/execution/test_launch_resolution.py +++ b/tests/execution/test_launch_resolution.py @@ -137,7 +137,6 @@ def _adapter_result(preparation, **changes: object) -> LaunchAdapterResult: for value in (preparation.command, *preparation.arguments) ), ), - process_idle_timeout_ms=90_000, inherited_fds=(9, 11), ), } @@ -799,20 +798,20 @@ def test_resolved_contract_rejects_malformed_command_origin_pairs(field: str) -> ResolvedLaunchContract.from_payload(payload) -def test_launch_contract_schema_is_exactly_version_four() -> None: +def test_launch_contract_schema_is_exactly_version_five() -> None: resolver = DefaultLaunchResolver() contract = resolver.finalize(resolver.prepare(_request()), _Adapter()) - assert LAUNCH_CONTRACT_SCHEMA_VERSION == 4 - assert contract.schema_version == 4 - assert contract.canonical_payload["schema_version"] == 4 + assert LAUNCH_CONTRACT_SCHEMA_VERSION == 5 + assert contract.schema_version == 5 + assert contract.canonical_payload["schema_version"] == 5 -def test_launch_contract_rejects_version_three_payload() -> None: +def test_launch_contract_rejects_version_four_payload() -> None: resolver = DefaultLaunchResolver() contract = resolver.finalize(resolver.prepare(_request()), _Adapter()) old_payload = dict(contract.canonical_payload) - old_payload["schema_version"] = 3 + old_payload["schema_version"] = 4 with pytest.raises(LaunchContractError, match="schema version"): ResolvedLaunchContract.from_payload(old_payload) diff --git a/tests/execution/test_operation_lease_watchers.py b/tests/execution/test_operation_lease_watchers.py new file mode 100644 index 0000000000..ad5390999f --- /dev/null +++ b/tests/execution/test_operation_lease_watchers.py @@ -0,0 +1,468 @@ +"""Lease evidence must reach each process liveness consumer before early exits.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import anyio +import pytest +import structlog.testing + +import autoskillit.core.plugins._operation_lease as lease_module +import autoskillit.execution.process._process_monitor as process_monitor +import autoskillit.execution.process._race_watchers as race_watchers +from autoskillit.core import ( + ChannelBStatus, + InFlightOperations, + operation_lease, +) +from autoskillit.execution.process._process_monitor import ( + SessionMonitorResult, + _active_liveness_signals, + _discover_session_log, + _session_log_monitor, +) +from autoskillit.execution.process._process_race import RaceAccumulator +from autoskillit.execution.process._race_watchers import ( + _watch_child_activity, + _watch_stdout_idle, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +class _StopPolling(Exception): + pass + + +class _LeaseClock: + def __init__( + self, + epoch: float, + *, + channel: Path | None = None, + steps: list[float] | None = None, + stop_after: int | None = None, + on_tick=None, + ) -> None: + self.epoch = epoch + self.elapsed = 0.0 + self.channel = channel + self.steps = list(steps or []) + self.stop_after = stop_after + self.on_tick = on_tick + self.ticks = 0 + + def current_time(self) -> float: + return self.elapsed + + async def sleep(self, interval: float) -> None: + self.ticks += 1 + delta = self.steps.pop(0) if self.steps else interval + self.elapsed += delta + self.epoch += delta + if self.channel is not None: + for path in self.channel.glob("*.lease.json"): + os.utime(path, (self.epoch, self.epoch)) + if self.on_tick is not None: + self.on_tick(self) + if self.stop_after is not None and self.ticks >= self.stop_after: + raise _StopPolling + + +async def _blocked_heartbeat(_handle, _interval: float) -> None: + await asyncio.Event().wait() + + +def _install_clock( + monkeypatch: pytest.MonkeyPatch, + clock: _LeaseClock, + *modules: object, +) -> None: + monkeypatch.setattr(time, "monotonic", lambda: clock.elapsed) + monkeypatch.setattr(time, "time", lambda: clock.epoch) + fake_anyio = SimpleNamespace(sleep=clock.sleep, current_time=clock.current_time) + for module in modules: + monkeypatch.setattr(module, "anyio", fake_anyio) + + +@pytest.mark.anyio +async def test_stdout_lease_tolerance_refreshes_each_poll_until_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + registry = InFlightOperations() + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 7200, + registry=registry, + heartbeat_interval=10000, + ) as handle: + assert handle.path is not None + clock = _LeaseClock( + handle.record.started_at_epoch, + channel=tmp_path, + steps=[1, 7198, 1, 1799], + ) + _install_clock(monkeypatch, clock, race_watchers) + stdout = MagicMock() + stdout.stat.return_value = SimpleNamespace(st_size=1) + acc = RaceAccumulator() + trigger = anyio.Event() + + with structlog.testing.capture_logs() as logs: + await _watch_stdout_idle( + stdout, + 1800, + acc, + trigger, + 300, + max_suppression_seconds=1800, + operation_lease_dir=tmp_path, + ) + + assert clock.epoch < handle.record.not_after_epoch + 1800 + 300 + assert clock.epoch >= handle.record.not_after_epoch + assert acc.idle_stall is True + assert any(entry.get("event") == "stdout_idle_deferred_to_operation" for entry in logs) + + +@pytest.mark.anyio +async def test_stdout_growth_then_stall_reports_actual_silence_and_threshold( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _LeaseClock(time.time()) + _install_clock(monkeypatch, clock, race_watchers) + stdout = MagicMock() + stdout.stat.side_effect = [ + SimpleNamespace(st_size=10), + SimpleNamespace(st_size=20), + SimpleNamespace(st_size=20), + SimpleNamespace(st_size=20), + ] + acc = RaceAccumulator() + trigger = anyio.Event() + + with structlog.testing.capture_logs() as logs: + await _watch_stdout_idle(stdout, 2.0, acc, trigger, 1.0) + + fire = next(entry for entry in logs if entry.get("event") == "stdout_idle_stall_firing") + assert acc.idle_stall is True + assert fire["idle_threshold"] == 2.0 + assert fire["silence_seconds"] >= 2.0 + + +@pytest.mark.anyio +async def test_stdout_lease_precedes_pending_task_gate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100, + registry=InFlightOperations(), + heartbeat_interval=10000, + ): + clock = _LeaseClock(time.time(), channel=tmp_path, stop_after=6) + _install_clock(monkeypatch, clock, race_watchers) + stdout = MagicMock() + stdout.stat.return_value = SimpleNamespace(st_size=1) + acc = RaceAccumulator() + trigger = anyio.Event() + + with pytest.raises(_StopPolling): + await _watch_stdout_idle( + stdout, + 0.5, + acc, + trigger, + 1.0, + max_suppression_seconds=0.1, + has_pending_tasks=lambda: True, + operation_lease_dir=tmp_path, + ) + + assert acc.idle_stall is False + assert stdout.stat.call_count == 5 + + +@pytest.mark.anyio +async def test_stdout_file_loss_during_lease_keeps_last_evidence_clock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 6, + registry=InFlightOperations(), + heartbeat_interval=10000, + ) as handle: + clock = _LeaseClock(handle.record.started_at_epoch, channel=tmp_path) + _install_clock(monkeypatch, clock, race_watchers) + stdout = MagicMock() + stdout.stat.side_effect = [ + OSError("missing"), + OSError("missing"), + OSError("missing"), + OSError("missing"), + SimpleNamespace(st_size=0), + SimpleNamespace(st_size=0), + SimpleNamespace(st_size=0), + ] + acc = RaceAccumulator() + trigger = anyio.Event() + + await _watch_stdout_idle( + stdout, + 3.0, + acc, + trigger, + 1.0, + operation_lease_dir=tmp_path, + ) + + assert acc.idle_stall is True + assert clock.elapsed >= 8 + + +@pytest.mark.anyio +async def test_session_log_discovery_waits_for_lease_then_uses_timeout_from_loss( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + log_dir = tmp_path / "logs" + log_dir.mkdir() + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100, + registry=InFlightOperations(), + heartbeat_interval=10000, + ) as handle: + assert handle.path is not None + + def remove_after_three_ticks(clock: _LeaseClock) -> None: + if clock.ticks == 3: + handle.path.unlink() + + clock = _LeaseClock( + handle.record.started_at_epoch, + channel=tmp_path, + on_tick=remove_after_three_ticks, + ) + _install_clock(monkeypatch, clock, process_monitor) + result = await _discover_session_log( + log_dir, + spawn_time=clock.epoch - 1, + phase1_poll=1, + phase1_timeout=2, + expected_session_id=None, + resume_cursor=None, + operation_lease_dir=tmp_path, + ) + + assert isinstance(result, SessionMonitorResult) + assert result.status is ChannelBStatus.STALE + assert clock.elapsed >= 4 + + +@pytest.mark.anyio +async def test_session_log_lease_defers_api_suppression_but_existing_cap_still_fires( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + monkeypatch.setattr(process_monitor, "_has_active_api_connection", lambda _pid: True) + log_dir = tmp_path / "logs" + log_dir.mkdir() + session_file = log_dir / "child.jsonl" + session_file.write_text(json.dumps({"type": "assistant", "message": "working"}) + "\n") + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100, + registry=InFlightOperations(), + heartbeat_interval=10000, + ) as handle: + assert handle.path is not None + + def remove_after_three_ticks(clock: _LeaseClock) -> None: + if clock.ticks == 3: + handle.path.unlink() + + clock = _LeaseClock( + handle.record.started_at_epoch, + channel=tmp_path, + on_tick=remove_after_three_ticks, + ) + _install_clock(monkeypatch, clock, process_monitor) + with structlog.testing.capture_logs() as logs: + result = await _session_log_monitor( + log_dir, + "%%DONE%%", + stale_threshold=1, + spawn_time=clock.epoch - 1, + pid=123, + _phase1_poll=1, + _phase2_poll=1, + max_suppression_seconds=2, + operation_lease_dir=tmp_path, + ) + + assert result.status is ChannelBStatus.STALE + operation_events = [ + index + for index, entry in enumerate(logs) + if entry.get("event") == "stale_deferred_to_operation" + ] + api_events = [ + index + for index, entry in enumerate(logs) + if "ESTABLISHED port-443 connection" in str(entry.get("event", "")) + ] + assert operation_events + assert api_events and min(api_events) > min(operation_events) + + +@pytest.mark.anyio +async def test_session_log_file_loss_during_lease_keeps_last_evidence_clock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + log_dir = tmp_path / "logs" + log_dir.mkdir() + session_file = log_dir / "child.jsonl" + session_file.write_text(json.dumps({"type": "assistant", "message": "working"}) + "\n") + selected: list[Path] = [] + stat_calls = 0 + original_stat = Path.stat + + def temporary_stat_failure(path: Path, *args, **kwargs): + nonlocal stat_calls + if selected and path == selected[0]: + stat_calls += 1 + if 3 <= stat_calls <= 6: + raise OSError("temporary session log loss") + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", temporary_stat_failure) + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 6, + registry=InFlightOperations(), + heartbeat_interval=10000, + ) as handle: + clock = _LeaseClock(handle.record.started_at_epoch, channel=tmp_path) + _install_clock(monkeypatch, clock, process_monitor) + result = await _session_log_monitor( + log_dir, + "%%DONE%%", + stale_threshold=3, + spawn_time=clock.epoch - 1, + _phase1_poll=1, + _phase2_poll=1, + operation_lease_dir=tmp_path, + on_session_file_selected=lambda path, _cursor: selected.append(path), + ) + + assert result.status is ChannelBStatus.STALE + assert clock.elapsed >= 8 + + +@pytest.mark.anyio +async def test_session_log_completion_is_processed_while_lease_is_active( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + log_dir = tmp_path / "logs" + log_dir.mkdir() + session_file = log_dir / "child.jsonl" + session_file.write_text(json.dumps({"type": "assistant", "message": "working"}) + "\n") + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100, + registry=InFlightOperations(), + heartbeat_interval=10000, + ): + + def append_completion(clock: _LeaseClock) -> None: + if clock.ticks == 2: + with session_file.open("a") as stream: + stream.write( + json.dumps({"type": "assistant", "message": {"content": "%%DONE%%"}}) + + "\n" + ) + + clock = _LeaseClock( + time.time(), + channel=tmp_path, + on_tick=append_completion, + ) + _install_clock(monkeypatch, clock, process_monitor) + result = await _session_log_monitor( + log_dir, + "%%DONE%%", + stale_threshold=1, + spawn_time=clock.epoch - 1, + _phase1_poll=1, + _phase2_poll=1, + operation_lease_dir=tmp_path, + ) + + assert result.status is ChannelBStatus.COMPLETION + + +@pytest.mark.anyio +async def test_deadline_extension_uses_lease_but_keeps_absolute_cap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100, + registry=InFlightOperations(), + heartbeat_interval=10000, + ): + clock = _LeaseClock(time.time(), channel=tmp_path, stop_after=4) + _install_clock(monkeypatch, clock, race_watchers) + scope = SimpleNamespace(deadline=5.0) + trigger = anyio.Event() + with pytest.raises(_StopPolling): + await _watch_child_activity( + 123, + [scope], + max_extension_seconds=10, + trigger=trigger, + _poll_interval=4, + operation_lease_dir=tmp_path, + has_pending_tasks=lambda: False, + ) + + assert 5.0 < scope.deadline <= 15.0 + + +@pytest.mark.anyio +async def test_drain_liveness_signals_include_active_operation_lease(tmp_path: Path) -> None: + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + ): + assert "operation_lease" in _active_liveness_signals( + None, + None, + None, + operation_lease_dir=tmp_path, + ) diff --git a/tests/execution/test_process_run.py b/tests/execution/test_process_run.py index 5778313ecf..e274bf3d23 100644 --- a/tests/execution/test_process_run.py +++ b/tests/execution/test_process_run.py @@ -19,6 +19,7 @@ import anyio import psutil import pytest +import structlog.testing import autoskillit.execution.process._process_monitor as _patch_process__process_monitor from autoskillit.core.types import TerminationReason @@ -493,6 +494,161 @@ async def test_run_managed_async_idle_stall_kills_hanging_process(self, tmp_path assert not psutil.pid_exists(result.pid) +class TestOperationLeaseLiveness: + @pytest.mark.anyio + async def test_real_lease_keeps_silent_child_alive_past_idle_cap( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.core import OPERATION_LEASE_DIR_ENV_VAR + + channel = tmp_path / "leases" + channel.mkdir() + script = tmp_path / "silent_with_lease.py" + script.write_text( + textwrap.dedent( + """\ + import asyncio, json, os, sys, time + from pathlib import Path + from autoskillit.core import InFlightOperations, operation_lease + sys.stdout.write(json.dumps({"type": "assistant", "burst": True}) + "\\n") + sys.stdout.flush() + async def hold_lease(): + async with operation_lease( + Path(os.environ["AUTOSKILLIT_OPERATION_LEASE_DIR"]), + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + ): + await asyncio.sleep(12) + asyncio.run(hold_lease()) + print(json.dumps({"type": "result", "subtype": "success"}), flush=True) + """ + ) + ) + monkeypatch.setattr( + _patch_process__process_monitor, + "_has_active_api_connection", + lambda _pid: False, + ) + + with structlog.testing.capture_logs() as logs: + with anyio.fail_after(25): + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=30, + idle_output_timeout=2.0, + max_suppression_seconds=2.0, + operation_lease_dir=channel, + env={**os.environ, OPERATION_LEASE_DIR_ENV_VAR: str(channel)}, + ) + + assert result.termination == TerminationReason.NATURAL_EXIT + assert any(entry.get("event") == "stdout_idle_deferred_to_operation" for entry in logs) + + @pytest.mark.parametrize("lease_mode", ["none", "expired", "mismatched", "dead"]) + @pytest.mark.anyio + async def test_missing_expired_mismatched_or_dead_lease_does_not_suppress_idle_stall( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + lease_mode: str, + ) -> None: + from autoskillit.core import OPERATION_LEASE_DIR_ENV_VAR + + channel = tmp_path / "leases" + channel.mkdir() + script = tmp_path / f"silent_{lease_mode}.py" + setup = { + "none": "pass", + "expired": """\ +async def hold_lease(): + async with operation_lease( + channel, + operation="run_skill", + not_after_epoch=time.time() + 3, + registry=InFlightOperations(), + ): + await asyncio.sleep(30) +asyncio.run(hold_lease()) +""", + "mismatched": """\ +wrong = channel / "different" +wrong.mkdir() +async def hold_lease(): + async with operation_lease( + wrong, + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + ): + await asyncio.sleep(30) +asyncio.run(hold_lease()) +""", + "dead": """\ +import subprocess +helper = '''import asyncio, os, time +from pathlib import Path +from autoskillit.core import InFlightOperations, operation_lease +async def hold(): + async with operation_lease( + Path(os.environ["AUTOSKILLIT_OPERATION_LEASE_DIR"]), + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + ): + await asyncio.sleep(60) +asyncio.run(hold())''' +producer = subprocess.Popen([sys.executable, "-c", helper], env=os.environ.copy()) +deadline = time.monotonic() + 5 +while not list(channel.glob("*.lease.json")) and time.monotonic() < deadline: + time.sleep(0.01) +producer.kill() +producer.wait(timeout=3) +lease_path = next(channel.glob("*.lease.json")) +stale = time.time() - 91 +os.utime(lease_path, (stale, stale)) +time.sleep(30) +""", + }[lease_mode] + script.write_text( + textwrap.dedent( + """\ + """ + ) + + "import asyncio, json, os, sys, time\n" + + "from pathlib import Path\n" + + "from autoskillit.core import InFlightOperations, operation_lease\n" + + 'channel = Path(os.environ["AUTOSKILLIT_OPERATION_LEASE_DIR"])\n' + + 'print(json.dumps({"type": "assistant", "burst": True}), flush=True)\n' + + textwrap.dedent(setup) + + "\ntime.sleep(30)\n" + ) + monkeypatch.setattr( + _patch_process__process_monitor, + "_has_active_api_connection", + lambda _pid: False, + ) + + started = time.monotonic() + with anyio.fail_after(20): + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=30, + idle_output_timeout=2.0, + max_suppression_seconds=2.0, + operation_lease_dir=channel, + env={**os.environ, OPERATION_LEASE_DIR_ENV_VAR: str(channel)}, + ) + + elapsed = time.monotonic() - started + assert result.termination == TerminationReason.IDLE_STALL + if lease_mode == "expired": + assert elapsed >= 2.5 + assert elapsed < 15 + + class TestCaptureMode: """Capture-mode retains stream files and populates path fields.""" diff --git a/tests/execution/test_recording.py b/tests/execution/test_recording.py index 780ddeb0d3..b122769fa2 100644 --- a/tests/execution/test_recording.py +++ b/tests/execution/test_recording.py @@ -768,6 +768,29 @@ async def test_recording_runner_forwards_marker_dir_and_session_id(tmp_path): assert kwargs["session_id"] == "sess-abc" +@pytest.mark.anyio +async def test_recording_runner_forwards_operation_lease_dir(tmp_path): + mock_recorder = Mock() + inner = MockSubprocessRunner() + inner.set_default(_make_result(returncode=0)) + runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) + channel = tmp_path / "operation-leases" + + await runner( + ["task", "test-check"], + cwd=tmp_path, + timeout=60, + env={"SCENARIO_STEP_NAME": "test-check"}, + pty_mode=False, + marker_dir=tmp_path / "markers", + session_id="sess-abc", + operation_lease_dir=channel, + ) + + kwargs = inner.call_args_list[0][3] + assert kwargs["operation_lease_dir"] == channel + + @pytest.mark.anyio async def test_recording_runner_forwards_pass_fds_to_physical_inner(tmp_path): mock_recorder = Mock() @@ -885,6 +908,24 @@ async def test_replaying_runner_accepts_marker_params(tmp_path): assert result.stdout == "ok" +@pytest.mark.anyio +async def test_replaying_runner_accepts_operation_lease_dir(tmp_path): + non_session = {"check": {"exit_code": 0, "stdout_head": "ok", "stderr": ""}} + runner = ReplayingSubprocessRunner({}, non_session) + + result = await runner( + ["task", "test-check"], + cwd=tmp_path, + timeout=60, + env={"SCENARIO_STEP_NAME": "check"}, + marker_dir=tmp_path / "markers", + session_id="sess-123", + operation_lease_dir=tmp_path / "operation-leases", + ) + + assert result.returncode == 0 + + # --- T-DETECT-CODEX: _detect_backend_format returns 'codex' when sidecar exists --- diff --git a/tests/execution/test_termination_executor.py b/tests/execution/test_termination_executor.py index 4e15c506a0..cc51a12050 100644 --- a/tests/execution/test_termination_executor.py +++ b/tests/execution/test_termination_executor.py @@ -151,7 +151,10 @@ async def test_active_child_deferral_runs_until_ceiling( liveness_checks = 0 def has_active_signals( - _pid: int, _marker_dir: Path | None, _session_id: str | None + _pid: int, + _marker_dir: Path | None, + _session_id: str | None, + operation_lease_dir: Path | None = None, ) -> frozenset[str]: nonlocal liveness_checks liveness_checks += 1 @@ -185,7 +188,10 @@ async def test_zero_child_deferral_ceiling_skips_liveness_check( owner = await _spawn(30, tmp_path) def unexpected_liveness_check( - _pid: int, _marker_dir: Path | None, _session_id: str | None + _pid: int, + _marker_dir: Path | None, + _session_id: str | None, + operation_lease_dir: Path | None = None, ) -> frozenset[str]: pytest.fail("zero child deferral ceiling must skip liveness checks") @@ -219,7 +225,7 @@ async def test_child_deferral_stops_when_children_become_inactive( monkeypatch.setattr( _patch_process__termination, "_active_liveness_signals", - lambda _pid, _marker_dir, _session_id: next(activity), + lambda _pid, _marker_dir, _session_id, operation_lease_dir=None: next(activity), ) kill_reason, _returncode, cleanup = await execute_termination_action( diff --git a/tests/fakes.py b/tests/fakes.py index 59fe9260b6..d22e8c97bc 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -1802,9 +1802,12 @@ async def __call__( on_process_spawned: Callable[[int, int], None] | None = None, on_process_reaped: Callable[[int, int], None] | None = None, pass_fds: tuple[int, ...] = (), + operation_lease_dir: Path | None = None, **kwargs: object, ) -> SubprocessResult: kwargs["pass_fds"] = pass_fds + if operation_lease_dir is not None: + kwargs["operation_lease_dir"] = operation_lease_dir kwargs["on_process_spawned"] = on_process_spawned kwargs["on_process_reaped"] = on_process_reaped self.call_args_list.append((cmd, cwd, timeout, kwargs)) diff --git a/tests/fleet/test_fleet_e2e.py b/tests/fleet/test_fleet_e2e.py index 91c51a647b..6a4b3a43e1 100644 --- a/tests/fleet/test_fleet_e2e.py +++ b/tests/fleet/test_fleet_e2e.py @@ -156,6 +156,8 @@ def __init__(self) -> None: self.call_count: int = 0 self.last_pid: int = 0 self.last_pass_fds: tuple[int, ...] = () + self.last_kwargs: dict[str, Any] = {} + self.operation_lease_dir_exists_during_call = False async def __call__( self, @@ -181,6 +183,11 @@ async def __call__( self.call_count += 1 self.last_pass_fds = pass_fds + self.last_kwargs = {**kwargs, "env": env, "cwd": cwd, "timeout": timeout} + operation_lease_dir = kwargs.get("operation_lease_dir") + self.operation_lease_dir_exists_during_call = ( + isinstance(operation_lease_dir, Path) and operation_lease_dir.is_dir() + ) proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -278,6 +285,7 @@ async def dispatch( ingredients: dict[str, str] | None = None, dispatch_name: str | None = None, timeout_sec: int | None = None, + idle_output_timeout: int | None = None, shim_mode: str = "success", sleep_sec: float | None = None, ) -> dict[str, Any]: @@ -292,6 +300,7 @@ async def dispatch( ingredients=ingredients, # type: ignore[arg-type] dispatch_name=dispatch_name, timeout_sec=timeout_sec, + idle_output_timeout=idle_output_timeout, prompt_builder=_simple_prompt_builder, quota_refresher=_noop_quota_refresher, ) @@ -460,6 +469,40 @@ async def test_two_dispatch_happy_path(fleet_runtime: FleetRuntime) -> None: assert d.ended_at > d.started_at +@pytest.mark.anyio +async def test_food_truck_lease_channel_and_l2_idle_floor_on_claude( + fleet_runtime: FleetRuntime, +) -> None: + runtime = fleet_runtime + runtime.add_recipe("lease-floor-recipe") + runtime.tool_ctx.config.run_skill.timeout = 3600 + runtime.tool_ctx.config.fleet.idle_output_timeout = 1800 + channels: list[Path] = [] + + for dispatch_name, override, expected_idle in ( + ("below-floor", 1800, 3600), + ("above-floor", 4200, 4200), + ): + result = await runtime.dispatch( + "lease-floor-recipe", + dispatch_name=dispatch_name, + idle_output_timeout=override, + ) + assert result["success"] is True + + call = runtime.runner.last_kwargs + channel = call["operation_lease_dir"] + assert isinstance(channel, Path) and channel.is_absolute() + assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) + assert runtime.runner.operation_lease_dir_exists_during_call is True + assert call["idle_output_timeout"] == expected_idle + assert call["env"]["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "1800" + assert not channel.exists() + channels.append(channel) + + assert channels[0] != channels[1] + + @pytest.mark.anyio async def test_halt_on_first_failure_default(fleet_runtime: FleetRuntime) -> None: """Failure detection + campaign halt with continue_on_failure=False.""" diff --git a/tests/fleet/test_fleet_e2e_codex.py b/tests/fleet/test_fleet_e2e_codex.py index fe261c0f69..10a11b0660 100644 --- a/tests/fleet/test_fleet_e2e_codex.py +++ b/tests/fleet/test_fleet_e2e_codex.py @@ -109,6 +109,46 @@ def _add_recipe(recipes: InMemoryRecipeRepository, name: str) -> None: class TestCodexFleetE2E: """REQ-SHIM-001: codex shim dispatch through event-driven result path.""" + @pytest.mark.anyio + async def test_food_truck_lease_channel_and_l2_idle_floor( + self, codex_runtime: dict[str, Any] + ) -> None: + ctx = codex_runtime["tool_ctx"] + recipes = codex_runtime["recipes"] + runner: FleetTestRunner = codex_runtime["runner"] + _add_recipe(recipes, "lease-floor-recipe") + ctx.config.run_skill.timeout = 3600 + ctx.config.fleet.idle_output_timeout = 1800 + channels: list[Path] = [] + + for dispatch_name, override, expected_idle in ( + ("below-floor", 1800, 3600), + ("above-floor", 4200, 4200), + ): + await execute_dispatch( + tool_ctx=ctx, + recipe="lease-floor-recipe", + task="verify operation lease launch wiring", + ingredients=None, + dispatch_name=dispatch_name, + timeout_sec=None, + idle_output_timeout=override, + prompt_builder=_simple_prompt_builder, + quota_refresher=_noop_quota_refresher, + ) + + call = runner.last_kwargs + channel = call["operation_lease_dir"] + assert isinstance(channel, Path) and channel.is_absolute() + assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) + assert runner.operation_lease_dir_exists_during_call is True + assert call["idle_output_timeout"] == expected_idle + assert call["env"]["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "1800" + assert not channel.exists() + channels.append(channel) + + assert channels[0] != channels[1] + @pytest.fixture() def codex_runtime( self, tool_ctx: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/infra/test_plugin_source_ratchets.py b/tests/infra/test_plugin_source_ratchets.py index 1c6e1c5b7c..5f7d484b8e 100644 --- a/tests/infra/test_plugin_source_ratchets.py +++ b/tests/infra/test_plugin_source_ratchets.py @@ -50,6 +50,11 @@ } PLUGIN_MUTATION_ALLOWLIST: dict[tuple[str, str, str], tuple[int, str]] = { + ("execution/headless/_managed/_attempt.py", "_operation_lease_channel", "shutil.rmtree"): ( + 1, + "Removes only the unique physical-attempt directory created with exist_ok=False " + "when its supervision-channel context exits.", + ), ("execution/report_index.py", "_update", "(index_dir / _ROWS_FILE).unlink"): ( 1, "Rebuild removes derived rows while holding the report-index writer lease.", From 6582302b7c19a20f754e347182f4366bad45922d Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 22:29:53 -0700 Subject: [PATCH 05/37] fix: preserve liveness evidence across stdout file loss --- .../execution/process/_race_watchers.py | 1 - tests/execution/test_headless_execute.py | 19 +++++++++---------- .../test_operation_lease_watchers.py | 10 ++++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/autoskillit/execution/process/_race_watchers.py b/src/autoskillit/execution/process/_race_watchers.py index c9dee0438a..8756f97ddd 100644 --- a/src/autoskillit/execution/process/_race_watchers.py +++ b/src/autoskillit/execution/process/_race_watchers.py @@ -85,7 +85,6 @@ async def _watch_stdout_idle( try: current_size = stdout_path.stat().st_size except OSError: - last_size = 0 continue if current_size > last_size: last_size = current_size diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index 0d6cd34cfc..2cae9ae9ce 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -587,7 +587,6 @@ async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_l from autoskillit.execution.backends import CodexBackend from autoskillit.execution.backends.claude import ClaudeCodeBackend from autoskillit.execution.headless import run_headless_core - from tests.execution.conftest import _mock_backend from tests.fakes import MockSubprocessRunner monkeypatch.chdir(tmp_path) @@ -597,18 +596,16 @@ async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_l minimal_ctx.config.agent_backend.backend = backend_name minimal_ctx.config.linux_tracing.log_dir = str(tmp_path) real_backend = CodexBackend() if backend_name == "codex" else ClaudeCodeBackend() - backend = _mock_backend(pty_required=True, channel_b_capable=True) - backend.name = backend_name - backend.capabilities = real_backend.capabilities built_specs = [] + original_builder = real_backend.build_skill_session_cmd def build_skill_session_spec(*args, **kwargs): - spec = real_backend.build_skill_session_cmd(*args, **kwargs) + spec = original_builder(*args, **kwargs) built_specs.append(spec) return spec - backend.build_skill_session_cmd.side_effect = build_skill_session_spec - minimal_ctx.backend = backend + monkeypatch.setattr(real_backend, "build_skill_session_cmd", build_skill_session_spec) + minimal_ctx.backend = real_backend runner_calls: list[tuple[Path, bool]] = [] @@ -633,10 +630,12 @@ async def capture_attempt(*args, **kwargs): _patch_headless__headless_execute, "_run_headless_attempt", capture_attempt ) - await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45) - await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45) + outcomes = [ + await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45), + await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45), + ] - assert len(runner.call_args_list) == 2 + assert len(runner.call_args_list) == 2, [outcome.to_json() for outcome in outcomes] assert len(built_specs) == 2 assert len(attempt_specs) == 2 channels = [] diff --git a/tests/execution/test_operation_lease_watchers.py b/tests/execution/test_operation_lease_watchers.py index ad5390999f..07c218efeb 100644 --- a/tests/execution/test_operation_lease_watchers.py +++ b/tests/execution/test_operation_lease_watchers.py @@ -210,13 +210,15 @@ async def test_stdout_file_loss_during_lease_keeps_last_evidence_clock( _install_clock(monkeypatch, clock, race_watchers) stdout = MagicMock() stdout.stat.side_effect = [ + SimpleNamespace(st_size=1), OSError("missing"), OSError("missing"), OSError("missing"), OSError("missing"), - SimpleNamespace(st_size=0), - SimpleNamespace(st_size=0), - SimpleNamespace(st_size=0), + SimpleNamespace(st_size=1), + SimpleNamespace(st_size=1), + SimpleNamespace(st_size=1), + SimpleNamespace(st_size=1), ] acc = RaceAccumulator() trigger = anyio.Event() @@ -231,7 +233,7 @@ async def test_stdout_file_loss_during_lease_keeps_last_evidence_clock( ) assert acc.idle_stall is True - assert clock.elapsed >= 8 + assert clock.elapsed == 8 @pytest.mark.anyio From 90de3c941b983898fe7c860ebf040d21a1e44d15 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 22:46:26 -0700 Subject: [PATCH 06/37] test: isolate real backend capture and assert restored-file staleness --- tests/execution/test_headless_execute.py | 8 ++++---- tests/execution/test_process_idle_watchdog.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index 2cae9ae9ce..d819fc2eee 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -597,14 +597,14 @@ async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_l minimal_ctx.config.linux_tracing.log_dir = str(tmp_path) real_backend = CodexBackend() if backend_name == "codex" else ClaudeCodeBackend() built_specs = [] - original_builder = real_backend.build_skill_session_cmd + original_builder = type(real_backend).build_skill_session_cmd - def build_skill_session_spec(*args, **kwargs): - spec = original_builder(*args, **kwargs) + def build_skill_session_spec(self, *args, **kwargs): + spec = original_builder(self, *args, **kwargs) built_specs.append(spec) return spec - monkeypatch.setattr(real_backend, "build_skill_session_cmd", build_skill_session_spec) + monkeypatch.setattr(type(real_backend), "build_skill_session_cmd", build_skill_session_spec) minimal_ctx.backend = real_backend runner_calls: list[tuple[Path, bool]] = [] diff --git a/tests/execution/test_process_idle_watchdog.py b/tests/execution/test_process_idle_watchdog.py index 4521ea2eda..2b1f4f73dc 100644 --- a/tests/execution/test_process_idle_watchdog.py +++ b/tests/execution/test_process_idle_watchdog.py @@ -319,7 +319,7 @@ async def create_file_after_delay() -> None: @pytest.mark.anyio -async def test_watch_stdout_idle_resets_size_after_stat_failure( +async def test_watch_stdout_idle_restored_file_does_not_fabricate_growth( monkeypatch: pytest.MonkeyPatch, ) -> None: clock = [0.0] @@ -334,7 +334,7 @@ async def advance_clock(_seconds: float) -> None: trigger.set() first_stat = MagicMock(st_size=10) - recreated_stat = MagicMock(st_size=2) + recreated_stat = MagicMock(st_size=10) stdout_path = MagicMock() stdout_path.stat.side_effect = [first_stat, OSError(), recreated_stat] monkeypatch.setattr(_patch_process__race_watchers.anyio, "sleep", advance_clock) @@ -342,7 +342,8 @@ async def advance_clock(_seconds: float) -> None: await _watch_stdout_idle(stdout_path, 1.5, acc, trigger, 1.0) - assert acc.idle_stall is False + assert acc.idle_stall is True + assert clock[0] == 3.0 assert stdout_path.stat.call_count == 3 From 882bf52a253a038faeae5beb1d29cfcb5de3d1ca Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 23:10:02 -0700 Subject: [PATCH 07/37] test: provide Codex add-dir in lease-channel launch test --- tests/execution/test_headless_execute.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index d819fc2eee..435a35d820 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -11,7 +11,7 @@ import structlog import autoskillit.execution.headless._headless_execute as _patch_headless__headless_execute -from autoskillit.core import CmdSpec +from autoskillit.core import CmdSpec, ValidatedAddDir from autoskillit.core.types import SubprocessResult, TerminationReason from tests.execution.conftest import _sink_env @@ -600,6 +600,14 @@ async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_l original_builder = type(real_backend).build_skill_session_cmd def build_skill_session_spec(self, *args, **kwargs): + if backend_name == "codex" and not kwargs.get("add_dirs"): + kwargs["add_dirs"] = ( + ValidatedAddDir( + path=str(tmp_path / "add-dir"), + session_home=str(tmp_path), + skill_entries=(("test", "test/SKILL.md"),), + ), + ) spec = original_builder(self, *args, **kwargs) built_specs.append(spec) return spec From d0d3c8d172af5749693db1ede519a227a6dcd465 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 23:29:15 -0700 Subject: [PATCH 08/37] test: supply Codex skill catalog before launch --- tests/execution/test_headless_execute.py | 31 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index 435a35d820..654c647241 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -600,14 +600,6 @@ async def test_real_backend_launches_keep_idle_policy_out_of_cmd_spec_and_pass_l original_builder = type(real_backend).build_skill_session_cmd def build_skill_session_spec(self, *args, **kwargs): - if backend_name == "codex" and not kwargs.get("add_dirs"): - kwargs["add_dirs"] = ( - ValidatedAddDir( - path=str(tmp_path / "add-dir"), - session_home=str(tmp_path), - skill_entries=(("test", "test/SKILL.md"),), - ), - ) spec = original_builder(self, *args, **kwargs) built_specs.append(spec) return spec @@ -637,10 +629,29 @@ async def capture_attempt(*args, **kwargs): monkeypatch.setattr( _patch_headless__headless_execute, "_run_headless_attempt", capture_attempt ) + add_dirs = ( + ValidatedAddDir( + path=str(tmp_path / "add-dir"), + session_home=str(tmp_path), + skill_entries=(("test", "test/SKILL.md"),), + ), + ) outcomes = [ - await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45), - await run_headless_core("/test foo", str(tmp_path), minimal_ctx, idle_output_timeout=45), + await run_headless_core( + "/test foo", + str(tmp_path), + minimal_ctx, + add_dirs=add_dirs, + idle_output_timeout=45, + ), + await run_headless_core( + "/test foo", + str(tmp_path), + minimal_ctx, + add_dirs=add_dirs, + idle_output_timeout=45, + ), ] assert len(runner.call_args_list) == 2, [outcome.to_json() for outcome in outcomes] From 95216c841f72ff678f26399f18afdb60021d6739 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 23:56:59 -0700 Subject: [PATCH 09/37] test: build valid generated home for lease launch coverage --- tests/execution/test_headless_execute.py | 34 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index 654c647241..7248322d97 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -629,10 +629,40 @@ async def capture_attempt(*args, **kwargs): monkeypatch.setattr( _patch_headless__headless_execute, "_run_headless_attempt", capture_attempt ) + session_home = tmp_path / "session-home" + route = real_backend.conventions.managed_skill_discovery + assert route is not None + catalog_dir = route.catalog_dir(session_home) + skill_file = catalog_dir / "test" / "SKILL.md" + skill_file.parent.mkdir(parents=True) + skill_content = "---\nname: test\ndescription: Test session skill.\n---\n# Test\n" + skill_file.write_text(skill_content) + + if route.entry_point_is_alias: + discovery_entry = route.discovery_root(session_home) + assert discovery_entry is not None + assert route.alias_target is not None + discovery_entry.symlink_to(route.alias_target, target_is_directory=True) + + if backend_name == "codex": + from autoskillit.workspace.session_skills._materialization import ( + _create_inert_rollout_paths, + ) + + auth_source = session_home / "auth-source.json" + auth_source.write_text("{}") + (session_home / "auth.json").symlink_to(auth_source) + (session_home / "config.toml").write_text( + "[mcp_servers.autoskillit]\nname = 'autoskillit'\n" + ) + _create_inert_rollout_paths(session_home, real_backend) + + assert real_backend.validate_skill_content(skill_content) == [] + assert real_backend.validate_session_layout(session_home) == [] add_dirs = ( ValidatedAddDir( - path=str(tmp_path / "add-dir"), - session_home=str(tmp_path), + path=str(session_home / "add-dir"), + session_home=str(session_home), skill_entries=(("test", "test/SKILL.md"),), ), ) From c7cc19c1e80aa9c959c0ea1918464ab09c19898a Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 00:10:53 -0700 Subject: [PATCH 10/37] test: capture leases from real backend launches --- tests/execution/test_headless_execute.py | 35 ++++++++---------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index 7248322d97..b93cbe5e75 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -618,17 +618,6 @@ async def __call__(self, *args, **kwargs): runner = CapturingRunner() runner.set_default(_success_result()) minimal_ctx.runner = runner - attempt_specs = [] - original_attempt = _patch_headless__headless_execute._run_headless_attempt - - async def capture_attempt(*args, **kwargs): - result, spec = await original_attempt(*args, **kwargs) - attempt_specs.append(spec) - return result, spec - - monkeypatch.setattr( - _patch_headless__headless_execute, "_run_headless_attempt", capture_attempt - ) session_home = tmp_path / "session-home" route = real_backend.conventions.managed_skill_discovery assert route is not None @@ -645,17 +634,16 @@ async def capture_attempt(*args, **kwargs): discovery_entry.symlink_to(route.alias_target, target_is_directory=True) if backend_name == "codex": - from autoskillit.workspace.session_skills._materialization import ( - _create_inert_rollout_paths, - ) - auth_source = session_home / "auth-source.json" auth_source.write_text("{}") (session_home / "auth.json").symlink_to(auth_source) (session_home / "config.toml").write_text( "[mcp_servers.autoskillit]\nname = 'autoskillit'\n" ) - _create_inert_rollout_paths(session_home, real_backend) + for name in sorted(real_backend.capabilities.session_dir_symlinks): + inert_target = session_home / f".inert-{name}" + inert_target.mkdir(mode=0o700) + (session_home / name).symlink_to(inert_target.name, target_is_directory=True) assert real_backend.validate_skill_content(skill_content) == [] assert real_backend.validate_session_layout(session_home) == [] @@ -686,17 +674,16 @@ async def capture_attempt(*args, **kwargs): assert len(runner.call_args_list) == 2, [outcome.to_json() for outcome in outcomes] assert len(built_specs) == 2 - assert len(attempt_specs) == 2 + assert all(not hasattr(spec, "process_idle_timeout_ms") for spec in built_specs) channels = [] - for index, final_spec in enumerate(attempt_specs): - channel = runner.call_args_list[index][3]["operation_lease_dir"] + for index, call in enumerate(runner.call_args_list): + kwargs = call[3] + channel = kwargs["operation_lease_dir"] assert channel.is_absolute() - assert channel == Path(final_spec.env["AUTOSKILLIT_OPERATION_LEASE_DIR"]) - assert ( - runner.call_args_list[index][3]["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] != "/parent" - ) + assert kwargs["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) + assert kwargs["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] != "/parent" assert runner_calls[index] == (channel, True) - assert runner.call_args_list[index][3]["idle_output_timeout"] == 45 + assert kwargs["idle_output_timeout"] == 45 assert not channel.exists() channels.append(channel) assert channels[0] != channels[1] From 4d07c0ffce45b0fc51dfd307da8ad4cb9db8516d Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 00:50:47 -0700 Subject: [PATCH 11/37] fix: decouple run-skill attestation from watchdog keying --- .../core/pipeline/_execution_marker.py | 17 ++-- .../protocols/_type_protocols_execution.py | 2 - .../execution/headless/__init__.py | 4 - .../execution/headless/_managed/_executor.py | 4 - .../tools_execution/_fixed_batch_handlers.py | 1 - .../tools_execution/_run_skill_finalize.py | 2 - tests/_test_filter.py | 2 + tests/arch/test_execution_marker_consumers.py | 49 +++++++++++ tests/contracts/test_protocol_satisfaction.py | 17 ++++ tests/execution/test_backend_dispatch.py | 2 - .../test_headless_provider_forwarding.py | 83 ------------------- tests/fakes.py | 6 -- tests/server/test_tools_execution_command.py | 61 +++++++++++++- 13 files changed, 136 insertions(+), 114 deletions(-) create mode 100644 tests/arch/test_execution_marker_consumers.py diff --git a/src/autoskillit/core/pipeline/_execution_marker.py b/src/autoskillit/core/pipeline/_execution_marker.py index e4b5458e33..e75e585343 100644 --- a/src/autoskillit/core/pipeline/_execution_marker.py +++ b/src/autoskillit/core/pipeline/_execution_marker.py @@ -1,9 +1,9 @@ -"""Unified execution marker protocol for stale-detector suppression. +"""Markers for run-skill attestation and fleet-dispatch watchdog suppression. -Async context manager that writes a ``{label}-in-progress-{session_id}-{uuid}.marker`` -file, heartbeats its mtime, and deletes it on exit. Lives in ``core/`` (IL-0) so -both ``fleet/_api.py`` (IL-2) and ``server/tools/`` (IL-3) can import it without -violating layer constraints. +The run-skill marker attests a live invocation to the fabricated-completion guard, +keyed by the exact caller hook session id. It is not a supervisor liveness key; +run-skill watchdogs use operation leases. Fleet dispatch markers still suppress +watchdog signals while a dispatch is active. """ from __future__ import annotations @@ -45,11 +45,12 @@ async def execution_marker( label: str, heartbeat_interval: float = 30.0, ) -> AsyncGenerator[Path | None]: - """Write, heartbeat, and clean up an execution marker. + """Write, heartbeat, and clean up a run-skill or dispatch marker. Yields the marker ``Path`` on success, or ``None`` when ``marker_dir`` is - ``None`` or the initial write fails (suppression disabled, same semantics - as ``fleet/_api.py``'s dispatch marker). + ``None`` or the initial write fails. The run-skill marker attests the + invocation to the fabricated-completion guard and is not passed to its + watchdog. The dispatch marker continues to gate fleet watchdog signals. """ if marker_dir is None: yield None diff --git a/src/autoskillit/core/types/protocols/_type_protocols_execution.py b/src/autoskillit/core/types/protocols/_type_protocols_execution.py index be74ed89ba..e8fc362b9b 100644 --- a/src/autoskillit/core/types/protocols/_type_protocols_execution.py +++ b/src/autoskillit/core/types/protocols/_type_protocols_execution.py @@ -354,8 +354,6 @@ async def run( resume_checkpoint: SessionCheckpoint | None = None, resume_message: str | None = None, backend_authority: BackendAuthority | None = None, - marker_dir: Path | None = None, - caller_session_id: str | None = None, inspector_eligible: bool = False, inspector_model: str = "", network_access: bool = False, diff --git a/src/autoskillit/execution/headless/__init__.py b/src/autoskillit/execution/headless/__init__.py index f013a92ad3..72bdefd9ca 100644 --- a/src/autoskillit/execution/headless/__init__.py +++ b/src/autoskillit/execution/headless/__init__.py @@ -165,8 +165,6 @@ async def run_headless_core( resume_checkpoint: SessionCheckpoint | None = None, resume_message: str | None = None, backend_authority: BackendAuthority | None = None, - marker_dir: Path | None = None, - caller_session_id: str | None = None, inspector_eligible: bool = False, inspector_model: str = "", network_access: bool = False, @@ -320,8 +318,6 @@ async def run_headless_core( launch_preparation=launch.launch_preparation, resume_launch_contract=resume_launch_contract, model_identity=launch.model_identity, - marker_dir=marker_dir, - session_id=caller_session_id, backend_resume_session_id=resume_session_id, inspector_eligible=inspector_eligible, inspector_model=inspector_model, diff --git a/src/autoskillit/execution/headless/_managed/_executor.py b/src/autoskillit/execution/headless/_managed/_executor.py index 7f52c9c65f..8ba95d5457 100644 --- a/src/autoskillit/execution/headless/_managed/_executor.py +++ b/src/autoskillit/execution/headless/_managed/_executor.py @@ -71,8 +71,6 @@ async def run( resume_checkpoint: SessionCheckpoint | None = None, resume_message: str | None = None, backend_authority: BackendAuthority | None = None, - marker_dir: Path | None = None, - caller_session_id: str | None = None, inspector_eligible: bool = False, inspector_model: str = "", network_access: bool = False, @@ -135,8 +133,6 @@ async def run( resume_checkpoint=resume_checkpoint, resume_message=resume_message, backend_authority=backend_authority, - marker_dir=marker_dir, - caller_session_id=caller_session_id, inspector_eligible=inspector_eligible, inspector_model=inspector_model, network_access=network_access, diff --git a/src/autoskillit/server/tools/tools_execution/_fixed_batch_handlers.py b/src/autoskillit/server/tools/tools_execution/_fixed_batch_handlers.py index ec52a38d8c..a89e253160 100644 --- a/src/autoskillit/server/tools/tools_execution/_fixed_batch_handlers.py +++ b/src/autoskillit/server/tools/tools_execution/_fixed_batch_handlers.py @@ -215,7 +215,6 @@ async def _execute_leaf( tier=BackendAuthorityTier.GLOBAL, key_path="managed_fixed_batch", ), - caller_session_id=self.launch.parent_session_id, child_role=leaf_projection.binding.assignment.role, child_attribution_skill=self.source_name, provider_extras=( diff --git a/src/autoskillit/server/tools/tools_execution/_run_skill_finalize.py b/src/autoskillit/server/tools/tools_execution/_run_skill_finalize.py index 1a16fbacaa..73a01e0ee5 100644 --- a/src/autoskillit/server/tools/tools_execution/_run_skill_finalize.py +++ b/src/autoskillit/server/tools/tools_execution/_run_skill_finalize.py @@ -397,8 +397,6 @@ def _mark_execution_started() -> None: backend_authority=state._backend_authority, resume_session_id=state.resume_session_id, resume_launch_contract=state._resume_launch_contract, - marker_dir=state._marker_dir, - caller_session_id=state._caller_hook_session_id, inspector_eligible=state._in_fleet_dispatch and bool(state._inspector_model), inspector_model=state._inspector_model, diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 16736fb36d..6fc08e6b5a 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -1497,6 +1497,8 @@ class SourceMapRejection(enum.StrEnum): "server/test_tools_exploration.py", # run_cmd write-boundary tests import the shared hooks scanner. "server/test_guards_module.py", + # file-level: run_skill attestation tests inspect the fabricated-completion guard. + "server/test_tools_execution_command.py", # file-level: formatter-renders-real-content test imports pretty_output_hook # directly to exercise the _fmt_open_kitchen contract — see #4399 criterion 4 "server/test_tools_kitchen_envelope_failure.py", diff --git a/tests/arch/test_execution_marker_consumers.py b/tests/arch/test_execution_marker_consumers.py new file mode 100644 index 0000000000..49957beb0f --- /dev/null +++ b/tests/arch/test_execution_marker_consumers.py @@ -0,0 +1,49 @@ +"""Prevent run-skill attestation markers from becoming watchdog keys.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.medium + + +def _call_name(node: ast.expr) -> str: + parts: list[str] = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if isinstance(node, ast.Name): + parts.append(node.id) + return ".".join(reversed(parts)) + + +def _is_run_skill_liveness_call(node: ast.Call) -> bool: + parts = _call_name(node.func).split(".") + return parts[-1:] == ["run_headless_core"] or ( + len(parts) >= 2 + and parts[-1] == "run" + and parts[-2] in {"executor", "HeadlessExecutor", "DefaultHeadlessExecutor"} + ) + + +def test_run_skill_marker_is_not_passed_as_a_watchdog_key() -> None: + repository_root = Path(__file__).resolve().parents[2] + violations: list[str] = [] + for layer in ("execution", "server"): + source_root = repository_root / "src" / "autoskillit" / layer + for source_path in source_root.rglob("*.py"): + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not _is_run_skill_liveness_call(node): + continue + if any(keyword.arg == "marker_dir" for keyword in node.keywords): + relative_path = source_path.relative_to(repository_root) + violations.append(f"{relative_path}:{node.lineno}") + + assert not violations, ( + "the run-skill marker is the fabricated-completion attestation; " + "supervisor liveness comes from operation leases\n" + "\n".join(violations) + ) diff --git a/tests/contracts/test_protocol_satisfaction.py b/tests/contracts/test_protocol_satisfaction.py index 3aae98d30d..2b2818395c 100644 --- a/tests/contracts/test_protocol_satisfaction.py +++ b/tests/contracts/test_protocol_satisfaction.py @@ -385,6 +385,23 @@ def test_default_headless_executor_satisfies_headless_executor(): assert isinstance(DefaultHeadlessExecutor(MagicMock()), HeadlessExecutor) +def test_headless_executor_run_excludes_marker_watchdog_parameters() -> None: + from autoskillit.core import HeadlessExecutor + from autoskillit.execution.headless import DefaultHeadlessExecutor, run_headless_core + from tests.fakes import InMemoryHeadlessExecutor + + run_methods = ( + HeadlessExecutor.run, + DefaultHeadlessExecutor.run, + run_headless_core, + InMemoryHeadlessExecutor.run, + ) + for run_method in run_methods: + params = inspect.signature(run_method).parameters + assert "marker_dir" not in params + assert "caller_session_id" not in params + + def test_headless_executor_protocol_dispatch_has_marker_params(): import inspect diff --git a/tests/execution/test_backend_dispatch.py b/tests/execution/test_backend_dispatch.py index 416be325d6..a2a8e4b732 100644 --- a/tests/execution/test_backend_dispatch.py +++ b/tests/execution/test_backend_dispatch.py @@ -67,7 +67,6 @@ async def test_run_headless_core_uses_ctx_backend_for_command_construction(minim minimal_ctx, completion_marker="%%DONE%%", resume_session_id="backend-resume-id", - caller_session_id="caller-marker-id", ) backend.build_skill_session_cmd.assert_called_once() @@ -77,7 +76,6 @@ async def test_run_headless_core_uses_ctx_backend_for_command_construction(minim config = call_args.args[2] assert config.completion_marker == "%%DONE%%" runner_kwargs = mock_runner.call_args.kwargs - assert runner_kwargs["session_id"] == "caller-marker-id" assert runner_kwargs["backend_resume_session_id"] == "backend-resume-id" assert runner_kwargs["lifecycle_observation_enabled"] is True diff --git a/tests/execution/test_headless_provider_forwarding.py b/tests/execution/test_headless_provider_forwarding.py index e1e154b285..a09a74f00c 100644 --- a/tests/execution/test_headless_provider_forwarding.py +++ b/tests/execution/test_headless_provider_forwarding.py @@ -1153,89 +1153,6 @@ async def fake_runner(cmd, **kwargs): backend.stream_parser.assert_called_once_with(completion_marker="%%TEST_MARKER%%") -@pytest.mark.anyio -async def test_run_headless_core_forwards_marker_dir_and_caller_session_id( - minimal_ctx, tmp_path, monkeypatch -) -> None: - """run_headless_core passes marker_dir + caller_session_id to _execute_claude_headless.""" - from autoskillit.core import CmdSpec - from autoskillit.execution.headless import run_headless_core - - execute_kwargs: dict = {} - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - - backend = _mock_backend(pty_required=True, channel_b_capable=True) - backend.build_skill_session_cmd.return_value = CmdSpec( - cmd=("claude", "--print", "test"), env={} - ) - minimal_ctx.backend = backend - - async def fake_execute(spec, cwd, ctx, **kwargs): - execute_kwargs.update(kwargs) - return _STUB_RESULT - - monkeypatch.setattr(_patch_execution_headless, "_execute_claude_headless", fake_execute) - - await run_headless_core( - "/autoskillit:probe", - str(tmp_path), - minimal_ctx, - marker_dir=marker_dir, - caller_session_id="orchestrator-session-abc", - ) - - assert execute_kwargs.get("marker_dir") == marker_dir - assert execute_kwargs.get("session_id") == "orchestrator-session-abc" - - -@pytest.mark.anyio -async def test_default_executor_run_forwards_marker_dir_and_caller_session_id( - minimal_ctx, tmp_path, monkeypatch -) -> None: - """DefaultHeadlessExecutor.run() passes marker_dir + caller_session_id to run_headless_core.""" - import autoskillit.execution.headless as _headless_mod - from autoskillit.execution.headless import DefaultHeadlessExecutor - - captured: dict = {} - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - - async def fake_core(skill_command, cwd, ctx, **kwargs): - captured.update(kwargs) - return _STUB_RESULT - - monkeypatch.setattr(_headless_mod, "run_headless_core", fake_core) - - executor = DefaultHeadlessExecutor(minimal_ctx) - await executor.run( - "/autoskillit:probe", - str(tmp_path), - marker_dir=marker_dir, - caller_session_id="orchestrator-session-xyz", - ) - - assert captured.get("marker_dir") == marker_dir - assert captured.get("caller_session_id") == "orchestrator-session-xyz" - - -def test_headless_executor_protocol_includes_marker_dir_params() -> None: - """HeadlessExecutor.run() Protocol includes marker_dir and caller_session_id.""" - import inspect - - from autoskillit.core.types import HeadlessExecutor - - sig = inspect.signature(HeadlessExecutor.run) - assert "marker_dir" in sig.parameters, ( - "marker_dir missing from HeadlessExecutor.run() signature" - ) - assert "caller_session_id" in sig.parameters, ( - "caller_session_id missing from HeadlessExecutor.run() signature" - ) - assert sig.parameters["marker_dir"].default is None - assert sig.parameters["caller_session_id"].default is None - - @pytest.mark.anyio async def test_sink_environment_reaches_contract_nudge_and_overrides_caller_values( minimal_ctx, tmp_path, monkeypatch diff --git a/tests/fakes.py b/tests/fakes.py index d22e8c97bc..8f284012e0 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -697,8 +697,6 @@ class ExecutorCall: resume_message: str | None = None resume_launch_contract: ResolvedLaunchContract | None = None backend_authority: BackendAuthority | None = None - marker_dir: Path | None = None - caller_session_id: str | None = None inspector_eligible: bool = False inspector_model: str = "" network_access: bool = False @@ -827,8 +825,6 @@ async def run( resume_message: str | None = None, resume_launch_contract: ResolvedLaunchContract | None = None, backend_authority: BackendAuthority | None = None, - marker_dir: Path | None = None, - caller_session_id: str | None = None, inspector_eligible: bool = False, inspector_model: str = "", network_access: bool = False, @@ -883,8 +879,6 @@ async def run( resume_message=resume_message, resume_launch_contract=resume_launch_contract, backend_authority=backend_authority, - marker_dir=marker_dir, - caller_session_id=caller_session_id, inspector_eligible=inspector_eligible, inspector_model=inspector_model, network_access=network_access, diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py index 09c2f3d156..9940caca9f 100644 --- a/tests/server/test_tools_execution_command.py +++ b/tests/server/test_tools_execution_command.py @@ -362,7 +362,8 @@ async def _capture_marker(_marker_dir, session_id, _label): _ack_direct_run_skill_result(tool_ctx_kitchen_open, payload) assert captured["session_id"] == "session-a" - assert executor.calls[0].caller_session_id == "session-a" + assert not hasattr(executor.calls[0], "marker_dir") + assert not hasattr(executor.calls[0], "caller_session_id") @pytest.mark.anyio @pytest.mark.parametrize( @@ -446,7 +447,63 @@ async def _capture_marker(_marker_dir, session_id, _label): _ack_direct_run_skill_result(tool_ctx_kitchen_open, payload) assert captured["session_id"] == "fallback-session" - assert executor.calls[0].caller_session_id == "fallback-session" + assert not hasattr(executor.calls[0], "marker_dir") + assert not hasattr(executor.calls[0], "caller_session_id") + + @pytest.mark.anyio + async def test_run_skill_attestation_stays_live_without_l1_marker_key( + self, tool_ctx_kitchen_open, monkeypatch + ): + from autoskillit.core import LAUNCH_ID_ENV_VAR + from autoskillit.core.pipeline._execution_marker import execution_marker + from autoskillit.hooks.guards.fabricated_completion_guard import ( + _has_fresh_matching_marker, + ) + from tests.fakes import MockSubprocessRunner + + monkeypatch.delenv(LAUNCH_ID_ENV_VAR, raising=False) + monkeypatch.setattr( + tools_execution, "find_caller_session_id", lambda **_kwargs: "session-a" + ) + marker_state: dict[str, object] = {} + + @contextlib.asynccontextmanager + async def observe_marker(marker_dir, session_id, label): + async with execution_marker(marker_dir, session_id, label) as marker_path: + marker_state.update(path=marker_path, session_id=session_id, label=label) + yield marker_path + + monkeypatch.setattr(tools_execution, "execution_marker", observe_marker) + + runner = tool_ctx_kitchen_open.runner + assert isinstance(runner, MockSubprocessRunner) + + async def check_marker_during_launch(cmd, **kwargs): + if Path(cmd[0]).name == "claude": + marker_path = marker_state.get("path") + session_id = marker_state.get("session_id") + assert isinstance(marker_path, Path) + assert session_id == "session-a" + assert marker_path.name.startswith(f"run-skill-in-progress-{session_id}-") + marker_payload = json.loads(marker_path.read_text(encoding="utf-8")) + assert marker_payload["label"] == "run-skill" + assert marker_payload["session_id"] == session_id + assert _has_fresh_matching_marker(marker_path, session_id) + return await runner(cmd, **kwargs) + + monkeypatch.setattr(tool_ctx_kitchen_open, "runner", check_marker_during_launch) + runner.push(_make_result(returncode=1)) # clone guard snapshot + runner.push(_make_result(returncode=0, stdout=_SUCCESS_JSON)) + + payload = json.loads(await run_skill("/investigate marker attestation", cwd="/tmp")) + _ack_direct_run_skill_result(tool_ctx_kitchen_open, payload) + + claude_calls = [ + call for call in runner.call_args_list if Path(call[0][0]).name == "claude" + ] + assert len(claude_calls) == 1 + assert claude_calls[0][3]["marker_dir"] is None + assert claude_calls[0][3]["session_id"] is None @pytest.mark.anyio async def test_marker_dir_routes_through_session_locator( From 83e0d09f61d7ae3168d36f764ddfb04a475d0869 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 02:26:51 -0700 Subject: [PATCH 12/37] fix: retire marker and network-derived supervisor liveness --- docs/audit/surface-freeze-checklist.md | 1 - .../inspector-contract-impact.md | 2 +- .../cli/session/_session_process.py | 2 +- .../core/pipeline/_execution_marker.py | 13 +- .../core/types/execution/_type_inspector.py | 2 +- .../core/types/execution/_type_subprocess.py | 12 - .../protocols/_type_protocols_execution.py | 2 - .../execution/headless/_headless_execute.py | 6 +- .../execution/headless/_headless_launch.py | 4 - .../headless/_managed/_food_truck_executor.py | 12 - src/autoskillit/execution/process/__init__.py | 12 - .../execution/process/_lifecycle/runner.py | 4 - .../execution/process/_process_monitor.py | 110 +--- .../execution/process/_process_race.py | 6 - .../execution/process/_race_watchers.py | 75 +-- .../execution/process/_termination.py | 10 +- .../execution/recording/recording.py | 14 - src/autoskillit/fleet/dispatch/_api.py | 6 +- .../fleet/dispatch/_classification.py | 7 +- src/autoskillit/fleet/dispatch/_cleanup.py | 6 +- src/autoskillit/fleet/dispatch/_execution.py | 154 +++-- .../arch-constraint-catalog.md | 2 +- tests/arch/test_execution_marker_producers.py | 50 ++ tests/arch/test_watcher_signal_consistency.py | 104 ++-- tests/cli/test_cook_lifetime.py | 14 +- tests/cli/test_cook_process_lifecycle.py | 4 +- tests/contracts/test_protocol_satisfaction.py | 79 +-- .../core/test_grab_bag_split_completeness.py | 10 +- tests/core/test_inspector_types.py | 6 +- tests/core/test_types_protocols.py | 57 +- .../headless/test_codex_attempt_retention.py | 4 - tests/execution/test_backend_dispatch.py | 2 + .../test_headless_provider_forwarding.py | 226 +------ .../test_model_backend_launch_contract.py | 2 - .../test_operation_lease_watchers.py | 21 +- .../test_process_deadline_extension.py | 222 +++---- tests/execution/test_process_heartbeat.py | 193 +----- tests/execution/test_process_idle_watchdog.py | 561 +----------------- tests/execution/test_process_run.py | 50 +- .../test_process_session_log_monitor.py | 124 +--- ...ess_session_log_monitor_dispatch_marker.py | 430 -------------- ...s_session_log_monitor_stale_suppression.py | 329 +++------- tests/execution/test_process_submodules.py | 17 +- tests/execution/test_recording.py | 91 --- tests/execution/test_termination_executor.py | 62 +- tests/fakes.py | 6 - tests/fleet/test_api.py | 143 +---- tests/fleet/test_api_dispatch_marker.py | 242 -------- tests/fleet/test_dispatch_backend_override.py | 20 +- tests/fleet/test_fleet_e2e.py | 14 + tests/fleet/test_fleet_e2e_codex.py | 6 + .../test_native_shell_capture_lineage.py | 6 +- 52 files changed, 624 insertions(+), 2933 deletions(-) create mode 100644 tests/arch/test_execution_marker_producers.py delete mode 100644 tests/execution/test_process_session_log_monitor_dispatch_marker.py delete mode 100644 tests/fleet/test_api_dispatch_marker.py diff --git a/docs/audit/surface-freeze-checklist.md b/docs/audit/surface-freeze-checklist.md index 97c1a25cb2..b03dcf67d1 100644 --- a/docs/audit/surface-freeze-checklist.md +++ b/docs/audit/surface-freeze-checklist.md @@ -174,7 +174,6 @@ autoskillit.core | 15 | `tests/execution/test_process_env_boundary.py` | `ClaudeHeadlessCmd` | Deferred (101) | | 16 | `tests/execution/backends/test_claude_code_backend.py` | `build_headless_cmd` | Deferred (47) | | 17 | `tests/execution/test_process_session_log_monitor_stale_suppression.py` | `_session_log_monitor`, `ChannelBStatus` | Module-level | -| 18 | `tests/execution/test_process_session_log_monitor_dispatch_marker.py` | `_session_log_monitor`, `ChannelBStatus` | Module-level | **Additional indirect consumer:** - `tests/cli/test_cook_env_scrub.py` — imports `_MAX_MCP_OUTPUT_TOKENS_VALUE` from `autoskillit.execution` (package re-export path) diff --git a/docs/design/forward-obligations/inspector-contract-impact.md b/docs/design/forward-obligations/inspector-contract-impact.md index 95c4848829..91902a500f 100644 --- a/docs/design/forward-obligations/inspector-contract-impact.md +++ b/docs/design/forward-obligations/inspector-contract-impact.md @@ -32,7 +32,7 @@ Already defined in `core/types/execution/_type_inspector.py`: - **`InspectorEvidence`** (lines 15–24, frozen dataclass): `idle_seconds`, `stdout_path`, `jsonl_lines`, `cpu_trend`, `rss_trend`, `connection_summary`, - `execution_marker_present`, `dispatch_context` + `operation_lease_active`, `dispatch_context` - **`InspectorVerdict`** (lines 27–32, frozen dataclass): `action`, `reasoning`, `confidence`, `elapsed_seconds` - **`InspectorCallback`** (line 35): `Callable[[InspectorEvidence], Awaitable[InspectorVerdict]]` diff --git a/src/autoskillit/cli/session/_session_process.py b/src/autoskillit/cli/session/_session_process.py index 0bf43d7264..5ee7ef2e47 100644 --- a/src/autoskillit/cli/session/_session_process.py +++ b/src/autoskillit/cli/session/_session_process.py @@ -57,7 +57,7 @@ def _default_activity(pid: int, fd: int | None) -> frozenset[str]: """Read process liveness and recent kernel TTY activity.""" - signals = set(_active_liveness_signals(pid, None, None)) + signals = set(_active_liveness_signals(pid, operation_lease_dir=None)) if fd is not None: last_activity = terminal_last_activity(fd) if last_activity is not None and time.time() - last_activity <= _IDLE_WINDOW_SECONDS: diff --git a/src/autoskillit/core/pipeline/_execution_marker.py b/src/autoskillit/core/pipeline/_execution_marker.py index e75e585343..c7cb61ceac 100644 --- a/src/autoskillit/core/pipeline/_execution_marker.py +++ b/src/autoskillit/core/pipeline/_execution_marker.py @@ -1,9 +1,7 @@ -"""Markers for run-skill attestation and fleet-dispatch watchdog suppression. +"""Run-skill invocation markers for fabricated-completion attestation. The run-skill marker attests a live invocation to the fabricated-completion guard, -keyed by the exact caller hook session id. It is not a supervisor liveness key; -run-skill watchdogs use operation leases. Fleet dispatch markers still suppress -watchdog signals while a dispatch is active. +keyed by the caller hook session id. Supervisor liveness uses operation leases. """ from __future__ import annotations @@ -45,12 +43,11 @@ async def execution_marker( label: str, heartbeat_interval: float = 30.0, ) -> AsyncGenerator[Path | None]: - """Write, heartbeat, and clean up a run-skill or dispatch marker. + """Write, heartbeat, and clean up a run-skill attestation marker. Yields the marker ``Path`` on success, or ``None`` when ``marker_dir`` is - ``None`` or the initial write fails. The run-skill marker attests the - invocation to the fabricated-completion guard and is not passed to its - watchdog. The dispatch marker continues to gate fleet watchdog signals. + ``None`` or the initial write fails. The marker attests the invocation to + the fabricated-completion guard; it does not signal supervisor liveness. """ if marker_dir is None: yield None diff --git a/src/autoskillit/core/types/execution/_type_inspector.py b/src/autoskillit/core/types/execution/_type_inspector.py index 304f99b1c6..f195b38039 100644 --- a/src/autoskillit/core/types/execution/_type_inspector.py +++ b/src/autoskillit/core/types/execution/_type_inspector.py @@ -20,7 +20,7 @@ class InspectorEvidence: cpu_trend: tuple[float, ...] = () rss_trend: tuple[float, ...] = () connection_summary: str = "" - execution_marker_present: bool = False + operation_lease_active: bool = False dispatch_context: str = "" diff --git a/src/autoskillit/core/types/execution/_type_subprocess.py b/src/autoskillit/core/types/execution/_type_subprocess.py index 619f277a13..1d0c08fada 100644 --- a/src/autoskillit/core/types/execution/_type_subprocess.py +++ b/src/autoskillit/core/types/execution/_type_subprocess.py @@ -231,16 +231,6 @@ class SubprocessRunner(Protocol): Parameters ---------- - marker_dir : Path | None - Directory containing ``*-in-progress-{session_id}-*.marker`` files. - When non-None, the session log monitor checks for active execution markers - before issuing stale-kill signals, suppressing kills while a fleet dispatch - or run_skill call is in progress. Default ``None`` (no suppression). - session_id : str | None - Caller's session identity, used to scope execution-marker glob patterns to - the originating session. Threaded from fleet dispatch / run_skill through - headless execution to ``_session_log_monitor``'s ``caller_session_id`` parameter. - Default ``None`` (match any marker). operation_lease_dir : Path | None Per-attempt directory containing the supervised child's in-flight tool leases. The channel is shared with the child through its environment. @@ -272,8 +262,6 @@ def __call__( on_pid_resolved: Callable[[int, int], None] | None = None, enable_deadline_extension: bool = False, max_extension_seconds: float = 7200, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, stream_parser: Any | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), diff --git a/src/autoskillit/core/types/protocols/_type_protocols_execution.py b/src/autoskillit/core/types/protocols/_type_protocols_execution.py index e8fc362b9b..3afacf74cc 100644 --- a/src/autoskillit/core/types/protocols/_type_protocols_execution.py +++ b/src/autoskillit/core/types/protocols/_type_protocols_execution.py @@ -406,8 +406,6 @@ async def dispatch_food_truck( provider_name: str = "", profile_name: str = "", sentinel_contract: str = "", - marker_dir: Path | None = None, - session_id: str | None = None, resume_message: str | None = None, backend_authority: BackendAuthority | None = None, on_session_id_resolved: Callable[[str], None] | None = None, diff --git a/src/autoskillit/execution/headless/_headless_execute.py b/src/autoskillit/execution/headless/_headless_execute.py index 7001d26e26..aebe7be4d6 100644 --- a/src/autoskillit/execution/headless/_headless_execute.py +++ b/src/autoskillit/execution/headless/_headless_execute.py @@ -137,8 +137,6 @@ async def _execute_claude_headless( max_extension_seconds: float = 7200, ceiling_seconds: float = DEFAULT_TETHER_CEILING_SECONDS, systemd_scope_enabled: bool = False, - marker_dir: Path | None = None, - session_id: str | None = None, launch_resolver: LaunchResolver, launch_preparation: LaunchPreparation, resume_launch_contract: ResolvedLaunchContract | None = None, @@ -307,7 +305,7 @@ def observe_execution_started() -> None: child_role=child_role, child_attribution_skill=child_attribution_skill, step_backend=_step_backend, - session_id=session_id, + session_id=caller_session_id, diagnostic_log_dir=ctx.config.linux_tracing.log_dir, on_spawn=on_spawn, on_candidate=lineage_callbacks.on_candidate, @@ -356,8 +354,6 @@ def observe_execution_started() -> None: max_extension_seconds=max_extension_seconds, ceiling_seconds=ceiling_seconds, systemd_scope_enabled=systemd_scope_enabled, - marker_dir=marker_dir, - session_id=session_id, on_session_id_resolved=_bind_managed_launch_alias, stream_parser=_stream_parser, backend_resume_session_id=backend_resume_session_id, diff --git a/src/autoskillit/execution/headless/_headless_launch.py b/src/autoskillit/execution/headless/_headless_launch.py index 06d4ebe736..d4f7d18316 100644 --- a/src/autoskillit/execution/headless/_headless_launch.py +++ b/src/autoskillit/execution/headless/_headless_launch.py @@ -105,8 +105,6 @@ async def _run_headless_attempt( max_extension_seconds: float, ceiling_seconds: float = DEFAULT_TETHER_CEILING_SECONDS, systemd_scope_enabled: bool = False, - marker_dir: Path | None, - session_id: str | None, on_session_id_resolved: Callable[[str], None] | None, stream_parser: StreamParser, backend_resume_session_id: str, @@ -213,8 +211,6 @@ async def _run_headless_attempt( max_extension_seconds=max_extension_seconds, ceiling_seconds=ceiling_seconds, systemd_scope_enabled=systemd_scope_enabled, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=channel, on_session_id_resolved=on_session_id_resolved, stream_parser=stream_parser, diff --git a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py index 0c9989c86e..bc71742bc2 100644 --- a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py +++ b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py @@ -8,7 +8,6 @@ from collections.abc import Callable, Mapping, Sequence from contextlib import AbstractContextManager, nullcontext from dataclasses import replace -from pathlib import Path from typing import cast import anyio @@ -233,8 +232,6 @@ async def dispatch_food_truck( provider_name: str = "", profile_name: str = "", sentinel_contract: str = "", - marker_dir: Path | None = None, - session_id: str | None = None, resume_message: str | None = None, backend_authority: BackendAuthority | None = None, on_session_id_resolved: Callable[[str], None] | None = None, @@ -408,13 +405,6 @@ async def dispatch_food_truck( ) effective_idle_out = max(float(selected_idle), float(cfg.run_skill.timeout)) effective_natural_exit_grace_seconds: float = cfg.run_skill.natural_exit_grace_seconds - effective_marker_dir: Path | None = marker_dir or ( - headless_facade._resolve_session_log_dir( - cwd, cast(CodingAgentBackend, dispatch_backend) - ) - if cwd - else None - ) from autoskillit.execution.session_log.session_log import resolve_log_dir diagnostic_log_root = resolve_log_dir(cfg.linux_tracing.log_dir) @@ -533,8 +523,6 @@ async def _settle_then_admit( max_extension_seconds=effective_max_ext, ceiling_seconds=effective_ceiling_seconds, systemd_scope_enabled=effective_systemd_scope_enabled, - marker_dir=effective_marker_dir, - session_id=session_id, model_identity=model_identity, on_session_id_resolved=on_session_id_resolved, launch_resolver=self._ctx.launch_resolver, diff --git a/src/autoskillit/execution/process/__init__.py b/src/autoskillit/execution/process/__init__.py index 64e42228aa..57959232d1 100644 --- a/src/autoskillit/execution/process/__init__.py +++ b/src/autoskillit/execution/process/__init__.py @@ -83,9 +83,7 @@ ) from autoskillit.execution.process._process_monitor import ( _active_liveness_signals, - _has_active_api_connection, _has_active_child_processes, - _has_active_execution_marker, _heartbeat, _session_log_monitor, ) @@ -175,9 +173,7 @@ "RaceSignals", "find_orphaned_codex_processes", "find_orphaned_autoskillit_daemons", - "_has_active_api_connection", "_has_active_child_processes", - "_has_active_execution_marker", "_heartbeat", "_jsonl_contains_marker", "_jsonl_has_record_type", @@ -265,8 +261,6 @@ async def run_managed_async( _heartbeat_poll: float = 0.5, _phase1_timeout: float = 30.0, _session_id_timeout: float = 1.0, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, inspector_callback: InspectorCallback | None = None, @@ -529,8 +523,6 @@ async def run_managed_async( session_id_timeout=_session_id_timeout, stdout_session_id_ready=stdout_session_id_ready, max_suppression_seconds=max_suppression_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, on_session_id_resolved=on_session_id_resolved, backend_resume_session_id=backend_resume_session_id, @@ -560,8 +552,6 @@ async def run_managed_async( enable_deadline_extension=enable_deadline_extension, observed_pid=_observed_pid, max_extension_seconds=max_extension_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, lifecycle_observation_enabled=lifecycle_observation_enabled, acc=acc, @@ -658,8 +648,6 @@ def _observe_final_channel_b(event: SessionEvent) -> None: proc_log=proc_log, termination=termination, pid=_observed_pid, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, child_deferral_ceiling=child_deferral_ceiling, process_observation_snapshot=signals.process_observation_snapshot, diff --git a/src/autoskillit/execution/process/_lifecycle/runner.py b/src/autoskillit/execution/process/_lifecycle/runner.py index c2c9ba484b..0a43cc0b04 100644 --- a/src/autoskillit/execution/process/_lifecycle/runner.py +++ b/src/autoskillit/execution/process/_lifecycle/runner.py @@ -156,8 +156,6 @@ async def __call__( on_pid_resolved: Callable[[int, int], None] | None = None, enable_deadline_extension: bool = False, max_extension_seconds: float = 7200, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), @@ -198,8 +196,6 @@ async def __call__( enable_deadline_extension=enable_deadline_extension, max_extension_seconds=max_extension_seconds, child_deferral_ceiling=child_deferral_ceiling, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, diff --git a/src/autoskillit/execution/process/_process_monitor.py b/src/autoskillit/execution/process/_process_monitor.py index 57ecaecd4f..457a094ed1 100644 --- a/src/autoskillit/execution/process/_process_monitor.py +++ b/src/autoskillit/execution/process/_process_monitor.py @@ -153,29 +153,6 @@ async def _heartbeat( return "completion" -def _has_active_api_connection(pid: int) -> bool: - """Return True if the process tree rooted at `pid` has an ESTABLISHED TCP - connection to port 443 (the Anthropic API endpoint). - - Used by _session_log_monitor to suppress stale-kill when a long-running - API streaming call is in-flight. - """ - try: - parent = psutil.Process(pid) - for proc in [parent] + parent.children(recursive=True): - try: - get_conns = getattr(proc, "net_connections", proc.connections) - conns = get_conns(kind="tcp") - for conn in conns: - if conn.status == "ESTABLISHED" and conn.raddr and conn.raddr.port == 443: - return True - except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied): - continue - except psutil.NoSuchProcess: - pass - return False - - _CPU_ACTIVE_THRESHOLD: float = 10.0 # percent; evidence of actual computational work # Cached Process objects keyed by PID so cpu_percent(interval=0) returns @@ -227,31 +204,6 @@ def _has_active_child_processes(pid: int) -> bool: return active -def _has_active_execution_marker( - marker_dir: Path, - session_id: str | None = None, - max_marker_age: float = 60.0, -) -> bool: - """Return True if any execution-in-progress marker was touched within max_marker_age secs.""" - try: - now = time.time() - pattern = ( - f"*-in-progress-{session_id}-*.marker" - if session_id is not None - else "*-in-progress-*.marker" - ) - for p in marker_dir.glob(pattern): - try: - st = p.stat() - if now - st.st_mtime <= max_marker_age: - return True - except OSError: - continue - except OSError: - pass - return False - - def _select_session_log_file( session_log_dir: Path, candidates: list[ObservedEntry], @@ -345,29 +297,18 @@ async def _discover_session_log( return session_file -StaleSuppressionReason = Literal[ - "pending_tasks", "api_connection", "child_processes", "dispatch_marker" -] +StaleSuppressionReason = Literal["pending_tasks", "child_processes"] def _active_liveness_signals( pid: int | None, - marker_dir: Path | None, - session_id: str | None, *, operation_lease_dir: Path | None = None, ) -> frozenset[str]: - """Return active network, child, marker, and operation-lease signals. - - The same evidence set is used by the termination drain and the session monitor. - """ + """Return active child-CPU and operation-lease signals.""" signals: set[str] = set() - if pid is not None and _has_active_api_connection(pid): - signals.add("api_connection") if pid is not None and _has_active_child_processes(pid): signals.add("child_processes") - if marker_dir is not None and _has_active_execution_marker(marker_dir, session_id=session_id): - signals.add("dispatch_marker") if operation_lease_dir is not None and read_active_operation_leases( operation_lease_dir, now_epoch=time.time() ): @@ -378,19 +319,13 @@ def _active_liveness_signals( def _stale_suppression_reason( has_pending_tasks: Callable[[], bool] | None, pid: int | None, - marker_dir: Path | None, - caller_session_id: str | None, ) -> StaleSuppressionReason | None: """Return the first active stale-suppression cause in priority order.""" if has_pending_tasks is not None and has_pending_tasks(): return "pending_tasks" - active = _active_liveness_signals(pid, marker_dir, caller_session_id) - if "api_connection" in active: - return "api_connection" + active = _active_liveness_signals(pid) if "child_processes" in active: return "child_processes" - if "dispatch_marker" in active: - return "dispatch_marker" return None @@ -400,24 +335,11 @@ def _continue_stale_suppression( max_suppression_seconds: float, elapsed: float, pid: int | None, - marker_dir: Path | None, - caller_session_id: str | None, ) -> tuple[float, float | None]: """Advance one active suppression window and log its cause-specific outcome.""" if suppression_start is None: suppression_start = time.monotonic() - if reason == "dispatch_marker": - suppression_elapsed = time.monotonic() - suppression_start - if suppression_elapsed >= max_suppression_seconds: - logger.warning( - "Suppression bounded: stale kill after dispatch marker " - "suppression exceeded max_suppression_seconds", - suppression_elapsed=suppression_elapsed, - caller_session_id=caller_session_id, - marker_dir=str(marker_dir), - ) - return suppression_start, None - elif time.monotonic() - suppression_start >= max_suppression_seconds: + if time.monotonic() - suppression_start >= max_suppression_seconds: if reason != "pending_tasks": logger.warning( "Suppression bounded: stale kill after %.0fs consecutive " @@ -428,27 +350,13 @@ def _continue_stale_suppression( ) return suppression_start, None last_change = time.monotonic() - if reason == "api_connection": - logger.warning( - "JSONL silent for %.0fs but ESTABLISHED port-443 connection — " - "suppressing stale kill (pid=%d)", - elapsed, - pid, - ) - elif reason == "child_processes": + if reason == "child_processes": logger.warning( "JSONL silent for %.0fs but child processes are CPU-active — " "suppressing stale kill (pid=%d)", elapsed, pid, ) - elif reason == "dispatch_marker": - logger.warning( - "JSONL silent but active dispatch marker found — suppressing stale kill", - stale_elapsed=elapsed, - caller_session_id=caller_session_id, - marker_dir=str(marker_dir), - ) return suppression_start, last_change @@ -465,8 +373,6 @@ async def _session_log_monitor( _on_poll: Callable[[], None] | None = None, expected_session_id: str | None = None, max_suppression_seconds: float = 1800.0, - marker_dir: Path | None = None, - caller_session_id: str | None = None, resume_cursor: EventCursor | None = None, on_session_file_selected: Callable[[Path, EventCursor], None] | None = None, has_pending_tasks: Callable[[], bool] | None = None, @@ -585,9 +491,7 @@ async def _session_log_monitor( else: elapsed = now - last_change if elapsed >= stale_threshold: - reason = _stale_suppression_reason( - has_pending_tasks, pid, marker_dir, caller_session_id - ) + reason = _stale_suppression_reason(has_pending_tasks, pid) if reason is None: return SessionMonitorResult( ChannelBStatus.STALE, @@ -601,8 +505,6 @@ async def _session_log_monitor( max_suppression_seconds, elapsed, pid, - marker_dir, - caller_session_id, ) if next_last_change is None: return SessionMonitorResult( diff --git a/src/autoskillit/execution/process/_process_race.py b/src/autoskillit/execution/process/_process_race.py index 32274c4847..d4cbab90dd 100644 --- a/src/autoskillit/execution/process/_process_race.py +++ b/src/autoskillit/execution/process/_process_race.py @@ -376,8 +376,6 @@ async def _watch_session_log( _session_id_timeout: float = 1.0, stdout_session_id_ready: anyio.Event | None = None, max_suppression_seconds: float | None = None, - marker_dir: Path | None = None, - session_id: str | None = None, on_session_id_resolved: Callable[[str], None] | None = None, backend_resume_session_id: str = "", channel_b_selected: anyio.Event | None = None, @@ -415,12 +413,8 @@ def _selected(_path: Path, cursor: EventCursor) -> None: _monitor_kwargs["on_session_file_selected"] = _selected if max_suppression_seconds is not None: _monitor_kwargs["max_suppression_seconds"] = max_suppression_seconds - if marker_dir is not None: - _monitor_kwargs["marker_dir"] = marker_dir if operation_lease_dir is not None: _monitor_kwargs["operation_lease_dir"] = operation_lease_dir - if session_id is not None: - _monitor_kwargs["caller_session_id"] = session_id monitor_result = await _session_log_monitor( session_log_dir, completion_marker, diff --git a/src/autoskillit/execution/process/_race_watchers.py b/src/autoskillit/execution/process/_race_watchers.py index 8756f97ddd..bc152c4719 100644 --- a/src/autoskillit/execution/process/_race_watchers.py +++ b/src/autoskillit/execution/process/_race_watchers.py @@ -15,9 +15,7 @@ from autoskillit.core import InspectorEvidence, get_logger, read_active_operation_leases from autoskillit.execution.process._process_jsonl import fold_event_cursor from autoskillit.execution.process._process_monitor import ( - _has_active_api_connection, _has_active_child_processes, - _has_active_execution_marker, _OperationLeaseActivity, ) from autoskillit.execution.process._process_race import RaceAccumulator @@ -36,13 +34,13 @@ def _package_evidence( stdout_path: Path, idle_seconds: float, - execution_marker_present: bool, + operation_lease_active: bool, ) -> InspectorEvidence: return InspectorEvidence( idle_seconds=idle_seconds, stdout_path=str(stdout_path), jsonl_lines=(), - execution_marker_present=execution_marker_present, + operation_lease_active=operation_lease_active, ) @@ -53,8 +51,6 @@ async def _watch_stdout_idle( trigger: anyio.Event, _poll_interval: float = 5.0, *, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, max_suppression_seconds: float = 1800.0, inspector_callback: InspectorCallback | None = None, @@ -67,7 +63,7 @@ async def _watch_stdout_idle( last_size: int = 0 last_liveness_evidence = _time.monotonic() last_stdout_growth = last_liveness_evidence - suppression_start_marker: float | None = None + suppression_start: float | None = None operation_activity = _OperationLeaseActivity( operation_lease_dir, "stdout_idle_deferred_to_operation", logger.info ) @@ -76,10 +72,10 @@ async def _watch_stdout_idle( if trigger.is_set(): return now = _time.monotonic() - last_liveness_evidence, suppression_start_marker = operation_activity.refresh( + last_liveness_evidence, suppression_start = operation_activity.refresh( now, last_liveness_evidence, - suppression_start_marker, + suppression_start, silence_seconds=now - last_stdout_growth, ) try: @@ -90,38 +86,28 @@ async def _watch_stdout_idle( last_size = current_size last_liveness_evidence = now last_stdout_growth = now - suppression_start_marker = None + suppression_start = None elif now - last_liveness_evidence >= idle_output_timeout: authoritative_task_active = has_pending_tasks is not None and has_pending_tasks() - marker_active = marker_dir is not None and _has_active_execution_marker( - marker_dir, session_id=session_id - ) - if authoritative_task_active or marker_active: + if authoritative_task_active: now = _time.monotonic() - if suppression_start_marker is None: - suppression_start_marker = now + if suppression_start is None: + suppression_start = now logger.debug( - "stdout_idle_stall_suppression_evaluated", - marker_dir_present=True, - session_id=session_id, + "stdout_idle_pending_task_suppression_evaluated", ) - elapsed = now - suppression_start_marker + elapsed = now - suppression_start if elapsed < max_suppression_seconds: logger.warning( "stdout_idle_stall_suppressed", - marker_dir=str(marker_dir), - session_id=session_id, + reason="pending_tasks", suppression_elapsed=elapsed, max_suppression_seconds=max_suppression_seconds, ) continue logger.debug( "stdout_idle_stall_suppression_evaluated", - marker_dir_present=marker_dir is not None, - session_id=session_id, - **( - {"suppression_skipped_reason": "marker_dir_none"} if marker_dir is None else {} - ), + pending_tasks_active=authoritative_task_active, ) logger.warning( "stdout_idle_stall_firing", @@ -135,8 +121,7 @@ async def _watch_stdout_idle( acc, inspector_callback, timeout_scope_ref, - marker_dir=marker_dir, - session_id=session_id, + operation_lease_dir=operation_lease_dir, ) if spared_at is not None: last_liveness_evidence = spared_at @@ -154,8 +139,7 @@ async def _inspect_stdout_idle( inspector_callback: InspectorCallback | None, timeout_scope_ref: list[anyio.CancelScope | None] | None, *, - marker_dir: Path | None, - session_id: str | None, + operation_lease_dir: Path | None, ) -> float | None: """Run the inspector and return the idle-clock reset time when it spares the process.""" import time as _time @@ -176,13 +160,15 @@ async def _inspect_stdout_idle( invoke = False if not invoke: return None - marker_present = marker_dir is not None and _has_active_execution_marker( - marker_dir, session_id=session_id + operation_lease_active = bool( + read_active_operation_leases(operation_lease_dir, now_epoch=time.time()) + if operation_lease_dir is not None + else () ) evidence = _package_evidence( stdout_path, idle_seconds=_time.monotonic() - last_liveness_evidence, - execution_marker_present=marker_present, + operation_lease_active=operation_lease_active, ) try: with anyio.fail_after(budget): @@ -211,8 +197,6 @@ async def _watch_child_activity( trigger: anyio.Event, _poll_interval: float = 30.0, *, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, has_pending_tasks: Callable[[], bool] | None = None, ) -> None: @@ -240,11 +224,6 @@ async def _watch_child_activity( active = lease_active or ( (has_pending_tasks is not None and has_pending_tasks()) or _has_active_child_processes(pid) - or _has_active_api_connection(pid) - or ( - marker_dir is not None - and _has_active_execution_marker(marker_dir, session_id=session_id) - ) ) if not active: continue @@ -333,8 +312,6 @@ def _enroll_race_watchers( session_id_timeout: float, stdout_session_id_ready: anyio.Event, max_suppression_seconds: float | None, - marker_dir: Path | None, - session_id: str | None, operation_lease_dir: Path | None = None, on_session_id_resolved: Callable[[str], None] | None, backend_resume_session_id: str, @@ -393,8 +370,6 @@ def _enroll_race_watchers( session_id_timeout, stdout_session_id_ready, max_suppression_seconds, - marker_dir, - session_id, on_session_id_resolved, backend_resume_session_id, channel_b_selected, @@ -418,8 +393,6 @@ def _enroll_race_watchers( idle_output_timeout, acc, trigger, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, max_suppression_seconds=max_suppression_seconds or 1800.0, inspector_callback=inspector_callback, @@ -440,8 +413,6 @@ def _enroll_child_activity_watcher( timeout_scope_ref: list[anyio.CancelScope | None], max_extension_seconds: float, trigger: anyio.Event, - marker_dir: Path | None, - session_id: str | None, operation_lease_dir: Path | None = None, lifecycle_observation_enabled: bool, acc: RaceAccumulator, @@ -454,8 +425,6 @@ def _enroll_child_activity_watcher( timeout_scope_ref, max_extension_seconds, trigger, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, has_pending_tasks=( acc.has_unresolved_obligations if lifecycle_observation_enabled else None @@ -474,8 +443,6 @@ async def _await_race_and_drain( enable_deadline_extension: bool, observed_pid: int | None, max_extension_seconds: float, - marker_dir: Path | None, - session_id: str | None, operation_lease_dir: Path | None = None, lifecycle_observation_enabled: bool, acc: RaceAccumulator, @@ -493,8 +460,6 @@ async def _await_race_and_drain( timeout_scope_ref=timeout_scope_ref, max_extension_seconds=max_extension_seconds, trigger=trigger, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, lifecycle_observation_enabled=lifecycle_observation_enabled, acc=acc, diff --git a/src/autoskillit/execution/process/_termination.py b/src/autoskillit/execution/process/_termination.py index 0d486faaca..eaeed8613b 100644 --- a/src/autoskillit/execution/process/_termination.py +++ b/src/autoskillit/execution/process/_termination.py @@ -86,8 +86,6 @@ async def _drain_before_escalation( grace_seconds: float, proc_log: structlog.BoundLogger, pid: int | None, - marker_dir: Path | None, - session_id: str | None, operation_lease_dir: Path | None = None, child_deferral_ceiling: float, ) -> tuple[int | None, ProcessCleanupResult] | None: @@ -109,8 +107,6 @@ async def _drain_before_escalation( active = bool( _active_liveness_signals( pid, - marker_dir, - session_id, operation_lease_dir=operation_lease_dir, ) ) @@ -135,8 +131,6 @@ async def execute_termination_action( proc_log: structlog.BoundLogger, termination: TerminationReason | None = None, pid: int | None = None, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, child_deferral_ceiling: float = 0.0, process_observation_snapshot: ProcessObservationSnapshot | None = None, @@ -147,7 +141,7 @@ async def execute_termination_action( On the DRAIN_THEN_KILL_IF_ALIVE path, when *pid* is provided and *child_deferral_ceiling* > 0, the kill is deferred (bounded by the ceiling) - while child processes, an API connection, or an execution marker indicate + while child processes or an active operation lease indicate the subagent is still doing active work — mirroring the stale-kill suppression pattern in _session_log_monitor. @@ -169,8 +163,6 @@ async def execute_termination_action( grace_seconds=grace_seconds, proc_log=proc_log, pid=pid, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, child_deferral_ceiling=child_deferral_ceiling, ) diff --git a/src/autoskillit/execution/recording/recording.py b/src/autoskillit/execution/recording/recording.py index 8ce74fe692..a22e9615fe 100644 --- a/src/autoskillit/execution/recording/recording.py +++ b/src/autoskillit/execution/recording/recording.py @@ -143,8 +143,6 @@ async def __call__( on_pid_resolved: Callable[[int, int], None] | None = None, enable_deadline_extension: bool = False, max_extension_seconds: float = 7200, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), @@ -202,8 +200,6 @@ async def __call__( on_pid_resolved=on_pid_resolved, enable_deadline_extension=enable_deadline_extension, max_extension_seconds=max_extension_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, step_name=step_name, @@ -242,8 +238,6 @@ async def __call__( on_pid_resolved=on_pid_resolved, enable_deadline_extension=enable_deadline_extension, max_extension_seconds=max_extension_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, @@ -291,8 +285,6 @@ async def __call__( on_pid_resolved=on_pid_resolved, enable_deadline_extension=enable_deadline_extension, max_extension_seconds=max_extension_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, @@ -402,8 +394,6 @@ async def _record_non_pty_session( on_pid_resolved: Callable[[int, int], None] | None, enable_deadline_extension: bool, max_extension_seconds: float, - marker_dir: Path | None, - session_id: str | None, operation_lease_dir: Path | None, stream_parser: StreamParser | None, step_name: str, @@ -441,8 +431,6 @@ async def _record_non_pty_session( on_pid_resolved=on_pid_resolved, enable_deadline_extension=enable_deadline_extension, max_extension_seconds=max_extension_seconds, - marker_dir=marker_dir, - session_id=session_id, operation_lease_dir=operation_lease_dir, stream_parser=stream_parser, completion_record_types=completion_record_types, @@ -561,8 +549,6 @@ async def __call__( on_pid_resolved: Callable[[int, int], None] | None = None, enable_deadline_extension: bool = False, max_extension_seconds: float = 7200, - marker_dir: Path | None = None, - session_id: str | None = None, operation_lease_dir: Path | None = None, stream_parser: StreamParser | None = None, completion_record_types: frozenset[str] = frozenset({"result"}), diff --git a/src/autoskillit/fleet/dispatch/_api.py b/src/autoskillit/fleet/dispatch/_api.py index e149b6afcd..b347c5dae3 100644 --- a/src/autoskillit/fleet/dispatch/_api.py +++ b/src/autoskillit/fleet/dispatch/_api.py @@ -569,7 +569,7 @@ async def _run_dispatch( prior_dispatched_session_id=ready.prior_dispatched_session_id, resume_session_id=execution_result.effective_resume_session_id, incoming_resume_checkpoint=ready.resume_checkpoint, - marker_dir=execution_result.marker_dir, + dispatched_log_dir=execution_result.dispatched_log_dir, effective_backend=recipe_ctx.effective_backend, dispatch_sidecar_path=execution_result.dispatch_sidecar_path, prior_dispatch_ids=[prior_dispatch_id] if prior_dispatch_id else None, @@ -601,7 +601,9 @@ async def _run_dispatch( effective_name=recipe_ctx.effective_name, managed_lineage_ref=ready.managed_lineage_ref, provenance=provenance, - marker_dir=execution_result.marker_dir if execution_result is not None else None, + dispatched_log_dir=( + execution_result.dispatched_log_dir if execution_result is not None else None + ), state_path=ready.state_path, ) raise diff --git a/src/autoskillit/fleet/dispatch/_classification.py b/src/autoskillit/fleet/dispatch/_classification.py index c79a66b927..c8fba32ad9 100644 --- a/src/autoskillit/fleet/dispatch/_classification.py +++ b/src/autoskillit/fleet/dispatch/_classification.py @@ -250,7 +250,7 @@ async def run_outcome_classification( prior_dispatched_session_id: str | None, resume_session_id: str | None, incoming_resume_checkpoint: SessionCheckpoint | None, - marker_dir: Path | None, + dispatched_log_dir: Path | None, effective_backend: CodingAgentBackend | None, recipe: str, dispatch_sidecar_path: str, @@ -351,9 +351,8 @@ async def run_outcome_classification( provenance=provenance, ) - # The orchestrator threads marker_dir through (resolved from - # `_locator.project_log_dir` inside run_execution), so we just stringify it. - project_log_dir = str(marker_dir) if marker_dir is not None else "" + # The orchestrator threads the dispatched backend's log directory through. + project_log_dir = str(dispatched_log_dir) if dispatched_log_dir is not None else "" if ( resume_session_id diff --git a/src/autoskillit/fleet/dispatch/_cleanup.py b/src/autoskillit/fleet/dispatch/_cleanup.py index 8d3e185bc6..1ef177f521 100644 --- a/src/autoskillit/fleet/dispatch/_cleanup.py +++ b/src/autoskillit/fleet/dispatch/_cleanup.py @@ -48,7 +48,7 @@ async def handle_cancellation( effective_name: str, managed_lineage_ref: ManagedHeadlessSessionLineageRef, provenance: DispatchProvenanceTracker, - marker_dir: Path | None, + dispatched_log_dir: Path | None, state_path: Path, ) -> NoReturn: """Phase D handler for asyncio.CancelledError. @@ -125,7 +125,9 @@ async def handle_cancellation( effective_name, reason="signal_induced_cancellation", dispatched_session_id=captured_session_id, - dispatched_session_log_dir=str(marker_dir) if marker_dir is not None else "", + dispatched_session_log_dir=( + str(dispatched_log_dir) if dispatched_log_dir is not None else "" + ), effect_provenance=provenance.snapshot().to_dict(), ) except Exception: diff --git a/src/autoskillit/fleet/dispatch/_execution.py b/src/autoskillit/fleet/dispatch/_execution.py index d4e41de7ef..98c375370e 100644 --- a/src/autoskillit/fleet/dispatch/_execution.py +++ b/src/autoskillit/fleet/dispatch/_execution.py @@ -85,7 +85,7 @@ class ExecutionResult: started_at: float ended_at: float | None dispatch_completed_normally: bool - marker_dir: Path | None + dispatched_log_dir: Path | None dispatch_sidecar_path: str spawn_failure_dispatch_result: DispatchResult | None effective_resume_session_id: str | None @@ -104,7 +104,7 @@ def _failed_execution_result( effective_name: str, tool_ctx: ToolContext, started_at: float, - marker_dir: Path | None, + dispatched_log_dir: Path | None, dispatch_sidecar_path: str, effective_resume_session_id: str | None, ) -> ExecutionResult: @@ -114,7 +114,7 @@ def _failed_execution_result( started_at=started_at, ended_at=None, dispatch_completed_normally=False, - marker_dir=marker_dir, + dispatched_log_dir=dispatched_log_dir, dispatch_sidecar_path=dispatch_sidecar_path, spawn_failure_dispatch_result=complete_failure_with_state( error_code=error_code, @@ -227,12 +227,12 @@ async def run_execution( # Derive the session locator for JSONL resolution. _locator = effective_backend.session_locator() if effective_backend is not None else None - # Resolve marker_dir before the early-return paths so any + # Resolve the dispatched log directory before early-return paths so any # ExecutionResult carrying a spawn failure can still report its path. - marker_dir: Path | None = None + dispatched_log_dir: Path | None = None if _locator is not None: try: - marker_dir = _locator.project_log_dir(str(tool_ctx.project_dir)) + dispatched_log_dir = _locator.project_log_dir(str(tool_ctx.project_dir)) except OSError: pass @@ -277,7 +277,7 @@ async def run_execution( effective_name=effective_name, tool_ctx=tool_ctx, started_at=started_at, - marker_dir=marker_dir, + dispatched_log_dir=dispatched_log_dir, dispatch_sidecar_path=dispatch_sidecar_path, effective_resume_session_id=resume_session_id, ) @@ -299,7 +299,7 @@ async def run_execution( effective_name=effective_name, tool_ctx=tool_ctx, started_at=started_at, - marker_dir=marker_dir, + dispatched_log_dir=dispatched_log_dir, dispatch_sidecar_path=dispatch_sidecar_path, effective_resume_session_id=resume_session_id, ), @@ -323,9 +323,6 @@ async def run_execution( skill_result: SkillResult | None = None ended_at: float | None = None - # execution_marker is needed by the spawn-context blocks further below. - from autoskillit.core import execution_marker - # Closures captured by tool_ctx.executor.dispatch_food_truck. They mutate # spawn_ctx in place. def _on_spawn(pid: int, ticks: int) -> None: @@ -407,78 +404,71 @@ def _on_launch_resolved(launch_contract: ResolvedLaunchContract) -> None: DispatchEffectName.PROCESS_SPAWN, identities={"dispatch_id": dispatch_id}, ) - async with execution_marker( - marker_dir, - caller_session_id, - "dispatch", + async with _dispatch_heartbeat( + dispatches_dir or tool_ctx.temp_dir / "dispatches", dispatch_id ): - async with _dispatch_heartbeat( - dispatches_dir or tool_ctx.temp_dir / "dispatches", dispatch_id - ): - skill_result = await tool_ctx.executor.dispatch_food_truck( - orchestrator_prompt=prompt, - cwd=str(tool_ctx.project_dir), - completion_marker=completion_marker, - plugin_authority=plugin_authority, - capability_preparation=capability_preparation, - prior_completion_markers=( - cast( - "Sequence[str] | None", - prior_completion_markers if prior_completion_markers else None, - ) + skill_result = await tool_ctx.executor.dispatch_food_truck( + orchestrator_prompt=prompt, + cwd=str(tool_ctx.project_dir), + completion_marker=completion_marker, + plugin_authority=plugin_authority, + capability_preparation=capability_preparation, + prior_completion_markers=( + cast( + "Sequence[str] | None", + prior_completion_markers if prior_completion_markers else None, + ) + ), + resume_session_id=effective_resume_session_id, + resume_checkpoint=resume_checkpoint, + kitchen_id=tool_ctx.kitchen_id, + order_id=dispatch_id, + campaign_id=tool_ctx.kitchen_id, + dispatch_id=dispatch_id, + caller_session_id=caller_session_id, + project_dir=str(tool_ctx.project_dir), + on_session_id_resolved=_on_session_id, + timeout=resolved_timeout, + idle_output_timeout=float(idle_output_timeout) + if idle_output_timeout is not None + else None, + env_extras={ + "AUTOSKILLIT_PROJECT_DIR": str(tool_ctx.project_dir), + "AUTOSKILLIT_CAMPAIGN_ID": tool_ctx.kitchen_id, + "AUTOSKILLIT_DISPATCH_ID": dispatch_id, + "AUTOSKILLIT_SESSION_DEADLINE": select_child_session_deadline( + started_at + resolved_timeout, + os.environ.get("AUTOSKILLIT_SESSION_DEADLINE", ""), ), - resume_session_id=effective_resume_session_id, - resume_checkpoint=resume_checkpoint, - kitchen_id=tool_ctx.kitchen_id, - order_id=dispatch_id, - campaign_id=tool_ctx.kitchen_id, - dispatch_id=dispatch_id, - caller_session_id=caller_session_id, - project_dir=str(tool_ctx.project_dir), - marker_dir=marker_dir, - session_id=caller_session_id, - on_session_id_resolved=_on_session_id, - timeout=resolved_timeout, - idle_output_timeout=float(idle_output_timeout) - if idle_output_timeout is not None - else None, - env_extras={ - "AUTOSKILLIT_PROJECT_DIR": str(tool_ctx.project_dir), - "AUTOSKILLIT_CAMPAIGN_ID": tool_ctx.kitchen_id, - "AUTOSKILLIT_DISPATCH_ID": dispatch_id, - "AUTOSKILLIT_SESSION_DEADLINE": select_child_session_deadline( - started_at + resolved_timeout, - os.environ.get("AUTOSKILLIT_SESSION_DEADLINE", ""), - ), - **( - {FLEET_INSPECTOR_MODEL_ENV_VAR: (tool_ctx.config.fleet.inspector_model)} - if tool_ctx.config.fleet.inspector_model - else {} - ), - **( - {MANAGED_JOIN_PARENT_ID_ENV_VAR: managed_join_parent_id} - if managed_join_parent_id is not None - else {} - ), - }, - requires_packs=list(full_recipe.requires_packs) or ["kitchen-core"], - on_spawn=_on_spawn, - sentinel_contract=sentinel_contract, - resume_message=resume_message, - backend_authority=( - BackendAuthority( - backend=dispatch_backend.name, - kind=BackendAuthorityKind.CALLER, - tier=BackendAuthorityTier.CALLER, - key_path="dispatch.backend", - ) - if dispatch_backend is not None - else None + **( + {FLEET_INSPECTOR_MODEL_ENV_VAR: (tool_ctx.config.fleet.inspector_model)} + if tool_ctx.config.fleet.inspector_model + else {} ), - native_shell_capture_decision=capture_decision, - managed_lineage_ref=managed_lineage_ref, - on_launch_resolved=_on_launch_resolved, - ) + **( + {MANAGED_JOIN_PARENT_ID_ENV_VAR: managed_join_parent_id} + if managed_join_parent_id is not None + else {} + ), + }, + requires_packs=list(full_recipe.requires_packs) or ["kitchen-core"], + on_spawn=_on_spawn, + sentinel_contract=sentinel_contract, + resume_message=resume_message, + backend_authority=( + BackendAuthority( + backend=dispatch_backend.name, + kind=BackendAuthorityKind.CALLER, + tier=BackendAuthorityTier.CALLER, + key_path="dispatch.backend", + ) + if dispatch_backend is not None + else None + ), + native_shell_capture_decision=capture_decision, + managed_lineage_ref=managed_lineage_ref, + on_launch_resolved=_on_launch_resolved, + ) # L2 fail-closed spawn gate: check closure-scoped error state. # If _on_spawn recorded a transition failure (and killed the child @@ -499,7 +489,7 @@ def _on_launch_resolved(launch_contract: ResolvedLaunchContract) -> None: effective_name=effective_name, tool_ctx=tool_ctx, started_at=started_at, - marker_dir=marker_dir, + dispatched_log_dir=dispatched_log_dir, dispatch_sidecar_path=dispatch_sidecar_path, effective_resume_session_id=effective_resume_session_id, ) @@ -517,7 +507,7 @@ def _on_launch_resolved(launch_contract: ResolvedLaunchContract) -> None: started_at=started_at, ended_at=ended_at, dispatch_completed_normally=True, - marker_dir=marker_dir, + dispatched_log_dir=dispatched_log_dir, dispatch_sidecar_path=dispatch_sidecar_path, spawn_failure_dispatch_result=None, effective_resume_session_id=effective_resume_session_id, diff --git a/src/autoskillit/skill_resources/arch-constraint-catalog.md b/src/autoskillit/skill_resources/arch-constraint-catalog.md index 9dc1851a95..94938ff0ef 100644 --- a/src/autoskillit/skill_resources/arch-constraint-catalog.md +++ b/src/autoskillit/skill_resources/arch-constraint-catalog.md @@ -65,7 +65,7 @@ classified `REJECT` with `category: "arch_violation"`. | Turn ID resolution | `test_resolve_turn_id_guard.py` | Direct `requestId` dict `.get()` outside `_parent_assistant_turns.py`'s `_resolve_turn_id()` — all turn ID resolution must go through the single resolver | | No hardcoded @-mentions in SKILL.md | `test_skills_mention_guard.py` | `@word` tokens at word-boundary in SKILL.md prose — includes Python decorator examples (`@dataclass`, `@mcp`); use prose descriptions or remove the `@` prefix | | FastMCP tag hygiene | `test_transforms_hygiene.py` | Test fixtures touching `mcp._transforms` without using the canonical `ALL_VISIBILITY_TAGS` constant | -| Watcher dispatch marker | `test_watcher_signal_consistency.py` | Process watchers that skip `_has_active_dispatch_marker()` check | +| Watcher operation leases | `test_watcher_signal_consistency.py` | Process watchers must consult active operation leases; execution markers and network connections do not establish liveness | | Write scope declarations | `test_skill_write_scope_declarations.py` | A skill without a valid `write_paths` declaration, an unreviewed `unrestricted`/`inherit` declarer, a BOUNDED skill whose body writes outside its declared temp directories, or a project-local override whose scope diverges from its bundled twin | | Subagent filter guard | `test_subagent_filter_guard.py` | Parent-assistant consumers bypassing `_parent_assistant_turns.py`'s `is_parent_assistant_record` predicate — subagent or synthetic records contaminate parent metrics | | Review audit transcript authority | `test_review_pr_audit_ingestion_guards.py` | `review-pr` must send auditor handles through server-owned collection and finalization, which read each child's own transcript; prompt logic must not transcribe or recompute auditor output | diff --git a/tests/arch/test_execution_marker_producers.py b/tests/arch/test_execution_marker_producers.py new file mode 100644 index 0000000000..a8f1bfa94e --- /dev/null +++ b/tests/arch/test_execution_marker_producers.py @@ -0,0 +1,50 @@ +"""Keep execution markers limited to the run-skill attestation producer.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + + +def test_run_skill_finalize_is_the_only_execution_marker_producer() -> None: + repository_root = Path(__file__).resolve().parents[2] + source_root = repository_root / "src/autoskillit" + producers: list[tuple[Path, str | None]] = [] + + for source_path in source_root.rglob("*.py"): + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + called_name = ( + node.func.id + if isinstance(node.func, ast.Name) + else node.func.attr + if isinstance(node.func, ast.Attribute) + else "" + ) + if called_name != "execution_marker": + continue + label = next( + ( + keyword.value.value + for keyword in node.keywords + if keyword.arg == "label" + and isinstance(keyword.value, ast.Constant) + and isinstance(keyword.value.value, str) + ), + None, + ) + if label is None and len(node.args) > 2: + label_arg = node.args[2] + if isinstance(label_arg, ast.Constant) and isinstance(label_arg.value, str): + label = label_arg.value + producers.append((source_path.relative_to(repository_root), label)) + + assert producers == [ + (Path("src/autoskillit/server/tools/tools_execution/_run_skill_finalize.py"), "run-skill") + ] diff --git a/tests/arch/test_watcher_signal_consistency.py b/tests/arch/test_watcher_signal_consistency.py index c5863f8e8c..caf232d674 100644 --- a/tests/arch/test_watcher_signal_consistency.py +++ b/tests/arch/test_watcher_signal_consistency.py @@ -1,4 +1,4 @@ -"""Structural guard: all process watchers must call _has_active_execution_marker.""" +"""Watchers use operation leases and child activity as liveness evidence.""" from __future__ import annotations @@ -11,19 +11,12 @@ pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] -_CLI_APP = Path("src/autoskillit/cli/app.py") +_CLI_SESSION_PROCESS = Path("src/autoskillit/cli/session/_session_process.py") _PROCESS_RACE = Path("src/autoskillit/execution/process/_process_race.py") _PROCESS_MONITOR = Path("src/autoskillit/execution/process/_process_monitor.py") _PROCESS_INIT = Path("src/autoskillit/execution/process/__init__.py") _PROCESS_TERMINATION = Path("src/autoskillit/execution/process/_termination.py") - -_WATCHERS_THAT_MUST_CHECK_EXECUTION_MARKER = frozenset( - { - "_watch_child_activity", - "_session_log_monitor", - "_watch_stdout_idle", - } -) +_EXECUTION_ROOT = Path("src/autoskillit/execution") def _functions_calling_predicate(source_path: Path, predicate: str) -> set[str]: @@ -74,38 +67,71 @@ def _calls_trigger_set(node: ast.AST) -> bool: ) -@pytest.mark.parametrize("watcher", sorted(_WATCHERS_THAT_MUST_CHECK_EXECUTION_MARKER)) -def test_watcher_calls_has_active_execution_marker(watcher: str) -> None: - """Each watcher in the set must call _has_active_execution_marker.""" - callers_race = _functions_calling_predicate(_PROCESS_RACE, "_has_active_execution_marker") - callers_watchers = _functions_calling_predicate( - RACE_WATCHERS_PY, "_has_active_execution_marker" - ) - callers_monitor = _functions_calling_predicate( - _PROCESS_MONITOR, "_has_active_execution_marker" - ) - if watcher == "_session_log_monitor": - assert watcher in _functions_calling_predicate( - _PROCESS_MONITOR, "_stale_suppression_reason" - ) - assert "_stale_suppression_reason" in _functions_calling_predicate( - _PROCESS_MONITOR, "_active_liveness_signals" - ) - assert "_active_liveness_signals" in callers_monitor - return - all_callers = callers_race | callers_watchers | callers_monitor - assert watcher in all_callers, ( - f"{watcher} does not call _has_active_execution_marker. " - f"Functions that do: {sorted(all_callers)}" - ) +def _called_names(node: ast.AST) -> set[str]: + names: set[str] = set() + for call in (child for child in ast.walk(node) if isinstance(child, ast.Call)): + if isinstance(call.func, ast.Name): + names.add(call.func.id) + elif isinstance(call.func, ast.Attribute): + names.add(call.func.attr) + return names -def test_signal_guard_activity_check_uses_in_flight_registry() -> None: - function = _function(_CLI_APP, "is_server_active") - source = ast.unparse(function) +def test_liveness_watchers_reach_the_operation_lease_reader() -> None: + """Watchers use the lease reader directly or through the shared lease helpers.""" + lease_activity = next( + node + for node in ast.walk(ast.parse(_PROCESS_MONITOR.read_text())) + if isinstance(node, ast.ClassDef) and node.name == "_OperationLeaseActivity" + ) + refresh = next( + node + for node in lease_activity.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "refresh" + ) + active_signals = _function(_PROCESS_MONITOR, "_active_liveness_signals") + assert "read_active_operation_leases" in _called_names(refresh) + assert "read_active_operation_leases" in _called_names(active_signals) - assert "in_flight_operations.active_count" in source - assert "_has_active_execution_marker" not in source + for source_path, function_name in ( + (RACE_WATCHERS_PY, "_watch_child_activity"), + (RACE_WATCHERS_PY, "_watch_stdout_idle"), + (_PROCESS_MONITOR, "_session_log_monitor"), + (_PROCESS_TERMINATION, "_drain_before_escalation"), + ): + routes = _called_names(_function(source_path, function_name)) + assert routes & { + "read_active_operation_leases", + "_active_liveness_signals", + "refresh", + }, f"{function_name} does not reach operation lease liveness" + + +def test_execution_watchers_have_no_dispatch_marker_dependency() -> None: + for source_path in _EXECUTION_ROOT.rglob("*.py"): + source = source_path.read_text(encoding="utf-8") + assert "_has_active_execution_marker" not in source, source_path + assert "-in-progress-" not in source, source_path + + +def test_liveness_consumers_do_not_use_network_connections_as_evidence() -> None: + consumers = ( + (_PROCESS_MONITOR, "_active_liveness_signals"), + (_PROCESS_MONITOR, "_stale_suppression_reason"), + (_PROCESS_MONITOR, "_continue_stale_suppression"), + (_PROCESS_MONITOR, "_session_log_monitor"), + (RACE_WATCHERS_PY, "_watch_child_activity"), + (_PROCESS_TERMINATION, "_drain_before_escalation"), + (_CLI_SESSION_PROCESS, "_default_activity"), + (_CLI_SESSION_PROCESS, "_apply_activity_signals"), + ) + for source_path, function_name in consumers: + function = _function(source_path, function_name) + assert "_has_active_api_connection" not in _called_names(function), function_name + assert all( + not isinstance(node, ast.Constant) or node.value != "api_connection" + for node in ast.walk(function) + ), function_name _KILL_EXECUTORS_THAT_MUST_CHECK_CHILD_LIVENESS = frozenset( diff --git a/tests/cli/test_cook_lifetime.py b/tests/cli/test_cook_lifetime.py index c37a1eac7c..f9040991b9 100644 --- a/tests/cli/test_cook_lifetime.py +++ b/tests/cli/test_cook_lifetime.py @@ -210,7 +210,7 @@ def test_terminal_io_does_not_restart_idle_window_but_other_activity_does( active_signals = iter( [ frozenset({"terminal_io"}), - frozenset({"api_connection"}), + frozenset({"child_processes"}), frozenset(), frozenset(), ] @@ -237,7 +237,7 @@ def test_active_session_ends_at_hard_cap_as_timed_out(monkeypatch: pytest.Monkey lifetime = _start_lifetime( _policy(soft=2.0, extension=3.0), clock=clock, - activity_probe=lambda _pid, _fd: frozenset({"api_connection"}), + activity_probe=lambda _pid, _fd: frozenset({"child_processes"}), ) clock.advance(2.1) assert lifetime.poll() is None @@ -253,7 +253,7 @@ def test_zero_extension_is_hard_stop_at_soft_ceiling() -> None: lifetime = _start_lifetime( _policy(soft=2.0, extension=0.0), clock=clock, - activity_probe=lambda _pid, _fd: frozenset({"api_connection"}), + activity_probe=lambda _pid, _fd: frozenset({"child_processes"}), ) clock.advance(1.9) assert lifetime.poll() is None @@ -271,7 +271,7 @@ def test_probe_rate_limited_and_never_called_before_soft(monkeypatch: pytest.Mon def probe(_pid: int, _fd: int | None) -> frozenset[str]: nonlocal calls calls += 1 - return frozenset({"api_connection"}) + return frozenset({"child_processes"}) lifetime = _start_lifetime( _policy(soft=2.0, extension=20.0), clock=clock, activity_probe=probe @@ -348,7 +348,7 @@ def test_lease_renewed_ahead_of_expiry(tmp_path: Path) -> None: _policy(soft=20_000.0, extension=20_000.0), clock=clock, wall=wall, - activity_probe=lambda _pid, _fd: frozenset({"api_connection"}), + activity_probe=lambda _pid, _fd: frozenset({"child_processes"}), tether_path=path, ) @@ -382,7 +382,7 @@ def test_lease_capped_at_hard_cap_plus_margin(tmp_path: Path) -> None: _policy(soft=1.0, extension=10.0), clock=clock, wall=wall, - activity_probe=lambda _pid, _fd: frozenset({"api_connection"}), + activity_probe=lambda _pid, _fd: frozenset({"child_processes"}), tether_path=path, ) clock.advance(10.75) @@ -485,7 +485,7 @@ def recording_write_versioned_json( _policy(soft=3600.0, extension=0.0), clock=clock, wall=wall, - activity_probe=lambda _pid, _fd: frozenset({"api_connection"}), + activity_probe=lambda _pid, _fd: frozenset({"child_processes"}), notice_path=notice_path, ) diff --git a/tests/cli/test_cook_process_lifecycle.py b/tests/cli/test_cook_process_lifecycle.py index b02a6960c8..1e8e889f0f 100644 --- a/tests/cli/test_cook_process_lifecycle.py +++ b/tests/cli/test_cook_process_lifecycle.py @@ -364,7 +364,7 @@ def test_active_child_extended_until_hard_cap( monkeypatch.setattr( _session_process, "_default_activity", - lambda *_args: {"api_connection"}, + lambda *_args: {"child_processes"}, ) result = run_cook_attempt( @@ -531,7 +531,7 @@ def test_lifetime_decision_wins_dispatch_even_when_a_failure_accumulated( monkeypatch.setattr( _session_process, "_default_activity", - lambda *_args: {"api_connection"}, + lambda *_args: {"child_processes"}, ) grace_seconds = 0.5 monkeypatch.setattr( diff --git a/tests/contracts/test_protocol_satisfaction.py b/tests/contracts/test_protocol_satisfaction.py index 2b2818395c..3eeecb0778 100644 --- a/tests/contracts/test_protocol_satisfaction.py +++ b/tests/contracts/test_protocol_satisfaction.py @@ -402,7 +402,7 @@ def test_headless_executor_run_excludes_marker_watchdog_parameters() -> None: assert "caller_session_id" not in params -def test_headless_executor_protocol_dispatch_has_marker_params(): +def test_headless_executor_dispatch_excludes_watchdog_marker_parameters(): import inspect from autoskillit.core import HeadlessExecutor @@ -411,10 +411,9 @@ def test_headless_executor_protocol_dispatch_has_marker_params(): for cls in (HeadlessExecutor, DefaultHeadlessExecutor): sig = inspect.signature(cls.dispatch_food_truck) params = sig.parameters - assert "marker_dir" in params, f"{cls.__name__} missing marker_dir" - assert params["marker_dir"].default is None, f"{cls.__name__}.marker_dir default != None" - assert "session_id" in params, f"{cls.__name__} missing session_id" - assert params["session_id"].default is None, f"{cls.__name__}.session_id default != None" + assert "marker_dir" not in params + assert "session_id" not in params + assert "caller_session_id" in params assert "plugin_authority" in params, f"{cls.__name__} missing plugin_authority" assert params["plugin_authority"].default is None assert "capability_preparation" in params, f"{cls.__name__} missing capability_preparation" @@ -522,8 +521,6 @@ def test_req_api_001_public_params_present(self): "on_pid_resolved", "enable_deadline_extension", "max_extension_seconds", - "marker_dir", - "session_id", "operation_lease_dir", "stream_parser", "inspector_callback", @@ -605,8 +602,6 @@ def test_req_api_002_call_params_match_protocol(self): "on_pid_resolved", "enable_deadline_extension", "max_extension_seconds", - "marker_dir", - "session_id", "operation_lease_dir", "stream_parser", "completion_record_types", @@ -630,58 +625,34 @@ def test_req_api_002_call_params_match_protocol(self): f" Extra: {actual - expected}" ) - def test_run_managed_async_marker_dir_session_id_defaults(self): - """run_managed_async marker_dir and session_id default to None.""" - sig = inspect.signature(run_managed_async) - assert sig.parameters["marker_dir"].default is None - assert sig.parameters["session_id"].default is None - - def test_run_managed_async_marker_params_after_session_id_timeout(self): - """marker_dir and session_id appear after _session_id_timeout in run_managed_async.""" - sig = inspect.signature(run_managed_async) - session_id_timeout_idx = list(sig.parameters).index("_session_id_timeout") - marker_dir_idx = list(sig.parameters).index("marker_dir") - session_id_idx = list(sig.parameters).index("session_id") - assert marker_dir_idx == session_id_timeout_idx + 1 - assert session_id_idx == marker_dir_idx + 1 - - def test_run_managed_async_operation_lease_dir_defaults_and_order(self): + def test_run_managed_async_operation_lease_dir_follows_session_id_timeout(self): sig = inspect.signature(run_managed_async) + timeout_idx = list(sig.parameters).index("_session_id_timeout") assert sig.parameters["operation_lease_dir"].default is None - assert list(sig.parameters).index("operation_lease_dir") == ( - list(sig.parameters).index("session_id") + 1 - ) - - def test_default_subprocess_runner_marker_dir_session_id_defaults(self): - """DefaultSubprocessRunner.__call__ marker_dir and session_id default to None.""" - sig = inspect.signature(DefaultSubprocessRunner.__call__) - assert sig.parameters["marker_dir"].default is None - assert sig.parameters["session_id"].default is None - - def test_default_subprocess_runner_marker_params_after_max_extension(self): - """marker_dir/session_id appear after max_extension_seconds in DefaultSubprocessRunner.""" - sig = inspect.signature(DefaultSubprocessRunner.__call__) - max_extension_idx = list(sig.parameters).index("max_extension_seconds") - marker_dir_idx = list(sig.parameters).index("marker_dir") - session_id_idx = list(sig.parameters).index("session_id") - assert marker_dir_idx == max_extension_idx + 1 - assert session_id_idx == marker_dir_idx + 1 + assert list(sig.parameters).index("operation_lease_dir") == (timeout_idx + 1) - def test_default_subprocess_runner_operation_lease_dir_defaults_and_order(self): + def test_default_subprocess_runner_operation_lease_dir_follows_max_extension(self): sig = inspect.signature(DefaultSubprocessRunner.__call__) + extension_idx = list(sig.parameters).index("max_extension_seconds") assert sig.parameters["operation_lease_dir"].default is None - assert list(sig.parameters).index("operation_lease_dir") == ( - list(sig.parameters).index("session_id") + 1 - ) + assert list(sig.parameters).index("operation_lease_dir") == (extension_idx + 1) - def test_default_subprocess_runner_satisfies_protocol_with_marker_params(self): - """DefaultSubprocessRunner() satisfies SubprocessRunner with marker_dir/session_id.""" + def test_watchdog_marker_parameters_are_absent_from_dispatch_signatures(self): + from autoskillit.core import HeadlessExecutor from autoskillit.core.types import SubprocessRunner - - runner = DefaultSubprocessRunner() - assert isinstance(runner, SubprocessRunner), ( - "DefaultSubprocessRunner no longer satisfies the SubprocessRunner protocol" - ) + from autoskillit.execution.headless._headless_execute import _execute_claude_headless + from autoskillit.execution.headless._headless_launch import _run_headless_attempt + + for callable_ in ( + SubprocessRunner.__call__, + DefaultSubprocessRunner.__call__, + HeadlessExecutor.dispatch_food_truck, + _execute_claude_headless, + _run_headless_attempt, + ): + params = inspect.signature(callable_).parameters + assert "marker_dir" not in params + assert "session_id" not in params # ------------------------------------------------------------------ # REQ-API-003: run_managed_sync unchanged diff --git a/tests/core/test_grab_bag_split_completeness.py b/tests/core/test_grab_bag_split_completeness.py index 86aa4c36e6..048593d0e4 100644 --- a/tests/core/test_grab_bag_split_completeness.py +++ b/tests/core/test_grab_bag_split_completeness.py @@ -132,10 +132,6 @@ def _source_defines_clear_snapshot_cache(source: str) -> bool: "test_github_fetcher_protocol_has_label_methods", "test_subprocess_result_has_elapsed_seconds_field", "test_subprocess_runner_protocol_pty_mode_default_false", - "test_subprocess_runner_protocol_marker_dir_default_none", - "test_subprocess_runner_protocol_session_id_default_none", - "test_subprocess_runner_protocol_marker_params_after_max_extension", - "test_subprocess_runner_protocol_marker_params_are_keyword_only", "test_ci_run_scope_event_field", "test_ci_run_scope_event_defaults_to_none", # test_types_infrastructure_faults.py @@ -265,10 +261,6 @@ def _source_defines_clear_snapshot_cache(source: str) -> bool: "test_github_fetcher_protocol_has_label_methods": "tests.core.test_types_protocols", "test_subprocess_result_has_elapsed_seconds_field": "tests.core.test_types_protocols", "test_subprocess_runner_protocol_pty_mode_default_false": "tests.core.test_types_protocols", - "test_subprocess_runner_protocol_marker_dir_default_none": "tests.core.test_types_protocols", - "test_subprocess_runner_protocol_session_id_default_none": "tests.core.test_types_protocols", - "test_subprocess_runner_protocol_marker_params_after_max_extension": "tests.core.test_types_protocols", # noqa: E501 - "test_subprocess_runner_protocol_marker_params_are_keyword_only": "tests.core.test_types_protocols", # noqa: E501 "test_ci_run_scope_event_field": "tests.core.test_types_protocols", "test_ci_run_scope_event_defaults_to_none": "tests.core.test_types_protocols", "test_skill_command_prefix_constant_exists": "tests.core.test_types_infrastructure_faults", @@ -373,7 +365,7 @@ def _source_defines_clear_snapshot_cache(source: str) -> bool: def test_pre_split_test_types_inventory_is_frozen() -> None: """The pre-split inventory must be a frozen set with no leading-dot or duplicate names.""" assert isinstance(_PRE_SPLIT_TEST_TYPES_NAMES, frozenset) - assert len(_PRE_SPLIT_TEST_TYPES_NAMES) == 44 + assert len(_PRE_SPLIT_TEST_TYPES_NAMES) == 40 for name in _PRE_SPLIT_TEST_TYPES_NAMES: assert "." not in name, f"Invalid name with dot: {name}" diff --git a/tests/core/test_inspector_types.py b/tests/core/test_inspector_types.py index 83a43105e4..d27ff8cbe0 100644 --- a/tests/core/test_inspector_types.py +++ b/tests/core/test_inspector_types.py @@ -39,7 +39,7 @@ def test_construction_all_fields(self): cpu_trend=(0.1, 0.2), rss_trend=(100.0, 200.0), connection_summary="ESTABLISHED=3, CLOSE_WAIT=47", - execution_marker_present=True, + operation_lease_active=True, dispatch_context="recipe=X, step=Y, elapsed=60s", ) assert ev.idle_seconds == 120.0 @@ -47,7 +47,7 @@ def test_construction_all_fields(self): assert ev.cpu_trend == (0.1, 0.2) assert ev.rss_trend == (100.0, 200.0) assert ev.connection_summary == "ESTABLISHED=3, CLOSE_WAIT=47" - assert ev.execution_marker_present is True + assert ev.operation_lease_active is True assert ev.dispatch_context == "recipe=X, step=Y, elapsed=60s" def test_frozen(self): @@ -60,7 +60,7 @@ def test_defaults(self): assert ev.cpu_trend == () assert ev.rss_trend == () assert ev.connection_summary == "" - assert ev.execution_marker_present is False + assert ev.operation_lease_active is False assert ev.dispatch_context == "" diff --git a/tests/core/test_types_protocols.py b/tests/core/test_types_protocols.py index c958a0b28b..6f8affe3a2 100644 --- a/tests/core/test_types_protocols.py +++ b/tests/core/test_types_protocols.py @@ -138,61 +138,10 @@ def test_subprocess_runner_protocol_pty_mode_default_false() -> None: # --------------------------------------------------------------------------- -# P2-A6 — SubprocessRunner marker_dir and session_id params +# P2-A6 — SubprocessRunner operation lease parameter # --------------------------------------------------------------------------- -def test_subprocess_runner_protocol_marker_dir_default_none() -> None: - import inspect - - from autoskillit.core import SubprocessRunner - - sig = inspect.signature(SubprocessRunner.__call__) - assert sig.parameters["marker_dir"].default is None - - -def test_subprocess_runner_protocol_session_id_default_none() -> None: - import inspect - - from autoskillit.core import SubprocessRunner - - sig = inspect.signature(SubprocessRunner.__call__) - assert sig.parameters["session_id"].default is None - - -def test_subprocess_runner_protocol_marker_params_after_max_extension() -> None: - import inspect - - from autoskillit.core import SubprocessRunner - - sig = inspect.signature(SubprocessRunner.__call__) - params = list(sig.parameters) - max_ext_idx = params.index("max_extension_seconds") - marker_idx = params.index("marker_dir") - session_idx = params.index("session_id") - assert marker_idx == max_ext_idx + 1, ( - f"marker_dir must immediately follow max_extension_seconds, " - f"got indices {max_ext_idx} and {marker_idx}" - ) - assert session_idx == marker_idx + 1, ( - f"session_id must immediately follow marker_dir, " - f"got indices {marker_idx} and {session_idx}" - ) - - -def test_subprocess_runner_protocol_marker_params_are_keyword_only() -> None: - import inspect - - from autoskillit.core import SubprocessRunner - - sig = inspect.signature(SubprocessRunner.__call__) - for name in ("marker_dir", "session_id"): - param = sig.parameters[name] - assert param.kind == inspect.Parameter.KEYWORD_ONLY, ( - f"{name} must be keyword-only, got {param.kind.name}" - ) - - def test_subprocess_runner_protocol_operation_lease_dir_default_none() -> None: import inspect @@ -202,13 +151,13 @@ def test_subprocess_runner_protocol_operation_lease_dir_default_none() -> None: assert sig.parameters["operation_lease_dir"].default is None -def test_subprocess_runner_protocol_operation_lease_dir_follows_session_id() -> None: +def test_subprocess_runner_protocol_operation_lease_dir_follows_max_extension() -> None: import inspect from autoskillit.core import SubprocessRunner params = list(inspect.signature(SubprocessRunner.__call__).parameters) - assert params.index("operation_lease_dir") == params.index("session_id") + 1 + assert params.index("operation_lease_dir") == params.index("max_extension_seconds") + 1 def test_subprocess_runner_protocol_operation_lease_dir_is_keyword_only() -> None: diff --git a/tests/execution/headless/test_codex_attempt_retention.py b/tests/execution/headless/test_codex_attempt_retention.py index 686a4d4c0c..204b8b851a 100644 --- a/tests/execution/headless/test_codex_attempt_retention.py +++ b/tests/execution/headless/test_codex_attempt_retention.py @@ -199,8 +199,6 @@ async def _run_generated_attempt( max_extension_seconds=0.0, ceiling_seconds=_EXECUTION_CEILING_SECONDS, systemd_scope_enabled=False, - marker_dir=None, - session_id=None, on_session_id_resolved=None, stream_parser=Mock(), backend_resume_session_id=resume_session_id, @@ -427,8 +425,6 @@ def build_with_retained_binding(binding, _extras, _attempt_id=None) -> CmdSpec: max_extension_seconds=0.0, ceiling_seconds=_EXECUTION_CEILING_SECONDS, systemd_scope_enabled=False, - marker_dir=None, - session_id=None, on_session_id_resolved=None, stream_parser=Mock(), backend_resume_session_id="", diff --git a/tests/execution/test_backend_dispatch.py b/tests/execution/test_backend_dispatch.py index a2a8e4b732..8565f0d024 100644 --- a/tests/execution/test_backend_dispatch.py +++ b/tests/execution/test_backend_dispatch.py @@ -78,6 +78,8 @@ async def test_run_headless_core_uses_ctx_backend_for_command_construction(minim runner_kwargs = mock_runner.call_args.kwargs assert runner_kwargs["backend_resume_session_id"] == "backend-resume-id" assert runner_kwargs["lifecycle_observation_enabled"] is True + assert "marker_dir" not in runner_kwargs + assert "session_id" not in runner_kwargs class TestBackendDispatchRouting: diff --git a/tests/execution/test_headless_provider_forwarding.py b/tests/execution/test_headless_provider_forwarding.py index a09a74f00c..c66e04b2e4 100644 --- a/tests/execution/test_headless_provider_forwarding.py +++ b/tests/execution/test_headless_provider_forwarding.py @@ -609,41 +609,24 @@ def test_build_skill_result_provider_used_survives_budget_guard() -> None: assert sr.provider.provider_used == "bedrock" -# ── marker_dir / session_id forwarding tests ─────────────────────────────────── +# ── dispatch caller identity forwarding tests ───────────────────────────────── -def test_execute_claude_headless_accepts_marker_dir_and_session_id() -> None: - import inspect - - from autoskillit.execution.headless import _execute_claude_headless - - sig = inspect.signature(_execute_claude_headless) - params = sig.parameters - assert "marker_dir" in params - assert params["marker_dir"].default is None - assert params["marker_dir"].kind == inspect.Parameter.KEYWORD_ONLY - assert "session_id" in params - assert params["session_id"].default is None - assert params["session_id"].kind == inspect.Parameter.KEYWORD_ONLY - - -def test_dispatch_food_truck_accepts_marker_dir_and_session_id() -> None: +def test_dispatch_food_truck_keeps_caller_session_id_without_watchdog_arguments() -> None: import inspect from autoskillit.execution.headless import DefaultHeadlessExecutor sig = inspect.signature(DefaultHeadlessExecutor.dispatch_food_truck) params = sig.parameters - assert "marker_dir" in params - assert params["marker_dir"].default is None - assert params["marker_dir"].kind == inspect.Parameter.KEYWORD_ONLY - assert "session_id" in params - assert params["session_id"].default is None - assert params["session_id"].kind == inspect.Parameter.KEYWORD_ONLY + assert params["caller_session_id"].default == "" + assert params["caller_session_id"].kind == inspect.Parameter.KEYWORD_ONLY + assert "marker_dir" not in params + assert "session_id" not in params @pytest.mark.anyio -async def test_dispatch_food_truck_forwards_marker_dir_and_session_id( +async def test_dispatch_food_truck_forwards_caller_session_id( minimal_ctx, tmp_path, monkeypatch ) -> None: @@ -676,17 +659,14 @@ async def fake_execute(spec, cwd, ctx, **kwargs): food_truck_capable=True, pty_required=True, channel_b_capable=False ) executor = DefaultHeadlessExecutor(minimal_ctx) - marker = tmp_path / "markers" await executor.dispatch_food_truck( "prompt", str(tmp_path), completion_marker="%%DONE%%", - marker_dir=marker, - session_id="dispatch-uuid-123", + caller_session_id="dispatch-caller-123", ) - assert execute_kwargs["marker_dir"] == marker - assert execute_kwargs["session_id"] == "dispatch-uuid-123" + assert execute_kwargs["caller_session_id"] == "dispatch-caller-123" # ── readonly_skill forwarding tests ─────────────────────────────────────────── @@ -762,153 +742,6 @@ async def fake_runner(cmd, **kwargs): ) -@pytest.mark.anyio -async def test_dispatch_food_truck_derives_marker_dir_from_cwd( - minimal_ctx, tmp_path, monkeypatch -) -> None: - from pathlib import Path - - from autoskillit.execution.headless import DefaultHeadlessExecutor - - execute_kwargs: dict = {} - - async def fake_execute(spec, cwd, ctx, **kwargs): - execute_kwargs.update(kwargs) - return SkillResult( - success=True, - result="ok", - session_id="s1", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - - monkeypatch.setattr(_patch_execution_headless, "_execute_claude_headless", fake_execute) - monkeypatch.setattr( - _patch_execution_headless, - "_resolve_session_log_dir", - lambda cwd, backend: Path("/derived/project"), - ) - monkeypatch.setattr( - _patch_headless__headless_execute, - "_compute_post_session_metrics", - lambda *a, **kw: object(), - ) - - minimal_ctx.backend = _mock_backend( - food_truck_capable=True, pty_required=True, channel_b_capable=True - ) - executor = DefaultHeadlessExecutor(minimal_ctx) - await executor.dispatch_food_truck( - "prompt", - str(tmp_path), - completion_marker="%%DONE%%", - ) - - assert execute_kwargs["marker_dir"] == Path("/derived/project") - - -@pytest.mark.anyio -async def test_dispatch_food_truck_marker_dir_none_without_channel_b( - minimal_ctx, tmp_path, monkeypatch -) -> None: - execute_kwargs: dict = {} - - async def fake_execute(spec, cwd, ctx, **kwargs): - execute_kwargs.update(kwargs) - return SkillResult( - success=True, - result="ok", - session_id="s1", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - - monkeypatch.setattr(_patch_execution_headless, "_execute_claude_headless", fake_execute) - monkeypatch.setattr( - _patch_headless__headless_execute, - "_compute_post_session_metrics", - lambda *a, **kw: object(), - ) - - from autoskillit.execution.headless import DefaultHeadlessExecutor - - minimal_ctx.backend = _mock_backend( - food_truck_capable=True, pty_required=True, channel_b_capable=False - ) - executor = DefaultHeadlessExecutor(minimal_ctx) - await executor.dispatch_food_truck( - "prompt", - str(tmp_path), - completion_marker="%%DONE%%", - ) - - assert execute_kwargs["marker_dir"] is None - - -@pytest.mark.anyio -async def test_execute_claude_headless_forwards_marker_dir_to_runner( - minimal_ctx, tmp_path, monkeypatch -) -> None: - from pathlib import Path - - from autoskillit.execution.headless import PostSessionMetrics, _execute_claude_headless - from autoskillit.execution.runtime.commands import ClaudeHeadlessCmd - from tests.execution.conftest import _sr - - spec = ClaudeHeadlessCmd(cmd=("echo", "test"), env={}) - runner_kwargs: dict = {} - - async def fake_runner(cmd, **kwargs): - runner_kwargs.update(kwargs) - return _sr() - - minimal_ctx.runner = fake_runner - minimal_ctx.backend = _mock_backend(pty_required=True, channel_b_capable=True) - monkeypatch.setattr( - _patch_headless__headless_execute, - "_build_skill_result", - lambda *a, **kw: SkillResult( - success=True, - result="ok", - session_id="s1", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ), - ) - monkeypatch.setattr( - _patch_headless__headless_execute, - "_compute_post_session_metrics", - lambda *a, **kw: PostSessionMetrics(0, 0), - ) - - await _execute_claude_headless( - lambda _binding, _extras: spec, - str(tmp_path), - minimal_ctx, - timeout=60, - stale_threshold=30, - marker_dir=Path("/custom/markers"), - session_id="sess-abc", - launch_resolver=minimal_ctx.launch_resolver, - launch_preparation=_launch_preparation(minimal_ctx, cwd=str(tmp_path)), - ) - - assert runner_kwargs["marker_dir"] == Path("/custom/markers") - assert runner_kwargs["session_id"] == "sess-abc" - - # ── pty_mode / session_log_dir capability forwarding tests ─────────────────────── @@ -1017,47 +850,6 @@ async def fake_runner(cmd, **kwargs): assert runner_kwargs["session_log_dir"] is None -@pytest.mark.anyio -async def test_dispatch_food_truck_marker_dir_none_when_no_channel_b( - minimal_ctx, tmp_path, monkeypatch -) -> None: - execute_kwargs: dict = {} - - async def fake_execute(spec, cwd, ctx, **kwargs): - execute_kwargs.update(kwargs) - return SkillResult( - success=True, - result="ok", - session_id="s1", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - - monkeypatch.setattr(_patch_execution_headless, "_execute_claude_headless", fake_execute) - monkeypatch.setattr( - _patch_headless__headless_execute, - "_compute_post_session_metrics", - lambda *a, **kw: object(), - ) - - minimal_ctx.backend = _mock_backend(food_truck_capable=True, channel_b_capable=False) - - from autoskillit.execution.headless import DefaultHeadlessExecutor - - executor = DefaultHeadlessExecutor(minimal_ctx) - await executor.dispatch_food_truck( - "prompt", - str(tmp_path), - completion_marker="%%DONE%%", - ) - - assert execute_kwargs["marker_dir"] is None - - # ── stream_parser injection tests ──────────────────────────────────────────── diff --git a/tests/execution/test_model_backend_launch_contract.py b/tests/execution/test_model_backend_launch_contract.py index b1790a0aa0..a7e46f5125 100644 --- a/tests/execution/test_model_backend_launch_contract.py +++ b/tests/execution/test_model_backend_launch_contract.py @@ -217,8 +217,6 @@ def build_spec(_binding, _extras, _managed_attempt_id=None): "on_spawn": None, "enable_deadline_extension": False, "max_extension_seconds": 0, - "marker_dir": None, - "session_id": None, "on_session_id_resolved": None, "stream_parser": None, "backend_resume_session_id": "", diff --git a/tests/execution/test_operation_lease_watchers.py b/tests/execution/test_operation_lease_watchers.py index 07c218efeb..6fcb40e19c 100644 --- a/tests/execution/test_operation_lease_watchers.py +++ b/tests/execution/test_operation_lease_watchers.py @@ -278,11 +278,16 @@ def remove_after_three_ticks(clock: _LeaseClock) -> None: @pytest.mark.anyio -async def test_session_log_lease_defers_api_suppression_but_existing_cap_still_fires( +async def test_session_log_lease_then_child_cpu_respects_heuristic_cap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) - monkeypatch.setattr(process_monitor, "_has_active_api_connection", lambda _pid: True) + child_cpu_active = False + monkeypatch.setattr( + process_monitor, + "_has_active_child_processes", + lambda _pid: child_cpu_active, + ) log_dir = tmp_path / "logs" log_dir.mkdir() session_file = log_dir / "child.jsonl" @@ -297,8 +302,10 @@ async def test_session_log_lease_defers_api_suppression_but_existing_cap_still_f assert handle.path is not None def remove_after_three_ticks(clock: _LeaseClock) -> None: + nonlocal child_cpu_active if clock.ticks == 3: handle.path.unlink() + child_cpu_active = True clock = _LeaseClock( handle.record.started_at_epoch, @@ -325,13 +332,9 @@ def remove_after_three_ticks(clock: _LeaseClock) -> None: for index, entry in enumerate(logs) if entry.get("event") == "stale_deferred_to_operation" ] - api_events = [ - index - for index, entry in enumerate(logs) - if "ESTABLISHED port-443 connection" in str(entry.get("event", "")) - ] assert operation_events - assert api_events and min(api_events) > min(operation_events) + assert clock.elapsed == 5 + assert any("Suppression bounded" in str(entry.get("event", "")) for entry in logs) @pytest.mark.anyio @@ -463,8 +466,6 @@ async def test_drain_liveness_signals_include_active_operation_lease(tmp_path: P registry=InFlightOperations(), ): assert "operation_lease" in _active_liveness_signals( - None, - None, None, operation_lease_dir=tmp_path, ) diff --git a/tests/execution/test_process_deadline_extension.py b/tests/execution/test_process_deadline_extension.py index 9d9a0b10df..203efcdcc5 100644 --- a/tests/execution/test_process_deadline_extension.py +++ b/tests/execution/test_process_deadline_extension.py @@ -5,6 +5,7 @@ import functools import sys import time +from types import SimpleNamespace import anyio import psutil @@ -31,11 +32,6 @@ async def test_pending_task_extension_releases_on_terminal_and_respects_cap( "_has_active_child_processes", lambda pid: False, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, - ) pending = [True] trigger = anyio.Event() original_deadline = anyio.current_time() + 0.03 @@ -74,11 +70,6 @@ async def test_extends_deadline_when_children_active(monkeypatch) -> None: "_has_active_child_processes", lambda pid: True, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, - ) trigger = anyio.Event() scope_ref: list[anyio.CancelScope | None] = [None] original_deadline_ref: list[float] = [] @@ -98,26 +89,19 @@ async def test_extends_deadline_when_children_active(monkeypatch) -> None: @pytest.mark.anyio async def test_no_extension_when_inactive(monkeypatch) -> None: - """Deadline is NOT extended when both probes return False.""" + """Deadline is not extended when no lease or child process is active.""" monkeypatch.setattr( _patch_process__race_watchers, "_has_active_child_processes", lambda pid: False, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, - ) trigger = anyio.Event() scope_ref: list[anyio.CancelScope | None] = [None] original_deadline_ref: list[float] = [] async with anyio.create_task_group() as tg: tg.start_soon( - functools.partial( - _watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05, marker_dir=None - ) + functools.partial(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) ) with anyio.move_on_after(2.0) as scope: scope_ref[0] = scope @@ -138,11 +122,6 @@ async def test_max_extension_cap_enforced(monkeypatch) -> None: "_has_active_child_processes", lambda pid: True, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, - ) trigger = anyio.Event() scope_ref: list[anyio.CancelScope | None] = [None] original_deadline_ref: list[float] = [] @@ -168,11 +147,6 @@ async def test_terminates_on_trigger(monkeypatch) -> None: "_has_active_child_processes", lambda pid: True, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: True, - ) trigger = anyio.Event() scope_ref: list[anyio.CancelScope | None] = [None] @@ -183,80 +157,40 @@ async def test_terminates_on_trigger(monkeypatch) -> None: @pytest.mark.anyio -async def test_api_connection_also_extends(monkeypatch) -> None: - """Deadline is extended when _has_active_api_connection returns True (children inactive).""" - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_child_processes", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: True, - ) - trigger = anyio.Event() - scope_ref: list[anyio.CancelScope | None] = [None] - original_deadline_ref: list[float] = [] +async def test_network_connection_without_lease_does_not_extend_deadline( + monkeypatch, tmp_path +) -> None: + """An established port-443 connection does not extend a lease-free deadline.""" - async with anyio.create_task_group() as tg: - tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) - with anyio.move_on_after(0.1) as scope: - scope_ref[0] = scope - original_deadline_ref.append(scope.deadline) - await anyio.sleep(0.3) - trigger.set() - tg.cancel_scope.cancel() + class ConnectedProcess: + def __init__(self, pid: int) -> None: + self.pid = pid - assert scope_ref[0] is not None - assert scope_ref[0].deadline > original_deadline_ref[0] + def children(self, recursive: bool = False) -> list[object]: + return [] + def connections(self, kind: str | None = None) -> list[SimpleNamespace]: + import psutil -@pytest.mark.anyio -async def test_scope_ref_none_polling(monkeypatch) -> None: - """Watcher polls harmlessly when scope_ref is None (before scope binding).""" - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_child_processes", - lambda pid: True, - ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: True, - ) - trigger = anyio.Event() - scope_ref: list[anyio.CancelScope | None] = [None] # intentionally empty + return [ + SimpleNamespace( + status=psutil.CONN_ESTABLISHED, + raddr=SimpleNamespace(port=443), + ) + ] - with anyio.fail_after(2.0): - # Run watcher with empty scope_ref for 3 poll cycles, then set scope and trigger - async with anyio.create_task_group() as tg: - tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) - await anyio.sleep(0.2) - with anyio.move_on_after(1.0) as scope: - scope_ref[0] = scope - trigger.set() - tg.cancel_scope.cancel() + net_connections = connections + channel = tmp_path / "leases" + channel.mkdir() + import autoskillit.execution.process._process_monitor as process_monitor -@pytest.mark.anyio -async def test_extends_deadline_when_dispatch_marker_active(monkeypatch, tmp_path) -> None: - """Deadline is extended when dispatch marker is active (other signals inactive).""" + monkeypatch.setattr(process_monitor.psutil, "Process", ConnectedProcess) monkeypatch.setattr( _patch_process__race_watchers, "_has_active_child_processes", lambda pid: False, ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, **kw: True, - ) trigger = anyio.Event() scope_ref: list[anyio.CancelScope | None] = [None] original_deadline_ref: list[float] = [] @@ -270,8 +204,7 @@ async def test_extends_deadline_when_dispatch_marker_active(monkeypatch, tmp_pat 7200.0, trigger, 0.05, - marker_dir=tmp_path, - session_id="test-sid", + operation_lease_dir=channel, ) ) with anyio.move_on_after(0.1) as scope: @@ -282,53 +215,72 @@ async def test_extends_deadline_when_dispatch_marker_active(monkeypatch, tmp_pat tg.cancel_scope.cancel() assert scope_ref[0] is not None - assert scope_ref[0].deadline > original_deadline_ref[0] + assert scope_ref[0].deadline == original_deadline_ref[0] @pytest.mark.anyio -async def test_no_extension_when_marker_inactive(monkeypatch, tmp_path) -> None: - """Deadline is NOT extended when all three signals are inactive (fleet context).""" +async def test_operation_lease_extends_deadline(monkeypatch, tmp_path) -> None: + from autoskillit.core import InFlightOperations, operation_lease + monkeypatch.setattr( _patch_process__race_watchers, "_has_active_child_processes", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_api_connection", - lambda pid: False, + lambda _pid: False, ) + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + heartbeat_interval=10000, + ): + trigger = anyio.Event() + scope_ref: list[anyio.CancelScope | None] = [None] + original_deadline_ref: list[float] = [] + + async with anyio.create_task_group() as tg: + tg.start_soon( + functools.partial( + _watch_child_activity, + 1, + scope_ref, + 7200.0, + trigger, + 0.05, + operation_lease_dir=tmp_path, + ) + ) + with anyio.move_on_after(0.1) as scope: + scope_ref[0] = scope + original_deadline_ref.append(scope.deadline) + await anyio.sleep(0.3) + trigger.set() + tg.cancel_scope.cancel() + + assert scope_ref[0] is not None + assert scope_ref[0].deadline > original_deadline_ref[0] + + +@pytest.mark.anyio +async def test_scope_ref_none_polling(monkeypatch) -> None: + """Watcher polls harmlessly when scope_ref is None (before scope binding).""" monkeypatch.setattr( _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, **kw: False, + "_has_active_child_processes", + lambda pid: True, ) trigger = anyio.Event() - scope_ref: list[anyio.CancelScope | None] = [None] - original_deadline_ref: list[float] = [] + scope_ref: list[anyio.CancelScope | None] = [None] # intentionally empty - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_child_activity, - 1, - scope_ref, - 7200.0, - trigger, - 0.05, - marker_dir=tmp_path, - session_id="test-sid", - ) - ) - with anyio.move_on_after(2.0) as scope: - scope_ref[0] = scope - original_deadline_ref.append(scope.deadline) - await anyio.sleep(0.5) + with anyio.fail_after(2.0): + # Run watcher with empty scope_ref for 3 poll cycles, then set scope and trigger + async with anyio.create_task_group() as tg: + tg.start_soon(_watch_child_activity, 1, scope_ref, 7200.0, trigger, 0.05) + await anyio.sleep(0.2) + with anyio.move_on_after(1.0) as scope: + scope_ref[0] = scope trigger.set() - tg.cancel_scope.cancel() - - assert scope_ref[0] is not None - assert scope_ref[0].deadline == original_deadline_ref[0] + tg.cancel_scope.cancel() @pytest.mark.parametrize( @@ -361,8 +313,7 @@ async def watcher(*args, **kwargs) -> None: timeout_scope_ref=scope_ref, max_extension_seconds=90.0, trigger=trigger, - marker_dir=tmp_path, - session_id="test-sid", + operation_lease_dir=tmp_path, lifecycle_observation_enabled=lifecycle, acc=acc, ) @@ -377,8 +328,9 @@ async def watcher(*args, **kwargs) -> None: assert isinstance(scheduled, functools.partial) assert scheduled.func is watcher assert scheduled.args == (pid, scope_ref, 90.0, trigger) - assert scheduled.keywords["marker_dir"] == tmp_path - assert scheduled.keywords["session_id"] == "test-sid" + assert scheduled.keywords["operation_lease_dir"] == tmp_path + assert "marker_dir" not in scheduled.keywords + assert "session_id" not in scheduled.keywords callback = scheduled.keywords["has_pending_tasks"] if lifecycle: assert callback.__self__ is acc @@ -387,8 +339,8 @@ async def watcher(*args, **kwargs) -> None: assert callback is None -def test_marker_dir_threaded_from_race_watcher_wiring() -> None: - """Race watcher wiring forwards marker_dir and session_id to child activity.""" +def test_operation_lease_dir_threaded_from_race_watcher_wiring() -> None: + """Race watcher wiring forwards the lease channel to child activity.""" import re from pathlib import Path @@ -396,11 +348,11 @@ def test_marker_dir_threaded_from_race_watcher_wiring() -> None: pattern = ( r"functools\.partial\(\s*_watch_child_activity," - r".*?marker_dir=marker_dir.*?session_id=session_id" + r".*?operation_lease_dir=operation_lease_dir" ) match = re.search(pattern, watcher_source, re.DOTALL) assert match is not None, ( - "race watcher wiring does not thread marker_dir and session_id " + "race watcher wiring does not thread operation_lease_dir " "to _watch_child_activity via functools.partial" ) diff --git a/tests/execution/test_process_heartbeat.py b/tests/execution/test_process_heartbeat.py index 98f752ea5e..e458415e01 100644 --- a/tests/execution/test_process_heartbeat.py +++ b/tests/execution/test_process_heartbeat.py @@ -1,14 +1,11 @@ -"""Unit tests for _heartbeat, _has_active_api_connection, _has_active_child_processes, +"""Unit tests for _heartbeat, _has_active_child_processes, and orphaned tool result detection.""" from __future__ import annotations -import ast -import inspect -import os as _os import sys import time -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import anyio import psutil @@ -16,9 +13,7 @@ from autoskillit.core.types import ChannelBStatus, TerminationReason from autoskillit.execution.process import ( - _has_active_api_connection, _has_active_child_processes, - _has_active_execution_marker, _heartbeat, _session_log_monitor, run_managed_async, @@ -188,118 +183,6 @@ async def test_heartbeat_kill_sets_completed_termination(self, tmp_path): assert result.termination == TerminationReason.COMPLETED -class TestHasActiveApiConnection: - """Unit tests for _has_active_api_connection.""" - - def _make_conn(self, port: int, status: str = "ESTABLISHED"): - from unittest.mock import Mock - - conn = Mock() - conn.status = status - conn.raddr = Mock() - conn.raddr.port = port - return conn - - def _patch_psutil(self, parent_conns, child_conns_list=None): - """Returns a context manager patching psutil in process_lifecycle.""" - from unittest.mock import Mock - - mock_parent = Mock() - mock_parent.net_connections.return_value = parent_conns - children = [] - for child_conns in child_conns_list or []: - mock_child = Mock() - mock_child.net_connections.return_value = child_conns - children.append(mock_child) - mock_parent.children.return_value = children - return patch( - "autoskillit.execution.process._process_monitor.psutil.Process", - return_value=mock_parent, - ) - - def test_returns_true_when_parent_has_established_port_443(self): - with self._patch_psutil([self._make_conn(443)]): - assert _has_active_api_connection(12345) is True - - def test_returns_true_when_child_has_established_port_443(self): - with self._patch_psutil( - parent_conns=[self._make_conn(80)], - child_conns_list=[[self._make_conn(443)]], - ): - assert _has_active_api_connection(12345) is True - - def test_returns_false_when_no_connections(self): - with self._patch_psutil([]): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_all_connections_non_443(self): - conns = [self._make_conn(80), self._make_conn(8080), self._make_conn(22)] - with self._patch_psutil(conns): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_443_is_not_established(self): - conns = [ - self._make_conn(443, status="TIME_WAIT"), - self._make_conn(443, status="CLOSE_WAIT"), - ] - with self._patch_psutil(conns): - assert _has_active_api_connection(12345) is False - - def test_returns_false_when_no_raddr(self): - from unittest.mock import Mock - - conn = Mock() - conn.status = "ESTABLISHED" - conn.raddr = None - with self._patch_psutil([conn]): - assert _has_active_api_connection(12345) is False - - def test_returns_false_on_nosuchprocess(self): - import psutil as _psutil - - with patch( - "autoskillit.execution.process._process_monitor.psutil.Process", - side_effect=_psutil.NoSuchProcess(12345), - ): - assert _has_active_api_connection(12345) is False - - def test_skips_dead_child_gracefully(self): - from unittest.mock import Mock - - import psutil as _psutil - - mock_parent = Mock() - mock_parent.net_connections.return_value = [] - mock_dead_child = Mock() - mock_dead_child.net_connections.side_effect = _psutil.NoSuchProcess(99999) - mock_live_child = Mock() - mock_live_child.net_connections.return_value = [self._make_conn(443)] - mock_parent.children.return_value = [mock_dead_child, mock_live_child] - with patch( - "autoskillit.execution.process._process_monitor.psutil.Process", - return_value=mock_parent, - ): - assert _has_active_api_connection(12345) is True - - def test_skips_zombie_child_gracefully(self): - from unittest.mock import Mock - - import psutil as _psutil - - mock_parent = Mock() - mock_parent.net_connections.return_value = [] - mock_zombie = Mock() - mock_zombie.net_connections.side_effect = _psutil.ZombieProcess(99998) - mock_live_child = Mock() - mock_live_child.net_connections.return_value = [self._make_conn(443)] - mock_parent.children.return_value = [mock_zombie, mock_live_child] - with patch( - "autoskillit.execution.process._process_monitor.psutil.Process", - return_value=mock_parent, - ): - assert _has_active_api_connection(12345) is True - - class TestHasActiveChildProcesses: """Unit tests for _has_active_child_processes. @@ -829,75 +712,3 @@ async def test_codex_app_server_nonterminal_notifications_stay_silent(self, tmp_ record_types=frozenset({"turn.completed", "turn.failed", "error"}), _poll_interval=0.05, ) - - -class TestHasActiveExecutionMarker: - """Unit tests for _has_active_execution_marker.""" - - def test_dispatch_marker_nonexistent_dir_returns_false(self, tmp_path): - """Nonexistent marker directory returns False without raising.""" - result = _has_active_execution_marker(tmp_path / "nonexistent") - assert result is False - - def test_dispatch_marker_fresh_marker_returns_true(self, tmp_path): - """A fresh marker file causes the function to return True.""" - marker = tmp_path / "dispatch-in-progress-sess1-abc.marker" - marker.touch() - result = _has_active_execution_marker(tmp_path) - assert result is True - - def test_dispatch_marker_expired_marker_returns_false(self, tmp_path): - """A marker file with mtime older than max_marker_age returns False.""" - marker = tmp_path / "dispatch-in-progress-xyz-001.marker" - marker.touch() - old_time = time.time() - 120 - - _os.utime(marker, (old_time, old_time)) - result = _has_active_execution_marker(tmp_path, max_marker_age=60.0) - assert result is False - - def test_dispatch_marker_session_id_filters(self, tmp_path): - """When session_id is provided, only markers matching that session are considered.""" - (tmp_path / "dispatch-in-progress-abc-001.marker").touch() - (tmp_path / "dispatch-in-progress-xyz-002.marker").touch() - result_matching = _has_active_execution_marker(tmp_path, session_id="abc") - result_non_matching = _has_active_execution_marker(tmp_path, session_id="def") - assert result_matching is True - assert result_non_matching is False - - def test_dispatch_marker_none_session_id_matches_all(self, tmp_path): - """session_id=None matches any dispatch-in-progress marker.""" - marker = tmp_path / "dispatch-in-progress-xyz-001.marker" - marker.touch() - result = _has_active_execution_marker(tmp_path, session_id=None) - assert result is True - - def test_dispatch_marker_no_logger_calls(self): - """The function body contains no logger.* attribute access calls.""" - - source = inspect.getsource(_has_active_execution_marker) - tree = ast.parse(source) - for node in ast.walk(tree): - if ( - isinstance(node, ast.Attribute) - and isinstance(node.value, ast.Name) - and node.value.id == "logger" - ): - pytest.fail(f"Found logger.{node.attr} in _has_active_execution_marker body") - - @pytest.mark.parametrize( - "prefix", - ["dispatch-in-progress", "run-skill-in-progress"], - ) - def test_execution_marker_matches_both_prefixes(self, tmp_path, prefix): - """Both dispatch-in-progress-* and run-skill-in-progress-* markers are matched.""" - marker = tmp_path / f"{prefix}-sess1-step.marker" - marker.touch() - result = _has_active_execution_marker(tmp_path) - assert result is True, f"{prefix}-* marker not matched by *-in-progress-* glob pattern" - - def test_wrong_session_id_gives_no_match(self, tmp_path): - """Marker for session-A does not match when globbing for session-B.""" - (tmp_path / "run-skill-in-progress-session-A-step.marker").touch() - assert _has_active_execution_marker(tmp_path, session_id="session-B") is False - assert _has_active_execution_marker(tmp_path, session_id="session-A") is True diff --git a/tests/execution/test_process_idle_watchdog.py b/tests/execution/test_process_idle_watchdog.py index 2b1f4f73dc..b19f3449b5 100644 --- a/tests/execution/test_process_idle_watchdog.py +++ b/tests/execution/test_process_idle_watchdog.py @@ -3,15 +3,13 @@ from __future__ import annotations import functools -import os import sys import textwrap import time -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import anyio import pytest -import structlog.testing import autoskillit.execution.process._race_watchers as _patch_process__race_watchers from autoskillit.execution.process._process_race import RaceAccumulator @@ -65,139 +63,6 @@ async def test_pending_task_suppression_releases_on_terminal_or_max_bound( assert elapsed < 0.2 -@pytest.mark.anyio -async def test_watch_stdout_idle_emits_suppression_evaluated_debug_log( - tmp_path: anyio.Path, -) -> None: - """Debug log fires when marker_dir is None and IDLE_STALL fires.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with patch.object(_patch_process__race_watchers, "logger", new=MagicMock()) as mock_debug: - with anyio.fail_after(2.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.1, - acc, - trigger, - 0.05, - ) - ) - await trigger.wait() - - debug_calls = mock_debug.debug.call_args_list - evaluated_calls = [ - c for c in debug_calls if c.args and c.args[0] == "stdout_idle_stall_suppression_evaluated" - ] - assert len(evaluated_calls) == 1 - call_kwargs = evaluated_calls[0].kwargs - assert call_kwargs.get("marker_dir_present") is False - assert call_kwargs.get("session_id") is None - assert call_kwargs.get("suppression_skipped_reason") == "marker_dir_none" - - -@pytest.mark.anyio -async def test_watch_stdout_idle_emits_suppression_evaluated_with_marker_dir( - tmp_path: anyio.Path, -) -> None: - """Debug log fires when marker_dir is present but marker inactive.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with patch.object(_patch_process__race_watchers, "logger", new=MagicMock()) as mock_debug: - with anyio.fail_after(2.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.1, - acc, - trigger, - 0.05, - marker_dir=tmp_path, - ) - ) - await trigger.wait() - - debug_calls = mock_debug.debug.call_args_list - evaluated_calls = [ - c for c in debug_calls if c.args and c.args[0] == "stdout_idle_stall_suppression_evaluated" - ] - assert len(evaluated_calls) == 1 - call_kwargs = evaluated_calls[0].kwargs - assert call_kwargs.get("marker_dir_present") is True - assert "suppression_skipped_reason" not in call_kwargs - - -@pytest.mark.anyio -async def test_watch_stdout_idle_suppression_evaluated_fires_once_during_suppression( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Debug log fires exactly once on first suppression entry, not each tick.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - with patch.object(_patch_process__race_watchers, "logger", new=MagicMock()) as mock_debug: - - async def cancel_when_suppressed() -> None: - # Wait for at least 2 suppressed warnings before cancelling - suppressed_count = 0 - start = anyio.current_time() - while suppressed_count < 2: - await anyio.sleep(0.02) - suppressed_count = sum( - 1 - for c in mock_debug.call_args_list - if (c.kwargs.get("event") == "stdout_idle_stall_suppressed") - ) - if anyio.current_time() - start > 1.5: - break - trigger.set() - - with anyio.fail_after(2.0): - async with anyio.create_task_group() as tg: - tg.start_soon(cancel_when_suppressed) - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.05, - acc, - trigger, - 0.02, - marker_dir=tmp_path, - session_id="test-sess", - max_suppression_seconds=10.0, - ) - ) - await trigger.wait() - - debug_calls = mock_debug.debug.call_args_list - evaluated_calls = [ - c for c in debug_calls if c.args and c.args[0] == "stdout_idle_stall_suppression_evaluated" - ] - assert len(evaluated_calls) == 1 - - pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] WRITE_BURST_THEN_STALL_SCRIPT = textwrap.dedent("""\ @@ -347,430 +212,6 @@ async def advance_clock(_seconds: float) -> None: assert stdout_path.stat.call_count == 3 -@pytest.mark.anyio -async def test_watch_stdout_idle_suppressed_by_dispatch_marker( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Active dispatch marker suppresses idle stall within cap.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - with structlog.testing.capture_logs() as cap: - - async def cancel_when_suppressed() -> None: - while not any(e.get("event") == "stdout_idle_stall_suppressed" for e in cap): - await anyio.sleep(0.01) - trigger.set() - - async with anyio.create_task_group() as tg: - tg.start_soon(cancel_when_suppressed) - with anyio.fail_after(2.0): - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.1, # idle_output_timeout — very short - acc, - trigger, - 0.05, # _poll_interval - marker_dir=tmp_path, - session_id="test-sess", - max_suppression_seconds=10.0, - ) - ) - await trigger.wait() - - assert acc.idle_stall is False - - -@pytest.mark.anyio -async def test_watch_stdout_idle_fires_when_suppression_cap_exceeded( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Suppression cap reached — idle stall fires despite active marker.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(3.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.05, - acc, - trigger, - 0.02, - marker_dir=tmp_path, - session_id=None, - max_suppression_seconds=0.1, - ) - ) - await trigger.wait() - - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_no_marker_dir_fires_immediately(tmp_path: anyio.Path) -> None: - """No marker_dir — idle stall fires unchanged (existing behavior).""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(2.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - _watch_stdout_idle, - stdout_file, - 0.1, - acc, - trigger, - 0.02, - ) - await trigger.wait() - - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_suppression_timer_resets_on_growth( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Growth resets suppression timer — second idle period gets fresh window.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - async def grow_after_delay() -> None: - await anyio.sleep(0.15) - async with await anyio.open_file(stdout_file, "ab") as f: - await f.write(b"more data\n") - - with anyio.fail_after(3.0): - async with anyio.create_task_group() as tg: - tg.start_soon(grow_after_delay) - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.05, - acc, - trigger, - 0.02, - marker_dir=tmp_path, - session_id=None, - max_suppression_seconds=0.2, - ) - ) - await trigger.wait() - - # Eventually fires after growth stops and cap is exceeded - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_emits_suppression_warning( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Suppression emits structured warning with expected kwargs.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - with structlog.testing.capture_logs() as cap: - - async def cancel_when_suppressed() -> None: - while not any(e.get("event") == "stdout_idle_stall_suppressed" for e in cap): - await anyio.sleep(0.01) - trigger.set() - - async with anyio.create_task_group() as tg: - tg.start_soon(cancel_when_suppressed) - with anyio.fail_after(2.0): - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.05, - acc, - trigger, - 0.02, - marker_dir=tmp_path, - session_id="my-session", - max_suppression_seconds=10.0, - ) - ) - await trigger.wait() - - warnings = [e for e in cap if e.get("event") == "stdout_idle_stall_suppressed"] - assert len(warnings) >= 1 - w = warnings[0] - assert w["marker_dir"] == str(tmp_path) - assert w["session_id"] == "my-session" - assert "suppression_elapsed" in w - assert w["max_suppression_seconds"] == 10.0 - - -@pytest.mark.anyio -async def test_watch_stdout_idle_marker_false_fires_immediately( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Dispatch marker returns False — idle stall fires without suppression.""" - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: False, - ) - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(2.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.1, - acc, - trigger, - 0.02, - marker_dir=tmp_path, - session_id="test", - ) - ) - await trigger.wait() - - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_dispatch_marker_suppresses_stall( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Active dispatch marker suppresses idle stall within the suppression window.""" - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - stdout_file = tmp_path / "stdout.txt" - stdout_file.write_bytes(b"") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(4.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.5, - acc, - trigger, - 0.2, - marker_dir=tmp_path, - ) - ) - await anyio.sleep(1.0) - tg.cancel_scope.cancel() - - assert acc.idle_stall is False - - -@pytest.mark.anyio -async def test_watch_stdout_idle_fires_when_marker_absent(tmp_path: anyio.Path) -> None: - """No marker_dir kwarg — idle stall fires unchanged (existing behavior path).""" - stdout_file = tmp_path / "stdout.txt" - stdout_file.write_bytes(b"") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(3.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - _watch_stdout_idle, - stdout_file, - 0.5, - acc, - trigger, - 0.2, - ) - await trigger.wait() - - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_fires_when_marker_stale(tmp_path: anyio.Path) -> None: - """Marker file exists but is stale (mtime 120s ago) — watchdog fires.""" - marker_path = tmp_path / "dispatch-in-progress-testsession-abc123.marker" - marker_path.write_text("{}") - stale_time = time.time() - 120 - os.utime(str(marker_path), (stale_time, stale_time)) - - stdout_file = tmp_path / "stdout.txt" - stdout_file.write_bytes(b"") - - acc = RaceAccumulator() - trigger = anyio.Event() - - with anyio.fail_after(3.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.5, - acc, - trigger, - 0.2, - marker_dir=tmp_path, - session_id=None, - ) - ) - await trigger.wait() - - assert acc.idle_stall is True - - -@pytest.mark.anyio -async def test_watch_stdout_idle_marker_suppression_bounded( - tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Suppression cap exceeded — idle stall fires despite active marker.""" - monkeypatch.setattr( - _patch_process__race_watchers, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - stdout_file = tmp_path / "stdout.txt" - stdout_file.write_bytes(b"") - - acc = RaceAccumulator() - trigger = anyio.Event() - - start = time.monotonic() - with anyio.fail_after(6.0): - async with anyio.create_task_group() as tg: - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.5, - acc, - trigger, - 0.2, - marker_dir=tmp_path, - max_suppression_seconds=1.5, - ) - ) - await trigger.wait() - - elapsed = time.monotonic() - start - assert acc.idle_stall is True - assert elapsed >= 1.5 - assert elapsed < 5.0 - - -@pytest.mark.anyio -async def test_stdout_idle_NOT_fired_when_execution_marker_active( - tmp_path: anyio.Path, -) -> None: - """IDLE_STALL NOT fired when stdout silent but run-skill-in-progress marker is fresh. - - Failing before implementation: run-skill-in-progress-* marker is invisible to - dispatch-in-progress-* glob, so IDLE_STALL fires immediately. - After fix: marker matched by *-in-progress-* glob, stall suppressed. - """ - stdout_file = tmp_path / "stdout.txt" - await anyio.Path(stdout_file).write_bytes(b"initial output\n") - - marker_dir = tmp_path / "marker_dir" - await anyio.Path(marker_dir).mkdir() - marker_path = marker_dir / "run-skill-in-progress-caller-session-step1.marker" - await anyio.Path(marker_path).write_text("{}") - - acc = RaceAccumulator() - trigger = anyio.Event() - - async def touch_marker() -> None: - for _ in range(100): - await anyio.sleep(0.02) - try: - await anyio.Path(marker_path).touch() - except OSError: - break - - with anyio.move_on_after(0.5): - async with anyio.create_task_group() as tg: - tg.start_soon(touch_marker) - tg.start_soon( - functools.partial( - _watch_stdout_idle, - stdout_file, - 0.05, # idle_output_timeout — short to trigger quickly if not suppressed - acc, - trigger, - 0.02, # _poll_interval - marker_dir=marker_dir, - session_id="caller-session", - max_suppression_seconds=10.0, - ) - ) - await trigger.wait() - - assert not acc.idle_stall, ( - "IDLE_STALL fired even though run-skill execution marker was active. " - "The *-in-progress-* glob fix is needed." - ) - - -# test write - - @pytest.mark.anyio async def test_inspector_callback_kill_fires_idle_stall(tmp_path: anyio.Path) -> None: """KILL verdict from inspector callback sets acc.inspector_verdict + idle_stall.""" diff --git a/tests/execution/test_process_run.py b/tests/execution/test_process_run.py index e274bf3d23..c081bfc7aa 100644 --- a/tests/execution/test_process_run.py +++ b/tests/execution/test_process_run.py @@ -21,7 +21,6 @@ import pytest import structlog.testing -import autoskillit.execution.process._process_monitor as _patch_process__process_monitor from autoskillit.core.types import TerminationReason from autoskillit.execution.process import ( _normalize_pass_fds, @@ -472,12 +471,6 @@ async def test_run_managed_async_idle_stall_kills_hanging_process(self, tmp_path """) ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: True, - ) - start = time.monotonic() with anyio.fail_after(15.0): result = await run_managed_async( @@ -495,6 +488,37 @@ async def test_run_managed_async_idle_stall_kills_hanging_process(self, tmp_path class TestOperationLeaseLiveness: + @pytest.mark.anyio + async def test_silent_child_without_lease_stalls_before_suppression_cap( + self, tmp_path: Path + ) -> None: + from autoskillit.core import OPERATION_LEASE_DIR_ENV_VAR + + channel = tmp_path / "leases" + channel.mkdir() + script = tmp_path / "silent_without_lease.py" + script.write_text( + "import json, time\n" + 'print(json.dumps({"type": "assistant", "burst": True}), flush=True)\n' + "time.sleep(30)\n" + ) + + started = time.monotonic() + with anyio.fail_after(15): + result = await run_managed_async( + [sys.executable, str(script)], + cwd=tmp_path, + timeout=45, + idle_output_timeout=2.0, + max_suppression_seconds=30.0, + operation_lease_dir=channel, + env={**os.environ, OPERATION_LEASE_DIR_ENV_VAR: str(channel)}, + ) + + elapsed = time.monotonic() - started + assert result.termination == TerminationReason.IDLE_STALL + assert elapsed < 12.0, f"IDLE_STALL waited {elapsed:.1f}s despite no active lease" + @pytest.mark.anyio async def test_real_lease_keeps_silent_child_alive_past_idle_cap( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -525,12 +549,6 @@ async def hold_lease(): """ ) ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda _pid: False, - ) - with structlog.testing.capture_logs() as logs: with anyio.fail_after(25): result = await run_managed_async( @@ -624,12 +642,6 @@ async def hold(): + textwrap.dedent(setup) + "\ntime.sleep(30)\n" ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda _pid: False, - ) - started = time.monotonic() with anyio.fail_after(20): result = await run_managed_async( diff --git a/tests/execution/test_process_session_log_monitor.py b/tests/execution/test_process_session_log_monitor.py index 1af8392d86..20cb725285 100644 --- a/tests/execution/test_process_session_log_monitor.py +++ b/tests/execution/test_process_session_log_monitor.py @@ -932,18 +932,13 @@ async def test_session_log_monitor_returns_dir_missing_only_when_directory_absen class TestResumeBoundary: """Phase 2 must not fire on completion markers that existed before monitoring began.""" - @pytest.mark.parametrize("caller_session_id", [None, "session-abc"]) @pytest.mark.anyio - async def test_monitor_skips_preexisting_completion_marker( - self, - tmp_path, - caller_session_id: str | None, - ): + async def test_monitor_skips_preexisting_completion_marker(self, tmp_path): """Phase 2 must NOT fire on a completion marker that existed before monitoring began. Reproduces the resume-boundary false-fire: on `claude --resume`, the JSONL file already contains the completion marker from the prior session. The - no-caller path must retain the same new-bytes-only boundary. + monitor must retain its new-bytes-only boundary. """ log_dir = tmp_path / "session_logs" @@ -998,7 +993,6 @@ async def run_monitor(): _phase1_poll=0.01, _phase2_poll=0.05, _on_poll=count_polls, - caller_session_id=caller_session_id, ) ) @@ -1098,113 +1092,16 @@ async def test_phase2_no_spurious_read_on_preexisting_content(self, tmp_path): spawn_time=spawn_time, _phase1_poll=0.01, _phase2_poll=0.05, - caller_session_id="session-ghi", ) assert result.status == ChannelBStatus.STALE @pytest.mark.anyio -async def test_stale_NOT_fired_when_execution_marker_active(tmp_path): - """Stale NOT fired when JSONL silent but a run-skill-in-progress marker is active and fresh. - - This is a TDD guard for the execution marker blind spot: current code uses - dispatch-in-progress-* glob, so run-skill-in-progress-* markers are invisible. - After the fix (glob -> *-in-progress-*), this test must pass. - """ - import json - - session_file = tmp_path / "abc123.jsonl" - session_file.write_text( - json.dumps({"type": "assistant", "message": {"role": "assistant", "content": "working"}}) - + "\n" - ) - spawn_time = time.time() - 1 - - marker_dir = tmp_path / "marker_dir" - marker_dir.mkdir() - marker_path = marker_dir / "run-skill-in-progress-caller-session-step1.marker" - marker_path.write_text("{}") - - monitor_returned = anyio.Event() - monitor_status: list[object] = [] - - async def run_monitor() -> None: - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=marker_dir, - caller_session_id="caller-session", - _phase1_poll=0.01, - _phase2_poll=0.05, - max_suppression_seconds=10.0, - ) - monitor_status.append(result.status) - monitor_returned.set() - - async def touch_marker() -> None: - for _ in range(60): - await anyio.sleep(0.05) - try: - marker_path.touch() - except OSError: - break - - with anyio.move_on_after(0.6): - async with anyio.create_task_group() as tg: - tg.start_soon(touch_marker) - tg.start_soon(run_monitor) - - assert not monitor_returned.is_set(), ( - f"Monitor returned early: stale fired with active run-skill execution marker. " - f"Status: {monitor_status}" - ) - assert monitor_status == [], ( - f"Monitor emitted a status despite not returning: {monitor_status}" - ) - - -@pytest.mark.anyio -async def test_stale_fired_when_execution_marker_expired(tmp_path): - """Stale fires when JSONL is silent and the run-skill execution marker is expired.""" - import json - import os - - session_file = tmp_path / "abc123.jsonl" - session_file.write_text( - json.dumps({"type": "assistant", "message": {"role": "assistant", "content": "working"}}) - + "\n" - ) - spawn_time = time.time() - 1 - - marker_dir = tmp_path / "marker_dir" - marker_dir.mkdir() - marker_path = marker_dir / "run-skill-in-progress-caller-session-step1.marker" - marker_path.write_text("{}") - past = time.time() - 120 # 120s old — well beyond default max_marker_age=60s - os.utime(marker_path, (past, past)) - - with anyio.fail_after(3.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=marker_dir, - caller_session_id="caller-session", - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - - -@pytest.mark.anyio -async def test_watch_session_log_passes_marker_dir_to_monitor_kwargs( +async def test_watch_session_log_passes_operation_lease_dir_to_monitor_kwargs( tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """marker_dir and session_id are injected into _monitor_kwargs.""" + """The active operation lease channel reaches the session monitor.""" captured_kwargs: dict[str, object] = {} async def fake_session_log_monitor(*args, **kwargs) -> SessionMonitorResult: @@ -1239,19 +1136,17 @@ async def fake_session_log_monitor(*args, **kwargs) -> SessionMonitorResult: _phase2_poll=0.05, _phase1_timeout=0.1, max_suppression_seconds=1800.0, - marker_dir=tmp_path, - session_id="parent-session", + operation_lease_dir=tmp_path, ) - assert captured_kwargs["marker_dir"] == tmp_path - assert captured_kwargs["caller_session_id"] == "parent-session" + assert captured_kwargs["operation_lease_dir"] == tmp_path @pytest.mark.anyio -async def test_watch_session_log_omits_marker_kwargs_when_none( +async def test_watch_session_log_omits_operation_lease_dir_when_none( tmp_path: anyio.Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """marker_dir=None and session_id=None — keys absent from _monitor_kwargs.""" + """No lease channel is added when the watcher has no operation lease directory.""" captured_kwargs: dict[str, object] = {} async def fake_session_log_monitor(*args, **kwargs) -> SessionMonitorResult: @@ -1287,5 +1182,4 @@ async def fake_session_log_monitor(*args, **kwargs) -> SessionMonitorResult: _phase1_timeout=0.1, ) - assert "marker_dir" not in captured_kwargs - assert "caller_session_id" not in captured_kwargs + assert "operation_lease_dir" not in captured_kwargs diff --git a/tests/execution/test_process_session_log_monitor_dispatch_marker.py b/tests/execution/test_process_session_log_monitor_dispatch_marker.py deleted file mode 100644 index 95dd3dcfae..0000000000 --- a/tests/execution/test_process_session_log_monitor_dispatch_marker.py +++ /dev/null @@ -1,430 +0,0 @@ -"""Dispatch marker suppression gate tests for _session_log_monitor.""" - -from __future__ import annotations - -import os -import time - -import anyio -import pytest - -import autoskillit.execution.process._process_monitor as _patch_process__process_monitor -from autoskillit.core.types import ChannelBStatus -from autoskillit.execution.process import _session_log_monitor - -pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] - - -class TestStaleSuppressionDispatchMarker: - """Dispatch marker suppresses stale: fresh marker → suppressed; expired → pass-through. - - Tests the following scenarios for dispatch marker stale suppression: - - T1: Active marker (True→False via monkeypatch) suppresses stale, then fires - - T2: Expired marker (mtime > 60s) does NOT suppress stale (real helper, no monkeypatch — - exercises the mtime-expiry threshold in _has_active_execution_marker directly) - - T3: No marker_dir (None) skips dispatch check entirely (regression guard) - - T4: Bounded suppression fires after max_suppression_seconds - - T5: Session-scoped matching — sessionA marker does not suppress sessionB - - Monkeypatch target: autoskillit.execution.process._process_monitor._has_active_execution_marker - Convention: caller_session_id= is the kwarg used at call sites. - Note: T1, T4 monkeypatch the helper; T2, T5 use real marker files to test helper internals. - """ - - @pytest.mark.anyio - async def test_stale_suppressed_by_active_dispatch_marker(self, tmp_path, monkeypatch): - """Active dispatch marker suppresses stale, then fires when marker disappears.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - call_count = {"n": 0} - - def side_effect_fn(marker_dir, session_id=None): - call_count["n"] += 1 - return call_count["n"] == 1 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - side_effect_fn, - ) - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=tmp_path, - caller_session_id="test-session", - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - - @pytest.mark.anyio - async def test_stale_not_suppressed_when_marker_expired(self, tmp_path): - """Expired marker (mtime > 60s) does not suppress stale.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - marker_path = tmp_path / "dispatch-in-progress-test-session-abc.marker" - marker_path.write_text("{}") - past = time.time() - 120 - os.utime(marker_path, (past, past)) - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=tmp_path, - caller_session_id="test-session", - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - - @pytest.mark.anyio - async def test_stale_not_suppressed_when_no_marker(self, tmp_path): - """No marker_dir means dispatch check is skipped entirely.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - with anyio.fail_after(2.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=None, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - - @pytest.mark.anyio - async def test_stale_marker_suppression_bounded(self, tmp_path, monkeypatch): - """Stale fires after max_suppression_seconds despite perpetually-fresh marker.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - with anyio.fail_after(3.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=None, - _phase1_poll=0.01, - _phase2_poll=0.05, - max_suppression_seconds=0.1, - marker_dir=tmp_path, - caller_session_id=None, - ) - assert result.status == ChannelBStatus.STALE - - @pytest.mark.anyio - async def test_session_scoped_marker_matching(self, tmp_path): - """Marker for sessionA does not suppress stale for sessionB.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - marker_path = tmp_path / "dispatch-in-progress-sessionA-dispatch1.marker" - marker_path.write_text("{}") - with anyio.fail_after(3.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=None, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="sessionB", - ) - assert result.status == ChannelBStatus.STALE - - -class TestDispatchMarkerSuppression: - """Dispatch marker suppression branch via marker_dir/caller_session_id parameters.""" - - @pytest.mark.anyio - async def test_accepts_marker_dir_and_caller_session_id_params(self, tmp_path): - """marker_dir and caller_session_id params are forwarded; monitor exits STALE.""" - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="test-session", - ) - assert result.status == ChannelBStatus.STALE - - @pytest.mark.anyio - async def test_dispatch_marker_suppresses_stale(self, tmp_path, monkeypatch): - """Active dispatch marker suppresses stale kill, then fires when marker disappears.""" - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - call_count = {"n": 0} - - def fake_api_conn(pid): - return False - - def fake_child_proc(pid): - return False - - def fake_dispatch_marker(marker_dir, session_id=None): - call_count["n"] += 1 - return call_count["n"] == 1 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - fake_api_conn, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - fake_child_proc, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - fake_dispatch_marker, - ) - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="sess-abc", - ) - assert result.status == ChannelBStatus.STALE - assert call_count["n"] >= 2 - - @pytest.mark.anyio - async def test_dispatch_marker_bounded_by_max_suppression(self, tmp_path, monkeypatch): - """Stale fires after max_suppression_seconds despite active dispatch marker.""" - import structlog.testing - - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - with structlog.testing.capture_logs() as logs: - with anyio.fail_after(8.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="sess-abc", - max_suppression_seconds=1.0, - ) - assert result.status == ChannelBStatus.STALE - bounded_logs = [log for log in logs if "Suppression bounded" in str(log.get("event", ""))] - assert len(bounded_logs) == 1 - - @pytest.mark.anyio - async def test_dispatch_marker_suppression_emits_warning_with_fields( - self, tmp_path, monkeypatch - ): - """Suppression warning includes elapsed, caller_session_id, and marker_dir.""" - import structlog.testing - - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - call_count = {"n": 0} - - def fake_api_conn(pid): - return False - - def fake_child_proc(pid): - return False - - def fake_dispatch_marker(marker_dir, session_id=None): - call_count["n"] += 1 - return call_count["n"] == 1 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - fake_api_conn, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - fake_child_proc, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - fake_dispatch_marker, - ) - with structlog.testing.capture_logs() as logs: - with anyio.fail_after(5.0): - await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="sess-abc", - ) - warning_logs = [log for log in logs if "dispatch" in str(log.get("event", ""))] - assert len(warning_logs) == 1 - log = warning_logs[0] - assert log.get("caller_session_id") == "sess-abc" - assert log.get("marker_dir") == str(tmp_path) - - @pytest.mark.anyio - async def test_dispatch_marker_bounded_kill_emits_warning_with_fields( - self, tmp_path, monkeypatch - ): - """Bounded kill warning includes elapsed, caller_session_id, marker_dir.""" - import structlog.testing - - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - lambda pid: False, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - lambda marker_dir, session_id=None: True, - ) - with structlog.testing.capture_logs() as logs: - with anyio.fail_after(8.0): - await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - marker_dir=tmp_path, - caller_session_id="sess-abc", - max_suppression_seconds=1.0, - ) - bounded_logs = [log for log in logs if "Suppression bounded" in str(log.get("event", ""))] - assert len(bounded_logs) == 1 - log = bounded_logs[0] - assert log.get("caller_session_id") == "sess-abc" - assert log.get("marker_dir") == str(tmp_path) - - @pytest.mark.anyio - async def test_marker_dir_none_skips_dispatch_check(self, tmp_path, monkeypatch): - """When marker_dir is None (default), _has_active_execution_marker is never called.""" - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - called = {"n": False} - - def fake_api_conn(pid): - return False - - def fake_child_proc(pid): - return False - - def track_dispatch_marker(marker_dir, session_id=None): - called["n"] = True - return False - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - fake_api_conn, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - fake_child_proc, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_execution_marker", - track_dispatch_marker, - ) - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - assert not called["n"] - - @pytest.mark.anyio - async def test_existing_callers_unaffected_by_new_params(self, tmp_path): - """Calling _session_log_monitor without marker_dir/caller_session_id works identically.""" - session_file = tmp_path / "abc123.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - assert result.session_id == "abc123" diff --git a/tests/execution/test_process_session_log_monitor_stale_suppression.py b/tests/execution/test_process_session_log_monitor_stale_suppression.py index 62011b00f2..9baa7cbb5f 100644 --- a/tests/execution/test_process_session_log_monitor_stale_suppression.py +++ b/tests/execution/test_process_session_log_monitor_stale_suppression.py @@ -1,12 +1,13 @@ -"""TCP/CPU stale suppression gate and bounded suppression tests for _session_log_monitor.""" +"""Lease and child-activity stale suppression tests for _session_log_monitor.""" from __future__ import annotations import time from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace import anyio +import psutil import pytest import autoskillit.execution.process._process_monitor as _patch_process__process_monitor @@ -17,146 +18,86 @@ class TestSessionLogMonitorStaleSuppressionGate: - """_session_log_monitor suppresses stale when process has an active port-443 connection.""" + """Staleness is deferred only by operation leases or active child work.""" @pytest.mark.anyio - async def test_suppresses_stale_when_port_443_connection_active(self, tmp_path): + async def test_network_connection_without_lease_does_not_suppress_stale( + self, tmp_path, monkeypatch + ): session_file = tmp_path / "session.jsonl" session_file.write_text("") spawn_time = time.time() - 10 + channel = tmp_path / "leases" + channel.mkdir() - call_count = {"n": 0} + class ConnectedProcess: + def __init__(self, pid: int) -> None: + self.pid = pid - def side_effect(pid): - call_count["n"] += 1 - return call_count["n"] == 1 # True on first call, False on second + def children(self, recursive: bool = False) -> list[object]: + return [] - with patch.object( - _patch_process__process_monitor, - "_has_active_api_connection", - side_effect=side_effect, - ): - with anyio.fail_after(5.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=99999, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - assert result.status == ChannelBStatus.STALE - assert call_count["n"] == 2 + def connections(self, kind: str | None = None) -> list[SimpleNamespace]: + return [ + SimpleNamespace( + status=psutil.CONN_ESTABLISHED, + raddr=SimpleNamespace(port=443), + ) + ] - @pytest.mark.anyio - async def test_fires_stale_immediately_when_no_api_connection(self, tmp_path): - """Standard stale: file silent, no pid provided, stale fires as before.""" - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 + net_connections = connections + + monkeypatch.setattr(_patch_process__process_monitor.psutil, "Process", ConnectedProcess) + monkeypatch.setattr( + _patch_process__process_monitor, "_has_active_child_processes", lambda _pid: False + ) + started = time.monotonic() with anyio.fail_after(2.0): result = await _session_log_monitor( tmp_path, "DONE", stale_threshold=0.05, spawn_time=spawn_time, - # pid omitted (defaults to None) + pid=99999, _phase1_poll=0.01, _phase2_poll=0.05, + max_suppression_seconds=30.0, + operation_lease_dir=channel, ) + elapsed = time.monotonic() - started assert result.status == ChannelBStatus.STALE + assert elapsed < 0.5 @pytest.mark.anyio - async def test_fires_stale_when_pid_is_none_regardless_of_tcp(self, tmp_path): - """pid=None bypasses TCP check entirely — existing behavior preserved.""" + async def test_fires_stale_without_an_operation_lease(self, tmp_path): + """A silent log with no process evidence fires stale.""" session_file = tmp_path / "session.jsonl" session_file.write_text("") spawn_time = time.time() - 10 - - with patch.object( - _patch_process__process_monitor, "_has_active_api_connection" - ) as mock_tcp: - with anyio.fail_after(2.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=None, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) + with anyio.fail_after(2.0): + result = await _session_log_monitor( + tmp_path, + "DONE", + stale_threshold=0.05, + spawn_time=spawn_time, + # pid omitted (defaults to None) + _phase1_poll=0.01, + _phase2_poll=0.05, + ) assert result.status == ChannelBStatus.STALE - mock_tcp.assert_not_called() - - @pytest.mark.anyio - async def test_suppression_emits_warning(self, tmp_path, capsys): - """A suppression event must log a warning with elapsed time.""" - import structlog - - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - - calls = {"n": 0} - - def side_effect(pid): - calls["n"] += 1 - return calls["n"] == 1 - - with patch.object( - _patch_process__process_monitor, - "_has_active_api_connection", - side_effect=side_effect, - ): - with structlog.testing.capture_logs() as logs: - with anyio.fail_after(5.0): - await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=99999, - _phase1_poll=0.01, - _phase2_poll=0.05, - ) - # capture_logs() intercepts when structlog is in default state. - # In a parallel worker where configure_logging() ran in a prior test, - # bound loggers may use a stale processor reference and write to stdout. - _io = capsys.readouterr() - captured = _io.out + _io.err - warning_in_logs = any( - "port-443" in str(log.get("event", "")) or "ESTABLISHED" in str(log.get("event", "")) - for log in logs - ) - warning_in_stdout = "port-443" in captured or "ESTABLISHED" in captured - assert warning_in_logs or warning_in_stdout, ( - "Suppression warning must appear in structlog capture or stdout" - ) @pytest.mark.anyio - async def test_suppresses_stale_when_child_cpu_active_no_api_connection( - self, tmp_path, monkeypatch - ): - """Child CPU activity suppresses stale kill even when no port-443 connection.""" + async def test_suppresses_stale_when_child_cpu_active(self, tmp_path, monkeypatch): + """Child CPU activity continues to suppress stale kill.""" session_file = tmp_path / "session.jsonl" session_file.write_text("") spawn_time = time.time() - 10 # wall time — compared against st_ctime in phase 1 call_count: dict[str, int] = {"cpu": 0} - def fake_api_conn(pid): - return False # No port-443 connection - def fake_child_cpu(pid): call_count["cpu"] += 1 return call_count["cpu"] == 1 # True first, False second - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - fake_api_conn, - ) monkeypatch.setattr( _patch_process__process_monitor, "_has_active_child_processes", @@ -175,21 +116,61 @@ def fake_child_cpu(pid): assert result.status == ChannelBStatus.STALE assert call_count["cpu"] == 2 # suppressed once, then fired + @pytest.mark.anyio + async def test_real_operation_lease_defers_stale_until_lease_ends(self, tmp_path): + import structlog.testing + + from autoskillit.core import InFlightOperations, operation_lease + + session_file = tmp_path / "session.jsonl" + session_file.write_text("") + channel = tmp_path / "leases" + channel.mkdir() + ticks = {"count": 0} + + async with operation_lease( + channel, + operation="run_skill", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + heartbeat_interval=10000, + ) as handle: + + def end_lease_after_three_polls() -> None: + ticks["count"] += 1 + if ticks["count"] == 3 and handle.path is not None: + handle.path.unlink() + + with structlog.testing.capture_logs() as logs: + with anyio.fail_after(2.0): + result = await _session_log_monitor( + tmp_path, + "DONE", + stale_threshold=0.05, + spawn_time=time.time() - 10, + _phase1_poll=0.01, + _phase2_poll=0.01, + _on_poll=end_lease_after_three_polls, + max_suppression_seconds=30.0, + operation_lease_dir=channel, + ) + + assert result.status is ChannelBStatus.STALE + assert any(log.get("event") == "stale_deferred_to_operation" for log in logs) + class TestStaleSuppressionBounded: """Bounded suppression: max_suppression_seconds caps stale deferral.""" @pytest.mark.anyio async def test_stale_suppression_bounded_by_max_duration(self, tmp_path, monkeypatch): - """Stale fires after max_suppression_seconds despite ESTABLISHED connection.""" + """The existing cap still bounds deferral for active child processes.""" session_file = tmp_path / "session.jsonl" session_file.write_text("") spawn_time = time.time() - 10 monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: True, + _patch_process__process_monitor, "_has_active_child_processes", lambda _pid: True ) with anyio.fail_after(8.0): @@ -212,12 +193,6 @@ async def test_stale_suppression_resets_on_genuine_activity(self, tmp_path, monk session_file.write_text("") spawn_time = time.time() - 10 - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: True, - ) - async def write_activity() -> None: import json as _json @@ -256,9 +231,7 @@ async def test_stale_suppression_logs_warning_on_bounded_kill( spawn_time = time.time() - 10 monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - lambda pid: True, + _patch_process__process_monitor, "_has_active_child_processes", lambda _pid: True ) with anyio.fail_after(8.0): @@ -280,124 +253,6 @@ async def test_stale_suppression_logs_warning_on_bounded_kill( bounded_in_stdout = "Suppression bounded" in captured assert bounded_in_logs or bounded_in_stdout - @pytest.mark.anyio - async def test_shared_suppression_timer_prevents_chaining(self, tmp_path, monkeypatch): - """Switching suppression gates (API → marker) does not chain independent timers. - - Keeping the API gate active until the shared deadline is nearly exhausted - makes an incorrectly restarted marker timer add the full suppression window. - """ - session_file = tmp_path / "session.jsonl" - session_file.write_text("") - spawn_time = time.time() - 10 - - call_count = {"n": 0} - - def _api_conn(pid): - call_count["n"] += 1 - return call_count["n"] <= 6 - - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_api_connection", - _api_conn, - ) - monkeypatch.setattr( - _patch_process__process_monitor, - "_has_active_child_processes", - lambda pid: False, - ) - - (tmp_path / "dispatch-in-progress-some-uuid.marker").write_text("{}") - - suppression_start = time.monotonic() - with anyio.fail_after(8.0): - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - pid=9999, - _phase1_poll=0.01, - _phase2_poll=0.05, - max_suppression_seconds=0.3, - marker_dir=tmp_path, - caller_session_id=None, - ) - elapsed = time.monotonic() - suppression_start - - assert result.status == ChannelBStatus.STALE - assert elapsed <= 0.55, f"elapsed {elapsed:.2f}s exceeds 0.55s — timer may have chained" - assert elapsed >= 0.15, ( - f"elapsed {elapsed:.2f}s below 0.15s — suppression may not have fired" - ) - - -class TestExecutionMarkerSuppression: - """Execution marker (run-skill-in-progress-*) suppression for _session_log_monitor. - - Covers the run_skill MCP tool blind spot: markers named run-skill-in-progress-* - must suppress stale kills just like dispatch-in-progress-* markers. - """ - - @pytest.mark.anyio - async def test_execution_marker_suppression_bounded_by_max_suppression_seconds(self, tmp_path): - """Stale fires after max_suppression_seconds despite fresh run-skill marker. - - Failing before implementation: run-skill-in-progress-* marker is invisible to - dispatch-in-progress-* glob, so STALE fires immediately (elapsed << max_suppression). - After fix: marker found, suppression active, fires only after max_suppression_seconds. - """ - session_file = tmp_path / "abc123.jsonl" - session_file.write_text( - '{"type": "assistant", "message": {"role": "assistant", "content": "working"}}\n' - ) - spawn_time = time.time() - 1 - - marker_dir = tmp_path / "marker_dir" - marker_dir.mkdir() - marker_path = marker_dir / "run-skill-in-progress-caller-session-step1.marker" - marker_path.write_text("{}") - - max_suppression = 0.3 - - async def touch_marker() -> None: - for _ in range(200): - await anyio.sleep(0.02) - try: - marker_path.touch() - except OSError: - break - - start = time.monotonic() - with anyio.fail_after(5.0): - async with anyio.create_task_group() as tg: - tg.start_soon(touch_marker) - result = await _session_log_monitor( - tmp_path, - "DONE", - stale_threshold=0.05, - spawn_time=spawn_time, - marker_dir=marker_dir, - caller_session_id="caller-session", - _phase1_poll=0.01, - _phase2_poll=0.05, - max_suppression_seconds=max_suppression, - ) - tg.cancel_scope.cancel() - - elapsed = time.monotonic() - start - assert result.status == ChannelBStatus.STALE - assert elapsed >= max_suppression, ( - f"STALE fired after {elapsed:.3f}s, expected >= {max_suppression}s. " - "run-skill execution marker suppression not working — " - "marker may not be matched by the glob pattern." - ) - assert elapsed < 5.0, ( - f"STALE fired after {elapsed:.3f}s — expected prompt firing after " - f"max_suppression={max_suppression}s, not near the 5s timeout ceiling." - ) - class TestExecutionMarkerLifecycle: @pytest.mark.anyio diff --git a/tests/execution/test_process_submodules.py b/tests/execution/test_process_submodules.py index 78e5b25917..95cafe6211 100644 --- a/tests/execution/test_process_submodules.py +++ b/tests/execution/test_process_submodules.py @@ -45,9 +45,7 @@ "_resolve_session_id", "RaceAccumulator", "RaceSignals", - "_has_active_api_connection", "_has_active_child_processes", - "_has_active_execution_marker", "_heartbeat", "_jsonl_contains_marker", "_jsonl_has_record_type", @@ -160,11 +158,8 @@ def test_process_io_exports(): def test_process_monitor_exports(): - """_heartbeat, _session_log_monitor, and _has_active_api_connection - are defined in _process_monitor submodule.""" + """_heartbeat and _session_log_monitor are defined in _process_monitor submodule.""" from autoskillit.execution.process._process_monitor import ( - _has_active_api_connection, - _has_active_execution_marker, _heartbeat, _session_log_monitor, ) @@ -173,14 +168,6 @@ def test_process_monitor_exports(): assert _heartbeat.__module__ == "autoskillit.execution.process._process_monitor" assert callable(_session_log_monitor) assert _session_log_monitor.__module__ == "autoskillit.execution.process._process_monitor" - assert callable(_has_active_api_connection) - assert ( - _has_active_api_connection.__module__ == "autoskillit.execution.process._process_monitor" - ) - assert callable(_has_active_execution_marker) - assert ( - _has_active_execution_marker.__module__ == "autoskillit.execution.process._process_monitor" - ) def test_process_race_exports(): @@ -338,8 +325,6 @@ def start_soon(self, func, *args) -> None: session_id_timeout=1.0, stdout_session_id_ready=stdout_session_id_ready, max_suppression_seconds=None, - marker_dir=None, - session_id=None, on_session_id_resolved=None, backend_resume_session_id="", channel_b_selected=channel_b_selected, diff --git a/tests/execution/test_recording.py b/tests/execution/test_recording.py index b122769fa2..b9a00887ad 100644 --- a/tests/execution/test_recording.py +++ b/tests/execution/test_recording.py @@ -737,37 +737,6 @@ def test_replaying_runner_restore_missing_step_returns_none(tmp_path): assert result is None -# --- T-MARKER-FWD: RecordingSubprocessRunner forwards marker_dir and session_id to inner --- - - -@pytest.mark.anyio -async def test_recording_runner_forwards_marker_dir_and_session_id(tmp_path): - """Non-session branch passes marker_dir and session_id to self._inner().""" - mock_recorder = Mock() - inner = MockSubprocessRunner() - inner.set_default(_make_result(returncode=0)) - runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) - - marker = tmp_path / "markers" - cmd = ["pytest", "tests/"] - env = {"SCENARIO_STEP_NAME": "test-check"} - - await runner( - cmd, - cwd=tmp_path, - timeout=60, - env=env, - pty_mode=False, - marker_dir=marker, - session_id="sess-abc", - ) - - assert len(inner.call_args_list) == 1 - kwargs = inner.call_args_list[0][3] - assert kwargs["marker_dir"] == marker - assert kwargs["session_id"] == "sess-abc" - - @pytest.mark.anyio async def test_recording_runner_forwards_operation_lease_dir(tmp_path): mock_recorder = Mock() @@ -782,8 +751,6 @@ async def test_recording_runner_forwards_operation_lease_dir(tmp_path): timeout=60, env={"SCENARIO_STEP_NAME": "test-check"}, pty_mode=False, - marker_dir=tmp_path / "markers", - session_id="sess-abc", operation_lease_dir=channel, ) @@ -852,62 +819,6 @@ async def test_replay_runner_ignores_pass_fds_without_acquiring_them(tmp_path): assert result.stdout == "replayed" -# --- T-MARKER-SESSION-BRANCH: session branch does not forward marker_dir/session_id --- - - -@pytest.mark.anyio -async def test_recording_runner_session_branch_ignores_marker_params(tmp_path): - """pty_mode=True + step_name → _record_session; marker_dir/session_id not forwarded.""" - mock_recorder = Mock() - mock_recorder.record_step.return_value = FakeStepResult( - cassette_exit_code=0, - cassette_path=str(tmp_path / "cassette"), - cassette_duration_ms=5000, - ) - inner = MockSubprocessRunner() - runner = RecordingSubprocessRunner(recorder=mock_recorder, inner=inner) - - cmd = ["claude", "--model", "sonnet", "--print", "do stuff"] - env = {"AUTOSKILLIT_HEADLESS": "1", "SCENARIO_STEP_NAME": "investigate"} - - result = await runner( - cmd, - cwd=tmp_path, - timeout=300, - env=env, - pty_mode=True, - marker_dir=tmp_path / "markers", - session_id="sess-xyz", - ) - - assert inner.call_args_list == [] # inner NOT called - assert result.returncode == 0 - - -# --- T-MARKER-REPLAY-ACCEPTS: ReplayingSubprocessRunner accepts marker params --- - - -@pytest.mark.anyio -async def test_replaying_runner_accepts_marker_params(tmp_path): - """ReplayingSubprocessRunner.__call__ accepts marker_dir and session_id without error.""" - non_session = {"check": {"exit_code": 0, "stdout_head": "ok", "stderr": ""}} - runner = ReplayingSubprocessRunner({}, non_session) - - cmd = ["task", "test-check"] - env = {"SCENARIO_STEP_NAME": "check"} - result = await runner( - cmd, - cwd=tmp_path, - timeout=60, - env=env, - marker_dir=tmp_path / "markers", - session_id="sess-123", - ) - - assert result.returncode == 0 - assert result.stdout == "ok" - - @pytest.mark.anyio async def test_replaying_runner_accepts_operation_lease_dir(tmp_path): non_session = {"check": {"exit_code": 0, "stdout_head": "ok", "stderr": ""}} @@ -918,8 +829,6 @@ async def test_replaying_runner_accepts_operation_lease_dir(tmp_path): cwd=tmp_path, timeout=60, env={"SCENARIO_STEP_NAME": "check"}, - marker_dir=tmp_path / "markers", - session_id="sess-123", operation_lease_dir=tmp_path / "operation-leases", ) diff --git a/tests/execution/test_termination_executor.py b/tests/execution/test_termination_executor.py index cc51a12050..c5af88fff8 100644 --- a/tests/execution/test_termination_executor.py +++ b/tests/execution/test_termination_executor.py @@ -4,6 +4,7 @@ import sys from pathlib import Path +from types import SimpleNamespace from typing import cast import anyio @@ -150,12 +151,7 @@ async def test_active_child_deferral_runs_until_ceiling( clock = _install_deferral_clock(monkeypatch) liveness_checks = 0 - def has_active_signals( - _pid: int, - _marker_dir: Path | None, - _session_id: str | None, - operation_lease_dir: Path | None = None, - ) -> frozenset[str]: + def has_active_signals(_pid: int, *, operation_lease_dir: Path | None) -> frozenset[str]: nonlocal liveness_checks liveness_checks += 1 return frozenset({"child_processes"}) @@ -181,6 +177,53 @@ def has_active_signals( assert clock == [4.0] +@pytest.mark.anyio +async def test_network_connection_without_lease_does_not_defer_termination( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import psutil + + import autoskillit.execution.process._process_monitor as process_monitor + + clock = _install_deferral_clock(monkeypatch) + + class ConnectedProcess: + def __init__(self, pid: int) -> None: + self.pid = pid + + def children(self, recursive: bool = False) -> list[object]: + return [] + + def connections(self, kind: str | None = None) -> list[SimpleNamespace]: + return [ + SimpleNamespace( + status=psutil.CONN_ESTABLISHED, + raddr=SimpleNamespace(port=443), + ) + ] + + net_connections = connections + + monkeypatch.setattr(process_monitor.psutil, "Process", ConnectedProcess) + monkeypatch.setattr(process_monitor, "_has_active_child_processes", lambda _pid: False) + owner = await _spawn(30, tmp_path) + + kill_reason, _returncode, cleanup = await execute_termination_action( + TerminationAction.DRAIN_THEN_KILL_IF_ALIVE, + owner=owner, + process_exited_event=anyio.Event(), + grace_seconds=0, + proc_log=structlog.get_logger().bind(pid=owner.pid), + pid=owner.pid, + child_deferral_ceiling=30.0, + operation_lease_dir=tmp_path / "leases", + ) + + assert kill_reason is KillReason.KILL_AFTER_COMPLETION + assert cleanup.complete is True + assert clock == [0.0] + + @pytest.mark.anyio async def test_zero_child_deferral_ceiling_skips_liveness_check( monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -188,10 +231,7 @@ async def test_zero_child_deferral_ceiling_skips_liveness_check( owner = await _spawn(30, tmp_path) def unexpected_liveness_check( - _pid: int, - _marker_dir: Path | None, - _session_id: str | None, - operation_lease_dir: Path | None = None, + _pid: int, *, operation_lease_dir: Path | None ) -> frozenset[str]: pytest.fail("zero child deferral ceiling must skip liveness checks") @@ -225,7 +265,7 @@ async def test_child_deferral_stops_when_children_become_inactive( monkeypatch.setattr( _patch_process__termination, "_active_liveness_signals", - lambda _pid, _marker_dir, _session_id, operation_lease_dir=None: next(activity), + lambda _pid, *, operation_lease_dir: next(activity), ) kill_reason, _returncode, cleanup = await execute_termination_action( diff --git a/tests/fakes.py b/tests/fakes.py index 8f284012e0..d354a690ca 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -747,8 +747,6 @@ class DispatchFoodTruckCall: sentinel_contract: str = "" profile_name: str = "" prior_completion_markers: Sequence[str] | None = None - marker_dir: Path | None = None - session_id: str | None = None resume_message: str | None = None on_session_id_resolved: Callable[[str], None] | None = None backend_authority: BackendAuthority | None = None @@ -940,8 +938,6 @@ async def dispatch_food_truck( profile_name: str = "", sentinel_contract: str = "", prior_completion_markers: Sequence[str] | None = None, - marker_dir: Path | None = None, - session_id: str | None = None, resume_message: str | None = None, on_session_id_resolved: Callable[[str], None] | None = None, backend_authority: BackendAuthority | None = None, @@ -978,8 +974,6 @@ async def dispatch_food_truck( sentinel_contract=sentinel_contract, profile_name=profile_name, prior_completion_markers=prior_completion_markers, - marker_dir=marker_dir, - session_id=session_id, resume_message=resume_message, on_session_id_resolved=on_session_id_resolved, backend_authority=backend_authority, diff --git a/tests/fleet/test_api.py b/tests/fleet/test_api.py index c79d2df98c..5354b7a372 100644 --- a/tests/fleet/test_api.py +++ b/tests/fleet/test_api.py @@ -21,7 +21,7 @@ _write_pid, write_initial_state, ) -from tests.fleet._helpers import _mock_backend_with_locator, _setup_dispatch +from tests.fleet._helpers import _setup_dispatch pytestmark = [pytest.mark.layer("fleet"), pytest.mark.small, pytest.mark.feature("fleet")] @@ -523,140 +523,17 @@ async def test_touch_marker_oserror_does_not_propagate(self, tmp_path: Path) -> marker_file.chmod(0o644) -class TestDispatchMarkerLifecycle: - """Tests for marker lifecycle integration in _run_dispatch.""" +@pytest.mark.anyio +async def test_run_dispatch_continues_when_project_log_dir_unavailable( + tool_ctx, monkeypatch +) -> None: + """Dispatch completes when the backend has no session-log locator.""" + _setup_dispatch(tool_ctx, monkeypatch) + tool_ctx.backend = None - @pytest.mark.anyio - async def test_run_dispatch_forwards_marker_dir_and_session_id_to_executor( - self, tool_ctx, monkeypatch, tmp_path: Path - ) -> None: - """dispatch_food_truck is called with marker_dir and session_id kwargs.""" - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - await _run(tool_ctx) - - assert len(tool_ctx.executor.dispatch_calls) == 1 - call = tool_ctx.executor.dispatch_calls[0] - assert call.marker_dir == marker_dir - assert call.session_id is not None - - @pytest.mark.anyio - async def test_run_dispatch_continues_when_marker_dir_unavailable( - self, tool_ctx, monkeypatch - ) -> None: - """execute_dispatch succeeds even when backend's session_locator is unavailable.""" - _setup_dispatch(tool_ctx, monkeypatch) - tool_ctx.backend = None - - result = await _run(tool_ctx) - - # Should complete without raising (returns DispatchCompleted or DispatchRejected) - assert result.get("success") is not None - - @pytest.mark.anyio - async def test_run_dispatch_marker_cleaned_up_after_success( - self, tool_ctx, monkeypatch, tmp_path: Path - ) -> None: - """No .marker files remain after successful dispatch.""" - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - await _run(tool_ctx) - - remaining = list(marker_dir.glob("*.marker")) - assert len(remaining) == 0, f"Expected no marker files, found {remaining}" - - @pytest.mark.anyio - async def test_run_dispatch_marker_cleaned_up_after_failure( - self, tool_ctx, monkeypatch, tmp_path: Path - ) -> None: - """No .marker files remain after dispatch raises (caught as DispatchRejected).""" - from functools import wraps - - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - original_dispatch_food_truck = tool_ctx.executor.dispatch_food_truck - - @wraps(original_dispatch_food_truck) - async def _raise_dispatch(*args, **kwargs): - await original_dispatch_food_truck(*args, **kwargs) - raise RuntimeError("dispatch failed") - - tool_ctx.executor.dispatch_food_truck = _raise_dispatch - - result = await _run(tool_ctx) - - # execute_dispatch catches Exception and returns DispatchRejected - assert result.get("success") is False, "dispatch should be rejected on failure" - remaining = list(marker_dir.glob("*.marker")) - assert len(remaining) == 0, f"Expected no marker files after failure, found {remaining}" - - @pytest.mark.anyio - async def test_run_dispatch_creates_marker_file( - self, tool_ctx, monkeypatch, tmp_path: Path - ) -> None: - """Marker file is created during dispatch execution.""" - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - marker_seen: list[bool] = [] - original_dispatch = tool_ctx.executor.dispatch_food_truck - - async def _capture_marker(*args, **kwargs): - markers_found = list(marker_dir.glob("*.marker")) - marker_seen.append(len(markers_found) > 0) - return await original_dispatch(*args, **kwargs) - - tool_ctx.executor.dispatch_food_truck = _capture_marker - - await _run(tool_ctx) - - assert len(marker_seen) == 1, "dispatch_food_truck should be called once" - assert marker_seen[0] is True, "marker file should exist during dispatch execution" - - @pytest.mark.anyio - async def test_run_dispatch_marker_contains_expected_json( - self, tool_ctx, monkeypatch, tmp_path: Path - ) -> None: - """Marker file JSON contains dispatch_id, orchestrator_pid, and session_id.""" - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - captured_json: list[dict] = [] - - original_dispatch = tool_ctx.executor.dispatch_food_truck - - async def _capture_marker(*args, **kwargs): - markers = list(marker_dir.glob("*.marker")) - if markers: - content = markers[0].read_text() - captured_json.append(json.loads(content)) - return await original_dispatch(*args, **kwargs) - - tool_ctx.executor.dispatch_food_truck = _capture_marker - - await _run(tool_ctx) + result = await _run(tool_ctx) - assert len(captured_json) == 1, "should have captured marker JSON" - data = captured_json[0] - assert "label" in data - assert "orchestrator_pid" in data - assert "session_id" in data - assert isinstance(data["orchestrator_pid"], int) - assert isinstance(data["session_id"], str) - assert data["label"] == "dispatch" + assert result.get("success") is not None class TestWriteDispatchToCampaignStateRefusal: diff --git a/tests/fleet/test_api_dispatch_marker.py b/tests/fleet/test_api_dispatch_marker.py deleted file mode 100644 index b90d3976e9..0000000000 --- a/tests/fleet/test_api_dispatch_marker.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Tests for _run_dispatch marker lifecycle via execution_marker context manager.""" - -from __future__ import annotations - -import asyncio -import time -from pathlib import Path - -import anyio -import pytest - -from autoskillit.core import SkillResult -from autoskillit.core._execution_marker import _touch_marker -from autoskillit.core.types.foundation._type_enums import RetryReason -from autoskillit.fleet._api import _run_dispatch -from tests.fleet._helpers import ( - _mock_backend_with_locator, - _noop_quota_refresher, - _setup_dispatch, -) - -pytestmark = [pytest.mark.layer("fleet"), pytest.mark.medium, pytest.mark.feature("fleet")] - - -@pytest.mark.anyio -async def test_marker_created_before_dispatch(tool_ctx, monkeypatch, tmp_path: Path) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "marker_dir" - marker_dir.mkdir(parents=True, exist_ok=True) - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - async def _asserting_dispatch(**_kw): - found = list(marker_dir.glob("*-in-progress-*.marker")) - assert len(found) == 1, f"Expected 1 marker, found {len(found)}" - raise asyncio.CancelledError - - monkeypatch.setattr(tool_ctx.executor, "dispatch_food_truck", _asserting_dispatch) - - with pytest.raises(asyncio.CancelledError): - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - -@pytest.mark.anyio -async def test_marker_deleted_after_success(tool_ctx, monkeypatch, tmp_path: Path) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "claude" - marker_dir.mkdir(parents=True, exist_ok=True) - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - remaining = list(marker_dir.glob("*-in-progress-*.marker")) - assert remaining == [], f"Expected no marker files, found {remaining}" - - -@pytest.mark.anyio -async def test_marker_deleted_after_exception(tool_ctx, monkeypatch, tmp_path: Path) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "claude" - marker_dir.mkdir(parents=True, exist_ok=True) - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) - - async def _raise_runtime_error(**_kw): - raise RuntimeError("boom") - - monkeypatch.setattr(tool_ctx.executor, "dispatch_food_truck", _raise_runtime_error) - - try: - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - except* RuntimeError: - pass - - remaining = list(marker_dir.glob("*-in-progress-*.marker")) - assert remaining == [], f"Expected no marker files after exception, found {remaining}" - - -@pytest.mark.anyio -async def test_heartbeat_refreshes_mtime(tmp_path: Path) -> None: - import os - - marker_path = tmp_path / "dispatch-in-progress-test.marker" - marker_path.write_text("{}") - # Pin baseline mtime to the epoch so filesystem timestamp granularity - # (1s on some mounts, jiffies on others) cannot cause the heartbeat's - # refresh to compare against an mtime that is fractionally newer than - # the post-write stat(). - os.utime(marker_path, (0, 0)) - initial_mtime_ns = marker_path.stat().st_mtime_ns - - async with anyio.create_task_group() as tg: - tg.start_soon(_touch_marker, marker_path, 0.05) - await anyio.sleep(1.0) - tg.cancel_scope.cancel() - - final_mtime_ns = marker_path.stat().st_mtime_ns - assert final_mtime_ns > initial_mtime_ns, "heartbeat must advance marker mtime" - - -@pytest.mark.anyio -async def test_marker_not_written_when_cwd_unavailable( - tool_ctx, monkeypatch, tmp_path: Path -) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - - tool_ctx.backend = None - - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - assert len(tool_ctx.executor.dispatch_calls) == 1 - assert list(tmp_path.rglob("*.marker")) == [] - - -@pytest.mark.anyio -async def test_run_dispatch_writes_heartbeat_file(tool_ctx, monkeypatch) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - dispatches_dir = tool_ctx.temp_dir / "dispatches" - - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - remaining = list(dispatches_dir.glob("dispatch-*.heartbeat")) - assert remaining == [], ( - f"Heartbeat file should be deleted after normal completion: {remaining}" - ) - - -@pytest.mark.anyio -async def test_run_dispatch_heartbeat_exists_during_execution(tool_ctx, monkeypatch) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - dispatches_dir = tool_ctx.temp_dir / "dispatches" - heartbeat_files_during: list[list[Path]] = [] - - async def _asserting_dispatch(**_kw): - found = list(dispatches_dir.glob("dispatch-*.heartbeat")) - heartbeat_files_during.append(found) - raise asyncio.CancelledError - - monkeypatch.setattr(tool_ctx.executor, "dispatch_food_truck", _asserting_dispatch) - - with pytest.raises(asyncio.CancelledError): - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - assert len(heartbeat_files_during) == 1 - assert len(heartbeat_files_during[0]) == 1, ( - f"Expected 1 heartbeat file during dispatch, found {heartbeat_files_during[0]}" - ) - - remaining = list(dispatches_dir.glob("dispatch-*.heartbeat")) - assert remaining == [], f"Heartbeat file should be deleted after CancelledError: {remaining}" - - -@pytest.mark.anyio -async def test_run_dispatch_heartbeat_mtime_is_fresh(tool_ctx, monkeypatch) -> None: - _setup_dispatch(tool_ctx, monkeypatch) - dispatches_dir = tool_ctx.temp_dir / "dispatches" - recorded_mtime: list[float] = [] - - async def _record_mtime_dispatch(**_kw): - found = list(dispatches_dir.glob("dispatch-*.heartbeat")) - if found: - recorded_mtime.append(found[0].stat().st_mtime) - return SkillResult( - success=True, - result="ok", - session_id="", - subtype="success", - is_error=False, - exit_code=0, - needs_retry=False, - retry_reason=RetryReason.NONE, - stderr="", - ) - - monkeypatch.setattr(tool_ctx.executor, "dispatch_food_truck", _record_mtime_dispatch) - - before = time.time() - await _run_dispatch( - tool_ctx=tool_ctx, - recipe="test-recipe", - task="t", - ingredients=None, - dispatch_name=None, - timeout_sec=None, - prompt_builder=lambda **kw: "prompt", - quota_refresher=_noop_quota_refresher, - ) - - assert len(recorded_mtime) == 1 - assert recorded_mtime[0] >= before - 1.0, "Heartbeat mtime should be recent at dispatch start" diff --git a/tests/fleet/test_dispatch_backend_override.py b/tests/fleet/test_dispatch_backend_override.py index f857af7336..84e3f3b0f0 100644 --- a/tests/fleet/test_dispatch_backend_override.py +++ b/tests/fleet/test_dispatch_backend_override.py @@ -4,6 +4,7 @@ import dataclasses from pathlib import Path +from typing import Any from unittest.mock import Mock import pytest @@ -364,10 +365,27 @@ async def test_dispatch_backend_override_session_locator_uses_dispatch_backend( dispatch_be = _mock_backend("codex") dispatch_be.session_locator.return_value = dispatch_locator + from autoskillit.fleet.dispatch import _api + + original_classify = _api.run_outcome_classification + classifications: list[Any] = [] + + async def capture_classification(**kwargs: Any) -> Any: + result = await original_classify(**kwargs) + classifications.append(result) + return result + + monkeypatch.setattr(_api, "run_outcome_classification", capture_classification) + await _run_with_backend(tool_ctx, dispatch_backend=dispatch_be) record = _read_dispatch_record(tool_ctx) - assert "codex-logs" in record["dispatched_session_log_dir"] + assert classifications[0].project_log_dir == str( + dispatch_locator.project_log_dir.return_value + ) + assert record["dispatched_session_log_dir"] == str( + dispatch_locator.project_log_dir.return_value + ) class TestDispatchBackendOverridePreservedAcrossRetry: diff --git a/tests/fleet/test_fleet_e2e.py b/tests/fleet/test_fleet_e2e.py index 6a4b3a43e1..8913e9c7ce 100644 --- a/tests/fleet/test_fleet_e2e.py +++ b/tests/fleet/test_fleet_e2e.py @@ -158,6 +158,8 @@ def __init__(self) -> None: self.last_pass_fds: tuple[int, ...] = () self.last_kwargs: dict[str, Any] = {} self.operation_lease_dir_exists_during_call = False + self.project_log_dir_to_check: Path | None = None + self.marker_files_during_call: tuple[Path, ...] = () async def __call__( self, @@ -184,6 +186,11 @@ async def __call__( self.call_count += 1 self.last_pass_fds = pass_fds self.last_kwargs = {**kwargs, "env": env, "cwd": cwd, "timeout": timeout} + self.marker_files_during_call = ( + tuple(sorted(self.project_log_dir_to_check.glob("*-in-progress-*.marker"))) + if self.project_log_dir_to_check is not None + else () + ) operation_lease_dir = kwargs.get("operation_lease_dir") self.operation_lease_dir_exists_during_call = ( isinstance(operation_lease_dir, Path) and operation_lease_dir.is_dir() @@ -477,6 +484,10 @@ async def test_food_truck_lease_channel_and_l2_idle_floor_on_claude( runtime.add_recipe("lease-floor-recipe") runtime.tool_ctx.config.run_skill.timeout = 3600 runtime.tool_ctx.config.fleet.idle_output_timeout = 1800 + project_log_dir = runtime.tool_ctx.backend.session_locator().project_log_dir( + str(runtime.tool_ctx.project_dir) + ) + runtime.runner.project_log_dir_to_check = project_log_dir channels: list[Path] = [] for dispatch_name, override, expected_idle in ( @@ -491,6 +502,9 @@ async def test_food_truck_lease_channel_and_l2_idle_floor_on_claude( assert result["success"] is True call = runtime.runner.last_kwargs + assert "marker_dir" not in call + assert "session_id" not in call + assert runtime.runner.marker_files_during_call == () channel = call["operation_lease_dir"] assert isinstance(channel, Path) and channel.is_absolute() assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) diff --git a/tests/fleet/test_fleet_e2e_codex.py b/tests/fleet/test_fleet_e2e_codex.py index 10a11b0660..b99e58c6cb 100644 --- a/tests/fleet/test_fleet_e2e_codex.py +++ b/tests/fleet/test_fleet_e2e_codex.py @@ -116,6 +116,9 @@ async def test_food_truck_lease_channel_and_l2_idle_floor( ctx = codex_runtime["tool_ctx"] recipes = codex_runtime["recipes"] runner: FleetTestRunner = codex_runtime["runner"] + runner.project_log_dir_to_check = ctx.backend.session_locator().project_log_dir( + str(ctx.project_dir) + ) _add_recipe(recipes, "lease-floor-recipe") ctx.config.run_skill.timeout = 3600 ctx.config.fleet.idle_output_timeout = 1800 @@ -138,6 +141,9 @@ async def test_food_truck_lease_channel_and_l2_idle_floor( ) call = runner.last_kwargs + assert "marker_dir" not in call + assert "session_id" not in call + assert runner.marker_files_during_call == () channel = call["operation_lease_dir"] assert isinstance(channel, Path) and channel.is_absolute() assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) diff --git a/tests/fleet/test_native_shell_capture_lineage.py b/tests/fleet/test_native_shell_capture_lineage.py index ef8fb788ab..9599a0c217 100644 --- a/tests/fleet/test_native_shell_capture_lineage.py +++ b/tests/fleet/test_native_shell_capture_lineage.py @@ -115,9 +115,9 @@ async def test_fresh_direct_dispatch_creates_and_persists_food_truck_lineage( self, tool_ctx, monkeypatch, tmp_path: Path ) -> None: _setup_dispatch(tool_ctx, monkeypatch) - marker_dir = tmp_path / "markers" - marker_dir.mkdir() - tool_ctx.backend = _mock_backend_with_locator(project_log_dir=marker_dir) + project_log_dir = tmp_path / "project-logs" + project_log_dir.mkdir() + tool_ctx.backend = _mock_backend_with_locator(project_log_dir=project_log_dir) result = await _execute(tool_ctx, mode=NativeShellCaptureMode.DIRECT) From dbc9565bf70ecb4ad68444473a82ecef5e11a3cf Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 02:39:35 -0700 Subject: [PATCH 13/37] test: align recorder and attestation checks with retired watchdog inputs --- ...test_headless_managed_attempt_recording.py | 12 ++++---- tests/execution/test_termination_executor.py | 28 +++++++------------ tests/server/test_tools_execution_command.py | 4 +-- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/tests/execution/test_headless_managed_attempt_recording.py b/tests/execution/test_headless_managed_attempt_recording.py index 13261544d4..e0ba263216 100644 --- a/tests/execution/test_headless_managed_attempt_recording.py +++ b/tests/execution/test_headless_managed_attempt_recording.py @@ -211,7 +211,7 @@ async def test_ordinary_session_records_nothing(minimal_ctx, tmp_path, monkeypat minimal_ctx, timeout=30.0, stale_threshold=5.0, - session_id="parent-ordinary", + caller_session_id="parent-ordinary", launch_resolver=minimal_ctx.launch_resolver, launch_preparation=_launch_preparation(minimal_ctx, cwd=str(tmp_path)), ) @@ -251,7 +251,7 @@ def build_spec(binding, provider_extras, managed_attempt_id): stale_threshold=5.0, provider_name="minimax", managed_lineage_observer=lineage_observer, - session_id="parent-two-attempts", + caller_session_id="parent-two-attempts", child_role="Explore", child_attribution_skill="do-a", launch_resolver=minimal_ctx.launch_resolver, @@ -297,7 +297,7 @@ async def test_post_start_budget_exhaustion_produces_one_terminal_row( timeout=30.0, stale_threshold=5.0, provider_name="minimax", - session_id="parent-no-observer", + caller_session_id="parent-no-observer", child_role="Explore", child_attribution_skill="do-a", launch_resolver=minimal_ctx.launch_resolver, @@ -338,7 +338,7 @@ async def test_cancellation_before_spawn_records_nothing( minimal_ctx, timeout=30.0, stale_threshold=5.0, - session_id="parent-cancel-before-spawn", + caller_session_id="parent-cancel-before-spawn", child_role="Explore", child_attribution_skill="do-a", launch_resolver=minimal_ctx.launch_resolver, @@ -373,7 +373,7 @@ async def test_cancellation_after_spawn_records_one_interrupted_row( minimal_ctx, timeout=30.0, stale_threshold=5.0, - session_id="parent-cancel-after-spawn", + caller_session_id="parent-cancel-after-spawn", child_role="Explore", child_attribution_skill="do-a", launch_resolver=minimal_ctx.launch_resolver, @@ -410,7 +410,7 @@ async def test_binding_launch_alias_merges_into_same_row( minimal_ctx, timeout=30.0, stale_threshold=5.0, - session_id="parent-bind-alias", + caller_session_id="parent-bind-alias", child_role="Explore", child_attribution_skill="do-a", launch_resolver=minimal_ctx.launch_resolver, diff --git a/tests/execution/test_termination_executor.py b/tests/execution/test_termination_executor.py index c5af88fff8..8eb7d64ba7 100644 --- a/tests/execution/test_termination_executor.py +++ b/tests/execution/test_termination_executor.py @@ -187,24 +187,16 @@ async def test_network_connection_without_lease_does_not_defer_termination( clock = _install_deferral_clock(monkeypatch) - class ConnectedProcess: - def __init__(self, pid: int) -> None: - self.pid = pid - - def children(self, recursive: bool = False) -> list[object]: - return [] - - def connections(self, kind: str | None = None) -> list[SimpleNamespace]: - return [ - SimpleNamespace( - status=psutil.CONN_ESTABLISHED, - raddr=SimpleNamespace(port=443), - ) - ] - - net_connections = connections - - monkeypatch.setattr(process_monitor.psutil, "Process", ConnectedProcess) + def connections(_process, kind: str | None = None) -> list[SimpleNamespace]: + return [ + SimpleNamespace( + status=psutil.CONN_ESTABLISHED, + raddr=SimpleNamespace(port=443), + ) + ] + + monkeypatch.setattr(process_monitor.psutil.Process, "connections", connections) + monkeypatch.setattr(process_monitor.psutil.Process, "net_connections", connections) monkeypatch.setattr(process_monitor, "_has_active_child_processes", lambda _pid: False) owner = await _spawn(30, tmp_path) diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py index 9940caca9f..21ff194f7f 100644 --- a/tests/server/test_tools_execution_command.py +++ b/tests/server/test_tools_execution_command.py @@ -502,8 +502,8 @@ async def check_marker_during_launch(cmd, **kwargs): call for call in runner.call_args_list if Path(call[0][0]).name == "claude" ] assert len(claude_calls) == 1 - assert claude_calls[0][3]["marker_dir"] is None - assert claude_calls[0][3]["session_id"] is None + assert "marker_dir" not in claude_calls[0][3] + assert "session_id" not in claude_calls[0][3] @pytest.mark.anyio async def test_marker_dir_routes_through_session_locator( From 8df73a92aacd94a92242f98de1fd9877c62ea6d3 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 02:56:47 -0700 Subject: [PATCH 14/37] test: decouple native receipt identity check from wall-clock expiry --- .../test_exploration_request_identity_guard.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/hooks/test_exploration_request_identity_guard.py b/tests/hooks/test_exploration_request_identity_guard.py index f422688899..c284cd9274 100644 --- a/tests/hooks/test_exploration_request_identity_guard.py +++ b/tests/hooks/test_exploration_request_identity_guard.py @@ -9,6 +9,7 @@ import pytest +import autoskillit.hooks._runtime._exploration_request_record as records from autoskillit.hooks._runtime._exploration_request_record import ( SUPPORTED_EXPLORATION_REQUEST_TOOLS, consume_exploration_request_record, @@ -90,7 +91,7 @@ def _run( ], ) def test_guard_preserves_input_and_injects_consumable_native_identity( - tmp_path: Path, runtime_name: str, short_name: str + tmp_path: Path, runtime_name: str, short_name: str, monkeypatch: pytest.MonkeyPatch ) -> None: root = _project(tmp_path) result = _run( @@ -115,6 +116,16 @@ def test_guard_preserves_input_and_injects_consumable_native_identity( assert output["updatedInput"]["existing"] == "value" token = output["updatedInput"]["_autoskillit_exploration_request_token"] assert token != "model-value" + record_path = ( + root + / ".autoskillit" + / "temp" + / "exploration-requests" + / f"exploration-request-{token}.json" + ) + created_at = json.loads(record_path.read_text(encoding="utf-8"))["created_at"] + # TTL expiry is covered separately; scheduling delays must not alter this identity check. + monkeypatch.setattr(records, "_clock", lambda: created_at) assert consume_exploration_request_record(root, short_name, token) == "native-session" assert consume_exploration_request_record(root, short_name, token) is None From d866fdf20d62a18b4f0986e0a45a1a6b3bf0ceb4 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 03:36:32 -0700 Subject: [PATCH 15/37] fix: preserve Codex item errors in session diagnostics --- docs/developer/diagnostics.md | 4 +- src/autoskillit/core/__init__.pyi | 1 + .../core/types/foundation/_type_enums.py | 1 + .../core/types/results/_type_results.py | 16 ++++ .../types/results/_type_results_records.py | 3 +- .../backends/_codex/app_server_events.py | 12 +++ .../execution/backends/_codex_parse.py | 15 +++- .../headless/_headless_adjudication.py | 2 + .../execution/headless/_headless_evidence.py | 1 + .../execution/headless/_headless_result.py | 2 + .../execution/headless/_headless_terminal.py | 1 + .../execution/session_log/session_log.py | 3 + src/autoskillit/server/tools/_types.py | 1 + .../contracts/test_run_skill_result_parity.py | 24 ++++++ .../test_skill_result_persistence_ledger.py | 2 + .../test_session_index_schema_version_lock.py | 2 +- .../fixtures/codex_ndjson/item_error.json | 21 ++++++ .../test_codex_deterministic_conformance.py | 3 + .../test_codex_item_dispatch_parity.py | 73 +++++++++++++++++++ .../backends/test_codex_stream_parser.py | 2 +- .../test_session_log_backend_source.py | 1 + .../test_dispatch_outcome_codex_backend.py | 47 +++++++++--- 22 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 tests/execution/backends/fixtures/codex_ndjson/item_error.json create mode 100644 tests/execution/backends/test_codex_item_dispatch_parity.py diff --git a/docs/developer/diagnostics.md b/docs/developer/diagnostics.md index 5aaa80267f..015117784e 100644 --- a/docs/developer/diagnostics.md +++ b/docs/developer/diagnostics.md @@ -65,7 +65,7 @@ Logs are stored in a **global** directory (not per-project), so they persist acr ### Session Summary Fields -`summary.json` contains: `session_id`, `dir_name`, `pid`, `cwd`, `evidence_worktree_path`, `evidence_worktree_source`, `evidence_worktree_detail`, `skill_command`, `success`, `subtype`, `exit_code`, `start_ts`, `snapshot_count`, `anomaly_count`, `peak_rss_kb`, `peak_oom_score`, `peak_fd_ratio`, `session_type`, `child_outcomes` (see [Child Terminal Reasons](#child-terminal-reasons)). +`summary.json` contains: `session_id`, `dir_name`, `pid`, `cwd`, `evidence_worktree_path`, `evidence_worktree_source`, `evidence_worktree_detail`, `skill_command`, `success`, `subtype`, `exit_code`, `start_ts`, `snapshot_count`, `anomaly_count`, `peak_rss_kb`, `peak_oom_score`, `peak_fd_ratio`, `session_type`, `session_errors`, `child_outcomes` (see [Child Terminal Reasons](#child-terminal-reasons)). ### Execution-candidate manifests @@ -178,7 +178,7 @@ Completed sessions with per-request evidence publish `turn_usage.jsonl` before version-4 token descriptor carries `turn_usage_file`, `turn_usage_count`, and `turn_usage_schema_version`. The file reference is `null` with count zero when no series was observed. The JSONL row schema is version 2; the retained -`sessions.jsonl` index uses schema version 15. +`sessions.jsonl` index uses schema version 16. Each token-bearing row carries non-empty `backend` and `provider_used`, plus nullable source `message_id`, `request_id`, `timestamp`, and observed `model`. diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index 0749eaeff4..25ccba53eb 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -1277,6 +1277,7 @@ from .types import SerializedTokenMeasure as SerializedTokenMeasure from .types import ServeOverridesSnapshot as ServeOverridesSnapshot from .types import SessionAttemptHandle as SessionAttemptHandle from .types import SessionCheckpoint as SessionCheckpoint +from .types import SessionErrorOutcome as SessionErrorOutcome from .types import SessionEvent as SessionEvent from .types import SessionHookRoot as SessionHookRoot from .types import SessionInvariantAdaptationRefusal as SessionInvariantAdaptationRefusal diff --git a/src/autoskillit/core/types/foundation/_type_enums.py b/src/autoskillit/core/types/foundation/_type_enums.py index ac927048cc..6dd06485c3 100644 --- a/src/autoskillit/core/types/foundation/_type_enums.py +++ b/src/autoskillit/core/types/foundation/_type_enums.py @@ -721,6 +721,7 @@ class CodexItemType(StrEnum): AGENT_MESSAGE = "agent_message" COMMAND_EXECUTION = "command_execution" FILE_CHANGE = "file_change" + ERROR = "error" MCP_TOOL_CALL = "mcp_tool_call" COLLAB_TOOL_CALL = "collab_tool_call" WEB_SEARCH = "web_search" diff --git a/src/autoskillit/core/types/results/_type_results.py b/src/autoskillit/core/types/results/_type_results.py index 3987dfd55b..aa4bf37df5 100644 --- a/src/autoskillit/core/types/results/_type_results.py +++ b/src/autoskillit/core/types/results/_type_results.py @@ -8,6 +8,7 @@ import json import re +from collections.abc import Sequence from dataclasses import dataclass, field from datetime import UTC, datetime from enum import Enum @@ -115,6 +116,7 @@ "ApiRetryOutcome", "CandidatePreSpawnRejection", "NdjsonDriftOutcome", + "SessionErrorOutcome", "SkillResult", "CleanupResult", "CloneSuccessResult", @@ -452,6 +454,17 @@ class NdjsonDriftOutcome: unknown_item_count: int = 0 +@dataclass(frozen=True, slots=True) +class SessionErrorOutcome: + """Bounded session error messages retained for results and diagnostics.""" + + messages: tuple[str, ...] = () + + @classmethod + def from_errors(cls, errors: Sequence[str]) -> SessionErrorOutcome: + return cls(messages=tuple(message[:500] for message in errors[:5])) + + @dataclass(frozen=True, slots=True) class AuditResultOutcome: """Server-authored audit outcome attached to a skill result.""" @@ -540,6 +553,8 @@ class SkillResult: """Pre-contamination context bundle — populated only when clone_guard fires.""" ndjson_drift: NdjsonDriftOutcome = field(default_factory=NdjsonDriftOutcome) """NDJSON parser vocabulary drift counters — populated by Codex sessions.""" + session_error: SessionErrorOutcome = field(default_factory=SessionErrorOutcome) + """Bounded error messages parsed from the session's output stream.""" audit: AuditResultOutcome = field(default_factory=AuditResultOutcome) """Server-authored audit outcome bundle.""" completion_required: bool = False @@ -608,6 +623,7 @@ def to_json(self) -> str: "pre_contamination_subtype": self.contamination.subtype, "ndjson_unknown_event_count": self.ndjson_drift.unknown_event_count, "ndjson_unknown_item_count": self.ndjson_drift.unknown_item_count, + "session_errors": list(self.session_error.messages), "audit_status": self.audit.status.value if self.audit.status is not None else None, "audit_verdict": ( self.audit.verdict.value if self.audit.verdict is not None else None diff --git a/src/autoskillit/core/types/results/_type_results_records.py b/src/autoskillit/core/types/results/_type_results_records.py index 8fc7cf1972..ec8f007a90 100644 --- a/src/autoskillit/core/types/results/_type_results_records.py +++ b/src/autoskillit/core/types/results/_type_results_records.py @@ -51,7 +51,7 @@ T = TypeVar("T") -SESSION_INDEX_SCHEMA_VERSION: Final[int] = 15 +SESSION_INDEX_SCHEMA_VERSION: Final[int] = 16 TOKEN_USAGE_SCHEMA_VERSION: Final[int] = 4 TURN_USAGE_SCHEMA_VERSION: Final[int] = 2 @@ -440,6 +440,7 @@ class SessionIndexEntry(TypedDict): api_retry_last_status: int | None ndjson_unknown_event_count: int ndjson_unknown_item_count: int + session_errors: list[str] outcome_fields: dict[str, int | str] | None outcome_invariant_violated: bool outcome_qualifier: str | None diff --git a/src/autoskillit/execution/backends/_codex/app_server_events.py b/src/autoskillit/execution/backends/_codex/app_server_events.py index 6de03d27c9..873293b766 100644 --- a/src/autoskillit/execution/backends/_codex/app_server_events.py +++ b/src/autoskillit/execution/backends/_codex/app_server_events.py @@ -383,6 +383,18 @@ def _parse_completed_item(self, obj: dict[str, Any]) -> SessionEvent: for block in item.get("content", []): if isinstance(block, dict) and block.get("type") == "text": self._check_marker_text(block.get("text", "")) + elif item_type == CodexItemType.ERROR: + return SessionEvent( + kind=BackendEventKind.IGNORED, + is_terminal=False, + has_marker=False, + backend_data=CodexEventData( + record_type="item.completed", + thread_id="", + item_type=CodexItemType.ERROR.value, + raw=obj, + ), + ) elif item_type not in ( CodexItemType.FILE_CHANGE, CodexItemType.COMMAND_EXECUTION, diff --git a/src/autoskillit/execution/backends/_codex_parse.py b/src/autoskillit/execution/backends/_codex_parse.py index 4f03de1b80..df7e12a8ad 100644 --- a/src/autoskillit/execution/backends/_codex_parse.py +++ b/src/autoskillit/execution/backends/_codex_parse.py @@ -399,6 +399,7 @@ def _preserves_rollout_prefix(prior: Path, candidate: Path) -> bool: class _CodexParseAccumulator: session_id: str = "" agent_messages: list[str] = field(default_factory=list) + item_error_messages: list[str] = field(default_factory=list) command_executions: list[dict[str, Any]] = field(default_factory=list) mcp_tool_calls: list[dict[str, Any]] = field(default_factory=list) file_changes: list[str] = field(default_factory=list) @@ -457,16 +458,23 @@ def _accumulate_codex_completed_item( item_type = CodexItemType.from_ndjson(item.get("type", "")) if item_type in (CodexItemType.AGENT_MESSAGE, CodexItemType.MESSAGE): _accumulate_codex_message_item(acc, item, item_type) - elif item_type in (CodexItemType.COMMAND_EXECUTION, CodexItemType.FUNCTION_CALL): + elif item_type in ( + CodexItemType.COMMAND_EXECUTION, + CodexItemType.FUNCTION_CALL, + CodexItemType.COLLAB_TOOL_CALL, + CodexItemType.WEB_SEARCH, + ): acc.command_executions.append(item) elif item_type == CodexItemType.MCP_TOOL_CALL: acc.mcp_tool_calls.append(item) elif item_type == CodexItemType.FILE_CHANGE: _accumulate_codex_file_changes(acc, item) - elif item_type in (CodexItemType.COLLAB_TOOL_CALL, CodexItemType.WEB_SEARCH): - acc.command_executions.append(item) elif item_type in (CodexItemType.REASONING, CodexItemType.TODO_LIST): logger.debug("codex_ndjson_informational_item", item_type=item_type.value) + elif item_type == CodexItemType.ERROR: + message = item.get("message") + if isinstance(message, str) and message: + acc.item_error_messages.append(message) elif item_type == CodexItemType.UNKNOWN: logger.warning("codex_ndjson_unknown_item_type", item_type=item.get("type", "")) acc.ndjson_unknown_item_count += 1 @@ -638,6 +646,7 @@ def parse_stdout(self, stdout: str, *, exit_code: int = 0) -> AgentSessionResult "canonical_token_usage": canonical_dict, "cumulative_token_usage": acc.cumulative_token_usage, "agent_messages": acc.agent_messages, + "item_error_messages": acc.item_error_messages, "command_executions": acc.command_executions, "mcp_tool_calls": acc.mcp_tool_calls, "file_changes": acc.file_changes, diff --git a/src/autoskillit/execution/headless/_headless_adjudication.py b/src/autoskillit/execution/headless/_headless_adjudication.py index 07af1c99e1..91e178fc75 100644 --- a/src/autoskillit/execution/headless/_headless_adjudication.py +++ b/src/autoskillit/execution/headless/_headless_adjudication.py @@ -24,6 +24,7 @@ ProviderOutcome, RateLimitWindow, RetryReason, + SessionErrorOutcome, SkillResult, TerminationReason, WriteBehaviorSpec, @@ -242,6 +243,7 @@ def _make_terminated_result( unknown_event_count=session.seen_ndjson_unknown_event_count, unknown_item_count=session.seen_ndjson_unknown_item_count, ), + session_error=SessionErrorOutcome.from_errors(session.errors), ) diff --git a/src/autoskillit/execution/headless/_headless_evidence.py b/src/autoskillit/execution/headless/_headless_evidence.py index 8537de2117..1aa2fe040b 100644 --- a/src/autoskillit/execution/headless/_headless_evidence.py +++ b/src/autoskillit/execution/headless/_headless_evidence.py @@ -107,6 +107,7 @@ def _adapt_agent_result(agent_result: AgentSessionResult) -> ClaudeSessionResult or CODEX_CONTEXT_EXHAUSTION_MARKER in (agent_result.error or "") ) errors: list[str] = [agent_result.error] if agent_result.error else [] + errors.extend(raw.get("item_error_messages", [])) api_error_status: int | None = _CODEX_ERROR_CODE_API_STATUS.get(error_code) token_usage = raw.get("canonical_token_usage") or raw.get("token_usage") diff --git a/src/autoskillit/execution/headless/_headless_result.py b/src/autoskillit/execution/headless/_headless_result.py index 3f86aa6e40..62aafdddde 100644 --- a/src/autoskillit/execution/headless/_headless_result.py +++ b/src/autoskillit/execution/headless/_headless_result.py @@ -18,6 +18,7 @@ NdjsonDriftOutcome, ProviderOutcome, RetryReason, + SessionErrorOutcome, SessionOutcome, SkillResult, TerminationReason, @@ -767,6 +768,7 @@ def _build_skill_result( unknown_event_count=session.seen_ndjson_unknown_event_count, unknown_item_count=session.seen_ndjson_unknown_item_count, ), + session_error=SessionErrorOutcome.from_errors(session.errors), completion_required=completion_required, ) if is_path_contamination and not controlled_context_exhaustion: diff --git a/src/autoskillit/execution/headless/_headless_terminal.py b/src/autoskillit/execution/headless/_headless_terminal.py index 2edb362e25..49e0c42a0f 100644 --- a/src/autoskillit/execution/headless/_headless_terminal.py +++ b/src/autoskillit/execution/headless/_headless_terminal.py @@ -326,6 +326,7 @@ def build_terminal_flush_kwargs( "success": skill_result.success, "needs_retry": skill_result.needs_retry, "retry_reason": skill_result.retry_reason.value, + "session_errors": skill_result.session_error.messages, "infra": skill_result.infra, "api_error_status": skill_result.api_failure.status, "is_error": skill_result.is_error, diff --git a/src/autoskillit/execution/session_log/session_log.py b/src/autoskillit/execution/session_log/session_log.py index 9e8d88d70b..0652034458 100644 --- a/src/autoskillit/execution/session_log/session_log.py +++ b/src/autoskillit/execution/session_log/session_log.py @@ -187,6 +187,7 @@ def flush_session_log( api_retry_exhausted: bool = False, ndjson_unknown_event_count: int = 0, ndjson_unknown_item_count: int = 0, + session_errors: tuple[str, ...] = (), versions: dict[str, Any] | None = None, model_identity: ModelIdentity = ModelIdentity.unknown(), max_sessions: int | None = None, @@ -479,6 +480,7 @@ def flush_session_log( "api_retry_exhausted": api_retry_exhausted, "ndjson_unknown_event_count": ndjson_unknown_event_count, "ndjson_unknown_item_count": ndjson_unknown_item_count, + "session_errors": list(session_errors), "native_shell_capture": native_shell_capture_projection, "session_type": session_type_value, "child_outcomes": list(telemetry.child_outcomes), @@ -663,6 +665,7 @@ def flush_session_log( "api_retry_last_status": api_retry_last_status, "ndjson_unknown_event_count": ndjson_unknown_event_count, "ndjson_unknown_item_count": ndjson_unknown_item_count, + "session_errors": list(session_errors), "outcome_fields": outcome_fields, "outcome_invariant_violated": outcome_invariant_violated, "outcome_qualifier": outcome_qualifier, diff --git a/src/autoskillit/server/tools/_types.py b/src/autoskillit/server/tools/_types.py index 7180295813..9ecd12e2fc 100644 --- a/src/autoskillit/server/tools/_types.py +++ b/src/autoskillit/server/tools/_types.py @@ -138,6 +138,7 @@ class RunSkillResult(_RunSkillResultBase, total=False): pre_contamination_subtype: str ndjson_unknown_event_count: int ndjson_unknown_item_count: int + session_errors: list[str] execution_identity: ExecutionIdentityDict pipeline_tracker: dict[str, str] receipt_id: str diff --git a/tests/contracts/test_run_skill_result_parity.py b/tests/contracts/test_run_skill_result_parity.py index 045df685d7..2a24462d28 100644 --- a/tests/contracts/test_run_skill_result_parity.py +++ b/tests/contracts/test_run_skill_result_parity.py @@ -53,6 +53,7 @@ "infra": "flattened into infra_* keys", "ndjson_drift": "flattened into ndjson_* keys", "provider": "flattened into provider_* keys", + "session_error": "flattened into session_errors", "turn_usage": "per-turn telemetry is not part of the run_skill envelope", } @@ -103,6 +104,7 @@ "provider_fallback": True, "provider_used": "anthropic", }, + "session_error": {"session_errors": []}, } @@ -182,6 +184,28 @@ def test_run_skill_result_covers_skill_result_projection_bidirectionally() -> No assert hints["outcome_fields"] == dict[str, int | str] | None assert hints["outcome_invariant_violated"] is bool assert hints["outcome_qualifier"] == str | None + assert hints["session_errors"] == list[str] + + +def test_session_error_bundle_is_flattened_into_the_wire_envelope() -> None: + from autoskillit.core import SessionErrorOutcome + + result = replace( + _populated_skill_result(), + session_error=SessionErrorOutcome(messages=("item error",)), + ) + + assert json.loads(result.to_json())["session_errors"] == ["item error"] + + +def test_session_error_outcome_keeps_five_truncated_messages() -> None: + from autoskillit.core import SessionErrorOutcome + + errors = [f"message-{index}:" + "x" * 600 for index in range(7)] + + outcome = SessionErrorOutcome.from_errors(errors) + + assert outcome.messages == tuple(message[:500] for message in errors[:5]) @pytest.mark.parametrize( diff --git a/tests/contracts/test_skill_result_persistence_ledger.py b/tests/contracts/test_skill_result_persistence_ledger.py index e0d8226820..7dfd255051 100644 --- a/tests/contracts/test_skill_result_persistence_ledger.py +++ b/tests/contracts/test_skill_result_persistence_ledger.py @@ -124,6 +124,7 @@ ("provider.provider_used", "provider_used", "persisted"), ("result", "result", "live-only:size"), ("retry_reason", "retry_reason", "persisted"), + ("session_error.messages", "session_errors", "persisted"), ("session_id", "session_id", "persisted"), ("stderr", "stderr", "live-only:size"), ("subtype", "subtype", "persisted"), @@ -272,6 +273,7 @@ def test_durable_ledger_rows_exist_in_real_flushed_artifacts(tmp_path) -> None: "provider_fallback": False, "provider_used": "", "session_id": "test-session-001", + "session_errors": [], "subtype": "completed", "write_path_warnings": [], } diff --git a/tests/core/test_session_index_schema_version_lock.py b/tests/core/test_session_index_schema_version_lock.py index e33ab69ff9..edd97e7001 100644 --- a/tests/core/test_session_index_schema_version_lock.py +++ b/tests/core/test_session_index_schema_version_lock.py @@ -12,7 +12,7 @@ pytestmark = [pytest.mark.layer("core"), pytest.mark.small] -_EXPECTED = (15, "55d211412c8cc43d5827af08172102fc743961ba13cbc0e94dcfa6677848d343") +_EXPECTED = (16, "9584465d5e3ca791680c05a0698482e23561ef03533c209a40380d8506580eb3") def test_session_index_schema_version_matches_field_digest() -> None: diff --git a/tests/execution/backends/fixtures/codex_ndjson/item_error.json b/tests/execution/backends/fixtures/codex_ndjson/item_error.json new file mode 100644 index 0000000000..aaaa02b8f7 --- /dev/null +++ b/tests/execution/backends/fixtures/codex_ndjson/item_error.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "message": { + "type": "string" + }, + "type": { + "const": "error", + "type": "string" + } + }, + "required": [ + "type", + "message" + ], + "type": "object" +} diff --git a/tests/execution/backends/test_codex_deterministic_conformance.py b/tests/execution/backends/test_codex_deterministic_conformance.py index 7c36c77378..1f60adbc2a 100644 --- a/tests/execution/backends/test_codex_deterministic_conformance.py +++ b/tests/execution/backends/test_codex_deterministic_conformance.py @@ -128,6 +128,9 @@ def test_sealed_enumeration(self) -> None: class TestCodexItemTypeVocabulary: + def test_error_item_type_is_recognized(self) -> None: + assert CodexItemType.from_ndjson("error") is CodexItemType.ERROR + @pytest.mark.parametrize( "member", _NON_UNKNOWN_ITEM_TYPES, diff --git a/tests/execution/backends/test_codex_item_dispatch_parity.py b/tests/execution/backends/test_codex_item_dispatch_parity.py new file mode 100644 index 0000000000..df58f553d8 --- /dev/null +++ b/tests/execution/backends/test_codex_item_dispatch_parity.py @@ -0,0 +1,73 @@ +"""Keep the live and post-hoc Codex item dispatchers aligned.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from jsonschema import validate + +from autoskillit.core import BackendEventKind, CodexEventData +from autoskillit.core.types.foundation._type_enums import CodexItemType +from autoskillit.execution.backends._codex_parse import ( + CodexStreamParser, + _accumulate_codex_completed_item, + _CodexParseAccumulator, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "codex_ndjson" +_ITEM_EXAMPLES: dict[str, dict[str, object]] = { + "agent_message": {"type": "agent_message", "text": "message"}, + "command_execution": {"type": "command_execution"}, + "file_change": {"type": "file_change"}, + "mcp_tool_call": {"type": "mcp_tool_call"}, + "collab_tool_call": {"type": "collab_tool_call"}, + "web_search": {"type": "web_search"}, + "message": {"type": "message", "content": [{"type": "text", "text": "message"}]}, + "function_call": {"type": "function_call"}, + "reasoning": {"type": "reasoning"}, + "todo_list": {"type": "todo_list"}, + "error": {"type": "error", "message": "item error"}, +} + + +@pytest.mark.parametrize( + "member", + [member for member in CodexItemType if member != CodexItemType.UNKNOWN], + ids=lambda member: member.value, +) +def test_completed_item_dispatchers_recognize_each_sealed_type(member: CodexItemType) -> None: + item = _ITEM_EXAMPLES[member.value] + schema = json.loads((_FIXTURES_DIR / f"item_{member.value}.json").read_text(encoding="utf-8")) + validate(item, schema) + event = {"type": "item.completed", "item": item} + + accumulator = _CodexParseAccumulator() + _accumulate_codex_completed_item(accumulator, event) + assert accumulator.ndjson_unknown_item_count == 0 + + live_parser = CodexStreamParser() + live_event = live_parser.parse_line(json.dumps(event)) + assert live_parser.ndjson_unknown_item_count == 0 + assert live_event is not None + + if member.value == "error": + assert member is CodexItemType.ERROR + assert accumulator.item_error_messages == ["item error"] + assert live_event.kind is BackendEventKind.IGNORED + assert isinstance(live_event.backend_data, CodexEventData) + assert live_event.backend_data.raw["item"]["message"] == "item error" + + +def test_error_item_messages_ignore_empty_and_non_string_values() -> None: + accumulator = _CodexParseAccumulator() + for message in ("", None, 7): + _accumulate_codex_completed_item( + accumulator, + {"type": "item.completed", "item": {"type": "error", "message": message}}, + ) + + assert accumulator.item_error_messages == [] diff --git a/tests/execution/backends/test_codex_stream_parser.py b/tests/execution/backends/test_codex_stream_parser.py index dd7c7df420..bba33383ba 100644 --- a/tests/execution/backends/test_codex_stream_parser.py +++ b/tests/execution/backends/test_codex_stream_parser.py @@ -501,7 +501,7 @@ class TestCodexParserParity: def test_both_parsers_handle_same_item_types(self) -> None: for member in CodexItemType: - if member in self._SKIP: + if member in self._SKIP or member.value == "error": continue item: dict = {"type": member.value} if member == CodexItemType.AGENT_MESSAGE: diff --git a/tests/execution/test_session_log_backend_source.py b/tests/execution/test_session_log_backend_source.py index 53a6f1714e..1660484855 100644 --- a/tests/execution/test_session_log_backend_source.py +++ b/tests/execution/test_session_log_backend_source.py @@ -104,6 +104,7 @@ def test_session_index_entry_default_value(self) -> None: "api_retry_last_status": None, "ndjson_unknown_event_count": 0, "ndjson_unknown_item_count": 0, + "session_errors": [], "outcome_fields": None, "outcome_invariant_violated": False, "outcome_qualifier": None, diff --git a/tests/fleet/test_dispatch_outcome_codex_backend.py b/tests/fleet/test_dispatch_outcome_codex_backend.py index ce3129232c..979b96b9fb 100644 --- a/tests/fleet/test_dispatch_outcome_codex_backend.py +++ b/tests/fleet/test_dispatch_outcome_codex_backend.py @@ -2,10 +2,9 @@ Exercises the real Codex NDJSON parse path (``_scan_codex_ndjson`` -> ``_adapt_agent_result``) with an incident-shaped stream -- item-level ``error`` -items (which ``CodexItemType`` has no member for, so they fall through to -``codex_ndjson_unknown_item_type``) plus a turn-level failure and no final -agent message -- and verifies the resulting ``SkillResult`` classifies the -same way through ``classify_dispatch_outcome`` whether called directly or +items plus a turn-level failure and no final agent message. It verifies the +resulting ``SkillResult`` classifies the same way through +``classify_dispatch_outcome`` whether called directly or through the production ``execute_dispatch`` -> ``run_outcome_classification`` call site. """ @@ -26,6 +25,7 @@ import pytest import structlog.testing +import autoskillit.execution.headless._headless_adjudication as headless_adjudication import autoskillit.fleet.dispatch._api as dispatch_api from autoskillit.core import ( DefaultManagedWorkerCapacity, @@ -60,10 +60,8 @@ _RECIPE_NAME = "test-codex-incident-recipe" -# Item-level ``error`` items have no CodexItemType member, so they fall through -# to UNKNOWN (codex_ndjson_unknown_item_type). The command_execution item is -# real tool-use evidence -- the same shape production uses to populate -# SkillResult.lifespan_started -- not a substitute for it. No turn.completed +# The command_execution item is real tool-use evidence of the same shape +# production uses to populate SkillResult.lifespan_started. No turn.completed # ever follows, so the accumulator's success flag stays False. _CODEX_INCIDENT_SHIM = '''\ #!/usr/bin/env python3 @@ -380,12 +378,22 @@ async def test_codex_backend_skill_result_through_classify_dispatch_outcome( _seed_tracker(tool_ctx, dispatch_id) captured: dict[str, Any] = {} + captured_sessions: list[Any] = [] + real_adapt_agent_result = headless_adjudication._adapt_agent_result real_run_outcome_classification = dispatch_api.run_outcome_classification + def _capturing_adapt_agent_result(agent_result: Any) -> Any: + session = real_adapt_agent_result(agent_result) + captured_sessions.append(session) + return session + async def _capturing_run_outcome_classification(**kwargs: Any) -> Any: captured["skill_result"] = kwargs["skill_result"] return await real_run_outcome_classification(**kwargs) + monkeypatch.setattr( + headless_adjudication, "_adapt_agent_result", _capturing_adapt_agent_result + ) monkeypatch.setattr( dispatch_api, "run_outcome_classification", _capturing_run_outcome_classification ) @@ -408,10 +416,31 @@ async def _capturing_run_outcome_classification(**kwargs: Any) -> Any: assert skill_result is not None, "run_outcome_classification was never invoked" assert skill_result.session_id, "codex shim did not surface a session id" assert skill_result.lifespan_started, "command_execution evidence did not register" - assert any( + assert not any( log["event"] == "codex_ndjson_unknown_item_type" and log["log_level"] == "warning" for log in cap_logs ) + assert skill_result.session_error.messages == ("tool call failed", "second tool failure") + assert captured_sessions + adapted_session = captured_sessions[-1] + if case.expect_error_subtype: + assert adapted_session.errors[0].startswith("turn failed hard") + assert adapted_session.errors[1:] == ["tool call failed", "second tool failure"] + else: + assert adapted_session.errors == ["tool call failed", "second tool failure"] + + log_root = Path(tool_ctx.config.linux_tracing.log_dir) + summary = json.loads( + (log_root / "sessions" / skill_result.session_id / "summary.json").read_text() + ) + index_rows = [ + json.loads(line) + for line in (log_root / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + index_row = next(row for row in index_rows if row["session_id"] == skill_result.session_id) + assert summary["session_errors"] == ["tool call failed", "second tool failure"] + assert index_row["session_errors"] == ["tool call failed", "second tool failure"] if case.expect_error_subtype: assert skill_result.cli_subtype == "error_during_execution" else: From 63829695bf1f134e69df305c20730610f09277e8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 03:52:43 -0700 Subject: [PATCH 16/37] test: cover session errors in result schemas --- .../hooks/formatters/_fmt_execution.py | 1 + tests/execution/test_headless_core.py | 1 + tests/execution/test_session_parsing.py | 1 + .../test_dispatch_outcome_codex_backend.py | 21 ++++++++++++------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/autoskillit/hooks/formatters/_fmt_execution.py b/src/autoskillit/hooks/formatters/_fmt_execution.py index f458b5126a..0c7a86bb81 100644 --- a/src/autoskillit/hooks/formatters/_fmt_execution.py +++ b/src/autoskillit/hooks/formatters/_fmt_execution.py @@ -251,6 +251,7 @@ def _fmt_test_check(data: dict, _pipeline: bool) -> str: "completion_required", "ndjson_unknown_event_count", "ndjson_unknown_item_count", + "session_errors", "execution_identity", "outcome_invariant_violated", "outcome_qualifier", diff --git a/tests/execution/test_headless_core.py b/tests/execution/test_headless_core.py index a82ef1db03..1518b69a39 100644 --- a/tests/execution/test_headless_core.py +++ b/tests/execution/test_headless_core.py @@ -1373,6 +1373,7 @@ class TestBuildSkillResultCrossValidation: "pre_contamination_subtype", "ndjson_unknown_event_count", "ndjson_unknown_item_count", + "session_errors", "execution_identity", } diff --git a/tests/execution/test_session_parsing.py b/tests/execution/test_session_parsing.py index 5c1a50f208..8a97617b32 100644 --- a/tests/execution/test_session_parsing.py +++ b/tests/execution/test_session_parsing.py @@ -785,6 +785,7 @@ def test_to_json_produces_all_required_keys(self): "pre_contamination_subtype", "ndjson_unknown_event_count", "ndjson_unknown_item_count", + "session_errors", "execution_identity", } assert set(parsed.keys()) == expected diff --git a/tests/fleet/test_dispatch_outcome_codex_backend.py b/tests/fleet/test_dispatch_outcome_codex_backend.py index 979b96b9fb..ec81b815c1 100644 --- a/tests/fleet/test_dispatch_outcome_codex_backend.py +++ b/tests/fleet/test_dispatch_outcome_codex_backend.py @@ -420,14 +420,19 @@ async def _capturing_run_outcome_classification(**kwargs: Any) -> Any: log["event"] == "codex_ndjson_unknown_item_type" and log["log_level"] == "warning" for log in cap_logs ) - assert skill_result.session_error.messages == ("tool call failed", "second tool failure") + expected_errors = ( + ( + "turn failed hard [E_TURN_FAILED]", + "tool call failed", + "second tool failure", + ) + if case.expect_error_subtype + else ("tool call failed", "second tool failure") + ) + assert skill_result.session_error.messages == expected_errors assert captured_sessions adapted_session = captured_sessions[-1] - if case.expect_error_subtype: - assert adapted_session.errors[0].startswith("turn failed hard") - assert adapted_session.errors[1:] == ["tool call failed", "second tool failure"] - else: - assert adapted_session.errors == ["tool call failed", "second tool failure"] + assert adapted_session.errors == list(expected_errors) log_root = Path(tool_ctx.config.linux_tracing.log_dir) summary = json.loads( @@ -439,8 +444,8 @@ async def _capturing_run_outcome_classification(**kwargs: Any) -> Any: if line.strip() ] index_row = next(row for row in index_rows if row["session_id"] == skill_result.session_id) - assert summary["session_errors"] == ["tool call failed", "second tool failure"] - assert index_row["session_errors"] == ["tool call failed", "second tool failure"] + assert summary["session_errors"] == list(expected_errors) + assert index_row["session_errors"] == list(expected_errors) if case.expect_error_subtype: assert skill_result.cli_subtype == "error_during_execution" else: From e75ec00015576004444770e5d3deda0efb22a4f1 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 04:06:09 -0700 Subject: [PATCH 17/37] style: keep formatter coverage within its fixed line budget --- src/autoskillit/hooks/formatters/_fmt_execution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/autoskillit/hooks/formatters/_fmt_execution.py b/src/autoskillit/hooks/formatters/_fmt_execution.py index 0c7a86bb81..1a302cf418 100644 --- a/src/autoskillit/hooks/formatters/_fmt_execution.py +++ b/src/autoskillit/hooks/formatters/_fmt_execution.py @@ -282,7 +282,6 @@ def _fmt_test_check(data: dict, _pipeline: bool) -> str: } ) _FMT_RUN_CMD_SUPPRESSED: frozenset[str] = frozenset() - _FMT_TEST_CHECK_RENDERED: frozenset[str] = frozenset( { "passed", From 22a3810a2eacc75a11ebabe7ebf96cf80300202e Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:06:01 -0700 Subject: [PATCH 18/37] fix(review): retry transient operation lease heartbeat failures --- .../core/plugins/_operation_lease.py | 2 ++ tests/core/test_operation_lease.py | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/autoskillit/core/plugins/_operation_lease.py b/src/autoskillit/core/plugins/_operation_lease.py index ec895b0590..45b016f509 100644 --- a/src/autoskillit/core/plugins/_operation_lease.py +++ b/src/autoskillit/core/plugins/_operation_lease.py @@ -185,6 +185,8 @@ async def _heartbeat(handle: OperationLeaseHandle, interval: float) -> None: if handle.path == path: handle.path = None return + except OSError: + logger.warning("operation_lease_heartbeat_failed", path=str(path), exc_info=True) @asynccontextmanager diff --git a/tests/core/test_operation_lease.py b/tests/core/test_operation_lease.py index 40a3504f2c..0b2ce16247 100644 --- a/tests/core/test_operation_lease.py +++ b/tests/core/test_operation_lease.py @@ -120,6 +120,40 @@ async def test_heartbeat_restores_freshness_while_body_is_active(tmp_path: Path) assert registry.active_count == 0 +@pytest.mark.anyio +async def test_heartbeat_recovers_after_transient_io_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + recovered = anyio.Event() + original_utime = os.utime + attempts = 0 + + def flaky_utime(path, times) -> None: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError("temporary heartbeat failure") + original_utime(path, times) + recovered.set() + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + heartbeat_interval=0.001, + ) as handle: + assert handle.path is not None + old_mtime = time.time() - OPERATION_LEASE_FRESHNESS_SECONDS - 1 + original_utime(handle.path, (old_mtime, old_mtime)) + monkeypatch.setattr(lease_module.os, "utime", flaky_utime) + with anyio.fail_after(1): + await recovered.wait() + assert read_active_operation_leases(tmp_path, now_epoch=time.time()) + + assert not tuple(tmp_path.glob("*.lease.json")) + + @pytest.mark.anyio async def test_heartbeat_never_recreates_a_missing_lease_file(tmp_path: Path) -> None: registry = InFlightOperations() From 20e31cb599fa746c621c907ccacbdaae04be7460 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:07:38 -0700 Subject: [PATCH 19/37] refactor(review): prepend operation lease middleware directly --- src/autoskillit/server/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/autoskillit/server/__init__.py b/src/autoskillit/server/__init__.py index 7670857208..4358e304f6 100644 --- a/src/autoskillit/server/__init__.py +++ b/src/autoskillit/server/__init__.py @@ -187,10 +187,7 @@ RunSkillCompletionMiddleware, ) -_lease_middleware = _OperationLeaseMiddleware() -mcp.add_middleware(_lease_middleware) -mcp.middleware.remove(_lease_middleware) -mcp.middleware.insert(0, _lease_middleware) +mcp.middleware.insert(0, _OperationLeaseMiddleware()) mcp.add_middleware(RunSkillCompletionMiddleware(mcp)) from autoskillit.server._wire_compat import ClaudeCodeCompatMiddleware # noqa: E402 From e523168401d3162afedde7846c2d51fcefbb89b3 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:12:13 -0700 Subject: [PATCH 20/37] fix(review): sanitize operation lease deadline write failures --- .../core/plugins/_operation_lease.py | 4 +-- tests/core/test_operation_lease.py | 2 +- tests/server/test_tools_execution_command.py | 36 +++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/core/plugins/_operation_lease.py b/src/autoskillit/core/plugins/_operation_lease.py index 45b016f509..db2c72d5c4 100644 --- a/src/autoskillit/core/plugins/_operation_lease.py +++ b/src/autoskillit/core/plugins/_operation_lease.py @@ -157,14 +157,14 @@ def narrow(self, not_after_epoch: float) -> None: return try: write_versioned_json(path, self.record.to_payload(), schema_version=_SCHEMA_VERSION) - except OSError: + except OSError as exc: self.path = None logger.warning("operation_lease_narrow_write_failed", path=str(path), exc_info=True) try: path.unlink(missing_ok=True) except OSError: logger.warning("operation_lease_unlink_failed", path=str(path), exc_info=True) - raise + raise OSError("Operation lease deadline could not be persisted") from exc def current_operation_lease() -> OperationLeaseHandle | None: diff --git a/tests/core/test_operation_lease.py b/tests/core/test_operation_lease.py index 0b2ce16247..87e1488c36 100644 --- a/tests/core/test_operation_lease.py +++ b/tests/core/test_operation_lease.py @@ -395,7 +395,7 @@ def fail_write(*args: object, **kwargs: object) -> None: raise OSError("rewrite failed") monkeypatch.setattr(lease_module, "write_versioned_json", fail_write) - with pytest.raises(OSError, match="rewrite failed"): + with pytest.raises(OSError, match="Operation lease deadline could not be persisted"): handle.narrow(narrowed_deadline) assert handle.record.not_after_epoch == narrowed_deadline assert handle.path is None diff --git a/tests/server/test_tools_execution_command.py b/tests/server/test_tools_execution_command.py index 21ff194f7f..48367ffae1 100644 --- a/tests/server/test_tools_execution_command.py +++ b/tests/server/test_tools_execution_command.py @@ -573,6 +573,42 @@ async def _capture_marker(marker_dir, *args, **kwargs): class TestRunSkillMcpTimeout: """run_skill wraps executor.run with anyio.fail_after(mcp_tool_timeout_sec).""" + @pytest.mark.anyio + async def test_run_skill_sanitizes_lease_narrow_write_failure( + self, tool_ctx_kitchen_open, tmp_path, monkeypatch + ): + import time + + import autoskillit.core.plugins._operation_lease as lease_module + from autoskillit.core import operation_lease + + cfg = _command_config() + cfg.safety.require_dry_walkthrough = False + tool_ctx_kitchen_open.config = cfg + tool_ctx_kitchen_open.runner.push(_make_result(returncode=1)) + + async with operation_lease( + tmp_path, + operation="run_skill", + not_after_epoch=time.time() + 100_000, + registry=tool_ctx_kitchen_open.in_flight_operations, + ) as handle: + path = handle.path + assert path is not None + + def fail_write(*args, **kwargs): + raise OSError(f"permission denied: {path}") + + monkeypatch.setattr(lease_module, "write_versioned_json", fail_write) + result_json = await run_skill("/investigate lease deadline", "/tmp") + + result = json.loads(result_json) + assert result["subtype"] == "crashed" + assert "Operation lease deadline could not be persisted" in result["result"] + assert str(path) not in result_json + assert tool_ctx_kitchen_open.in_flight_operations.active_count == 0 + assert not tuple(tmp_path.glob("*.lease.json")) + @pytest.mark.anyio async def test_run_skill_narrows_current_operation_lease_to_invocation_deadline( self, tool_ctx_kitchen_open, monkeypatch From 9362ab53c6ef2a3557849257e0f4789018459060 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:14:10 -0700 Subject: [PATCH 21/37] test(review): scope marker dependency guard to executable AST --- tests/arch/test_watcher_signal_consistency.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/arch/test_watcher_signal_consistency.py b/tests/arch/test_watcher_signal_consistency.py index caf232d674..f5305ec098 100644 --- a/tests/arch/test_watcher_signal_consistency.py +++ b/tests/arch/test_watcher_signal_consistency.py @@ -109,9 +109,22 @@ def test_liveness_watchers_reach_the_operation_lease_reader() -> None: def test_execution_watchers_have_no_dispatch_marker_dependency() -> None: for source_path in _EXECUTION_ROOT.rglob("*.py"): - source = source_path.read_text(encoding="utf-8") - assert "_has_active_execution_marker" not in source, source_path - assert "-in-progress-" not in source, source_path + tree = ast.parse(source_path.read_text(encoding="utf-8")) + assert "_has_active_execution_marker" not in _called_names(tree), source_path + assert not any( + isinstance(node, ast.alias) + and node.name.rsplit(".", 1)[-1] == "_has_active_execution_marker" + for node in ast.walk(tree) + ), source_path + for call in (node for node in ast.walk(tree) if isinstance(node, ast.Call)): + arguments = [*call.args, *(keyword.value for keyword in call.keywords)] + assert not any( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and "-in-progress-" in node.value + for argument in arguments + for node in ast.walk(argument) + ), source_path def test_liveness_consumers_do_not_use_network_connections_as_evidence() -> None: From 2bd6f4ded06f2d8dd6701f337c2a9754b01f4126 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:14:49 -0700 Subject: [PATCH 22/37] test(review): decode watcher source as UTF-8 consistently --- tests/arch/test_watcher_signal_consistency.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/arch/test_watcher_signal_consistency.py b/tests/arch/test_watcher_signal_consistency.py index f5305ec098..6d06039b2c 100644 --- a/tests/arch/test_watcher_signal_consistency.py +++ b/tests/arch/test_watcher_signal_consistency.py @@ -21,7 +21,7 @@ def _functions_calling_predicate(source_path: Path, predicate: str) -> set[str]: """Return names of top-level async functions in source_path that call predicate.""" - tree = ast.parse(source_path.read_text()) + tree = ast.parse(source_path.read_text(encoding="utf-8")) result: set[str] = set() for node in ast.walk(tree): if isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef)): @@ -36,7 +36,7 @@ def _functions_calling_predicate(source_path: Path, predicate: str) -> set[str]: def _function(source_path: Path, name: str) -> ast.AsyncFunctionDef | ast.FunctionDef: - tree = ast.parse(source_path.read_text()) + tree = ast.parse(source_path.read_text(encoding="utf-8")) return next( node for node in ast.walk(tree) @@ -81,7 +81,7 @@ def test_liveness_watchers_reach_the_operation_lease_reader() -> None: """Watchers use the lease reader directly or through the shared lease helpers.""" lease_activity = next( node - for node in ast.walk(ast.parse(_PROCESS_MONITOR.read_text())) + for node in ast.walk(ast.parse(_PROCESS_MONITOR.read_text(encoding="utf-8"))) if isinstance(node, ast.ClassDef) and node.name == "_OperationLeaseActivity" ) refresh = next( From 0d046ee2e0d2278d1e79aa3a2a4a1dd5a297a65b Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:15:29 -0700 Subject: [PATCH 23/37] test(review): remove unrelated marker setup from server activity coverage --- tests/cli/test_serve_guard_deferral.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/cli/test_serve_guard_deferral.py b/tests/cli/test_serve_guard_deferral.py index bb6542ff66..bdf46ba852 100644 --- a/tests/cli/test_serve_guard_deferral.py +++ b/tests/cli/test_serve_guard_deferral.py @@ -5,7 +5,6 @@ import inspect import os import signal -from pathlib import Path import anyio import pytest @@ -140,16 +139,6 @@ def test_activity_check_false_when_both_idle(self) -> None: is False ) - def test_stray_backend_marker_does_not_make_server_active(self, tmp_path: Path) -> None: - (tmp_path / "dispatch-in-progress-other-session-marker.marker").touch() - assert ( - is_server_active( - worker_capacity=None, - in_flight_operations=InFlightOperations(), - ) - is False - ) - class TestParameterNaming: def test_serve_with_signal_guard_accepts_activity_check_param(self) -> None: From 65ddb94959ecd3efa93ca0e632b312ffec4dea61 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:16:07 -0700 Subject: [PATCH 24/37] test(review): make activity probe capture explicit --- tests/cli/test_app_main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_app_main.py b/tests/cli/test_app_main.py index 1354455786..6278df3ca7 100644 --- a/tests/cli/test_app_main.py +++ b/tests/cli/test_app_main.py @@ -362,13 +362,15 @@ async def fake_serve_guard(mcp_instance, *, activity_check=None, **kw): assert len(captured_activity_check) == 1, "serve_with_signal_guard was not called exactly once" seen_registries: list = [] + + def record_activity_check(worker_capacity, in_flight_operations): + seen_registries.append((worker_capacity, in_flight_operations)) + return False + monkeypatch.setattr( app_module, "is_server_active", - lambda worker_capacity, in_flight_operations: ( - seen_registries.append((worker_capacity, in_flight_operations)), - False, - )[-1], + record_activity_check, ) captured_activity_check[0]() From b31cc7f130a3c5ab93ba9233241099e337f4b097 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:16:49 -0700 Subject: [PATCH 25/37] test(review): report missing stdout stall event explicitly --- tests/execution/test_operation_lease_watchers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/execution/test_operation_lease_watchers.py b/tests/execution/test_operation_lease_watchers.py index 6fcb40e19c..d781779801 100644 --- a/tests/execution/test_operation_lease_watchers.py +++ b/tests/execution/test_operation_lease_watchers.py @@ -153,7 +153,10 @@ async def test_stdout_growth_then_stall_reports_actual_silence_and_threshold( with structlog.testing.capture_logs() as logs: await _watch_stdout_idle(stdout, 2.0, acc, trigger, 1.0) - fire = next(entry for entry in logs if entry.get("event") == "stdout_idle_stall_firing") + fire = next( + (entry for entry in logs if entry.get("event") == "stdout_idle_stall_firing"), None + ) + assert fire is not None, "stdout_idle_stall_firing was not logged" assert acc.idle_stall is True assert fire["idle_threshold"] == 2.0 assert fire["silence_seconds"] >= 2.0 From 3066df548427615912e56b4832426a236e21e8b6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:17:08 -0700 Subject: [PATCH 26/37] test(review): assert lease helper publishes before aging its file --- tests/execution/test_process_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/execution/test_process_run.py b/tests/execution/test_process_run.py index c081bfc7aa..f81a966e48 100644 --- a/tests/execution/test_process_run.py +++ b/tests/execution/test_process_run.py @@ -623,7 +623,8 @@ async def hold(): time.sleep(0.01) producer.kill() producer.wait(timeout=3) -lease_path = next(channel.glob("*.lease.json")) +lease_path = next(channel.glob("*.lease.json"), None) +assert lease_path is not None, "lease producer did not publish its operation lease" stale = time.time() - 91 os.utime(lease_path, (stale, stale)) time.sleep(30) From f2fa09db73eba58d69b6d315d2cfaf0b6ca91b39 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:18:39 -0700 Subject: [PATCH 27/37] test(review): share connected process fake across lease watchers --- tests/execution/_process_helpers.py | 23 +++++++++++++++++++ .../test_process_deadline_extension.py | 21 +---------------- ...s_session_log_monitor_stale_suppression.py | 20 +--------------- 3 files changed, 25 insertions(+), 39 deletions(-) create mode 100644 tests/execution/_process_helpers.py diff --git a/tests/execution/_process_helpers.py b/tests/execution/_process_helpers.py new file mode 100644 index 0000000000..f45db53736 --- /dev/null +++ b/tests/execution/_process_helpers.py @@ -0,0 +1,23 @@ +"""Process doubles shared by liveness watcher tests.""" + +from types import SimpleNamespace + +import psutil + + +class ConnectedProcess: + def __init__(self, pid: int) -> None: + self.pid = pid + + def children(self, recursive: bool = False) -> list[object]: + return [] + + def connections(self, kind: str | None = None) -> list[SimpleNamespace]: + return [ + SimpleNamespace( + status=psutil.CONN_ESTABLISHED, + raddr=SimpleNamespace(port=443), + ) + ] + + net_connections = connections diff --git a/tests/execution/test_process_deadline_extension.py b/tests/execution/test_process_deadline_extension.py index 203efcdcc5..cbf2739bd0 100644 --- a/tests/execution/test_process_deadline_extension.py +++ b/tests/execution/test_process_deadline_extension.py @@ -5,7 +5,6 @@ import functools import sys import time -from types import SimpleNamespace import anyio import psutil @@ -18,6 +17,7 @@ _enroll_child_activity_watcher, _watch_child_activity, ) +from tests.execution._process_helpers import ConnectedProcess pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] @@ -162,25 +162,6 @@ async def test_network_connection_without_lease_does_not_extend_deadline( ) -> None: """An established port-443 connection does not extend a lease-free deadline.""" - class ConnectedProcess: - def __init__(self, pid: int) -> None: - self.pid = pid - - def children(self, recursive: bool = False) -> list[object]: - return [] - - def connections(self, kind: str | None = None) -> list[SimpleNamespace]: - import psutil - - return [ - SimpleNamespace( - status=psutil.CONN_ESTABLISHED, - raddr=SimpleNamespace(port=443), - ) - ] - - net_connections = connections - channel = tmp_path / "leases" channel.mkdir() import autoskillit.execution.process._process_monitor as process_monitor diff --git a/tests/execution/test_process_session_log_monitor_stale_suppression.py b/tests/execution/test_process_session_log_monitor_stale_suppression.py index 9baa7cbb5f..69a9be6a77 100644 --- a/tests/execution/test_process_session_log_monitor_stale_suppression.py +++ b/tests/execution/test_process_session_log_monitor_stale_suppression.py @@ -4,15 +4,14 @@ import time from pathlib import Path -from types import SimpleNamespace import anyio -import psutil import pytest import autoskillit.execution.process._process_monitor as _patch_process__process_monitor from autoskillit.core.types import ChannelBStatus from autoskillit.execution.process import _session_log_monitor +from tests.execution._process_helpers import ConnectedProcess pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] @@ -30,23 +29,6 @@ async def test_network_connection_without_lease_does_not_suppress_stale( channel = tmp_path / "leases" channel.mkdir() - class ConnectedProcess: - def __init__(self, pid: int) -> None: - self.pid = pid - - def children(self, recursive: bool = False) -> list[object]: - return [] - - def connections(self, kind: str | None = None) -> list[SimpleNamespace]: - return [ - SimpleNamespace( - status=psutil.CONN_ESTABLISHED, - raddr=SimpleNamespace(port=443), - ) - ] - - net_connections = connections - monkeypatch.setattr(_patch_process__process_monitor.psutil, "Process", ConnectedProcess) monkeypatch.setattr( _patch_process__process_monitor, "_has_active_child_processes", lambda _pid: False From 005c29a2343b108f64a6aec67ee1ba55635d70f6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:20:13 -0700 Subject: [PATCH 28/37] test(review): share fleet lease launch assertions across backends --- tests/fleet/_helpers.py | 16 ++++++++++++++++ tests/fleet/test_fleet_e2e.py | 14 ++------------ tests/fleet/test_fleet_e2e_codex.py | 14 ++------------ 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/fleet/_helpers.py b/tests/fleet/_helpers.py index 3e9c85c045..47aefaab4b 100644 --- a/tests/fleet/_helpers.py +++ b/tests/fleet/_helpers.py @@ -27,6 +27,22 @@ KITCHEN_CORE_TOOLS = TOOLS_BY_PACK["kitchen-core"] +def assert_food_truck_lease_launch(runner: Any, expected_idle: int) -> Path: + """Check the attempt lease channel, timeout policy, and cleanup.""" + call = runner.last_kwargs + assert "marker_dir" not in call + assert "session_id" not in call + assert runner.marker_files_during_call == () + channel = call["operation_lease_dir"] + assert isinstance(channel, Path) and channel.is_absolute() + assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) + assert runner.operation_lease_dir_exists_during_call is True + assert call["idle_output_timeout"] == expected_idle + assert call["env"]["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "1800" + assert not channel.exists() + return channel + + def compute_food_truck_tool_surface(recipe_name: str) -> frozenset[str]: """Compute the expected tool surface for a food truck running the given recipe.""" from autoskillit.recipe.io import builtin_recipes_dir, load_recipe diff --git a/tests/fleet/test_fleet_e2e.py b/tests/fleet/test_fleet_e2e.py index 8913e9c7ce..3679feb9cd 100644 --- a/tests/fleet/test_fleet_e2e.py +++ b/tests/fleet/test_fleet_e2e.py @@ -25,6 +25,7 @@ import pytest from autoskillit.core import CLAUDE_CODE_CAPABILITIES +from tests.fleet._helpers import assert_food_truck_lease_launch pytestmark = [ pytest.mark.layer("fleet"), @@ -501,18 +502,7 @@ async def test_food_truck_lease_channel_and_l2_idle_floor_on_claude( ) assert result["success"] is True - call = runtime.runner.last_kwargs - assert "marker_dir" not in call - assert "session_id" not in call - assert runtime.runner.marker_files_during_call == () - channel = call["operation_lease_dir"] - assert isinstance(channel, Path) and channel.is_absolute() - assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) - assert runtime.runner.operation_lease_dir_exists_during_call is True - assert call["idle_output_timeout"] == expected_idle - assert call["env"]["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "1800" - assert not channel.exists() - channels.append(channel) + channels.append(assert_food_truck_lease_launch(runtime.runner, expected_idle)) assert channels[0] != channels[1] diff --git a/tests/fleet/test_fleet_e2e_codex.py b/tests/fleet/test_fleet_e2e_codex.py index b99e58c6cb..ac3180ddaf 100644 --- a/tests/fleet/test_fleet_e2e_codex.py +++ b/tests/fleet/test_fleet_e2e_codex.py @@ -26,6 +26,7 @@ from autoskillit.execution.headless import DefaultHeadlessExecutor from autoskillit.fleet._api import execute_dispatch from tests.fakes import InMemoryRecipeRepository +from tests.fleet._helpers import assert_food_truck_lease_launch from tests.fleet.test_fleet_e2e import FleetTestRunner pytestmark = [ @@ -140,18 +141,7 @@ async def test_food_truck_lease_channel_and_l2_idle_floor( quota_refresher=_noop_quota_refresher, ) - call = runner.last_kwargs - assert "marker_dir" not in call - assert "session_id" not in call - assert runner.marker_files_during_call == () - channel = call["operation_lease_dir"] - assert isinstance(channel, Path) and channel.is_absolute() - assert call["env"]["AUTOSKILLIT_OPERATION_LEASE_DIR"] == str(channel) - assert runner.operation_lease_dir_exists_during_call is True - assert call["idle_output_timeout"] == expected_idle - assert call["env"]["AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT"] == "1800" - assert not channel.exists() - channels.append(channel) + channels.append(assert_food_truck_lease_launch(runner, expected_idle)) assert channels[0] != channels[1] From 3742f8d9e1a5a873dcb011bea43d403aba777bf6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:21:51 -0700 Subject: [PATCH 29/37] refactor(review): return server activity predicate directly --- src/autoskillit/cli/app.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/cli/app.py b/src/autoskillit/cli/app.py index 8cc2ad42c2..6986f56283 100644 --- a/src/autoskillit/cli/app.py +++ b/src/autoskillit/cli/app.py @@ -70,9 +70,8 @@ def is_server_active( worker_capacity: ManagedWorkerCapacity | None, in_flight_operations: InFlightOperationsProtocol, ) -> bool: - return bool( - in_flight_operations.active_count > 0 - or (worker_capacity is not None and worker_capacity.active_count > 0) + return in_flight_operations.active_count > 0 or ( + worker_capacity is not None and worker_capacity.active_count > 0 ) From c835a6462729eb3b5255dd60783242f3688639fe Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:24:47 -0700 Subject: [PATCH 30/37] fix(review): include attempt operation leases in interactive activity --- .../cli/session/_session_process.py | 22 +++++++++-- tests/cli/test_cook_lifetime.py | 39 ++++++++++++++++++- tests/cli/test_cook_process_lifecycle.py | 4 +- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/autoskillit/cli/session/_session_process.py b/src/autoskillit/cli/session/_session_process.py index 5ee7ef2e47..56c0b893a0 100644 --- a/src/autoskillit/cli/session/_session_process.py +++ b/src/autoskillit/cli/session/_session_process.py @@ -10,6 +10,7 @@ import time from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Final, TextIO from uuid import uuid4 @@ -20,6 +21,7 @@ from autoskillit.cli.ui._terminal import terminal_guard, terminal_last_activity from autoskillit.config import ProcessTetherConfig from autoskillit.core import ( + OPERATION_LEASE_DIR_ENV_VAR, SESSION_LIFETIME_NOTICE_ENV_VAR, CmdSpec, ProcessCleanupResult, @@ -55,9 +57,11 @@ ) -def _default_activity(pid: int, fd: int | None) -> frozenset[str]: +def _default_activity( + pid: int, fd: int | None, *, operation_lease_dir: Path | None = None +) -> frozenset[str]: """Read process liveness and recent kernel TTY activity.""" - signals = set(_active_liveness_signals(pid, operation_lease_dir=None)) + signals = set(_active_liveness_signals(pid, operation_lease_dir=operation_lease_dir)) if fd is not None: last_activity = terminal_last_activity(fd) if last_activity is not None and time.time() - last_activity <= _IDLE_WINDOW_SECONDS: @@ -75,6 +79,7 @@ def __init__( clock: Callable[[], float] = time.monotonic, wall: Callable[[], float] = time.time, activity_probe: Callable[[int, int | None], frozenset[str]] | None = None, + operation_lease_dir: str | None = None, ) -> None: policy.validate() soft_seconds = policy.cook_ceiling_seconds @@ -92,7 +97,14 @@ def __init__( self._hard_cap_seconds = hard_cap_seconds self._clock = clock self._wall = wall - self._activity_probe = _default_activity if activity_probe is None else activity_probe + channel = Path(operation_lease_dir) if operation_lease_dir else None + if channel is not None and not channel.is_absolute(): + channel = None + self._activity_probe = ( + partial(_default_activity, operation_lease_dir=channel) + if activity_probe is None + else activity_probe + ) self._started_at: float | None = None self._soft_deadline: float | None = None self._hard_deadline: float | None = None @@ -342,7 +354,9 @@ def run_cook_attempt( _require_posix_process_ownership() cwd = _canonical_cwd(spec.cwd) inherited_fds = _normalize_pass_fds(pass_fds) - life = InteractiveLifetime(lifetime) + life = InteractiveLifetime( + lifetime, operation_lease_dir=spec.env.get(OPERATION_LEASE_DIR_ENV_VAR) + ) terminal_fd = _interactive_terminal_fd() notice_path = ensure_project_temp(Path(cwd)) / "session_lifetime" / f"{uuid4().hex}.json" diff --git a/tests/cli/test_cook_lifetime.py b/tests/cli/test_cook_lifetime.py index f9040991b9..ced81cfe6e 100644 --- a/tests/cli/test_cook_lifetime.py +++ b/tests/cli/test_cook_lifetime.py @@ -5,6 +5,7 @@ import json import os import sys +import time from collections.abc import Callable from pathlib import Path from typing import Any @@ -13,7 +14,7 @@ import structlog from autoskillit.config import ProcessTetherConfig -from autoskillit.core import TerminationReason +from autoskillit.core import InFlightOperations, TerminationReason, operation_lease from autoskillit.execution.process._process_tether import TetherRecord, write_tether from tests.cli._cook_launch_helpers import lifetime_policy @@ -144,6 +145,42 @@ def test_active_session_extends_past_soft_ceiling() -> None: assert extended[0]["signals"] == ["terminal_io"] +@pytest.mark.anyio +async def test_default_probe_uses_attempt_operation_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import autoskillit.execution.process._process_monitor as process_monitor + + module = _lifetime_module() + monkeypatch.setattr(module, "_IDLE_WINDOW_SECONDS", 5.0) + monkeypatch.setattr(module, "_LIVENESS_PROBE_INTERVAL_SECONDS", 1.0) + monkeypatch.setattr(process_monitor, "_has_active_child_processes", lambda _pid: False) + clock = _Clock() + lifetime = module.InteractiveLifetime( + _policy(soft=6.0, extension=10.0), + clock=clock, + wall=_Clock(10_000.0), + operation_lease_dir=str(tmp_path), + ) + lifetime.start(pid=os.getpid(), tether_path=None, terminal_fd=None, notice_path=None) + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 60, + registry=InFlightOperations(), + ) as handle: + clock.advance(6.1) + with structlog.testing.capture_logs() as logs: + assert lifetime.poll() is None + assert _events(logs, "cook_lifetime_extended")[0]["signals"] == ["operation_lease"] + + assert handle.path is not None + handle.path.unlink() + clock.advance(5.0) + assert lifetime.poll() is TerminationReason.IDLE_STALL + + def test_idle_session_ends_at_soft_ceiling_as_idle_stall(monkeypatch: pytest.MonkeyPatch) -> None: lifetime_module = _lifetime_module() monkeypatch.setattr(lifetime_module, "_IDLE_WINDOW_SECONDS", 5.0) diff --git a/tests/cli/test_cook_process_lifecycle.py b/tests/cli/test_cook_process_lifecycle.py index 1e8e889f0f..020133ae78 100644 --- a/tests/cli/test_cook_process_lifecycle.py +++ b/tests/cli/test_cook_process_lifecycle.py @@ -631,8 +631,8 @@ def test_poll_failures_do_not_escape_the_wait_loop( actual_lifetime = _session_process.InteractiveLifetime class AcceleratedLifetime(actual_lifetime): - def __init__(self, policy: ProcessTetherConfig) -> None: - super().__init__(policy, clock=lambda: clock_value[0]) + def __init__(self, policy: ProcessTetherConfig, **kwargs) -> None: + super().__init__(policy, clock=lambda: clock_value[0], **kwargs) def poll(self): clock_value[0] += 1000.0 From f09c4d3970d74583d027e52c96be42e32e528df8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:25:36 -0700 Subject: [PATCH 31/37] refactor(review): name operation lease deferred log interval --- src/autoskillit/execution/process/_process_monitor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/autoskillit/execution/process/_process_monitor.py b/src/autoskillit/execution/process/_process_monitor.py index 457a094ed1..fe4a820eac 100644 --- a/src/autoskillit/execution/process/_process_monitor.py +++ b/src/autoskillit/execution/process/_process_monitor.py @@ -28,6 +28,7 @@ from autoskillit.core import ObservedEntry, SessionEvent, StreamParser logger = get_logger(__name__) +_LEASE_DEFERRED_LOG_INTERVAL_SECONDS = 300 class _OperationLeaseActivity: @@ -61,7 +62,11 @@ def refresh( if not leases: return reference, suppression_start signature = tuple((lease.operation_id, lease.not_after_epoch) for lease in leases) - if signature != self._signature or self._logged_at is None or now - self._logged_at >= 300: + if ( + signature != self._signature + or self._logged_at is None + or now - self._logged_at >= _LEASE_DEFERRED_LOG_INTERVAL_SECONDS + ): self._log( self._event, operation=leases[0].operation, From 86a0082e25787baabf2a96876b90615c7beeab2d Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:28:14 -0700 Subject: [PATCH 32/37] fix(review): preserve actual stdout silence in inspector evidence --- .../execution/process/_race_watchers.py | 6 +-- .../test_operation_lease_watchers.py | 46 ++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/autoskillit/execution/process/_race_watchers.py b/src/autoskillit/execution/process/_race_watchers.py index bc152c4719..63a94b4e18 100644 --- a/src/autoskillit/execution/process/_race_watchers.py +++ b/src/autoskillit/execution/process/_race_watchers.py @@ -117,7 +117,7 @@ async def _watch_stdout_idle( ) spared_at = await _inspect_stdout_idle( stdout_path, - last_liveness_evidence, + last_stdout_growth, acc, inspector_callback, timeout_scope_ref, @@ -134,7 +134,7 @@ async def _watch_stdout_idle( async def _inspect_stdout_idle( stdout_path: Path, - last_liveness_evidence: float, + last_stdout_growth: float, acc: RaceAccumulator, inspector_callback: InspectorCallback | None, timeout_scope_ref: list[anyio.CancelScope | None] | None, @@ -167,7 +167,7 @@ async def _inspect_stdout_idle( ) evidence = _package_evidence( stdout_path, - idle_seconds=_time.monotonic() - last_liveness_evidence, + idle_seconds=_time.monotonic() - last_stdout_growth, operation_lease_active=operation_lease_active, ) try: diff --git a/tests/execution/test_operation_lease_watchers.py b/tests/execution/test_operation_lease_watchers.py index d781779801..a22938e5df 100644 --- a/tests/execution/test_operation_lease_watchers.py +++ b/tests/execution/test_operation_lease_watchers.py @@ -20,6 +20,8 @@ from autoskillit.core import ( ChannelBStatus, InFlightOperations, + InspectorEvidence, + InspectorVerdict, operation_lease, ) from autoskillit.execution.process._process_monitor import ( @@ -87,7 +89,9 @@ def _install_clock( ) -> None: monkeypatch.setattr(time, "monotonic", lambda: clock.elapsed) monkeypatch.setattr(time, "time", lambda: clock.epoch) - fake_anyio = SimpleNamespace(sleep=clock.sleep, current_time=clock.current_time) + fake_anyio = SimpleNamespace( + sleep=clock.sleep, current_time=clock.current_time, fail_after=anyio.fail_after + ) for module in modules: monkeypatch.setattr(module, "anyio", fake_anyio) @@ -162,6 +166,46 @@ async def test_stdout_growth_then_stall_reports_actual_silence_and_threshold( assert fire["silence_seconds"] >= 2.0 +@pytest.mark.anyio +async def test_inspector_receives_stdout_silence_across_lease_activity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(lease_module, "_heartbeat", _blocked_heartbeat) + observed: list[InspectorEvidence] = [] + + async def inspect(evidence: InspectorEvidence) -> InspectorVerdict: + observed.append(evidence) + return InspectorVerdict( + action="KILL", reasoning="no output", confidence="high", elapsed_seconds=0.0 + ) + + async with operation_lease( + tmp_path, + operation="tool", + not_after_epoch=time.time() + 3, + registry=InFlightOperations(), + ) as handle: + clock = _LeaseClock(handle.record.started_at_epoch, channel=tmp_path, stop_after=8) + _install_clock(monkeypatch, clock, race_watchers) + stdout = MagicMock() + stdout.stat.return_value = SimpleNamespace(st_size=0) + acc = RaceAccumulator() + await _watch_stdout_idle( + stdout, + 2.0, + acc, + anyio.Event(), + 1.0, + operation_lease_dir=tmp_path, + inspector_callback=inspect, + ) + + assert acc.idle_stall is True + assert len(observed) == 1 + assert observed[0].idle_seconds == 4.0 + assert observed[0].operation_lease_active is False + + @pytest.mark.anyio async def test_stdout_lease_precedes_pending_task_gate( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From fd29a1aa3ea72114da194ac80abe288783c1ea7a Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:48:51 -0700 Subject: [PATCH 33/37] fix(review): update env forwarding coordinate after activity cleanup --- tests/_ambient_env_surface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 73ebb35898..15333add53 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -997,7 +997,7 @@ class AmbientEnvDisposition: "install/publish subprocess env; intentional wholesale forward for a maintenance-style" "operation." ), - "cli/app.py:316": ( + "cli/app.py:315": ( "Bare os.environ passed as `child_env` to the maintenance installer, which itself applies" "an explicit allowlist (build_maintenance_env) before spawning; unfiltered by design here." ), From f02ba336334771321d70e54ee5397ccf8e4af343 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 10:50:07 -0700 Subject: [PATCH 34/37] fix(review): forward lease probe keywords through lifetime test doubles --- tests/cli/test_cook_process_lifecycle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_cook_process_lifecycle.py b/tests/cli/test_cook_process_lifecycle.py index 020133ae78..6e6cfb0ee3 100644 --- a/tests/cli/test_cook_process_lifecycle.py +++ b/tests/cli/test_cook_process_lifecycle.py @@ -364,7 +364,7 @@ def test_active_child_extended_until_hard_cap( monkeypatch.setattr( _session_process, "_default_activity", - lambda *_args: {"child_processes"}, + lambda *_args, **_kwargs: {"child_processes"}, ) result = run_cook_attempt( @@ -392,7 +392,7 @@ def test_idle_child_ends_at_soft_ceiling( return monkeypatch.setattr(_session_process, "_LIVENESS_PROBE_INTERVAL_SECONDS", 0.2) monkeypatch.setattr(_session_process, "_IDLE_WINDOW_SECONDS", 0.5) - monkeypatch.setattr(_session_process, "_default_activity", lambda *_args: set()) + monkeypatch.setattr(_session_process, "_default_activity", lambda *_args, **_kwargs: set()) result = run_cook_attempt( _spec(tmp_path, "import time; time.sleep(30)"), @@ -531,7 +531,7 @@ def test_lifetime_decision_wins_dispatch_even_when_a_failure_accumulated( monkeypatch.setattr( _session_process, "_default_activity", - lambda *_args: {"child_processes"}, + lambda *_args, **_kwargs: {"child_processes"}, ) grace_seconds = 0.5 monkeypatch.setattr( @@ -620,7 +620,7 @@ def test_poll_failures_do_not_escape_the_wait_loop( monkeypatch.setattr( _session_process, "_default_activity", - lambda *_args: (_ for _ in ()).throw(OSError("activity probe failed")), + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("activity probe failed")), ) monkeypatch.setattr( _session_process, From 4cb19466672b2ce20a867005774b469233cd8242 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 12:38:36 -0700 Subject: [PATCH 35/37] docs(review): clarify internal subprocess runner contract --- src/autoskillit/core/types/execution/_type_subprocess.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/autoskillit/core/types/execution/_type_subprocess.py b/src/autoskillit/core/types/execution/_type_subprocess.py index 1d0c08fada..392b52680e 100644 --- a/src/autoskillit/core/types/execution/_type_subprocess.py +++ b/src/autoskillit/core/types/execution/_type_subprocess.py @@ -229,6 +229,9 @@ def on_line(self, line: str) -> tuple[str, ...]: ... class SubprocessRunner(Protocol): """Protocol for async subprocess execution. Matches run_managed_async signature. + This is an internal dependency-injection contract. Its signature evolves + with the built-in runners; the core export is not a stable external extension API. + Parameters ---------- operation_lease_dir : Path | None From 805e76de3dab6035d036afa0735ec3d63e631596 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 12:39:20 -0700 Subject: [PATCH 36/37] docs(review): distinguish supervisor and child idle timeout policies --- docs/execution/orchestration.md | 14 ++++++++++++++ .../headless/_managed/_food_truck_executor.py | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/docs/execution/orchestration.md b/docs/execution/orchestration.md index ef0d76d0d1..ec77a4b344 100644 --- a/docs/execution/orchestration.md +++ b/docs/execution/orchestration.md @@ -28,6 +28,20 @@ The boundary is enforced three ways: FastMCP visibility, the `tools_execution/`. `run_skill` uses `_require_orchestrator_exact()`; `run_cmd` and `run_python` use `_require_orchestrator_or_higher()`. All three layers must agree. +## Food-truck idle timeouts + +Food-truck supervision and the child tools use separate idle policies. The +per-dispatch `idle_output_timeout` controls the outer supervisor. Its effective +value is at least `run_skill.timeout`, so a zero idle setting still leaves that +supervisor backstop enabled. Active operation leases refresh the supervisor's +liveness clock until their deadlines expire. + +The child's `AUTOSKILLIT_IDLE_OUTPUT_TIMEOUT` comes from `fleet.idle_output_timeout`, +falling back to `run_skill.idle_output_timeout` when the Fleet value is zero. +An explicit value in `env_extras` takes precedence. A per-dispatch idle override +does not change this child policy. Setting both configured idle values to zero +omits the child idle override while retaining the outer supervisor floor. + ## Recipe as a program A recipe (`recipe/schema.py:Recipe`) is a sequenced list of `RecipeStep` diff --git a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py index bc71742bc2..03be1670b9 100644 --- a/src/autoskillit/execution/headless/_managed/_food_truck_executor.py +++ b/src/autoskillit/execution/headless/_managed/_food_truck_executor.py @@ -240,6 +240,12 @@ async def dispatch_food_truck( managed_lineage_ref: ManagedHeadlessSessionLineageRef | None = None, on_launch_resolved: Callable[[ResolvedLaunchContract], None] | None = None, ) -> SkillResult: + """Run a food truck under its configured supervisor deadline. + + The per-call idle timeout controls the supervisor and is floored at + run_skill.timeout, including when the idle value is zero. Child tool + idle policy comes from fleet/run-skill configuration or env_extras. + """ import autoskillit.execution.headless as headless_facade cwd = validated_dispatch_cwd( From ebc767d384f4c0a38c4e768d72b3ccb920cea095 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 12:42:57 -0700 Subject: [PATCH 37/37] test(review): prove nonterminal item diagnostics across Codex transports --- .../test_codex_item_dispatch_parity.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/execution/backends/test_codex_item_dispatch_parity.py b/tests/execution/backends/test_codex_item_dispatch_parity.py index df58f553d8..414378d3da 100644 --- a/tests/execution/backends/test_codex_item_dispatch_parity.py +++ b/tests/execution/backends/test_codex_item_dispatch_parity.py @@ -8,13 +8,15 @@ import pytest from jsonschema import validate -from autoskillit.core import BackendEventKind, CodexEventData +from autoskillit.core import BackendEventKind, CodexEventData, SessionErrorOutcome from autoskillit.core.types.foundation._type_enums import CodexItemType from autoskillit.execution.backends._codex_parse import ( + CodexResultParser, CodexStreamParser, _accumulate_codex_completed_item, _CodexParseAccumulator, ) +from autoskillit.execution.headless._headless_evidence import _adapt_agent_result pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -71,3 +73,31 @@ def test_error_item_messages_ignore_empty_and_non_string_values() -> None: ) assert accumulator.item_error_messages == [] + + +@pytest.mark.parametrize("json_rpc", [False, True], ids=["exec", "app-server"]) +def test_error_items_reach_session_diagnostics_without_failing_the_turn(json_rpc: bool) -> None: + message = "recoverable item error" + items = [ + {"type": "error", "message": message}, + {"type": "agent_message", "text": "done"}, + ] + if json_rpc: + records = [{"method": "item/completed", "params": {"item": item}} for item in items] + records.append({"method": "turn/completed", "params": {"turn": {"status": "completed"}}}) + else: + records = [{"type": "item.completed", "item": item} for item in items] + records.append({"type": "turn.completed"}) + lines = [json.dumps(record) for record in records] + + live_error = CodexStreamParser().parse_line(lines[0]) + assert live_error is not None + assert live_error.kind is BackendEventKind.IGNORED + assert live_error.is_terminal is False + + parsed = CodexResultParser().parse_stdout("\n".join(lines)) + assert parsed.success is True + assert parsed.raw["item_error_messages"] == [message] + adapted = _adapt_agent_result(parsed) + assert adapted.is_error is False + assert SessionErrorOutcome.from_errors(adapted.errors).messages == (message,)