From c34beb6bdffc5d3e8c58693e34510acc7bd4ddfe Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 01:08:37 +0000 Subject: [PATCH 01/22] Define the background child run lifecycle contract --- docs/prd/PRD-001-netclaw-mvp.md | 15 + .../background-child-runs/.openspec.yaml | 2 + .../changes/background-child-runs/design.md | 352 +++++++++++++++ .../changes/background-child-runs/proposal.md | 56 +++ .../specs/background-subagent-runs/spec.md | 420 ++++++++++++++++++ .../specs/netclaw-session/spec.md | 32 ++ .../specs/netclaw-subagents/spec.md | 120 +++++ .../specs/progressive-tool-disclosure/spec.md | 31 ++ .../specs/session-state-machine/spec.md | 112 +++++ .../specs/skill-execution-routing/spec.md | 46 ++ .../specs/tool-authorization/spec.md | 80 ++++ .../changes/background-child-runs/tasks.md | 85 ++++ 12 files changed, 1351 insertions(+) create mode 100644 openspec/changes/background-child-runs/.openspec.yaml create mode 100644 openspec/changes/background-child-runs/design.md create mode 100644 openspec/changes/background-child-runs/proposal.md create mode 100644 openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md create mode 100644 openspec/changes/background-child-runs/specs/netclaw-session/spec.md create mode 100644 openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md create mode 100644 openspec/changes/background-child-runs/specs/progressive-tool-disclosure/spec.md create mode 100644 openspec/changes/background-child-runs/specs/session-state-machine/spec.md create mode 100644 openspec/changes/background-child-runs/specs/skill-execution-routing/spec.md create mode 100644 openspec/changes/background-child-runs/specs/tool-authorization/spec.md create mode 100644 openspec/changes/background-child-runs/tasks.md diff --git a/docs/prd/PRD-001-netclaw-mvp.md b/docs/prd/PRD-001-netclaw-mvp.md index 71be8143c..85cd37c76 100644 --- a/docs/prd/PRD-001-netclaw-mvp.md +++ b/docs/prd/PRD-001-netclaw-mvp.md @@ -288,6 +288,21 @@ the current daemon instance running. Valid changes SHALL trigger coordinated daemon restart: close new ingress, drain active sessions, restart, relaunch the sessions that were active, and resume from the last durable checkpoint. +### FR-017 Background Child Delegation + +Netclaw shall return a durable accepted run identifier for explicit and routed child tasks before child completion. +The parent shall accept other inputs and retain status and cancellation controls for its accepted children. +Child results shall return later as attributed tool-origin content under the original requester authority. +Child approval prompts shall remain valid for the live run after its start call ends. +Cancellation shall stop new task dispatch and preserve recorded partial evidence through a five-second framework-only grace period. + +Restart shall report interrupted children as lost, retain terminal delivery, and expire child approval prompts without automatic child relaunch. +The release shall prove a parent model response before a held child request completes under the selected supported provider capacity contract. +Provider capacity remains an explicit delivery gate; actor acknowledgements alone do not prove model responsiveness. +Private agent messages, peer discovery, questions, and live steering remain outside this delivery. + +See `openspec/changes/background-child-runs/` for the second PR's review contract. + ## Operational Requirements - Daemon deploys as a user-level systemd service on `pi1` (no sudo required) diff --git a/openspec/changes/background-child-runs/.openspec.yaml b/openspec/changes/background-child-runs/.openspec.yaml new file mode 100644 index 000000000..01660bab0 --- /dev/null +++ b/openspec/changes/background-child-runs/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-08 diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md new file mode 100644 index 000000000..99d3d33b7 --- /dev/null +++ b/openspec/changes/background-child-runs/design.md @@ -0,0 +1,352 @@ +## Context + +Baseline: `2e6bc4f014dc96b606566709df1bd11f90ecf34a` on `feature/background-subagents`. +The branch will receive PR 1 before implementation. Its frozen dependency is `per-call-tool-recurrence` in the sibling worktree. +See [proposal.md](proposal.md) for the objective and PRD traceability. +The approved [plan](../../../.systematize/plans/background-subagents/plan.html) owns delivery sequencing when that artifact is present in the combined branch. + +Current code establishes these seams: + +- `SubAgentSpawner.SpawnRunAsync` creates `ChildRunScope`, waits through `Ask`, and enriches the terminal result. +- `ToolRunScope.SpawnChildActor` delegates actor creation to the parent. It does not express durable run acceptance. +- `SkillLoadTool` and the parent's slash route await final child output. +- `ParentSessionApprovalBridge` binds the original requester, but its child prompt waits remain live-only in the start scope. +- `SubAgentActor` stops immediately after its terminal reply. It has no durable resume state or compaction. +- `SessionState.ActiveBackgroundJobs` belongs to shell jobs. Its delivery dedup is not a durable child-result contract. +- `RoleBasedFailoverRouter` shares the main pipeline with compaction unless a distinct compaction model exists. +- No provider admission scheduler, priority queue, or reserved parent inference slot exists. + +Relevant engineering references: `SPEC-002`, `SPEC-016`, and the [glossary](../../../docs/spec/GLOSSARY.md). +The old subagent wall-clock timeout statement conflicts with the implemented per-operation watchdog. +This package changes that statement without introducing a task-lifetime ceiling. + +## Goals / Non-Goals + +**Goals:** + +- Free the parent tool phase after durable start acceptance. +- Preserve original run authority through every later parent turn and approval prompt. +- Guarantee duplicate start prevention and durable result admission within one session journal. +- Stop new child dispatch before a confirmed cancellation cutover. +- Preserve framework-recorded evidence without another model call. +- Report explicit restart loss and truthful delivery failures. + +**Non-Goals:** + +- A child that resumes after a daemon restart. +- Agent messages, questions, peer discovery, or live steering. +- A workflow engine, a general provider scheduler, or a new test platform. +- Exactly-once external side effects from a model continuation that crashes midway. +- Another tool-count, task-age, or task-token ceiling. + +## Decisions + +### 1. Reuse authority and the owner-session seam + +Evolve the existing owner-session callback into typed start and control operations. +Keep actor references and prepared actor properties inside the adapter. +Reuse `ChildRunScope.Authority`, the admitted `TurnContextRecord`, and existing session storage bindings. +Do not add parallel audience, path-root, requester, or channel fields to the live start contract. + +Persist the existing serializable turn record plus run identifiers, the initial working snapshot, and lifecycle facts. +Do not serialize `ToolRunScope`, actor references, callbacks, cancellation tokens, or provider clients. +The child retains the same inherited policy and profile contracts as PR 1. + +Alternative: put agents in the shell-job manager. Its shell launch and grant contracts do not describe child authority or results. + +### 2. Durable acceptance fixes one start identity + +The parent owns run identifiers and the acceptance ledger. +The pipeline stamps correlation before asynchronous setup. The model cannot supply or replace it. + +Use these exact logical keys: + +| Start path | Durable key | +| --- | --- | +| `spawn_agent` or routed `skill_load` | Owner session ID, original admitted turn ID, original tool call ID | +| Slash activation | Owner session ID, admitted input ID, original turn ID, framework activation slot `0` | +| Scheduled slash activation | The slash key from the already admitted scheduled input; reuse its persisted occurrence correlation | + +Do not derive keys from task text, current turn state, actor names, or a fresh ID on retry. +Retain the original admitted input IDs as provenance even when a tool start belongs to a buffered multi-input turn. +Compare a bounded digest of canonical start arguments and resolved profile/overlay content against the committed acceptance. +A reused key with conflicting content returns `start_conflict`. It cannot mutate or duplicate the accepted run. +Duplicate scheduled ingress follows existing ingress authority. This contract does not promise indefinite dedup of separately admitted schedule occurrences. + +The parent persists acceptance before actor creation, model/tool execution, or an accepted response. +Concurrent duplicates share one pending persistence operation and receive the same committed run ID. +After persistence, a start failure becomes a terminal failure of that accepted run. +Before persistence, failure returns an explicit rejection and creates no child. +If the accepted response disappears, a retry returns the recorded run. Recovery never creates the child again. + +Schematic; validation, authorization, and persistence failure paths are abbreviated: + +```text +Start(key, preparedScope, argumentsDigest) + -> reject conflicting key or unavailable admission + -> persist Accepted(runId, originalContext, key, argumentsDigest) + -> create child with a run-owned lifetime + -> reply Accepted(runId, state, controlTool) + +Retry(same key, same digest) + -> reply with the existing run and current recorded state +``` + +### 3. The accepted run outlives its start call + +The start token governs setup and acceptance only. +After acceptance commits, its cancellation or disposal cannot cancel the run. +A later parent turn also cannot replace the run's authority or lifetime token. +Explicit run cancellation, explicit parent stop, and owner loss remain terminal boundaries. +Idle passivation defers while a run remains live or a durable result still needs continuation admission. + +All first-party routed starts use this contract. There is no permanent synchronous routed exception. +Inline skills remain inline. Routed failures remain explicit and cannot fall back to inline execution. +No later terminal result uses the completed start tool's activity channel. + +### 4. Machine-actionable outputs describe lifecycle separately from outcome + +The accepted output contains `run_id`, `scope_id`, `state`, and `control_tool`. +Its canonical tool body is one JSON object with four required string fields. +Each required field occurs once. Run and scope identifiers use their nonempty canonical owner values. +Those strings use the existing `SubAgentRunId.Value` and `SubAgentScopeId.Value` representations. +The first response reports the committed initial state `Accepted`. +That state describes admission. The current status can advance before the parent observes this response. +An equivalent retry can report the current recorded state with the same identifiers. + +Example; identifier values are illustrative: + +```json +{ + "run_id": "e1c231a7977f4024903660ad451bb044", + "scope_id": "console/example/subagent/summarizer/e1c231a7977f4024903660ad451bb044", + "state": "Accepted", + "control_tool": "check_agent_run" +} +``` + +The owner supplies these facts only after acceptance commits. Child text cannot supply them. +`spawn_agent` and routed `skill_load` return the bare JSON body without prose or Markdown fences. +The adapter formats it once for `ToolResultOutput.Result` and the model's `FunctionResultContent.Result` string. +Both consumers receive the same body. Field order and insignificant whitespace have no semantic meaning. +Direct slash adapters format human acknowledgement prose from the same owner facts. +They cannot reconstruct the owner facts from that prose or replace identifiers, state, or the control name. +This tool representation does not replace framework-owned journal or snapshot serialization. + +Negative example: `{"run_id":"","state":"Completed"}` is not a valid first acceptance. +An uncommitted object with valid-looking fields is also not acceptance. +The consumer reports the invalid acceptance rather than inferring it from a child's final output. +Later terminal results retain their distinct delivery correlation; they do not replace this completed start result. + +The control tool is `check_agent_run(run_id, cancel=false)` and uses the existing parameter-name conventions. +It remains Deferred under the existing non-shell tool policy. Children cannot discover, load, or dispatch it. +The current parent invocation must pass normal policy and match the owner session and original eligible requester context. +A mismatched child, session, or requester receives a non-disclosing denial. +The original context is the one authority source for this comparison; do not create a second requester ledger. +Root confirmed this conservative first-release boundary under the approved other-speaker negative controls. +Broader control by another eligible same-session user requires a later explicit policy decision. + +States are `Accepted`, `Running`, `Cancelling`, `Completed`, `Partial`, `Failed`, `Cancelled`, and `Lost`. +These states are lifecycle facts. They do not expand `SubAgentRunOutcome` implicitly. +Status includes the state, cancellation admission, dispatch-closure state, and the recorded terminal result when present. +Accepted and live status outputs do not claim task completion. + +Preserve the existing wire outcome names `Completed`, `Partial`, and `Failed`. +A cancelled completion keeps `Success = false`, wire outcome `Failed`, and reason `CancelledByParent`. +Extend the typed cancelled completion with a confirmed working-context delta. +Its lifecycle state is `Cancelled`; its useful evidence does not turn it into `Completed` or successful `Partial`. +`Lost` also uses a failed wire outcome with an explicit owner-loss reason. +Add new reason fields and durable record variants through the framework schema and conversion contracts. + +### 4a. Reuse existing owners for lifecycle acknowledgements + +The table names current mechanisms. It does not prescribe new C# message types. +New acknowledgements need the run correlation that a session-only `CommandAck` cannot carry. +The owner acceptance and terminal records are durable. The JSON body is a call-local view of those committed facts. +Child results, dispatch admission state, actor references, and cancellation sources remain live runtime state. + +| Boundary | Existing owner and mechanism | Required extension | +| --- | --- | --- | +| Acceptance before execution | `LlmSessionActor` owns the journal. Its `AdmitInput` callback applies a committed event before work and acknowledgement. | Use the same post-commit pattern for child acceptance through the existing owner-session adapter. Create no child or accepted body before that callback. | +| Start-token independence | The owner adapter already creates the child. `SubAgentActor` already owns execution and external cancellation sources. | Bind accepted child cancellation to the owner run lifetime. Remove the start token from that lifetime after commit. Keep pre-admission setup cancellation effective. | +| Durable terminal acknowledgement | `SubAgentActor.Complete` already creates one typed result. The parent already applies events in `Persist` callbacks. | Retain the child result and acknowledge its run correlation only after the owner commits its terminal receipt. Reuse sender/owner boundaries and existing state conversion. | +| Cancellation closure | The child owns provider calls and async tool workers. The parent owns approval answers and authorization grants. | Close run dispatch at actual provider/tool/retry entry points. Acknowledge closure only after local entry admission closes. Include the parent approval path in that boundary. | + +`CommandAck` proves only the semantics of its specific acknowledged command. +An input acknowledgement does not acknowledge child acceptance, terminal persistence, or dispatch closure. +`JoinSession` proves a mailbox response. It does not prove a journal commit or a model reply. +`IApprovalChannel.TryClaim` rejects an absent live wait before a broader authorization grant. +Keep that check. Also retain run-liveness checks across asynchronous grant persistence and the exact child retry. +Cancellation tokens alone do not prove that a queued worker cannot dispatch after closure acknowledgement. + +The current test base selects its journal in sealed host setup and has no journal-write fault barrier. +Add only a narrow test-owned journal barrier that holds the real acceptance or terminal write and returns its real acknowledgement. +Keep the existing in-memory journal, real owner actor, normal policy, and supported serializer setup around that barrier. +Do not replace the owner with a probe that emits the desired acceptance or terminal acknowledgement. + +### 5. Terminal precedence follows durable owner order + +The parent accepts terminal messages only from the current recorded child generation. +The first committed terminal receipt wins over later cancellation requests. +If cancellation admission commits first, later child success cannot become the terminal outcome. +The parent can retain its confirmed evidence inside the cancelled result. +Actor mailbox arrival alone is insufficient: journal commit determines the winner. + +The child closes normal dispatch, sends one terminal result, and waits for the owner acknowledgement. +The owner acknowledges only after terminal persistence succeeds. +A duplicate terminal message receives the same acknowledgement and cannot schedule a second continuation. +If persistence fails, the owner does not acknowledge successful receipt. +Existing actor/persistence failure policy remains visible; it must not silently discard the result. +Owner termination stops the child. Recovery resolves an accepted run without a terminal receipt as `Lost`. + +Alternative: acknowledge before persistence. A crash can then lose a result after the child discards its only copy. + +### 6. Cancellation distinguishes admission from dispatch closure + +The parent persists cancellation admission before it reports `Cancelling`. +Then it closes child work admission, cancels active calls, and settles run-owned approval prompts. +Closure must cover pending thread-pool tool dispatch and queued provider requests, not just the actor mailbox. +The child/runtime adapter sends an explicit dispatch-closure acknowledgement after no new task dispatch can start. +The control response reports `dispatch_closed = false` until that acknowledgement exists. +It must never describe cancellation admission alone as proof of stopped dispatch. + +Once closure is acknowledged, no fresh model request, tool launch, or approval retry can start for that run. +Already launched remote work can ignore cancellation. Record its external effect as unknown when confirmation is absent. +The runtime does not claim that it reverses existing effects or stops remote computation. +If a child dies before closure acknowledgement, owner termination establishes local closure and preserves the cancellation outcome. + +Five seconds is the accepted initial finalization default after local dispatch closure. This change adds no operator configuration knob. +The finalization token is separate from the cancelled execution token. +Use injected time and explicitly controllable actor timers; `FakeTimeProvider` alone does not advance Akka timers. +Finalization permits only framework-owned checkpoint preservation and an atomic run-local report write. +It permits no model call, task tool, project edit, external request, or new authorization grant. + +Preserve confirmed tool receipts and file activity during normal execution after each completed feedback round. +Bound inline evidence through existing output rules. Store retained reports inside the assigned run artifact directory. +On grace expiry or report-write failure, retain the last committed checkpoint and expose an explicit finalization reason. +Do not claim a report exists before its atomic write commits. Do not invent findings from interrupted model text. + +### 7. Durable delivery has one idempotent admission boundary + +Use a deterministic delivery ID derived from owner session ID and run ID. +There is one terminal delivery per run. An enrichment retry keeps the same delivery ID. + +```text +child terminal + -> persist terminal receipt and acknowledge child + -> validate/enrich the recorded result + -> persist delivery-ready payload + -> persist continuation admission and its input/transcript item atomically + -> run parent review under original context +``` + +The last persistence operation writes both the dedup marker and the continuation item in one authoritative event. +Two unrelated records with a crash gap do not establish reliable admission. +Recovery derives pending continuations from that event and the existing input-completion records. +An admitted but unfinished parent continuation remains pending. Do not create another admission. +Completed parent input cannot be reintroduced by terminal delivery replay. + +The original start call already received its acceptance result. +The later result uses a fresh framework delivery correlation and a valid synthetic tool-call/result pair. +It retains the originating `spawn_agent` or routed activation attribution without executing that source call again. +The pair is framework-origin materialization, not a newly requested tool dispatch or an authorization receipt. +Do not attach a second result to the completed original provider call ID. +Provider compatibility tests must inspect the actual paired messages. + +Review can run after the active parent turn or compaction finishes. +Use a stable queue in terminal journal order. Coalesce only records with compatible original authority. +Progress and status changes do not start model turns. Terminal receipt schedules one attributed continuation. +Parent continuation failure remains visible through run status and the existing turn-failure path. +Do not automatically retry completed external effects or relaunch children. + +If enrichment fails, retain the original terminal outcome and expose a delivery warning. +The framework can admit a bounded factual result from that durable receipt without a model or unsafe path inference. +Failed persistence remains an explicit undelivered state until recovery or a recorded retry succeeds. + +### 8. Original approval prompts remain run-owned + +Create the child bridge from the immutable original turn context, not the parent's later current turn. +Keep the prompt identifier, exact child call, authorization attempt, and run ID correlated for the run lifetime. +Persist prompt lifecycle facts before display so recovery can visibly expire child prompts. +Do not persist live waiters, retry authorization objects, or actor references. +Unlike parent prompts, child prompts cannot resume an interrupted child after recovery. + +A later ordinary user message cannot abandon a child prompt merely because the start call ended. +A valid original requester can answer while the parent handles other input. +The exact retry repeats the full authorization and run-liveness checks. +Cancellation and terminal loss close the prompt. A stale approval prompt answer creates neither a retry nor a persistent authorization grant. +An answer that wins before cancellation remains subject to the later dispatch-closure gate. +The documentation term becomes **approval prompt**. Code identifiers such as `ToolApprovalRequested` retain their names. + +### 9. Recovery, upgrade, and rollback do not replay a child + +Snapshot and journal recovery preserve acceptance keys, original contexts, checkpoints, terminal receipts, and delivery markers. +After recovery, an accepted run without a durable terminal receipt becomes `Lost` exactly once. +A previously admitted cancellation remains `Cancelled` with retained evidence; recovery does not replace it with successful output. +Recovered child prompts receive visible expired/cancelled disposition. +No recovery branch recreates the child, replays task tools, or adopts the current requester. + +Captured pre-change records contain no background child ledger. +Read them as an explicit empty ledger and retain conversation, storage bindings, and parent approval state. +Do not infer accepted children from historical text or transform old unresolved synchronous tool calls into live background runs. +Use the existing healing path to close interrupted legacy calls with explicit loss. + +New journal variants need an old-reader compatibility probe. +If the prior binary cannot read them, rollback requires the pre-upgrade database backup. +Stop new admissions and settle active runs before rollback. Restoring the database cannot undo external effects. +This package authorizes no daemon restart, rollout, or data replacement. + +### 10. Blocking owner decision: provider capacity + +Root asked the owner to choose verified spare/separate capacity or runtime preemption for a single-slot provider. +The decision remains pending. Do not weaken the held-child response gate to actor responsiveness. +The actor can process controls while an inference server still delays the parent's model request. +There is no inspected runtime mechanism that reserves or preempts provider capacity. + +Before dependent runtime edits, record the selected capacity contract and its admission consequences in this package. +Do not invent a child concurrency limit or an unbounded accepted queue. +The selected contract must define accepted, queued, and rejected starts, queue bounds, and response under saturation. +The required real-model gate holds a child request and proves a parent response before child release. +An actor probe proves local control response only. It cannot satisfy this model-server gate. + +### 11. Proof and delivery boundary + +PR 2 depends on PR 1. Validate the combined stack, not the old baseline alone. +The runtime, every routed consumer, cancellation, prompt lifetime, and recovery form one coherent PR. +Do not release the intermediate state that frees the parent but loses results or control. + +Reuse actor persistence fixtures, provider barriers, session subscribers, and the existing background/cycle eval relays. +Add child-request correlation to the existing relay before it claims background-agent coverage. +Use actual dispatch counts, persisted state, path contents, and prompt settlement as assertions. +Keep scripted protocol evidence separate from real-model evidence. +Keep the inherited PR 1 recurrence gates. This change cannot reset their state through child-result admission. + +Use paired controls and fault injection at acceptance, closure, terminal receipt, enrichment, continuation admission, and restart. +Require five successful trials per critical real-model case on the owner-authorized target. +Do not put that target or credentials in commit messages or PR descriptions. +Require independent review of integrated evidence before the background behavior is called verified. + +## Risks / Trade-offs + +- [Single-slot starvation] -> Resolve the owner capacity decision; retain the held-child model gate as unpassed. +- [Acceptance acknowledgement loss duplicates work] -> Persist one stamped start key and reject digest conflicts. +- [A late dispatch follows cancellation] -> Acknowledge closure only after all local dispatch paths close. +- [Cancellation loses useful results] -> Commit checkpoints during normal work and preserve them without another model call. +- [A late success wins after cancellation] -> Select terminal precedence by durable owner order. +- [A crash loses terminal delivery] -> Atomically admit the dedup marker and continuation item. +- [A result creates trusted automation authority] -> Materialize attributed tool-origin content under the original context. +- [A later speaker answers an approval prompt] -> Compare the live prompt with its immutable original requester. +- [Cancelled output masquerades as success] -> Keep failed wire outcome and explicit cancelled lifecycle state. +- [Old readers reject new records] -> Prove compatibility or use the pre-upgrade backup procedure. +- [Days-long claims exceed implementation] -> State that this child cannot resume after restart or compact its context. + +## Migration Plan + +1. Root reviews this package and resolves provider capacity. Retain the accepted five-second framework-only grace default. +2. Rebase this branch on PR 1 and reconcile its frozen wire and detector contracts. +3. Prepare independent lifecycle fixtures and captured pre-change records before runtime edits. +4. Implement the complete background contract with the minimum runtime guidance. +5. Run targeted deterministic checks, fault controls, compatibility probes, and real-model cases. +6. Review missing evidence and false failures before merge approval. +7. Sync accepted specification deltas through the OpenSpec skill. +8. Prepare a database backup and approved rollback procedure before a separately authorized rollout. diff --git a/openspec/changes/background-child-runs/proposal.md b/openspec/changes/background-child-runs/proposal.md new file mode 100644 index 000000000..9f9e4c17f --- /dev/null +++ b/openspec/changes/background-child-runs/proposal.md @@ -0,0 +1,56 @@ +## Why + +A child currently holds its start call until completion and delays the parent's response to later input. +The approved second PR gives the parent durable child ownership, explicit controls, and later results. + +Source requirements: [PRD-001](../../../docs/prd/PRD-001-netclaw-mvp.md), FR-001, FR-002, FR-003, FR-005, FR-007, FR-011, and FR-017. +Routed scheduled skills also trace to FR-012 and [PRD-008](../../../docs/prd/PRD-008-scheduling-and-periodic-tasks.md). +Use the [engineering glossary](../../../docs/spec/GLOSSARY.md) for shared terms. + +## What Changes + +- **BREAKING:** `spawn_agent` and every routed skill activation return accepted run identifiers instead of final child output. +- Persist child acceptance, original authority, terminal receipts, delivery admission, and the identifiers that prevent duplicate work. +- Give each accepted run a lifetime independent of its start-tool token and later parent turns. +- Return later child results as attributed tool-origin content under the original authority. +- Add one deferred parent control: `check_agent_run(run_id, cancel=false)`. +- Retain the original requester for child approval prompts after the start call ends. +- Cancel task execution before a five-second framework-only grace period preserves recorded partial evidence. +- Defer idle passivation while live children or undelivered terminal results require their owner. +- Report restart loss explicitly. Do not replay interrupted child tools or relaunch lost children. +- Preserve the first PR's recurrence contract and removed tool ceilings. +- Use **approval prompt** as the glossary's displayed term. Keep existing code identifiers and the legacy anchor unchanged. + +In scope: background execution, parent status/cancel, minimum runtime instructions, lifecycle tests, and targeted subagent evals. +Out of scope: private agent messages, questions, peer discovery, steering, automatic relaunch, and general workflow infrastructure. +The third PR owns the coordination skill and general worker profile. + +## Capabilities + +### New Capabilities + +- `background-subagent-runs`: Durable start acceptance, later terminal delivery, cancellation, recovery, and their evidence gates. + +### Modified Capabilities + +- `netclaw-subagents`: Child execution ends through the owner acknowledgement contract and operation health checks. +- `skill-execution-routing`: Every routed activation uses the same background start and later-result contract. +- `session-state-machine`: Live child ownership and terminal delivery defer idle passivation. +- `tool-authorization`: Child prompts retain original authority; parent controls enforce session and role ownership. +- `progressive-tool-disclosure`: The child control stays deferred and remains unavailable to children. +- `netclaw-session`: Session context exposes attributed run status without creating a new authority source. + +## Impact + +Reuse `ChildRunScope.Authority`, `ToolRunScope`, `TurnContextRecord`, the owner-session seam, and existing session storage. +Update the spawner, child actor, parent actor, routed skill consumers, approval bridge, session state, snapshots, and framework wire schema. +Update runtime `AGENTS.md`, `subagent-authoring`, `netclaw-operations`, and the authorization architecture document in the runtime PR. +The first PR's frozen `per-call-tool-recurrence` contract is a dependency. This package does not redefine its detector. + +### Security and operational impact + +Acceptance does not grant extra tool or path authority. Child output cannot answer an approval prompt or create a trusted automation input. +New lifecycle records require upgrade and rollback proof. No actor reference, token, callback, or prepared actor property enters durable records. +The current provider router has no capacity reservation or priority scheduler. +The owner must resolve the capacity decision before implementation claims parent response under a held child request. +Five seconds is the accepted initial framework-only cancellation grace default. This change adds no operator configuration knob. diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md new file mode 100644 index 000000000..f65d87361 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -0,0 +1,420 @@ +## Purpose + +Define durable acceptance, parent controls, later child results, cancellation, and explicit restart loss for background child tasks. +Use the [engineering glossary](../../../../../docs/spec/GLOSSARY.md) for shared terms. +Source requirements: PRD-001 FR-001, FR-002, FR-003, FR-005, FR-007, FR-011, and FR-017; PRD-008 scheduled skill execution. + +## ADDED Requirements + +### Requirement: All child starts return durable acceptance + +Every explicit or routed child start SHALL return a machine-actionable accepted run identifier before the child completes. +Acceptance SHALL commit before child execution or the accepted response. +The response SHALL include the run ID, child scope ID, lifecycle state, and the deferred control tool name. +An accepted response SHALL NOT claim that the task succeeded or expose a final result before receipt. +Pre-admission failure SHALL return an explicit rejected start and create no child. +Post-admission failure SHALL retain the accepted identifier and produce a terminal failure for that run. + +#### Scenario: Parent receives acceptance while the child remains held + +- **GIVEN** an authorized child start and a child provider barrier +- **WHEN** the owner commits acceptance and the child reaches that barrier +- **THEN** the start call returns its accepted identifier before the barrier releases +- **AND** the parent can leave its tool phase and process later input + +#### Scenario: Acceptance persistence fails + +- **GIVEN** a prepared child start +- **WHEN** acceptance persistence fails +- **THEN** no child model or task tool executes +- **AND** the caller receives an explicit failure rather than a false accepted identifier + +#### Scenario: Child creation fails after acceptance + +- **GIVEN** acceptance commits before child creation +- **WHEN** child creation fails +- **THEN** the accepted run remains queryable +- **AND** one later terminal result describes the start failure + +### Requirement: Tool acceptance has one canonical JSON representation + +The tool acceptance body SHALL be one JSON object without a prose prefix, Markdown fence, or suffix. +The object SHALL contain `run_id`, `scope_id`, `state`, and `control_tool` as JSON strings. +Each required field SHALL occur exactly once. +The owner SHALL supply nonempty canonical run and scope identifier values from its committed acceptance record. +The identifier strings SHALL use `SubAgentRunId.Value` and `SubAgentScopeId.Value` from that record. +The first accepted response SHALL use `state: "Accepted"` and `control_tool: "check_agent_run"`. +That initial state SHALL describe the committed admission fact, rather than claim that the current live state cannot advance. +The adapter SHALL NOT derive identifiers or acceptance from child text, actor names, or an uncommitted proposal. +An equivalent start retry SHALL retain those identifiers and may report the owner's current recorded lifecycle state. +JSON field order and insignificant whitespace SHALL NOT change the contract. + +`spawn_agent` and routed `skill_load` SHALL expose this body as their tool result string. +`ToolResultOutput.Result` and the corresponding model `FunctionResultContent.Result` string SHALL carry the same JSON object. +The adapter SHALL format that body once and reuse it for both consumers. +Direct slash adapters SHALL use the same committed owner values for their acceptance acknowledgement. +Their human prose may differ. Their identifiers, state, and control name SHALL NOT differ. +An acceptance body SHALL NOT contain a child's final output or claim successful task completion. +The later terminal result SHALL remain a distinct owner delivery under its fresh result correlation. + +#### Scenario: A held child produces one accepted tool body + +- **GIVEN** the owner commits acceptance and the child provider remains held +- **WHEN** the start adapter returns its tool result +- **THEN** the output event and model result contain the same JSON object +- **AND** its first state is `Accepted` with canonical nonempty owner identifiers and the exact control name +- **AND** the parent does not receive the child's final output as that acceptance + +#### Scenario: A malformed or uncommitted body is not acceptance + +- **GIVEN** a result has a missing, duplicate, non-string, or empty required field, or no committed owner acceptance +- **WHEN** a start consumer validates that result +- **THEN** it reports an invalid acceptance rather than a live accepted run +- **AND** it cannot infer acceptance or launch another child from human prose or child output +- **AND** a failed acceptance write still creates no child model request or task effect + +### Requirement: Start retries preserve one durable run identity + +The runtime SHALL stamp start correlation from the original admitted input and call context. +A tool start key SHALL contain the owner session ID, original admitted turn ID, and original tool call ID. +A slash start key SHALL contain the owner session ID, admitted input ID, original turn ID, and framework activation slot zero. +A scheduled slash start SHALL reuse the admitted scheduled input and its persisted occurrence correlation. +The model SHALL NOT supply these keys or mint a replacement key during retry. +Concurrent or recovered duplicates with the same key and canonical start digest SHALL return the same run. +A conflicting digest under an accepted key SHALL fail explicitly and SHALL NOT alter that run. + +#### Scenario: A lost accepted response does not duplicate execution + +- **GIVEN** the owner commits acceptance but the response never reaches the caller +- **WHEN** the runtime retries the same stamped start +- **THEN** it returns the existing run ID +- **AND** no second child or external task effect starts + +#### Scenario: Conflicting retry fails without mutation + +- **GIVEN** one key belongs to an accepted task +- **WHEN** a retry reuses that key with different task or resolved profile content +- **THEN** the runtime returns `start_conflict` +- **AND** the accepted task and its authority remain unchanged + +#### Scenario: Two concurrent duplicates share acceptance + +- **GIVEN** two equivalent start requests arrive before the acceptance write completes +- **WHEN** the write commits +- **THEN** both callers receive the same run ID +- **AND** exactly one child starts + +### Requirement: The run lifetime is independent of its start call + +After acceptance commits, the child SHALL use a run-owned lifetime. +Start-call completion, cancellation, or timeout SHALL NOT cancel that accepted run. +A later parent turn SHALL NOT replace the run's original authority or lifetime. +Explicit run cancellation, explicit parent stop, and owner loss SHALL end that lifetime through the defined terminal contract. +No task-lifetime time, token, or tool-count ceiling SHALL be introduced by this capability. +Existing per-operation health checks and the shared recurrence contract SHALL remain effective. + +#### Scenario: Start-tool cancellation after acceptance leaves the child live + +- **GIVEN** the owner committed a child acceptance +- **WHEN** the original start token cancels or its tool response times out +- **THEN** the child remains live until a run-level terminal boundary +- **AND** parent status and cancellation remain available + +#### Scenario: A later user turn does not cancel a child + +- **GIVEN** a live child that belongs to an earlier parent turn +- **WHEN** the parent accepts a later authorized user turn +- **THEN** the child retains its original context and lifetime +- **AND** the parent does not wait for that child's completion merely because it exists + +### Requirement: Parent controls describe recorded state without widening authority + +`check_agent_run(run_id, cancel=false)` SHALL return the owner's recorded run state. +The parent caller SHALL pass normal tool policy and match the owner session and original eligible requester context. +A child or mismatched session/requester SHALL receive a denial without target details. +The control SHALL NOT route through the shell-job manager or use shell authority merely because it resembles shell-job cancellation. +Status SHALL expose cancellation admission separately from confirmed local dispatch closure. +Repeated status or cancellation requests SHALL NOT create another child, terminal result, or model continuation. + +#### Scenario: Original parent inspects its live run + +- **GIVEN** the original authorized parent requester and its accepted run +- **WHEN** the parent loads and calls the control +- **THEN** it receives the run ID, recorded state, and cancellation/dispatch-closure facts +- **AND** the call does not wait for task completion + +#### Scenario: Another session cannot inspect or cancel the run + +- **GIVEN** another session supplies a valid run identifier +- **WHEN** it attempts status or cancellation +- **THEN** the runtime reveals no target state, paths, or requester details +- **AND** the target remains live + +### Requirement: Cancellation has a durable admission and a confirmed local cutover + +The owner SHALL persist cancellation admission before it reports `Cancelling`. +It SHALL then close normal task admission, cancel active requests, and settle run-owned approval prompts. +The closure SHALL include pending tool-dispatch workers and queued provider requests. +The runtime SHALL acknowledge dispatch closure only after no new model request, task tool, or approval retry can start. +An admission acknowledgement SHALL NOT claim that dispatch closure already occurred. +Already launched remote work that lacks a confirmed outcome SHALL remain explicitly unknown. + +#### Scenario: Cancellation admission precedes closure + +- **GIVEN** a live child with a held local dispatch worker +- **WHEN** cancellation admission commits before that worker acknowledges closure +- **THEN** status reports `Cancelling` with dispatch closure unconfirmed +- **AND** the runtime does not claim complete task termination + +#### Scenario: No fresh dispatch follows closure acknowledgement + +- **GIVEN** the runtime acknowledges local dispatch closure +- **WHEN** stale provider output or a late approval answer attempts more task work +- **THEN** no new tool, provider request, retry, or grant is created +- **AND** the recorded cancellation outcome remains effective + +#### Scenario: Remote work ignores cancellation + +- **GIVEN** a remote operation started before cancellation cutover +- **WHEN** that operation ignores cancellation and provides no final receipt +- **THEN** local settlement completes without waiting for remote cooperation +- **AND** the result identifies the unconfirmed external effect + +### Requirement: Cancellation preserves recorded partial evidence through framework-only finalization + +The runtime SHALL retain confirmed results and file activity during normal execution after each completed feedback round. +After local dispatch closure, finalization SHALL have a separate five-second grace deadline by default. +Finalization SHALL use a token independent of the cancelled task token. +It SHALL permit only framework-owned checkpoint preservation and atomic writes inside the assigned run artifact directory. +It SHALL NOT invoke a model, task tool, external request, project edit, or approval prompt action. +Grace expiry or report-write failure SHALL retain the last committed checkpoint and expose an explicit finalization reason. +The result SHALL distinguish useful partial evidence from successful task completion. + +#### Scenario: Cancellation returns useful recorded evidence + +- **GIVEN** a child completed useful file operations and recorded a checkpoint +- **WHEN** the parent cancels it +- **THEN** one cancelled terminal result retains that confirmed activity and valid artifact references +- **AND** no new model request occurs to produce the report + +#### Scenario: Report write fails + +- **GIVEN** cancellation finalization attempts an atomic run-local report write +- **WHEN** the write fails +- **THEN** the runtime returns retained checkpoint evidence with an explicit write-failure reason +- **AND** it does not expose a nonexistent report path + +#### Scenario: Finalization does not finish within grace + +- **GIVEN** a held framework finalization operation +- **WHEN** the five-second grace deadline expires +- **THEN** one cancelled terminal result uses the retained checkpoint +- **AND** late finalization cannot replace the terminal winner or start task work + +### Requirement: Terminal precedence follows durable owner order + +The owner SHALL accept terminal messages only from the recorded child generation. +The first committed terminal receipt SHALL win over later cancellation admission. +Cancellation admission that commits first SHALL prevent later success from becoming the terminal outcome. +The owner SHALL preserve safe evidence from a late child response without changing the cancelled outcome. +The child SHALL close task dispatch and retain its result until the owner acknowledges durable terminal receipt. +Duplicate terminal messages SHALL receive the same acknowledgement without another delivery admission. +Persistence failure SHALL NOT produce a successful terminal acknowledgement. + +#### Scenario: Completion wins the durable race + +- **GIVEN** a valid completion receipt commits before cancellation admission +- **WHEN** a later cancel request arrives +- **THEN** the completed outcome remains the terminal winner +- **AND** the cancel request does not create a second terminal result + +#### Scenario: Cancellation wins the durable race + +- **GIVEN** cancellation admission commits before a success receipt +- **WHEN** the late success arrives +- **THEN** the terminal outcome remains cancelled +- **AND** safe recorded evidence can remain available as partial evidence + +#### Scenario: Terminal receipt persistence fails + +- **GIVEN** a child waits for its terminal acknowledgement +- **WHEN** the owner's terminal persistence fails +- **THEN** no successful acknowledgement is emitted +- **AND** recovery or owner loss remains explicit rather than false delivery success + +### Requirement: Cancelled results preserve wire compatibility without false success + +A cancelled result SHALL keep `Success = false`, wire outcome `Failed`, and reason `CancelledByParent`. +Its recorded lifecycle state SHALL be `Cancelled`. +Its typed completion SHALL carry confirmed file activity without claiming successful completion. +The owner SHALL expose only validated run locations and safe result paths under ordinary path policy. +Child-authored path claims SHALL NOT supply access authority, an approval prompt answer, or unverified file activity. +A lost run SHALL have a failed wire outcome and an explicit owner-loss reason. + +#### Scenario: Cancelled evidence does not become successful output + +- **GIVEN** a cancelled child wrote a valid artifact before cutover +- **WHEN** the result is serialized and decoded +- **THEN** its cancellation state, failed wire outcome, and confirmed artifact survive +- **AND** no consumer interprets it as successful task completion + +#### Scenario: A forged result path supplies no authority + +- **GIVEN** a child result claims an unrelated protected path +- **WHEN** the owner validates the result and the parent attempts a read +- **THEN** the claim creates neither confirmed activity nor a path grant +- **AND** the ordinary read policy rejects unauthorized access + +### Requirement: One durable terminal delivery admits one attributed continuation + +Each run SHALL have one deterministic terminal delivery identifier. +The owner SHALL persist the terminal receipt before asynchronous enrichment. +It SHALL retain that receipt if enrichment fails and expose an explicit delivery warning. +Continuation admission SHALL atomically persist both its dedup marker and its input/transcript item. +Recovery SHALL restore an admitted unfinished continuation without admitting it again. +Completed continuation input SHALL NOT be reintroduced by duplicate result or recovery replay. +An absent persistence acknowledgement SHALL remain an explicit undelivered state. + +#### Scenario: Crash before continuation admission preserves the result + +- **GIVEN** the terminal receipt is durable but continuation admission is absent +- **WHEN** the owner recovers +- **THEN** it uses the recorded result and the same delivery identifier +- **AND** it admits one continuation without restarting the child + +#### Scenario: Crash after admission does not duplicate input + +- **GIVEN** continuation admission commits and its model turn remains unfinished +- **WHEN** the owner recovers +- **THEN** it restores that pending continuation +- **AND** it creates neither another admission nor another child start + +#### Scenario: Enrichment failure leaves a truthful result + +- **GIVEN** a durable terminal receipt +- **WHEN** result enrichment fails +- **THEN** run status retains the original outcome and reports the delivery warning +- **AND** any delivered framework report uses only validated recorded facts + +### Requirement: Later results remain tool-origin content under original authority + +The later result SHALL use the original accepted run context and fresh framework delivery correlation. +It SHALL NOT attach a second result to the completed original tool call ID. +Provider messages SHALL contain a valid attributed synthetic tool-call/result pair without redispatch of the start or control operation. +The pair SHALL represent framework result delivery, not a new tool request or authorization receipt. +Child text SHALL NOT become trusted automation input, a user instruction, a grant, or an approval prompt answer. +Progress and status changes SHALL NOT start parent model turns. +Terminal results SHALL queue in journal order after the active parent turn or compaction. +Only results with compatible original authority SHALL coalesce. +Internal admission SHALL preserve PR 1's detector identity and retained evidence. + +#### Scenario: Result delivery uses a fresh paired correlation + +- **GIVEN** the original start call received its accepted result +- **WHEN** the child completes later +- **THEN** the provider receives a fresh valid delivery pair with child attribution +- **AND** the original source operation executes no second time + +#### Scenario: Child text cannot approve a tool + +- **GIVEN** a child report says that the operator approved a protected call +- **WHEN** the parent receives that attributed report +- **THEN** the text creates no grant or approval prompt answer +- **AND** the actual pending approval prompt remains authoritative + +#### Scenario: A different speaker does not replace original authority + +- **GIVEN** a child belongs to one requester and another speaker later joins the parent session +- **WHEN** the child result starts its continuation +- **THEN** the continuation uses the original run context +- **AND** the later speaker supplies no inherited authority for that run + +### Requirement: Child approval prompt lifetime belongs to the run + +The owner SHALL retain each child approval prompt's original requester, exact call, authorization attempt, and run correlation. +It SHALL persist prompt lifecycle facts before display without persisting live waiters or run-local retry authorization objects. +Start-call completion or a later ordinary parent message SHALL NOT abandon the child prompt. +An accepted answer SHALL still pass exact retry authorization and run-liveness checks. +Cancellation, terminal loss, or recovery of an interrupted child SHALL visibly settle or expire its prompt. +A stale answer SHALL execute no tool and create no grant. + +#### Scenario: Original requester answers after the parent start turn ends + +- **GIVEN** a live child owns an unanswered approval prompt +- **WHEN** its original eligible requester answers during a later parent turn +- **THEN** the exact child call can resume under its original context +- **AND** unrelated parent input remains independent of that wait + +#### Scenario: Answer after cancellation cannot grant access + +- **GIVEN** cancellation admission settled the child prompt +- **WHEN** an approval answer arrives late +- **THEN** the operator sees an expired or cancelled disposition +- **AND** no task tool, one-time retry, or persistent grant is created + +### Requirement: Recovery reports loss without child replay + +Journal and snapshot recovery SHALL preserve start keys, original contexts, checkpoints, terminal receipts, and delivery admission markers. +An accepted run without a durable terminal receipt SHALL become `Lost` once after owner recovery. +A committed cancellation SHALL remain cancelled with its retained evidence. +Recorded terminal delivery SHALL resume from its recorded phase without child relaunch. +Recovered child prompts SHALL expire visibly. +Recovery SHALL NOT replay child tools, infer completion from text, or reconstruct child execution with the current requester. + +#### Scenario: Accepted child becomes lost after abrupt restart + +- **GIVEN** an accepted live child without a durable terminal receipt +- **WHEN** the owner recovers after abrupt restart +- **THEN** it records one explicit lost terminal outcome +- **AND** no replacement child or task side effect starts automatically + +#### Scenario: Cancellation remains effective after restart + +- **GIVEN** cancellation admission committed before restart +- **WHEN** the owner recovers without a child finalization response +- **THEN** it settles a cancelled result from retained evidence +- **AND** it cannot turn a stale success into the terminal outcome + +### Requirement: Upgrade and rollback preserve explicit lifecycle limits + +The candidate SHALL load captured pre-change journals and snapshots without fabricating accepted background runs. +It SHALL retain existing conversations, parent approval state, and session storage bindings. +Interrupted legacy synchronous calls SHALL use explicit loss/healing behavior rather than background relaunch. +Rollback SHALL either prove prior-reader compatibility for new records or require the pre-upgrade database backup. +No rollback procedure SHALL claim reversal of completed external effects. + +#### Scenario: Captured old state remains readable + +- **GIVEN** a pre-change snapshot and journal with parent approval and conversation state +- **WHEN** the candidate loads that captured state +- **THEN** it preserves those existing records and uses an explicit empty child ledger +- **AND** it infers no live child from old transcript text + +#### Scenario: The previous binary cannot read a new record + +- **GIVEN** a compatibility probe demonstrates that the previous binary rejects a new record +- **WHEN** rollback instructions are reviewed +- **THEN** they require the pre-upgrade database backup and active-run settlement +- **AND** they retain the warning that external effects remain + +### Requirement: Responsiveness evidence reaches the provider boundary + +The release SHALL prove a parent model response before release of a held child provider request under the selected supported capacity contract. +An actor acknowledgement alone SHALL NOT satisfy that model-response claim. +The capacity contract SHALL define admission outcomes and bounded queue behavior before dependent runtime implementation. +Status and cancellation SHALL remain locally usable under provider contention. +The runtime SHALL NOT silently add a provider scheduler, require unproven spare capacity, or weaken the response claim. + +#### Scenario: Held child permits a parent model response + +- **GIVEN** the selected supported provider configuration and a held child request +- **WHEN** the user submits a parent status question +- **THEN** the parent returns its model response before child release +- **AND** evidence records both provider requests and their barrier order + +#### Scenario: Single-slot starvation remains an unpassed gate + +- **GIVEN** a provider permits only the held child request +- **WHEN** no chosen contract supplies parent capacity +- **THEN** the model-response gate remains unpassed +- **AND** local actor control success cannot replace that evidence diff --git a/openspec/changes/background-child-runs/specs/netclaw-session/spec.md b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md new file mode 100644 index 000000000..7e6fe97e5 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Parent context carries recorded child run status + +The owning session SHALL include a bounded status summary for its accepted and undelivered child runs in parent context. +The summary SHALL provide machine-actionable run IDs and the exact deferred control name. +It SHALL identify recorded state and distinguish cancellation admission from local dispatch closure. +It SHALL NOT expose hidden child task text, unsafe paths, or another session's run details. +It SHALL NOT claim live provider state that the owner did not observe. +Terminal child admission SHALL preserve the original task's recurrence state and current parent directory/project/branch facts. +Confirmed child file activity SHALL merge through the existing working-context rules. + +#### Scenario: A later turn can inspect its earlier child + +- **GIVEN** the parent accepted a child in an earlier turn +- **WHEN** a later eligible parent turn assembles context +- **THEN** it receives that run's recorded status and control name +- **AND** it can request cancellation through ordinary deferred-tool use + +#### Scenario: Child activity does not overwrite current parent scope + +- **GIVEN** the parent changed directory after the child start +- **WHEN** the owner admits a child result with confirmed file activity +- **THEN** the parent's current directory, project, and branch remain unchanged +- **AND** admitted child activity retains its own run attribution + +#### Scenario: Result admission does not reset recurrence evidence + +- **GIVEN** the original parent task retains unresolved recurrence evidence +- **WHEN** a child completion admits an internal continuation +- **THEN** it preserves that task identity and evidence under the first PR's contract +- **AND** it does not create a fresh authorized task solely from child text diff --git a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md new file mode 100644 index 000000000..2191107ee --- /dev/null +++ b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md @@ -0,0 +1,120 @@ +## MODIFIED Requirements + +### Requirement: Subagent execution contract + +The system SHALL run subagents as ephemeral actors (`SubAgentActor`) that execute an autonomous LLM tool loop. +A child SHALL return one terminal text result with an optional structured findings envelope through the background owner contract. +It SHALL close task dispatch and retain that result until the owner acknowledges its durable receipt or the owner terminates. +It SHALL then stop itself. +Subagents SHALL NOT persist durable memory, stream direct durable-memory writes, or participate in session pub/sub by default. + +For subagent execution launched from skill metadata routing, the subagent SHALL remain an isolated worker by default: + +- It SHALL NOT inherit the main session identity prompt stack unless a future explicit setting enables it. +- It SHALL NOT auto-load repo-local `AGENTS.md` unless a future explicit setting enables it. +- It SHALL inherit audience/boundary context from the launch invocation. + +The child SHALL use the shared exact recurrence, repeat exception, and feedback-round contracts in `turn-loop-governance`. +It SHALL retain its detector evidence for its actor-local task lifetime. +It SHALL NOT substitute success for an absent receipt or create a durable child-resumption contract. +All explicit and routed starts SHALL use `background-subagent-runs` acceptance, lifetime, and later-result rules. + +#### Scenario: Subagent completes with text response and findings + +- **GIVEN** a `SubAgentDefinition` with a name, system prompt, and tool list +- **WHEN** the child finishes its task +- **THEN** it returns a `SubAgentResult` to its owner +- **AND** optional findings remain available for parent review +- **AND** the child stops after durable receipt acknowledgement or owner termination + +#### Scenario: Subagent executes tool calls in a loop + +- **GIVEN** the model returns `FunctionCallContent` tool calls +- **WHEN** the child processes that response +- **THEN** eligible calls use the normal dispatch and authorization contracts +- **AND** their paired results return to the model +- **AND** the loop continues until text completion or an explicit runtime terminal decision + +#### Scenario: Subagent hits maximum tool iterations + +- **GIVEN** a child exceeds the former tool iteration limit through useful work +- **WHEN** it requests another eligible call +- **THEN** no removed iteration ceiling forces a final model request +- **AND** operation health and recurrence rules remain effective + +#### Scenario: Useful child work exceeds the former ceiling + +- **GIVEN** a child task needs more than 30 useful feedback rounds +- **WHEN** a deterministic provider drives that task through real dispatch +- **THEN** no count-based stop ends the task +- **AND** independent artifact checks prove its required result + +#### Scenario: Child recurrence settles without another model response + +- **GIVEN** the shared recurrence contract requires terminal settlement +- **WHEN** the child settles that run +- **THEN** it returns `Partial` with `ToolCycleStopped` and confirmed activity +- **AND** it requests no final model response +- **AND** the owner persists and delivers that terminal result through the background contract + +#### Scenario: Default subagent cannot write durable memory directly + +- **GIVEN** a default child executes within a user-facing session +- **WHEN** it attempts a direct durable cross-session memory write +- **THEN** that path remains unavailable or denied +- **AND** the child returns findings for parent policy review instead + +#### Scenario: Routed subagent does not inherit main identity prompt stack + +- **GIVEN** a slash skill routes through `metadata.subagent` +- **WHEN** its isolated child prompt is assembled +- **THEN** the main session identity stack remains absent by default + +#### Scenario: Routed subagent does not auto-load repo AGENTS + +- **GIVEN** a slash skill routes through `metadata.subagent` +- **WHEN** its isolated child prompt is assembled +- **THEN** repo-local `AGENTS.md` remains absent by default + +#### Scenario: Routed subagent inherits launch audience + +- **GIVEN** a routed launch has audience `Team` +- **WHEN** the child executes a task tool +- **THEN** that tool uses inherited `Team` authority +- **AND** background lifetime does not widen its audience + +### Requirement: Subagent timeout enforcement + +The system SHALL enforce child operation health checks, not a wall-clock task-lifetime timeout. +The checks SHALL distinguish first substantive output, inter-delta inactivity, and content-free keepalives. +A child stall SHALL return an explicit failed terminal result through the background owner contract. +An authorized human approval wait SHALL remain independent of operation inactivity and remain cancellable. +Healthy task duration alone SHALL NOT cause terminal timeout. + +#### Scenario: Subagent times out + +- **GIVEN** a child provider operation has an applicable inactivity deadline +- **WHEN** the operation exceeds that deadline without the required progress +- **THEN** the child returns a failed terminal result with the corresponding timeout reason +- **AND** the owner records it and the child stops after receipt acknowledgement + +#### Scenario: LLM call failure returns failure result + +- **GIVEN** a child model call throws an exception +- **WHEN** the child processes that failure +- **THEN** it returns an explicit failed result +- **AND** background delivery preserves the failure rather than success + +#### Scenario: Healthy child outlives one operation deadline + +- **GIVEN** a child repeatedly completes healthy model and tool operations +- **WHEN** total task duration exceeds any individual operation deadline +- **THEN** task age alone does not stop the child +- **AND** individual stalled operations still trigger their health checks + +#### Scenario: A live approval wait does not become a stall + +- **GIVEN** a child waits on its run-owned approval prompt +- **WHEN** operation inactivity would otherwise expire +- **THEN** the legitimate approval wait remains open +- **AND** explicit run cancellation can still settle it diff --git a/openspec/changes/background-child-runs/specs/progressive-tool-disclosure/spec.md b/openspec/changes/background-child-runs/specs/progressive-tool-disclosure/spec.md new file mode 100644 index 000000000..341f0f846 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/progressive-tool-disclosure/spec.md @@ -0,0 +1,31 @@ +## ADDED Requirements + +### Requirement: Parent child control remains deferred and absent from child exposure + +`check_agent_run` SHALL use Deferred registration and normal audience policy. +Accepted-run context SHALL name it so the parent can call `load_tool` by its exact name. +Start acceptance SHALL NOT automatically expose its schema or change the bounded core set. +Children SHALL NOT discover, load, inherit, or directly dispatch this parent control. +Loading SHALL NOT grant run ownership, requester eligibility, or path access. +This change SHALL NOT expose agent message, child question, steering, or peer tools. + +#### Scenario: Parent loads one known control + +- **GIVEN** a parent receives accepted-run context with `check_agent_run` +- **WHEN** it loads that exact permitted name +- **THEN** the next parent request can use that control schema +- **AND** unrelated deferred schemas remain absent + +#### Scenario: Acceptance does not expand core exposure + +- **GIVEN** a parent starts a child +- **WHEN** acceptance commits +- **THEN** the initial core name snapshot remains unchanged +- **AND** the control stays absent until explicit load under normal policy + +#### Scenario: Child cannot discover or dispatch parent control + +- **GIVEN** a child searches, loads, or directly calls `check_agent_run` +- **WHEN** exposure or dispatch evaluates the child policy +- **THEN** search/load reveals no hidden schema or existence detail +- **AND** direct invocation cannot inspect or cancel any run diff --git a/openspec/changes/background-child-runs/specs/session-state-machine/spec.md b/openspec/changes/background-child-runs/specs/session-state-machine/spec.md new file mode 100644 index 000000000..d4f0766f8 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/session-state-machine/spec.md @@ -0,0 +1,112 @@ +## MODIFIED Requirements + +### Requirement: Passivating behavior + +The session actor SHALL enter `Passivating` after idle timeout only when no subscriber, live child, or unadmitted child result requires it. +A live child includes accepted, running, cancelling, and terminal-acknowledgement states that still need their owner. +An admitted pending result SHALL remain protected through the ordinary pending-input lifecycle. +After those obligations end, normal idle passivation SHALL resume. + +In `Passivating`, the actor SHALL request final memory distillation from the observer actor when present. +It SHALL wait up to five seconds, save a snapshot, notify the lifecycle observer, and stop itself. +Idle passivation SHALL retain the post-snapshot grace window that lets racing input abort the stop and return to `Ready`. +Child ownership SHALL NOT alter shell-job reap semantics. + +#### Scenario: Idle timeout triggers passivation with no subscribers + +- **GIVEN** the session is `Ready` with no subscriber or remaining child obligation +- **WHEN** `ReceiveTimeout` fires +- **THEN** it transitions to `Passivating` +- **AND** it requests final distillation when an observer exists + +#### Scenario: Idle timeout deferred when subscribers active + +- **GIVEN** the session is `Ready` with active subscribers +- **WHEN** `ReceiveTimeout` fires +- **THEN** it remains `Ready` +- **AND** it does not start idle passivation + +#### Scenario: Passivation completes after distillation + +- **GIVEN** the session is `Passivating` +- **WHEN** final distillation completes +- **THEN** it saves a snapshot and notifies the lifecycle observer +- **AND** it stops itself + +#### Scenario: Passivation completes on timeout + +- **GIVEN** the session is `Passivating` +- **WHEN** five seconds pass without final distillation completion +- **THEN** it saves a snapshot and stops +- **AND** it does not wait indefinitely for the observer + +#### Scenario: Passivation without observer actor + +- **GIVEN** the session has no memory observer +- **WHEN** eligible idle passivation starts +- **THEN** it saves a snapshot and stops through the existing grace contract +- **AND** it requests no final distillation + +#### Scenario: Messages buffered during passivation + +- **GIVEN** idle passivation remains inside its interruptible grace window +- **WHEN** a user message arrives +- **THEN** the session aborts that idle stop and returns to `Ready` +- **AND** it handles the message under normal input rules + +#### Scenario: Live child prevents idle owner loss + +- **GIVEN** the session has no subscribers but owns a live background child +- **WHEN** idle timeout fires +- **THEN** the owner remains available for terminal receipt, prompts, status, and cancellation +- **AND** it cannot stop merely because the original start turn ended + +#### Scenario: Undelivered result prevents idle loss + +- **GIVEN** a terminal child receipt is durable but continuation admission remains pending +- **WHEN** idle timeout fires +- **THEN** the owner retains its delivery obligation +- **AND** normal idle passivation resumes after admission and ordinary pending-input settlement + +### Requirement: Restart-drain passivation is non-interruptible + +When coordinated restart requests drain, the actor SHALL reject new work and complete its current parent turn or compaction. +It SHALL also cancel accepted live children through the run cancellation contract and bounded framework-only finalization. +It SHALL retain child terminal and prompt-settlement records before passivation whenever the owner remains operational. +Abrupt owner loss SHALL use the explicit recovery-loss contract instead. +Once restart-drain reaches `Passivating`, new user messages and approval prompt answers SHALL NOT abort shutdown. + +#### Scenario: Restart drain finishes current turn before passivating + +- **GIVEN** the parent is `Processing` or `Compacting` +- **WHEN** coordinated restart requests drain +- **THEN** it rejects new work and completes its current parent phase +- **AND** accepted children settle through cancellation rather than an unbounded lifetime wait + +#### Scenario: Restart-drain passivation rejects racing inbound work + +- **GIVEN** restart-drain reached `Passivating` +- **WHEN** new user input or an approval prompt answer arrives +- **THEN** it does not abort shutdown +- **AND** a settled child prompt cannot resume task work + +#### Scenario: Response in Compacting is buffered and replayed + +- **GIVEN** an ordinary live approval response arrives during `Compacting` +- **WHEN** the response is admitted under the existing run or parent prompt authority +- **THEN** its safe response transition respects the existing compaction boundary +- **AND** it does not redispatch a parent batch or cancelled child mid-compaction + +#### Scenario: Unknown call id does not transition phase + +- **GIVEN** the session has no live or reconstructable prompt for a call ID +- **WHEN** an answer arrives +- **THEN** the actor retains its phase +- **AND** it reports an expired approval prompt without tool execution + +#### Scenario: Coordinated restart settles a live child + +- **GIVEN** an accepted child still runs when drain begins +- **WHEN** the owner closes dispatch and finalization reaches its bounded terminal result +- **THEN** that cancelled result and prompt disposition remain durable +- **AND** recovery cannot automatically relaunch the child diff --git a/openspec/changes/background-child-runs/specs/skill-execution-routing/spec.md b/openspec/changes/background-child-runs/specs/skill-execution-routing/spec.md new file mode 100644 index 000000000..4c98209fb --- /dev/null +++ b/openspec/changes/background-child-runs/specs/skill-execution-routing/spec.md @@ -0,0 +1,46 @@ +## ADDED Requirements + +### Requirement: Routed activations use background acceptance and later result delivery + +Every first-party routed activation SHALL use the `background-subagent-runs` start contract. +This includes `skill_load`, slash commands, scheduled slash payloads, and reminder skill activation. +The activation SHALL validate metadata, reload and validate the target, and scan the skill before acceptance. +It SHALL return an accepted run identifier without waiting for final child output. +The later result SHALL preserve skill, profile, original input, and original authority attribution. +The skill body SHALL remain an additive child system overlay on the routed path. +Inline activation and deterministic failure without inline fallback SHALL remain unchanged. + +#### Scenario: Tool-driven routed skill returns acceptance + +- **GIVEN** an authorized routed skill and a non-empty task +- **WHEN** `skill_load` activates that skill +- **THEN** it returns the accepted run identifier before child completion +- **AND** the later child result arrives through the owner's attributed continuation + +#### Scenario: Slash activation frees the parent phase + +- **GIVEN** a valid slash skill with `metadata.subagent` +- **WHEN** its child acceptance commits +- **THEN** the parent emits an acceptance reply and completes the slash input +- **AND** a later user input does not wait for that child merely because the skill routed + +#### Scenario: Scheduled activation preserves its original correlation + +- **GIVEN** an admitted scheduled slash payload +- **WHEN** activation retries after its accepted response disappears +- **THEN** it uses the original admitted occurrence and activation key +- **AND** it creates no second child + +#### Scenario: Invalid route cannot become inline work + +- **GIVEN** an unknown target, invalid metadata, or rejected skill content +- **WHEN** routed activation runs its pre-admission checks +- **THEN** it returns the existing explicit failure with useful remediation +- **AND** neither a child nor inline execution starts + +#### Scenario: A loaded routed skill retains its overlay + +- **GIVEN** a scanned routed skill body +- **WHEN** its background child starts +- **THEN** the body remains an additive system overlay +- **AND** acceptance does not inject that body into the parent's inline path diff --git a/openspec/changes/background-child-runs/specs/tool-authorization/spec.md b/openspec/changes/background-child-runs/specs/tool-authorization/spec.md new file mode 100644 index 000000000..2537b8589 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/tool-authorization/spec.md @@ -0,0 +1,80 @@ +## RENAMED Requirements + +- FROM: `TA-12 Subagent consent goes through the parent session` +- TO: `TA-12 Subagent approval prompts go through the parent session` + +## MODIFIED Requirements + +### Requirement: TA-12 Subagent approval prompts go through the parent session + +A subagent SHALL request an approval prompt through its owner session with the existing computed policy request contract. +The prompt SHALL retain the original parent requester, inherited working scope, exact child call, authorization attempt, and accepted run correlation. +The child SHALL use the parent's chat authorization grants. Its private grants SHALL NOT reach the parent. + +- Prompt lifecycle facts SHALL persist before display; the execution waiter remains run-local. +- Start-tool completion or a later ordinary parent message SHALL NOT abandon a live child prompt. +- A restart cannot resume an interrupted child. Its old prompt SHALL expire visibly under the loss contract. +- Cancellation or terminal loss SHALL settle the prompt. A late answer SHALL create no retry or authorization grant. +- A child without a safe parent bridge SHALL fail with `ToolExecutionFailed` rather than synthesize a requester. +- A one-time authorization grant SHALL cover only the exact prompted retry and remain subject to launch and run-liveness checks. +- Background acceptance and child text SHALL create no approval authority. + +#### Scenario: Subagent prompt reaches the parent requester + +- **GIVEN** a live child belongs to an interactive Personal parent run +- **WHEN** it requests a protected shell call +- **THEN** the parent channel displays an approval prompt for the original eligible requester +- **AND** a valid answer can resume the exact child call under normal authorization + +#### Scenario: Subagent prompt expires after restart + +- **GIVEN** an unanswered child prompt and an interrupted child after restart +- **WHEN** the original requester answers the old prompt +- **THEN** the system reports an expired prompt +- **AND** no child call or new authorization grant results + +#### Scenario: Another speaker cannot answer the original prompt + +- **GIVEN** a child prompt belongs to one original requester +- **WHEN** another speaker joins a later parent turn and answers that prompt +- **THEN** the answer cannot replace the recorded requester +- **AND** the protected child call remains unexecuted + +#### Scenario: Start completion does not end a live prompt + +- **GIVEN** the start call returned accepted and the child later requests approval +- **WHEN** the eligible requester answers through the parent +- **THEN** the run-owned wait remains available for exact retry authorization +- **AND** its lifetime does not depend on the completed start tool + +## ADDED Requirements + +### Requirement: TA-17 Child controls enforce parent session and original requester ownership + +The deferred child control SHALL require an authorized parent invocation in the owning session and original eligible requester context. +It SHALL use the existing original authority record, rather than parallel audience, requester, or path-root fields. +It SHALL be unavailable to children through discovery, schema load, and direct dispatch. +Unauthorized requests SHALL reveal no target existence, state, task text, result paths, or requester details. +Cancellation SHALL acknowledge local cutover separately from admission and SHALL create no tool or path grant. +Child result locations SHALL remain data for ordinary file access, not access authority. + +#### Scenario: Eligible parent cancels its child + +- **GIVEN** an authorized parent invocation under the original eligible requester context +- **WHEN** it cancels its own accepted run +- **THEN** the owner admits cancellation and later reports local dispatch closure +- **AND** no broader shell or path authority results + +#### Scenario: Child cannot cancel a sibling + +- **GIVEN** a child knows a sibling run identifier +- **WHEN** it attempts direct child-control dispatch +- **THEN** the runtime denies the operation without target details +- **AND** the sibling remains live + +#### Scenario: Known identifier does not authorize cross-session status + +- **GIVEN** another session obtains a child identifier from text +- **WHEN** it requests status or cancellation +- **THEN** the runtime returns a non-disclosing denial +- **AND** no state mutation or result access occurs diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md new file mode 100644 index 000000000..96c0831e4 --- /dev/null +++ b/openspec/changes/background-child-runs/tasks.md @@ -0,0 +1,85 @@ +## 1. Freeze prerequisites and independent proof fixtures + +- [ ] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract remains unchanged. +- [ ] 1.2 Record the owner's pending capacity decision before dependent runtime edits; verify the selected admission and held-child model-response clauses are explicit. +- [ ] 1.3 Have an independent test author derive lifecycle assertions from the delta; verify the author lists paired controls and deliberate faulty variants. +- [ ] 1.4 Extend existing actor fixtures with start, provider, dispatch, terminal, and persistence barriers; verify assertions use acknowledgements rather than sleeps. +- [ ] 1.5 Capture a pre-change journal/snapshot fixture with conversation and parent approval state; verify the fixture predates new lifecycle records. + +## 2. Durable acceptance and independent lifetime + +- [ ] 2.1 Extend the owner-session seam with typed accepted-run operations and reuse `ChildRunScope.Authority`; verify no parallel requester or path-root inputs exist. +- [ ] 2.2 Stamp tool and routed activation keys before asynchronous setup; verify retries preserve original call/input IDs and conflicting digests fail. +- [ ] 2.3 Add framework-owned acceptance records and snapshot conversion; verify round-trip and journal replay preserve the same key, context, and run ID. +- [ ] 2.4 Persist acceptance before execution and acknowledgement; fault the write and verify zero child requests or task effects. +- [ ] 2.5 Deduplicate concurrent and recovered starts; lose the accepted response and verify one child plus the same returned run ID. +- [ ] 2.6 Separate accepted-run lifetime from the start token; cancel that token after commit and verify the child remains controllable. +- [ ] 2.7 Report post-acceptance creation failure through the accepted run; verify one durable terminal failure and no false rejection/restart. + +## 3. Terminal receipt and attributed continuation + +- [ ] 3.1 Add child terminal acknowledgement and close task dispatch before terminal output; verify the child retains its result until durable acknowledgement. +- [ ] 3.2 Authenticate terminal sender and generation; replay stale and duplicate results and verify one committed winner. +- [ ] 3.3 Persist terminal receipt before enrichment; fail persistence and verify no successful acknowledgement, delivery claim, or model request. +- [ ] 3.4 Move result enrichment to recorded terminal handling; fail enrichment and verify the original outcome survives with an explicit warning. +- [ ] 3.5 Atomically persist delivery dedup and continuation input; crash before/after admission and verify one pending input without child replay. +- [ ] 3.6 Materialize a fresh attributed provider tool-call/result pair; inspect actual messages and verify the original start call receives no second result. +- [ ] 3.7 Preserve original authority and compatible journal-order queues; verify another speaker and active compaction cannot change result authority. +- [ ] 3.8 Keep internal result admission inside PR 1's original detector task; verify retained recurrence evidence and current parent directory/project/branch survive. +- [ ] 3.9 Keep failed parent continuations visible; verify replay does not admit completed input or automatically restart external effects. + +## 4. Every routed start and deferred parent controls + +- [ ] 4.1 Convert `spawn_agent` to the canonical acceptance JSON. Verify identical output/model bodies, valid owner identifiers, and initial `Accepted` state. Hold the child and verify parent replies before release. Reject malformed or uncommitted acceptance. +- [ ] 4.2 Convert routed `skill_load`; verify target validation, content scan, overlay semantics, and explicit failures remain intact. +- [ ] 4.3 Convert slash, scheduled slash, and reminder skill activation; verify each returns acceptance and retains its original occurrence/call correlation. +- [ ] 4.4 Preserve inline activation and no-inline-fallback failures; verify unknown/internal targets create neither a child nor inline work. +- [ ] 4.5 Add Deferred `check_agent_run(run_id, cancel=false)` through normal non-shell policy; verify explicit load, unchanged core, and direct ownership checks. +- [ ] 4.6 Deny child, cross-session, and wrong-requester controls; verify no target details and keep the valid parent control operational. +- [ ] 4.7 Add bounded recorded-run context with the exact control name; verify it cannot claim unobserved live provider state or expose hidden results. + +## 5. Cancellation, partial evidence, and approval prompts + +- [ ] 5.1 Persist cancellation admission and acknowledge local dispatch closure separately; hold a dispatch worker and verify the acknowledgement cannot precede closure. +- [ ] 5.2 Close queued model/tool dispatch and exact approval retries; release stale callbacks and verify no new task operation after confirmed cutover. +- [ ] 5.3 Commit bounded checkpoints after completed child rounds; verify confirmed receipts and paths survive a later cancel without a new model response. +- [ ] 5.4 Add five-second framework-only finalization with a separate token; verify report writes stay run-local and no task tool or project edit occurs. +- [ ] 5.5 Fault report writes and expire controlled finalization timers; verify retained checkpoint evidence, explicit reason, and one cancelled result. +- [ ] 5.6 Extend cancelled completion with confirmed activity; verify `Success=false`, `Failed`, and `CancelledByParent` survive wire conversion. +- [ ] 5.7 Exercise both durable completion/cancel orders; verify the first permitted terminal winner and accurate unconfirmed external-effect reports. +- [ ] 5.8 Retain original approval prompt correlation beyond the start call; verify an original eligible answer resumes only the exact live child retry. +- [ ] 5.9 Persist child prompt lifecycle facts and settle them on cancel/loss; verify late answers create no tool execution or authorization grant. +- [ ] 5.10 Keep child prompt waits independent of later ordinary parent input; verify that input neither adopts requester authority nor abandons the child prompt. + +## 6. Passivation, restart, and compatibility + +- [ ] 6.1 Defer idle passivation for live children and pending result admission; verify normal idle behavior resumes when those obligations end. +- [ ] 6.2 Retain shell-job reap behavior as a nearby control; run its existing integration cases and verify no child state enters shell-job records. +- [ ] 6.3 Cancel children during coordinated drain with bounded finalization; verify parent snapshot retains terminal and prompt disposition. +- [ ] 6.4 Recover accepted children as explicit `Lost` without relaunch; verify committed cancellation remains cancelled and stale success cannot win. +- [ ] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child or completed side-effect replay. +- [ ] 6.6 Load the captured pre-change fixture; verify existing history, parent prompts, and storage bindings survive with an explicit empty child ledger. +- [ ] 6.7 Probe prior-reader compatibility against new events; verify rollback instructions require backup restoration when compatibility fails. + +## 7. Integrated health, evals, and sensitive fault controls + +- [ ] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. +- [ ] 7.2 Add targeted Subagents and background cases for acceptance, routed starts, controls, cancellation, prompts, and recovery; verify actual dispatch and artifact evidence. +- [ ] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. +- [ ] 7.4 Exercise the chosen admission contract under saturation; verify bounded queue state, explicit outcomes, and locally responsive status/cancel. +- [ ] 7.5 Hold a child on the selected real provider and request a parent reply; verify that reply precedes child release in five critical-case trials. +- [ ] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. +- [ ] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. +- [ ] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. +- [ ] 7.9 Require an independent verifier to inspect integrated results and assertions; verify no skipped cases, weakened gates, or author-only pass claims remain. + +## 8. Docs, operational guidance, and handoff + +- [ ] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. +- [ ] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. +- [ ] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. +- [ ] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. +- [ ] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. +- [ ] 8.6 Run applicable builds, required tests, Slopwatch, and copyright-header verification; retain exact candidate and meaningful test counts. +- [ ] 8.7 Validate the OpenSpec change strictly and sync implemented deltas through the appropriate skill; verify the PRD and active plan trace to the final contract. +- [ ] 8.8 Prepare rollback and evidence handoff for the combined PR; verify no merge, rollout, publication, or database replacement is implied by local passes. From cd1e05d44b2cd7068f9a426314b8c42854df9b9b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 04:51:29 +0000 Subject: [PATCH 02/22] Retain parent detector evidence for child continuations --- docs/spec/GLOSSARY.md | 10 ++-- .../changes/background-child-runs/design.md | 56 ++++++++++++++++-- .../specs/background-subagent-runs/spec.md | 59 ++++++++++++++++++- .../specs/netclaw-session/spec.md | 3 + .../changes/background-child-runs/tasks.md | 23 ++++++-- 5 files changed, 136 insertions(+), 15 deletions(-) diff --git a/docs/spec/GLOSSARY.md b/docs/spec/GLOSSARY.md index 87345b633..6839a8235 100644 --- a/docs/spec/GLOSSARY.md +++ b/docs/spec/GLOSSARY.md @@ -533,11 +533,13 @@ nothing. The retry passes every check again. **Code anchors:** `OneTimeApprovalKeys`, `ToolApprovalAttempt` -### Consent request + -A consent request asks the operator to answer for the uncovered candidates of -one call. It carries the options that the operator can choose and a display -text with secrets removed. +### Approval prompt + +An approval prompt asks the operator to answer for the uncovered candidates of one tool call. +It shows the available options and text with secrets removed. +Parent and child calls use this term for the displayed request and its lifecycle. **Code anchors:** `ToolApprovalContext`, `ToolInteractionRequest` diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md index 99d3d33b7..2d2cf1605 100644 --- a/openspec/changes/background-child-runs/design.md +++ b/openspec/changes/background-child-runs/design.md @@ -1,7 +1,9 @@ ## Context -Baseline: `2e6bc4f014dc96b606566709df1bd11f90ecf34a` on `feature/background-subagents`. -The branch will receive PR 1 before implementation. Its frozen dependency is `per-call-tool-recurrence` in the sibling worktree. +The original baseline is `2e6bc4f014dc96b606566709df1bd11f90ecf34a`. +The branch now includes PR 1 at `893435238b7f18e1ba7f45e181255bab95225ad1`. +The dependency is `per-call-tool-recurrence`. Its runtime types remain unchanged by this contract clarification. +PR 1 retains its open proof gates. This rebase does not establish background runtime behavior. See [proposal.md](proposal.md) for the objective and PRD traceability. The approved [plan](../../../.systematize/plans/background-subagents/plan.html) owns delivery sequencing when that artifact is present in the combined branch. @@ -254,7 +256,8 @@ Do not attach a second result to the completed original provider call ID. Provider compatibility tests must inspect the actual paired messages. Review can run after the active parent turn or compaction finishes. -Use a stable queue in terminal journal order. Coalesce only records with compatible original authority. +Use a stable queue in terminal journal order. +Coalesce only records with compatible original authority and the same original detector task identity. Progress and status changes do not start model turns. Terminal receipt schedules one attributed continuation. Parent continuation failure remains visible through run status and the existing turn-failure path. Do not automatically retry completed external effects or relaunch children. @@ -263,6 +266,51 @@ If enrichment fails, retain the original terminal outcome and expose a delivery The framework can admit a bounded factual result from that durable receipt without a model or unsafe path inference. Failed persistence remains an explicit undelivered state until recovery or a recorded retry succeeds. +#### Parent detector evidence remains separate from child partial results + +The owner stores parent detector evidence in the already planned durable child run ledger. +Reuse `ToolLoopCheckpoint` and the existing receipt-failure fact. Do not add another task ledger or use shell-job records for children. +These facts differ from the child's partial-result checkpoint and its actor-local detector state. +The checkpoint's `TaskId` identifies the original detector task. Do not derive it from `TurnContextRecord.TurnId`. +Delivery authority can identify a later continuation while its detector task still identifies the original causal task. +`HasSameAuthority` alone cannot establish detector identity. + +Committed parent admissions and results refresh outstanding runs that belong to the same detector task. +For tool-based starts, continuation restoration waits until the original batch settles and commits its detector evidence. +A normal batch includes the start receipt and completed feedback round. +A missing mandatory receipt retains the receipt-failure fact and invokes PR 1's terminal settlement without another model request. +Direct activation retains canonical task evidence without a fabricated tool receipt. +Parent completion and fresh input retain evidence that an outstanding child continuation needs. +Sibling runs use the latest retained parent checkpoint for their task. +Canonical continuation adoption validates the accepted run and commits restored evidence before the next model request. +The synthetic delivery pair supplies no execution receipt. It cannot clear receipt failure or reopen the completed start call. + +Schematic; authorization, admission, and persistence failure paths are abbreviated: + +```text +Parent task A accepts child X + -> run ledger retains A's ToolLoopCheckpoint and receipt-failure fact +Committed A admission or result + -> refresh every outstanding run whose checkpoint TaskId matches A + -> tool start: wait for the original batch's committed settlement + -> normal settlement: retain the start receipt and completed round + -> missing mandatory receipt: retain failure; settle without another model request + -> direct activation: retain canonical task evidence without a fabricated receipt +Parent completes A; fresh user task B starts + -> retain A's evidence in X; keep B's current checkpoint separate +Child X terminal receipt + -> admit one attributed continuation through durable delivery + -> validate X; durably restore the latest retained A checkpoint + -> if evidence permits, request the model under X's original authority +Continuation completes + -> settle only this delivery; retain evidence for outstanding siblings +``` + +Positive example: two children from A share its latest committed checkpoint before either continuation requests the model. +Negative example: children from A and B cannot coalesce solely because the same requester owns both tasks. +Recovery preserves parent detector evidence without child relaunch or a fresh-task reset. +Result admission preserves the parent's current directory, project, and branch. + ### 8. Original approval prompts remain run-owned Create the child bridge from the immutable original turn context, not the parent's later current turn. @@ -280,7 +328,7 @@ The documentation term becomes **approval prompt**. Code identifiers such as `To ### 9. Recovery, upgrade, and rollback do not replay a child -Snapshot and journal recovery preserve acceptance keys, original contexts, checkpoints, terminal receipts, and delivery markers. +Snapshot and journal recovery preserve acceptance keys, original contexts, parent detector evidence, child partial checkpoints, terminal receipts, and delivery markers. After recovery, an accepted run without a durable terminal receipt becomes `Lost` exactly once. A previously admitted cancellation remains `Cancelled` with retained evidence; recovery does not replace it with successful output. Recovered child prompts receive visible expired/cancelled disposition. diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md index f65d87361..4b761db3c 100644 --- a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -305,8 +305,22 @@ The pair SHALL represent framework result delivery, not a new tool request or au Child text SHALL NOT become trusted automation input, a user instruction, a grant, or an approval prompt answer. Progress and status changes SHALL NOT start parent model turns. Terminal results SHALL queue in journal order after the active parent turn or compaction. -Only results with compatible original authority SHALL coalesce. +Only results with compatible original authority and the same original detector task identity SHALL coalesce. Internal admission SHALL preserve PR 1's detector identity and retained evidence. +The owner SHALL retain the parent `ToolLoopCheckpoint` and receipt-failure fact in the durable child run ledger. +These facts SHALL remain separate from child partial-result checkpoints and actor-local child detector state. +The checkpoint's `TaskId` SHALL identify the detector task independently from `TurnContextRecord.TurnId`. +Committed parent admissions and results SHALL refresh outstanding runs that belong to the same detector task. +For tool-based starts, continuation restoration SHALL wait until the original batch settles and commits its detector evidence. +A normal batch SHALL include the start receipt and completed feedback round. +A missing mandatory receipt SHALL retain the receipt-failure fact and invoke PR 1's terminal settlement without another model request. +Direct activation SHALL retain canonical task evidence without a fabricated tool receipt. +Parent completion and fresh input SHALL retain evidence required by outstanding child continuations. +Sibling runs SHALL use the latest retained parent checkpoint for their task. +Canonical continuation adoption SHALL validate the accepted run and commit restored evidence before the next model request. +The synthetic delivery pair SHALL NOT clear receipt failure or reopen the completed start call. +Delivery completion SHALL retain evidence required by another outstanding run. +Recovery SHALL preserve these facts without child relaunch or a fresh-task reset. #### Scenario: Result delivery uses a fresh paired correlation @@ -322,6 +336,49 @@ Internal admission SHALL preserve PR 1's detector identity and retained evidence - **THEN** the text creates no grant or approval prompt answer - **AND** the actual pending approval prompt remains authoritative +#### Scenario: Fresh parent input preserves a child's original detector task + +- **GIVEN** task A accepted a child and committed its start receipt and feedback round +- **AND** the parent completed A and admitted fresh user task B +- **WHEN** the owner adopts the child's later result continuation +- **THEN** it commits A's latest retained checkpoint and receipt-failure fact before the next model request +- **AND** it uses the child's original authority without a fresh detector task + +#### Scenario: Sibling results use the latest shared task evidence + +- **GIVEN** task A accepted two children and later committed additional detector evidence +- **WHEN** either child result starts a continuation +- **THEN** that continuation uses A's latest retained parent checkpoint +- **AND** its completion retains evidence required by the other child + +#### Scenario: Equal authority cannot merge distinct detector tasks + +- **GIVEN** one requester owns child runs from distinct detector tasks A and B +- **WHEN** their terminal results await parent continuation +- **THEN** the owner keeps separate continuation groups and parent checkpoints +- **AND** equal authority alone cannot merge or reset their detector evidence + +#### Scenario: Recovery cannot erase a retained receipt failure + +- **GIVEN** an outstanding child run retains its parent task's receipt-failure fact +- **WHEN** recovery restores that run and admits its result continuation +- **THEN** the original detector task and receipt failure remain authoritative +- **AND** the delivery pair creates no execution receipt or child relaunch + +#### Scenario: An early terminal result waits for the start batch + +- **GIVEN** a tool-based child start commits acceptance before its parent batch settles +- **WHEN** the child reports a terminal result before the original start receipt commits +- **THEN** continuation restoration waits for the original batch's committed settlement +- **AND** a missing mandatory receipt invokes terminal settlement without another model request + +#### Scenario: Direct slash activation creates no tool receipt + +- **GIVEN** direct slash activation accepted a child without a parent tool call +- **WHEN** its terminal result admits a continuation +- **THEN** the owner retains the canonical original detector evidence +- **AND** it fabricates no start receipt or completed tool feedback round + #### Scenario: A different speaker does not replace original authority - **GIVEN** a child belongs to one requester and another speaker later joins the parent session diff --git a/openspec/changes/background-child-runs/specs/netclaw-session/spec.md b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md index 7e6fe97e5..401b7cf38 100644 --- a/openspec/changes/background-child-runs/specs/netclaw-session/spec.md +++ b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md @@ -8,6 +8,9 @@ It SHALL identify recorded state and distinguish cancellation admission from loc It SHALL NOT expose hidden child task text, unsafe paths, or another session's run details. It SHALL NOT claim live provider state that the owner did not observe. Terminal child admission SHALL preserve the original task's recurrence state and current parent directory/project/branch facts. +The durable child run ledger SHALL retain parent detector checkpoints independently from the session's current task checkpoint. +Fresh input SHALL NOT replace evidence required by an outstanding child continuation. +Continuation adoption SHALL restore the latest retained original-task checkpoint before the next model request. Confirmed child file activity SHALL merge through the existing working-context rules. #### Scenario: A later turn can inspect its earlier child diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 96c0831e4..752634afd 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -1,10 +1,21 @@ ## 1. Freeze prerequisites and independent proof fixtures -- [ ] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract remains unchanged. +- [x] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract remains unchanged. - [ ] 1.2 Record the owner's pending capacity decision before dependent runtime edits; verify the selected admission and held-child model-response clauses are explicit. -- [ ] 1.3 Have an independent test author derive lifecycle assertions from the delta; verify the author lists paired controls and deliberate faulty variants. +- [x] 1.3 Have an independent test author derive lifecycle assertions from the delta; verify the author lists paired controls and deliberate faulty variants. - [ ] 1.4 Extend existing actor fixtures with start, provider, dispatch, terminal, and persistence barriers; verify assertions use acknowledgements rather than sleeps. -- [ ] 1.5 Capture a pre-change journal/snapshot fixture with conversation and parent approval state; verify the fixture predates new lifecycle records. +- [x] 1.5 Capture a pre-change journal/snapshot fixture with conversation and parent approval state; verify the fixture predates new lifecycle records. + +Preparation evidence: + +- The branch includes PR 1 at `893435238`. Its existing runtime types remain unchanged. +- Design section 7 assigns retained parent detector evidence to the planned child run ledger. +- An independent review derives sixteen assertion groups, paired controls, and named faults from the deltas. +- The [approved plan](../../../.systematize/plans/background-subagents/plan.html) owns the adversarial method and acceptance gates. +- The original `2e6bc4f` capture already supplies conversation, original requester, and parent approval facts. +- Reuse `src/Netclaw.Actors.Tests/Sessions/Fixtures/LegacyPersistence/legacy-session-v0.json`; do not recapture or infer child records. +- Its SHA-256 is `bf400d15ea1541d1a59b8be238705c99eb7f489413c863d93b17976ab1cfc94b`. +- These preparation tasks establish no background runtime or provider-capacity pass. ## 2. Durable acceptance and independent lifetime @@ -25,7 +36,7 @@ - [ ] 3.5 Atomically persist delivery dedup and continuation input; crash before/after admission and verify one pending input without child replay. - [ ] 3.6 Materialize a fresh attributed provider tool-call/result pair; inspect actual messages and verify the original start call receives no second result. - [ ] 3.7 Preserve original authority and compatible journal-order queues; verify another speaker and active compaction cannot change result authority. -- [ ] 3.8 Keep internal result admission inside PR 1's original detector task; verify retained recurrence evidence and current parent directory/project/branch survive. +- [ ] 3.8 Retain and refresh parent detector checkpoints in the child run ledger. Verify committed start settlement and direct activation without fabricated receipts. Verify late results after fresh input, sibling checkpoint updates, distinct-task separation, receipt failure, and recovery. Preserve current parent directory/project/branch. - [ ] 3.9 Keep failed parent continuations visible; verify replay does not admit completed input or automatically restart external effects. ## 4. Every routed start and deferred parent controls @@ -58,7 +69,7 @@ - [ ] 6.3 Cancel children during coordinated drain with bounded finalization; verify parent snapshot retains terminal and prompt disposition. - [ ] 6.4 Recover accepted children as explicit `Lost` without relaunch; verify committed cancellation remains cancelled and stale success cannot win. - [ ] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child or completed side-effect replay. -- [ ] 6.6 Load the captured pre-change fixture; verify existing history, parent prompts, and storage bindings survive with an explicit empty child ledger. +- [ ] 6.6 Load the captured pre-change fixture. Verify retained history, parent prompts, and an explicit empty child ledger. Capture a baseline SQLite storage binding and marker through the real resolver. Verify the candidate preserves that row and marker. The journal fixture alone supplies no storage-binding proof. - [ ] 6.7 Probe prior-reader compatibility against new events; verify rollback instructions require backup restoration when compatibility fails. ## 7. Integrated health, evals, and sensitive fault controls @@ -75,7 +86,7 @@ ## 8. Docs, operational guidance, and handoff -- [ ] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. +- [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. - [ ] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. - [ ] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. - [ ] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. From b95ea58cdb561727ba9154a8f4b5dbced849e129 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 05:02:18 +0000 Subject: [PATCH 03/22] Capture legacy storage bindings for child-run verification --- .../changes/background-child-runs/tasks.md | 5 + .../Fixtures/LegacyPersistence/README.md | 45 ++++++++ .../LegacyPersistence/capture-baseline.patch | 67 +++++++++++ .../legacy-storage-binding-v0.json | 12 ++ .../Gateway/PreChangeStorageBindingTests.cs | 106 ++++++++++++++++++ .../Netclaw.Daemon.Tests.csproj | 1 + 6 files changed, 236 insertions(+) create mode 100644 src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/README.md create mode 100644 src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/capture-baseline.patch create mode 100644 src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/legacy-storage-binding-v0.json create mode 100644 src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingTests.cs diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 752634afd..b348dadca 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -15,6 +15,11 @@ Preparation evidence: - The original `2e6bc4f` capture already supplies conversation, original requester, and parent approval facts. - Reuse `src/Netclaw.Actors.Tests/Sessions/Fixtures/LegacyPersistence/legacy-session-v0.json`; do not recapture or infer child records. - Its SHA-256 is `bf400d15ea1541d1a59b8be238705c99eb7f489413c863d93b17976ab1cfc94b`. +- A separate baseline capture supplies one real SQLite storage binding and its neutral workspace marker. +- Reuse `src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/legacy-storage-binding-v0.json` with its bundled capture source and provenance. +- The original resolver creates the captured row. The consumer restores all four columns and the marker under a new home. +- The combined candidate passes 33 storage-resolver cases without failures or skips. +- Task 6.6 remains open for the new empty child ledger and the integrated lifecycle recovery proof. - These preparation tasks establish no background runtime or provider-capacity pass. ## 2. Durable acceptance and independent lifetime diff --git a/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/README.md b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/README.md new file mode 100644 index 000000000..cdc9b026b --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/README.md @@ -0,0 +1,45 @@ +# Pre-change session storage fixture + +Baseline: `2e6bc4f014dc96b606566709df1bd11f90ecf34a`. +Session: `signalr/pre-change-persistence-proof`. +The existing conversation and approval fixture uses this same session identifier. + +The baseline migrations create the schema. The baseline `SqliteSessionStorageResolver` creates the binding. +The producer reads its actual row and writes one neutral workspace marker. +The fixture retains all four row columns, the marker's relative path, its bytes, and its SHA-256 digest. +The capture creates the binding before any journal seed, as a normal new session does. +The producer uses a fixed clock and deletes its test home after capture. +Its old absolute root names only a test-owned temporary directory. + +The consumer restores the exact row under the current schema and the marker under a different test home. +It calls the real resolver twice through separate instances. +It requires the current envelope and workspace paths, the original marker bytes, and one unchanged database row. +The consumer uses a later clock, so a replacement binding cannot pass the original-row assertion. + +Re-root semantics are explicit: the resolver maps the old `sessions/` suffix below the current sessions directory. +It retains the old absolute database value and creation time. +The consumer does not rewrite the stored row or use the old absolute path for a file write. +Both separator styles remain part of the existing resolver contract. + +Fixture SHA-256: `a3fadd7978ffbecc1667120860486c0cebbb01a46d87854ce3151f3f2bd4326a`. + +The bundled `capture-baseline.patch` adds only the capture test. +Apply it in an isolated baseline checkout after the consumer and fixture patch. +Then capture to an owned artifact path: + +```bash +NETCLAW_STORAGE_FIXTURE_OUTPUT="$PWD/artifacts/pre-change-storage/legacy-storage-binding-v0.json" \ + dotnet test src/Netclaw.Daemon.Tests/Netclaw.Daemon.Tests.csproj \ + --filter 'FullyQualifiedName~PreChangeStorageBindingCaptureTests' \ + --logger 'trx;LogFileName=baseline-storage-capture.trx' \ + --results-directory artifacts/pre-change-storage +``` + +The normal suite contains only `PreChangeStorageBindingTests`, which consumes the captured asset. +Keep the capture test outside that suite. +A new capture uses another temporary home, so its whole-file digest can differ. +Retain its baseline revision, producer source, payload digest, and actual test receipt. + +This fixture captures one actual storage row. It does not import an old complete SQLite database or plugin envelope. +It does not prove a live compaction save, old-reader compatibility, child recovery, or an empty new child ledger. +Those proof gates remain separate. diff --git a/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/capture-baseline.patch b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/capture-baseline.patch new file mode 100644 index 000000000..6661b1ce8 --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/capture-baseline.patch @@ -0,0 +1,67 @@ +diff --git a/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingCaptureTests.cs b/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingCaptureTests.cs +new file mode 100644 +index 000000000..cdd319a3c +--- /dev/null ++++ b/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingCaptureTests.cs +@@ -0,0 +1,61 @@ ++// ----------------------------------------------------------------------- ++// ++// Copyright (C) 2026 - 2026 Petabridge, LLC ++// ++// ----------------------------------------------------------------------- ++using System.Security.Cryptography; ++using System.Text; ++using System.Text.Json; ++using Microsoft.Extensions.Logging.Abstractions; ++using Microsoft.Extensions.Time.Testing; ++using Netclaw.Actors.Protocol; ++using Netclaw.Configuration; ++using Netclaw.Daemon.Gateway; ++using Netclaw.Daemon.Services; ++using Netclaw.Tests.Utilities; ++using Xunit; ++ ++namespace Netclaw.Daemon.Tests.Gateway; ++ ++public sealed class PreChangeStorageBindingCaptureTests ++{ ++ [Fact] ++ public async Task Capture_one_neutral_binding_and_marker_through_the_original_resolver() ++ { ++ var outputPath = Environment.GetEnvironmentVariable("NETCLAW_STORAGE_FIXTURE_OUTPUT") ++ ?? throw new InvalidOperationException("Set NETCLAW_STORAGE_FIXTURE_OUTPUT to an owned artifact path."); ++ using var directory = new DisposableTempDir(); ++ var paths = new NetclawPaths(directory.Path); ++ paths.EnsureDirectoriesExist(); ++ try ++ { ++ await new SchemaMigrator(paths, NullLogger.Instance) ++ .MigrateAsync(paths.SqliteDbPath, TestContext.Current.CancellationToken); ++ var session = new SessionId("signalr/pre-change-persistence-proof"); ++ var clock = new FakeTimeProvider(new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero)); ++ var storage = new SqliteSessionStorageResolver(paths, clock).Resolve(session); ++ Assert.NotNull(storage.Binding); ++ var row = CapturedStorageRow.Read(paths); ++ Assert.Equal(storage.Binding.EnvelopeRoot.Value, row.EnvelopeRoot); ++ Assert.Equal(clock.GetUtcNow().ToUnixTimeMilliseconds(), row.CreatedAt); ++ Directory.CreateDirectory(storage.SessionDirectory.Value); ++ var markerPath = Path.Combine(storage.SessionDirectory.Value, "neutral-marker.txt"); ++ await File.WriteAllBytesAsync(markerPath, Encoding.UTF8.GetBytes("The neutral session marker remains.\n"), ++ TestContext.Current.CancellationToken); ++ var markerBytes = await File.ReadAllBytesAsync(markerPath, TestContext.Current.CancellationToken); ++ var markerRelativePath = Path.GetRelativePath(row.EnvelopeRoot, markerPath).Replace(Path.DirectorySeparatorChar, '/'); ++ var fixture = new CapturedStorageFixture( ++ "2e6bc4f014dc96b606566709df1bd11f90ecf34a", row, markerRelativePath, ++ Convert.ToBase64String(markerBytes), Convert.ToHexString(SHA256.HashData(markerBytes))); ++ Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputPath)) ++ ?? throw new InvalidOperationException("The fixture needs a parent directory.")); ++ await File.WriteAllTextAsync(outputPath, ++ JsonSerializer.Serialize(fixture, new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true }) + "\n", ++ TestContext.Current.CancellationToken); ++ } ++ finally ++ { ++ SqliteTestPools.Clear(paths); ++ } ++ } ++} diff --git a/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/legacy-storage-binding-v0.json b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/legacy-storage-binding-v0.json new file mode 100644 index 000000000..c7f4319fb --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Gateway/Fixtures/LegacyPersistence/legacy-storage-binding-v0.json @@ -0,0 +1,12 @@ +{ + "baseline": "2e6bc4f014dc96b606566709df1bd11f90ecf34a", + "row": { + "sessionId": "signalr/pre-change-persistence-proof", + "layoutVersion": 2, + "envelopeRoot": "/tmp/netclaw-testrun-92f7815e981241148c98995416e0584d/netclaw-test-PreChangeStorageBindingCaptureTests-2939e2f75ac04d0684568ad42e7b0065/sessions/signalr_pre-change-persistence-proof-2ffc6df70e782831", + "createdAt": 1577836800000 + }, + "markerRelativePath": "workspace/neutral-marker.txt", + "markerBase64": "VGhlIG5ldXRyYWwgc2Vzc2lvbiBtYXJrZXIgcmVtYWlucy4K", + "markerSha256": "5827F5E607A6B6DF758D8379EB00089DC0E30DB9626D65A240E1F2788C8AFCA2" +} diff --git a/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingTests.cs b/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingTests.cs new file mode 100644 index 000000000..fb9ad4abf --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Gateway/PreChangeStorageBindingTests.cs @@ -0,0 +1,106 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Protocol; +using Netclaw.Configuration; +using Netclaw.Daemon.Gateway; +using Netclaw.Daemon.Services; +using Netclaw.Tests.Utilities; +using Xunit; + +namespace Netclaw.Daemon.Tests.Gateway; + +public sealed class PreChangeStorageBindingTests +{ + [Fact] + public async Task Captured_old_binding_and_marker_survive_a_home_move_without_a_new_binding() + { + var fixturePath = Path.Combine(AppContext.BaseDirectory, "LegacyPersistence", "legacy-storage-binding-v0.json"); + var fixture = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(fixturePath, TestContext.Current.CancellationToken), + new JsonSerializerOptions(JsonSerializerDefaults.Web)) + ?? throw new InvalidOperationException("The storage fixture is empty."); + Assert.Equal("2e6bc4f014dc96b606566709df1bd11f90ecf34a", fixture.Baseline); + Assert.Equal("signalr/pre-change-persistence-proof", fixture.Row.SessionId); + Assert.Equal(2, fixture.Row.LayoutVersion); + var segments = fixture.Row.EnvelopeRoot.Split('/', '\\'); + Assert.Equal("sessions", segments[^2]); + var envelopeName = segments[^1]; + Assert.NotEmpty(envelopeName); + Assert.DoesNotContain(envelopeName, new[] { ".", ".." }); + var markerBytes = Convert.FromBase64String(fixture.MarkerBase64); + Assert.Equal(fixture.MarkerSha256, Convert.ToHexString(SHA256.HashData(markerBytes))); + + using var directory = new DisposableTempDir(); + var paths = new NetclawPaths(directory.Path); + paths.EnsureDirectoriesExist(); + try + { + await new SchemaMigrator(paths, NullLogger.Instance) + .MigrateAsync(paths.SqliteDbPath, TestContext.Current.CancellationToken); + using (var connection = new SqliteConnection($"Data Source={paths.SqliteDbPath};Pooling=False")) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + INSERT INTO session_storage_bindings(session_id, layout_version, envelope_root, created_at) + VALUES ($id, $version, $root, $created); + """; + command.Parameters.AddWithValue("$id", fixture.Row.SessionId); + command.Parameters.AddWithValue("$version", fixture.Row.LayoutVersion); + command.Parameters.AddWithValue("$root", fixture.Row.EnvelopeRoot); + command.Parameters.AddWithValue("$created", fixture.Row.CreatedAt); + command.ExecuteNonQuery(); + } + var expectedRoot = Path.Combine(paths.SessionsDirectory, envelopeName); + var markerPath = Path.Combine(expectedRoot, fixture.MarkerRelativePath.Replace('/', Path.DirectorySeparatorChar)); + Assert.StartsWith(expectedRoot + Path.DirectorySeparatorChar, Path.GetFullPath(markerPath), StringComparison.Ordinal); + Directory.CreateDirectory(Path.GetDirectoryName(markerPath) + ?? throw new InvalidOperationException("The marker needs a parent directory.")); + await File.WriteAllBytesAsync(markerPath, markerBytes, TestContext.Current.CancellationToken); + + var clock = new FakeTimeProvider(new DateTimeOffset(2035, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var session = new SessionId(fixture.Row.SessionId); + var storage = new SqliteSessionStorageResolver(paths, clock).Resolve(session); + Assert.NotNull(storage.Binding); + Assert.Equal(expectedRoot, storage.Binding.EnvelopeRoot.Value); + Assert.NotEqual(fixture.Row.EnvelopeRoot, storage.Binding.EnvelopeRoot.Value); + Assert.Equal(Path.Combine(storage.SessionDirectory.Value, "neutral-marker.txt"), markerPath); + Assert.Equal(markerBytes, await File.ReadAllBytesAsync(markerPath, TestContext.Current.CancellationToken)); + Assert.Equal(fixture.Row, CapturedStorageRow.Read(paths)); + var cold = new SqliteSessionStorageResolver(paths, clock).Resolve(session); + Assert.Equal(storage.Binding, cold.Binding); + Assert.Equal(fixture.Row, CapturedStorageRow.Read(paths)); + } + finally + { + SqliteTestPools.Clear(paths); + } + } +} + +internal sealed record CapturedStorageFixture( + string Baseline, CapturedStorageRow Row, string MarkerRelativePath, string MarkerBase64, string MarkerSha256); + +internal sealed record CapturedStorageRow(string SessionId, int LayoutVersion, string EnvelopeRoot, long CreatedAt) +{ + public static CapturedStorageRow Read(NetclawPaths paths) + { + using var connection = new SqliteConnection($"Data Source={paths.SqliteDbPath};Pooling=False"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT session_id, layout_version, envelope_root, created_at FROM session_storage_bindings"; + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + var row = new CapturedStorageRow(reader.GetString(0), reader.GetInt32(1), reader.GetString(2), reader.GetInt64(3)); + Assert.False(reader.Read()); + return row; + } +} diff --git a/src/Netclaw.Daemon.Tests/Netclaw.Daemon.Tests.csproj b/src/Netclaw.Daemon.Tests/Netclaw.Daemon.Tests.csproj index a2746f742..f4bbd920e 100644 --- a/src/Netclaw.Daemon.Tests/Netclaw.Daemon.Tests.csproj +++ b/src/Netclaw.Daemon.Tests/Netclaw.Daemon.Tests.csproj @@ -41,6 +41,7 @@ + From 1b867691e41a0ebdad730e2729facb8665d1cfaf Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 15:56:28 +0000 Subject: [PATCH 04/22] Remove the provider capacity gate from the child run contract --- .../changes/background-child-runs/design.md | 25 ++++----------- .../changes/background-child-runs/proposal.md | 2 -- .../specs/background-subagent-runs/spec.md | 32 +++++++++---------- .../changes/background-child-runs/tasks.md | 16 +++++----- 4 files changed, 31 insertions(+), 44 deletions(-) diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md index 2d2cf1605..f60c55efd 100644 --- a/openspec/changes/background-child-runs/design.md +++ b/openspec/changes/background-child-runs/design.md @@ -16,7 +16,6 @@ Current code establishes these seams: - `SubAgentActor` stops immediately after its terminal reply. It has no durable resume state or compaction. - `SessionState.ActiveBackgroundJobs` belongs to shell jobs. Its delivery dedup is not a durable child-result contract. - `RoleBasedFailoverRouter` shares the main pipeline with compaction unless a distinct compaction model exists. -- No provider admission scheduler, priority queue, or reserved parent inference slot exists. Relevant engineering references: `SPEC-002`, `SPEC-016`, and the [glossary](../../../docs/spec/GLOSSARY.md). The old subagent wall-clock timeout statement conflicts with the implemented per-operation watchdog. @@ -37,7 +36,8 @@ This package changes that statement without introducing a task-lifetime ceiling. - A child that resumes after a daemon restart. - Agent messages, questions, peer discovery, or live steering. -- A workflow engine, a general provider scheduler, or a new test platform. +- A workflow engine or a new test platform. +- Provider-slot limits, request-capacity reservations, or parent/child inference preemption. - Exactly-once external side effects from a model continuation that crashes midway. - Another tool-count, task-age, or task-token ceiling. @@ -333,6 +333,9 @@ After recovery, an accepted run without a durable terminal receipt becomes `Lost A previously admitted cancellation remains `Cancelled` with retained evidence; recovery does not replace it with successful output. Recovered child prompts receive visible expired/cancelled disposition. No recovery branch recreates the child, replays task tools, or adopts the current requester. +The parent model decides further work from the recorded session and the retained loss or partial-result facts. +The framework imposes no retry or operator-review policy for uncertain external effects. +A new model decision can start a new child under ordinary authority. It cannot revive a lost run or duplicate its acceptance key. Captured pre-change records contain no background child ledger. Read them as an explicit empty ledger and retain conversation, storage bindings, and parent approval state. @@ -344,20 +347,7 @@ If the prior binary cannot read them, rollback requires the pre-upgrade database Stop new admissions and settle active runs before rollback. Restoring the database cannot undo external effects. This package authorizes no daemon restart, rollout, or data replacement. -### 10. Blocking owner decision: provider capacity - -Root asked the owner to choose verified spare/separate capacity or runtime preemption for a single-slot provider. -The decision remains pending. Do not weaken the held-child response gate to actor responsiveness. -The actor can process controls while an inference server still delays the parent's model request. -There is no inspected runtime mechanism that reserves or preempts provider capacity. - -Before dependent runtime edits, record the selected capacity contract and its admission consequences in this package. -Do not invent a child concurrency limit or an unbounded accepted queue. -The selected contract must define accepted, queued, and rejected starts, queue bounds, and response under saturation. -The required real-model gate holds a child request and proves a parent response before child release. -An actor probe proves local control response only. It cannot satisfy this model-server gate. - -### 11. Proof and delivery boundary +### 10. Proof and delivery boundary PR 2 depends on PR 1. Validate the combined stack, not the old baseline alone. The runtime, every routed consumer, cancellation, prompt lifetime, and recovery form one coherent PR. @@ -376,7 +366,6 @@ Require independent review of integrated evidence before the background behavior ## Risks / Trade-offs -- [Single-slot starvation] -> Resolve the owner capacity decision; retain the held-child model gate as unpassed. - [Acceptance acknowledgement loss duplicates work] -> Persist one stamped start key and reject digest conflicts. - [A late dispatch follows cancellation] -> Acknowledge closure only after all local dispatch paths close. - [Cancellation loses useful results] -> Commit checkpoints during normal work and preserve them without another model call. @@ -390,7 +379,7 @@ Require independent review of integrated evidence before the background behavior ## Migration Plan -1. Root reviews this package and resolves provider capacity. Retain the accepted five-second framework-only grace default. +1. Apply the confirmed implementation decisions. Retain the accepted five-second framework-only grace default. 2. Rebase this branch on PR 1 and reconcile its frozen wire and detector contracts. 3. Prepare independent lifecycle fixtures and captured pre-change records before runtime edits. 4. Implement the complete background contract with the minimum runtime guidance. diff --git a/openspec/changes/background-child-runs/proposal.md b/openspec/changes/background-child-runs/proposal.md index 9f9e4c17f..84e56204d 100644 --- a/openspec/changes/background-child-runs/proposal.md +++ b/openspec/changes/background-child-runs/proposal.md @@ -51,6 +51,4 @@ The first PR's frozen `per-call-tool-recurrence` contract is a dependency. This Acceptance does not grant extra tool or path authority. Child output cannot answer an approval prompt or create a trusted automation input. New lifecycle records require upgrade and rollback proof. No actor reference, token, callback, or prepared actor property enters durable records. -The current provider router has no capacity reservation or priority scheduler. -The owner must resolve the capacity decision before implementation claims parent response under a held child request. Five seconds is the accepted initial framework-only cancellation grace default. This change adds no operator configuration knob. diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md index 4b761db3c..576d14f6f 100644 --- a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -454,24 +454,24 @@ No rollback procedure SHALL claim reversal of completed external effects. - **THEN** they require the pre-upgrade database backup and active-run settlement - **AND** they retain the warning that external effects remain -### Requirement: Responsiveness evidence reaches the provider boundary +### Requirement: Parent and child model requests remain independent -The release SHALL prove a parent model response before release of a held child provider request under the selected supported capacity contract. -An actor acknowledgement alone SHALL NOT satisfy that model-response claim. -The capacity contract SHALL define admission outcomes and bounded queue behavior before dependent runtime implementation. -Status and cancellation SHALL remain locally usable under provider contention. -The runtime SHALL NOT silently add a provider scheduler, require unproven spare capacity, or weaken the response claim. +The parent SHALL issue its model request without waiting for a live child's completion or a Netclaw provider-slot gate. +The inference backend SHALL own its request queues and capacity. +Netclaw SHALL NOT add provider-slot limits, capacity reservations, or parent/child inference preemption. +Status and cancellation SHALL remain locally usable while a child request remains pending. +An actor acknowledgement SHALL NOT substitute for required real-model evidence. -#### Scenario: Held child permits a parent model response +#### Scenario: A held child does not block the parent request -- **GIVEN** the selected supported provider configuration and a held child request -- **WHEN** the user submits a parent status question -- **THEN** the parent returns its model response before child release -- **AND** evidence records both provider requests and their barrier order +- **GIVEN** a live child with a held model request +- **WHEN** the user submits a parent question +- **THEN** the parent's model request reaches the backend before child release +- **AND** Netclaw does not wait for or allocate a provider slot -#### Scenario: Single-slot starvation remains an unpassed gate +#### Scenario: The backend owns a queued request -- **GIVEN** a provider permits only the held child request -- **WHEN** no chosen contract supplies parent capacity -- **THEN** the model-response gate remains unpassed -- **AND** local actor control success cannot replace that evidence +- **GIVEN** the backend queues the parent's request behind a child request +- **WHEN** Netclaw awaits the parent response +- **THEN** Netclaw does not reserve capacity or preempt the child to change that order +- **AND** local status and cancellation remain available diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index b348dadca..8cac6c44f 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -1,14 +1,14 @@ ## 1. Freeze prerequisites and independent proof fixtures -- [x] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract remains unchanged. -- [ ] 1.2 Record the owner's pending capacity decision before dependent runtime edits; verify the selected admission and held-child model-response clauses are explicit. +- [x] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract includes the confirmed model-owned recovery boundary. +- [x] 1.2 Record the confirmed backend-owned capacity boundary. Remove provider-slot coordination and the pending approval gate. Verify independent request and local control clauses. - [x] 1.3 Have an independent test author derive lifecycle assertions from the delta; verify the author lists paired controls and deliberate faulty variants. - [ ] 1.4 Extend existing actor fixtures with start, provider, dispatch, terminal, and persistence barriers; verify assertions use acknowledgements rather than sleeps. - [x] 1.5 Capture a pre-change journal/snapshot fixture with conversation and parent approval state; verify the fixture predates new lifecycle records. Preparation evidence: -- The branch includes PR 1 at `893435238`. Its existing runtime types remain unchanged. +- The branch inherits PR 1. This contract reconciliation leaves its existing runtime types unchanged. - Design section 7 assigns retained parent detector evidence to the planned child run ledger. - An independent review derives sixteen assertion groups, paired controls, and named faults from the deltas. - The [approved plan](../../../.systematize/plans/background-subagents/plan.html) owns the adversarial method and acceptance gates. @@ -20,7 +20,7 @@ Preparation evidence: - The original resolver creates the captured row. The consumer restores all four columns and the marker under a new home. - The combined candidate passes 33 storage-resolver cases without failures or skips. - Task 6.6 remains open for the new empty child ledger and the integrated lifecycle recovery proof. -- These preparation tasks establish no background runtime or provider-capacity pass. +- These preparation tasks establish no background runtime pass. The owner confirmed backend-owned capacity and model-owned recovery. ## 2. Durable acceptance and independent lifetime @@ -42,7 +42,7 @@ Preparation evidence: - [ ] 3.6 Materialize a fresh attributed provider tool-call/result pair; inspect actual messages and verify the original start call receives no second result. - [ ] 3.7 Preserve original authority and compatible journal-order queues; verify another speaker and active compaction cannot change result authority. - [ ] 3.8 Retain and refresh parent detector checkpoints in the child run ledger. Verify committed start settlement and direct activation without fabricated receipts. Verify late results after fresh input, sibling checkpoint updates, distinct-task separation, receipt failure, and recovery. Preserve current parent directory/project/branch. -- [ ] 3.9 Keep failed parent continuations visible; verify replay does not admit completed input or automatically restart external effects. +- [ ] 3.9 Keep failed parent continuations visible. Verify committed input is not readmitted and the model receives retained loss facts before it chooses further work. ## 4. Every routed start and deferred parent controls @@ -73,7 +73,7 @@ Preparation evidence: - [ ] 6.2 Retain shell-job reap behavior as a nearby control; run its existing integration cases and verify no child state enters shell-job records. - [ ] 6.3 Cancel children during coordinated drain with bounded finalization; verify parent snapshot retains terminal and prompt disposition. - [ ] 6.4 Recover accepted children as explicit `Lost` without relaunch; verify committed cancellation remains cancelled and stale success cannot win. -- [ ] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child or completed side-effect replay. +- [ ] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child, committed result loss, or framework tool execution during recovery. - [ ] 6.6 Load the captured pre-change fixture. Verify retained history, parent prompts, and an explicit empty child ledger. Capture a baseline SQLite storage binding and marker through the real resolver. Verify the candidate preserves that row and marker. The journal fixture alone supplies no storage-binding proof. - [ ] 6.7 Probe prior-reader compatibility against new events; verify rollback instructions require backup restoration when compatibility fails. @@ -82,8 +82,8 @@ Preparation evidence: - [ ] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. - [ ] 7.2 Add targeted Subagents and background cases for acceptance, routed starts, controls, cancellation, prompts, and recovery; verify actual dispatch and artifact evidence. - [ ] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. -- [ ] 7.4 Exercise the chosen admission contract under saturation; verify bounded queue state, explicit outcomes, and locally responsive status/cancel. -- [ ] 7.5 Hold a child on the selected real provider and request a parent reply; verify that reply precedes child release in five critical-case trials. +- [ ] 7.4 Hold several accepted child requests. Verify independent parent dispatch and locally responsive status/cancel without provider-capacity coordination. +- [ ] 7.5 Hold a child on the selected real backend that accepts concurrent requests. Verify a parent reply before child release in five critical-case trials. - [ ] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. - [ ] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. - [ ] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. From 6d544bc5456c31c017e7cd4c84a4df4df14be7fb Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 22:29:47 +0000 Subject: [PATCH 05/22] Implement durable background child runs and parent controls --- .github/workflows/pr_validation.yml | 1 + TOOLING.md | 24 +- docs/architecture/tool-authorization.md | 44 +- docs/prd/PRD-001-netclaw-mvp.md | 4 +- docs/runbooks/subagents.md | 142 +++- ...SPEC-002-session-lifecycle-and-protocol.md | 111 ++- ...C-016-tool-liveness-and-stall-detection.md | 347 ++++----- evals/README.md | 21 + evals/child_run_evals.py | 727 ++++++++++++++++++ evals/fixtures/agents/child-run-worker.md | 11 + .../fixtures/child-runs/HeldChildProtocol.cs | 118 +++ evals/fixtures/child-runs/ProtocolControls.cs | 117 +++ evals/fixtures/child-runs/README.md | 123 +++ evals/fixtures/child-runs/SessionObserver.cs | 251 ++++++ .../child-runs/SessionObserver.csproj | 11 + evals/run-background-evals.sh | 20 +- evals/run-evals.sh | 129 +++- evals/test_child_run_evals.py | 639 +++++++++++++++ .../.system/files/netclaw-operations/SKILL.md | 14 +- .../references/child-runs.md | 98 +++ .../.system/files/subagent-authoring/SKILL.md | 24 +- .../changes/background-child-runs/design.md | 19 +- .../specs/background-subagent-runs/spec.md | 20 + .../changes/background-child-runs/tasks.md | 1 + scripts/run-tool-task-adoption-mutations.sh | 103 +-- .../BackgroundChildControlMutationTests.cs | 290 +++++++ ...hildApprovalGrantAdversarialTests.Drain.cs | 120 +++ ...roundChildApprovalGrantAdversarialTests.cs | 352 +++++++++ ...dOwnerAdversarialTests.AdmittedRecovery.cs | 266 +++++++ ...ChildOwnerAdversarialTests.Cancellation.cs | 190 +++++ ...rAdversarialTests.CancellationCallbacks.cs | 168 ++++ ...dChildOwnerAdversarialTests.ClosureCuts.cs | 202 +++++ ...ChildOwnerAdversarialTests.CrossSession.cs | 102 +++ ...oundChildOwnerAdversarialTests.Delivery.cs | 274 +++++++ ...dOwnerAdversarialTests.DeliveryRecovery.cs | 132 ++++ ...dChildOwnerAdversarialTests.Obligations.cs | 371 +++++++++ ...hildOwnerAdversarialTests.RecoveryDrain.cs | 218 ++++++ ...ChildOwnerAdversarialTests.SpawnFailure.cs | 60 ++ ...hildOwnerAdversarialTests.StartAdapters.cs | 253 ++++++ ...hildOwnerAdversarialTests.StartLifetime.cs | 322 ++++++++ ...ChildOwnerAdversarialTests.TerminalCuts.cs | 306 ++++++++ .../BackgroundChildOwnerAdversarialTests.cs | 468 +++++++++++ .../BackgroundChildOwnerTests.Approvals.cs | 96 +++ .../BackgroundChildOwnerTests.Controls.cs | 94 +++ .../BackgroundChildOwnerTests.Findings.cs | 80 ++ .../BackgroundChildOwnerTests.Recovery.cs | 104 +++ .../Sessions/BackgroundChildOwnerTests.cs | 374 +++++++++ ...kgroundChildReportGraceAdversarialTests.cs | 83 ++ .../Sessions/LlmSessionIntegrationTests.cs | 2 +- .../BackgroundChildStampedAcceptanceTests.cs | 171 ++++ ...SpawnIntegrationTests.AcceptanceReplies.cs | 93 +++ .../SubAgentSpawnIntegrationTests.Recovery.cs | 107 +++ .../Sessions/SubAgentSpawnIntegrationTests.cs | 252 +++--- .../ToolRecurrenceAdversarialTests.cs | 4 +- ...ackgroundChildAcceptanceTests.Lifecycle.cs | 179 +++++ ...undChildAcceptanceTests.ToolsAndContext.cs | 91 +++ .../BackgroundChildAcceptanceTests.cs | 211 +++++ ...ndChildLedgerAdversarialTests.Lifecycle.cs | 250 ++++++ ...edgerAdversarialTests.TerminalPositions.cs | 150 ++++ .../BackgroundChildLedgerAdversarialTests.cs | 334 ++++++++ .../SubAgents/SpawnAgentStreamingTests.cs | 271 ------- .../SubAgents/SubAgentActorTests.cs | 52 ++ .../SubAgentSpawnObservabilityTests.cs | 7 +- .../SubAgents/SubAgentSpawnerTests.cs | 317 ++------ .../Protocol/SessionSnapshot.cs | 3 + .../NetclawProtoMapper.ChildLifecycle.cs | 156 ++++ .../NetclawProtoMapper.ChildRuns.cs | 168 ++++ .../Serialization/NetclawProtoMapper.cs | 20 +- .../NetclawProtobufSerializer.cs | 14 + .../Protos/netclaw_messages.proto | 132 ++++ .../LlmSessionActor.ChildApprovals.cs | 192 +++++ .../Sessions/LlmSessionActor.ChildRecovery.cs | 158 ++++ .../Sessions/LlmSessionActor.ChildRuns.cs | 595 ++++++++++++++ .../Sessions/LlmSessionActor.cs | 315 ++++---- .../Sessions/ParentSessionApprovalBridge.cs | 1 + .../Pipelines/SessionToolExecutionPipeline.cs | 29 +- .../Sessions/SessionMessageAssembler.cs | 35 + .../Sessions/SessionProtocol.ChildRuns.cs | 30 + .../Sessions/SessionProtocol.Events.cs | 16 + .../Sessions/SessionState.ChildRuns.cs | 186 +++++ src/Netclaw.Actors/Sessions/SessionState.cs | 75 +- .../Sessions/ToolApprovalState.cs | 1 + .../SubAgents/BackgroundChildRun.cs | 159 ++++ .../SubAgents/CheckAgentRunTool.cs | 55 ++ .../SubAgents/ChildRunDelivery.cs | 47 ++ .../SubAgents/ChildRunOperations.cs | 128 +++ .../SubAgents/SpawnAgentTool.cs | 78 +- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 286 ++++--- .../SubAgents/SubAgentSpawner.cs | 296 ++----- src/Netclaw.Actors/Tools/SkillLoadTool.cs | 13 +- src/Netclaw.Actors/Tools/ToolAccessPolicy.cs | 3 +- .../Tools/ToolLivenessValidator.cs | 3 +- .../Resources/identity/AGENTS.template.md | 12 +- src/Netclaw.Configuration/Resources/AGENTS.md | 18 + .../SubAgentToolPolicy.cs | 1 + .../Mcp/ToolIndexUpdaterTests.cs | 3 + src/Netclaw.Daemon/Mcp/ToolIndexUpdater.cs | 5 +- .../SubAgentRunMetadata.cs | 1 + .../ToolExecutionContext.cs | 5 +- 99 files changed, 12224 insertions(+), 1550 deletions(-) create mode 100644 evals/child_run_evals.py create mode 100644 evals/fixtures/agents/child-run-worker.md create mode 100644 evals/fixtures/child-runs/HeldChildProtocol.cs create mode 100644 evals/fixtures/child-runs/ProtocolControls.cs create mode 100644 evals/fixtures/child-runs/README.md create mode 100644 evals/fixtures/child-runs/SessionObserver.cs create mode 100644 evals/fixtures/child-runs/SessionObserver.csproj create mode 100644 evals/test_child_run_evals.py create mode 100644 feeds/skills/.system/files/netclaw-operations/references/child-runs.md create mode 100644 src/Netclaw.Actors.MutationTests/BackgroundChildControlMutationTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.Drain.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Cancellation.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CancellationCallbacks.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ClosureCuts.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CrossSession.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.DeliveryRecovery.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Obligations.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.RecoveryDrain.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.SpawnFailure.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartAdapters.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.TerminalCuts.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Approvals.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Controls.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Findings.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Recovery.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildReportGraceAdversarialTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/Pipelines/BackgroundChildStampedAcceptanceTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.AcceptanceReplies.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.Recovery.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs delete mode 100644 src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs create mode 100644 src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildLifecycle.cs create mode 100644 src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildRuns.cs create mode 100644 src/Netclaw.Actors/Sessions/LlmSessionActor.ChildApprovals.cs create mode 100644 src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRecovery.cs create mode 100644 src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs create mode 100644 src/Netclaw.Actors/Sessions/SessionProtocol.ChildRuns.cs create mode 100644 src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs create mode 100644 src/Netclaw.Actors/SubAgents/BackgroundChildRun.cs create mode 100644 src/Netclaw.Actors/SubAgents/CheckAgentRunTool.cs create mode 100644 src/Netclaw.Actors/SubAgents/ChildRunDelivery.cs create mode 100644 src/Netclaw.Actors/SubAgents/ChildRunOperations.cs diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 845b6bc20..a5fe4d191 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -326,6 +326,7 @@ jobs: bash -n evals/run-background-evals.sh bash -n evals/cycle_evals.sh python3 -m unittest discover -s evals -p test_background_evals.py -v + python3 -m unittest discover -s evals -p test_child_run_evals.py -v python3 -m unittest discover -s evals -p test_cycle_evals.py -v python3 -m unittest discover -s evals -p test_cli_detector_evals.py -v python3 -m unittest discover -s evals -p test_spill_steer_evals.py -v diff --git a/TOOLING.md b/TOOLING.md index 4502e75bf..a26da8065 100644 --- a/TOOLING.md +++ b/TOOLING.md @@ -43,31 +43,48 @@ coverage. They do not replace positive and negative behavior tests. ### Task Adoption Gate Run `./scripts/run-tool-task-adoption-mutations.sh` from the repository root. -The gate uses the existing xUnit 2 harness and selects 13 tests through a temporary Stryker JSON configuration. +The gate uses the existing xUnit 2 harness and selects 14 tests through a temporary Stryker JSON configuration. +It runs the five prior targets with their 13 existing controls. +It runs the child-authority target separately with its one owner test. +An unrelated broken parent-input guard can prevent that owner test from starting a child. +Separate filters keep each mutation with the tests that reject it. The script deletes that configuration on exit. -The gate targets five exact mutations: +The gate targets six exact mutations: - Replace full task-authority equality. - Remove the ordered-prefix rejection. - Replace `Take` with `Skip` in the ordered prefix. - Replace actual-outcome equality in the recurrence tracker. - Change stale-callback rejection to acceptance. +- Remove the original-requester rejection for a child control call. The callback target changes the final `false` result in `OwnsToolExecution` to `true`. Its exact mutator name is `Boolean mutation`. The mailbox test replays a captured prior result during a new dispatch with the same provider call ID. The test verifies the current authority attempt, result pair, cancellation token, and confirmed disk effects. +The child control target removes the `!` from `TurnContext.HasSameAuthority(original, current)`. +Its exact mutator name is `LogicalNotExpression to un-LogicalNotExpression mutation`. +The test uses a real owner with a held child and the actual status tool. +It checks original-owner access, opaque foreign denial, canonical log paths, and normal file-policy rejection. + Every expected mutant must exist and have the `Killed` status. The gate rejects expected compile errors, timeouts, survivors, and absent mutants. It also rejects extra executable mutants. Stryker can emit a non-target `Count`-to-`Sum` mutation that cannot compile for the typed input list. The exact-name check excludes that invalid mutation; it does not exclude an expected target error. -The current independent run killed all five targets in 187.564 seconds. +The earlier independent run killed all five prior targets in 187.564 seconds. The prior four-target run took 3m05s. +The first combined six-target run took 232.52 seconds and failed. +Four targets were killed. The two ordered-prefix targets timed out. +The gate correctly rejected those timeouts. +The separate-filter run killed all six targets in 339.85 seconds. +Its reports contain no tested timeout or survivor. +The separate build adds 152.286 seconds over the earlier five-target run. CI runs this gate in `shell-analysis` and uploads `artifacts/stryker/tool-task-adoption` with the group reports. +The `adoption` and `child-control` directories each retain their own mutation report. The local duration does not establish the combined CI duration. ### Current Targets @@ -76,6 +93,7 @@ The local duration does not establish the combined CI duration. |--------|-----------------|------------------|---------| | `SessionState` task adoption and `TurnStateTracker` actual outcomes | Full authority context and the ordered input prefix control adoption; changed actual outcomes do not trigger exact recurrence | 4 killed | `./scripts/run-tool-task-adoption-mutations.sh` | | `LlmSessionActor.OwnsToolExecution` | A captured prior reply cannot affect the current dispatch, even when the provider call ID repeats | 1 killed | `./scripts/run-tool-task-adoption-mutations.sh` | +| `LlmSessionActor.ChildRuns` original-requester check | Child controls retain the original authority; foreign callers receive no state or paths | 1 killed | `./scripts/run-tool-task-adoption-mutations.sh` | | `PathAccessPolicy.AddSessionRoots` and `PathAccessPolicy.IsReadableByAudience` | Only a Personal context receives shared session roots; a reviewed phrase uses the read authority of the audience, attended or not (D2), only for a fully qualified host path of the shell's own style that is not protected | 4 killed | `./scripts/run-path-access-mutations.sh` | | `ToolAccessPolicy.AdmitMcpAudience` | Server and tool audience grants precede approval | 2 killed | `./scripts/run-tool-authorization-mutations.sh` | | `ToolAccessPolicy.ScreenHardDeny` | A shell hard denial precedes approval | 1 killed | `./scripts/run-tool-authorization-mutations.sh` | diff --git a/docs/architecture/tool-authorization.md b/docs/architecture/tool-authorization.md index bd47ac858..195762cf1 100644 --- a/docs/architecture/tool-authorization.md +++ b/docs/architecture/tool-authorization.md @@ -83,12 +83,15 @@ Durable approval answers retain their canonical pending context and existing req A recovered durable approval can use the existing redrive path after the old source ends. That redrive preserves the original authority and authorization attempt and creates a new local dispatch source. -Foreground child spawn requests carry the original parent token before child creation. -The parent receives foreground child activity through a token-bound local mailbox envelope before public emission. -Routed foreground skills use the same existing source for their spawn, activity, result, and failure callbacks. -These checks change no public output format or transport protocol. +Child start requests carry the original parent token until the owner commits acceptance. +The accepted run then owns its lifetime and immutable original authority. +The pipeline validates the returned start key, authority, and source operation against the stamped request. +The slash adapter applies the same checks to its original activation. +The spawner rejects malformed identifiers, digests, profiles, and state before it emits success. +A duplicate returns the recorded run identity and its current state. +The owner binds child results to the accepted run, actor identity, and live dispatch gate. +The parent start call does not own later child approval prompts or terminal delivery. Shell background job deliveries retain their separate durable origin and input-admission contract. -Future accepted background children need their separate run owner after parent foreground settlement. See [TA-1](../../openspec/specs/tool-authorization/spec.md#requirement-ta-1-trust-context-is-explicit-and-fails-loud). ## Background job result authority @@ -721,7 +724,7 @@ Leaks today: `false`) need no consent. - Grant match for one shell candidate can run up to 5 times in one gate run. -### 3.6 Consent delivery +### 3.6 Approval prompt delivery | Item | Current state | | --- | --- | @@ -729,7 +732,7 @@ Leaks today: | Classes | Approval part of [`LlmSessionActor`](../../src/Netclaw.Actors/Sessions/LlmSessionActor.cs), [`ToolApprovalState`](../../src/Netclaw.Actors/Sessions/ToolApprovalState.cs), [`IApprovalChannel`](../../src/Netclaw.Actors/Sessions/IApprovalChannel.cs), [`ParentSessionApprovalBridge`](../../src/Netclaw.Actors/Sessions/ParentSessionApprovalBridge.cs), [`IParentApprovalBridge`](../../src/Netclaw.Tools.Abstractions/IParentApprovalBridge.cs), the approval loop in [`SubAgentActor`](../../src/Netclaw.Actors/SubAgents/SubAgentActor.cs), [`ApprovalResponseFlow`](../../src/Netclaw.Channels/ApprovalResponseFlow.cs), [`PendingApprovalLookup`](../../src/Netclaw.Channels/PendingApprovalLookup.cs), [`ApprovalButtonValueCodec`](../../src/Netclaw.Actors/Protocol/ApprovalButtonValueCodec.cs), [`ApprovalOptionKeys`](../../src/Netclaw.Actors/Protocol/ApprovalOptionKeys.cs), the Slack, Discord, and Mattermost prompt builders | | Published contract | Output `ToolInteractionRequest` and input `ToolInteractionResponse` ([`SessionProtocol.Outputs.cs`](../../src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs)). Journal events `ToolApprovalRequested` and `ToolApprovalResolved`. Option key strings such as `approve_once`. | | Must not know | Policy rules. It renders the options that Consent offers. | -| Data | Durable session journal. Actor-local state for an unanswered request. A subagent request is live-only. | +| Data | Durable session journal for prompt lifecycle and original requester facts. A child execution waiter remains run-local. | | Rules | [TA-10](../../openspec/specs/tool-authorization/spec.md#requirement-ta-10-consent-prompts-offer-only-safe-options), [TA-11](../../openspec/specs/tool-authorization/spec.md#requirement-ta-11-an-unanswered-consent-request-survives-restart), [TA-12](../../openspec/specs/tool-authorization/spec.md#requirement-ta-12-subagent-consent-goes-through-the-parent-session) | Leaks today: @@ -839,15 +842,38 @@ Facts behind the diagram (current code): through one prompt, `ParentSessionApprovalBridge`, which waits with the session's approval timeout. The daemon sets that timeout to infinite. The request waits until the operator answers, the run is cancelled, or a new user - message abandons the parked batch. + message abandons the parked parent batch. An ordinary parent message does not abandon a live child prompt. - A subagent request goes to the parent through `ParentSessionApprovalBridge`. - It is live-only. After a restart, Netclaw rejects the old prompt as expired. + Its lifecycle facts retain the original requester, exact call, authorization attempt, and accepted run. + `LlmSessionActor` stores its request and resolution in the durable `BackgroundChildRun.Approvals` ledger. + The ordinary parent-turn approval map does not own that child prompt. + Its execution waiter remains run-local. After restart, Netclaw rejects the old child prompt as expired. A subagent without a parent bridge cannot ask. Its whole run fails with `ToolExecutionFailed`; it does not return a per-tool denial. +- Child cancellation or loss closes that run's prompt. A late answer creates no grant or retry. +- The owner checks run liveness across any await before a broad grant or prompted retry can commit. +- The owner claims the exact live approval wait before it persists a reusable grant. +- A completed start call does not own the accepted child's prompt lifetime. - A channel that cannot post a prompt sends `Deny` for that call (`ChannelOutputEngine`, `SlackThreadBindingActor`). This is the only automatic denial of a consent request. +The parent loads `check_agent_run` explicitly through `load_tool` for child status or cancellation. +Normal tool policy, the owner session, and the original eligible requester define access. +Children cannot discover, load, or dispatch that control. A foreign caller receives no target details. +The owner uses the original authority record. It does not copy requester or path authority into a second policy structure. +Cancellation creates no authorization grant. A report path still requires ordinary file access policy. +After authorization, the control adapter obtains `log_path` and `artifact_directory` from the existing canonical child storage binding. +These call-local output paths are data. They create no grant or separate durable authority. +The parent uses normal file tools for log access. A denied control reveals neither path. + +Positive example: the original parent requester answers its live child's exact approval prompt after the start call completes. +Negative example: a later speaker cannot answer that prompt with replacement requester authority. +Positive control example: the eligible parent cancels its accepted run and receives separate closure evidence. +Negative control example: a child that knows a sibling run ID cannot inspect or cancel it. +Positive log example: the original requester uses the returned live log path with `file_read` under normal file policy. +Negative log example: a returned path does not bypass a denied file read. + A non-shell call follows the same path, with two differences. `ToolAuthorizer` applies the rules for other tools and one `StoredGrantCheck` in place of the shell coverage. The launch step does not exist. diff --git a/docs/prd/PRD-001-netclaw-mvp.md b/docs/prd/PRD-001-netclaw-mvp.md index 85cd37c76..6e5e5a8f9 100644 --- a/docs/prd/PRD-001-netclaw-mvp.md +++ b/docs/prd/PRD-001-netclaw-mvp.md @@ -297,8 +297,8 @@ Child approval prompts shall remain valid for the live run after its start call Cancellation shall stop new task dispatch and preserve recorded partial evidence through a five-second framework-only grace period. Restart shall report interrupted children as lost, retain terminal delivery, and expire child approval prompts without automatic child relaunch. -The release shall prove a parent model response before a held child request completes under the selected supported provider capacity contract. -Provider capacity remains an explicit delivery gate; actor acknowledgements alone do not prove model responsiveness. +The release shall prove a parent model response before a held child request completes on the selected real backend. +Actor acknowledgements alone do not prove model responsiveness. The inference backend owns capacity and request queues. Private agent messages, peer discovery, questions, and live steering remain outside this delivery. See `openspec/changes/background-child-runs/` for the second PR's review contract. diff --git a/docs/runbooks/subagents.md b/docs/runbooks/subagents.md index 8ffbea1dc..c911d6f37 100644 --- a/docs/runbooks/subagents.md +++ b/docs/runbooks/subagents.md @@ -1,11 +1,10 @@ # Subagents -Subagents are specialist workers that the main Netclaw agent can delegate tasks -to. Each subagent runs autonomously with its own system prompt, inherited -audience-scoped tool surface, and timeout — then returns a synthesized result to -the main agent. The delegation protects the main session's context window from -token-heavy work (deep research, broad exploration, long summarization) and lets -the main agent stay focused on conversation and coordination. +Subagents are workers that the main Netclaw agent can use for independent tasks. +Each child has its own system prompt, inherited audience/profile tool policy, and operation health checks. +The owner accepts the task durably before the child completes it. +The parent can continue its work and accept later user input. A terminal child result arrives later. +Use [the engineering glossary](../spec/GLOSSARY.md) for shared terms. ## How it works @@ -89,20 +88,70 @@ first user message is just the raw task, identical to the pre-context protocol. policy, then `SubAgentToolPolicy` removes tools that are statically denied to subagents (`spawn_agent`). Agent definition `tools:` metadata is advisory and does not narrow runtime authorization. -3. A `SubAgentActor` is spawned as a **child of the session actor** (supervised, - lifecycle-managed — stops when the session stops). -4. The subagent runs an autonomous LLM loop: call tools, process results, repeat. -5. A final response, exact recurrence stop, cancellation, or inactivity timeout - causes the subagent to return a terminal run result. -6. The main agent receives the `spawn_agent` tool result as an explicit text - envelope: agent name, run id, outcome (`completed`, `partial`, or `failed`), - optional reason, diagnostics pointer, and either a `Summary:` or `Error:` - section containing the subagent's final text. - -Child creation is marshaled back onto the session actor thread, so supervision -stays within Akka's actor-thread rules. If the parent tool call is cancelled or -times out, the subagent is cancelled too. The timeout is an inactivity budget: a -responsive subagent is not stopped merely because wall-clock time has elapsed. +3. The owner session commits the accepted run before it creates a `SubAgentActor` or returns acceptance. +4. The first tool response contains `run_id`, `scope_id`, `state`, and `control_tool` as strings. +5. The first state is `Accepted`. The control name is `check_agent_run`. +6. The child executes its model/tool loop with its original authority and a run-owned lifetime. +7. The owner records the terminal result before it acknowledges the child. +8. The owner admits one attributed continuation with a fresh tool-call/result pair after the original start batch settles. + +Acceptance proves admission. It does not prove successful task completion or the current live state. +The tool output and model history contain the same canonical acceptance JSON. +Routed `skill_load` uses the same contract. Direct slash activation uses a human acknowledgement from the same owner facts. +Inline skills remain inline. A failed child route cannot silently execute inline. +Durable adoption preserves the fresh child result pair if its first parent review fails before the next turn record. +Owner recovery retains the pair without a child relaunch or duplicate result admission. + +Child creation occurs on the owner actor thread. +Start-call completion, start-token cancellation after acceptance, and ordinary later parent input do not cancel the accepted child. +Operation health checks still apply. A healthy child can continue without a total task-age or tool-count limit. + +### Status And Cancellation + +The parent explicitly calls `load_tool` for `check_agent_run` when status or cancellation is necessary. +The control requires normal policy, the owning session, and the original eligible requester. +Children and foreign callers receive a denial without target details. +Run IDs and result paths grant no access authority. + +Cancellation has four separate steps: + +1. The owner commits cancellation admission. +2. The owner closes new model, tool, and approval-retry admission, then cancels active calls. +3. The owner confirms local closure after each admitted invocation returns its Task. +4. The framework retains a terminal result with confirmed partial evidence. + +Cancellation admission alone does not prove dispatch closure. +An admitted invocation can delay closure before it returns its Task. The parent must still accept status requests. +Closure does not wait for that returned Task to complete. An external service can ignore its cancellation token. +After closure, a separate five-second deadline permits framework checkpoint retention and atomic run-local report writes. +The grace period permits no model request, task tool, project edit, external request, or approval action. +A failed report write or grace expiry preserves the last confirmed checkpoint and an explicit reason. +The deadline bounds the owner wait. It does not prove that an operating-system file operation stops. +Cancellation cannot undo an earlier effect. An external service can ignore cancellation and leave that effect's outcome unknown. + +Cancelled completion retains `Success=false`, wire outcome `Failed`, and reason `CancelledByParent`. +Confirmed partial evidence does not turn cancellation into success. +The parent does not merge a working-context delta from a failed or cancelled child. +The first durable terminal or cancellation admission determines the terminal outcome. + +### Approval Prompts And Recovery + +A live child's approval prompt retains its original requester, exact call, authorization attempt, and accepted run. +The prompt can outlive the start call. Ordinary later parent input does not abandon it. +Child cancellation or loss expires it. A late answer creates no grant or retry. + +Idle passivation defers live children and pending result admission. +Explicit stop and coordinated drain use the same child cancellation path. +Owner restart records unresolved accepted children as `Lost`. It does not resume or recreate them. +Committed cancellation retains its cancelled outcome. Committed terminal facts and pending delivery survive recovery. +The model chooses further work from the recorded evidence. The framework does not automatically replay uncertain effects. + +Use cancellation and a revised child task when instructions must change. +Private agent messages, peer discovery, and live steering remain outside this release. + +Assign one writer per workspace. Concurrent writers need separate authorized worktrees. +Preserve the operator's dirty checkout. Check child artifacts and verification evidence before the parent delivers them. +Only the parent sends user messages and attaches files under normal policy. ### Observability @@ -120,9 +169,8 @@ are suppressed in Slack. Completion events are emitted for every finished subagent run, even when the subagent returns no structured findings. In that case `FindingsCount` is `0` and the memory-decision fields are empty because there was nothing to review. -The completion event carries the same terminal outcome and reason used by the -tool-result envelope, so operators can distinguish a useful partial summary from -a failed run. +The completion event carries the recorded terminal outcome and reason. +Operators can distinguish a successful partial summary from a failed or cancelled run. Structured findings are conservative, parent-reviewed durable-memory candidates. They should be emitted as explicit conclusion envelopes with review metadata, @@ -243,27 +291,27 @@ Each child receives its own `temp_dir`, `artifact_dir`, and `log_path`. The child also receives the session `session_dir` as its workspace base. Project instructions come from the inherited project root. -A successful `spawn_agent` result returns the child run identifier. -It also returns the exact child log path and artifact directory. -The parent can inspect that log with `file_read`, `file_list`, or `file_search`. -The parent must not use a shell search to discover the log. +A successful `spawn_agent` result returns durable acceptance and the child run identifier. +The authorized `check_agent_run` response supplies the exact `log_path` and `artifact_directory` before terminal completion. +The parent can inspect that log with `file_read` or `file_search` under normal file policy. +Use bounded reads and targeted searches. Do not derive the path or search a global log tree. +A log shows diagnostic evidence. It does not prove current health, dispatch closure, or task completion. +Use the owner's recorded status and terminal result for lifecycle facts. Example: The parent passes the returned log path to `file_read`. Counterexample: The parent does not search a global log tree for the child. -If a subagent hits an approval-gated tool, the prompt is routed through the -parent session's approval channel and requester context. Human approval time does -not count as subagent inactivity or parent `spawn_agent` tool inactivity; both -watchdogs resume after the approval wait settles. If no parent approval bridge -is available (an unattended parent), the authorizer denies the gated tool with -`approval_required_unattended`. The tool does not run, and the subagent gets the -denial as a tool result (decision D2). If the bridge has no requester -authority context, the gated tool fails closed as a failed subagent run and is -not executed. Subagent approval waits are live-only: if the -daemon or parent session restarts before the user responds, the stale prompt is -expired and the interrupted parent `spawn_agent` call is closed before the next -turn continues. +Before reuse in eval fixtures or shared reports, scrub personal data and secrets from a derived transcript copy. +Preserve the original log unchanged. Use stable opaque replacements when correlation or equality matters. + +The parent session routes a child's approval prompt with the original requester and exact call context. +Human approval time does not count as child inactivity. The child health check resumes after the approval wait settles. +The accepted run owns that wait after the start call completes. +A missing parent approval bridge or requester authority fails the run without tool execution. +Cancellation or loss expires the prompt. A late answer creates no grant or retry. +After owner restart, unresolved children become `Lost`; committed cancellation retains its cancelled outcome. +See [Approval Prompts And Recovery](#approval-prompts-and-recovery) for the lifecycle contract. ## Built-in agents @@ -344,3 +392,19 @@ Rollback requires a tested compatible reader or restoration of that recorded pre Backup restoration loses journal state after the backup. It cannot undo external effects from completed tools or jobs. The plan documents this rollback limit; no rollback procedure receives proof without an executed test. + +## Background Child Upgrade And Rollback + +Stop ingress and complete the coordinated drain before you take a pre-upgrade backup. +Retain the session journal, snapshots, storage catalog, child logs, and run artifacts together. +Record the binary revision, backup path, and backup time. Test restoration against a copy. + +New child events use registered manifests `cra-v1` and `cre-v1`. +An old binary cannot assume it can read these events. A protobuf field that an old reader ignores does not prove compatibility. +Direct binary rollback requires explicit reader proof. Otherwise, restore the recorded pre-upgrade backup before you start the prior binary. +Backup restoration loses journal state after the backup and cannot undo external effects. + +After restart, inspect recorded `Lost` or cancelled runs and retained partial artifacts. +Do not recreate a child merely because its acceptance response exists. +The model decides whether to finish, revise the task, or request a new authorized attempt from the recorded session. +Reader and restoration test receipts establish only their stated storage scope. They do not prove a live operational rollback. diff --git a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md index af2d72fa5..6023b2ed5 100644 --- a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md +++ b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md @@ -7,6 +7,7 @@ Research: `docs/research/context-management-patterns.md` Define session identity, message protocol, persistence events, subscriber model, context management, and compaction behavior for `LlmSessionActor`. +Use [the engineering glossary](GLOSSARY.md) for shared terms. ## Session Identity @@ -85,10 +86,107 @@ Each admitted subagent receives a `ChildRunScope`: a fork of immutable tool authority plus the parent's working-context snapshot. The child owns fresh activity tracking and mutable tool-call state; neither is shared with the parent or sibling runs. Terminal results use typed completion variants. -Completed and partial runs carry a `WorkingContextDelta`; failed and cancelled -runs cannot carry one. The parent merges only files the child confirms it -changed through first-party tools. Git-observed dirty files remain diagnostic -context and are never attributed to the child. +Completed and partial runs carry a `WorkingContextDelta`. +Cancelled runs retain confirmed activity as partial evidence and keep a failed outcome. +The parent merges a delta only from completed or partial success. +The parent attributes only files that first-party tools confirm the child changed. +Git-observed dirty files remain diagnostic context. They do not prove child authorship. + +### Background Child Ownership + +`LlmSessionActor` owns durable child acceptance, cancellation admission, dispatch closure, terminal receipt, and result delivery. +`SubAgentActor` owns live task execution, operation health, and confirmed partial evidence. +The live-only `ChildRunDispatch` gate serializes new model and tool dispatch against closure. +The accepted run retains its original `ChildRunScope.Authority` and initial snapshot. +Runtime shell host objects stay live-only. Recovery does not reconstruct child execution. + +The framework stamps a start key before asynchronous setup. +A tool key uses the owner session, original admitted turn, and original call ID. +A slash key uses the owner session, admitted input, original turn, and framework activation slot zero. +Equivalent retries return one accepted run. Conflicting task or resolved-profile digests fail without mutation. + +Schematic flow; normal policy, path checks, and approval gates remain required: + +```text +prepare the start from canonical admitted authority +owner commits ChildRunAccepted before child execution or acceptance output +owner creates the child with a run-owned lifetime +owner commits Started before it dispatches the child task +adapter returns the owner's canonical acceptance JSON +parent continues independent work +child closes task dispatch and retains its terminal result +owner authenticates the child sender and generation +owner commits the terminal receipt before acknowledging the child +owner records the actual journal position as TerminalSequenceNr +owner completes enrichment and waits for the original start batch to settle +owner atomically commits delivery identity and attributed continuation input +parent adopts the continuation under its original authority and detector checkpoint +owner materializes each fresh child call/result pair in the durable adoption fold +parent continues from the recorded result or settles a retained receipt defect +``` + +The initial acceptance JSON contains `run_id`, `scope_id`, `state`, and `control_tool` as strings. +Its first state is `Accepted`. Its control name is `check_agent_run`. +Tool output and model history use the same formatted body. +The later result uses a fresh call/result pair. It does not create a second result for the original start call. +Durable adoption retains that pair even when the first review fails before a tool batch or turn record. +Recovery validates each ordered sibling input before it adopts the prefix. +Repeated adoption of the same pending prefix adds no duplicate pair. A changed pair fails validation. + +The owner admits terminal results in journal order, even when enrichment completes in reverse order. +Equal timestamps do not change this order. +`ResultPrepared` retains the terminal's `TerminalSequenceNr`. +Recovery rejects a terminal stamp that differs from its journal position. +Snapshot recovery rejects a terminal stamp beyond the snapshot's journal position. +The owner serializes terminal writes through their acknowledgements. A queued write does not advance Akka's `LastSequenceNr`. + +The deferred control requires explicit `load_tool` use and normal parent policy. +It also requires the owner session and original eligible requester. +Children and foreign callers receive a denial without target details. +Identifiers and report paths grant no authority. +After authorization, status includes the canonical `log_path` and `artifact_directory`, including for an accepted or live child. +The adapter uses the existing child storage binding. It creates no separate path authority or durable path ledger. +Normal file tools enforce file policy when the parent reads the returned log path. +The log is diagnostic evidence. It does not prove current health, dispatch closure, or completion. + +Positive example: the original requester reads its held child's returned log path through `file_read` under normal policy. +Negative example: another requester receives neither target state nor paths, even with a valid run identifier. + +The owner stores child approval requests and resolutions in `BackgroundChildRun.Approvals`. +These records retain the original context, child call, prompt call, and authorization attempt. +The parent-turn approval map does not own them. An ordinary parent message does not close a live child prompt. +The owner claims the exact live wait before it persists a reusable grant. +The live dispatch gate also applies to that grant operation and the child retry. +Cancellation, terminal settlement, and recovery expire unresolved child prompts. + +Cancellation admission, confirmed local dispatch closure, and terminal settlement are separate lifecycle facts. +After closure, framework-only finalization has a separate five-second deadline. +It retains checkpoints and permits atomic report writes only inside the assigned run artifact directory. +The complete report operation runs outside the actor thread, including path checks, directory creation, serialization, and file flushes. +The owner deadline bounds its wait even when the filesystem operation ignores cancellation. +A late worker cannot publish a successful report result or replace the retained terminal outcome. +It permits no model call, task tool, project edit, external request, or approval action. +A cancelled result retains useful confirmed evidence and remains a failure outcome. +An external effect without a confirmed result remains explicitly unknown. + +The first durable terminal or cancellation admission determines the terminal outcome. +A late result cannot replace an earlier cancellation outcome or admit another continuation. +The owner retains terminal results and pending delivery through snapshots and journal recovery. +Owner restart marks unresolved accepted children `Lost` and never recreates them. +The model decides its next action from the recorded evidence. The framework does not automatically retry uncertain effects. + +Idle passivation defers live children and pending delivery. +Explicit stop and coordinated drain cancel children through the bounded closure path. +The inference backend owns request queues and capacity. Netclaw adds no parent or child slot coordinator. + +Positive example: a held child stays live while the parent answers later input, then delivers one attributed result. +Negative example: another session obtains its run ID but cannot inspect, cancel, or read protected result paths. +Positive cancellation example: the parent receives closure evidence and a partial report after confirmed file effects. +Negative cancellation example: a late approval answer creates no grant or retry after child cancellation. + +Testable contracts live in [background-subagent-runs](../../openspec/specs/background-subagent-runs/spec.md), +[netclaw-subagents](../../openspec/specs/netclaw-subagents/spec.md), and +[tool-authorization](../../openspec/specs/tool-authorization/spec.md). ## Subscriber Model @@ -179,7 +277,7 @@ are summarized as "Used {tool} for {purpose} → {outcome}". ## Persistence Rules -- protobuf-net serialization only for events and snapshots +- registered protobuf serialization only for events and snapshots - framework-owned message envelopes only - no direct persistence of `Microsoft.Extensions.AI` model types - system prompt is always slot 0 in history — compaction must preserve it @@ -192,8 +290,11 @@ are summarized as "Used {tool} for {purpose} → {outcome}". | `TurnRecorded` | Completed turn (user message + assistant reply) | | `SessionTitleSet` | Title generated or updated | | `SessionCompacted` | History compacted with summary + retained messages | +| `ChildRunAccepted` | Original child authority, start identity, working facts, and parent detector evidence | ## Snapshot `SessionSnapshot` captures `History`, `TurnCount`, `Title` for fast recovery. Taken periodically per `SessionConfig.SnapshotInterval` and after compaction. +It also retains the child ledger, original authority, parent checkpoints, lifecycle facts, and pending delivery. +Old captured snapshots decode with an empty child ledger. Transcript text never creates a live child record. diff --git a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md index 1c4ced6f6..b6dc00b7c 100644 --- a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md +++ b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md @@ -2,222 +2,141 @@ Source PRDs: `PRD-001`, `PRD-006` -Status: Planning note for issue #1450. The corresponding OpenSpec artifacts must -be updated through the OpenSpec workflow before implementation is closed. - -> **Superseded in part by #1472 / PR #1481 ("addition through subtraction").** The -> "parent first-item startup guard for self-monitoring tools" described below was -> removed. The parent no longer supervises self-monitoring tools at all — they own -> their liveness end to end and are drained with no parent watchdog, bounded only by -> their own internal watchdog or caller (turn/user) cancellation; an unanswered human -> approval blocks the run until it is answered or the turn is cancelled. The -> authoritative, reconciled spec is the `Per-call liveness by tool class` requirement -> in the `streaming-tool-call-execution` OpenSpec change -> (`specs/netclaw-tools/spec.md`). Treat the parent-startup-guard clauses below as -> historical. +Use [the engineering glossary](GLOSSARY.md) for shared terms. +This document describes operation health and the background child lifecycle. +Testable rules live in [netclaw-tools](../../openspec/specs/netclaw-tools/spec.md), +[netclaw-subagents](../../openspec/specs/netclaw-subagents/spec.md), +[background-subagent-runs](../../openspec/specs/background-subagent-runs/spec.md), and +[turn-loop-governance](../../openspec/specs/turn-loop-governance/spec.md). ## Purpose -Define the simplified stall-detection contract for tool calls. The goal is to -make long-running tool calls reliable without adding another competing timeout or -turning streamed output into a fake progress signal. - -## Problem - -`spawn_agent` is currently consumed through the same generic streaming-tool -watchdog as ordinary tools. The parent session uses -`Session.ToolExecutionTimeoutSeconds` as a reset-on-item inactivity budget. That -means a healthy sub-agent can be killed when it opens a quiet window longer than -the parent budget, even though the child actor has its own progress-aware -watchdogs. - -The wrong abstraction is "did the stream emit anything?" The useful question is -"who can actually tell whether this operation has stalled?" - -## Decision - -Tool calls SHALL use two liveness classes. - -### Opaque Tools - -An opaque tool does not expose reliable internal stall detection. The parent -tool-execution pipeline SHALL apply one explicit wall-clock budget to the whole -call. Streaming output, if any, is display-only and SHALL NOT extend the budget. - -Default/generated tools are opaque unless they opt into another mode. - -Examples: - -- MCP tools with no mapped MCP progress notification -- most generated first-party tools -- `web_fetch` -- `shell_execute` - -For `shell_execute`, stdout and stderr may still be streamed to subscribers as -live output, but a command that prints forever is not making proven forward -progress. `_timeout_seconds` or `Session.ToolExecutionTimeoutSeconds` remains the -process wall-clock budget. - -### Self-Monitoring Tools - -A self-monitoring tool owns its own stall detection because the worker can see -more than the parent can. The parent session SHALL keep ownership of -cancellation, persistence, and final turn response, but it SHALL NOT run the -generic inter-item inactivity watchdog after the tool has produced its first -sign of life. - -The parent SHALL still apply a startup guard: a self-monitoring tool must produce -its first stream item within the existing startup/first-item budget. This bounds -the irreducibly blind window where the parent does not know whether the tool -actually began executing. - -`spawn_agent` is the first self-monitoring tool. Its child `SubAgentActor` -already owns: - -- wait-for-first-delta prefill budget -- inter-delta liveness budget after model output starts -- keepalive-immune no-progress budget -- approval-wait suspension and external cancellation -- tool-iteration cap - -If a sub-agent stalls, the child actor SHALL complete the `spawn_agent` call with -a failed `SubAgentResult`. The parent records that terminal tool result and -continues the turn according to the normal tool-batch rules. - -## Non-Goals - -- No new coarse parent backstop in this change. -- No new taxonomy of progress event records in this change. -- No requirement for every tool to emit progress events. -- No MCP progress mapping in this change. -- No behavior where stdout, stderr, or heartbeat output extends an opaque tool's -budget. - -## Runtime Contract - -`INetclawTool` SHOULD expose a liveness classification with an opaque default. - -```csharp -public enum ToolLivenessMode -{ - Opaque, - SelfMonitoring -} +Long tasks can continue while each operation remains healthy. +Task age, token totals, and tool-call counts do not define a task-lifetime deadline. +Explicit cancellation, operation health checks, authorization, and exact recurrence protection still apply. + +## Decision Owners + +| Owner | Decision | Data lifetime | +| --- | --- | --- | +| Tool execution pipeline | Apply the resolved tool's operation-health contract | Call-local | +| `SubAgentActor` | Detect stalled child model/tool operations and close task dispatch | Actor-local | +| `LlmSessionActor` | Commit acceptance, cancellation, dispatch closure, terminal receipt, and later result delivery | Durable journal and snapshots | +| `ChildRunDispatch` | Reject new model and tool dispatch after local closure | Run-local; live-only | +| `LlmSessionActor` child approval handler | Retain the request and resolution under the accepted run's original authority | Durable `BackgroundChildRun.Approvals` ledger | +| `ParentSessionApprovalBridge` | Correlate a live child approval wait with its owner and original requester | Run-local waiter | +| Inference backend | Queue and admit provider requests according to backend capacity | Backend-owned | + +Netclaw adds no inference-slot limits, reservations, parent preemption, or cooperative provider scheduler. +A held child request does not prevent the parent from issuing its own request. +The backend can queue either request. Local parent status and cancellation do not require an inference slot. + +## Opaque Operations + +Opaque tools retain a wall-clock operation deadline. +Output does not extend that deadline. +Most MCP calls, `web_fetch`, and `shell_execute` use this contract. +For shell, `_timeout_seconds` or `Session.ToolExecutionTimeoutSeconds` bounds the process operation. + +Positive example: a quiet shell operation completes before its deadline. +Negative example: a shell process prints forever but still stops at its operation deadline. + +## Child Operation Health + +A child owns its operation health rather than the parent's generic tool watchdog. +The pipeline does not add a first-item or inter-item watchdog for a self-monitoring operation. +The child distinguishes: + +- Wait for the first substantive model output. +- Inactivity between model deltas after output starts. +- A no-progress deadline that content-free keepalives cannot refresh. +- Tool operation deadlines and explicit cancellation. +- An authorized approval wait, which is intentional suspension rather than a model stall. + +A completed healthy operation can lead to another operation without a total task-time limit. +A stalled operation returns an explicit failed terminal result through the owner. +Exact recurrence can produce a partial terminal result without a final model request. +No static parent or child tool-call ceiling remains. + +Positive example: a child completes useful work through more than the former 30 feedback rounds. +Negative example: heartbeat-only output cannot keep a child model operation alive indefinitely. + +## Acceptance And Lifetime + +All explicit and routed child starts use durable acceptance. +The owner commits acceptance before child execution and before the accepted response. +The response identifies the accepted run; it does not wait for child completion. +After acceptance, a run-owned lifetime replaces dependence on the start call's token. +An ordinary later parent message does not cancel that run or its approval prompt. + +Schematic sequence; normal authorization and persistence gates remain required: + +```text +prepare the scoped start from its admitted input and call +commit acceptance +start the run-owned child +return canonical accepted identifiers +continue independent parent work +receive and durably acknowledge the child's terminal result +admit one attributed parent continuation after the original start batch settles ``` -For source-generated tools, the Roslyn generator may generate the overridden -liveness property from `NetclawToolAttribute`, but the default remains `Opaque`. -The generator can enforce that any generated first-party tool declaring -`SelfMonitoring` also overrides `ExecuteStreamAsync`. - -The tool-execution pipeline SHALL choose the watchdog shape from the resolved -tool's liveness mode: - -- `Opaque`: wall-clock budget for the whole call. Stream items do not reset it. -- `SelfMonitoring`: first-item startup guard only. After the first item, parent - liveness is disabled for that call and child/tool cancellation remains linked - to parent turn cancellation. - -Approval waits remain intentional suspension owned by the layer that issued the -approval request. For sub-agents, that is the live `SubAgentActor` run. - -## Shell Example - -`shell_execute` remains opaque. - -Expected stream shape: - -1. Arguments and policy are validated. -2. The process starts. -3. Optional stdout/stderr chunks are streamed for user-visible output. -4. The process exits, is killed by the wall-clock budget, or is cancelled. -5. The terminal `ToolCompletedUpdate` carries the model-facing result. - -The stdout/stderr chunks are not progress for stall detection. A command such as -`while true; do echo .; sleep 1; done` must still be killed by the wall-clock -budget. - -## Sub-Agent Example - -`spawn_agent` is self-monitoring. - -Expected stream shape: - -1. The parent resolves the agent and asks the session actor to create the child. -2. The child accepts `RunSubAgent` and emits the first stream item, such as - `calling the model`. -3. The parent first-item guard is satisfied and parent inter-item liveness is no - longer applied to this call. -4. The child watchdog governs prefill, model deltas, keepalive-only wedges, - tool-loop progress, approval waits, cancellation, and iteration exhaustion. -5. The terminal `ToolCompletedUpdate` carries an explicit sub-agent run envelope - produced by the child: run id, outcome, optional reason, diagnostics pointer, - and the final summary or error text. - -## Validation Strategy - -Validation must prove the failure mode directly: a self-monitoring sub-agent can -legitimately remain quiet longer than `Session.ToolExecutionTimeoutSeconds` -without the parent killing it, while opaque tools remain bounded. - -### Unit And Contract Tests - -- `StreamingToolWatchdog` or its replacement SHALL have deterministic tests with - `FakeTimeProvider` proving opaque calls use wall-clock timeout, not - reset-on-output inactivity. -- A chatty opaque stream SHALL still time out at the wall-clock budget even when - it emits output items before the budget expires. -- A self-monitoring stream SHALL fail if no first item arrives before the startup - guard. -- A self-monitoring stream SHALL remain alive after its first item even when no - later item arrives before the opaque default budget. -- Parallel calls SHALL be independent: a self-monitoring call that disables its - parent inter-item liveness must not extend or disable an opaque sibling's - budget. -- The tool generator SHALL prove generated tools default to `Opaque` and that - `spawn_agent` explicitly resolves as `SelfMonitoring`. - -### Actor Integration Tests - -- A `spawn_agent` call with `Session.ToolExecutionTimeoutSeconds` set shorter - than the child prefill budget SHALL not be cancelled by the parent after the - first child stream item. -- A silent child prefill SHALL be terminated by the child prefill watchdog, and - the resulting `spawn_agent` tool result SHALL contain the child timeout reason, - not the parent generic `produced no activity` timeout. -- A keepalive-only child stream SHALL be terminated by the child no-progress - watchdog. -- A sub-agent approval wait longer than the parent tool timeout SHALL remain - pending until approval, denial, approval timeout, or parent cancellation. -- Three parallel `spawn_agent` calls SHALL produce one terminal tool result per - call when one child stalls and the others complete. - -### Session And Persistence Tests - -- `ActiveToolBatchTracker` SHALL not complete the tool batch until every expected - call id has one recorded terminal tool result and the execution task has - finished. -- A timed-out child sub-agent SHALL be recorded as a `ToolCallRecorded` event for - the `spawn_agent` call id, not as a whole-turn `ToolExecutionFailed`. -- Recovery from `ToolBatchStarted` with missing tool results SHALL remain loud and - deterministic; side-effecting tools must not be silently re-run. - -### Diagnostics And Manual Repro - -- Logs SHALL correlate parent session id, sub-agent run id, child watchdog - reason, terminal outcome, and terminal `spawn_agent` result. -- Manual repro should run parallel `spawn_agent` calls where one child opens a - quiet window longer than the parent tool timeout. The expected result is no - parent `produced no activity` failure; either the child completes or the child - watchdog reports the stall. - -## Implementation Order - -1. Add the liveness classification with opaque default. -2. Mark `spawn_agent` as self-monitoring. -3. Change the tool pipeline to apply wall-clock budget for opaque tools and - first-item-only guard for self-monitoring tools. -4. Keep shell opaque and ensure streamed stdout/stderr does not reset the budget. -5. Add the tests listed above. -6. Update OpenSpec artifacts through the OpenSpec workflow and then sync stale - main specs that still describe tool timeouts as whole-batch failures. +A failed acceptance commit starts no child and returns no false acceptance. +Equivalent start retries return the same run. A conflicting start digest fails without mutation. +A start-tool timeout after acceptance does not cancel the accepted run. + +## Cancellation And Partial Evidence + +The parent explicitly loads `check_agent_run` for authorized status or cancellation. +The owner commits cancellation admission before it reports that fact. +The owner closes the run's local model, tool, and approval-retry admission gate. +The owner then commits the dispatch closure fact. +That committed fact proves that no new local task work can start. +It does not prove that an earlier external effect stopped or that its outcome is known. + +After closure, framework-only finalization uses a separate token and a five-second grace deadline. +It can preserve confirmed checkpoints and atomically write a report inside the assigned run artifact directory. +It cannot request model output, run task tools, edit the project, call an external service, or create approval authority. +A failed write or grace expiry preserves the last confirmed checkpoint and an explicit reason. +Cancellation remains a failure outcome even when useful partial evidence exists. + +Positive example: cancellation returns confirmed file activity and an existing partial artifact without another model call. +Negative example: late provider output or an approval answer cannot start task work after dispatch closure. + +The first committed terminal or cancellation admission determines the terminal outcome. +Duplicate child results create no second delivery or parent continuation. +Terminal persistence failure produces no successful receipt acknowledgement. + +## Recovery, Passivation, And Drain + +Idle passivation defers live children and pending delivery. +Explicit parent stop and coordinated drain cancel children through the bounded closure path. +Owner restart marks unresolved accepted runs `Lost`; it does not recreate or resume child execution. +A prior durable cancellation retains its cancelled outcome. +Committed terminal facts and pending delivery remain available through journal and snapshot recovery. +Old child approval prompts expire visibly after loss or restart. + +The framework restores committed evidence and truthful unknown outcomes. +The model decides its next action from that recorded session. +The framework does not automatically replay uncertain effects or impose an operator-review policy. +No physical external exactly-once guarantee follows from local closure or journal recovery. + +## Verification + +Use deterministic provider and tool barriers. Do not use sleeps to infer closure. +Required evidence includes: + +- Acceptance commit before execution or accepted output, including failed and held writes. +- Equivalent start deduplication and conflicting-digest rejection. +- A held child plus independent parent work and usable local controls. +- Distinct cancellation-admission and dispatch-closure acknowledgements. +- No late model, tool, grant, or approval retry after closure. +- Confirmed partial evidence after report failure or grace expiry. +- Both durable terminal/cancel race orders and duplicate terminal delivery. +- Snapshot and journal recovery with explicit loss and no child relaunch. +- Original authority and parent detector evidence across later input and sibling results. +- Opaque wall-clock deadlines and child health checks that retain their distinct owners. + +Targeted model evals supplement these deterministic contracts. +Retain raw evidence and classify instrumentation, prompt, capability, and infrastructure failures separately. +A model claim or an artifact path does not replace an independent effect or file-content check. diff --git a/evals/README.md b/evals/README.md index 247673a7b..ff253ea93 100644 --- a/evals/README.md +++ b/evals/README.md @@ -549,3 +549,24 @@ skips persistence. whose LLM calls fail at request time. Not relevant to the eval path itself (the script always supplies valid config) but noted for anyone exploring the Docker image directly. + +## Background child acceptance and later results + +The child cases use the existing background runner and a persistent `DaemonClient` consumer. +Read [the child consumer procedure](fixtures/child-runs/README.md) before a trial. +The product headless CLI exits at its first turn boundary. +A child result can arrive later, so the affected Subagents cases retain a connection through automatic continuation. +The adapter labels its evidence source and retains the actual output DTOs. +It does not fabricate a second result for the original start call. + +Select `child_run_held_parent`, `child_run_partial_cancel`, or `child_run_cli_acceptance` with `NETCLAW_EVAL_CASE`. +Run each trial through `run-background-evals.sh` with `NETCLAW_EVAL_RUNS=1`. +Set `NETCLAW_CHILD_OBSERVER` to the canonical observer DLL from the final combined source. +Use five fresh invocations for each critical held-child and cancel case. +The runner archives actual provider bytes, child paths, commit diagnostics, and artifact evidence. +The CLI acceptance smoke remains separate from the persistent-result cases. + +The existing Subagents consumers also require an exact case selector and one fresh invocation. +Their original project, scope, shell, and artifact assertions remain mandatory. +The queue-grant and shell lifecycle controls retain their original paths and oracles. +This source preparation supplies no current model pass or complete background health claim. diff --git a/evals/child_run_evals.py b/evals/child_run_evals.py new file mode 100644 index 000000000..7c2034537 --- /dev/null +++ b/evals/child_run_evals.py @@ -0,0 +1,727 @@ +#!/usr/bin/env python3 +"""A child-request barrier and narrow oracles for actual background-child trials.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import subprocess +import threading +import time +import urllib.request +import uuid +from http.server import ThreadingHTTPServer + +from background_fixture import Fixture, handler_for, message_text + +CHILD_CONTRACT = "[Subagent Execution Contract]" +CASES = {"child_run_held_parent", "child_run_partial_cancel", "child_run_cli_acceptance"} + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, "JSON repeats an object key.") + result[key] = value + return result + + +def acceptance(text): + value = json.loads(text, object_pairs_hook=unique_object) + require(isinstance(value, dict), "Acceptance is not an object.") + require(all(isinstance(value.get(key), str) and value[key].strip() + for key in ("run_id", "scope_id", "state", "control_tool")), "Acceptance lacks owner identifiers.") + require(value["state"] == "Accepted" and value["control_tool"] == "check_agent_run", "Wrong acceptance contract.") + return value + + +def context_paths(request): + system = "\n".join(message_text(m.get("content")) for m in request.get("messages", []) if m.get("role") == "system") + if CHILD_CONTRACT not in system: + return None + users = "\n".join(message_text(m.get("content")) for m in request.get("messages", []) if m.get("role") == "user") + paths = {} + for key in ("session_dir", "temp_dir", "artifact_dir", "log_path"): + matches = re.findall(r"^" + key + r": (.+)$", users, re.MULTILINE) + if len(matches) != 1: + return None + paths[key] = matches[0] + return paths + + +def bind_request(request, accepted, status, nonce): + require(status.get("run_id") == accepted["run_id"] and status.get("scope_id") == accepted["scope_id"], + "Status belongs to another accepted child.") + require(status.get("state") in {"Accepted", "Running", "Cancelling"}, "The child is not live at the barrier.") + paths = context_paths(request) + require(paths is not None, "The held request is not an actual child request with runtime storage context.") + require(paths["log_path"] == status.get("log_path") and paths["artifact_dir"] == status.get("artifact_directory"), + "The child request does not use the status-returned paths.") + users = "\n".join(message_text(m.get("content")) for m in request["messages"] if m.get("role") == "user") + require(nonce in users, "The child task lacks the exact trial nonce.") + for path in paths.values(): + require(path.startswith("/") and ".." not in PurePosixPath(path).parts, "A child path is not canonical.") + return paths + + +def write_completed(request, filename): + calls = {} + for message in request.get("messages", []): + for call in message.get("tool_calls", []): + function = call.get("function", {}) + if function.get("name") == "file_write": + arguments = json.loads(function.get("arguments", "{}")) + path = arguments.get("Path", arguments.get("path", "")) + if PurePosixPath(path).name == filename: + calls[call["id"]] = path + if message.get("role") == "tool" and message.get("tool_call_id") in calls: + # The final oracle also checks the actual file and this paired result. + return {"call_id": message["tool_call_id"], "path": calls[message["tool_call_id"]], + "result": message_text(message.get("content"))} + return None + + +class ChildFixture(Fixture): + def __init__(self, upstream, model, api_key, evidence): + super().__init__(upstream, model, api_key) + self.mode = "model" + self.evidence = Path(evidence) + self.evidence.mkdir(parents=True, exist_ok=False) + self.records = [] + self.candidate = None + self.binding = None + self.release = False + self.release_ns = 0 + self.abort = False + self.case = "" + + def control(self, action, data): + with self.condition: + if action == "child-setup": + require(not self.records and data["case"] in CASES, "The child fixture is not fresh.") + self.nonce, self.case = data["nonce"], data["case"] + elif action == "child-wait": + if not self.condition.wait_for(lambda: self.candidate is not None and self.candidate["upstream_first_payload_ns"] > 0, timeout=30): + raise TimeoutError("The child upstream response did not reach the payload barrier.") + elif action == "child-bind": + require(self.binding is None and not self.release, "The child binding repeats or follows release.") + accepted = acceptance(json.dumps(data["accepted"])) + if not self.condition.wait_for(lambda: self.candidate is not None and self.candidate["upstream_first_payload_ns"] > 0, timeout=30): + raise TimeoutError("The child upstream response did not reach the payload barrier.") + paths = bind_request(self.candidate["request"], accepted, data["status"], self.nonce) + if self.case == "child_run_partial_cancel": + filename = f"partial-{self.nonce}.txt" + write = write_completed(self.candidate["request"], filename) + require(write is not None and write["path"] == paths["artifact_dir"] + "/" + filename, + "The held request lacks the exact partial artifact result.") + contents = actual_file(os.environ["EVAL_HOME"], write["path"]).read_text() + require(contents == "PARTIAL-" + self.nonce, "The partial write lacks actual file evidence before cancellation.") + self.binding = {"accepted": accepted, "paths": paths, "request_id": self.candidate["request_id"], + "arrived_ns": self.candidate["admitted_ns"], + "upstream_first_payload_ns": self.candidate["upstream_first_payload_ns"], "bound_ns": time.monotonic_ns()} + elif action == "child-release": + require(self.binding is not None and not self.release, "Release requires one canonical child binding.") + self.release_ns = time.monotonic_ns() + self.release = True + self.condition.notify_all() + elif action == "child-drained": + require(self.release, "The held request remains unreleased.") + if not self.condition.wait_for(lambda: self.candidate["forward_complete"], timeout=30): + raise TimeoutError("The released provider request did not complete its local forward operation.") + elif action == "child-consumed": + pending = [row for row in self.records if row["response_first_payload_ns"] + and not row["response_payload_written"] and not row["forward_complete"] + and any(terminal_pairs([row["request"]], start["accepted"], start["call_id"], start["source_operation"]) + for start in data["expected"])] + if pending and not self.condition.wait_for( + lambda: all(row["response_payload_written"] or row["forward_complete"] for row in pending), timeout=30): + raise TimeoutError("A provider payload write lacks its local acknowledgement.") + return consumed_deliveries(self.records, data["expected"], data["parent_boundary_ns"]) + elif action == "child-abort": + self.abort = True + self.condition.notify_all() + elif action != "snapshot": + raise ValueError("Unknown child fixture control.") + return self.snapshot() + + def snapshot(self): + return {"nonce": self.nonce, "case": self.case, "binding": self.binding, "released": self.release, "release_ns": self.release_ns, + "candidate": None if self.candidate is None else {k: self.candidate[k] for k in + ("request_id", "admitted_ns", "upstream_first_payload_ns", "forward_complete")}, + "requests": [{k: row[k] for k in ("request_id", "admitted_ns", "child", "held", "forward_complete", "upstream_first_payload_ns", "response_first_payload_ns", "response_payload_written")} + for row in self.records]} + + def completion(self, request): + with self.condition: + number = len(self.records) + 1 + child = context_paths(request) is not None + nonce_present = any(self.nonce in message_text(m.get("content")) for m in request.get("messages", []) + if m.get("role") == "user") and bool(self.nonce) + hold = self.case in {"child_run_held_parent", "child_run_partial_cancel"} and child and nonce_present and self.candidate is None + if self.case == "child_run_partial_cancel": + hold = hold and write_completed(request, f"partial-{self.nonce}.txt") is not None + row = {"request_id": number, "admitted_ns": time.monotonic_ns(), "child": child, + "held": bool(hold), "forward_complete": False, "upstream_first_payload_ns": 0, "response_first_payload_ns": 0, "response_payload_written": False, "request": request} + self.records.append(row) + (self.evidence / f"request-{number:04}.json").write_bytes(json.dumps(request).encode()) + if hold: + self.candidate = row + + return None + + +def child_handler(fixture): + base = handler_for(fixture) + + class Handler(base): + response_status = 0 + + def send_response(self, code, message=None): + self.response_status = code + return super().send_response(code, message) + + def do_POST(self): + try: + super().do_POST() + except AssertionError as error: + self.send_json(400, {"error": str(error)}) + + def forward(self, body): + # Use the actual base forward path. Capture each byte that it writes. + with fixture.condition: + row = next(row for row in fixture.records if row["request"] is body) + original = self.wfile + handler = self + path = fixture.evidence / f'response-{row["request_id"]:04}.wire' + with path.open("xb") as capture: + class Tee: + writes = 0 + def write(self, value): + capture.write(value) + capture.flush() + self.writes += 1 + first_payload = (self.writes > 1 and value and 200 <= handler.response_status < 300 + and row["upstream_first_payload_ns"] == 0) + if first_payload: + with fixture.condition: + row["upstream_first_payload_ns"] = time.monotonic_ns() + fixture.condition.notify_all() + if row["held"]: + if not fixture.condition.wait_for(lambda: fixture.release or fixture.abort, timeout=600): + raise TimeoutError("The owner did not release the held upstream response payload.") + if fixture.abort and not fixture.release: + raise ValueError("The task-owned trial aborted before response release.") + row["response_first_payload_ns"] = time.monotonic_ns() + written = original.write(value) + if first_payload: + with fixture.condition: + row["response_payload_written"] = True + fixture.condition.notify_all() + return written + def flush(self): + return original.flush() + self.wfile = Tee() + try: + super().forward(body) + finally: + self.wfile = original + with fixture.condition: + row["forward_complete"] = True + fixture.condition.notify_all() + return Handler + + +def terminal_pairs(requests, accepted, original_start_call_id, original_start_operation): + require(isinstance(original_start_operation, str) and original_start_operation, "The actual start operation is absent.") + pairs = {} + for request in requests: + if context_paths(request) is not None: + continue + calls = {} + delivered_in_request = set() + for message in request.get("messages", []): + for call in message.get("tool_calls", []): + function = call.get("function", {}) + arguments = json.loads(function.get("arguments", "{}"), object_pairs_hook=unique_object) + require(isinstance(arguments, dict), "The tool arguments are not a JSON object.") + if arguments.get("run_id") == accepted["run_id"] and arguments.get("source_operation"): + require(arguments == {"run_id": accepted["run_id"], "source_operation": original_start_operation}, + "The terminal arguments differ from the canonical two-key object.") + require(message.get("role") == "assistant", "The terminal call lacks the assistant role.") + require(arguments["source_operation"] == function.get("name") == original_start_operation, + "The terminal source differs from the actual original start operation.") + require(call["id"] != original_start_call_id, "The terminal call reused the original start identifier.") + require(call["id"] not in calls, "The terminal call repeats within an unresolved occurrence.") + calls[call["id"]] = function["name"] + if message.get("role") == "tool" and message.get("tool_call_id") in calls: + identifier = message["tool_call_id"] + require(identifier not in delivered_in_request, "The terminal pair repeats inside one provider history.") + delivered_in_request.add(identifier) + source = calls.pop(identifier) + body = json.loads(message_text(message.get("content")), object_pairs_hook=unique_object) + require(isinstance(body, dict), "The terminal body is not a JSON object.") + require(body.get("run_id") == accepted["run_id"] and body.get("scope_id") == accepted["scope_id"], + "The terminal pair has a foreign owner.") + require(body.get("source_operation") == source, "The terminal source operation differs.") + require(identifier not in pairs or pairs[identifier] == body, "A terminal pair changed between requests.") + pairs[identifier] = body + require(len(pairs) <= 1, "The accepted run has multiple terminal call identifiers.") + return pairs + + +def canonical_pairs(requests, accepted, original_start_call_id, original_start_operation): + pairs = terminal_pairs(requests, accepted, original_start_call_id, original_start_operation) + require(len(pairs) == 1, "The accepted run lacks exactly one attributed terminal call/result pair.") + return next(iter(pairs.items())) + + +def consumed_deliveries(records, expected, parent_boundary_ns): + if not expected: + return {"complete": False, "deliveries": []} + result = [] + for start in expected: + accepted = acceptance(json.dumps(start["accepted"])) + require(isinstance(start.get("call_id"), str) and start["call_id"], "The original start call identifier is absent.") + observations = [] + all_pairs = terminal_pairs([row["request"] for row in records], accepted, start["call_id"], start["source_operation"]) + for row in records: + pairs = terminal_pairs([row["request"]], accepted, start["call_id"], start["source_operation"]) + payload_ns = row.get("response_first_payload_ns", 0) + if pairs and row.get("response_payload_written") and 0 < row["admitted_ns"] < payload_ns < parent_boundary_ns: + identifier, terminal = next(iter(pairs.items())) + observations.append({"accepted": accepted, "call_id": identifier, "terminal": terminal, + "request_id": row["request_id"], "request_admitted_ns": row["admitted_ns"], + "response_first_payload_ns": payload_ns, "parent_boundary_ns": parent_boundary_ns}) + if not observations: + return {"complete": False, "deliveries": []} + require(len(all_pairs) == 1, "The consumed child has no stable attributed terminal pair.") + result.append(observations[0]) + return {"complete": True, "deliveries": result} + + +def committed_positions(log, session, run_id, call_id): + result = {} + for name in ("accepted", "terminal_recorded", "result_prepared", "delivery_admitted"): + pattern = (r"child_run_" + name + r" owner=" + re.escape(session) + r" runId=" + re.escape(run_id) + + r" journalSequence=(\d+)(?: inputId=([^\s]+) callId=([^\s]+))?") + matches = re.findall(pattern, log) + require(len(matches) == 1, "The run lacks exactly one post-commit diagnostic: " + name) + result[name] = int(matches[0][0]) + if name == "delivery_admitted": + require(matches[0][1] and matches[0][2] == call_id, "The admitted input lacks the actual terminal call ID.") + require(list(result.values()) == sorted(set(result.values())), "Child journal positions are not distinct and ordered.") + return result + + +def actual_file(home, canonical): + prefix = "/home/netclaw/.netclaw/" + require(isinstance(canonical, str) and canonical.startswith(prefix), "The runtime path is outside the eval-owned mount.") + relative = canonical[len(prefix):] + require(".." not in PurePosixPath(relative).parts, "The runtime path escapes the eval-owned home.") + root = Path(home).resolve() / "data" + target = root / relative + require(target.resolve().is_relative_to(root.resolve()), "A link escapes the eval-owned home.") + return target + + +def verify_child_actions(requests, paths, nonce, cancel): + calls = {} + allowed = {"load_tool", "search_tools", "file_list", "file_read", "file_write"} + for request in requests: + if context_paths(request) != paths: + continue + write_occurrences = 0 + for message in request.get("messages", []): + for call in message.get("tool_calls", []): + function = call.get("function", {}) + require(function.get("name") in allowed, "The child called an action outside the assigned task.") + if function.get("name") == "file_write": + write_occurrences += 1 + require(write_occurrences == 1, "One child history contains multiple artifact write occurrences.") + args = json.loads(function["arguments"]) + require(call["id"] not in calls or calls[call["id"]] == args, "The child write changed between requests.") + calls[call["id"]] = args + require(len(calls) == 1, "The child did not execute exactly one artifact write.") + arguments = next(iter(calls.values())) + expected = paths["artifact_dir"] + (f"/partial-{nonce}.txt" if cancel else f"/complete-{nonce}.txt") + require(arguments.get("Path", arguments.get("path")) == expected + and arguments.get("Content", arguments.get("content")) == ("PARTIAL-" if cancel else "COMPLETE-") + nonce, + "The child artifact call has the wrong path or content.") + + +def verify_trial(receipt, requests, snapshot, log, home, nonce, cancel): + accepted = acceptance(json.dumps(receipt["accepted_run"])) + require(receipt["status"] == "observed" and receipt["completed_turns"] >= 2, "The persistent parent flow did not complete.") + consumption = receipt["delivery_observations"] + require(consumption.get("complete") and len(consumption["deliveries"]) == 1, "The parent never consumed the actual terminal pair.") + require(receipt["user_inputs"] == 2, "New input replaced automatic continuation.") + binding = snapshot["binding"] + require(binding and binding["accepted"] == accepted and snapshot["released"], "The child barrier lacks a canonical binding.") + held_rows = [row for row in snapshot["requests"] if row["held"]] + require(len(held_rows) == 1 and held_rows[0]["child"] + and held_rows[0]["request_id"] == binding["request_id"] + and held_rows[0]["admitted_ns"] == binding["arrived_ns"] + and held_rows[0]["upstream_first_payload_ns"] == binding["upstream_first_payload_ns"], + "The child binding differs from the actual held upstream request record.") + require(0 < receipt["first_turn_ns"] < snapshot["release_ns"] <= receipt["release_ns"] + and binding["arrived_ns"] < binding["upstream_first_payload_ns"] < receipt["second_turn_ns"] < snapshot["release_ns"] + and binding["upstream_first_payload_ns"] <= binding["bound_ns"] <= snapshot["release_ns"], + "The parent probe did not complete under the actual child barrier.") + start_calls = [row for row in receipt["calls"] if row["name"] == "spawn_agent" + and row.get("result") and acceptance(row["result"])["run_id"] == accepted["run_id"]] + require(len(start_calls) == 1, "The terminal lacks the exact original start occurrence.") + call_id, terminal = canonical_pairs(requests, accepted, start_calls[0]["id"], start_calls[0]["name"]) + positions = committed_positions(log, receipt["session_id"], accepted["run_id"], call_id) + consumed = consumption["deliveries"][0] + require(consumed["accepted"] == accepted and consumed["call_id"] == call_id and consumed["terminal"] == terminal + and 0 < consumed["request_admitted_ns"] < consumed["response_first_payload_ns"] < consumed["parent_boundary_ns"], + "The final parent response lacks actual model-consumption correlation.") + observed_request = next((row for row in snapshot["requests"] if row["request_id"] == consumed["request_id"]), None) + require(observed_request is not None and not observed_request["child"] and observed_request["response_payload_written"] + and observed_request["admitted_ns"] == consumed["request_admitted_ns"] + and observed_request["response_first_payload_ns"] == consumed["response_first_payload_ns"], + "The consumption receipt differs from the actual relay request record.") + require(0 < consumed["request_id"] <= len(requests), "The consumed request is absent from raw capture.") + require(canonical_pairs([requests[consumed["request_id"] - 1]], accepted, start_calls[0]["id"], start_calls[0]["name"]) == (call_id, terminal), + "The consumption receipt names a request without the actual terminal pair.") + require(terminal.get("outcome") == ("Failed" if cancel else "Completed"), "The child terminal outcome differs.") + if cancel: + require(terminal.get("state") == "Cancelled" and terminal.get("reason") == "cancelled_by_parent", + "The local cancellation disposition differs.") + require(terminal.get("log_path") == binding["paths"]["log_path"] + and terminal.get("artifact_directory") == binding["paths"]["artifact_dir"], "The terminal paths differ from bound child storage.") + calls = receipt["calls"] + starts = [row for row in calls if row["name"] == "spawn_agent"] + require(len(starts) == 1 and starts[0]["arguments"].get("Agent", starts[0]["arguments"].get("agent")) == "child-run-worker", + "The fixed flow did not start the assigned child exactly once.") + verify_child_actions(requests, binding["paths"], nonce, cancel) + controls = [row for row in calls if row["name"] == "check_agent_run"] + require(any(row["arguments"].get("RunId", row["arguments"].get("runId")) == accepted["run_id"] + and row["arguments"].get("Cancel", row["arguments"].get("cancel", False)) is cancel and row.get("success") and row["turn"] == 2 for row in controls), + "The parent did not execute the exact deferred child control.") + require(any(row["name"] == "load_tool" and row["arguments"].get("Name", row["arguments"].get("name")) == "check_agent_run" + and row.get("success") and row["turn"] == 2 for row in calls), "The parent did not load the deferred child control.") + read_path = binding["paths"]["log_path"] + log_bytes = actual_file(home, read_path).read_text() + require(log_bytes, "The actual live child log is empty.") + require(any(row["name"] == "file_read" and row["arguments"].get("Path", row["arguments"].get("path")) == read_path + and row.get("success") and row["turn"] == 2 and any(line in row.get("result", "") for line in log_bytes.splitlines() if len(line) >= 16) + for row in calls), "The parent did not read actual live child log content.") + artifact = binding["paths"]["artifact_dir"] + (f"/partial-{nonce}.txt" if cancel else f"/complete-{nonce}.txt") + contents = actual_file(home, artifact).read_text() + require(contents == ("PARTIAL-" if cancel else "COMPLETE-") + nonce, "The actual child artifact content differs.") + if cancel: + held = next(row for row in snapshot["requests"] if row["held"]) + actual_write = write_completed(requests[held["request_id"] - 1], f"partial-{nonce}.txt") + require(actual_write is not None, "The held request lacks its actual partial write result.") + verify_partial_report(terminal, home, accepted, artifact, actual_write["result"]) + require(not any(row["child"] and row["request_id"] > held["request_id"] for row in snapshot["requests"]), + "The child entered a new provider operation after the cancellation barrier.") + require(any(row["name"] == "file_read" and row["arguments"].get("Path", row["arguments"].get("path")) == artifact + and row.get("success") and consumed["response_first_payload_ns"] < row["observed_ns"] < consumed["parent_boundary_ns"] + and contents in row.get("result", "") for row in calls), + "The parent did not read the actual artifact during automatic continuation.") + require(("PARTIAL-" if cancel else "COMPLETE-") + nonce in receipt["last_reply"], "The automatic parent reply lacks actual artifact evidence.") + return {"passed": True, "journal_positions": positions, "terminal_call_id": call_id, + "artifact_sha256": hashlib.sha256(contents.encode()).hexdigest(), + "limit": "Local cancellation does not prove that an external provider stopped its accepted operation."} + + + +def verify_partial_report(terminal, home, accepted, artifact, confirmed_write_result): + checkpoint = terminal.get("checkpoint") + require(isinstance(checkpoint, dict) and isinstance(checkpoint.get("CompletedRound"), int) + and not isinstance(checkpoint["CompletedRound"], bool) and checkpoint["CompletedRound"] > 0, + "The cancelled child lacks a completed durable checkpoint.") + summary = checkpoint.get("Summary") + activity = checkpoint.get("ConfirmedActivity") + require(isinstance(summary, str) and summary.strip() and confirmed_write_result + and confirmed_write_result in summary and isinstance(activity, dict) + and activity.get("ConfirmedChangedFiles") == [artifact], + "The cancelled checkpoint lacks confirmed partial artifact activity.") + report_path = terminal["artifact_directory"] + "/cancelled-results.json" + require(report_path in terminal.get("output", "") and not terminal.get("warning"), + "The fixed local-report case lacks a confirmed framework report.") + report = json.loads(actual_file(home, report_path).read_bytes()) + require(report.get("run_id") == accepted["run_id"] and report.get("state") == "Cancelled" + and report.get("summary") == summary and report.get("confirmed_activity") == activity, + "The actual partial report differs from the durable terminal checkpoint.") + require(report.get("external_effects") == "Recorded receipts describe known local results. They do not prove external effects stopped.", + "The partial report omits the canonical external-effect limit.") + return report_path + +def prompts(nonce, cancel): + kind = "partial" if cancel else "complete" + content = ("PARTIAL-" if cancel else "COMPLETE-") + nonce + task = (f"Trial {nonce}. Write exactly {content}, without a newline, to {kind}-{nonce}.txt in your runtime artifact_dir. " + "Use file_write. Use no shell command. Return the full artifact path and its content in your final result.") + initial = (f"Delegate this task once to child-run-worker: {task} " + "Acknowledge the accepted child immediately with a brief visible reply. Do not wait for its result. " + "When its later result arrives, read its artifact with file_read and report the actual content. " + "If the child is cancelled, inspect its report and read the confirmed partial artifact. " + "Report the actual content and the local cancellation state.") + probe = ("For accepted child {{RUN_ID}}, inspect its current state and its authorized log path. " + "Read that log with file_read. Use the deferred child control through normal tool discovery. " + + ("After that read, cancel this child. Do not start another child. " if cancel else "Keep the child active. ") + + f"Reply with PARENT-PROBE-{nonce} and a concrete fact from the actual log.") + return initial, probe + + +def observer_command(): + path = Path(os.environ["NETCLAW_CHILD_OBSERVER"]).resolve() + require(path.is_file(), "Build the canonical SessionObserver project before this run.") + return ["dotnet", str(path)] if path.suffix == ".dll" else [str(path)] + + +def invoke_observer(port, prompt, evidence, mode, nonce, probe="", session="", output_format="json"): + evidence = Path(evidence) + evidence.mkdir(parents=True, exist_ok=False) + data = {"DaemonEndpoint": "http://127.0.0.1:" + os.environ["EVAL_PORT"], + "FixtureEndpoint": f"http://127.0.0.1:{port}", "Nonce": nonce, + "InitialPrompt": prompt, "ProbePrompt": probe, "ProbeMarker": f"PARENT-PROBE-{nonce}", + "EvidenceDirectory": str(evidence), "TimeoutSeconds": int(os.environ["PROMPT_TIMEOUT"]), + "Mode": mode, "SessionId": session, "OutputFormat": output_format} + input_path = evidence / "observer-input.json" + input_path.write_text(json.dumps(data, indent=2)) + command = observer_command() + [str(input_path)] + result = subprocess.run(command, capture_output=True, text=True, + timeout=int(os.environ["PROMPT_TIMEOUT"]) + 15) + (evidence / "observer.stdout").write_text(result.stdout) + (evidence / "observer.stderr").write_text(result.stderr) + require(result.returncode == 0, "The persistent observer failed; inspect its receipt and raw outputs.") + receipt = json.loads((evidence / "observer-receipt.json").read_text()) + validate_prompt_receipt(receipt, data) + return receipt, result.stdout + + +def control(port, action, **body): + request = urllib.request.Request(f"http://127.0.0.1:{port}/control/{action}", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=40) as response: + return json.load(response) + + +def evidence_requests(directory): + return [json.loads(path.read_bytes()) for path in sorted(Path(directory).glob("request-*.json"))] + + +def daemon_logs(home): + paths = sorted((Path(home) / "logs").glob("daemon*.log")) + require(paths, "The eval-owned daemon diagnostic log is absent.") + return "\n".join(path.read_text(errors="replace") for path in paths) + + +def legacy_observer_mode(case, prompt_ordinal): + require(isinstance(prompt_ordinal, int) and not isinstance(prompt_ordinal, bool), "The prompt ordinal is invalid.") + if case == "coding_context_worktree_handoff": + require(1 <= prompt_ordinal <= 4, "The worktree handoff prompt ordinal is outside its four-prompt contract.") + return "collect" if prompt_ordinal == 3 else "turn" + require(case in {"subagent_headless_ambiguous_task", "subagent_specialization_precedence", + "subagent_project_scope_declaration", "subagent_session_scratch_disposable", + "approval_natural_subagent_project_review"} and prompt_ordinal == 1, + "The legacy child case or prompt ordinal is invalid.") + return "collect" + + +def validate_prompt_receipt(receipt, data): + require(receipt.get("status") == "observed", "The current prompt did not complete successfully.") + require(receipt.get("prompt_nonce") == data["Nonce"] and receipt.get("observer_mode") == data["Mode"], + "The observer receipt belongs to another prompt invocation.") + require(receipt.get("initial_prompt_sha256") == hashlib.sha256(data["InitialPrompt"].encode()).hexdigest(), + "The observer receipt has another prompt payload.") + require(isinstance(receipt.get("last_reply"), str) and receipt["last_reply"].strip(), "The current final reply is absent.") + require(isinstance(receipt.get("session_id"), str) and receipt["session_id"].strip(), "The current session identifier is absent.") + if data["SessionId"]: + require(receipt.get("session_id") == data["SessionId"], "The observer receipt has another resumed session.") + + +def collect(port, prompt, session, output_format, evidence, case, prompt_ordinal): + mode = legacy_observer_mode(case, prompt_ordinal) + receipt, output = invoke_observer(port, prompt, evidence, mode, uuid.uuid4().hex, session=session, + output_format=output_format) + receipt["case"] = case + receipt["prompt_ordinal"] = prompt_ordinal + if mode == "turn": + receipt["verified_deliveries"] = [] + (Path(evidence) / "verified-receipt.json").write_text(json.dumps(receipt, indent=2)) + print(output, end="") + return + requests = evidence_requests(Path(os.environ["TMPDIR_EVAL"]) / "child-runs/relay") + log = daemon_logs(os.environ["EVAL_HOME"]) + require(receipt["accepted_runs"] and receipt["delivery_observations"]["complete"], + "The legacy response lacks an accepted child and actual terminal consumption.") + deliveries = [] + for accepted in receipt["accepted_runs"]: + starts = [call for call in receipt["calls"] if call["name"] == "spawn_agent" + and call.get("result") and acceptance(call["result"])["run_id"] == accepted["run_id"]] + require(len(starts) == 1, "The legacy terminal lacks the exact original start occurrence.") + call_id, terminal = canonical_pairs(requests, accepted, starts[0]["id"], starts[0]["name"]) + require(terminal.get("outcome") == "Completed", "A legacy child did not complete normally.") + positions = committed_positions(log, receipt["session_id"], accepted["run_id"], call_id) + deliveries.append({"accepted": accepted, "terminal": terminal, "journal_positions": positions}) + receipt["verified_deliveries"] = deliveries + (Path(evidence) / "verified-receipt.json").write_text(json.dumps(receipt, indent=2)) + print(output, end="") + if output_format == "text": + for row in deliveries: + print("[child:result] " + json.dumps(row["terminal"])) + + + +def cli_acceptance(port, root, nonce): + session = "eval/child-cli-" + nonce + initial, _ = prompts(nonce, False) + initial += " Include CLI-ACCEPTED-" + nonce + " in your immediate reply." + environment = {**os.environ, "NETCLAW_HOME": os.environ["EVAL_HOME"], + "NETCLAW_DAEMON_ENDPOINT": "http://127.0.0.1:" + os.environ["EVAL_PORT"]} + result = subprocess.run([os.environ["NETCLAW_BIN"], "chat", "-p", "--resume", session, initial], + env=environment, capture_output=True, text=True, timeout=int(os.environ["PROMPT_TIMEOUT"])) + (root / "actual-cli.stdout").write_text(result.stdout) + (root / "actual-cli.stderr").write_text(result.stderr) + require(result.returncode == 0, "The actual headless CLI failed.") + return verify_cli_acceptance(result.stdout, evidence_requests(root / "relay"), + daemon_logs(os.environ["EVAL_HOME"]), session, nonce) + + +def verify_cli_acceptance(stdout, requests, log, session, nonce): + bodies = re.findall(r"^\[tool:result\] spawn_agent → (.+)$", stdout, re.MULTILINE) + require(len(bodies) == 1, "The actual CLI lacks one start acceptance result.") + accepted = acceptance(bodies[0]) + require("CLI-ACCEPTED-" + nonce in stdout, "The actual CLI lacks the parent's acceptance reply.") + matches = {} + for request in requests: + if context_paths(request) is not None: + continue + calls = {call["id"]: call for message in request.get("messages", []) for call in message.get("tool_calls", [])} + for message in request.get("messages", []): + identifier = message.get("tool_call_id") + if message.get("role") != "tool" or identifier not in calls: + continue + call = calls[identifier] + if call.get("function", {}).get("name") != "spawn_agent": + continue + try: + body = json.loads(message_text(message.get("content"))) + except ValueError: + continue + if body == accepted: + matches[identifier] = body + require(len(matches) == 1, "The model history lacks the same actual CLI acceptance object.") + pattern = r"child_run_accepted owner=" + re.escape(session) + r" runId=" + re.escape(accepted["run_id"]) + r" journalSequence=(\d+)" + positions = re.findall(pattern, log) + require(len(positions) == 1, "The actual CLI acceptance lacks one durable accepted diagnostic.") + return {"passed": True, "session_id": session, "accepted_run": accepted, + "accepted_journal_position": int(positions[0]), + "limit": "The CLI smoke proves initial acceptance only. Persistent trials prove later results separately."} + +def run(port): + require(os.environ.get("RUNS") == "1", "Each child trial requires one fresh harness invocation with RUNS=1.") + case = os.environ["NETCLAW_EVAL_CASE"] + require(case in CASES, "Select one child case.") + nonce = uuid.uuid4().hex + root = Path(os.environ["TMPDIR_EVAL"]) / "child-runs" + root.mkdir(exist_ok=True) + report = {"case": case, "nonce": nonce, "passed": False, "fresh_invocation": True, + "home": os.environ["EVAL_HOME"], "container_name": os.environ["EVAL_CONTAINER_NAME"], + "image_reference": os.environ["NETCLAW_IMAGE"], "user_inputs": 1 if case == "child_run_cli_acceptance" else 2, + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "cli_sha256": hashlib.sha256(Path(os.environ["NETCLAW_BIN"]).read_bytes()).hexdigest(), + "observer_sha256": hashlib.sha256(Path(os.environ["NETCLAW_CHILD_OBSERVER"]).read_bytes()).hexdigest()} + error = None + try: + report["container_id"] = subprocess.check_output( + ["docker", "inspect", "--format", "{{.Id}}", os.environ["EVAL_CONTAINER_NAME"]], text=True).strip() + control(port, "child-setup", nonce=nonce, case=case) + if case == "child_run_cli_acceptance": + report.update(cli_acceptance(port, root, nonce)) + return 0 + initial, probe = prompts(nonce, case == "child_run_partial_cancel") + receipt, _ = invoke_observer(port, initial, root / "observer", "cancel" if case.endswith("cancel") else "held", + nonce, probe) + snapshot = control(port, "snapshot") + (root / "fixture-snapshot.json").write_text(json.dumps(snapshot, indent=2)) + report.update(verify_trial(receipt, evidence_requests(root / "relay"), snapshot, + daemon_logs(os.environ["EVAL_HOME"]), os.environ["EVAL_HOME"], nonce, + case == "child_run_partial_cancel")) + report["session_id"] = receipt["session_id"] + artifact_path = snapshot["binding"]["paths"]["artifact_dir"] + ( + f"/partial-{nonce}.txt" if case.endswith("cancel") else f"/complete-{nonce}.txt") + (root / "actual-artifact.txt").write_bytes(actual_file(os.environ["EVAL_HOME"], artifact_path).read_bytes()) + except (AssertionError, KeyError, ValueError, OSError, subprocess.SubprocessError) as failure: + error = type(failure).__name__ + ": " + str(failure) + report["error"] = error + finally: + # Preserve an aborted barrier state before teardown. Never remove another trial's resource. + snapshot = control(port, "snapshot") + (root / "fixture-final-snapshot.json").write_text(json.dumps(snapshot, indent=2)) + if not snapshot["released"]: + control(port, "child-abort") + (root / "trial-receipt.json").write_text(json.dumps(report, indent=2)) + (Path(os.environ["TMPDIR_EVAL"]) / "stdout_background-results.txt").write_text(json.dumps( + {"runtime": [], "model": [report], "errors": [error] if error else [], "passed": report["passed"]}, indent=2)) + return 0 if report["passed"] else 1 + + + +def verified_final_response(evidence): + directory = Path(evidence) + data = json.loads((directory / "observer-input.json").read_text()) + receipt = json.loads((directory / "verified-receipt.json").read_text()) + validate_prompt_receipt(receipt, data) + return receipt["last_reply"] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["serve", "run", "collect", "assert-delivery", "log-path", "final-response"]) + parser.add_argument("--port", type=int) + parser.add_argument("--prompt-file") + parser.add_argument("--session", default="") + parser.add_argument("--format", choices=["json", "text"], default="json") + parser.add_argument("--evidence") + parser.add_argument("--agent", default="") + parser.add_argument("--case", default="") + parser.add_argument("--prompt-ordinal", type=int, default=1) + args = parser.parse_args() + if args.action == "final-response": + print(verified_final_response(args.evidence), end="") + return 0 + if args.action in {"assert-delivery", "log-path"}: + receipts = [json.loads(path.read_text()) for path in Path(args.evidence).glob("observer-*/verified-receipt.json")] + deliveries = [row for receipt in receipts for row in receipt.get("verified_deliveries", [])] + require(deliveries, "No durable later child delivery exists.") + if args.agent: + require(any(call["name"] == "spawn_agent" and call["arguments"].get("Agent", call["arguments"].get("agent")) == args.agent + for receipt in receipts for call in receipt["calls"]), "The requested child profile was not used.") + if args.action == "log-path": + require(len(deliveries) == 1, "The fixed consumer requires one actual child log.") + path = actual_file(os.environ["EVAL_HOME"], deliveries[0]["terminal"]["log_path"]) + require(path.is_file(), "The actual child log is absent.") + print(path) + return 0 + if args.action == "serve": + fixture = ChildFixture(os.environ["BACKGROUND_EVAL_UPSTREAM"], os.environ["NETCLAW_EVAL_MODEL_ID"], + os.environ.get("NETCLAW_EVAL_PROVIDER_API_KEY", ""), + Path(os.environ["TMPDIR_EVAL"]) / "child-runs/relay") + server = ThreadingHTTPServer(("127.0.0.1", 0), child_handler(fixture)) + print(server.server_port, flush=True) + try: + server.serve_forever() + finally: + server.server_close() + elif args.action == "collect": + collect(args.port, Path(args.prompt_file).read_text(), args.session, args.format, args.evidence, args.case, args.prompt_ordinal) + else: + return run(args.port) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/fixtures/agents/child-run-worker.md b/evals/fixtures/agents/child-run-worker.md new file mode 100644 index 000000000..324502539 --- /dev/null +++ b/evals/fixtures/agents/child-run-worker.md @@ -0,0 +1,11 @@ +--- +name: child-run-worker +description: Eval worker that writes one assigned artifact through the normal file tool. +modelRole: Main +--- + +Use your actual runtime artifact directory for the assigned file. +Use file_write for that file. +Use no shell command. +Do not create another file. +Return the full artifact path and the actual file content. diff --git a/evals/fixtures/child-runs/HeldChildProtocol.cs b/evals/fixtures/child-runs/HeldChildProtocol.cs new file mode 100644 index 000000000..b694add7d --- /dev/null +++ b/evals/fixtures/child-runs/HeldChildProtocol.cs @@ -0,0 +1,118 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text; +using System.Text.Json; +using Netclaw.Actors.Protocol; + +namespace Netclaw.Evals.ChildRuns; + +internal sealed record AcceptedRun(string RunId, string ScopeId); + +internal sealed class HeldChildProtocol(string sessionId) +{ + private readonly Dictionary _calls = new(StringComparer.Ordinal); + private readonly StringBuilder _text = new(); + private bool _hasDelta; + private int _lastTurn; + + public AcceptedRun? Acceptance { get; private set; } + public int CompletedTurns { get; private set; } + public bool ChildHeld { get; private set; } + public bool Released { get; private set; } + public string LastReply { get; private set; } = ""; + + public void Observe(SessionOutputDto output) + { + Require(output.SessionId == sessionId, "A foreign session output reached the observer."); + switch (output.Type) + { + case SessionOutputTypes.ToolCall: + Require(!string.IsNullOrWhiteSpace(output.CallId) && !string.IsNullOrWhiteSpace(output.ToolName), + "A tool call lacks its canonical identifiers."); + Require(_calls.TryAdd(output.CallId!, output.ToolName!), "A tool call identifier was reused."); + break; + case SessionOutputTypes.ToolResult: + Require(output.CallId is not null && _calls.Remove(output.CallId, out var name) + && name == output.ToolName, + "A tool result lacks one matching call or repeats a result."); + if (output.ToolName == "spawn_agent") + { + Require(output.ToolFailureCode is null, "The child start failed."); + Acceptance ??= ParseAcceptance(output.Result ?? ""); + } + break; + case SessionOutputTypes.TextDelta: + _hasDelta = true; + _text.Append(output.Text); + break; + case SessionOutputTypes.Text: + // The canonical headless consumer also excludes final text after stream deltas. + if (!_hasDelta) + _text.Append(output.Text); + break; + case SessionOutputTypes.TurnCompleted: + Require(output.TurnNumber is { } number && number.Value > _lastTurn, + "A completed turn lacks a fresh ordinal."); + Require(output.TurnOutcome == "completed", "The parent turn did not complete normally."); + _lastTurn = output.TurnNumber!.Value.Value; + LastReply = _text.ToString(); + _text.Clear(); + _hasDelta = false; + CompletedTurns++; + break; + case SessionOutputTypes.Error: + throw new InvalidDataException("The daemon emitted an error: " + output.ErrorMessage); + } + } + + public void ConfirmHeld(JsonElement snapshot, string nonce, string slot) + { + Require(!Released && !ChildHeld, "The held-child barrier was repeated or followed release."); + Require(Acceptance is not null, "The child binding precedes acceptance."); + Require(slot == "child" && snapshot.GetProperty("nonce").GetString() == nonce, + "The fixture acknowledged a foreign trial."); + var binding = snapshot.GetProperty("binding"); + Require(binding.GetProperty("accepted").GetProperty("run_id").GetString() == Acceptance!.RunId + && binding.GetProperty("accepted").GetProperty("scope_id").GetString() == Acceptance.ScopeId + && binding.GetProperty("request_id").GetInt32() > 0, + "The fixture did not bind the exact accepted child request."); + ChildHeld = true; + } + + public void ConfirmRelease(string probeMarker) + { + Require(ChildHeld && !Released && Acceptance is not null && CompletedTurns == 2, + "Child release requires acceptance and two parent turn boundaries under the barrier."); + Require(!string.IsNullOrWhiteSpace(probeMarker) + && LastReply.Contains(probeMarker, StringComparison.Ordinal), + "The second parent reply lacks the independent probe marker."); + Released = true; + } + + public static AcceptedRun ParseAcceptance(string body) + { + using var document = JsonDocument.Parse(body); + Require(document.RootElement.ValueKind == JsonValueKind.Object, "Acceptance is not one JSON object."); + var fields = new Dictionary(StringComparer.Ordinal); + foreach (var field in document.RootElement.EnumerateObject()) + { + Require(field.Value.ValueKind == JsonValueKind.String && fields.TryAdd(field.Name, field.Value.GetString()!), + "Acceptance contains a duplicate or non-string field."); + } + foreach (var key in new[] { "run_id", "scope_id", "state", "control_tool" }) + Require(fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value), + "Acceptance lacks a nonempty required field: " + key); + Require(fields["state"] == "Accepted" && fields["control_tool"] == "check_agent_run", + "Acceptance has the wrong initial state or control tool."); + return new AcceptedRun(fields["run_id"], fields["scope_id"]); + } + + private static void Require(bool condition, string message) + { + if (!condition) + throw new InvalidDataException(message); + } +} diff --git a/evals/fixtures/child-runs/ProtocolControls.cs b/evals/fixtures/child-runs/ProtocolControls.cs new file mode 100644 index 000000000..e6886a86f --- /dev/null +++ b/evals/fixtures/child-runs/ProtocolControls.cs @@ -0,0 +1,117 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Netclaw.Actors.Protocol; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Evals.ChildRuns; + +internal static class ProtocolControls +{ + private const string Session = "signalr/observer-control"; + private const string Body = "{\"run_id\":\"owner-run\",\"scope_id\":\"owner-scope\",\"state\":\"Accepted\",\"control_tool\":\"check_agent_run\"}"; + + public static int Run() + { + var count = 0; + foreach (var malformed in new[] + { + "{}", "[" + Body + "]", "prose " + Body, "```json\n" + Body + "\n```", + Change("\"owner-run\"", "null"), Change("owner-scope", " "), + Change("Accepted", "Completed"), Change("check_agent_run", "check_background_job"), + Change("\"run_id\":\"owner-run\"", "\"run_id\":\"owner-run\",\"run_id\":\"other\"") + }) + Reject(() => HeldChildProtocol.ParseAcceptance(malformed)); + var accepted = HeldChildProtocol.ParseAcceptance(" \n" + Body + "\n"); + Check(accepted == new AcceptedRun("owner-run", "owner-scope")); + var valid = New(); + using var barrier = JsonDocument.Parse("{\"nonce\":\"trial\",\"binding\":{\"accepted\":{\"run_id\":\"owner-run\",\"scope_id\":\"owner-scope\"},\"request_id\":1}}"); + AddAcceptance(valid); + valid.ConfirmHeld(barrier.RootElement, "trial", "child"); + AddTurn(valid, 1, "The child was accepted."); + AddTurn(valid, 2, "probe-answer"); + valid.ConfirmRelease("probe-answer"); + Check(valid.Released && valid.CompletedTurns == 2); + Reject(() => New().ConfirmRelease("probe-answer")); + var early = New(); AddAcceptance(early); early.ConfirmHeld(barrier.RootElement, "trial", "child"); + AddTurn(early, 1, "probe-answer"); + Reject(() => early.ConfirmRelease("probe-answer")); + var wrong = New(); AddAcceptance(wrong); wrong.ConfirmHeld(barrier.RootElement, "trial", "child"); + AddTurn(wrong, 1, "accepted"); AddTurn(wrong, 2, "different"); + Reject(() => wrong.ConfirmRelease("probe-answer")); + Reject(() => New().ConfirmHeld(barrier.RootElement, "foreign", "child")); + foreach (var malformedBinding in new[] + { + "{\"nonce\":\"trial\",\"binding\":{\"accepted\":{\"run_id\":\"foreign\",\"scope_id\":\"owner-scope\"},\"request_id\":1}}", + "{\"nonce\":\"trial\",\"binding\":{\"accepted\":{\"run_id\":\"owner-run\",\"scope_id\":\"foreign\"},\"request_id\":1}}", + "{\"nonce\":\"trial\",\"binding\":{\"accepted\":{\"run_id\":\"owner-run\",\"scope_id\":\"owner-scope\"},\"request_id\":0}}" + }) + { + var invalid = New(); + AddAcceptance(invalid); + using var snapshot = JsonDocument.Parse(malformedBinding); + Reject(() => invalid.ConfirmHeld(snapshot.RootElement, "trial", "child")); + } + Reject(() => New().Observe(Result())); + var duplicate = New(); AddAcceptance(duplicate); + Reject(() => duplicate.Observe(Result())); + Reject(() => New().Observe(Dto(new TextOutput("foreign") { SessionId = new SessionId("foreign") }))); + Reject(() => valid.Observe(Dto(new TurnCompleted + { SessionId = new SessionId(Session), TurnNumber = new TurnNumber(2) }))); + Reject(() => New().Observe(Dto(new TurnCompleted + { SessionId = new SessionId(Session), TurnNumber = new TurnNumber(1), Outcome = TurnOutcome.Failed }))); + var mismatch = New(); mismatch.Observe(Call()); + Reject(() => mismatch.Observe(Result() with { ToolName = "skill_load" })); + var deltas = New(); + deltas.Observe(Dto(new TextDeltaOutput("one") { SessionId = new SessionId(Session) })); + deltas.Observe(Dto(new TextOutput("one") { SessionId = new SessionId(Session) })); + deltas.Observe(Dto(new TurnCompleted { SessionId = new SessionId(Session), TurnNumber = new TurnNumber(1) })); + Check(deltas.LastReply == "one"); + var reused = New(); + var readCall = Call() with { ToolName = "file_read" }; + var readResult = Result() with { ToolName = "file_read", Result = "neutral read result" }; + reused.Observe(readCall); + Reject(() => reused.Observe(readCall)); + reused.Observe(readResult); + AddTurn(reused, 1, "Initial acknowledgement with REQUIRED-MARKER"); + reused.Observe(readCall); + reused.Observe(readResult); + AddTurn(reused, 2, "Actual final response"); + Check(reused.CompletedTurns == 2 && reused.LastReply == "Actual final response" + && !reused.LastReply.Contains("REQUIRED-MARKER", StringComparison.Ordinal)); + Reject(() => reused.Observe(readResult)); + Console.WriteLine(JsonSerializer.Serialize(new { passed = count, failed = 0, scope = "protocol/barrier controls; no model or daemon proof" })); + return 0; + + void Check(bool condition) + { + if (!condition) throw new InvalidDataException("A positive protocol control failed."); + count++; + } + void Reject(Action action) + { + try { action(); } + catch (Exception error) when (error is InvalidDataException or JsonException) { count++; return; } + throw new InvalidDataException("A negative protocol control accepted unsafe evidence."); + } + } + + private static string Change(string before, string after) => Body.Replace(before, after, StringComparison.Ordinal); + private static HeldChildProtocol New() => new(Session); + private static SessionOutputDto Dto(SessionOutput output) => + SessionOutputDtoMapper.ToDto(SessionOutputDtoMapper.FromDto(SessionOutputDtoMapper.ToDto(output))); + private static SessionOutputDto Call() => Dto(new ToolCallOutput + { SessionId = new SessionId(Session), CallId = new ToolCallId("start"), ToolName = new ToolName("spawn_agent") }); + private static SessionOutputDto Result() => Dto(new ToolResultOutput + { SessionId = new SessionId(Session), CallId = new ToolCallId("start"), ToolName = new ToolName("spawn_agent"), Result = Body }); + private static void AddAcceptance(HeldChildProtocol protocol) { protocol.Observe(Call()); protocol.Observe(Result()); } + private static void AddTurn(HeldChildProtocol protocol, int number, string text) + { + protocol.Observe(Dto(new TextOutput(text) { SessionId = new SessionId(Session) })); + protocol.Observe(Dto(new TurnCompleted { SessionId = new SessionId(Session), TurnNumber = new TurnNumber(number) })); + } +} diff --git a/evals/fixtures/child-runs/README.md b/evals/fixtures/child-runs/README.md new file mode 100644 index 000000000..6d094fe5e --- /dev/null +++ b/evals/fixtures/child-runs/README.md @@ -0,0 +1,123 @@ +# Persistent child-run eval consumer + +The helper uses one real `DaemonClient` for each trial. +It records actual `SessionOutputDto` values through the public mapper. +It does not change the product CLI or attach another session connection. + +## Fixed flow + +1. Create a fresh headless session and submit one child task. +2. Read the paired canonical acceptance before a child ownership assertion. +3. Observe the initial parent reply. +4. Wait for the first actual upstream child response payload, then submit one independent parent probe. +5. Read an authorized status result and the actual child log. +6. Bind the held provider request to the accepted identifiers and returned paths. +7. Require the probe reply before child release. +8. Release the held upstream response payload. +9. Observe actual terminal consumption and the parent response without another user input. +10. Verify commit logs, actual provider history, and actual artifact bytes. + +The partial-cancel case holds an upstream response after a paired artifact-write result. +The parent probe reads the live log before it requests cancellation. +The harness then releases the old response payload and checks the partial evidence. +The oracle rejects any later child provider admission. +The cancel result can share the second parent probe response. +A fast result can share the initial response in a legacy case. +Compatible sibling results can share one continuation. +The consumer requires actual terminal call/result pairs and later response bytes in parent provider requests. +It does not assign one turn to each accepted child. +Local cancellation does not prove that remote computation stopped. + +The relay forwards actual requests and responses through `background_fixture.py`. +It saves requests before it forwards them. +The barrier holds the first actual successful upstream response payload before Netclaw receives that payload. +The receipt separates upstream payload arrival from local payload delivery. +This proves upstream request and response activity. +It does not prove durable upstream admission or remote effect completion. +It copies response bytes from the actual forward path. +It does not replace parent or child responses. +A nonce, arrival order, or process ID alone cannot establish child ownership. + +## Local controls + +```bash +python3 -m unittest discover -s evals -p test_child_run_evals.py -v +python3 -m unittest discover -s evals -p test_background_evals.py -v +# Run these commands only after the combined source and compiler window are available. +dotnet build evals/fixtures/child-runs/SessionObserver.csproj -c Release +dotnet evals/fixtures/child-runs/bin/Release/net10.0/SessionObserver.dll --protocol-controls +``` + +The Python controls reject foreign paths, missing commit logs, extra actions, and false artifact claims. +The C# controls use the real mapper and reject malformed acceptance, unmatched results, stale turns, and early release. +Neither control suite proves real model behavior. + +## Model execution + +Set `NETCLAW_CHILD_OBSERVER` to the compiled observer DLL from the exact combined source. +Use the existing provider, image, CLI, and asset variables from the eval README. +Select one exact case with `NETCLAW_EVAL_RUNS=1` for each invocation. +Use five separate fresh invocations for each critical case. +Each invocation owns its container, home, session, relay, recorder, and archive. +Multiple turns in one trial retain its one connection and container. + +| Case | Required evidence | +| --- | --- | +| `child_run_held_parent` | Acceptance, bound child request, independent parent status/log reply, later result, and actual artifact read | +| `child_run_partial_cancel` | Confirmed partial write, bound request, parent cancellation, cancelled evidence, and no later child request | +| `child_run_cli_acceptance` | Actual CLI acceptance matches the model tool result and a committed accepted record | + +`run-background-evals.sh` owns these cases. +The existing queue-grant and shell lifecycle paths stay separate. +The CLI smoke proves initial acceptance only. +The persistent cases prove later result observation separately. + +The six affected cases in `run-evals.sh` use the persistent consumer. +Select each case explicitly with one fresh invocation. +The adapter emits an eval response envelope; it does not claim to emit the CLI envelope. +The response field contains the actual final parent reply. +Each prompt owns a fresh nonce, prompt digest, input file, and receipt directory. +The final-response assertion reads only the latest prompt directory. +A failed latest prompt cannot select an earlier successful receipt. +The raw receipt retains all earlier parent replies. + +The worktree handoff case has four prompts. +Prompts 1, 2, and 4 require one parent turn; they do not require child acceptance. +Prompt 3 requires acceptance and actual later terminal consumption. +An observer failure fails the case before its assertions. +A tool call ID identifies one pending occurrence; later completed occurrences can reuse that ID. +Text output labels canonical provider-history results with `[child:result]`. +The adapter never invents a second result for the original start call. +The old shell/project/artifact assertions still apply. +The adapter takes child log paths from canonical terminal data. + +## Evidence and limits + +The archive retains observer inputs, canonical DTOs, provider bytes, snapshots, and trial receipts under `child-runs/`. +The oracle requires these distinct post-commit diagnostics: + +- `child_run_accepted` +- `child_run_terminal_recorded` +- `child_run_result_prepared` +- `child_run_delivery_admitted`, with the actual input and terminal call identifiers + +A terminal call/result pair must occur in the actual parent provider request. +Its source operation must match the actual original start call. +Repeated copies of that pair in later request history count as one delivery. +A later turn alone cannot establish admission or terminal attribution. +No private journal decoder, test endpoint, or provider-capacity controller exists here. + +The fixed partial-cancel case requires a successful framework report. +Its exact checkpoint must confirm the partial artifact and its actual tool result. +The actual report must match the run, state, summary, and confirmed activity. +Explicit report-grace failures remain separate outcomes; they cannot pass this fixed success case. + +A foreign child with the same nonce can reach the sole candidate barrier. +The exact path bind rejects that child. +This fixed one-child relay does not prove selection among several candidates. +The separate multi-child gate remains open. + +The current source preparation does not prove a build or a model pass. +The combined runtime must supply the post-commit result-preparation diagnostic. +The final stack still needs the canonical build, protocol controls, and fresh model trials. +Routed-start, approval-lifetime, recovery, and coordination cases retain their separate acceptance gates. diff --git a/evals/fixtures/child-runs/SessionObserver.cs b/evals/fixtures/child-runs/SessionObserver.cs new file mode 100644 index 000000000..3e0db6d64 --- /dev/null +++ b/evals/fixtures/child-runs/SessionObserver.cs @@ -0,0 +1,251 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Diagnostics; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading.Channels; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; +using Netclaw.Cli.Daemon; +using R3; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Evals.ChildRuns; + +internal sealed record ObserverInput( + string DaemonEndpoint, string FixtureEndpoint, string Nonce, string InitialPrompt, + string ProbePrompt, string ProbeMarker, string EvidenceDirectory, int TimeoutSeconds, + string Mode, string SessionId, string OutputFormat); + +internal static class SessionObserver +{ + public static async Task Main(string[] args) + { + if (args is ["--protocol-controls"]) + return ProtocolControls.Run(); + if (args.Length != 1) + throw new ArgumentException("Supply one observer input JSON file or --protocol-controls."); + var input = JsonSerializer.Deserialize(await File.ReadAllTextAsync(args[0])) + ?? throw new InvalidDataException("Observer input is null."); + Validate(input); + Directory.CreateDirectory(input.EvidenceDirectory); + var receiptPath = Path.Combine(input.EvidenceDirectory, "observer-receipt.json"); + if (File.Exists(receiptPath)) + throw new IOException("The observer receipt already exists."); + await using var events = new StreamWriter(new FileStream( + Path.Combine(input.EvidenceDirectory, "session-output.jsonl"), FileMode.CreateNew, FileAccess.Write)) + { AutoFlush = true }; + await using var client = new DaemonClient(input.DaemonEndpoint); + var outputs = Channel.CreateUnbounded<(SessionOutput Output, long ObservedNs)>(new UnboundedChannelOptions + { SingleReader = true, SingleWriter = false }); + using var subscription = client.SessionOutput.Subscribe(output => outputs.Writer.TryWrite((output, MonotonicNs()))); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(input.TimeoutSeconds)); + using var http = new HttpClient(); + var token = deadline.Token; + var sessionId = ""; + var sequence = 0; + var userInputs = 0; + long firstTurnNs = 0, secondTurnNs = 0, releaseNs = 0; + HeldChildProtocol? protocol = null; + var pendingCalls = new Dictionary(StringComparer.Ordinal); + var calls = new List(); + var starts = new Dictionary(StringComparer.Ordinal); + var acceptances = new Dictionary(StringComparer.Ordinal); + var transcript = new StringBuilder(); + var replies = new List(); + var deliveryObservations = JsonSerializer.SerializeToElement(new { complete = false, deliveries = Array.Empty() }); + var bound = false; + var statusBodies = new List(); + try + { + await client.ConnectAsync(token); + sessionId = string.IsNullOrEmpty(input.SessionId) + ? await client.CreateSessionAsync(ChannelType.Headless, token) + : await client.ResumeSessionAsync(input.SessionId, ChannelType.Headless, token); + protocol = new HeldChildProtocol(sessionId); + await SendAsync(input.InitialPrompt); + await ReadThroughAsync(() => protocol.CompletedTurns >= 1); + if (input.Mode == "collect") + { + Require(acceptances.Count > 0, "The initial parent turn lacks an accepted child."); + await ReadThroughAsync(() => deliveryObservations.GetProperty("complete").GetBoolean()); + } + else if (input.Mode != "turn") + { + Require(protocol.Acceptance is not null && !string.IsNullOrWhiteSpace(protocol.LastReply), + "The initial parent turn lacks acceptance or a visible reply."); + // This acknowledgement proves candidate arrival only. Status binds ownership below. + using var candidate = await ControlAsync("child-wait", new { }); + await SendAsync(input.ProbePrompt.Replace("{{RUN_ID}}", protocol.Acceptance!.RunId, StringComparison.Ordinal)); + await ReadThroughAsync(() => protocol.CompletedTurns >= 2); + Require(bound, "The parent probe lacks an authorized live status for child ownership."); + protocol.ConfirmRelease(input.ProbeMarker); + using var released = await ControlAsync("child-release", new { }); + releaseNs = MonotonicNs(); + await ReadThroughAsync(() => deliveryObservations.GetProperty("complete").GetBoolean()); + using var drained = await ControlAsync("child-drained", new { }); + } + await WriteReceiptAsync("observed", null); + if (input.OutputFormat == "json") + Console.WriteLine(JsonSerializer.Serialize(new + { + sessionId, response = protocol.LastReply, + toolCalls = calls.Select(call => new + { callId = call.Id, toolName = call.Name, argumentsJson = call.Arguments.GetRawText() }), + evidenceSource = "persistent DaemonClient output; not the headless CLI envelope" + })); + else + Console.Write(transcript.ToString()); + return 0; + } + catch (Exception error) when (error is not OutOfMemoryException) + { + await WriteReceiptAsync("incomplete", error.ToString()); + throw; + } + + async Task SendAsync(string prompt) + { + userInputs++; + await client.SendAsync(prompt, token); + } + + async Task ControlAsync(string action, object body) + { + using var response = await http.PostAsJsonAsync( + input.FixtureEndpoint.TrimEnd('/') + "/control/" + action, body, token); + var text = await response.Content.ReadAsStringAsync(token); + Require(response.IsSuccessStatusCode, "The fixture rejected the control: " + text); + return JsonDocument.Parse(text); + } + + async Task ReadThroughAsync(Func complete) + { + var currentProtocol = protocol ?? throw new InvalidDataException("The observer session is not initialized."); + while (!complete()) + { + var output = await outputs.Reader.ReadAsync(token); + var dto = SessionOutputDtoMapper.ToDto(output.Output); + await events.WriteLineAsync(JsonSerializer.Serialize(new + { sequence = ++sequence, observed_ns = output.ObservedNs, output = dto })); + currentProtocol.Observe(dto); + if (dto.Type == SessionOutputTypes.ToolCall) + { + using var arguments = JsonDocument.Parse(dto.ArgumentsJson ?? "{}"); + var call = new ObservedCall(dto.CallId!, dto.ToolName!, arguments.RootElement.Clone(), + currentProtocol.CompletedTurns + 1, output.ObservedNs, calls.Count + 1); + Require(pendingCalls.TryAdd(dto.CallId!, call), "A pending call identifier repeats within its occurrence."); + calls.Add(call); + transcript.AppendLine($"[tool:call] {dto.ToolName}({dto.ArgumentsJson})"); + } + else if (dto.Type == SessionOutputTypes.ToolResult) + { + Require(pendingCalls.Remove(dto.CallId!, out var matched), "The result lacks its pending call occurrence."); + var call = matched!; + call.Result = dto.Result ?? ""; + call.Success = dto.ToolFailureCode is null; + transcript.AppendLine($"[tool:result] {dto.ToolName} → {dto.Result}"); + if (dto.ToolName == "spawn_agent" && call.Success) + { + var accepted = HeldChildProtocol.ParseAcceptance(call.Result); + Require(acceptances.TryAdd(accepted.RunId, accepted), "The accepted run identifier repeats."); + starts.Add(accepted.RunId, call); + } + if (dto.ToolName == "check_agent_run" && call.Success) + { + using var status = JsonDocument.Parse(call.Result); + statusBodies.Add(status.RootElement.Clone()); + if (!bound && (input.Mode is "held" or "cancel") + && status.RootElement.TryGetProperty("state", out var state) + && state.GetString() is "Accepted" or "Running" or "Cancelling") + { + Require(currentProtocol.Acceptance is not null, "Status precedes the paired start acceptance."); + using var held = await ControlAsync("child-bind", new + { + accepted = AcceptanceBody(currentProtocol.Acceptance!), status = status.RootElement + }); + currentProtocol.ConfirmHeld(held.RootElement, input.Nonce, "child"); + bound = true; + } + } + } + else if (dto.Type == SessionOutputTypes.TurnCompleted) + { + if (firstTurnNs == 0) + firstTurnNs = output.ObservedNs; + if (currentProtocol.CompletedTurns == 2) + secondTurnNs = output.ObservedNs; + transcript.AppendLine(currentProtocol.LastReply); + replies.Add(currentProtocol.LastReply); + using var consumption = await ControlAsync("child-consumed", new + { + expected = acceptances.Values.Select(accepted => new + { accepted = AcceptanceBody(accepted), call_id = starts[accepted.RunId].Id, + source_operation = starts[accepted.RunId].Name }), + parent_boundary_ns = output.ObservedNs + }); + deliveryObservations = consumption.RootElement.Clone(); + } + } + } + + async Task WriteReceiptAsync(string status, string? error) + { + await File.WriteAllTextAsync(receiptPath, JsonSerializer.Serialize(new + { + status, session_id = sessionId, prompt_nonce = input.Nonce, observer_mode = input.Mode, + initial_prompt_sha256 = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(input.InitialPrompt))).ToLowerInvariant(), + accepted_run = protocol?.Acceptance is { } accepted ? AcceptanceBody(accepted) : null, + accepted_runs = acceptances.Values.Select(AcceptanceBody), + completed_turns = protocol?.CompletedTurns, user_inputs = userInputs, + first_turn_ns = firstTurnNs, second_turn_ns = secondTurnNs, release_ns = releaseNs, + last_reply = protocol?.LastReply, all_replies = replies, delivery_observations = deliveryObservations, + status_bodies = statusBodies, error, + calls = calls.Select(call => new + { id = call.Id, occurrence = call.Occurrence, observed_ns = call.ObservedNs, + name = call.Name, arguments = call.Arguments, turn = call.Turn, success = call.Success, result = call.Result }), + limit = "Post-commit diagnostics, actual provider history, and files require the separate Python oracle." + }, new JsonSerializerOptions { WriteIndented = true })); + } + } + + private static object AcceptanceBody(AcceptedRun accepted) => new + { run_id = accepted.RunId, scope_id = accepted.ScopeId, state = "Accepted", control_tool = "check_agent_run" }; + private static long MonotonicNs() => (long)(Stopwatch.GetTimestamp() * (1_000_000_000d / Stopwatch.Frequency)); + private static void Require(bool condition, string message) + { + if (!condition) throw new InvalidDataException(message); + } + + private static void Validate(ObserverInput input) + { + foreach (var value in new[] { input.DaemonEndpoint, input.FixtureEndpoint, input.Nonce, + input.InitialPrompt, input.EvidenceDirectory }) + Require(!string.IsNullOrWhiteSpace(value), "An observer input field is empty."); + foreach (var endpoint in new[] { input.DaemonEndpoint, input.FixtureEndpoint }) + Require(Uri.TryCreate(endpoint, UriKind.Absolute, out var uri) && uri.IsLoopback, + "The observer requires task-owned loopback endpoints."); + Require(input.TimeoutSeconds > 0 && input.Mode is "held" or "cancel" or "collect" or "turn", "The observer mode or deadline is invalid."); + Require(input.OutputFormat is "text" or "json", "The observer output format is invalid."); + if (input.Mode is "held" or "cancel") + Require(!string.IsNullOrWhiteSpace(input.ProbeMarker) + && input.ProbePrompt.Contains(input.ProbeMarker, StringComparison.Ordinal), "The probe lacks its reply marker."); + } + + private sealed class ObservedCall(string id, string name, JsonElement arguments, int turn, long observedNs, int occurrence) + { + public string Id { get; } = id; + public int Turn { get; } = turn; + public long ObservedNs { get; } = observedNs; + public int Occurrence { get; } = occurrence; + public string Name { get; } = name; + public JsonElement Arguments { get; } = arguments; + public string Result { get; set; } = ""; + public bool Success { get; set; } + } +} diff --git a/evals/fixtures/child-runs/SessionObserver.csproj b/evals/fixtures/child-runs/SessionObserver.csproj new file mode 100644 index 000000000..44916f784 --- /dev/null +++ b/evals/fixtures/child-runs/SessionObserver.csproj @@ -0,0 +1,11 @@ + + + net10.0 + Exe + enable + enable + + + + + diff --git a/evals/run-background-evals.sh b/evals/run-background-evals.sh index f219d2344..b532753ee 100755 --- a/evals/run-background-evals.sh +++ b/evals/run-background-evals.sh @@ -14,6 +14,15 @@ elif [[ $# != 0 ]]; then fi case "$FILTER_CASE" in ""|queued_grant_revoked) ;; + child_run_held_parent|child_run_partial_cancel|child_run_cli_acceptance) + [[ "${1:-}" != --runtime-only && "$RUNS" == 1 ]] || { + echo "ERROR: child cases require a real model and RUNS=1 per fresh invocation." >&2 + exit 2 + } + [[ -f "${NETCLAW_CHILD_OBSERVER:-}" ]] || { + echo "ERROR: set NETCLAW_CHILD_OBSERVER to the compiled canonical observer." >&2 + exit 2 + } ;; tool_background_job_lifecycle) if [[ "${1:-}" == --runtime-only ]]; then echo "ERROR: the lifecycle case requires a real model." >&2 @@ -35,7 +44,12 @@ build_local_image export BACKGROUND_EVAL_UPSTREAM="$EVAL_PROVIDER_ENDPOINT" export NETCLAW_EVAL_MODEL_ID="$EVAL_MODEL_ID" -coproc BACKGROUND_FIXTURE { exec python3 "$REPO_ROOT/evals/background_evals.py" serve; } +fixture_module=background_evals.py +if [[ "$FILTER_CASE" == child_run_* ]]; then + fixture_module=child_run_evals.py +fi +export TMPDIR_EVAL EVAL_HOME +coproc BACKGROUND_FIXTURE { exec python3 "$REPO_ROOT/evals/$fixture_module" serve; } fixture_pid=$BACKGROUND_FIXTURE_PID trap 'cleanup_eval_env; kill "$fixture_pid" 2>/dev/null || true; wait "$fixture_pid" 2>/dev/null || true' EXIT read -r -t 15 fixture_port <&"${BACKGROUND_FIXTURE[0]}" @@ -57,9 +71,9 @@ FILTER_CATEGORY="Background launch" THRESHOLD=1 NETCLAW_VER=$("$NETCLAW_BIN" --version) start_eval_daemon -export EVAL_HOME EVAL_PORT EVAL_CONTAINER_NAME NETCLAW_BIN TMPDIR_EVAL RUNS PROMPT_TIMEOUT +export EVAL_HOME EVAL_PORT EVAL_CONTAINER_NAME NETCLAW_BIN NETCLAW_IMAGE TMPDIR_EVAL RUNS PROMPT_TIMEOUT result=0 -python3 "$REPO_ROOT/evals/background_evals.py" run --port "$fixture_port" "$@" || result=$? +python3 "$REPO_ROOT/evals/$fixture_module" run --port "$fixture_port" "$@" || result=$? report="$TMPDIR_EVAL/stdout_background-results.txt" if [[ -s "$report" ]]; then TOTAL_CASES=$(jq '[.runtime[], .model[]] | length' "$report") diff --git a/evals/run-evals.sh b/evals/run-evals.sh index 1c0fe2b3e..d31558bb4 100755 --- a/evals/run-evals.sh +++ b/evals/run-evals.sh @@ -217,6 +217,10 @@ cleanup_eval_env() { kill "$CYCLE_FIXTURE_PID_SAVED" 2>/dev/null || true wait "$CYCLE_FIXTURE_PID_SAVED" 2>/dev/null || true fi + if [[ -n "${CHILD_FIXTURE_PID_SAVED:-}" ]]; then + kill "$CHILD_FIXTURE_PID_SAVED" 2>/dev/null || true + wait "$CHILD_FIXTURE_PID_SAVED" 2>/dev/null || true + fi # TMPDIR_EVAL only holds host-owned per-prompt stdout/stderr captures, so a # plain rm always succeeds — no force_rmrf fallback needed. if [[ -n "${TMPDIR_EVAL:-}" && -d "$TMPDIR_EVAL" ]]; then @@ -277,6 +281,9 @@ archive_eval_run() { mkdir -p "$archive_dir/stdout" cp "$TMPDIR_EVAL"/stdout_*.txt "$archive_dir/stdout/" 2>/dev/null || true cp "$TMPDIR_EVAL"/stderr_*.txt "$archive_dir/stdout/" 2>/dev/null || true + if [[ -d "$TMPDIR_EVAL/child-runs" ]]; then + cp -r "$TMPDIR_EVAL/child-runs" "$archive_dir/child-runs" + fi fi # Write run metadata, including the immutable image identity so before/after @@ -944,6 +951,27 @@ check_daemon_alive() { } +child_result_consumer() { + case "${case_name:-$FILTER_CASE}" in + subagent_headless_ambiguous_task|subagent_specialization_precedence|subagent_project_scope_declaration|subagent_session_scratch_disposable|approval_natural_subagent_project_review|coding_context_worktree_handoff) return 0 ;; + *) return 1 ;; + esac +} + +observe_child_result() { + local prompt="$1" session="$2" output_format="$3" prompt_ordinal="${4:-1}" + CHILD_LAST_EVIDENCE="$TMPDIR_EVAL/child-runs/observer-$(date +%s%N)" + local prompt_file="$TMPDIR_EVAL/child-prompt-$(date +%s%N).txt" + printf '%s' "$prompt" > "$prompt_file" + python3 "$REPO_ROOT/evals/child_run_evals.py" collect --port "$CHILD_FIXTURE_PORT" \ + --prompt-file "$prompt_file" --session "$session" --format "$output_format" --evidence "$CHILD_LAST_EVIDENCE" \ + --case "${case_name:-$FILTER_CASE}" --prompt-ordinal "$prompt_ordinal" +} + +assert_child_delivery_receipt() { + python3 "$REPO_ROOT/evals/child_run_evals.py" assert-delivery --evidence "$TMPDIR_EVAL/child-runs" --agent "${1:-}" +} + run_prompt() { local prompt="$1" local output_format="${2:-text}" @@ -974,13 +1002,21 @@ run_prompt() { # "[error] Unknown output type from daemon: ...") to stderr, and a # trailing diagnostic line breaks jq's parse of the JSON envelope on # stdout, producing a false eval failure rather than a real one. + if child_result_consumer; then + [[ -n "${CHILD_FIXTURE_PORT:-}" ]] || { echo "ERROR: select this child case explicitly with RUNS=1." >&2; return 2; } + observe_child_result "$prompt" "" "$output_format" > "$STDOUT_FILE" 2> "$STDERR_FILE" || return $? + else NETCLAW_DAEMON_ENDPOINT="http://127.0.0.1:$EVAL_PORT" \ NETCLAW_HOME="$EVAL_HOME" \ timeout "$PROMPT_TIMEOUT" stdbuf -oL -eL "$NETCLAW_BIN" chat -p "${output_args[@]}" "$prompt" \ > "$STDOUT_FILE" 2> "$STDERR_FILE" || true - # Brief pause for daemon log flush - sleep 2 + fi + + # Preserve the existing CLI flush path. The child adapter uses explicit evidence. + if ! child_result_consumer; then + sleep 2 + fi } ## Runs a prompt against an existing (or new) named session via `chat -p --resume`. @@ -993,6 +1029,8 @@ run_prompt_resume() { local session_id="$1" local prompt="$2" local output_format="${3:-text}" + local prompt_ordinal="${4:-1}" + local observer_exit=0 local ts ts="$(date +%s%N)" local turn_file="$TMPDIR_EVAL/stdout_${ts}_turn.txt" @@ -1028,12 +1066,18 @@ run_prompt_resume() { output_args+=(--json) fi + if child_result_consumer; then + [[ -n "${CHILD_FIXTURE_PORT:-}" ]] || { echo "ERROR: select this child case explicitly with RUNS=1." >&2; return 2; } + observe_child_result "$prompt" "$session_id" "$output_format" "$prompt_ordinal" > "$turn_file" 2> "$turn_stderr_file" || observer_exit=$? + else NETCLAW_DAEMON_ENDPOINT="http://127.0.0.1:$EVAL_PORT" \ NETCLAW_HOME="$EVAL_HOME" \ timeout "$PROMPT_TIMEOUT" stdbuf -oL -eL "$NETCLAW_BIN" chat -p --resume "$session_id" \ "${output_args[@]}" "$prompt" \ > "$turn_file" 2> "$turn_stderr_file" || true + fi + # Append this turn's output to the shared files so assertions see all turns. cat "$turn_file" >> "$STDOUT_FILE" cat "$turn_stderr_file" >> "$STDERR_FILE" @@ -1050,7 +1094,10 @@ run_prompt_resume() { LAST_TURN_USAGE_LINE=$(grep -ao '\[usage\].*' "$turn_file" 2>/dev/null | tail -1 || echo "") fi - sleep 2 + if ! child_result_consumer; then + sleep 2 + fi + return "$observer_exit" } ## Runs a named multi-turn case. Each prompt is sent via --resume against @@ -1093,6 +1140,7 @@ run_multi_turn_case() { fi local turn=1 + local prompt_failed=false local prompt for prompt in "${prompts[@]}"; do local rendered_prompt="$prompt" @@ -1101,15 +1149,19 @@ run_multi_turn_case() { rendered_prompt="${rendered_prompt//\{\{TARGET_BRANCH\}\}/${CODING_CONTEXT_TARGET_BRANCH:-}}" rendered_prompt="${rendered_prompt//\{\{TARGET_FILE\}\}/${CODING_CONTEXT_TARGET_FILE:-}}" rendered_prompt="${rendered_prompt//\{\{DIRECT_ATTACHMENT_SOURCE\}\}/${DIRECT_ATTACHMENT_SOURCE_PATH:-}}" - run_prompt_resume "$session_id" "$rendered_prompt" "$output_format" + if ! run_prompt_resume "$session_id" "$rendered_prompt" "$output_format" "$turn"; then + prompt_failed=true + break + fi store_metrics "$case_name" "$run" "$turn" "$LAST_TURN_USAGE_LINE" turn=$((turn + 1)) done local passed=0 local details="fail" + [[ "$prompt_failed" == false ]] || details="observer_failed" EVAL_ASSERTION_DETAILS="" - if $assert_fn 2>/dev/null; then + if [[ "${prompt_failed:-false}" == false ]] && $assert_fn 2>/dev/null; then passed=1 passes=$((passes + 1)) details="pass" @@ -1155,10 +1207,22 @@ stdout_not_contains() { } stdout_response_contains() { + if child_result_consumer; then + local child_final_response + child_final_response=$(python3 "$REPO_ROOT/evals/child_run_evals.py" final-response --evidence "${CHILD_LAST_EVIDENCE:?The current observer evidence path is absent.}") || return 1 + grep -qia "$1" <<< "$child_final_response" + return + fi grep -av '^\[tool:call\]' "$STDOUT_FILE" 2>/dev/null | grep -qia "$1" } stdout_response_not_contains() { + if child_result_consumer; then + local child_final_response + child_final_response=$(python3 "$REPO_ROOT/evals/child_run_evals.py" final-response --evidence "${CHILD_LAST_EVIDENCE:?The current observer evidence path is absent.}") || return 1 + ! grep -qia "$1" <<< "$child_final_response" + return + fi if grep -av '^\[tool:call\]' "$STDOUT_FILE" 2>/dev/null | grep -qia "$1"; then return 1 fi @@ -1832,7 +1896,7 @@ assert_deployment_mission_sales_email() { # Category 6b: Subagents assert_subagent_headless_ambiguous_task() { stdout_tool_called 'spawn_agent' && \ - stdout_contains '\[subagent:done\] headless-analyst (completed' && \ + assert_child_delivery_receipt headless-analyst && \ stdout_response_contains 'assumption' && \ stdout_response_not_contains 'which.*include' && \ stdout_response_not_contains 'what.*include' && \ @@ -1842,7 +1906,7 @@ assert_subagent_headless_ambiguous_task() { assert_subagent_specialization_precedence() { stdout_tool_called 'spawn_agent' && \ - stdout_contains '\[subagent:done\] headless-analyst (completed' && \ + assert_child_delivery_receipt headless-analyst && \ stdout_contains 'SPECIALIZED ANALYST BRIEF' && \ stdout_response_contains '^Subject:' && \ stdout_response_contains 'Would Tuesday or Wednesday work for a 15-minute call?' @@ -1870,15 +1934,14 @@ printf "%s\n" "// changed" >> /home/netclaw/.netclaw/workspaces/project-scope-ta assert_subagent_project_scope_declaration() { stdout_tool_called 'spawn_agent' || return 1 - stdout_contains '\[subagent:done\] project-scope-analyst (completed' || return 1 + assert_child_delivery_receipt || return 1 stdout_response_contains 'Project.csproj' || return 1 stdout_response_contains 'src' || return 1 local child_log local -a child_logs - mapfile -t child_logs < <(find "$EVAL_HOME/data/sessions" -type f \ - -path '*/subagents/*/logs/session.log' \ - -newer "$PROJECT_SCOPE_LOG_MARKER" 2>/dev/null) + mapfile -t child_logs < <(python3 "$REPO_ROOT/evals/child_run_evals.py" log-path \ + --evidence "$TMPDIR_EVAL/child-runs" --agent project-scope-analyst) [[ "${#child_logs[@]}" -eq 1 ]] || return 1 child_log="${child_logs[0]}" @@ -1937,14 +2000,13 @@ setup_approval_natural_subagent_project_review() { assert_approval_natural_subagent_project_review() { stdout_tool_called 'spawn_agent' || return 1 ! stdout_tool_called 'set_working_directory' || return 1 - stdout_contains '\[subagent:done\] project-scope-analyst (completed' || return 1 + assert_child_delivery_receipt || return 1 stdout_response_contains 'Project.csproj' || return 1 local child_log local -a child_logs - mapfile -t child_logs < <(find "$EVAL_HOME/data/sessions" -type f \ - -path '*/subagents/*/logs/session.log' \ - -newer "$PROJECT_SCOPE_LOG_MARKER" 2>/dev/null) + mapfile -t child_logs < <(python3 "$REPO_ROOT/evals/child_run_evals.py" log-path \ + --evidence "$TMPDIR_EVAL/child-runs" --agent project-scope-analyst) [[ "${#child_logs[@]}" -eq 1 ]] || return 1 child_log="${child_logs[0]}" @@ -1998,6 +2060,7 @@ setup_subagent_session_scratch_disposable() { assert_subagent_session_scratch_disposable() { stdout_json_tool_called 'spawn_agent' || return 1 + assert_child_delivery_receipt || return 1 local spawn_call child_task child_log spawn_call=$(stdout_json_tool_call_arguments 'spawn_agent' | head -1) @@ -2008,9 +2071,8 @@ assert_subagent_session_scratch_disposable() { <<<"$child_task" || return 1 local child_relative expected_temp_dir - child_log=$(find "$EVAL_HOME/data/sessions" -type f \ - -path '*/logs/session.log' \ - -newer "$SUBAGENT_SCRATCH_LOG_MARKER" 2>/dev/null | head -1) + child_log=$(python3 "$REPO_ROOT/evals/child_run_evals.py" log-path \ + --evidence "$TMPDIR_EVAL/child-runs" --agent disposable-diagnostic) [[ -n "$child_log" ]] || return 1 grep -aq \ 'SubAgent \[disposable-diagnostic\] completed (success=True, outcome=Completed' \ @@ -2084,6 +2146,7 @@ setup_coding_context_worktree_handoff() { } assert_coding_context_worktree_handoff() { + assert_child_delivery_receipt || return 1 if ! docker exec --user netclaw \ -e "EVAL_FIRST=$CODING_CONTEXT_FIRST" \ -e "EVAL_SECOND=$CODING_CONTEXT_SECOND" \ @@ -3042,11 +3105,15 @@ run_case() { rendered_prompt="${rendered_prompt//\{\{MANAGED_WORKTREE_BRANCH\}\}/${MANAGED_WORKTREE_BRANCH:-}}" rendered_prompt="${rendered_prompt//\{\{CYCLE_PROMPT\}\}/${CYCLE_PROMPT:-}}" rendered_prompt="${rendered_prompt//\{\{EVAL_REMINDER_TARGET\}\}/${EVAL_REMINDER_TARGET:-}}" - run_prompt "$rendered_prompt" "$output_format" + local prompt_failed=false + if ! run_prompt "$rendered_prompt" "$output_format"; then + prompt_failed=true + fi local passed=0 local details="fail" - if $assert_fn 2>/dev/null; then + [[ "$prompt_failed" == false ]] || details="observer_failed" + if [[ "${prompt_failed:-false}" == false ]] && $assert_fn 2>/dev/null; then passed=1 passes=$((passes + 1)) details="pass" @@ -3596,6 +3663,28 @@ main() { start_cycle_fixture cycle_cases=true fi + if child_result_consumer; then + [[ "$RUNS" == 1 && -f "${NETCLAW_CHILD_OBSERVER:-}" ]] || { + echo "ERROR: child consumers require RUNS=1 and NETCLAW_CHILD_OBSERVER." >&2 + exit 2 + } + [[ "$EVAL_PROVIDER_TYPE" == openai-compatible && "$EVAL_PROVIDER_ENDPOINT" == */v1 ]] || { + echo "ERROR: child consumers require the approved OpenAI-compatible target." >&2 + exit 2 + } + [[ "$EVAL_PROVIDER_API_KEY" != ENC:* && -z "$EVAL_DATA_PROTECTION_KEYS" ]] || { + echo "ERROR: the child relay requires a plain upstream key when authentication is needed." >&2 + exit 2 + } + export BACKGROUND_EVAL_UPSTREAM="$EVAL_PROVIDER_ENDPOINT" NETCLAW_EVAL_MODEL_ID="$EVAL_MODEL_ID" TMPDIR_EVAL + coproc CHILD_FIXTURE { exec python3 "$REPO_ROOT/evals/child_run_evals.py" serve; } + CHILD_FIXTURE_PID_SAVED=$CHILD_FIXTURE_PID + read -r -t 15 CHILD_FIXTURE_PORT <&"${CHILD_FIXTURE[0]}" + [[ "$CHILD_FIXTURE_PORT" =~ ^[0-9]+$ ]] + EVAL_PROVIDER_ENDPOINT="http://127.0.0.1:$CHILD_FIXTURE_PORT/v1" + EVAL_PROVIDER_API_KEY="" + export EVAL_HOME EVAL_PORT PROMPT_TIMEOUT NETCLAW_CHILD_OBSERVER TMPDIR_EVAL + fi start_eval_daemon init_db seed_eval_memories diff --git a/evals/test_child_run_evals.py b/evals/test_child_run_evals.py new file mode 100644 index 000000000..8b2c91cea --- /dev/null +++ b/evals/test_child_run_evals.py @@ -0,0 +1,639 @@ +"""Controls for child attribution, durable evidence, and actual-file oracles.""" + +import copy +from contextlib import contextmanager +import json +import hashlib +import io +import subprocess +from contextlib import redirect_stdout +import os +import re +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import tempfile +import threading +import time +import urllib.request +import unittest +from unittest.mock import patch + +from child_run_evals import (CHILD_CONTRACT, ChildFixture, acceptance, actual_file, bind_request, + canonical_pairs, child_handler, committed_positions, consumed_deliveries, collect, legacy_observer_mode, validate_prompt_receipt, verified_final_response, verify_child_actions, verify_cli_acceptance, verify_trial, write_completed) + +ACCEPTED = {"run_id": "run-neutral", "scope_id": "scope-neutral", "state": "Accepted", "control_tool": "check_agent_run"} +ROOT = "/home/netclaw/.netclaw/sessions/neutral/subagents/neutral" +PATHS = {"session_dir": ROOT, "temp_dir": ROOT + "/tmp", "artifact_dir": ROOT + "/artifacts", "log_path": ROOT + "/logs/session.log"} +STATUS = {**ACCEPTED, "state": "Running", "log_path": PATHS["log_path"], "artifact_directory": PATHS["artifact_dir"]} + + +def child(): + return {"messages": [{"role": "system", "content": CHILD_CONTRACT}, {"role": "user", "content": + "Context:\n[session]\n" + "\n".join(key + ": " + value for key, value in PATHS.items()) + "\nTask:\nTrial neutral-nonce."}]} + + +def terminal(cancel=False): + return {"run_id": ACCEPTED["run_id"], "scope_id": ACCEPTED["scope_id"], "source_operation": "spawn_agent", + "state": "Cancelled" if cancel else "Completed", "outcome": "Failed" if cancel else "Completed", + "reason": "cancelled_by_parent" if cancel else None, "log_path": PATHS["log_path"], + "artifact_directory": PATHS["artifact_dir"], + "output": "The child was cancelled. The last durable partial report is at " + PATHS["artifact_dir"] + "/cancelled-results.json.", + "warning": None, "checkpoint": {"CompletedRound": 1, "Summary": "File written: partial-neutral-nonce.txt", + "ConfirmedActivity": {"ProjectDirectory": None, "Worktree": None, "Branch": None, "Head": None, + "ConfirmedChangedFiles": [PATHS["artifact_dir"] + "/partial-neutral-nonce.txt"], + "ReadFiles": [], "ObservedChangedFiles": []}} if cancel else None} + + +def parent(value=None, identifier="delivery-neutral"): + return {"messages": [{"role": "assistant", "tool_calls": [{"id": identifier, "function": { + "name": "spawn_agent", "arguments": json.dumps({"run_id": ACCEPTED["run_id"], "source_operation": "spawn_agent"})}}]}, + {"role": "tool", "tool_call_id": identifier, "content": json.dumps(value or terminal())}]} + + + +def child_after_write(cancel=False): + request = child() + filename = "/partial-neutral-nonce.txt" if cancel else "/complete-neutral-nonce.txt" + content = ("PARTIAL-" if cancel else "COMPLETE-") + "neutral-nonce" + request["messages"].extend([{"role": "assistant", "tool_calls": [{"id": "write-neutral", "function": { + "name": "file_write", "arguments": json.dumps({"Path": PATHS["artifact_dir"] + filename, "Content": content})}}]}, + {"role": "tool", "tool_call_id": "write-neutral", "content": "File written"}]) + return request + +def logs(): + return "\n".join(f"child_run_{name} owner=session-neutral runId=run-neutral journalSequence={number}" + + (" inputId=input-neutral callId=delivery-neutral" if name == "delivery_admitted" else "") + for name, number in [("accepted", 1), ("terminal_recorded", 3), ("result_prepared", 4), ("delivery_admitted", 5)]) + + +class ChildAttributionControls(unittest.TestCase): + def test_acceptance_requires_canonical_fields_and_unique_keys(self): + self.assertEqual(ACCEPTED, acceptance(json.dumps(ACCEPTED))) + for value in [{**ACCEPTED, "scope_id": " "}, {**ACCEPTED, "state": "Completed"}, + {**ACCEPTED, "control_tool": "check_background_job"}]: + with self.subTest(value=value), self.assertRaises(AssertionError): + acceptance(json.dumps(value)) + with self.assertRaises(AssertionError): + acceptance('{"run_id":"first",' + json.dumps(ACCEPTED)[1:]) + + def test_cancellation_wire_uses_reason_value_and_enum_names_for_state_and_outcome(self): + source = Path(__file__).resolve().parents[1] / "src/Netclaw.Tools.Abstractions/SubAgentRunMetadata.cs" + match = re.search(r'CancelledByParent\s*=\s*new\("([^"\n]+)"\)', source.read_text()) + self.assertIsNotNone(match, "The canonical outcome-reason definition changed; review its wire value.") + wire = json.loads('{"state":"Cancelled","outcome":"Failed","reason":"cancelled_by_parent"}') + self.assertEqual(match.group(1), wire["reason"]) + self.assertEqual(wire, {name: terminal(True)[name] for name in wire}) + + def test_binding_requires_actual_child_contract_nonce_and_returned_paths(self): + self.assertEqual(PATHS, bind_request(child(), ACCEPTED, STATUS, "neutral-nonce")) + mutations = [lambda r: r["messages"][0].update(content="ordinary parent prompt"), + lambda r: r["messages"][0].update(content="Summarize memory"), + lambda r: r["messages"][1].update(content=r["messages"][1]["content"].replace(ROOT, ROOT + "-foreign")), + lambda r: r["messages"][1].update(content=r["messages"][1]["content"].replace("neutral-nonce", "foreign"))] + for mutate in mutations: + request = child() + mutate(request) + with self.subTest(request=request), self.assertRaises(AssertionError): + bind_request(request, ACCEPTED, STATUS, "neutral-nonce") + for key in ["run_id", "scope_id", "artifact_directory", "log_path"]: + with self.subTest(key=key), self.assertRaises(AssertionError): + bind_request(child(), ACCEPTED, {**STATUS, key: "foreign"}, "neutral-nonce") + + def test_terminal_pair_needs_actual_parent_history_and_one_stable_call(self): + self.assertEqual(("delivery-neutral", terminal()), canonical_pairs([parent(), parent()], ACCEPTED, "start-neutral", "spawn_agent")) + for requests in [[], [child()], [parent(), parent(identifier="other")], + [parent({**terminal(), "scope_id": "foreign"})], + [parent({**terminal(), "source_operation": "shell_execute"})], + [parent(), parent({**terminal(), "outcome": "Failed"})]]: + with self.subTest(requests=requests), self.assertRaises(AssertionError): + canonical_pairs(requests, ACCEPTED, "start-neutral", "spawn_agent") + + def test_terminal_pair_requires_fresh_id_consistent_argument_source_and_assistant_role(self): + for mutate in [lambda r: r["messages"][0].update(role="user"), + lambda r: r["messages"][0]["tool_calls"][0]["function"].update(arguments=json.dumps({"run_id": ACCEPTED["run_id"], "source_operation": "shell_execute"}))]: + request = parent() + mutate(request) + with self.subTest(request=request), self.assertRaises(AssertionError): + canonical_pairs([request], ACCEPTED, "start-neutral", "spawn_agent") + with self.assertRaises(AssertionError): + canonical_pairs([parent(identifier="start-neutral")], ACCEPTED, "start-neutral", "spawn_agent") + duplicated = parent() + duplicated["messages"].extend(copy.deepcopy(duplicated["messages"])) + with self.assertRaises(AssertionError): + canonical_pairs([duplicated], ACCEPTED, "start-neutral", "spawn_agent") + + def test_consumption_accepts_first_response_and_compatible_siblings_in_one_response(self): + sibling = {**ACCEPTED, "run_id": "run-sibling", "scope_id": "scope-sibling"} + sibling_body = {**terminal(), "run_id": sibling["run_id"], "scope_id": sibling["scope_id"]} + sibling_request = parent(sibling_body, "delivery-sibling") + sibling_request["messages"][0]["tool_calls"][0]["function"]["arguments"] = json.dumps( + {"run_id": sibling["run_id"], "source_operation": "spawn_agent"}) + request = parent() + request["messages"].extend(sibling_request["messages"]) + records = [{"request": request, "request_id": 1, "admitted_ns": 2, + "response_first_payload_ns": 3, "response_payload_written": True}] + expected = [{"accepted": ACCEPTED, "call_id": "start-neutral", "source_operation": "spawn_agent"}] + self.assertTrue(consumed_deliveries(records, expected, 4)["complete"]) + self.assertFalse(consumed_deliveries(records, [], 4)["complete"]) + repeated = records + [{**records[0], "request_id": 2, "admitted_ns": 5, "response_first_payload_ns": 6}] + self.assertEqual(1, consumed_deliveries(repeated, expected, 7)["deliveries"][0]["request_id"]) + expected.append({"accepted": sibling, "call_id": "start-sibling", "source_operation": "spawn_agent"}) + result = consumed_deliveries(records, expected, 4) + self.assertTrue(result["complete"]) + self.assertEqual(2, len(result["deliveries"])) + self.assertEqual({1}, {row["request_id"] for row in result["deliveries"]}) + for rows, boundary in [([], 4), ([{**records[0], "response_first_payload_ns": 0}], 4), + ([{**records[0], "response_payload_written": False}], 4), (records, 2)]: + with self.subTest(rows=rows, boundary=boundary): + self.assertFalse(consumed_deliveries(rows, expected, boundary)["complete"]) + + def test_final_response_requires_the_exact_current_prompt_receipt(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = root / "observer-0001" + current = root / "observer-0002" + old.mkdir(); current.mkdir() + data = {"Nonce": "prompt-four", "Mode": "turn", "InitialPrompt": "Report the actual final changed files.", + "SessionId": "session-neutral"} + receipt = {"status": "observed", "prompt_nonce": data["Nonce"], "observer_mode": data["Mode"], + "initial_prompt_sha256": hashlib.sha256(data["InitialPrompt"].encode()).hexdigest(), + "session_id": "session-neutral", "last_reply": "Actual final response", "all_replies": ["REQUIRED-MARKER initial acknowledgement", "Actual final response"]} + (old / "verified-receipt.json").write_text(json.dumps({**receipt, "prompt_nonce": "prompt-three", "last_reply": "STALE REQUIRED-MARKER"})) + (current / "observer-input.json").write_text(json.dumps(data)) + with self.assertRaises(FileNotFoundError): + verified_final_response(current) + (current / "verified-receipt.json").write_text((old / "verified-receipt.json").read_text()) + with self.assertRaises(AssertionError): + verified_final_response(current) + (current / "verified-receipt.json").write_text(json.dumps(receipt)) + self.assertEqual("Actual final response", verified_final_response(current)) + self.assertNotIn("REQUIRED-MARKER", verified_final_response(current)) + for fields in [{"status": "incomplete"}, {"observer_mode": "collect"}, {"initial_prompt_sha256": "foreign"}, + {"session_id": "foreign"}, {"session_id": ""}, {"last_reply": ""}, {"last_reply": None}]: + with self.subTest(fields=fields), self.assertRaises(AssertionError): + validate_prompt_receipt({**receipt, **fields}, data) + + def test_legacy_handoff_selects_only_its_child_prompt_for_terminal_collection(self): + self.assertEqual(["turn", "turn", "collect", "turn"], + [legacy_observer_mode("coding_context_worktree_handoff", number) for number in range(1, 5)]) + self.assertEqual("collect", legacy_observer_mode("subagent_specialization_precedence", 1)) + for case, number in [("coding_context_worktree_handoff", 0), ("coding_context_worktree_handoff", 5), + ("subagent_specialization_precedence", 2), ("foreign", 1)]: + with self.subTest(case=case, number=number), self.assertRaises(AssertionError): + legacy_observer_mode(case, number) + # These unit fixtures represent observer receipts, not actual CLI output. + for number in [1, 2, 4]: + with self.subTest(number=number), tempfile.TemporaryDirectory() as directory: + receipt = {"accepted_runs": [], "last_reply": f"Neutral parent reply {number}"} + with patch("child_run_evals.invoke_observer", return_value=(receipt, "neutral observer text")) as observer: + with redirect_stdout(io.StringIO()): + collect(1, "neutral task", "session-neutral", "text", directory, "coding_context_worktree_handoff", number) + self.assertEqual("turn", observer.call_args.args[3]) + saved = json.loads((Path(directory) / "verified-receipt.json").read_text()) + self.assertEqual(number, saved["prompt_ordinal"]) + self.assertEqual([], saved["verified_deliveries"]) + + def shell_functions(self, names): + source = (Path(__file__).resolve().parents[1] / "evals/run-evals.sh").read_text() + return "\n".join(re.search(r"^" + re.escape(name) + r"\(\) \{\n.*?^\}", source, + re.MULTILINE | re.DOTALL).group(0) for name in names) + + def test_failed_current_observer_cannot_reuse_an_earlier_final_reply(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + previous = root / "child-runs/observer-previous" + previous.mkdir(parents=True) + (previous / "verified-receipt.json").write_text(json.dumps({"last_reply": "STALE-ANSWER"})) + functions = self.shell_functions(["child_result_consumer", "observe_child_result", "run_prompt_resume", + "stdout_response_contains", "stdout_response_not_contains"]) + script = functions + r""" +set -euo pipefail +case_name=coding_context_worktree_handoff +resolve_daemon_log() { DAEMON_LOG=/missing-neutral-log; } +CHILD_LAST_EVIDENCE="$TMPDIR_EVAL/child-runs/observer-previous" +previous="$CHILD_LAST_EVIDENCE" +if run_prompt_resume session-neutral 'Final neutral prompt.' text 4; then exit 41; fi +[[ "$CHILD_LAST_EVIDENCE" != "$previous" ]] +[[ -f "$CHILD_LAST_EVIDENCE/observer-input.json" ]] +[[ ! -f "$CHILD_LAST_EVIDENCE/verified-receipt.json" ]] +if stdout_response_contains STALE-ANSWER; then exit 42; fi +if stdout_response_not_contains arbitrary; then exit 43; fi +printf '%s' "$CHILD_LAST_EVIDENCE" +""" + env = {**os.environ, "REPO_ROOT": str(Path(__file__).resolve().parents[1]), "TMPDIR_EVAL": directory, + "NETCLAW_BIN": "/bin/true", "CHILD_FIXTURE_PORT": "1", "EVAL_PORT": "1", "PROMPT_TIMEOUT": "1", + "NETCLAW_CHILD_OBSERVER": str(root / "missing-observer.dll"), "FILTER_CASE": "coding_context_worktree_handoff"} + result = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True, timeout=10) + self.assertEqual(0, result.returncode, result.stderr) + data = json.loads((Path(result.stdout) / "observer-input.json").read_text()) + self.assertEqual("turn", data["Mode"]) + self.assertEqual("Final neutral prompt.", data["InitialPrompt"]) + + def test_failed_fourth_prompt_blocks_the_legacy_case_assertion(self): + with tempfile.TemporaryDirectory() as directory: + script = self.shell_functions(["run_multi_turn_case"]) + r""" +set -euo pipefail +CATEGORY_SKIPPED=false +FILTER_CASE=coding_context_worktree_handoff +RUNS=1 +THRESHOLD=1 +CATEGORY_CASES=0; TOTAL_CASES=0; FAILED_CASES=0; CATEGORY_PASSED=0; PASSED_CASES=0 +check_daemon_alive() { :; } +setup_coding_context_worktree_handoff() { :; } +# The control supplies return codes only. It supplies no CLI output. +run_prompt_resume() { LAST_TURN_USAGE_LINE=""; [[ "$4" -lt 4 ]]; } +store_metrics() { :; } +store_result() { printf '%s' "$4" > "$CONTROL_ROOT/stored-pass"; printf '%s' "$5" > "$CONTROL_ROOT/stored-detail"; } +assert_coding_context_worktree_handoff() { printf called > "$CONTROL_ROOT/assertion-called"; return 0; } +run_multi_turn_case coding_context_worktree_handoff neutral one two three four +[[ ! -f "$CONTROL_ROOT/assertion-called" ]] +[[ "$(cat "$CONTROL_ROOT/stored-pass")" == 0 ]] +[[ "$(cat "$CONTROL_ROOT/stored-detail")" == observer_failed ]] +[[ "$FAILED_CASES" == 1 ]] +""" + result = subprocess.run(["bash", "-c", script], env={**os.environ, "CONTROL_ROOT": directory}, + text=True, capture_output=True, timeout=10) + self.assertEqual(0, result.returncode, result.stderr) + + def test_consistent_terminal_source_still_must_equal_the_actual_start(self): + wrong = parent() + function = wrong["messages"][0]["tool_calls"][0]["function"] + function["name"] = "shell_execute" + function["arguments"] = json.dumps({"run_id": ACCEPTED["run_id"], "source_operation": "shell_execute"}) + wrong["messages"][1]["content"] = json.dumps({**terminal(), "source_operation": "shell_execute"}) + with self.assertRaises(AssertionError): + canonical_pairs([wrong], ACCEPTED, "start-neutral", "spawn_agent") + self.assertEqual("delivery-neutral", canonical_pairs([wrong], ACCEPTED, "start-neutral", "shell_execute")[0]) + record = {"request": wrong, "request_id": 1, "admitted_ns": 1, "response_first_payload_ns": 2, "response_payload_written": True} + with self.assertRaises(AssertionError): + consumed_deliveries([record], [{"accepted": ACCEPTED, "call_id": "start-neutral", "source_operation": "spawn_agent"}], 3) + + def test_post_commit_positions_reject_absence_duplicate_reorder_and_foreign_call(self): + self.assertEqual(5, committed_positions(logs(), "session-neutral", "run-neutral", "delivery-neutral")["delivery_admitted"]) + for value in [logs().replace("child_run_result_prepared", "unobserved"), logs() + "\n" + logs(), + logs().replace("journalSequence=4", "journalSequence=2"), + logs().replace("callId=delivery-neutral", "callId=foreign")]: + with self.subTest(log=value), self.assertRaises(AssertionError): + committed_positions(value, "session-neutral", "run-neutral", "delivery-neutral") + + @contextmanager + def actual_held_response(self, directory, partial=False, stream=False): + body = {**(child_after_write(True) if partial else child()), "stream": stream} + payload = b'data: {"actual":"held-neutral"}\n\ndata: [DONE]\n\n' if stream else b'{"actual":"held-neutral"}' + upstream_requests = [] + first_client_byte = threading.Event() + received = [] + errors = [] + class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_): + return + def do_POST(self): + upstream_requests.append(self.rfile.read(int(self.headers["Content-Length"]))) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream" if stream else "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + with ThreadingHTTPServer(("127.0.0.1", 0), Upstream) as upstream: + fixture = ChildFixture(f"http://127.0.0.1:{upstream.server_port}/v1", "neutral", "", Path(directory) / "relay") + fixture.control("child-setup", {"case": "child_run_partial_cancel" if partial else "child_run_held_parent", "nonce": "neutral-nonce"}) + with ThreadingHTTPServer(("127.0.0.1", 0), child_handler(fixture)) as relay: + servers = [upstream, relay] + threads = [threading.Thread(target=server.serve_forever) for server in servers] + for thread in threads: + thread.start() + def call(): + try: + request = urllib.request.Request(f"http://127.0.0.1:{relay.server_port}/v1/chat/completions", + data=json.dumps(body).encode()) + with urllib.request.urlopen(request, timeout=5) as response: + first = response.read(1) + first_client_byte.set() + received.append(first + response.read()) + except Exception as error: + errors.append(error) + worker = threading.Thread(target=call) + worker.start() + try: + fixture.control("child-wait", {}) + self.assertEqual([json.dumps(body).encode()], upstream_requests) + self.assertFalse(first_client_byte.is_set()) + self.assertTrue(worker.is_alive()) + row = fixture.snapshot()["requests"][0] + self.assertGreater(row["upstream_first_payload_ns"], row["admitted_ns"]) + self.assertEqual(0, row["response_first_payload_ns"]) + self.assertFalse(row["response_payload_written"]) + self.assertTrue((fixture.evidence / "response-0001.wire").read_bytes().endswith(payload)) + yield fixture + finally: + fixture.control("child-abort", {}) + worker.join(5) + for server in servers: + server.shutdown() + for thread in threads: + thread.join(2) + self.assertFalse(worker.is_alive()) + self.assertEqual([], errors) + self.assertEqual([payload], received) + self.assertTrue(first_client_byte.is_set()) + fixture.control("child-drained", {}) + self.assertTrue(fixture.snapshot()["requests"][0]["response_payload_written"]) + + def test_barrier_holds_actual_upstream_payload_then_requires_binding_before_release(self): + for stream in [False, True]: + with self.subTest(stream=stream), tempfile.TemporaryDirectory() as directory: + with self.actual_held_response(directory, stream=stream) as fixture: + with self.assertRaises(AssertionError): + fixture.control("child-release", {}) + snapshot = fixture.control("child-bind", {"accepted": ACCEPTED, "status": STATUS}) + self.assertEqual(1, snapshot["binding"]["request_id"]) + self.assertGreater(snapshot["binding"]["upstream_first_payload_ns"], snapshot["binding"]["arrived_ns"]) + with self.assertRaises(AssertionError): + fixture.control("child-bind", {"accepted": ACCEPTED, "status": STATUS}) + fixture.control("child-release", {}) + + def test_partial_binding_checks_actual_file_before_parent_cancellation(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"EVAL_HOME": directory}): + path = actual_file(directory, PATHS["artifact_dir"] + "/partial-neutral-nonce.txt") + path.parent.mkdir(parents=True) + path.write_text("wrong bytes") + with self.actual_held_response(directory, partial=True) as fixture: + with self.assertRaises(AssertionError): + fixture.control("child-bind", {"accepted": ACCEPTED, "status": STATUS}) + self.assertIsNone(fixture.binding) + path.write_text("PARTIAL-neutral-nonce") + self.assertIsNotNone(fixture.control("child-bind", {"accepted": ACCEPTED, "status": STATUS})["binding"]) + fixture.control("child-release", {}) + + def test_parent_sidecar_and_foreign_requests_do_not_consume_child_barrier(self): + with tempfile.TemporaryDirectory() as directory: + fixture = ChildFixture("http://127.0.0.1:1/v1", "neutral", "", Path(directory) / "relay") + fixture.control("child-setup", {"case": "child_run_held_parent", "nonce": "neutral-nonce"}) + requests = [parent(), {"messages": [{"role": "system", "content": "memory distillation"}, + {"role": "user", "content": "neutral-nonce"}]}, child()] + requests[-1]["messages"][1]["content"] = requests[-1]["messages"][1]["content"].replace("neutral-nonce", "foreign") + for request in requests: + self.assertIsNone(fixture.completion(request)) + self.assertIsNone(fixture.candidate) + self.assertEqual(3, len(list((Path(directory) / "relay").glob("request-*.json")))) + + def test_actual_forward_preserves_stream_and_json_bytes_and_captures_admission(self): + received = [] + class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_): + return + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + received.append(body) + parsed = json.loads(body) + stream = parsed["stream"] + if parsed.get("fail"): + self.send_error(503, "Neutral upstream failure") + return + reply = b'data: {"actual":"neutral"}\n\ndata: [DONE]\n\n' if stream else b'{"actual":"neutral"}' + self.send_response(200) + self.send_header("Content-Type", "text/event-stream" if stream else "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + with tempfile.TemporaryDirectory() as directory, ThreadingHTTPServer(("127.0.0.1", 0), Upstream) as upstream: + fixture = ChildFixture(f"http://127.0.0.1:{upstream.server_port}/v1", "neutral", "", Path(directory) / "relay") + with ThreadingHTTPServer(("127.0.0.1", 0), child_handler(fixture)) as relay: + threads = [threading.Thread(target=server.serve_forever) for server in [upstream, relay]] + for thread in threads: + thread.start() + try: + for stream in [False, True]: + body = {**parent(), "stream": stream} + request = urllib.request.Request(f"http://127.0.0.1:{relay.server_port}/v1/chat/completions", + data=json.dumps(body).encode()) + with urllib.request.urlopen(request, timeout=2) as response: + actual = response.read() + expected = b'data: {"actual":"neutral"}\n\ndata: [DONE]\n\n' if stream else b'{"actual":"neutral"}' + self.assertEqual(expected, actual) + self.assertEqual(json.dumps(body).encode(), received[-1]) + index = len(received) + self.assertEqual(body, json.loads((fixture.evidence / f"request-{index:04}.json").read_text())) + wire = (fixture.evidence / f"response-{index:04}.wire").read_bytes() + self.assertTrue(wire.endswith(expected)) + snapshot = fixture.control("snapshot", {}) + row = snapshot["requests"][-1] + self.assertTrue(row["response_payload_written"]) + self.assertGreater(row["response_first_payload_ns"], row["admitted_ns"]) + consumed = fixture.control("child-consumed", {"expected": [ + {"accepted": ACCEPTED, "call_id": "start-neutral", "source_operation": "spawn_agent"}], "parent_boundary_ns": time.monotonic_ns()}) + self.assertTrue(consumed["complete"]) + self.assertEqual("delivery-neutral", consumed["deliveries"][0]["call_id"]) + failed = {**parent(), "stream": False, "fail": True} + request = urllib.request.Request(f"http://127.0.0.1:{relay.server_port}/v1/chat/completions", + data=json.dumps(failed).encode()) + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(request, timeout=2) + self.assertEqual(502, error.exception.code) + error.exception.close() + row = fixture.records[-1] + self.assertEqual(0, row["upstream_first_payload_ns"]) + self.assertEqual(0, row["response_first_payload_ns"]) + self.assertFalse(row["response_payload_written"]) + self.assertFalse(consumed_deliveries([row], [{"accepted": ACCEPTED, "call_id": "start-neutral", "source_operation": "spawn_agent"}], + time.monotonic_ns())["complete"]) + finally: + upstream.shutdown() + relay.shutdown() + for thread in threads: + thread.join(2) + + def test_partial_hold_needs_real_paired_write_result(self): + request = child() + request["messages"] += [{"role": "assistant", "tool_calls": [{"id": "write", "function": { + "name": "file_write", "arguments": json.dumps({"Path": PATHS["artifact_dir"] + "/partial-neutral-nonce.txt"})}}]}] + self.assertIsNone(write_completed(request, "partial-neutral-nonce.txt")) + request["messages"].append({"role": "tool", "tool_call_id": "foreign", "content": "Wrote file"}) + self.assertIsNone(write_completed(request, "partial-neutral-nonce.txt")) + request["messages"].append({"role": "tool", "tool_call_id": "write", "content": "Wrote file"}) + self.assertEqual("write", write_completed(request, "partial-neutral-nonce.txt")["call_id"]) + + def test_child_action_oracle_rejects_extra_shell_and_wrong_write(self): + verify_child_actions([child_after_write()], PATHS, "neutral-nonce", False) + repeated = child_after_write() + repeated["messages"].extend(copy.deepcopy(repeated["messages"][-2:])) + with self.assertRaises(AssertionError): + verify_child_actions([repeated], PATHS, "neutral-nonce", False) + for mutate in [lambda r: r["messages"][-2]["tool_calls"][0]["function"].update(name="shell_execute"), + lambda r: r["messages"][-2]["tool_calls"][0]["function"].update(arguments=json.dumps({"Path": "/tmp/foreign", "Content": "wrong"}))]: + request = child_after_write() + mutate(request) + with self.subTest(request=request), self.assertRaises(AssertionError): + verify_child_actions([request], PATHS, "neutral-nonce", False) + + def test_actual_cli_acceptance_requires_same_model_object_and_committed_start(self): + stdout = "[tool:result] spawn_agent → " + json.dumps(ACCEPTED) + "\nCLI-ACCEPTED-neutral-nonce\n" + request = {"messages": [{"role": "assistant", "tool_calls": [{"id": "start-neutral", "function": { + "name": "spawn_agent", "arguments": json.dumps({"Agent": "child-run-worker"})}}]}, + {"role": "tool", "tool_call_id": "start-neutral", "content": json.dumps(ACCEPTED)}]} + self.assertTrue(verify_cli_acceptance(stdout, [request], logs(), "session-neutral", "neutral-nonce")["passed"]) + for output, requests, log in [(stdout, [], logs()), (stdout, [request], ""), + (stdout.replace("Accepted", "Completed"), [request], logs()), + (stdout.replace("CLI-ACCEPTED-neutral-nonce", "no reply"), [request], logs()), + (stdout, [parent()], logs())]: + with self.subTest(output=output, requests=requests), self.assertRaises(AssertionError): + verify_cli_acceptance(output, requests, log, "session-neutral", "neutral-nonce") + + def test_actual_file_rejects_foreign_paths_and_link_escapes(self): + with tempfile.TemporaryDirectory() as home, tempfile.TemporaryDirectory() as foreign: + root = Path(home) / "data" + root.mkdir() + (root / "escape").symlink_to(foreign) + for path in ["/tmp/outside", "/home/netclaw/.netclaw/../outside", "/home/netclaw/.netclaw/escape/file"]: + with self.subTest(path=path), self.assertRaises(AssertionError): + actual_file(home, path) + + +class TrialOracleControls(unittest.TestCase): + def sample(self, home, cancel=False): + for canonical in [PATHS["log_path"], PATHS["artifact_dir"] + ("/partial-neutral-nonce.txt" if cancel else "/complete-neutral-nonce.txt")]: + path = actual_file(home, canonical) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("actual neutral child log line" if canonical == PATHS["log_path"] else + ("PARTIAL-" if cancel else "COMPLETE-") + "neutral-nonce") + calls = [{"id": "start-neutral", "name": "spawn_agent", "arguments": {"Agent": "child-run-worker"}, + "result": json.dumps(ACCEPTED), "success": True, "turn": 1}, + {"name": "load_tool", "arguments": {"Name": "check_agent_run"}, "success": True, "turn": 2}, + {"name": "check_agent_run", "arguments": {"RunId": ACCEPTED["run_id"], "Cancel": cancel}, "success": True, "turn": 2}, + {"name": "file_read", "arguments": {"Path": PATHS["log_path"]}, "success": True, "turn": 2, "result": "actual neutral child log line"}, + {"name": "file_read", "arguments": {"Path": PATHS["artifact_dir"] + ("/partial-neutral-nonce.txt" if cancel else "/complete-neutral-nonce.txt")}, + "success": True, "turn": 2 if cancel else 3, "observed_ns": 35 if cancel else 80, + "result": ("PARTIAL-" if cancel else "COMPLETE-") + "neutral-nonce"}] + receipt = {"accepted_run": ACCEPTED, "status": "observed", "completed_turns": 3, "user_inputs": 2, + "first_turn_ns": 10, "second_turn_ns": 40, "release_ns": 50, "session_id": "session-neutral", + "calls": calls, "last_reply": ("PARTIAL-" if cancel else "COMPLETE-") + "neutral-nonce"} + snapshot = {"binding": {"accepted": ACCEPTED, "paths": PATHS, "request_id": 1, "arrived_ns": 5, "upstream_first_payload_ns": 6, "bound_ns": 30}, + "released": True, "release_ns": 49, "requests": [{"held": True, "child": True, "request_id": 1, "admitted_ns": 5, "upstream_first_payload_ns": 6, "response_first_payload_ns": 0, "response_payload_written": False}, + {"held": False, "child": False, "request_id": 2, "admitted_ns": 32 if cancel else 60, + "response_first_payload_ns": 33 if cancel else 70, "response_payload_written": True}]} + receipt["delivery_observations"] = {"complete": True, "deliveries": [{"accepted": ACCEPTED, + "call_id": "delivery-neutral", "terminal": terminal(cancel), "request_id": 2, + "request_admitted_ns": 32 if cancel else 60, "response_first_payload_ns": 33 if cancel else 70, + "parent_boundary_ns": 40 if cancel else 90}]} + if cancel: + checkpoint = terminal(True)["checkpoint"] + report = {"run_id": ACCEPTED["run_id"], "state": "Cancelled", "summary": checkpoint["Summary"], + "confirmed_activity": checkpoint["ConfirmedActivity"], + "external_effects": "Recorded receipts describe known local results. They do not prove external effects stopped."} + actual_file(home, PATHS["artifact_dir"] + "/cancelled-results.json").write_text(json.dumps(report)) + return receipt, snapshot + + def test_valid_held_flow_requires_runtime_provider_and_actual_artifact_evidence(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home) + self.assertTrue(verify_trial(receipt, [child_after_write(), parent()], snapshot, logs(), home, "neutral-nonce", False)["passed"]) + for mutate in [lambda r: r.update(user_inputs=3), lambda r: r.update(completed_turns=1), + lambda r: r.update(first_turn_ns=0), lambda r: r.update(last_reply="imagined artifact"), + lambda r: r["calls"].pop(), lambda r: r["calls"][3].update(success=False), + lambda r: r["calls"][2]["arguments"].update(RunId="foreign"), lambda r: r["calls"][3].update(turn=3), + lambda r: r["calls"][4].update(observed_ns=60), lambda r: r["calls"][4].update(observed_ns=100)]: + value = copy.deepcopy(receipt) + mutate(value) + with self.subTest(receipt=value), self.assertRaises(AssertionError): + verify_trial(value, [child_after_write(), parent()], snapshot, logs(), home, "neutral-nonce", False) + for mutate in [lambda row: row["binding"].update(upstream_first_payload_ns=0), + lambda row: row.update(release_ns=39), + lambda row: row["binding"].update(request_id=2), + lambda row: row["requests"][0].update(upstream_first_payload_ns=0)]: + invalid = copy.deepcopy(snapshot) + mutate(invalid) + with self.subTest(snapshot=invalid), self.assertRaises(AssertionError): + verify_trial(receipt, [child_after_write(), parent()], invalid, logs(), home, "neutral-nonce", False) + actual_file(home, PATHS["artifact_dir"] + "/complete-neutral-nonce.txt").write_text("incorrect") + with self.assertRaises(AssertionError): + verify_trial(receipt, [child_after_write(), parent()], snapshot, logs(), home, "neutral-nonce", False) + + def test_full_trial_requires_exact_unique_terminal_argument_keys(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home) + def check(arguments): + request = parent() + request["messages"][0]["tool_calls"][0]["function"]["arguments"] = arguments + return verify_trial(receipt, [child_after_write(), request], snapshot, logs(), home, "neutral-nonce", False) + self.assertTrue(check(' { "source_operation" : "spawn_agent",\n "run_id" : "run-neutral" } ')["passed"]) + for arguments in [ + '{"run_id":"run-neutral","source_operation":"spawn_agent","unexpected_authority":"foreign"}', + '{"run_id":"foreign","run_id":"run-neutral","source_operation":"spawn_agent"}', + '{"run_id":"run-neutral","run_id":"foreign","source_operation":"spawn_agent"}', + '{"run_id":"run-neutral","run_id":"run-neutral","source_operation":"spawn_agent"}', + '{"run_id":"run-neutral","source_operation":"shell_execute","source_operation":"spawn_agent"}', + '{"run_id":"run-neutral","source_operation":"spawn_agent","source_operation":"spawn_agent"}', + '{"run_id":"run-neutral","source_operation":"spawn_agent","extra":{"scope":"foreign","scope":"owner"}}', + '{"run_id":"run-neutral","source_operation":{"name":"foreign","name":"spawn_agent"}}', + ]: + with self.subTest(arguments=arguments), self.assertRaises(AssertionError): + check(arguments) + + def test_full_trial_rejects_duplicate_terminal_body_keys_at_each_object_scope(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home) + for body in [ + '{"run_id":"foreign",' + json.dumps(terminal())[1:], + json.dumps(terminal())[:-1] + ',"extra":{"scope":"foreign","scope":"owner"}}', + ]: + request = parent() + request["messages"][1]["content"] = body + observed = copy.deepcopy(receipt) + observed["delivery_observations"]["deliveries"][0]["terminal"] = json.loads(body) + with self.subTest(body=body), self.assertRaises(AssertionError): + verify_trial(observed, [child_after_write(), request], snapshot, logs(), home, "neutral-nonce", False) + + def test_cancel_consumption_can_complete_inside_second_probe_before_release(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home, True) + receipt["completed_turns"] = 2 + receipt["last_reply"] += " PARENT-PROBE-neutral-nonce" + self.assertTrue(verify_trial(receipt, [child_after_write(True), parent(terminal(True))], snapshot, + logs(), home, "neutral-nonce", True)["passed"]) + + def test_partial_requires_exact_checkpoint_and_actual_run_local_report(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home, True) + original = terminal(True) + def check(body): + value = copy.deepcopy(receipt) + value["delivery_observations"]["deliveries"][0]["terminal"] = body + return verify_trial(value, [child_after_write(True), parent(body)], snapshot, logs(), home, "neutral-nonce", True) + self.assertTrue(check(original)["passed"]) + for mutate in [lambda b: b.update(checkpoint=None), + lambda b: b["checkpoint"].update(CompletedRound=0), + lambda b: b["checkpoint"].update(Summary=""), + lambda b: b["checkpoint"]["ConfirmedActivity"].update(ConfirmedChangedFiles=[]), + lambda b: b.update(output=PATHS["artifact_dir"] + "/partial-neutral-nonce.txt"), + lambda b: b.update(warning="The framework report failed. No report path is confirmed.")]: + body = copy.deepcopy(original) + mutate(body) + with self.subTest(body=body), self.assertRaises(AssertionError): + check(body) + report_path = actual_file(home, PATHS["artifact_dir"] + "/cancelled-results.json") + report = json.loads(report_path.read_text()) + for field, value in [("run_id", "foreign"), ("state", "Completed"), ("summary", "altered"), + ("confirmed_activity", {"ConfirmedChangedFiles": []}), ("external_effects", "remote provider stopped")]: + report_path.write_text(json.dumps({**report, field: value})) + with self.subTest(field=field), self.assertRaises(AssertionError): + check(original) + report_path.unlink() + with self.assertRaises(FileNotFoundError): + check(original) + + def test_cancel_requires_partial_evidence_and_rejects_later_child_admission(self): + with tempfile.TemporaryDirectory() as home: + receipt, snapshot = self.sample(home, True) + self.assertTrue(verify_trial(receipt, [child_after_write(True), parent(terminal(True))], snapshot, logs(), home, "neutral-nonce", True)["passed"]) + snapshot["requests"].append({"held": False, "child": True, "request_id": 3}) + with self.assertRaises(AssertionError): + verify_trial(receipt, [child_after_write(True), parent(terminal(True))], snapshot, logs(), home, "neutral-nonce", True) + snapshot["requests"].pop() + for fields in [{"reason": "Timeout"}, {"reason": "CancelledByParent"}, + {"state": "cancelled"}, {"outcome": "failed"}]: + with self.subTest(fields=fields), self.assertRaises(AssertionError): + verify_trial(receipt, [child_after_write(True), parent({**terminal(True), **fields})], snapshot, logs(), home, "neutral-nonce", True) + + +if __name__ == "__main__": + unittest.main() diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 2e0ec57f6..491bbbdca 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -1,9 +1,9 @@ --- name: netclaw-operations -description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." +description: "REQUIRED when the user asks about schedules, reminders, cron jobs, timers, background jobs, or child status and cancellation. Also use for diagnostics, troubleshooting, MCP tools, daemon health, identity updates, Netclaw capabilities, and maintenance." metadata: author: netclaw - version: "2.108.1" + version: "2.109.0" --- # Netclaw Operations @@ -22,6 +22,7 @@ a reference file — load the one matching the user's intent with | User intent | Where | |-------------|-------| | Schedule reminders/cron; run background shell jobs | `skill_read_resource('netclaw-operations', 'references/scheduling.md')` | +| Start, inspect, or cancel a background child task | `skill_read_resource('netclaw-operations', 'references/child-runs.md')` | | How tool arguments are validated | [Tool argument validation](#tool-argument-validation) | | Handle very large tool output | [Large tool output](#large-tool-output) | | Understand approval prompts | [Approval Prompts](#approval-prompts) | @@ -76,6 +77,7 @@ not know the name, and wait for its result before a shell call. | Inbound webhooks: list, create or change, delete | `list_webhooks`, `set_webhook`, `delete_webhook` | | Memories: search, read, save, change or delete | `find_memories`, `get_memories`, `store_memory`, `update_memory` | | Skills: load, read a bundled file, create or edit or delete | `skill_load`, `skill_read_resource`, `skill_manage` | +| Child tasks: inspect an owned run or request its cancellation | `check_agent_run` | | Read `netclaw.json`, the saved shell grants (`tool-approvals.json`), or a log | `file_read` | A `netclaw` command for an operation in this table, such as `netclaw reminder list` @@ -688,7 +690,7 @@ the operators `>|`, `>&`, and `<>`, and `x=1 y=2`, `a=(1 2)`, or `x+=1`. A prompt never shows an empty name. For an unknown program word (`$cmd > x`, `eval x`), it shows the full command text. -**Prompts survive passivation and restart.** Pending approval prompts are +**Parent task prompts survive passivation and restart.** Pending approval prompts are journaled with their requester and trust context, so if the session goes idle or the daemon restarts before the user clicks, the click is still honored when it arrives. Completed sibling tool results are journaled per call, so recovery @@ -706,6 +708,12 @@ can temporarily lag the session state. Shutdown cancellation does not mean the approval expired. An active tool with a possible external effect keeps the bounded drain path. +An accepted child's approval prompt belongs to that live run and its original requester. +Start-call completion and ordinary later parent input do not abandon that prompt. +Child cancellation or loss expires it. A late answer creates no grant or retry. +Restart does not resume an interrupted child. Its old approval prompt expires visibly. +Read `references/child-runs.md` for child cancellation and partial-result behavior. + An interrupted model call can create a short-lived restart reminder. The session restores accepted input and its original authority from the journal. The reminder expires ten minutes after the interruption. A completed turn, diff --git a/feeds/skills/.system/files/netclaw-operations/references/child-runs.md b/feeds/skills/.system/files/netclaw-operations/references/child-runs.md new file mode 100644 index 000000000..6e8a0f695 --- /dev/null +++ b/feeds/skills/.system/files/netclaw-operations/references/child-runs.md @@ -0,0 +1,98 @@ +# Background Child Tasks + +Use a child for useful independent work within the current audience and profile policy. +Give it a precise objective, source context, authorized workspace, output artifact, and acceptance checks. +Use the optional `context` argument to specialize an existing profile. +Do not infer capabilities from a profile name. Check `[available-subagents]`. + +## Start And Continue + +`spawn_agent` and a skill that routes to a child return durable acceptance before task completion. +The first tool response contains this JSON shape: + +```json +{"run_id":"","scope_id":"","state":"Accepted","control_tool":"check_agent_run"} +``` + +Keep the returned identifiers. Do not derive them from child text or actor names. +Acceptance proves admission. It does not prove a successful task or a current live state. +Continue independent parent work while the child executes. +The owner later delivers one terminal result with child attribution and a fresh result correlation. +That result does not replace the original acceptance response. + +Ordinary parent input, start-tool completion, and start-tool timeout do not cancel an accepted child. +The child retains its original requester, audience, working scope, and operation policy. +The inference backend owns capacity and request queues. Netclaw reserves no parent or child slots. + +## Status And Cancellation + +Call `load_tool` with the exact name `check_agent_run` when status or cancellation is necessary. +Use the loaded schema and the returned run identifier. +Avoid tight status polls. Normal completion uses the later terminal result. + +The control requires the owning parent session and original eligible requester under current policy. +A known identifier grants no access. Children and other sessions cannot inspect or cancel that run. +Use ordinary file tools for permitted result paths. A path grants no file authority. + +### Inspect A Live Child Log + +Use status first when a child needs diagnosis. +An authorized `check_agent_run` response supplies the exact `log_path` and `artifact_directory`, including before terminal completion. +Pass that log path to `file_read` or `file_search` under normal file policy. +Use bounded reads and targeted searches. Do not search a global log tree or derive child paths. +A log can explain a delay or show prior effects. It does not prove current health, dispatch closure, or completion. +Use the owner's recorded status and terminal result for those lifecycle facts. +Future private messages do not replace this diagnostic path. + +Before reuse in eval fixtures or shared reports, scrub personal data and secrets from a derived transcript copy. +Use stable opaque replacements when correlation or equality matters. Preserve the original log unchanged. + +### Cancel A Child + +Set the control's cancellation argument when the task must stop. +Distinguish these facts: + +- Cancellation admission is durable. The child can still have local work that must close. +- Dispatch closure confirms that no new model call, task tool, or approval retry can start locally. +- A terminal result records the final outcome and retained evidence. +- An earlier remote effect can remain unknown when its external service ignores cancellation. + +An admitted invocation can delay closure before it returns its Task. The parent must still accept status requests. +Closure does not wait for that returned Task to complete. +After dispatch closure, the framework has a separate five-second deadline for finalization. +It can retain checkpoints and atomically write a report inside the assigned run artifact directory. +It cannot call a model, run task tools, edit the project, or create approval authority during that grace period. +Report failure or grace expiry preserves the last confirmed checkpoint and an explicit reason. +Do not claim that a failed report write created an artifact. +The deadline bounds the owner wait. It does not prove that an operating-system file operation stops. + +Cancellation does not undo prior effects. A cancelled result is a failure outcome with useful partial evidence. +Confirmed partial evidence does not turn cancellation into successful completion. +Failed and cancelled runs do not merge a working-context delta into the parent. + +## Approval Prompts + +A child can request an approval prompt through its owner session when the channel supports it. +The prompt retains the original eligible requester, exact child call, authorization attempt, and accepted run. +An ordinary later parent message does not end that wait. +Cancellation or child loss expires the prompt. A late answer creates no grant or retry. +An approval prompt is a security gate. It is not a child question or a private message channel. + +## Loss And Replacement + +An owner restart records unresolved accepted runs as `Lost`. It does not recreate their children. +A prior durable cancellation remains cancellation. A committed terminal result retains its recorded outcome. +Read the recorded effects, unknown outcomes, and partial artifacts before you choose the next action. +The model decides whether to finish, revise the task, or request a new authorized attempt. +The framework does not replay an uncertain tool effect automatically or impose an operator-review policy. + +Use cancellation and a revised child task when the instructions must change. +Private agent messages, peer discovery, and live steering are unavailable in this release. + +## Workspace And Delivery + +Assign one writer per workspace. Concurrent writers need separate authorized worktrees. +Preserve a dirty operator checkout. Read-only children can share source inputs and use distinct report paths. +A child report does not prove its contents. Check its revision, scope, artifact contents, and verification evidence. +Only the parent sends user messages and delivers files through `attach_file` under normal policy. +An artifact on disk does not prove delivery. diff --git a/feeds/skills/.system/files/subagent-authoring/SKILL.md b/feeds/skills/.system/files/subagent-authoring/SKILL.md index 4f60d1379..7fc29940e 100644 --- a/feeds/skills/.system/files/subagent-authoring/SKILL.md +++ b/feeds/skills/.system/files/subagent-authoring/SKILL.md @@ -3,7 +3,7 @@ name: subagent-authoring description: "How to create and troubleshoot file-defined subagents in ~/.netclaw/agents. Load when the user asks to add, edit, or debug subagent definitions, or when a skill routes via metadata.subagent." metadata: author: netclaw - version: "1.4.1" + version: "1.5.0" --- # Subagent Authoring @@ -151,11 +151,23 @@ Cancellation, inactivity timeout, authorization, and operation deadlines still a The runtime returns known partial results when it stops a recurrent run. It does not request another model response to produce the terminal result. -The parent-facing `spawn_agent` result is an explicit terminal text envelope: -agent name, run id, outcome (`completed`, `partial`, or `failed`), optional -reason, diagnostics pointer, and a `Summary:` or `Error:` section. Structured -findings, when enabled, are a separate parent-reviewed memory-candidate path; -do not rely on them as the visible result returned to the parent model. +`spawn_agent` returns durable acceptance before child completion. +Its first JSON result contains `run_id`, `scope_id`, `state`, and `control_tool`. +The first state is `Accepted`; the control is `check_agent_run`. +Acceptance does not prove successful task completion. +The owner later delivers the recorded terminal result through a fresh result correlation. +Structured findings remain a separate memory-candidate path that the parent reviews. + +Start-tool completion and ordinary later parent input do not cancel an accepted child. +Load `check_agent_run` through `load_tool` for parent status or cancellation. +That control requires the owning session and original eligible requester under current policy. +Children cannot use it to control peers. +Cancellation admission, local dispatch closure, and terminal settlement are separate facts. +The framework retains partial evidence during its five-second finalization grace period after closure. +That grace period permits no model call, task tool, project edit, or approval action. +An owner restart marks unresolved accepted runs `Lost`; it does not resume child execution. +Child cancellation or loss expires its approval prompt. A late answer creates no grant or retry. +Load `netclaw-operations`, then read `references/child-runs.md` for the full contract. ## Fail-loud loader behavior diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md index f60c55efd..892585c54 100644 --- a/openspec/changes/background-child-runs/design.md +++ b/openspec/changes/background-child-runs/design.md @@ -152,6 +152,23 @@ States are `Accepted`, `Running`, `Cancelling`, `Completed`, `Partial`, `Failed` These states are lifecycle facts. They do not expand `SubAgentRunOutcome` implicitly. Status includes the state, cancellation admission, dispatch-closure state, and the recorded terminal result when present. Accepted and live status outputs do not claim task completion. +After owner authorization, the adapter supplies canonical `log_path` and `artifact_directory` through the existing child storage binding. +The adapter uses `SessionStorage.ForChild(run.RunId, run.ScopeId)` or its existing owner equivalent. +It adds no durable path field, path authority, or special log-reader tool. +The parent passes the exact path to ordinary file tools under normal file policy. +The log supports diagnosis. It does not establish current health, dispatch closure, or a terminal outcome. +A denied control reveals neither path. + +Schematic diagnostic flow; normal tool and file authorization remain required: + +```text +parent loads check_agent_run +owner checks current caller against original child authority +authorized adapter resolves the existing canonical child storage binding +status returns recorded lifecycle facts and exact diagnostic paths +parent reads a bounded portion of log_path through ordinary file tools +parent uses recorded status and terminal evidence for lifecycle decisions +``` Preserve the existing wire outcome names `Completed`, `Partial`, and `Failed`. A cancelled completion keeps `Success = false`, wire outcome `Failed`, and reason `CancelledByParent`. @@ -165,7 +182,7 @@ Add new reason fields and durable record variants through the framework schema a The table names current mechanisms. It does not prescribe new C# message types. New acknowledgements need the run correlation that a session-only `CommandAck` cannot carry. The owner acceptance and terminal records are durable. The JSON body is a call-local view of those committed facts. -Child results, dispatch admission state, actor references, and cancellation sources remain live runtime state. +Prepared child executions, dispatch admission gates, actor references, and cancellation sources remain live runtime state. | Boundary | Existing owner and mechanism | Required extension | | --- | --- | --- | diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md index 576d14f6f..01ab42bf1 100644 --- a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -135,6 +135,10 @@ A child or mismatched session/requester SHALL receive a denial without target de The control SHALL NOT route through the shell-job manager or use shell authority merely because it resembles shell-job cancellation. Status SHALL expose cancellation admission separately from confirmed local dispatch closure. Repeated status or cancellation requests SHALL NOT create another child, terminal result, or model continuation. +After authorization, status SHALL include the canonical `log_path` and `artifact_directory` before terminal completion. +The adapter SHALL obtain these paths from the existing child storage binding without a separate durable path authority. +The parent SHALL use ordinary file tools under normal file policy for diagnostic log access. +A returned path SHALL NOT grant file authority or prove current health, dispatch closure, or completion. #### Scenario: Original parent inspects its live run @@ -143,6 +147,22 @@ Repeated status or cancellation requests SHALL NOT create another child, termina - **THEN** it receives the run ID, recorded state, and cancellation/dispatch-closure facts - **AND** the call does not wait for task completion +#### Scenario: Parent reads a live child's exact log path + +- **GIVEN** an accepted child remains held before terminal completion +- **WHEN** the original authorized requester calls the deferred status tool +- **THEN** the response includes the exact canonical child log path and artifact directory +- **AND** the parent can pass that path to ordinary file tools under normal file policy +- **AND** the response does not claim a terminal outcome from log contents + +#### Scenario: A returned path does not bypass file policy + +- **GIVEN** an authorized status response supplies a child log path +- **AND** current file policy denies the requested read +- **WHEN** the parent calls a file tool with that path +- **THEN** the file tool denies the read +- **AND** the status response creates no file grant + #### Scenario: Another session cannot inspect or cancel the run - **GIVEN** another session supplies a valid run identifier diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 8cac6c44f..95a86c6ca 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -53,6 +53,7 @@ Preparation evidence: - [ ] 4.5 Add Deferred `check_agent_run(run_id, cancel=false)` through normal non-shell policy; verify explicit load, unchanged core, and direct ownership checks. - [ ] 4.6 Deny child, cross-session, and wrong-requester controls; verify no target details and keep the valid parent control operational. - [ ] 4.7 Add bounded recorded-run context with the exact control name; verify it cannot claim unobserved live provider state or expose hidden results. +- [ ] 4.8 Return canonical live `log_path` and `artifact_directory` after owner authorization. Hold a real child, inspect the actual status JSON, and read its exact log with ordinary file tools. Verify foreign controls reveal no paths and returned paths cannot bypass file policy. ## 5. Cancellation, partial evidence, and approval prompts diff --git a/scripts/run-tool-task-adoption-mutations.sh b/scripts/run-tool-task-adoption-mutations.sh index 165d1dc93..82c80b394 100755 --- a/scripts/run-tool-task-adoption-mutations.sh +++ b/scripts/run-tool-task-adoption-mutations.sh @@ -11,7 +11,9 @@ fi # Resolve exact boundaries. An absent or duplicate marker must fail before Stryker starts. spans="$( perl -Mopen=:std,:encoding\(UTF-8\) -0777 -ne ' - @targets = $ARGV =~ /LlmSessionActor/ + @targets = $ARGV =~ /LlmSessionActor\.ChildRuns\.cs$/ + ? (["!TurnContext.HasSameAuthority(original, current)", 1]) + : $ARGV =~ /LlmSessionActor\.cs$/ ? (["_log.Debug(\"Ignoring a stale local tool callback\");\n return false;", 1, "false"]) : $ARGV =~ /TurnStateTracker/ ? (["previous.Outcome == group.Outcome", 1]) @@ -35,56 +37,71 @@ spans="$( } ' "$repo_root/src/Netclaw.Actors/Sessions/SessionState.cs" \ "$repo_root/src/Netclaw.Actors/Sessions/Handlers/TurnStateTracker.cs" \ - "$repo_root/src/Netclaw.Actors/Sessions/LlmSessionActor.cs" + "$repo_root/src/Netclaw.Actors/Sessions/LlmSessionActor.cs" \ + "$repo_root/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs" )" -mutate_args=() -expected_total=0 -while read -r file span_start span_end first last count; do - mutate_args+=(--mutate "$file{$span_start..$span_end}") - expected_total=$((expected_total + count)) -done <<< "$spans" - # Stryker 5 accepts this filter in its JSON configuration only. config_path="$(mktemp --suffix=.json)" trap 'rm -f "$config_path"' EXIT -jq '."stryker-config"."test-case-filter" = - "FullyQualifiedName~ToolTaskAdoptionMutationTests|FullyQualifiedName~ToolRecurrenceMutationTests|FullyQualifiedName~LateToolReplyMutationTests"' \ - "$test_project/stryker-config.json" > "$config_path" -( - cd "$test_project" - dotnet stryker \ - --config-file "$config_path" \ - "${mutate_args[@]}" \ - --output "$output_path" \ - --skip-version-check -) +run_group() { + local group="$1" group_spans="$2" test_filter="$3" required_count="$4" + local file span_start span_end first last count report + local expected_total=0 + local mutate_args=() + while read -r file span_start span_end first last count; do + mutate_args+=(--mutate "$file{$span_start..$span_end}") + expected_total=$((expected_total + count)) + done <<< "$group_spans" + if [[ "$expected_total" != "$required_count" ]]; then + echo "Expected exactly $required_count target mutations in $group." >&2 + exit 1 + fi + + jq --arg filter "$test_filter" '."stryker-config"."test-case-filter" = $filter' \ + "$test_project/stryker-config.json" > "$config_path" + ( + cd "$test_project" + dotnet stryker \ + --config-file "$config_path" \ + "${mutate_args[@]}" \ + --output "$output_path/$group" \ + --skip-version-check + ) -report="$output_path/reports/mutation-report.json" -while read -r file span_start span_end first last count; do - jq -e --arg source "$repo_root/src/Netclaw.Actors/$file" \ - --argjson first "$first" --argjson last "$last" --argjson count "$count" ' - (if $source | endswith("LlmSessionActor.cs") then ["Boolean mutation"] - elif $count == 2 then ["Linq method mutation (Take() to Skip())", - "LogicalNotExpression to un-LogicalNotExpression mutation"] - else ["Equality mutation"] end) as $expected - | [.files[$source].mutants[] | select(.status != "Ignored") - | select(.location.start.line >= $first and .location.start.line <= $last) - | select(.mutatorName as $name | $expected | index($name))] as $mutants - | ($mutants | length) == $count and all($mutants[]; .status == "Killed") - and (($mutants | map(.mutatorName) | sort) == $expected) + report="$output_path/$group/reports/mutation-report.json" + while read -r file span_start span_end first last count; do + jq -e --arg source "$repo_root/src/Netclaw.Actors/$file" \ + --argjson first "$first" --argjson last "$last" --argjson count "$count" ' + (if $source | endswith("LlmSessionActor.ChildRuns.cs") then ["LogicalNotExpression to un-LogicalNotExpression mutation"] + elif $source | endswith("LlmSessionActor.cs") then ["Boolean mutation"] + elif $count == 2 then ["Linq method mutation (Take() to Skip())", + "LogicalNotExpression to un-LogicalNotExpression mutation"] + else ["Equality mutation"] end) as $expected + | [.files[$source].mutants[] | select(.status != "Ignored") + | select(.location.start.line >= $first and .location.start.line <= $last) + | select(.mutatorName as $name | $expected | index($name))] as $mutants + | ($mutants | length) == $count and all($mutants[]; .status == "Killed") + and (($mutants | map(.mutatorName) | sort) == $expected) + ' "$report" > /dev/null || { + echo "Expected $count killed adoption mutants in $file at lines $first-$last." >&2 + exit 1 + } + done <<< "$group_spans" + + # Unrelated compiler errors can precede the source filter. Target errors still fail above. + jq -e --argjson count "$expected_total" ' + [.files[].mutants[] | select(.status != "Ignored" and .status != "CompileError")] + | length == $count ' "$report" > /dev/null || { - echo "Expected $count killed adoption mutants in $file at lines $first-$last." >&2 + echo "Expected exactly $expected_total adoption mutants in $group." >&2 exit 1 } -done <<< "$spans" - -# Unrelated compiler errors can precede the source filter. Target errors still fail above. -jq -e --argjson count "$expected_total" ' - [.files[].mutants[] | select(.status != "Ignored" and .status != "CompileError")] - | length == $count -' "$report" > /dev/null || { - echo "Expected exactly $expected_total adoption mutants." >&2 - exit 1 } + +# The live child fixture cannot start under an unrelated broken parent-input guard. +run_group adoption "$(awk '$1 != "Sessions/LlmSessionActor.ChildRuns.cs"' <<< "$spans")" \ + 'FullyQualifiedName~ToolTaskAdoptionMutationTests|FullyQualifiedName~ToolRecurrenceMutationTests|FullyQualifiedName~LateToolReplyMutationTests' 5 +run_group child-control "$(awk '$1 == "Sessions/LlmSessionActor.ChildRuns.cs"' <<< "$spans")" \ + 'FullyQualifiedName~BackgroundChildControlMutationTests' 1 diff --git a/src/Netclaw.Actors.MutationTests/BackgroundChildControlMutationTests.cs b/src/Netclaw.Actors.MutationTests/BackgroundChildControlMutationTests.cs new file mode 100644 index 000000000..0551d72da --- /dev/null +++ b/src/Netclaw.Actors.MutationTests/BackgroundChildControlMutationTests.cs @@ -0,0 +1,290 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading.Channels; +using Akka.Actor; +using Akka.Hosting; +using Akka.Persistence.Hosting; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Serialization; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.MutationTests; + +public sealed class BackgroundChildControlMutationTests : IAsyncLifetime +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(30); + private readonly NetclawPaths _paths = new(Path.Combine(Path.GetTempPath(), $"netclaw-child-control-mutations-{Guid.NewGuid():N}")); + private readonly SessionId _session = new("mutation/child-control"); + private readonly ParentClient _parent = new(); + private readonly HeldChildClient _child = new(); + private readonly ToolConfig _toolConfig = new(); + private readonly Channel _outputs = Channel.CreateUnbounded(); + private IHost _host = null!; + private IActorRef _owner = null!; + private IActorRef _observer = null!; + private ControlExecutor _executor = null!; + + public async Task InitializeAsync() + { + var clients = new RoleProvider(_parent, _child); + var registry = new ToolRegistry(); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "start_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "control_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "file_read"), "file"); + var prompt = new StaticSystemPromptProvider("Use the assigned tools."); + var snapshots = new SnapshotProvider(); + var policy = new ToolAccessPolicy(_paths, _toolConfig, + new EffectivePolicyDefaults(DeploymentPosture.Personal, TrustAudience.Personal, + ShellExecutionMode.HostAllowed, UsedStrictFallback: false), + new ShellCommandPolicy(), new ToolPathPolicy([])); + var spawner = new SubAgentSpawner(clients, registry, policy, null, prompt, snapshots, + NullLogger.Instance); + _executor = new ControlExecutor(spawner); + var storage = SessionStoragePaths.CreateLegacy(Path.Combine(_paths.BasePath, "session"), + Path.Combine(_paths.BasePath, "logs"), _session.Value); + var services = new SessionServices(clients, prompt, [], snapshots, TimeProvider.System, _paths, new StorageResolver(storage)); + var tools = new SessionToolServices(_executor, registry, policy, null, null); + var builder = Host.CreateApplicationBuilder(); + builder.Services.AddAkka($"child-control-mutations-{Guid.NewGuid():N}", akka => akka + .WithInMemoryJournal().WithInMemorySnapshotStore().WithNetclawSerialization() + .WithActors((system, _) => + { + _observer = system.ActorOf(Props.Create(() => new OutputObserver(_outputs.Writer))); + _owner = system.ActorOf(Props.Create(() => new LlmSessionActor(_session.Value, + new ModelCapabilities { ModelId = "fake", ContextWindowTokens = 2_000_000 }, + new SessionConfig { Tuning = new SessionTuning { TitleGenerationInterval = 0, CompactionThreshold = 0.99 } }, + services, tools))); + })); + _host = builder.Build(); + await _host.StartAsync(); + _owner.Tell(new JoinSession(_observer) + { SessionId = _session, Filter = OutputFilter.Full | OutputFilter.ProcessingState }, _observer); + await ReadUntilAsync(message => message is SessionJoined); + } + + [Fact] + public async Task A_foreign_tool_cannot_control_a_child_but_its_original_requester_can() + { + _parent.NextCall = new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the assigned task." }); + await SendAsync("operator-a", "Start the child."); + await _child.Entered.Task.WaitAsync(Ceiling); + var started = await ReadUntilAsync(message => message is ProcessingStateOutput { IsProcessing: false }); + Assert.Single(started.OfType()); + var run = Assert.IsType(_executor.Prepared); + var startReceipt = Assert.Single(started.OfType()); + using var accepted = JsonDocument.Parse(startReceipt.Result); + Assert.Equal(run.RunId.Value, accepted.RootElement.GetProperty("run_id").GetString()); + + _parent.NextCall = ControlCall("foreign-control", run.RunId, true); + await SendAsync("operator-b", "Cancel the child."); + var foreign = await ReadUntilAsync(message => message is ProcessingStateOutput { IsProcessing: false }); + Assert.Single(foreign.OfType()); + var denied = Assert.Single(_executor.Controls); + Assert.Equal("operator-b", denied.Requester); + Assert.Null(denied.Reply.Run); + Assert.False(_child.Token.IsCancellationRequested); + var foreignResult = Assert.Single(foreign.OfType()); + Assert.Equal("Error: the child run was not found or is not accessible from this session.", foreignResult.Result); + Assert.DoesNotContain(run.RunId.Value, foreignResult.Result); + Assert.DoesNotContain(run.Execution.Scope.ScopeId.Value, foreignResult.Result); + Assert.DoesNotContain("log_path", foreignResult.Result); + Assert.DoesNotContain("artifact_directory", foreignResult.Result); + + _parent.NextCall = ControlCall("original-control", run.RunId, false); + await SendAsync("operator-a", "Inspect the child."); + var original = await ReadUntilAsync(message => message is ProcessingStateOutput { IsProcessing: false }); + Assert.Single(original.OfType()); + Assert.Equal(2, _executor.Controls.Count); + var allowed = _executor.Controls.Last(); + Assert.Equal("operator-a", allowed.Requester); + Assert.NotEqual(denied.AuthorizationAttempt, allowed.AuthorizationAttempt); + var visible = Assert.IsType(allowed.Reply.Run); + Assert.Equal(run.RunId, visible.RunId); + Assert.Equal(BackgroundChildState.Running, visible.State); + Assert.Null(visible.CancellationRequestedAtMs); + Assert.Null(visible.DispatchClosedAtMs); + Assert.Null(visible.Terminal); + Assert.False(_child.Token.IsCancellationRequested); + Assert.Equal(1, _child.Count); + Assert.Equal(6, _parent.Count); + var originalResult = Assert.Single(original.OfType()); + using var status = JsonDocument.Parse(originalResult.Result); + var storage = Assert.IsType(run.Execution.Scope.Authority.Session).Storage; + Assert.Equal(storage.LogPath.Value, status.RootElement.GetProperty("log_path").GetString()); + Assert.Equal(storage.ArtifactDirectory.Value, status.RootElement.GetProperty("artifact_directory").GetString()); + Assert.DoesNotContain(storage.LogPath.Value, foreignResult.Result); + Assert.DoesNotContain(storage.ArtifactDirectory.Value, foreignResult.Result); + Assert.True(File.Exists(storage.LogPath.Value)); + var readOutputs = allowed.Context.Outputs.Fork(); + var readContext = new ToolInvocationContext(allowed.Context.RunScope, allowed.Context.ExecutionTimeout, readOutputs); + var deniedRead = await new FileReadTool(_toolConfig, _paths, new ToolPathPolicy([storage.LogPath.Value])) + .ExecuteAsync(new Dictionary { ["Path"] = storage.LogPath.Value }, readContext); + Assert.IsType(readOutputs.Receipt); + Assert.Contains("Access denied", deniedRead); + Assert.StartsWith("Error:", deniedRead); + Assert.Empty(readOutputs.FileAttachments); + Assert.Empty(readOutputs.ModelInputFiles); + var pairs = _parent.Requests.Last().SelectMany(message => message.Contents.OfType()); + Assert.Equal(originalResult.Result, + Assert.IsType(Assert.Single(pairs, result => result.CallId == "original-control").Result)); + } + + private Task SendAsync(string requester, string content) => _owner.Ask(new SendUserMessage + { + SessionId = _session, Content = content, Source = new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId(requester), Audience = TrustAudience.Personal, + Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted), + DefaultDeliveryTarget = new ChannelDeliveryTargetInfo("signalr", "destination", requester, requester) + } + }, Ceiling); + + private static FunctionCallContent ControlCall(string id, SubAgentRunId runId, bool cancel) => new(id, "control_probe", + new Dictionary { ["run_id"] = runId.Value, ["cancel"] = cancel, ["_rationale"] = "Control the recorded child." }); + + private async Task> ReadUntilAsync(Func condition) + { + using var ceiling = new CancellationTokenSource(Ceiling); + var items = new List(); + while (true) + { + var item = await _outputs.Reader.ReadAsync(ceiling.Token); + items.Add(item); + if (condition(item)) return items; + } + } + + public async Task DisposeAsync() + { + try { if (_host is not null) { await _host.StopAsync(); _host.Dispose(); } } + finally { if (Directory.Exists(_paths.BasePath)) Directory.Delete(_paths.BasePath, true); } + } + + private sealed class OutputObserver : ReceiveActor + { + public OutputObserver(ChannelWriter writer) => ReceiveAny(message => writer.TryWrite(message)); + } + + private sealed class StorageResolver(SessionStoragePaths storage) : ISessionStorageResolver + { + public SessionStoragePaths Resolve(SessionId sessionId) => storage; + } + + private sealed class SnapshotProvider : IWorkingContextSnapshotProvider + { + public Task CreateAsync(WorkingContext context, TrustAudience audience, CancellationToken cancellationToken) + => Task.FromResult(new WorkingContextSnapshot { WorkingContext = context, Git = new GitWorkingContextInspection.Skipped() }); + } + + private sealed class RoleProvider(IChatClient parent, IChatClient child) : IChatClientProvider + { + public IChatClient GetClient(ModelRole role) => role == ModelRole.Main ? parent : child; + } + + private sealed class ControlExecutor(SubAgentSpawner spawner) : IToolExecutor + { + public sealed record Control(string? Requester, AuthorizationAttemptId AuthorizationAttempt, ChildControlReply Reply, ToolExecutionContext Context); + public ConcurrentQueue Controls { get; } = new(); + public PreparedChildRun? Prepared { get; private set; } + public async Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) + { + var factory = context.Invocation.SpawnChildActor + ?? throw new InvalidOperationException("The owner factory is required."); + if (call.Name == "control_probe") + { + var controlScope = context.RunScope with { SpawnChildActor = async (payload, name, token) => + { + Assert.IsType(payload); + var reply = Assert.IsType(await factory(payload, name, token)); + Controls.Enqueue(new Control(context.RunScope.DefaultDeliveryTarget?.DestinationId, context.Approval.AuthorizationAttemptId, reply, context)); + return reply; + }}; + return await new CheckAgentRunTool().ExecuteAsync(new Dictionary + { + ["RunId"] = call.Arguments!["run_id"]!.ToString(), + ["Cancel"] = JsonSerializer.SerializeToElement(call.Arguments).GetProperty("cancel").GetBoolean() + }, new ToolInvocationContext(controlScope, context.ExecutionTimeout, context.Outputs), ct); + } + var scope = context.RunScope with { SpawnChildActor = async (payload, name, token) => + { + Prepared = Assert.IsType(payload); + return await factory(payload, name, token); + }}; + return await spawner.StartRunAsync(new SubAgentProfile + { + Name = "worker", Description = "Complete the neutral task.", SystemPrompt = "Complete the assigned task.", + ModelRole = ModelRole.Compaction, ToolNames = ["file_read"], EmitStructuredFindings = false + }, "Inspect the neutral fixture.", null, new ToolInvocationContext(scope, context.ExecutionTimeout, context.Outputs), ct, null); + } + } + + private sealed class ParentClient : IChatClient + { + private int _count; + public int Count => Volatile.Read(ref _count); + public FunctionCallContent? NextCall { get; set; } + public ConcurrentQueue Requests { get; } = new(); + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + Requests.Enqueue(messages.ToArray()); + Interlocked.Increment(ref _count); + var call = NextCall; + NextCall = null; + return Task.FromResult(new ChatResponse(new ChatMessage(Microsoft.Extensions.AI.ChatRole.Assistant, + call is null ? [new TextContent("The parent task is complete.")] : [call]))); + } + public async IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, + ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var response = await GetResponseAsync(messages, options, cancellationToken); + yield return new ChatResponseUpdate { Role = Microsoft.Extensions.AI.ChatRole.Assistant, Contents = response.Messages[0].Contents }; + } + public object? GetService(Type serviceType, object? serviceKey = null) => null; + public void Dispose() { } + } + + private sealed class HeldChildClient : IChatClient + { + private int _count; + public int Count => Volatile.Read(ref _count); + public CancellationToken Token { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + Token = cancellationToken; + Interlocked.Increment(ref _count); + Entered.TrySetResult(); + await new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously).Task.WaitAsync(cancellationToken); + throw new InvalidOperationException("The held child requires cancellation."); + } + public async IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, + ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await GetResponseAsync(messages, options, cancellationToken); + yield break; + } + public object? GetService(Type serviceType, object? serviceKey = null) => null; + public void Dispose() { } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.Drain.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.Drain.cs new file mode 100644 index 000000000..5841b0b2c --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.Drain.cs @@ -0,0 +1,120 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Persistence; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildApprovalGrantAdversarialTests +{ + [Fact] + public async Task Coordinated_drain_commits_the_live_child_prompt_disposition_before_terminal_snapshot_and_acknowledgement() + { + var gate = InstallWaitReplyGate(); + try + { + var (manager, _, prompt) = await StartPromptAsync(); + var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(Session.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var admission = Assert.Single(before.OfType()); + var requested = Assert.Single(before.OfType()); + Assert.Equal(admission.Run.RunId, requested.SourceChildRunId); + Assert.Equal(prompt.CallId.Value, requested.CallId); + Assert.Equal(prompt.AuthorizationAttemptId, requested.AuthorizationAttemptId); + Assert.Equal(new SenderId("operator-a"), requested.RequesterSenderId); + Assert.Equal(new ToolCallId("child-approval-1"), requested.OriginalChildCallId); + Assert.Empty(before.OfType()); + Assert.Empty(before.OfType()); + var child = await Sys.ActorSelection($"{owner.Path}/child-run-{admission.Run.RunId.Value}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + var ownerWatcher = CreateTestProbe(); + var childWatcher = CreateTestProbe(); + ownerWatcher.Watch(owner); + childWatcher.Watch(child); + var drain = manager.Ask(new PrepareForDaemonRestart(Session, "daemon-stop"), + Ceiling, TestContext.Current.CancellationToken); + var ack = await drain; + Assert.Equal(Session, ack.SessionId); + Assert.Null(ack.RestartReminder); + await ownerWatcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await childWatcher.ExpectTerminatedAsync(child, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + SessionSnapshot? snapshot = null; + await AwaitAssertAsync(async () => + { + snapshot = await ReadPromptDrainSnapshotAsync(); + Assert.NotNull(snapshot); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var run = Assert.Single(snapshot!.ChildRuns); + run.Validate(); + Assert.Equal(admission.Run.RunId, run.RunId); + Assert.Equal(BackgroundChildState.Cancelled, run.State); + Assert.NotNull(run.CancellationRequestedAtMs); + Assert.NotNull(run.DispatchClosedAtMs); + Assert.True(run.TerminalSequenceNr > 0); + Assert.False(run.Terminal!.Lost); + Assert.IsType(run.Terminal.Result.Completion); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, run.Terminal.Result.OutcomeReason); + Assert.False(run.Terminal.Result.Success); + var approval = Assert.Single(run.Approvals); + Assert.Equal(requested.CallId, approval.Request.CallId); + Assert.Equal(requested.AuthorizationAttemptId, approval.Request.AuthorizationAttemptId); + Assert.Equal(requested.RequesterSenderId, approval.Request.RequesterSenderId); + Assert.Equal(requested.OriginalChildCallId, approval.Request.OriginalChildCallId); + Assert.NotNull(approval.Resolution); + Assert.Equal(admission.Run.RunId, approval.Resolution.SourceChildRunId); + Assert.Equal(requested.CallId, approval.Resolution.CallId); + Assert.Equal(requested.AuthorizationAttemptId, approval.Resolution.AuthorizationAttemptId); + Assert.Equal("Denied", approval.Resolution.Decision); + Assert.DoesNotContain(run.Approvals, item => item.Resolution is null); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + Assert.Empty(_store.Load().Audiences); + Assert.Equal(1, _child.CallCount); + Assert.Equal(2, _main.CallCount); + } + finally + { + ReleaseOwnedSignals(gate); + } + } + + private async Task ReadPromptDrainSnapshotAsync() + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reader = Sys.ActorOf(Props.Create(() => new PromptDrainSnapshotReader($"session-{Session.Value}", done))); + var watcher = CreateTestProbe(); + watcher.Watch(reader); + try { return await done.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); } + finally + { + Sys.Stop(reader); + await watcher.ExpectTerminatedAsync(reader, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + } + + private sealed class PromptDrainSnapshotReader : ReceivePersistentActor + { + public PromptDrainSnapshotReader(string persistenceId, TaskCompletionSource done) + { + PersistenceId = persistenceId; + SessionSnapshot? snapshot = null; + Recover(offer => + { + snapshot = Assert.IsType(offer.Snapshot); + _ = SessionState.FromSnapshot(snapshot); + }); + Recover(_ => { }); + Recover(_ => done.TrySetResult(snapshot)); + } + public override string PersistenceId { get; } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.cs new file mode 100644 index 000000000..6c6bc6684 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildApprovalGrantAdversarialTests.cs @@ -0,0 +1,352 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Runtime.CompilerServices; +using Akka.Actor; +using Akka.Configuration; +using Akka.Dispatch; +using Akka.Dispatch.MessageQueues; +using Akka.Hosting; +using Akka.Persistence; +using Akka.Routing; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tests.Utilities; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildApprovalGrantAdversarialTests(ITestOutputHelper output) : LlmSessionTestBase(output) +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(30); + private static readonly SessionId Session = new("signalr/child-approval-grant-adversarial"); + private readonly FakeChatClient _main = new(); + private readonly FakeChatClient _child = new(); + private readonly TaskCompletionSource _firstChildResponse = NewSignal(); + private readonly TaskCompletionSource _nextChildResponse = NewSignal(); + private readonly ControllableWorkingContextSnapshotProvider _snapshots = new(); + private ToolApprovalStore _store = null!; + private int _approvedEffects; + + protected override void ConfigureSessionServices(IServiceCollection services) + { + var clients = new RoleProvider(_main, _child); + services.AddSingleton(clients); + services.AddSingleton(new ModelCapabilities { ModelId = "fake", ContextWindowTokens = 2_000_000 }); + services.AddSingleton(new SessionConfig + { + Tuning = new SessionTuning { SnapshotInterval = 1000, TitleGenerationInterval = 0, CompactionThreshold = 0.99 } + }); + var prompt = new StaticSystemPromptProvider("Use the assigned tools."); + services.AddSingleton(prompt); + services.AddSingleton(_snapshots); + var registry = new ToolRegistry(); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "start_probe"), "builtin"); + registry.RegisterCore(new FakeNetclawTool("approval_probe", "The actual approved result.", "builtin", invocation => + { + Interlocked.Increment(ref _approvedEffects); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + })); + services.AddSingleton(registry); + var config = new ToolConfig(); + config.AudienceProfiles.Personal.ApprovalPolicy = new ToolApprovalConfig + { + ToolOverrides = new Dictionary(StringComparer.Ordinal) + { ["approval_probe"] = ToolApprovalMode.Approval } + }; + var policy = new ToolAccessPolicy(TestPaths, config, + new EffectivePolicyDefaults(DeploymentPosture.Personal, TrustAudience.Personal, + ShellExecutionMode.HostAllowed, UsedStrictFallback: false), + new ShellCommandPolicy(), new ToolPathPolicy([])); + services.AddSingleton(policy); + _store = new ToolApprovalStore(TestPaths.ToolApprovalsPath, TimeProvider.System); + services.AddSingleton(_store); + services.AddSingleton(); + services.AddSingleton(provider => new StartExecutor(new SubAgentSpawner( + clients, registry, policy, provider.GetRequiredService(), prompt, _snapshots, + NullLogger.Instance))); + _main.ToolCallsOnFirstCall = [new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the neutral task." })]; + _child.ToolCallsOnFirstCall = [new FunctionCallContent("child-approval-1", "approval_probe", + new Dictionary { ["_rationale"] = "Request the neutral operation." })]; + _child.NextResponseGate = _firstChildResponse; + } + + [Fact] + public async Task A_live_original_requester_grants_one_reusable_approval_and_one_exact_retry() + { + var gate = InstallWaitReplyGate(); + try + { + var (manager, _, prompt) = await StartPromptAsync(); + var denied = Assert.IsType(await manager.Ask(Answer(prompt, "operator-b"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(ApprovalNackReasons.WrongRequester, denied.Reason); + Assert.Empty(_store.Load().Audiences); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + Assert.IsType(await manager.Ask(Answer(prompt, "operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + await AwaitAssertAsync(() => Assert.Equal(2, _child.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, Volatile.Read(ref _approvedEffects)); + var stored = _store.Load(); + var audience = Assert.Single(stored.Audiences); + Assert.Equal("personal", audience.Key); + var tool = Assert.Single(audience.Value); + Assert.Equal("approval_probe", tool.Key); + Assert.Equal("approval_probe", Assert.Single(tool.Value).Verb); + var committed = await ReadJournalAsync(); + var requested = Assert.Single(committed.OfType()); + var resolved = Assert.Single(committed.OfType()); + Assert.Equal(requested.SourceChildRunId, resolved.SourceChildRunId); + Assert.Equal(requested.AuthorizationAttemptId, resolved.AuthorizationAttemptId); + Assert.Equal("ApprovedEverywhere", resolved.Decision); + Assert.Equal(new SenderId("operator-a"), requested.RequesterSenderId); + Assert.Equal(new ToolCallId("child-approval-1"), requested.OriginalChildCallId); + var retry = Assert.Single(_child.ReceivedMessages[^1] + .SelectMany(message => message.Contents.OfType())); + Assert.Equal("child-approval-1", retry.CallId); + Assert.Equal("The actual approved result.\n[approval: always anywhere]", Assert.IsType(retry.Result)); + var beforeDuplicate = await File.ReadAllBytesAsync(TestPaths.ToolApprovalsPath, TestContext.Current.CancellationToken); + var stale = Assert.IsType(await manager.Ask(Answer(prompt, "operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(ApprovalNackReasons.PromptExpired, stale.Reason); + Assert.Equal(beforeDuplicate, await File.ReadAllBytesAsync(TestPaths.ToolApprovalsPath, TestContext.Current.CancellationToken)); + Assert.Equal(1, Volatile.Read(ref _approvedEffects)); + Assert.Equal(2, _child.CallCount); + Assert.Equal(2, _main.CallCount); + Assert.False(_nextChildResponse.Task.IsCompleted); + Assert.Empty((await ReadJournalAsync()).OfType()); + } + finally { ReleaseOwnedSignals(gate); } + } + + [Fact] + public async Task A_reusable_answer_cannot_grant_after_the_real_live_wait_is_claimed_and_denied() + { + var gate = InstallWaitReplyGate(); + try + { + var (manager, _, prompt) = await StartPromptAsync(); + gate.Arm(); + Directory.CreateDirectory(TestPaths.ToolApprovalsPath); + await EventFilter.Error(contains: "Child approval decision failed").ExpectOneAsync(async () => + { + var failed = Assert.IsType(await manager.Ask(Answer(prompt, "operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(ApprovalNackReasons.PersistFailed, failed.Reason); + }, cancellationToken: TestContext.Current.CancellationToken); + var captured = await gate.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal("ChildApprovalWaitFinished", captured.Envelope.Message.GetType().Name); + Assert.True(Directory.Exists(TestPaths.ToolApprovalsPath)); + await AwaitAssertAsync(() => Assert.Equal(2, _child.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var pending = await ReadJournalAsync(); + var requested = Assert.Single(pending.OfType()); + Assert.Equal(prompt.CallId.Value, requested.CallId); + Assert.Equal(prompt.AuthorizationAttemptId, requested.AuthorizationAttemptId); + Assert.Equal(new SenderId("operator-a"), requested.RequesterSenderId); + Assert.Equal(new ToolCallId("child-approval-1"), requested.OriginalChildCallId); + Assert.Empty(pending.OfType()); + Assert.Empty(pending.OfType()); + Assert.Empty(pending.OfType()); + Assert.Empty(pending.OfType()); + Assert.Single(pending.OfType()); + var deniedResult = Assert.Single(_child.ReceivedMessages[^1] + .SelectMany(message => message.Contents.OfType())); + Assert.Equal("child-approval-1", deniedResult.CallId); + Assert.Equal("Tool access denied: approval_denied_by_user", Assert.IsType(deniedResult.Result)); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + Directory.Delete(TestPaths.ToolApprovalsPath); + Assert.Empty(_store.Load().Audiences); + var expired = Assert.IsType(await manager.Ask(Answer(prompt, "operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(ApprovalNackReasons.PromptExpired, expired.Reason); + Assert.Empty(_store.Load().Audiences); + Assert.False(File.Exists(TestPaths.ToolApprovalsPath)); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + Assert.Equal(2, _child.CallCount); + Assert.Equal(2, _main.CallCount); + Assert.False(_nextChildResponse.Task.IsCompleted); + Assert.Empty((await ReadJournalAsync()).OfType()); + gate.Release(); + await AwaitAssertAsync(async () => + { + var resolved = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal("Denied", resolved.Decision); + Assert.Equal(requested.SourceChildRunId, resolved.SourceChildRunId); + Assert.Equal(requested.AuthorizationAttemptId, resolved.AuthorizationAttemptId); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Empty(_store.Load().Audiences); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + } + finally + { + try { if (Directory.Exists(TestPaths.ToolApprovalsPath)) Directory.Delete(TestPaths.ToolApprovalsPath); } + finally { ReleaseOwnedSignals(gate); } + } + } + + private async Task<(IActorRef Manager, Akka.TestKit.TestProbe Subscriber, ToolInteractionRequest Prompt)> StartPromptAsync() + { + Assert.Empty(_store.Load().Audiences); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = Session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Start the neutral child task.", Source = new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), TurnId = new TurnId("original"), + MessageId = "original", Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, + Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + } + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _child.NextResponseGate = _nextChildResponse; + _firstChildResponse.TrySetResult(); + var prompt = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolInteractionRequest, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + Assert.Equal(new SenderId("operator-a"), prompt.RequesterSenderId); + Assert.Contains(prompt.Options, option => option.Key == ApprovalOptionKeys.ApproveEverywhereKey); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + var requested = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.NotNull(requested.SourceChildRunId); + Assert.Equal(prompt.CallId.Value, requested.CallId); + return (manager, subscriber, prompt); + } + + private static ToolInteractionResponse Answer(ToolInteractionRequest prompt, string sender) => new() + { + SessionId = Session, CallId = prompt.CallId, SenderId = new SenderId(sender), SelectedKey = ApprovalOptionKeys.ApproveEverywhereKey + }; + + private void ReleaseOwnedSignals(WaitReplyGate gate) + { + gate.Release(); + _firstChildResponse.TrySetResult(); + _nextChildResponse.TrySetResult(); + } + + private WaitReplyGate InstallWaitReplyGate() + { + Sys.Settings.InjectTopLevelFallback(ConfigurationFactory.ParseString( + $"child-approval-reply-mailbox {{ mailbox-type = \"{typeof(WaitReplyMailbox).AssemblyQualifiedName}\" }}")); + ((ExtendedActorSystem)Sys).Provider.Deployer.SetDeploy(new Deploy( + $"/session-manager/{Uri.EscapeDataString(Session.Value)}", Config.Empty, NoRouter.Instance, LocalScope.Instance, + Deploy.NoDispatcherGiven, "child-approval-reply-mailbox")); + return WaitReplyMailbox.Gates.GetOrCreateValue(Sys); + } + + private async Task ReadJournalAsync() + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reader = Sys.ActorOf(Props.Create(() => new JournalReader($"session-{Session.Value}", done))); + var watcher = CreateTestProbe(); + watcher.Watch(reader); + try { return await done.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); } + finally + { + Sys.Stop(reader); + await watcher.ExpectTerminatedAsync(reader, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + } + + private sealed class JournalReader : ReceivePersistentActor + { + public JournalReader(string id, TaskCompletionSource done) + { + PersistenceId = id; + var events = new List(); + Recover(events.Add); + Recover(_ => done.TrySetResult(events.ToArray())); + } + public override string PersistenceId { get; } + public override Recovery Recovery => new(SnapshotSelectionCriteria.None); + } + + private sealed class RoleProvider(IChatClient main, IChatClient child) : IChatClientProvider + { + public IChatClient GetClient(ModelRole role) => role == ModelRole.Main ? main : child; + } + + private sealed class StartExecutor(SubAgentSpawner spawner) : IToolExecutor + { + public Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) + => spawner.StartRunAsync(new SubAgentProfile + { + Name = "worker", Description = "Complete the neutral task.", SystemPrompt = "Complete the assigned task.", + ModelRole = ModelRole.Compaction, ToolNames = ["approval_probe"], EmitStructuredFindings = false + }, "Request the neutral operation.", null, + new ToolInvocationContext(context.RunScope, context.ExecutionTimeout, context.Outputs), ct, null); + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + public sealed class WaitReplyGate + { + private readonly object _sync = new(); + private bool _armed; + public string MessageName { get; set; } = "ChildApprovalWaitFinished"; + public sealed record Captured(IActorRef Receiver, Envelope Envelope); + public TaskCompletionSource Held { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void Arm() { lock (_sync) _armed = true; } + public bool TryHold(IActorRef receiver, Envelope envelope) + { + lock (_sync) + { + return _armed && Held.TrySetResult(new Captured(receiver, envelope)); + } + } + public void Release() + { + Captured? captured; + lock (_sync) + { + if (!_armed) return; + _armed = false; + captured = Held.Task.IsCompletedSuccessfully ? Held.Task.Result : null; + } + captured?.Receiver.Tell(captured.Envelope.Message, captured.Envelope.Sender); + } + } + + public sealed class WaitReplyMailbox(Settings settings, Config config) : MailboxType(settings, config), IProducesMessageQueue + { + internal static readonly ConditionalWeakTable Gates = new(); + public override IMessageQueue Create(IActorRef owner, ActorSystem system) => new WaitReplyQueue(Gates.GetOrCreateValue(system)); + } + + public sealed class WaitReplyQueue(WaitReplyGate gate) : IMessageQueue, IUnboundedDequeBasedMessageQueueSemantics + { + private readonly UnboundedDequeMessageQueue _inner = new(); + public int Count => _inner.Count; + public bool HasMessages => _inner.HasMessages; + public void Enqueue(IActorRef receiver, Envelope envelope) + { + if (envelope.Message.GetType().Name == gate.MessageName && gate.TryHold(receiver, envelope)) return; + _inner.Enqueue(receiver, envelope); + } + public void EnqueueFirst(Envelope envelope) => _inner.EnqueueFirst(envelope); + public bool TryDequeue(out Envelope envelope) => _inner.TryDequeue(out envelope); + public void CleanUp(IActorRef owner, IMessageQueue deadletters) => _inner.CleanUp(owner, deadletters); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs new file mode 100644 index 000000000..88a2e0085 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs @@ -0,0 +1,266 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Reminders; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Actors.SubAgents; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task Cold_recovery_resumes_an_already_adopted_child_input_without_duplicate_adoption_or_detector_reset() + { + PrepareAdmittedRecoveryRounds(); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Delegate after the two confirmed rounds.", + Source = DeliverySource("original", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var oldResponse = NewSignal(); + var recoveredResponse = NewSignal(); + _main.NextResponseGate = oldResponse; + try + { + _childRelease.TrySetResult(); + await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var prefix = await ReadJournalPositionsAsync(); + var accepted = Assert.Single(prefix.Select(row => row.Event).OfType()); + var delivery = Assert.Single(prefix.Select(row => row.Event).OfType()); + var adopted = Assert.Single(prefix.Select(row => row.Event).OfType(), + evt => evt.ContinuedChildRunId == accepted.Run.RunId); + Assert.Equal(new[] { delivery.Input.InputId }, adopted.InputIds); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, adopted.TurnContext)); + Assert.Empty(prefix.Select(row => row.Event).OfType()); + Assert.Single(prefix.Select(row => row.Event).OfType()); + AssertAdmittedRecoveryPair(_main.ReceivedMessages[^1], delivery, accepted.Run); + Assert.Equal(new[] { "prefix-probe-1", "prefix-probe-2" }, _start!.ProbeCallIds); + + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + _main.NextResponseGate = recoveredResponse; + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-refused-3")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-stopped-4")]); + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, + recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(owner, await OwnerAsync()); + await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var resumed = await ReadJournalPositionsAsync(); + Assert.Equal(prefix.Select(row => row.SequenceNr), resumed.Select(row => row.SequenceNr)); + AssertAdmittedRecoveryPair(_main.ReceivedMessages[^1], delivery, accepted.Run); + Assert.Single(resumed.Select(row => row.Event).OfType(), + evt => evt.ContinuedChildRunId == accepted.Run.RunId); + Assert.Equal(1, _child.CallCount); + recoveredResponse.TrySetResult(); + await CompletedAsync(recoveredSubscriber); + var final = await ReadJournalPositionsAsync(); + AssertAdmittedRecoverySettlement(final, delivery, accepted.Run, ["prefix-probe-1", "prefix-probe-2"]); + Assert.Equal(7, _main.CallCount); + } + finally + { + oldResponse.TrySetResult(); + recoveredResponse.TrySetResult(); + } + } + + [Fact] + public async Task Cold_recovery_keeps_an_ordinary_head_before_its_pending_child_result() + { + PrepareAdmittedRecoveryRounds(); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Delegate after the two confirmed rounds.", + Source = DeliverySource("original", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var oldResponse = NewSignal(); + var recoveredResponse = NewSignal(); + _main.NextResponseGate = oldResponse; + try + { + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Complete the ordinary head first.", + Source = DeliverySource("ordinary-head", "operator-b") + }, Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _childRelease.TrySetResult(); + ChildRunEvent.DeliveryAdmitted? delivery = null; + await AwaitAssertAsync(async () => + { + delivery = Assert.Single((await ReadJournalAsync()).OfType()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var prefix = await ReadJournalPositionsAsync(); + var accepted = Assert.Single(prefix.Select(row => row.Event).OfType()); + var ordinary = Assert.Single(prefix.Select(row => row.Event).OfType(), + input => input.TurnContext.TurnId == "ordinary-head"); + var ordinaryAdoption = Assert.Single(prefix.Select(row => row.Event).OfType(), + evt => evt.TurnContext.TurnId == "ordinary-head"); + Assert.True(SessionState.SameCanonicalContext(ordinary.TurnContext, ordinaryAdoption.TurnContext)); + Assert.Equal(new SenderId("operator-b"), ordinaryAdoption.TurnContext.RequesterSenderId); + Assert.DoesNotContain(prefix, row => row.Event is ToolTaskAdopted { ContinuedChildRunId: not null }); + Assert.True(Assert.Single(prefix, row => row.Event is InputAdmitted input + && input.InputId == ordinary.InputId).SequenceNr < Assert.Single(prefix, + row => row.Event is ChildRunEvent.DeliveryAdmitted).SequenceNr); + Assert.Single(prefix.Select(row => row.Event).OfType()); + + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, + recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var recovered = await OwnerAsync(); + Assert.NotEqual(owner, recovered); + await AssertTerminalCutOwnerBarrierAsync(recovered, "ordinary-prefix-before-resume"); + Assert.Equal(5, _main.CallCount); + var waiting = await ReadJournalPositionsAsync(); + Assert.Equal(prefix.Select(row => row.SequenceNr), waiting.Select(row => row.SequenceNr)); + + _main.NextResponseGate = recoveredResponse; + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("ordinary-probe")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-refused-3")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-stopped-4")]); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Resume the work that was interrupted by the daemon restart.", + Source = DeliverySource("restart-resume-child-prefix:1", "reminder-system") with + { + ReminderId = new ReminderId("restart-resume-child-prefix:1"), + Principal = PrincipalClassification.VerifiedAutomation + } + }, Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var ordinaryRequest = _main.ReceivedMessages[^1]; + Assert.Contains(ordinaryRequest, message => message.Role == Microsoft.Extensions.AI.ChatRole.User + && message.Text == ordinary.UserMessage.Content); + var deliveryId = delivery!.Input.UserMessage.ToolCallId!.Value.Value; + Assert.DoesNotContain(ordinaryRequest.SelectMany(message => message.Contents.OfType()), + call => call.CallId == deliveryId); + Assert.DoesNotContain(ordinaryRequest.SelectMany(message => message.Contents.OfType()), + result => result.CallId == deliveryId); + Assert.DoesNotContain(await ReadJournalAsync(), evt => evt is ToolTaskAdopted { ContinuedChildRunId: not null }); + recoveredResponse.TrySetResult(); + await CompletedAsync(recoveredSubscriber); + var final = await ReadJournalPositionsAsync(); + AssertAdmittedRecoverySettlement(final, delivery, accepted.Run, + ["prefix-probe-1", "prefix-probe-2", "ordinary-probe"]); + Assert.Equal(new[] { "operator-a", "operator-a", "operator-b" }, _start!.ProbeRequesters); + Assert.Single(final.Select(row => row.Event).OfType(), + evt => evt.TurnContext.TurnId == "ordinary-head"); + var childAdoption = Assert.Single(final, + row => row.Event is ToolTaskAdopted { ContinuedChildRunId: not null }); + var ordinaryBatch = Assert.Single(final, row => row.Event is ToolBatchStarted batch + && batch.ConsumedInputIds.Contains(ordinary.InputId)); + Assert.True(ordinaryBatch.SequenceNr < childAdoption.SequenceNr); + Assert.Equal(new[] { ordinary.InputId, delivery.Input.InputId }, final.SelectMany(row => row.Event switch + { + ToolBatchStarted batch => batch.ConsumedInputIds, + TurnRecorded turn => turn.ConsumedInputIds, + InputClosed closed => closed.InputIds, + _ => Array.Empty() + }).Where(id => id == ordinary.InputId || id == delivery.Input.InputId)); + AssertAdmittedRecoveryPair(_main.ReceivedMessages[6], delivery, accepted.Run); + Assert.Equal(8, _main.CallCount); + } + finally + { + oldResponse.TrySetResult(); + recoveredResponse.TrySetResult(); + } + } + + private void PrepareAdmittedRecoveryRounds() + { + _main.ToolCallsOnFirstCall = null; + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-1")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-2")]); + _main.PlannedResponses.Enqueue([new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Inspect the neutral fixture." })]); + _main.PlannedResponses.Enqueue([new TextContent("The original child run was accepted.")]); + } + + private static FunctionCallContent AdmittedRecoveryProbe(string id) + => new(id, "neutral_probe", new Dictionary()); + + private static void AssertAdmittedRecoveryPair(IReadOnlyList request, + ChildRunEvent.DeliveryAdmitted delivery, BackgroundChildRun run) + { + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var call = Assert.Single(request.SelectMany(message => message.Contents.OfType()), + item => item.CallId == id); + Assert.Equal(ChildRunDelivery.ToolName(run), call.Name); + Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single( + request.SelectMany(message => message.Contents.OfType()), + item => item.CallId == id).Result)); + Assert.Single(request.SelectMany(message => message.Contents.OfType()), + item => item.CallId == "start-1"); + } + + private void AssertAdmittedRecoverySettlement((ISessionEvent Event, long SequenceNr)[] rows, + ChildRunEvent.DeliveryAdmitted delivery, BackgroundChildRun run, string[] executed) + { + var events = rows.Select(row => row.Event).ToArray(); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + var adoption = Assert.Single(events.OfType(), evt => evt.ContinuedChildRunId == run.RunId); + Assert.Equal(new[] { delivery.Input.InputId }, adoption.InputIds); + Assert.True(SessionState.SameCanonicalContext(run.OriginalContext, adoption.TurnContext)); + Assert.Equal(new SenderId("operator-a"), adoption.TurnContext.RequesterSenderId); + Assert.Equal(executed, _start!.ProbeCallIds); + var refused = Assert.Single(events.OfType(), + evt => evt.ToolResult.ToolCallId == new ToolCallId("prefix-refused-3")); + Assert.True(Assert.IsType(refused.LoopObservation).Synthetic); + Assert.StartsWith(ToolCycleMessages.Correction, refused.ToolResult.Content); + Assert.DoesNotContain(events.OfType(), + evt => evt.AssistantMessage.ToolCalls.Any(call => call.CallId == new ToolCallId("prefix-stopped-4"))); + var refusedBatch = Assert.Single(events.OfType(), + evt => evt.AssistantMessage.ToolCalls.Any(call => call.CallId == new ToolCallId("prefix-refused-3"))); + Assert.Equal(run.ParentCheckpoint.TaskId, refusedBatch.LoopAdmission!.TaskId); + Assert.Contains("prefix-refused-3", refusedBatch.LoopAdmission.RefusedCallIds); + Assert.Equal(new[] { delivery.Input.InputId }, events.SelectMany(evt => evt switch + { + ToolBatchStarted batch => batch.ConsumedInputIds, + TurnRecorded turn => turn.ConsumedInputIds, + InputClosed closed => closed.InputIds, + _ => Array.Empty() + }).Where(id => id == delivery.Input.InputId)); + var settlement = Assert.Single(events.OfType(), + evt => evt.AssistantReply.Content == ToolCycleMessages.Final); + Assert.Equal(ToolCycleMessages.Final, settlement.AssistantReply.Content); + Assert.Empty(events.OfType()); + Assert.Equal(1, _child.CallCount); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Cancellation.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Cancellation.cs new file mode 100644 index 000000000..349852cf8 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Cancellation.cs @@ -0,0 +1,190 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Actors.Tools; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_checkpoint_commit_precedes_child_progress_and_cancel_retains_its_actual_evidence(bool reportFailure) + { + var path = Path.Combine(_directory!.Path, "confirmed-child-file.txt"); + var toolCalls = 0; + var read = new FakeNetclawTool("file_read", "The actual child result is retained.", "file", invocation => + { + Interlocked.Increment(ref toolCalls); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([new ToolFileActivity(path, ToolFileActivityKind.Read)], null)); + }); + Host.Services.GetRequiredService().ReplaceCore(read); + _child.ToolCallsOnFirstCall = [new FunctionCallContent("child-read-1", "file_read", new Dictionary { ["_rationale"] = "Read the neutral fixture." })]; + var checkpointEntered = NewSignal(); + var checkpointRelease = NewSignal(); + var closureEntered = NewSignal(); + var closureRelease = NewSignal(); + var nextChildResponse = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is ChildRunEvent.Checkpointed) + { + checkpointEntered.TrySetResult(); + await checkpointRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + } + if (record.Payload is ChildRunEvent.DispatchClosed) + { + closureEntered.TrySetResult(); + await closureRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + } + return false; + }); + var (_, manager, subscriber) = await CreateOwnerAsync(); + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _child.NextResponseGate = nextChildResponse; + _childRelease.TrySetResult(); + await checkpointEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(1, Volatile.Read(ref toolCalls)); + Assert.Equal(1, _child.CallCount); + Assert.Empty((await ReadJournalAsync()).OfType()); + checkpointRelease.TrySetResult(); + await AwaitAssertAsync(() => Assert.Equal(2, _child.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + var checkpoint = Assert.Single(before.OfType()); + Assert.Equal(accepted.Run.RunId, checkpoint.RunId); + Assert.Equal(1, checkpoint.Checkpoint.CompletedRound); + Assert.Equal("The actual child result is retained.", checkpoint.Checkpoint.Summary); + Assert.Equal(path, Assert.Single(checkpoint.Checkpoint.ConfirmedActivity.ReadFiles)); + Assert.Equal("operator-a", read.LastContext!.RunScope.DefaultDeliveryTarget!.DestinationId); + var childPairs = _child.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()); + Assert.Equal("The actual child result is retained.", Assert.IsType(Assert.Single(childPairs).Result)); + var storage = Assert.IsType(_start!.Prepared!.Execution.Scope.Authority.Session).Storage; + var report = Path.Combine(storage.ArtifactDirectory.Value, "cancelled-results.json"); + if (reportFailure) Directory.CreateDirectory(report); + _main.PlannedResponses.Enqueue([ControlCall("cancel-1", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel this child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await closureEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var control = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(control.Run); + Assert.Equal(BackgroundChildState.Cancelling, control.Run.State); + Assert.NotNull(control.Run.CancellationRequestedAtMs); + Assert.Null(control.Run.DispatchClosedAtMs); + var admitted = await ReadJournalAsync(); + Assert.Single(admitted.OfType()); + Assert.Empty(admitted.OfType()); + Assert.Empty(admitted.OfType()); + closureRelease.TrySetResult(); + ChildRunEvent.TerminalRecorded? terminal = null; + await AwaitAssertAsync(async () => + { + terminal = Assert.Single((await ReadJournalAsync()).OfType()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotNull(terminal); + Assert.False(terminal.Terminal.Lost); + Assert.False(terminal.Terminal.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, terminal.Terminal.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var cancelled = Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(path, Assert.Single(cancelled.ConfirmedActivity!.ReadFiles)); + Assert.True(nextChildResponse.Task.IsCanceled); + Assert.Equal(2, _child.CallCount); + Assert.Equal(1, Volatile.Read(ref toolCalls)); + if (reportFailure) + { + Assert.Contains("No report path is confirmed", terminal.Terminal.EvidenceWarning, StringComparison.Ordinal); + Assert.DoesNotContain(report, terminal.Terminal.Result.Output, StringComparison.Ordinal); + Assert.False(File.Exists(report)); + } + else + { + using var body = JsonDocument.Parse(await File.ReadAllTextAsync(report, TestContext.Current.CancellationToken)); + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Cancelled", body.RootElement.GetProperty("state").GetString()); + Assert.Equal(checkpoint.Checkpoint.Summary, body.RootElement.GetProperty("summary").GetString()); + Assert.Contains(report, terminal.Terminal.Result.Output, StringComparison.Ordinal); + Assert.Null(terminal.Terminal.EvidenceWarning); + } + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Equal(2, _child.CallCount); + } + finally + { + checkpointRelease.TrySetResult(); + closureRelease.TrySetResult(); + nextChildResponse.TrySetResult(); + } + } + + [Fact] + public async Task A_genuine_foreign_parent_tool_cannot_inspect_or_cancel_an_older_child() + { + var (_, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + _main.PlannedResponses.Enqueue([ControlCall("foreign-cancel", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Inspect that child.", Source = DeliverySource("foreign", "operator-b") }, + Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var denied = await _start!.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Null(denied.Run); + var after = await ReadJournalAsync(); + Assert.Empty(after.OfType()); + Assert.Empty(after.OfType()); + Assert.Empty(after.OfType()); + Assert.Single(after.OfType()); + Assert.False(_childRelease.Task.IsCompleted); + Assert.Equal(1, _child.CallCount); + var result = Assert.Single(after.OfType(), evt => evt.ToolResult.ToolCallId == new ToolCallId("foreign-cancel")); + using var body = JsonDocument.Parse(result.ToolResult.Content); + Assert.False(body.RootElement.GetProperty("found").GetBoolean()); + Assert.DoesNotContain(accepted.Run.RunId.Value, result.ToolResult.Content, StringComparison.Ordinal); + Assert.DoesNotContain(accepted.Run.ScopeId.Value, result.ToolResult.Content, StringComparison.Ordinal); + _main.PlannedResponses.Enqueue([ControlCall("own-cancel", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the owned child.", Source = DeliverySource("own", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(async () => + { + Assert.Single((await ReadJournalAsync()).OfType()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Equal(1, _child.CallCount); + } + + private static FunctionCallContent ControlCall(string id, SubAgentRunId runId) => new(id, "control_probe", + new Dictionary { ["run_id"] = runId.Value, ["_rationale"] = "Control the recorded child." }); +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CancellationCallbacks.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CancellationCallbacks.cs new file mode 100644 index 000000000..8d0fe8c51 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CancellationCallbacks.cs @@ -0,0 +1,168 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_held_execution_cancellation_callback_cannot_block_the_owner_terminal(bool naturalCompletion) + { + var callbackEntered = NewSignal(); + var callbackExited = NewSignal(); + using var callbackRelease = new ManualResetEventSlim(); + var provider = new UncooperativeChildClient(_child) + { + CancellationCallback = () => + { + callbackEntered.TrySetResult(); + try { callbackRelease.Wait(TestContext.Current.CancellationToken); } + finally { callbackExited.TrySetResult(); } + } + }; + Assert.IsType(Host.Services.GetRequiredService()).Child = provider; + var path = Path.Combine(_directory!.Path, "confirmed-before-held-cancellation-callback.txt"); + var effects = 0; + Host.Services.GetRequiredService().ReplaceCore(new FakeNetclawTool( + "file_read", "The real receipt precedes the callback hold.", "file", invocation => + { + Interlocked.Increment(ref effects); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([new ToolFileActivity(path, ToolFileActivityKind.Read)], null)); + })); + _child.ToolCallsOnFirstCall = [ReadCall("before-held-cancellation-callback")]; + var controlRelease = NewSignal(); + var (_, manager, subscriber) = await CreateOwnerAsync(); + Task? statusTask = null; + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _childRelease.TrySetResult(); + await provider.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + var checkpoint = Assert.Single(before.OfType()); + Assert.Equal(path, Assert.Single(checkpoint.Checkpoint.ConfirmedActivity.ReadFiles)); + Assert.Equal(new SenderId("operator-a"), accepted.Run.OriginalContext.RequesterSenderId); + var pair = Assert.Single(provider.HeldMessages.SelectMany(message => message.Contents.OfType())); + Assert.Equal("before-held-cancellation-callback", pair.CallId); + Assert.Equal(checkpoint.Checkpoint.Summary, Assert.IsType(pair.Result)); + Assert.False(provider.Token.IsCancellationRequested); + Assert.False(callbackEntered.Task.IsCompleted); + if (naturalCompletion) + { + provider.Pending.TrySetResult(new ChatResponse(new ChatMessage(Microsoft.Extensions.AI.ChatRole.Assistant, "The neutral child task is complete."))); + } + else + { + _start!.ControlCompletionGate = controlRelease; + _main.PlannedResponses.Enqueue([ControlCall("cancel-held-callback", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the accepted child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + var cancelled = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(BackgroundChildState.Cancelling, cancelled.Run!.State); + } + await callbackEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.True(provider.Token.IsCancellationRequested); + Assert.False(callbackExited.Task.IsCompleted); + ChildRunEvent.TerminalRecorded? terminal = null; + await AwaitAssertAsync(async () => + { + var records = await ReadJournalAsync(); + Assert.Single(records.OfType()); + Assert.Single(records.OfType()); + terminal = Assert.Single(records.OfType()); + Assert.Equal(accepted.Run.RunId, terminal.RunId); + if (naturalCompletion) + { + Assert.Empty(records.OfType()); + Assert.Empty(records.OfType()); + Assert.True(terminal.Terminal.Result.Success); + Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal("The neutral child task is complete.", terminal.Terminal.Result.Output); + } + else + { + Assert.Single(records.OfType()); + Assert.Single(records.OfType()); + Assert.False(terminal.Terminal.Result.Success); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var cancelled = Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(path, Assert.Single(cancelled.ConfirmedActivity!.ReadFiles)); + } + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotNull(terminal); + if (naturalCompletion) + { + _main.PlannedResponses.Enqueue([new FunctionCallContent("status-held-callback", "status_json_probe", + new Dictionary { ["run_id"] = accepted.Run.RunId.Value, ["cancel"] = false, + ["_rationale"] = "Inspect the owned child terminal." })]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Read the owned child status.", Source = DeliverySource("status", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(async () => + { + var records = await ReadJournalAsync(); + var result = Assert.Single(records.OfType(), record => record.ToolResult.ToolCallId == new ToolCallId("status-held-callback")); + using var status = JsonDocument.Parse(result.ToolResult.Content); + Assert.Equal(accepted.Run.RunId.Value, status.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Completed", status.RootElement.GetProperty("state").GetString()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + else + { + var factory = _start!.ControlInvocation!.SpawnChildActor!; + statusTask = factory(new ChildControlRequest(accepted.Run.RunId, false), "status-with-held-callback", _start.ControlToken); + var status = Assert.IsType(await statusTask.WaitAsync(Ceiling, TestContext.Current.CancellationToken)); + Assert.Equal(BackgroundChildState.Cancelled, status.Run!.State); + Assert.NotNull(status.Run.DispatchClosedAtMs); + Assert.Equal(checkpoint.Checkpoint, status.Run.ChildCheckpoint); + Assert.False(provider.Pending.Task.IsCompleted); + } + Assert.False(callbackExited.Task.IsCompleted); + Assert.Equal(2, provider.Count); + Assert.Equal(1, Volatile.Read(ref effects)); + } + finally + { + callbackRelease.Set(); + _childRelease.TrySetResult(); + provider.Pending.TrySetCanceled(TestContext.Current.CancellationToken); + try + { + if (callbackEntered.Task.IsCompletedSuccessfully) + await callbackExited.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + if (provider.Held.Task.IsCompletedSuccessfully) + await provider.Exited.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + if (statusTask is not null) + await statusTask.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + } + finally + { + controlRelease.TrySetResult(); + provider.Dispose(); + } + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ClosureCuts.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ClosureCuts.cs new file mode 100644 index 000000000..3439bf02f --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ClosureCuts.cs @@ -0,0 +1,202 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Configuration; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Actors.Tools; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_synchronous_child_dispatch_prefix_cannot_block_the_parent_after_cancel_admission() + { + var workerEntered = NewSignal(); + var workerExited = NewSignal(); + using var workerRelease = new ManualResetEventSlim(); + var controlRelease = NewSignal(); + var path = Path.Combine(_directory!.Path, "confirmed-before-held-entry.txt"); + var effects = 0; + var read = new FakeNetclawTool("file_read", "The actual child receipt is retained.", "file", invocation => + { + var effect = Interlocked.Increment(ref effects); + if (effect == 2) + { + workerEntered.TrySetResult(); + try { workerRelease.Wait(TestContext.Current.CancellationToken); } + finally { workerExited.TrySetResult(); } + } + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([new ToolFileActivity(path, ToolFileActivityKind.Read)], null)); + }); + Host.Services.GetRequiredService().ReplaceCore(read); + _child.PlannedResponses.Enqueue([ReadCall("confirmed-read")]); + _child.PlannedResponses.Enqueue([ReadCall("held-read")]); + var (_, manager, subscriber) = await CreateOwnerAsync(); + Task? statusTask = null; + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _childRelease.TrySetResult(); + await workerEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + var checkpoint = Assert.Single(before.OfType()); + Assert.Equal(1, checkpoint.Checkpoint.CompletedRound); + Assert.Equal("The actual child receipt is retained.", checkpoint.Checkpoint.Summary); + Assert.Equal(path, Assert.Single(checkpoint.Checkpoint.ConfirmedActivity.ReadFiles)); + Assert.Equal(2, _child.CallCount); + Assert.Equal(2, Volatile.Read(ref effects)); + Assert.Equal("operator-a", read.LastContext!.RunScope.DefaultDeliveryTarget!.DestinationId); + _start!.ControlCompletionGate = controlRelease; + _main.PlannedResponses.Enqueue([ControlCall("held-entry-cancel", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the owned child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + var cancelled = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(cancelled.Run); + Assert.Equal(BackgroundChildState.Cancelling, cancelled.Run.State); + Assert.Null(cancelled.Run.DispatchClosedAtMs); + Assert.False(workerExited.Task.IsCompleted); + var admitted = await ReadJournalAsync(); + Assert.Single(admitted.OfType()); + Assert.Empty(admitted.OfType()); + Assert.Empty(admitted.OfType()); + var control = _start.ControlInvocation!.SpawnChildActor!; + statusTask = control(new ChildControlRequest(accepted.Run.RunId, false), "status-during-held-prefix", _start.ControlToken); + var status = Assert.IsType(await statusTask.WaitAsync(Ceiling, TestContext.Current.CancellationToken)); + Assert.NotNull(status.Run); + Assert.Equal(BackgroundChildState.Cancelling, status.Run.State); + Assert.Null(status.Run.DispatchClosedAtMs); + Assert.Equal(checkpoint.Checkpoint, status.Run.ChildCheckpoint); + Assert.False(workerExited.Task.IsCompleted); + Assert.Equal(2, _child.CallCount); + } + finally + { + workerRelease.Set(); + if (workerEntered.Task.IsCompletedSuccessfully) + await workerExited.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + try + { + if (statusTask is not null) await statusTask.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + } + finally { controlRelease.TrySetResult(); } + } + await AwaitAssertAsync(async () => + { + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + var terminal = Assert.Single(final.OfType()); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(2, _child.CallCount); + Assert.Equal(2, Volatile.Read(ref effects)); + } + + [Fact] + public async Task A_real_checkpoint_ack_cannot_start_a_provider_request_after_local_dispatch_closure() + { + const string mailbox = "child-checkpoint-ack-cut-mailbox"; + Sys.Settings.InjectTopLevelFallback(ConfigurationFactory.ParseString( + $"{mailbox} {{ mailbox-type = \"{typeof(BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox).AssemblyQualifiedName}\" }}")); + var gate = BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox.Gates.GetOrCreateValue(Sys); + gate.MessageName = nameof(BackgroundChildCheckpointAck); + gate.Arm(); + _start!.ChildMailbox = mailbox; + var closureEntered = NewSignal(); + var closureRelease = NewSignal(); + var nextProvider = NewSignal(); + var effects = 0; + var path = Path.Combine(_directory!.Path, "confirmed-before-held-ack.txt"); + Host.Services.GetRequiredService().ReplaceCore(new FakeNetclawTool( + "file_read", "The acknowledged child result is retained.", "file", invocation => + { + Interlocked.Increment(ref effects); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([new ToolFileActivity(path, ToolFileActivityKind.Read)], null)); + })); + _child.ToolCallsOnFirstCall = [ReadCall("checkpoint-read")]; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is ChildRunEvent.DispatchClosed) + { + closureEntered.TrySetResult(); + await closureRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + } + return false; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _child.NextResponseGate = nextProvider; + _childRelease.TrySetResult(); + var captured = await gate.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var ack = Assert.IsType(captured.Envelope.Message); + Assert.Equal(owner, captured.Envelope.Sender); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + var checkpoint = Assert.Single(before.OfType()); + Assert.Equal(accepted.Run.RunId, ack.RunId); + Assert.Equal(checkpoint.Checkpoint.CompletedRound, ack.CompletedRound); + Assert.Equal(1, Volatile.Read(ref effects)); + Assert.Equal(1, _child.CallCount); + _main.PlannedResponses.Enqueue([ControlCall("held-ack-cancel", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the owned child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await closureEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var cancel = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(BackgroundChildState.Cancelling, cancel.Run!.State); + Assert.Null(cancel.Run.DispatchClosedAtMs); + Assert.Empty((await ReadJournalAsync()).OfType()); + gate.Release(); + var barrier = await captured.Receiver.Ask(new Identify("after-closed-checkpoint-ack"), + Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(captured.Receiver, barrier.Subject); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, Volatile.Read(ref effects)); + Assert.False(nextProvider.Task.IsCompleted); + closureRelease.TrySetResult(); + await AwaitAssertAsync(async () => + { + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + var terminal = Assert.Single(final.OfType()); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var completion = Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(path, Assert.Single(completion.ConfirmedActivity!.ReadFiles)); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, Volatile.Read(ref effects)); + } + finally + { + gate.Release(); + closureRelease.TrySetResult(); + nextProvider.TrySetResult(); + } + } + + private static FunctionCallContent ReadCall(string id) => new(id, "file_read", + new Dictionary { ["_rationale"] = "Read the neutral fixture." }); +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CrossSession.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CrossSession.cs new file mode 100644 index 000000000..4b7544b58 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.CrossSession.cs @@ -0,0 +1,102 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Another_session_cannot_read_or_cancel_a_child_with_the_same_requester(bool cancel) + { + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Delegate the neutral task.", + Source = DeliverySource("original", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var original = await ReadJournalAsync(); + var accepted = Assert.Single(original.OfType()); + var storage = Assert.IsType(_start!.Prepared!.Execution.Scope.Authority.Session).Storage; + var foreignSession = new SessionId("signalr/other-owner-adversarial"); + var foreignSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(foreignSubscriber) + { SessionId = foreignSession, Filter = OutputFilter.Full }, foreignSubscriber.Ref); + await foreignSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var foreignOwner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(foreignSession.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + Assert.NotEqual(owner, foreignOwner); + _main.PlannedResponses.Enqueue([StatusCall("cross-session-control", cancel)]); + await manager.Ask(new SendUserMessage + { + SessionId = foreignSession, Content = "Inspect the recorded child.", + Source = DeliverySource("foreign-session", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + var denied = Assert.IsType(await foreignSubscriber.FishForMessageAsync( + message => message is ToolResultOutput, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + await CompletedAsync(foreignSubscriber); + Assert.Equal("Error: the child run was not found or is not accessible from this session.", denied.Result); + foreach (var detail in new[] { accepted.Run.RunId.Value, accepted.Run.ScopeId.Value, + storage.LogPath.Value, storage.ArtifactDirectory.Value, "log_path", "artifact_directory" }) + Assert.DoesNotContain(detail, denied.Result, StringComparison.Ordinal); + var deniedPair = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType())); + Assert.Equal("cross-session-control", deniedPair.CallId); + Assert.Equal(denied.Result, Assert.IsType(deniedPair.Result)); + var retained = await ReadJournalAsync(); + Assert.Equal(original.Length, retained.Length); + Assert.Empty(retained.OfType()); + Assert.Empty(retained.OfType()); + Assert.Empty(retained.OfType()); + Assert.False(_childRelease.Task.IsCompleted); + Assert.Equal(1, _child.CallCount); + Assert.Equal(4, _main.CallCount); + + _main.PlannedResponses.Enqueue([StatusCall("original-session-control", false)]); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Inspect the child in its original session.", + Source = DeliverySource("original-session-control", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + var allowed = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolResultOutput, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + await CompletedAsync(subscriber); + using var body = JsonDocument.Parse(allowed.Result); + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(accepted.Run.ScopeId.Value, body.RootElement.GetProperty("scope_id").GetString()); + Assert.Equal("Running", body.RootElement.GetProperty("state").GetString()); + Assert.Equal(storage.LogPath.Value, body.RootElement.GetProperty("log_path").GetString()); + Assert.Equal(storage.ArtifactDirectory.Value, body.RootElement.GetProperty("artifact_directory").GetString()); + Assert.False(body.RootElement.GetProperty("cancellation_requested").GetBoolean()); + Assert.False(body.RootElement.GetProperty("dispatch_closed").GetBoolean()); + var result = Assert.Single((await ReadJournalAsync()).OfType(), + record => record.ToolResult.ToolCallId == new ToolCallId("original-session-control")); + Assert.Equal(allowed.Result, result.ToolResult.Content); + var currentPair = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()), + pair => pair.CallId == "original-session-control"); + Assert.Equal(allowed.Result, Assert.IsType(currentPair.Result)); + Assert.False(_childRelease.Task.IsCompleted); + Assert.Equal(1, _child.CallCount); + Assert.Equal(6, _main.CallCount); + + FunctionCallContent StatusCall(string id, bool requestCancel) => new(id, "status_json_probe", + new Dictionary + { + ["run_id"] = accepted.Run.RunId.Value, ["cancel"] = requestCancel, + ["_rationale"] = "Inspect the child through its session owner." + }); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs new file mode 100644 index 000000000..c1d081bc6 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs @@ -0,0 +1,274 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Configuration; +using Netclaw.Actors.SubAgents; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Actual_terminals_keep_journal_order_and_original_evidence_after_reversed_enrichment(bool receiptFailure) + { + _main.ToolCallsOnFirstCall = null; + static FunctionCallContent Probe(string id) => new(id, "neutral_probe", new Dictionary()); + _main.PlannedResponses.Enqueue([Probe("probe-1")]); + _main.PlannedResponses.Enqueue([Probe("probe-2")]); + var starts = new List + { + new FunctionCallContent("start-1", "start_probe", new Dictionary { ["_rationale"] = "Inspect the first part." }), + new FunctionCallContent("start-2", "start_probe", new Dictionary { ["_rationale"] = "Inspect the second part." }) + }; + if (receiptFailure) + starts.Add(new FunctionCallContent("defect-1", "missing_receipt_probe", new Dictionary())); + _main.PlannedResponses.Enqueue(starts); + if (!receiptFailure) _main.PlannedResponses.Enqueue([new TextContent("The original task has its two child runs.")]); + _main.PlannedResponses.Enqueue([new TextContent("The fresh task has no tool effect.")]); + _main.PlannedResponses.Enqueue([Probe("probe-3")]); + _main.PlannedResponses.Enqueue([Probe("probe-4")]); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await _secondChild.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var before = await ReadJournalAsync(); + var accepted = before.OfType().ToArray(); + Assert.Equal(2, accepted.Length); + Assert.Equal(2, _start!.ProbeCallIds.Count(id => id.StartsWith("probe-", StringComparison.Ordinal))); + Assert.All(_start.ProbeRequesters, requester => Assert.Equal("operator-a", requester)); + var oldModelCount = _main.CallCount; + Assert.Equal(receiptFailure ? 3 : 4, oldModelCount); + if (receiptFailure) + Assert.Equal(ToolCycleMessages.MissingReceipt, Assert.Single(before.OfType()).AssistantReply.Content); + var parentGate = NewSignal(); + var enrichmentGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _main.NextResponseGate = parentGate; + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Inspect the fresh task.", Source = DeliverySource("fresh", "operator-b") + }, Ceiling, TestContext.Current.CancellationToken); + try + { + await AwaitAssertAsync(() => Assert.Equal(oldModelCount + 1, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _snapshots.InvocationStarted = NewSignal(); + _snapshots.Pending = enrichmentGate; + _childRelease.TrySetResult(); + await _snapshots.InvocationStarted.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var firstPrefix = await ReadJournalPositionsAsync(); + var first = Assert.Single(firstPrefix, record => record.Event is ChildRunEvent.TerminalRecorded); + var firstTerminal = Assert.IsType(first.Event); + Assert.Equal(first.SequenceNr, firstTerminal.TerminalSequenceNr); + _snapshots.Pending = null; + _secondChildRelease.TrySetResult(); + (ISessionEvent Event, long SequenceNr)[] preparedSecond = []; + await AwaitAssertAsync(async () => + { + preparedSecond = await ReadJournalPositionsAsync(); + var prepared = Assert.Single(preparedSecond.Select(record => record.Event).OfType()); + Assert.NotEqual(firstTerminal.RunId, prepared.RunId); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + enrichmentGate.TrySetResult(new WorkingContextSnapshot + { WorkingContext = WorkingContext.Empty, Git = new GitWorkingContextInspection.Skipped() }); + (ISessionEvent Event, long SequenceNr)[] queued = []; + await AwaitAssertAsync(async () => + { + queued = await ReadJournalPositionsAsync(); + Assert.Equal(2, queued.Count(record => record.Event is ChildRunEvent.DeliveryAdmitted)); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var terminalRows = queued.Where(record => record.Event is ChildRunEvent.TerminalRecorded).ToArray(); + var terminals = terminalRows.Select(record => Assert.IsType(record.Event)).ToArray(); + Assert.Equal(2, terminals.Length); + Assert.Equal(terminals[0].RecordedAtMs, terminals[1].RecordedAtMs); + Assert.True(terminals[0].TerminalSequenceNr < terminals[1].TerminalSequenceNr); + Assert.All(terminalRows, record => Assert.Equal(record.SequenceNr, + Assert.IsType(record.Event).TerminalSequenceNr)); + var preparations = queued.Select(record => record.Event).OfType().ToArray(); + Assert.Equal(terminals.Select(terminal => terminal.RunId).Reverse(), preparations.Select(prepared => prepared.RunId)); + var deliveries = queued.Select(record => record.Event).OfType().ToArray(); + Assert.Equal(terminals.Select(terminal => terminal.RunId), deliveries.Select(delivery => delivery.RunId)); + Assert.Equal(2, deliveries.Select(delivery => delivery.Input.UserMessage.ToolCallId).Distinct().Count()); + Assert.All(deliveries, delivery => + { + Assert.Equal("original", delivery.Input.TurnContext.TurnId); + Assert.Equal(new SenderId("operator-a"), delivery.Input.TurnContext.RequesterSenderId); + Assert.DoesNotContain(delivery.Input.UserMessage.ToolCallId!.Value.Value, new[] { "start-1", "start-2" }); + }); + Assert.Equal(oldModelCount + 1, _main.CallCount); + parentGate.TrySetResult(); + await CompletedAsync(subscriber); + await CompletedAsync(subscriber); + var after = await ReadJournalAsync(); + var childAdoption = Assert.Single(after.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(deliveries.Select(delivery => delivery.Input.InputId), childAdoption.InputIds); + Assert.Equal("original", childAdoption.TurnContext.TurnId); + Assert.Equal(new SenderId("operator-a"), childAdoption.TurnContext.RequesterSenderId); + var freshAdoption = Assert.Single(after.OfType(), evt => evt.TurnContext.TurnId == "fresh"); + Assert.Equal(new SenderId("operator-b"), freshAdoption.TurnContext.RequesterSenderId); + var final = after.OfType().Last(); + var deliveryIds = deliveries.Select(delivery => delivery.Input.InputId).ToArray(); + var consumed = after.SelectMany(evt => evt switch + { + ToolBatchStarted batch => batch.ConsumedInputIds, + TurnRecorded turn => turn.ConsumedInputIds, + InputClosed closed => closed.InputIds, + _ => Array.Empty() + }).Where(deliveryIds.Contains); + Assert.Equal(deliveryIds, consumed); + Assert.Equal(receiptFailure ? ToolCycleMessages.MissingReceipt : ToolCycleMessages.Final, final.AssistantReply.Content); + Assert.Equal(oldModelCount + (receiptFailure ? 1 : 3), _main.CallCount); + Assert.Equal(new[] { "probe-1", "probe-2" }, _start.ProbeCallIds.Where(id => id.StartsWith("probe-", StringComparison.Ordinal))); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, _secondChild.CallCount); + Assert.Equal(2, after.OfType().Count()); + Assert.Equal(2, after.OfType().Count()); + Assert.Equal(2, after.OfType().Count()); + Assert.Equal(2, after.OfType().Count()); + Assert.Equal(2, after.OfType().Count()); + if (!receiptFailure) + { + var request = _main.ReceivedMessages[oldModelCount + 1]; + var calls = request.SelectMany(message => message.Contents.OfType()).ToArray(); + var results = request.SelectMany(message => message.Contents.OfType()).ToArray(); + foreach (var delivery in deliveries) + { + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + Assert.Single(calls, call => call.CallId == id); + Assert.Equal(delivery.Input.UserMessage.Content, + Assert.IsType(Assert.Single(results, result => result.CallId == id).Result)); + } + var correction = Assert.Single(after.OfType(), evt => evt.ToolResult.ToolCallId == new ToolCallId("probe-3")); + Assert.True(correction.LoopObservation!.Synthetic); + Assert.StartsWith(ToolCycleMessages.Correction, correction.ToolResult.Content); + } + } + finally + { + _snapshots.Pending = null; + enrichmentGate.TrySetResult(new WorkingContextSnapshot + { WorkingContext = WorkingContext.Empty, Git = new GitWorkingContextInspection.Skipped() }); + parentGate.TrySetResult(); + } + } + + [Fact] + public async Task A_failed_first_child_review_retains_one_canonical_pair_after_closed_input_and_cold_recovery() + { + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var original = await ReadJournalAsync(); + var accepted = Assert.Single(original.OfType()); + var deliveryInputId = new InputId($"child-result-{accepted.Run.RunId.Value}"); + var closeEntered = NewSignal(); + var closeRelease = NewSignal(); + var freshRelease = NewSignal(); + var failure = new InvalidOperationException("The first child review failed at the actual provider boundary."); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not InputClosed closed || !closed.InputIds.Contains(deliveryInputId)) return false; + closeEntered.TrySetResult(); + await closeRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + _main.PlannedExceptions.Enqueue(failure); + try + { + await EventFilter.Error(contains: "turn_failed").ExpectOneAsync(async () => + { + _childRelease.TrySetResult(); + await closeEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var prefix = await ReadJournalAsync(); + Assert.Single(prefix.OfType()); + Assert.Single(prefix.OfType(), evt => evt.ContinuedChildRunId == accepted.Run.RunId); + Assert.Empty(prefix.OfType()); + closeRelease.TrySetResult(); + var error = await subscriber.FishForMessageAsync(message => message is ErrorOutput, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Same(failure, Assert.IsType(error).Cause); + var completed = await CompletedAsync(subscriber); + Assert.Equal(TurnOutcome.Failed, Assert.IsType(completed).Outcome); + }, cancellationToken: TestContext.Current.CancellationToken); + var closedPrefix = await ReadJournalAsync(); + var delivery = Assert.Single(closedPrefix.OfType()); + var closure = Assert.Single(closedPrefix.OfType()); + Assert.Equal(new[] { deliveryInputId }, closure.InputIds); + Assert.Equal("original", closure.TaskId); + Assert.Single(closedPrefix.OfType()); + Assert.Single(closedPrefix.OfType()); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = Session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var recoveredOwner = await OwnerAsync(); + Assert.NotEqual(owner, recoveredOwner); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + _main.NextResponseGate = freshRelease; + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Inspect the recovered record.", Source = DeliverySource("fresh", "operator-b") }, + Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Equal(3, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var request = _main.ReceivedMessages[^1]; + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var calls = request.SelectMany(message => message.Contents.OfType()).ToArray(); + var results = request.SelectMany(message => message.Contents.OfType()).ToArray(); + var call = Assert.Single(calls, candidate => candidate.CallId == id); + Assert.Equal(ChildRunDelivery.ToolName(accepted.Run), call.Name); + Assert.Equal(delivery.Input.UserMessage.Content, + Assert.IsType(Assert.Single(results, candidate => candidate.CallId == id).Result)); + Assert.Equal(calls.Select(candidate => candidate.CallId).Order(), results.Select(candidate => candidate.CallId).Order()); + freshRelease.TrySetResult(); + await CompletedAsync(subscriber); + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + var childAdoption = Assert.Single(final.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, childAdoption.TurnContext)); + Assert.DoesNotContain(final.OfType(), evt => evt.ToolResult.ToolCallId == delivery.Input.UserMessage.ToolCallId); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + Assert.Equal(0, _secondChild.CallCount); + } + finally + { + closeRelease.TrySetResult(); + freshRelease.TrySetResult(); + } + } + + private static MessageSource DeliverySource(string turn, string requester) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId(requester), TurnId = new TurnId(turn), MessageId = turn, + Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted), + DefaultDeliveryTarget = new ChannelDeliveryTargetInfo("signalr", "destination", requester, requester) + }; +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.DeliveryRecovery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.DeliveryRecovery.cs new file mode 100644 index 000000000..1bf050788 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.DeliveryRecovery.cs @@ -0,0 +1,132 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_failed_coalesced_review_restores_every_durable_child_pair_after_cold_recovery() + { + _main.ToolCallsOnFirstCall!.Add(new FunctionCallContent("start-2", "start_probe", + new Dictionary { ["_rationale"] = "Inspect the second part." })); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate both parts.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await _secondChild.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var accepted = (await ReadJournalAsync()).OfType().ToArray(); + Assert.Equal(2, accepted.Length); + var parentRelease = NewSignal(); + var closeEntered = NewSignal(); + var closeRelease = NewSignal(); + var freshRelease = NewSignal(); + var failure = new InvalidOperationException("The actual coalesced review provider failed."); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not InputClosed closed || closed.InputIds.Count != 2) return false; + closeEntered.TrySetResult(); + await closeRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + _main.NextResponseGate = parentRelease; + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Inspect a separate task.", Source = DeliverySource("intervening", "operator-b") }, + Ceiling, TestContext.Current.CancellationToken); + try + { + await AwaitAssertAsync(() => Assert.Equal(3, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _childRelease.TrySetResult(); + _secondChildRelease.TrySetResult(); + ChildRunEvent.DeliveryAdmitted[] deliveries = []; + await AwaitAssertAsync(async () => + { + deliveries = (await ReadJournalAsync()).OfType().ToArray(); + Assert.Equal(2, deliveries.Length); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var ids = deliveries.Select(delivery => delivery.Input.InputId).ToArray(); + Assert.Equal(2, ids.Distinct().Count()); + _main.PlannedExceptions.Enqueue(failure); + await EventFilter.Error(contains: "turn_failed").ExpectOneAsync(async () => + { + parentRelease.TrySetResult(); + await closeEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var prefix = await ReadJournalAsync(); + var adoption = Assert.Single(prefix.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(ids, adoption.InputIds); + Assert.True(SessionState.SameCanonicalContext(accepted[0].Run.OriginalContext, adoption.TurnContext)); + Assert.Empty(prefix.OfType()); + closeRelease.TrySetResult(); + var error = await subscriber.FishForMessageAsync(message => message is ErrorOutput, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Same(failure, Assert.IsType(error).Cause); + Assert.Equal(TurnOutcome.Failed, Assert.IsType(await CompletedAsync(subscriber)).Outcome); + }, cancellationToken: TestContext.Current.CancellationToken); + var beforeStop = await ReadJournalAsync(); + Assert.Equal(ids, Assert.Single(beforeStop.OfType()).InputIds); + Assert.Equal(2, beforeStop.OfType().Count()); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, _secondChild.CallCount); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = Session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(owner, await OwnerAsync()); + Assert.Equal(3, _main.CallCount); + _main.NextResponseGate = freshRelease; + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Inspect the recovered pairs.", Source = DeliverySource("fresh", "operator-b") }, + Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Equal(4, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var request = _main.ReceivedMessages[^1]; + var calls = request.SelectMany(message => message.Contents.OfType()).ToArray(); + var results = request.SelectMany(message => message.Contents.OfType()).ToArray(); + foreach (var delivery in deliveries) + { + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var run = Assert.Single(accepted, admission => admission.Run.RunId == delivery.RunId).Run; + Assert.Equal(ChildRunDelivery.ToolName(run), Assert.Single(calls, call => call.CallId == id).Name); + Assert.Equal(delivery.Input.UserMessage.Content, + Assert.IsType(Assert.Single(results, result => result.CallId == id).Result)); + } + Assert.Equal(calls.Select(call => call.CallId).Order(), results.Select(result => result.CallId).Order()); + freshRelease.TrySetResult(); + await CompletedAsync(subscriber); + var final = await ReadJournalAsync(); + Assert.Equal(2, final.OfType().Count()); + Assert.Equal(2, final.OfType().Count()); + Assert.Equal(2, final.OfType().Count()); + Assert.Equal(2, final.OfType().Count()); + Assert.Equal(ids, final.OfType().Select(delivery => delivery.Input.InputId)); + Assert.Single(final.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(4, _main.CallCount); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, _secondChild.CallCount); + } + finally + { + parentRelease.TrySetResult(); + closeRelease.TrySetResult(); + freshRelease.TrySetResult(); + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Obligations.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Obligations.cs new file mode 100644 index 000000000..5600f0aeb --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Obligations.cs @@ -0,0 +1,371 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_child_result_waits_for_actual_compaction_and_restores_its_original_authority() + { + _main.ToolCallsOnFirstCall = null; + var (owner, manager, subscriber) = await CreateObligationOwnerAsync(); + for (var seed = 0; seed < 3; seed++) + { + await SendObligationInputAsync(manager, $"seed-{seed}", "operator-a"); + await CompletedObligationTurnAsync(subscriber); + } + _main.PlannedResponses.Enqueue([new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the original task." })]); + await SendObligationInputAsync(manager, "original", "operator-a"); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal(5, _main.CallCount); + var observerRelease = NewSignal(); + var reviewRelease = NewSignal(); + var snapshotRelease = NewSignal(); + Assert.Equal(1, _child.CallCount); + Assert.Single(_child.ReceivedMessages); + Assert.False(_childRelease.Task.IsCompleted); + await AwaitAssertAsync(() => Assert.Null(_child.NextResponseGate), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _child.NextResponseGate = observerRelease; + _child.ObservationResponseOverride = "The prior neutral turns completed. The original child remains independent."; + static bool IsSummarizer(IReadOnlyList request) + => request.Any(message => message.Role == Microsoft.Extensions.AI.ChatRole.System + && message.Text?.Contains("You are a session summarizer", StringComparison.Ordinal) == true); + await Snapshots.OnSave.FailIf(async (persistenceId, _) => + { + if (persistenceId == $"session-{Session.Value}") + await snapshotRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + try + { + _main.PlannedUsageOverrides.Enqueue(new UsageDetails { InputTokenCount = 100000, OutputTokenCount = 1 }); + await SendObligationInputAsync(manager, "foreign-before-compaction", "operator-b"); + await CompletedAsync(subscriber); + await AwaitAssertAsync(() => Assert.Equal(2, _child.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Null(_child.NextResponseGate), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Single(_child.ReceivedMessages, IsSummarizer); + Assert.Single(_child.ReceivedMessages, request => !IsSummarizer(request)); + Assert.False(observerRelease.Task.IsCompleted); + Assert.False(_childRelease.Task.IsCompleted); + _main.NextResponseGate = reviewRelease; + _childRelease.TrySetResult(); + await AwaitAssertAsync(async () => Assert.Single((await ReadJournalAsync()).OfType()), + Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var held = await ReadJournalAsync(); + Assert.Single(held.OfType()); + Assert.Single(held.OfType()); + Assert.Empty(held.OfType()); + Assert.DoesNotContain(held.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(6, _main.CallCount); + observerRelease.TrySetResult(); + var notice = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is CompactionOutput, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + Assert.True(notice.Summarized); + Assert.True(notice.MessagesAfter < notice.MessagesBefore); + await AwaitAssertAsync(() => Assert.Equal(7, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var after = await ReadJournalPositionsAsync(); + var compactedRow = Assert.Single(after, row => row.Event is SessionCompacted); + var compacted = Assert.IsType(compactedRow.Event); + Assert.Equal(_child.ObservationResponseOverride, compacted.Summary); + var adoptionRow = Assert.Single(after, row => row.Event is ToolTaskAdopted { ContinuedChildRunId: not null }); + Assert.True(compactedRow.SequenceNr < adoptionRow.SequenceNr); + var adoption = Assert.IsType(adoptionRow.Event); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, adoption.TurnContext)); + var delivery = Assert.Single(after.Select(row => row.Event).OfType()); + Assert.Equal(new[] { delivery.Input.InputId }, adoption.InputIds); + AssertObligationProviderPair(delivery, _main.ReceivedMessages[^1]); + Assert.Equal(owner, await OwnerAsync()); + Assert.Equal(2, _child.CallCount); + Assert.Single(_child.ReceivedMessages, IsSummarizer); + Assert.Single(_child.ReceivedMessages, request => !IsSummarizer(request)); + reviewRelease.TrySetResult(); + await CompletedAsync(subscriber); + Assert.Equal(7, _main.CallCount); + } + finally + { + observerRelease.TrySetResult(); + reviewRelease.TrySetResult(); + snapshotRelease.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task The_actual_first_durable_completion_or_cancel_keeps_one_terminal_winner(bool cancelFirst) + { + var gate = InstallTerminalCutMailbox(nameof(BackgroundChildTerminal), child: false); + var (owner, manager, subscriber) = await CreateObligationOwnerAsync(); + var controlRelease = NewSignal(); + try + { + await SendObligationInputAsync(manager, "original", "operator-a"); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + _childRelease.TrySetResult(); + var captured = await gate.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var completion = Assert.IsType(captured.Envelope.Message); + Assert.Equal(accepted.Run.RunId, completion.RunId); + Assert.True(completion.Result.Success); + if (!cancelFirst) + { + gate.Release(); + await CompletedObligationTurnAsync(subscriber); + var committed = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.True(committed.Terminal.Result.Success); + Assert.Empty((await ReadJournalAsync()).OfType()); + } + _start!.ControlCompletionGate = controlRelease; + _main.PlannedResponses.Enqueue([ControlCall("race-cancel", accepted.Run.RunId)]); + await SendObligationInputAsync(manager, "race-cancel", "operator-a"); + var reply = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(reply.Run); + Assert.Equal(accepted.Run.RunId, reply.Run.RunId); + Assert.Equal(cancelFirst ? BackgroundChildState.Cancelling : BackgroundChildState.Completed, reply.Run.State); + if (cancelFirst) + { + Assert.Single((await ReadJournalAsync()).OfType()); + gate.Release(); + } + await AssertTerminalCutOwnerBarrierAsync(owner, "after-competing-terminal"); + await AwaitAssertAsync(async () => Assert.Single((await ReadJournalAsync()).OfType()), + Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var rows = await ReadJournalPositionsAsync(); + var terminalRow = Assert.Single(rows, row => row.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + Assert.Equal(terminalRow.SequenceNr, terminal.TerminalSequenceNr); + Assert.Equal(!cancelFirst, terminal.Terminal.Result.Success); + if (cancelFirst) + { + var cancellationRow = Assert.Single(rows, row => row.Event is ChildRunEvent.CancellationRequested); + Assert.True(cancellationRow.SequenceNr < terminalRow.SequenceNr); + Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(SubAgentRunOutcome.Failed, terminal.Terminal.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var storage = Assert.IsType(_start.Prepared!.Execution.Scope.Authority.Session).Storage; + using var report = JsonDocument.Parse(await File.ReadAllTextAsync( + Path.Combine(storage.ArtifactDirectory.Value, "cancelled-results.json"), TestContext.Current.CancellationToken)); + Assert.Equal(accepted.Run.RunId.Value, report.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Cancelled", report.RootElement.GetProperty("state").GetString()); + Assert.Equal("Recorded receipts describe known local results. They do not prove external effects stopped.", + report.RootElement.GetProperty("external_effects").GetString()); + } + else + { + Assert.DoesNotContain(rows, row => row.Event is ChildRunEvent.CancellationRequested or ChildRunEvent.DispatchClosed); + Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(completion.Result.Output, terminal.Terminal.Result.Output); + } + controlRelease.TrySetResult(); + await CompletedObligationTurnAsync(subscriber); + await AwaitAssertAsync(async () => + { + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + var delivery = Assert.Single(final.OfType()); + Assert.Single(final.OfType(), turn => turn.ConsumedInputIds.Contains(delivery.Input.InputId)); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, _child.CallCount); + } + finally { gate.Release(); controlRelease.TrySetResult(); } + } + + [Fact] + public async Task Child_idle_obligations_defer_owner_loss_then_allow_ordinary_passivation() + { + var (owner, manager, subscriber) = await CreateObligationOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + var enrichmentRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reviewRelease = NewSignal(); + var control = CreateTestProbe(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + owner.Tell(new LeaveSession(subscriber) { SessionId = Session }, control.Ref); + _time.Advance(TimeSpan.FromMinutes(31)); + owner.Tell(ReceiveTimeout.Instance, control.Ref); + owner.Tell(new Identify("live-child-idle-obligation"), control.Ref); + Assert.Equal(owner, (await control.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken)).Subject); + Assert.Equal(owner, await OwnerAsync()); + Assert.Null(await ReadDrainedChildSnapshotAsync()); + Assert.Equal(1, _child.CallCount); + Assert.Equal(2, _main.CallCount); + + _snapshots.InvocationStarted = NewSignal(); + _snapshots.Pending = enrichmentRelease; + _childRelease.TrySetResult(); + await _snapshots.InvocationStarted.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var terminalOnly = await ReadJournalAsync(); + Assert.Single(terminalOnly.OfType()); + Assert.Empty(terminalOnly.OfType()); + Assert.Empty(terminalOnly.OfType()); + _time.Advance(TimeSpan.FromMinutes(31)); + owner.Tell(ReceiveTimeout.Instance, control.Ref); + owner.Tell(new Identify("undelivered-child-idle-obligation"), control.Ref); + Assert.Equal(owner, (await control.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken)).Subject); + Assert.Null(await ReadDrainedChildSnapshotAsync()); + _main.NextResponseGate = reviewRelease; + enrichmentRelease.TrySetResult(new WorkingContextSnapshot + { WorkingContext = WorkingContext.Empty, Git = new GitWorkingContextInspection.Skipped() }); + await AwaitAssertAsync(() => Assert.Equal(3, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(owner, await OwnerAsync()); + var delivery = Assert.Single((await ReadJournalAsync()).OfType()); + AssertObligationProviderPair(delivery, _main.ReceivedMessages[^1]); + var reviewSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(reviewSubscriber) + { SessionId = Session, Filter = OutputFilter.Full | OutputFilter.ProcessingState }, reviewSubscriber.Ref); + await reviewSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + reviewRelease.TrySetResult(); + await CompletedObligationTurnAsync(reviewSubscriber); + var settled = await ReadJournalAsync(); + Assert.Single(settled.OfType(), turn => turn.ConsumedInputIds.Contains(delivery.Input.InputId)); + owner.Tell(new LeaveSession(reviewSubscriber) { SessionId = Session }, control.Ref); + _time.Advance(TimeSpan.FromMinutes(31)); + owner.Tell(ReceiveTimeout.Instance, control.Ref); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + SessionSnapshot? snapshot = null; + await AwaitAssertAsync(async () => + { + snapshot = await ReadDrainedChildSnapshotAsync(); + Assert.NotNull(snapshot); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var stored = Assert.Single(snapshot!.ChildRuns); + Assert.Equal(accepted.Run.RunId, stored.RunId); + Assert.Equal(delivery.Input.InputId, stored.DeliveryInputId); + Assert.True(stored.Terminal!.Result.Success); + Assert.Empty(snapshot.PendingInputs); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + finally + { + _snapshots.Pending = null; + enrichmentRelease.TrySetResult(new WorkingContextSnapshot + { WorkingContext = WorkingContext.Empty, Git = new GitWorkingContextInspection.Skipped() }); + reviewRelease.TrySetResult(); + } + } + + [Fact] + public async Task Two_held_child_providers_do_not_block_parent_status_or_cancel_the_other_run() + { + _main.ToolCallsOnFirstCall!.Add(new FunctionCallContent("start-2", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the second independent part." })); + var (_, manager, subscriber) = await CreateObligationOwnerAsync(); + await SendObligationInputAsync(manager, "original", "operator-a"); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await _secondChild.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var accepted = (await ReadJournalAsync()).OfType().ToArray(); + Assert.Equal(2, accepted.Length); + var first = Assert.Single(accepted, admission => Assert.IsType(admission.Run.StartKey).CallId.Value == "start-1"); + var second = Assert.Single(accepted, admission => Assert.IsType(admission.Run.StartKey).CallId.Value == "start-2"); + await CheckAsync(first.Run.RunId, "held-first-status", false, "Running"); + Assert.False(_childRelease.Task.IsCompleted); + Assert.False(_secondChildRelease.Task.IsCompleted); + await CheckAsync(first.Run.RunId, "held-first-cancel", true, "Cancelling"); + await AwaitAssertAsync(async () => + { + var records = await ReadJournalAsync(); + var terminal = Assert.Single(records.OfType()); + Assert.Equal(first.Run.RunId, terminal.RunId); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var delivery = Assert.Single(records.OfType()); + Assert.Single(records.OfType(), turn => turn.ConsumedInputIds.Contains(delivery.Input.InputId)); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await AssertTerminalCutOwnerBarrierAsync(await OwnerAsync(), "after-first-child-settlement"); + await CheckAsync(second.Run.RunId, "held-second-status", false, "Running"); + var final = await ReadJournalAsync(); + Assert.Equal(first.Run.RunId, Assert.Single(final.OfType()).RunId); + Assert.Equal(first.Run.RunId, Assert.Single(final.OfType()).RunId); + Assert.Single(final.OfType()); + Assert.Equal(2, final.OfType().Count()); + Assert.Equal(2, final.OfType().Count()); + Assert.False(_secondChildRelease.Task.IsCompleted); + Assert.Equal(1, _child.CallCount); + Assert.Equal(1, _secondChild.CallCount); + + async Task CheckAsync(SubAgentRunId runId, string id, bool cancel, string expectedState) + { + _main.PlannedResponses.Enqueue([new FunctionCallContent(id, "status_json_probe", + new Dictionary { ["run_id"] = runId.Value, ["cancel"] = cancel, + ["_rationale"] = "Inspect the specific accepted child." })]); + await SendObligationInputAsync(manager, id, "operator-a"); + var output = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolResultOutput result && result.CallId.Value == id, + Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + using var body = JsonDocument.Parse(output.Result); + Assert.Equal(runId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(expectedState, body.RootElement.GetProperty("state").GetString()); + await subscriber.FishForMessageAsync(message => message is ProcessingStateOutput { IsProcessing: false }, + Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var result = Assert.Single((await ReadJournalAsync()).OfType(), + record => record.ToolResult.ToolCallId == new ToolCallId(id)); + Assert.Equal(output.Result, result.ToolResult.Content); + } + } + + private async Task<(IActorRef Owner, IActorRef Manager, Akka.TestKit.TestProbe Subscriber)> CreateObligationOwnerAsync() + { + var created = await CreateOwnerAsync(); + created.Manager.Tell(new JoinSession(created.Subscriber) + { SessionId = Session, Filter = OutputFilter.Full | OutputFilter.ProcessingState }, created.Subscriber.Ref); + await created.Subscriber.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + return created; + } + + private static async Task CompletedObligationTurnAsync(Akka.TestKit.TestProbe subscriber) + { + await CompletedAsync(subscriber); + await subscriber.FishForMessageAsync(message => message is ProcessingStateOutput { IsProcessing: false }, + Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + + private static Task SendObligationInputAsync(IActorRef manager, string turn, string requester) + => manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Complete the neutral request.", Source = DeliverySource(turn, requester) + }, Ceiling, TestContext.Current.CancellationToken); + + private static void AssertObligationProviderPair(ChildRunEvent.DeliveryAdmitted delivery, IReadOnlyList request) + { + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var call = Assert.Single(request.SelectMany(message => message.Contents.OfType()), item => item.CallId == id); + Assert.Equal(delivery.Input.UserMessage.Name, call.Name); + Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single( + request.SelectMany(message => message.Contents.OfType()), item => item.CallId == id).Result)); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.RecoveryDrain.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.RecoveryDrain.cs new file mode 100644 index 000000000..13d9a701e --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.RecoveryDrain.cs @@ -0,0 +1,218 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Persistence; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Cold_recovery_preserves_committed_cancellation_before_its_terminal(bool closureCommitted) + { + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Delegate the neutral task.", + Source = DeliverySource("original", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + var selected = closureCommitted + ? record.Payload is ChildRunEvent.TerminalRecorded + : record.Payload is ChildRunEvent.DispatchClosed; + if (!selected) return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + _main.PlannedResponses.Enqueue([ControlCall("cancel-before-recovery", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Cancel the accepted child.", + Source = DeliverySource("cancel-before-recovery", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + var control = await _start!.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(control.Run); + Assert.Equal(accepted.Run.RunId, control.Run.RunId); + Assert.Equal(BackgroundChildState.Cancelling, control.Run.State); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, control.Run.OriginalContext)); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + Assert.Equal(accepted.Run.RunId, Assert.Single(before.OfType()).RunId); + Assert.Equal(closureCommitted ? 1 : 0, before.OfType().Count()); + Assert.Empty(before.OfType()); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + release.TrySetResult(); + await Journal.OnWrite.Pass(); + Assert.True(_childRelease.Task.IsCanceled); + + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) + { SessionId = Session, Filter = OutputFilter.Full }, recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var recovered = await OwnerAsync(); + Assert.NotEqual(owner, recovered); + var rows = await ReadJournalPositionsAsync(); + var events = rows.Select(row => row.Event).ToArray(); + var terminalRow = Assert.Single(rows, row => row.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + Assert.Equal(terminalRow.SequenceNr, terminal.TerminalSequenceNr); + Assert.Equal(accepted.Run.RunId, terminal.RunId); + Assert.False(terminal.Terminal.Lost); + Assert.IsType(terminal.Terminal.Result.Completion); + Assert.False(terminal.Terminal.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, terminal.Terminal.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + var cancellationRow = Assert.Single(rows, row => row.Event is ChildRunEvent.CancellationRequested); + var closureRow = Assert.Single(rows, row => row.Event is ChildRunEvent.DispatchClosed); + Assert.True(cancellationRow.SequenceNr < closureRow.SequenceNr); + Assert.True(closureRow.SequenceNr < terminalRow.SequenceNr); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + var retry = Assert.IsType(await recovered.Ask(Retry(accepted), + Ceiling, TestContext.Current.CancellationToken)); + Assert.Equal(BackgroundChildState.Cancelled, retry.State); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, retry.Run.OriginalContext)); + await AwaitAssertAsync(() => + { + var delivered = _main.ReceivedMessages.SelectMany(messages => messages) + .SelectMany(message => message.Contents.OfType()) + .Where(result => result.CallId.StartsWith("child-result-", StringComparison.Ordinal)).ToArray(); + Assert.NotEmpty(delivered); + Assert.All(delivered, result => + { + using var body = JsonDocument.Parse(Assert.IsType(result.Result)); + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Cancelled", body.RootElement.GetProperty("state").GetString()); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent.Value, body.RootElement.GetProperty("reason").GetString()); + }); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, _child.CallCount); + Assert.Single((await ReadJournalAsync()).OfType()); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task Daemon_restart_drain_cancels_a_live_child_before_acknowledgement_and_retains_its_terminal_snapshot() + { + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + var child = await Sys.ActorSelection($"{owner.Path}/child-run-{accepted.Run.RunId.Value}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + var ownerWatcher = CreateTestProbe(); + var childWatcher = CreateTestProbe(); + ownerWatcher.Watch(owner); + childWatcher.Watch(child); + var entered = NewSignal(); + var release = NewSignal(); + long terminalSequence = 0; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunEvent.TerminalRecorded) return false; + Interlocked.Exchange(ref terminalSequence, record.SequenceNr); + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + try + { + var drain = manager.Ask(new PrepareForDaemonRestart(Session, "daemon-stop"), + Ceiling, TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + Assert.Equal(accepted.Run.RunId, Assert.Single(before.OfType()).RunId); + Assert.Equal(accepted.Run.RunId, Assert.Single(before.OfType()).RunId); + Assert.Empty(before.OfType()); + Assert.True(_childRelease.Task.IsCanceled); + Assert.False(drain.IsCompleted); + release.TrySetResult(); + var acknowledgement = await drain; + Assert.Equal(Session, acknowledgement.SessionId); + Assert.Null(acknowledgement.RestartReminder); + await ownerWatcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await childWatcher.ExpectTerminatedAsync(child, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + SessionSnapshot? snapshot = null; + await AwaitAssertAsync(async () => + { + snapshot = await ReadDrainedChildSnapshotAsync(); + Assert.NotNull(snapshot); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var stored = Assert.Single(snapshot!.ChildRuns); + Assert.Equal(accepted.Run.RunId, stored.RunId); + Assert.Equal(BackgroundChildState.Cancelled, stored.State); + Assert.NotNull(stored.CancellationRequestedAtMs); + Assert.NotNull(stored.DispatchClosedAtMs); + Assert.Equal(Volatile.Read(ref terminalSequence), stored.TerminalSequenceNr); + Assert.False(stored.Terminal!.Lost); + Assert.False(stored.Terminal.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, stored.Terminal.Result.Outcome); + Assert.IsType(stored.Terminal.Result.Completion); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, stored.Terminal.Result.OutcomeReason); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, stored.OriginalContext)); + Assert.Equal(1, _child.CallCount); + } + finally { release.TrySetResult(); } + } + + private async Task ReadDrainedChildSnapshotAsync() + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reader = Sys.ActorOf(Props.Create(() => new DrainedChildSnapshotReader($"session-{Session.Value}", done))); + var watcher = CreateTestProbe(); + watcher.Watch(reader); + try { return await done.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); } + finally + { + Sys.Stop(reader); + await watcher.ExpectTerminatedAsync(reader, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + } + + private sealed class DrainedChildSnapshotReader : ReceivePersistentActor + { + public DrainedChildSnapshotReader(string persistenceId, TaskCompletionSource done) + { + PersistenceId = persistenceId; + SessionSnapshot? snapshot = null; + Recover(offer => + { + snapshot = (SessionSnapshot)offer.Snapshot; + _ = SessionState.FromSnapshot(snapshot); + }); + // This probe returns the stored snapshot. Later journal events do not alter that artifact. + Recover(_ => { }); + Recover(_ => done.TrySetResult(snapshot)); + } + public override string PersistenceId { get; } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.SpawnFailure.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.SpawnFailure.cs new file mode 100644 index 000000000..fb1b66720 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.SpawnFailure.cs @@ -0,0 +1,60 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_synchronous_child_creation_failure_retains_one_acceptance_and_one_spawn_error() + { + // Akka resolves the named mailbox inside ActorOf, before it returns a child reference. + _start!.ChildMailbox = "unconfigured-child-owner-mailbox"; + Assert.False(Sys.Settings.Config.HasPath(_start.ChildMailbox)); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + ChildRunAccepted? accepted = null; + await EventFilter.Error(contains: "Child creation failed after acceptance").ExpectOneAsync(async () => + { + await SendOriginalAsync(manager); + await AwaitAssertAsync(async () => + { + var records = await ReadJournalAsync(); + accepted = Assert.Single(records.OfType()); + Assert.Empty(records.OfType()); + var terminal = Assert.Single(records.OfType()); + Assert.Equal(accepted.Run.RunId, terminal.RunId); + Assert.False(terminal.Terminal.Result.Success); + Assert.False(terminal.Terminal.Lost); + Assert.Equal(SubAgentOutcomeReason.SpawnError, terminal.Terminal.Result.OutcomeReason); + Assert.IsType(terminal.Terminal.Result.Completion); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotNull(accepted); + await CompletedAsync(subscriber); + Assert.Equal(0, _child.CallCount); + var repeated = Assert.IsType(await owner.Ask(Retry(accepted), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(accepted.Run.RunId, repeated.Run.RunId); + Assert.Equal(accepted.Run.StartKey, repeated.Run.StartKey); + Assert.Equal(accepted.Run.ArgumentsDigest, repeated.Run.ArgumentsDigest); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, repeated.Run.OriginalContext)); + Assert.Equal(BackgroundChildState.Failed, repeated.Run.State); + Assert.Equal(SubAgentOutcomeReason.SpawnError, repeated.Run.Terminal!.Result.OutcomeReason); + var after = await ReadJournalPositionsAsync(); + Assert.Single(after, record => record.Event is ChildRunAccepted); + Assert.DoesNotContain(after, record => record.Event is ChildRunEvent.Started); + var terminalRow = Assert.Single(after, record => record.Event is ChildRunEvent.TerminalRecorded); + var terminalEvent = Assert.IsType(terminalRow.Event); + Assert.Equal(terminalRow.SequenceNr, terminalEvent.TerminalSequenceNr); + Assert.Equal(SubAgentOutcomeReason.SpawnError, terminalEvent.Terminal.Result.OutcomeReason); + Assert.Equal(0, _child.CallCount); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartAdapters.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartAdapters.cs new file mode 100644 index 000000000..4f1fa2cb0 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartAdapters.cs @@ -0,0 +1,253 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Skills; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security.Skills; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using ModelChatRole = Microsoft.Extensions.AI.ChatRole; +using SkillScanResult = Netclaw.Security.Skills.SkillScanResult; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData("scope")] + [InlineData("digest")] + [InlineData("agent")] + [InlineData("owner")] + [InlineData("state")] + public async Task Spawn_agent_rejects_malformed_typed_owner_acceptance_before_a_success_receipt(string defect) + { + var tool = ConfigureCanonicalSpawnAdapter(); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var admission = Assert.Single((await ReadJournalAsync()).OfType()); + var original = Assert.IsType(await owner.Ask(Retry(admission), + Ceiling, TestContext.Current.CancellationToken)); + Assert.Equal(BackgroundChildState.Running, original.State); + var run = original.Run; + var state = original.State; + switch (defect) + { + case "scope": run = run with { ScopeId = new SubAgentScopeId("signalr/foreign/subagent/worker/other") }; break; + case "digest": run = run with { ArgumentsDigest = new string('F', 64) }; break; + case "agent": run = run with { AgentName = new AgentName("other-worker"), + ScopeId = new SubAgentScopeId($"{Session.Value}/subagent/other-worker/{run.RunId.Value}") }; break; + case "owner": + var foreign = new SessionId("signalr/foreign-owner"); + run = run with { StartKey = run.StartKey with { SessionId = foreign }, + OriginalContext = run.OriginalContext with { SessionId = foreign }, + ScopeId = new SubAgentScopeId($"{foreign.Value}/subagent/worker/{run.RunId.Value}") }; + break; + case "state": state = BackgroundChildState.Completed; break; + default: throw new ArgumentOutOfRangeException(nameof(defect)); + } + if (defect != "scope") run.Validate(); + var outputs = new ToolExecutionOutputs(); + var calls = 0; + var scope = _start!.StartInvocation!.RunScope with { SpawnChildActor = (payload, _, _) => + { + var prepared = Assert.IsType(payload); + Assert.Equal(admission.Run.ArgumentsDigest, prepared.ArgumentsDigest); + Interlocked.Increment(ref calls); + return Task.FromResult(new ChildStartReply.Accepted(run, state)); + }}; + await Assert.ThrowsAsync(() => tool.ExecuteAsync(SpawnArguments(), + new ToolInvocationContext(scope, _start.StartInvocation.ExecutionTimeout, outputs), TestContext.Current.CancellationToken)); + Assert.Equal(1, Volatile.Read(ref calls)); + Assert.Null(outputs.Receipt); + Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + } + + [Fact] + public async Task Spawn_agent_accepts_a_valid_running_retry_with_the_original_run_instead_of_the_new_proposal() + { + var tool = ConfigureCanonicalSpawnAdapter(); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var admission = Assert.Single((await ReadJournalAsync()).OfType()); + var retry = Assert.IsType(await owner.Ask(Retry(admission), + Ceiling, TestContext.Current.CancellationToken)); + Assert.Equal(BackgroundChildState.Running, retry.State); + var outputs = new ToolExecutionOutputs(); + var scope = _start!.StartInvocation!.RunScope with { SpawnChildActor = (payload, _, _) => + { + var proposed = Assert.IsType(payload); + Assert.NotEqual(retry.Run.RunId, proposed.RunId); + Assert.Equal(retry.Run.ArgumentsDigest, proposed.ArgumentsDigest); + return Task.FromResult(retry); + }}; + var result = await tool.ExecuteAsync(SpawnArguments(), + new ToolInvocationContext(scope, _start.StartInvocation.ExecutionTimeout, outputs), TestContext.Current.CancellationToken); + using var body = JsonDocument.Parse(result); + Assert.Equal(retry.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(retry.Run.ScopeId.Value, body.RootElement.GetProperty("scope_id").GetString()); + Assert.Equal("Running", body.RootElement.GetProperty("state").GetString()); + Assert.Equal("check_agent_run", body.RootElement.GetProperty("control_tool").GetString()); + Assert.IsType(outputs.Receipt); + Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + } + + [Fact] + public async Task Spawn_agent_cannot_return_acceptance_from_a_failed_owner_commit() + { + ConfigureCanonicalSpawnAdapter(); + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunAccepted) return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var (owner, manager, _) = await CreateOwnerAsync(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + await EventFilter.Error(contains: "Failed to persist event type").ExpectOneAsync(async () => + { + await SendOriginalAsync(manager); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(0, _child.CallCount); + Assert.Null(_start!.StartInvocation!.Outputs.Receipt); + release.TrySetResult(); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + var events = await ReadJournalAsync(); + Assert.Single(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Null(_start!.StartInvocation!.Outputs.Receipt); + Assert.Equal(0, _child.CallCount); + Assert.Equal(1, _main.CallCount); + } + finally { release.TrySetResult(); } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Routed_skill_load_scans_before_durable_acceptance_and_preserves_the_additive_overlay(bool reject) + { + const string content = "---\nname: route-probe\ndescription: Neutral route.\nmetadata:\n subagent: worker\n---\n\nInspect the neutral route marker.\n"; + var file = Path.Combine(_directory!.Paths.SkillsDirectory, "route-probe", "SKILL.md"); + Directory.CreateDirectory(Path.GetDirectoryName(file)!); + await File.WriteAllTextAsync(file, content, TestContext.Current.CancellationToken); + var skills = new SkillRegistry(); + var scanned = SkillScanner.Scan(_directory.Paths.SkillsDirectory); + Assert.Empty(scanned.Issues); + skills.ReplaceAll(scanned.AcceptedSkills); + var scanner = new AdapterContentScanner(reject); + _start!.StartTool = new SkillLoadTool(skills, scanner, new AdapterPromptLoader(), + subAgentRegistry: AdapterProfiles(), subAgentSpawner: _start.Spawner); + _main.ToolCallsOnFirstCall = [new FunctionCallContent("start-1", "skill_load", new Dictionary + { ["Name"] = "route-probe", ["Task"] = "Inspect the neutral fixture.", ["Context"] = "route-context-marker", + ["_rationale"] = "Use the neutral routed skill." })]; + var (_, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + var output = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolResultOutput, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + await CompletedAsync(subscriber); + var scan = Assert.Single(scanner.Calls); + Assert.Equal("route-probe", scan.Name); + Assert.Equal(content, scan.Content); + var events = await ReadJournalAsync(); + var recorded = Assert.Single(events.OfType()); + Assert.Equal(output.Result, recorded.ToolResult.Content); + var paired = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType())); + Assert.Equal("start-1", paired.CallId); + Assert.Equal(output.Result, Assert.IsType(paired.Result)); + if (reject) + { + Assert.Equal("Skill 'route-probe' blocked by content scan: neutral rejection", output.Result); + Assert.Empty(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Equal(0, _child.CallCount); + return; + } + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var admission = Assert.Single(events.OfType()); + admission.Run.Validate(); + Assert.Equal("skill_load", admission.Run.SourceOperation); + Assert.Equal(new ToolCallId("start-1"), Assert.IsType(admission.Run.StartKey).CallId); + using var body = JsonDocument.Parse(output.Result); + Assert.Equal(4, body.RootElement.EnumerateObject().Count()); + Assert.Equal(admission.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(admission.Run.ScopeId.Value, body.RootElement.GetProperty("scope_id").GetString()); + Assert.Equal("Accepted", body.RootElement.GetProperty("state").GetString()); + Assert.Equal("check_agent_run", body.RootElement.GetProperty("control_tool").GetString()); + var system = string.Join("\n", _child.ReceivedMessages[0].Where(message => message.Role == ModelChatRole.System).Select(message => message.Text)); + Assert.Contains("Complete the assigned task.", system, StringComparison.Ordinal); + Assert.Contains("[Skill Overlay]\nInspect the neutral route marker.", system, StringComparison.Ordinal); + Assert.Contains(_child.ReceivedMessages[0], message => message.Role == ModelChatRole.User + && message.Text.Contains("route-context-marker", StringComparison.Ordinal)); + Assert.Empty(events.OfType()); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + } + + private SpawnAgentTool ConfigureCanonicalSpawnAdapter() + { + var tool = new SpawnAgentTool(AdapterProfiles(), _start!.Spawner, _directory!.Paths); + _start.StartTool = tool; + _main.ToolCallsOnFirstCall = [new FunctionCallContent("start-1", "spawn_agent", SpawnArguments())]; + return tool; + } + + private static Dictionary SpawnArguments() => new() + { ["Agent"] = "worker", ["Task"] = "Inspect the neutral fixture.", ["_rationale"] = "Delegate the neutral task." }; + + private static SubAgentDefinitionRegistry AdapterProfiles() + { + var profiles = new SubAgentDefinitionRegistry(); + profiles.Register(new SubAgentProfile + { + Name = "worker", Description = "Complete the neutral task.", SystemPrompt = "Complete the assigned task.", + ModelRole = ModelRole.Compaction, ToolNames = ["file_read"], EmitStructuredFindings = false, + Visibility = SubAgentVisibility.UserFacing + }); + return profiles; + } + + private sealed class AdapterContentScanner(bool reject) : ISkillContentScanner + { + public List<(string Name, string Content)> Calls { get; } = []; + public Task ScanAsync(string skillName, string content, CancellationToken cancellationToken = default) + { + Calls.Add((skillName, content)); + return Task.FromResult(reject ? SkillScanResult.Reject("neutral rejection") : SkillScanResult.Allow()); + } + } + + private sealed class AdapterPromptLoader : IMcpPromptSkillLoader + { + public ValueTask LoadAsync(McpPromptSkillSource source, + IReadOnlyDictionary? arguments, ToolInvocationContext context, CancellationToken cancellationToken) + => throw new InvalidOperationException("The file route must not request an MCP prompt."); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs new file mode 100644 index 000000000..3f18c69f6 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs @@ -0,0 +1,322 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Runtime.CompilerServices; +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task Concurrent_equivalent_starts_recover_a_lost_actual_acceptance_reply_without_a_second_child() + { + using var lostReply = new CancellationTokenSource(); + _start!.StartReplyWaitCancellation = lostReply.Token; + var startedEntered = NewSignal(); + var startedRelease = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is ChildRunEvent.Started) + { + startedEntered.TrySetResult(); + await startedRelease.Task.WaitAsync(TestContext.Current.CancellationToken); + } + return false; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + try + { + await SendOriginalAsync(manager); + await startedEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + Assert.Empty(before.OfType()); + Assert.Equal(0, _child.CallCount); + lostReply.Cancel(); + await _start.StartReplyWaitCancelled.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var retry = Retry(accepted); + var first = CreateTestProbe(); + var second = CreateTestProbe(); + owner.Tell(retry, first.Ref); + owner.Tell(retry, second.Ref); + startedRelease.TrySetResult(); + var firstReply = await first.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var secondReply = await second.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + foreach (var reply in new[] { firstReply, secondReply }) + { + Assert.Equal(accepted.Run.RunId, reply.Run.RunId); + Assert.Equal(accepted.Run.StartKey, reply.Run.StartKey); + Assert.Equal(accepted.Run.ArgumentsDigest, reply.Run.ArgumentsDigest); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, reply.Run.OriginalContext)); + Assert.Equal(BackgroundChildState.Running, reply.State); + } + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var after = await ReadJournalAsync(); + Assert.Single(after.OfType()); + Assert.Single(after.OfType()); + Assert.Empty(after.OfType()); + var result = Assert.Single(after.OfType()); + Assert.Contains("Error executing tool:", result.ToolResult.Content, StringComparison.Ordinal); + Assert.NotNull(result.LoopObservation); + Assert.NotEqual((int)ToolInvocationOutcomeCategory.Success, result.LoopObservation.Category); + Assert.False(result.LoopObservation.MissingReceipt); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + var child = await Sys.ActorSelection($"{owner.Path}/child-run-{accepted.Run.RunId.Value}") + .Ask(new Identify("single-child-after-concurrent-retry"), Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(child.Subject); + var recorded = Assert.IsType(await owner.Ask(retry, Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(accepted.Run.RunId, recorded.Run.RunId); + Assert.Equal(after.Length, (await ReadJournalAsync()).Length); + } + finally { startedRelease.TrySetResult(); } + } + + [Fact] + public async Task Cancellation_of_the_original_start_invocation_does_not_cancel_the_accepted_child_or_its_controls() + { + using var invocationCancellation = new CancellationTokenSource(); + _start!.StartInvocationCancellation = invocationCancellation.Token; + var entered = NewSignal(); + var release = NewSignal(); + var nextProvider = NewSignal(); + var calls = 0; + Host.Services.GetRequiredService().ReplaceCore(new FakeNetclawTool( + "file_read", "The child continued after its original dispatch ended.", "file", invocation => + { + Interlocked.Increment(ref calls); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + })); + _child.ToolCallsOnFirstCall = [ReadCall("after-start-dispatch")]; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is ChildRunEvent.Started) + { + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + } + return false; + }); + var (_, manager, subscriber) = await CreateOwnerAsync(); + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + var startToken = _start!.Token; + Assert.True(startToken.CanBeCanceled); + Assert.False(startToken.IsCancellationRequested); + Assert.Equal(invocationCancellation.Token, startToken); + Assert.Equal(startToken, _start.Prepared!.Execution.Cancellation); + Assert.False(_start.StartReplyWaitCancelled.Task.IsCompleted); + Assert.False(_child.FirstCallEntered.Task.IsCompleted); + await invocationCancellation.CancelAsync(); + await _start.StartReplyWaitCancelled.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.True(startToken.IsCancellationRequested); + Assert.False(_child.FirstCallEntered.Task.IsCompleted); + release.TrySetResult(); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var startRecords = await ReadJournalAsync(); + Assert.Single(startRecords.OfType()); + Assert.Single(startRecords.OfType()); + Assert.Empty(startRecords.OfType()); + var startResult = Assert.Single(startRecords.OfType()); + Assert.Contains("Error executing tool:", startResult.ToolResult.Content, StringComparison.Ordinal); + Assert.NotNull(startResult.LoopObservation); + Assert.NotEqual((int)ToolInvocationOutcomeCategory.Success, startResult.LoopObservation.Category); + Assert.False(startResult.LoopObservation.MissingReceipt); + Assert.False(_childRelease.Task.IsCompleted); + _main.PlannedResponses.Enqueue([new FunctionCallContent("independent-status", "status_json_probe", + new Dictionary + { + ["run_id"] = accepted.Run.RunId.Value, ["cancel"] = false, + ["_rationale"] = "Inspect the child after the original invocation is cancelled." + })]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Inspect the accepted child.", Source = DeliverySource("status", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + var status = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolResultOutput, Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + await CompletedAsync(subscriber); + using (var body = JsonDocument.Parse(status.Result)) + { + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Running", body.RootElement.GetProperty("state").GetString()); + Assert.False(body.RootElement.GetProperty("cancellation_requested").GetBoolean()); + } + _child.NextResponseGate = nextProvider; + _childRelease.TrySetResult(); + await AwaitAssertAsync(() => Assert.Equal(2, _child.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var continued = await ReadJournalAsync(); + var checkpoint = Assert.Single(continued.OfType()); + Assert.Equal(accepted.Run.RunId, checkpoint.RunId); + Assert.Equal("The child continued after its original dispatch ended.", checkpoint.Checkpoint.Summary); + Assert.Equal(1, Volatile.Read(ref calls)); + Assert.False(nextProvider.Task.IsCompleted); + Assert.Empty(continued.OfType()); + _main.PlannedResponses.Enqueue([ControlCall("independent-cancel", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the accepted child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + var control = await _start.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(BackgroundChildState.Cancelling, control.Run!.State); + await AwaitAssertAsync(async () => + { + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + var terminal = Assert.Single(final.OfType()); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(2, _child.CallCount); + Assert.Equal(1, Volatile.Read(ref calls)); + } + finally + { + release.TrySetResult(); + nextProvider.TrySetResult(); + } + } + + [Fact] + public async Task A_child_provider_that_ignores_cancellation_cannot_delay_owner_terminal_settlement() + { + var provider = new UncooperativeChildClient(_child); + Assert.IsType(Host.Services.GetRequiredService()).Child = provider; + var path = Path.Combine(_directory!.Path, "confirmed-before-uncooperative-provider.txt"); + var calls = 0; + Host.Services.GetRequiredService().ReplaceCore(new FakeNetclawTool( + "file_read", "The confirmed checkpoint precedes the held remote task.", "file", invocation => + { + Interlocked.Increment(ref calls); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([new ToolFileActivity(path, ToolFileActivityKind.Read)], null)); + })); + _child.ToolCallsOnFirstCall = [ReadCall("before-uncooperative-provider")]; + var (_, manager, subscriber) = await CreateOwnerAsync(); + try + { + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Delegate the neutral task.", Source = DeliverySource("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _childRelease.TrySetResult(); + await provider.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + var checkpoint = Assert.Single(before.OfType()); + Assert.Equal(path, Assert.Single(checkpoint.Checkpoint.ConfirmedActivity.ReadFiles)); + var pair = Assert.Single(provider.HeldMessages.SelectMany(message => message.Contents.OfType())); + Assert.Equal("before-uncooperative-provider", pair.CallId); + Assert.Equal(checkpoint.Checkpoint.Summary, Assert.IsType(pair.Result)); + Assert.Equal(2, provider.Count); + Assert.False(provider.Pending.Task.IsCompleted); + _main.PlannedResponses.Enqueue([ControlCall("cancel-uncooperative-provider", accepted.Run.RunId)]); + await manager.Ask(new SendUserMessage + { SessionId = Session, Content = "Cancel the accepted child.", Source = DeliverySource("cancel", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + var control = await _start!.ControlReply.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(BackgroundChildState.Cancelling, control.Run!.State); + ChildRunEvent.TerminalRecorded? terminal = null; + await AwaitAssertAsync(async () => + { + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + terminal = Assert.Single(final.OfType()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotNull(terminal); + Assert.False(terminal.Terminal.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, terminal.Terminal.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); + var completion = Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Equal(path, Assert.Single(completion.ConfirmedActivity!.ReadFiles)); + Assert.True(provider.Token.IsCancellationRequested); + Assert.False(provider.Pending.Task.IsCompleted); + Assert.Equal(2, provider.Count); + Assert.Equal(1, Volatile.Read(ref calls)); + var storage = Assert.IsType(_start.Prepared!.Execution.Scope.Authority.Session).Storage; + var report = Path.Combine(storage.ArtifactDirectory.Value, "cancelled-results.json"); + using var body = JsonDocument.Parse(await File.ReadAllTextAsync(report, TestContext.Current.CancellationToken)); + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(checkpoint.Checkpoint.Summary, body.RootElement.GetProperty("summary").GetString()); + Assert.Equal("Cancelled", body.RootElement.GetProperty("state").GetString()); + Assert.Null(terminal.Terminal.EvidenceWarning); + Assert.False(provider.Pending.Task.IsCompleted); + } + finally + { + _childRelease.TrySetResult(); + provider.Pending.TrySetCanceled(TestContext.Current.CancellationToken); + if (provider.Held.Task.IsCompletedSuccessfully) + await provider.Exited.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + } + } + + private sealed class UncooperativeChildClient(IChatClient first) : IChatClient + { + private int _count; + private CancellationTokenRegistration _registration; + public Action? CancellationCallback { get; init; } + public int Count => Volatile.Read(ref _count); + public TaskCompletionSource Held { get; } = NewSignal(); + public TaskCompletionSource Exited { get; } = NewSignal(); + public TaskCompletionSource Pending { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken Token { get; private set; } + public IReadOnlyList HeldMessages { get; private set; } = []; + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _count) == 1) + return first.GetResponseAsync(messages, options, cancellationToken); + Token = cancellationToken; + HeldMessages = messages.ToArray(); + if (CancellationCallback is { } callback) + _registration = cancellationToken.Register(callback); + Held.TrySetResult(); + return Pending.Task; + } + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, + ChatOptions? options = null, CancellationToken cancellationToken = default) + => StreamAsync(messages, options, cancellationToken); + private async IAsyncEnumerable StreamAsync(IEnumerable messages, + ChatOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken) + { + try + { + var response = await GetResponseAsync(messages, options, cancellationToken); + foreach (var update in response.ToChatResponseUpdates()) yield return update; + } + finally { if (Count > 1) Exited.TrySetResult(); } + } + public object? GetService(Type serviceType, object? serviceKey = null) => first.GetService(serviceType, serviceKey); + public void Dispose() + { + _registration.Dispose(); + first.Dispose(); + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.TerminalCuts.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.TerminalCuts.cs new file mode 100644 index 000000000..a3f39b9d5 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.TerminalCuts.cs @@ -0,0 +1,306 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Configuration; +using Akka.Routing; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_captured_terminal_requires_its_actual_child_sender_and_has_one_durable_winner() + { + var gate = InstallTerminalCutMailbox(nameof(BackgroundChildTerminal), child: false); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + try + { + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _snapshots.InvocationStarted = NewSignal(); + _childRelease.TrySetResult(); + var captured = await gate.Held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var message = Assert.IsType(captured.Envelope.Message); + Assert.Equal(owner, captured.Receiver); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal(accepted.Run.RunId, message.RunId); + Assert.NotEqual(owner, captured.Envelope.Sender); + var foreign = CreateTestProbe(); + owner.Tell(message, foreign.Ref); + owner.Tell(new Identify("after-foreign-terminal"), foreign.Ref); + Assert.Equal(owner, (await foreign.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken)).Subject); + var rejected = await ReadJournalAsync(); + Assert.Empty(rejected.OfType()); + Assert.Empty(rejected.OfType()); + Assert.Empty(rejected.OfType()); + Assert.False(_snapshots.InvocationStarted.Task.IsCompleted); + Assert.Equal(2, _main.CallCount); + + gate.Release(); + await CompletedAsync(subscriber); + var committed = await ReadJournalPositionsAsync(); + var terminalRow = Assert.Single(committed, row => row.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + Assert.Equal(terminalRow.SequenceNr, terminal.TerminalSequenceNr); + Assert.Equal(message.Result.Output, terminal.Terminal.Result.Output); + Assert.Equal(message.Result.Outcome, terminal.Terminal.Result.Outcome); + Assert.Single(committed, row => row.Event is ChildRunEvent.ResultPrepared); + Assert.Single(committed, row => row.Event is ChildRunEvent.DeliveryAdmitted); + owner.Tell(message, captured.Envelope.Sender); + await AssertTerminalCutOwnerBarrierAsync(owner, "after-duplicate-terminal"); + Assert.Equal(3, _main.CallCount); + + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var recovered = await OwnerAsync(); + Assert.NotEqual(owner, recovered); + recovered.Tell(message, captured.Envelope.Sender); + await AssertTerminalCutOwnerBarrierAsync(recovered, "after-old-generation-terminal"); + var final = await ReadJournalPositionsAsync(); + Assert.Equal(committed.Select(row => row.SequenceNr), final.Select(row => row.SequenceNr)); + Assert.Single(final, row => row.Event is ChildRunAccepted); + Assert.Single(final, row => row.Event is ChildRunEvent.Started); + Assert.Single(final, row => row.Event is ChildRunEvent.TerminalRecorded); + Assert.Single(final, row => row.Event is ChildRunEvent.ResultPrepared); + Assert.Single(final, row => row.Event is ChildRunEvent.DeliveryAdmitted); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + finally { gate.Release(); } + } + + [Fact] + public async Task A_failed_terminal_write_cannot_acknowledge_enrich_or_admit_the_child_result() + { + var gate = InstallTerminalCutMailbox(nameof(BackgroundChildTerminalAck), child: true); + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunEvent.TerminalRecorded) return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _snapshots.InvocationStarted = NewSignal(); + await EventFilter.Error(contains: "Failed to persist event type").ExpectOneAsync(async () => + { + _childRelease.TrySetResult(); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.False(gate.Held.Task.IsCompleted); + Assert.False(_snapshots.InvocationStarted.Task.IsCompleted); + release.TrySetResult(); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Empty(final.OfType()); + Assert.Empty(final.OfType()); + Assert.Empty(final.OfType()); + Assert.False(gate.Held.Task.IsCompleted); + Assert.False(_snapshots.InvocationStarted.Task.IsCompleted); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + finally { release.TrySetResult(); gate.Release(); } + } + + [Fact] + public async Task A_real_enrichment_failure_preserves_the_committed_outcome_and_warns_in_its_delivery() + { + var (_, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _snapshots.InvocationStarted = NewSignal(); + _snapshots.Failure = new InvalidOperationException("The neutral terminal snapshot failed."); + _childRelease.TrySetResult(); + await _snapshots.InvocationStarted.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var final = await ReadJournalPositionsAsync(); + var terminalRow = Assert.Single(final, row => row.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + var preparedRow = Assert.Single(final, row => row.Event is ChildRunEvent.ResultPrepared); + var prepared = Assert.IsType(preparedRow.Event); + var deliveryRow = Assert.Single(final, row => row.Event is ChildRunEvent.DeliveryAdmitted); + var delivery = Assert.IsType(deliveryRow.Event); + Assert.True(terminal.Terminal.Result.Success); + Assert.IsType(terminal.Terminal.Result.Completion); + Assert.Null(terminal.Terminal.EvidenceWarning); + Assert.Equal(terminal.Terminal.Result.Output, prepared.Terminal.Result.Output); + Assert.Equal(terminal.Terminal.Result.Outcome, prepared.Terminal.Result.Outcome); + Assert.Equal(terminal.Terminal.Result.OutcomeReason, prepared.Terminal.Result.OutcomeReason); + Assert.True(prepared.Terminal.Result.Success); + Assert.Equal(terminal.TerminalSequenceNr, prepared.TerminalSequenceNr); + Assert.True(terminalRow.SequenceNr < preparedRow.SequenceNr); + Assert.True(preparedRow.SequenceNr < deliveryRow.SequenceNr); + Assert.Equal("Result enrichment failed. The original terminal receipt remains unchanged.", prepared.Terminal.EvidenceWarning); + using var body = JsonDocument.Parse(delivery.Input.UserMessage.Content!); + Assert.Equal(prepared.Terminal.EvidenceWarning, body.RootElement.GetProperty("warning").GetString()); + Assert.Equal(terminal.Terminal.Result.Output, body.RootElement.GetProperty("output").GetString()); + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var request = _main.ReceivedMessages[^1]; + Assert.Single(request.SelectMany(message => message.Contents.OfType()), call => call.CallId == id); + Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single( + request.SelectMany(message => message.Contents.OfType()), result => result.CallId == id).Result)); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + + [Theory] + [InlineData("terminal")] + [InlineData("prepared")] + [InlineData("admitted")] + public async Task Cold_recovery_preserves_the_actual_terminal_prefix_without_child_or_delivery_replay(string cut) + { + var entered = NewSignal(); + var release = NewSignal(); + var intercepted = 0; + ISessionEvent? pending = null; + await Journal.OnWrite.FailIf(async record => + { + var matches = cut switch + { + "terminal" => record.Payload is ChildRunEvent.TerminalRecorded, + "prepared" => record.Payload is ChildRunEvent.ResultPrepared, + _ => record.Payload is ChildRunEvent.DeliveryAdmitted + }; + if (!matches || Interlocked.Exchange(ref intercepted, 1) != 0) return false; + pending = Assert.IsAssignableFrom(record.Payload); + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + var resume = NewSignal(); + try + { + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + _childRelease.TrySetResult(); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var prefix = await ReadJournalPositionsAsync(); + var acceptance = Assert.Single(prefix.Select(row => row.Event).OfType()); + var terminal = cut == "terminal" + ? Assert.IsType(pending) + : Assert.IsType(Assert.Single(prefix, + row => row.Event is ChildRunEvent.TerminalRecorded).Event); + Assert.Equal(cut == "terminal" ? 0 : 1, prefix.Count(row => row.Event is ChildRunEvent.TerminalRecorded)); + Assert.True(terminal.Terminal.Result.Success); + Assert.Equal(cut == "admitted" ? 1 : 0, prefix.Count(row => row.Event is ChildRunEvent.ResultPrepared)); + Assert.DoesNotContain(prefix, row => row.Event is ChildRunEvent.DeliveryAdmitted); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + release.TrySetResult(); + await AwaitAssertAsync(async () => + { + var stored = await ReadJournalAsync(); + Assert.Single(stored.OfType()); + Assert.Equal(cut == "terminal" ? 0 : 1, stored.OfType().Count()); + Assert.Equal(cut == "admitted" ? 1 : 0, stored.OfType().Count()); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + _main.NextResponseGate = resume; + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(owner, await OwnerAsync()); + await AwaitAssertAsync(() => Assert.Equal(3, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var resumed = await ReadJournalPositionsAsync(); + var actualTerminal = Assert.Single(resumed, row => row.Event is ChildRunEvent.TerminalRecorded); + Assert.Equal(terminal.TerminalSequenceNr, actualTerminal.SequenceNr); + var actual = Assert.IsType(actualTerminal.Event); + Assert.False(actual.Terminal.Lost); + Assert.Equal(terminal.Terminal.Result.Output, actual.Terminal.Result.Output); + Assert.Equal(terminal.Terminal.Result.Outcome, actual.Terminal.Result.Outcome); + Assert.Single(resumed, row => row.Event is ChildRunEvent.ResultPrepared); + var delivery = Assert.IsType(Assert.Single(resumed, + row => row.Event is ChildRunEvent.DeliveryAdmitted).Event); + Assert.Equal(acceptance.Run.RunId, delivery.RunId); + Assert.True(SessionState.SameCanonicalContext(acceptance.Run.OriginalContext, delivery.Input.TurnContext)); + var adoption = Assert.Single(resumed.Select(row => row.Event).OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(new[] { delivery.Input.InputId }, adoption.InputIds); + Assert.True(SessionState.SameCanonicalContext(acceptance.Run.OriginalContext, adoption.TurnContext)); + var id = delivery.Input.UserMessage.ToolCallId!.Value.Value; + var request = _main.ReceivedMessages[^1]; + var call = Assert.Single(request.SelectMany(message => message.Contents.OfType()), item => item.CallId == id); + Assert.Equal(ChildRunDelivery.ToolName(acceptance.Run), call.Name); + Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single( + request.SelectMany(message => message.Contents.OfType()), item => item.CallId == id).Result)); + using var body = JsonDocument.Parse(delivery.Input.UserMessage.Content!); + Assert.Equal(terminal.Terminal.Result.Output, body.RootElement.GetProperty("output").GetString()); + Assert.Equal(1, _child.CallCount); + Assert.Empty(_start!.ProbeCallIds); + resume.TrySetResult(); + await CompletedAsync(recoveredSubscriber); + var final = await ReadJournalAsync(); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType(), evt => evt.ContinuedChildRunId is not null); + var consumed = final.OfType().SelectMany(evt => evt.ConsumedInputIds) + .Where(inputId => inputId == delivery.Input.InputId); + Assert.Equal(new[] { delivery.Input.InputId }, consumed); + Assert.Equal(3, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + finally { release.TrySetResult(); resume.TrySetResult(); } + } + + private BackgroundChildApprovalGrantAdversarialTests.WaitReplyGate InstallTerminalCutMailbox(string messageName, bool child) + { + const string mailbox = "child-terminal-cut-mailbox"; + Sys.Settings.InjectTopLevelFallback(ConfigurationFactory.ParseString( + $"{mailbox} {{ mailbox-type = \"{typeof(BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox).AssemblyQualifiedName}\" }}")); + var gate = BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox.Gates.GetOrCreateValue(Sys); + gate.MessageName = messageName; + gate.Arm(); + if (child) _start!.ChildMailbox = mailbox; + else ((ExtendedActorSystem)Sys).Provider.Deployer.SetDeploy(new Deploy( + $"/session-manager/{Uri.EscapeDataString(Session.Value)}", Config.Empty, NoRouter.Instance, + LocalScope.Instance, Deploy.NoDispatcherGiven, mailbox)); + return gate; + } + + private async Task AssertTerminalCutOwnerBarrierAsync(IActorRef owner, string label) + { + var barrier = CreateTestProbe(); + owner.Tell(new Identify(label), barrier.Ref); + Assert.Equal(owner, (await barrier.ExpectMsgAsync(Ceiling, + cancellationToken: TestContext.Current.CancellationToken)).Subject); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs new file mode 100644 index 000000000..f80e0192d --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs @@ -0,0 +1,468 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Persistence; +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Jobs; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Reminders; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tests.Utilities; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests(ITestOutputHelper output) + : PersistenceTestKit(output: output), IAsyncDisposable +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(30); + private readonly FakeChatClient _main = new(); + private readonly FakeChatClient _child = new(); + private readonly FakeChatClient _secondChild = new(); + private readonly TaskCompletionSource _secondChildRelease = NewSignal(); + private readonly FakeTimeProvider _time = new(); + private readonly ControllableWorkingContextSnapshotProvider _snapshots = new(); + private readonly TaskCompletionSource _childRelease = NewSignal(); + private TestSessionTempDirectory? _directory; + private StartExecutor? _start; + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + { + base.ConfigureAkka(builder, provider); + builder.WithNetclawSerialization() + .WithNetclawActors(provider.GetRequiredService()); + } + + protected override void ConfigureServices(HostBuilderContext context, IServiceCollection services) + { + _directory = TestSessionTempDirectory.Create("netclaw-child-owner-"); + services.AddSingleton(_directory); + services.AddSingleton(_directory.Paths); + services.AddSingleton(new FakeCapabilityResolver()); + services.AddSingleton(_time); + services.AddSingleton(_snapshots); + services.AddSingleton(SecurityPolicyDefaults.Resolve(null)); + services.AddSingleton(); + services.AddSingleton(new SchedulingConfig()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(NullNotificationSink.Instance); + services.AddSingleton(NullReminderChannelNotifier.Instance); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + var clients = new RoleProvider(_main, _child, _secondChild); + services.AddSingleton(clients); + services.AddSingleton(new ModelCapabilities { ModelId = "fake-model", ContextWindowTokens = 128000 }); + services.AddSingleton(new SessionConfig + { + Tuning = new SessionTuning { SnapshotInterval = 1000, TitleGenerationInterval = 0 } + }); + var prompt = new StaticSystemPromptProvider("Use the assigned tools."); + services.AddSingleton(prompt); + var registry = new ToolRegistry(); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "start_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "spawn_agent"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "skill_load"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "control_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "status_json_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "file_read"), "file"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "neutral_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "missing_receipt_probe"), "builtin"); + services.AddSingleton(registry); + var policy = new ToolAccessPolicy(_directory.Paths, new ToolConfig(), + new EffectivePolicyDefaults(DeploymentPosture.Personal, TrustAudience.Personal, + ShellExecutionMode.HostAllowed, UsedStrictFallback: false), + new ShellCommandPolicy(), new ToolPathPolicy([])); + var spawner = new SubAgentSpawner(clients, registry, policy, null, prompt, _snapshots, + NullLogger.Instance); + _start = new StartExecutor(spawner, registry, policy); + services.AddSingleton(_start); + services.AddLlmSessionCompositeRecords(); + _main.ToolCallsOnFirstCall = [new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the assigned task." })]; + _child.NextResponseGate = _childRelease; + _secondChild.NextResponseGate = _secondChildRelease; + } + + [Theory] + [InlineData("acceptance")] + [InlineData("started")] + public async Task Neither_durable_start_boundary_can_dispatch_the_child_before_its_write_completes(string boundary) + { + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (boundary == "acceptance" ? record.Payload is not ChildRunAccepted : record.Payload is not ChildRunEvent.Started) + return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + try + { + await SendOriginalAsync(manager); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var childIdentity = await Sys.ActorSelection($"{owner.Path}/child-run-{_start!.Prepared!.RunId.Value}") + .Ask(new Identify("held-start-boundary"), Ceiling, TestContext.Current.CancellationToken); + if (boundary == "acceptance") + Assert.Null(childIdentity.Subject); + else + Assert.NotNull(childIdentity.Subject); + Assert.Equal(0, _child.CallCount); + Assert.Equal(1, _main.CallCount); + release.TrySetResult(); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var events = await ReadJournalAsync(); + var admission = Assert.Single(events.OfType()); + var started = Assert.Single(events.OfType()); + Assert.True(Array.IndexOf(events, admission) < Array.IndexOf(events, started)); + Assert.Equal(admission.Run.RunId, started.RunId); + Assert.Equal("original", admission.Run.OriginalContext.TurnId); + Assert.Equal(new SenderId("operator-a"), admission.Run.OriginalContext.RequesterSenderId); + Assert.Single(admission.Run.OriginInputIds); + Assert.Equal(1, _child.CallCount); + Assert.Equal(2, _main.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + var result = Assert.Single(events.OfType()); + Assert.Equal(new ToolCallId("start-1"), result.ToolResult.ToolCallId); + Assert.False(result.LoopObservation!.Synthetic); + using var body = JsonDocument.Parse(result.ToolResult.Content); + Assert.Equal("Accepted", body.RootElement.GetProperty("state").GetString()); + Assert.Equal(admission.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("check_agent_run", body.RootElement.GetProperty("control_tool").GetString()); + var paired = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType())); + Assert.Equal("start-1", paired.CallId); + Assert.Equal(result.ToolResult.Content, paired.Result!.ToString()); + Assert.IsType(await owner.Ask(Retry(admission), Ceiling, + TestContext.Current.CancellationToken)); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task A_failed_acceptance_has_no_durable_child_or_child_provider_request() + { + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunAccepted) return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var (owner, manager, _) = await CreateOwnerAsync(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + await EventFilter.Error(contains: "Failed to persist event type").ExpectOneAsync(async () => + { + await SendOriginalAsync(manager); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(0, _child.CallCount); + release.TrySetResult(); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + var events = await ReadJournalAsync(); + Assert.Single(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Empty(events.OfType()); + Assert.Equal(0, _child.CallCount); + Assert.Equal(1, _main.CallCount); + } + finally { release.TrySetResult(); } + } + + [Fact] + public async Task A_recorded_retry_survives_expired_dispatch_but_cannot_change_arguments_or_original_authority() + { + var (owner, manager, subscriber) = await CreateOwnerAsync(); + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var before = await ReadJournalAsync(); + var admission = Assert.Single(before.OfType()); + var retry = Retry(admission); + var staleNew = retry with + { + StartKey = new ChildRunStartKey.Tool(new ToolCallId("unadmitted-new-start")) + { SessionId = Session, TurnId = new TurnId("original") } + }; + var stale = await Assert.ThrowsAnyAsync(() => owner.Ask(staleNew, Ceiling, + TestContext.Current.CancellationToken)); + Assert.Contains("no longer owns", stale.Message, StringComparison.Ordinal); + var repeated = Assert.IsType(await owner.Ask(retry, Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(admission.Run.RunId, repeated.Run.RunId); + Assert.Equal(admission.Run.StartKey, repeated.Run.StartKey); + Assert.Equal(admission.Run.ArgumentsDigest, repeated.Run.ArgumentsDigest); + Assert.True(SessionState.SameCanonicalContext(admission.Run.OriginalContext, repeated.Run.OriginalContext)); + Assert.Equal(BackgroundChildState.Running, repeated.State); + var conflict = retry with { Prepared = retry.Prepared with { ArgumentsDigest = new string('b', 64) } }; + Assert.IsType(await owner.Ask(conflict, Ceiling, + TestContext.Current.CancellationToken)); + var foreign = retry with + { + InvocationContext = retry.InvocationContext with { RequesterSenderId = new SenderId("operator-b") } + }; + var denial = await Assert.ThrowsAsync(() => owner.Ask(foreign, Ceiling, + TestContext.Current.CancellationToken)); + Assert.DoesNotContain(admission.Run.RunId.Value, denial.Message, StringComparison.Ordinal); + Assert.DoesNotContain(admission.Run.ScopeId.Value, denial.Message, StringComparison.Ordinal); + Assert.IsType(await owner.Ask(retry, Ceiling, + TestContext.Current.CancellationToken)); + var after = await ReadJournalAsync(); + Assert.Equal(before.Length, after.Length); + Assert.Single(after.OfType()); + Assert.Single(after.OfType()); + Assert.Equal(1, _child.CallCount); + Assert.Equal(2, _main.CallCount); + } + + [Fact] + public async Task Cold_recovery_of_accepted_but_unstarted_child_returns_the_recorded_run_without_relaunch() + { + var entered = NewSignal(); + var release = NewSignal(); + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunEvent.Started) return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var (owner, manager, _) = await CreateOwnerAsync(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + await SendOriginalAsync(manager); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(0, _child.CallCount); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + release.TrySetResult(); + await Journal.OnWrite.Pass(); + var before = await ReadJournalAsync(); + var admission = Assert.Single(before.OfType()); + Assert.Empty(before.OfType()); + Assert.Empty(before.OfType()); + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var recovered = await OwnerAsync(); + Assert.NotEqual(owner, recovered); + var repeated = Assert.IsType(await recovered.Ask(Retry(admission), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(admission.Run.RunId, repeated.Run.RunId); + Assert.Equal(BackgroundChildState.Lost, repeated.State); + Assert.Equal(admission.Run.StartKey, repeated.Run.StartKey); + Assert.Equal(admission.Run.ArgumentsDigest, repeated.Run.ArgumentsDigest); + var lost = Assert.IsType(repeated.Run.Terminal); + Assert.True(lost.Lost); + Assert.Equal(SubAgentRunOutcome.Failed, lost.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.OwnerRestartLost, lost.Result.OutcomeReason); + Assert.True(SessionState.SameCanonicalContext(admission.Run.OriginalContext, repeated.Run.OriginalContext)); + Assert.Equal(admission.Run.OriginInputIds, repeated.Run.OriginInputIds); + var after = await ReadJournalPositionsAsync(); + Assert.Single(after.Select(record => record.Event).OfType()); + Assert.Empty(after.Select(record => record.Event).OfType()); + var terminalRow = Assert.Single(after, record => record.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + Assert.Equal(terminalRow.SequenceNr, terminal.TerminalSequenceNr); + Assert.Equal(admission.Run.RunId, terminal.RunId); + Assert.True(terminal.Terminal.Lost); + Assert.Equal(SubAgentOutcomeReason.OwnerRestartLost, terminal.Terminal.Result.OutcomeReason); + await CompletedAsync(recoveredSubscriber); + var settled = await ReadJournalAsync(); + Assert.Single(settled.OfType()); + var delivery = Assert.Single(settled.OfType()); + Assert.Equal(admission.Run.RunId, delivery.RunId); + var adoption = Assert.Single(settled.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.True(SessionState.SameCanonicalContext(admission.Run.OriginalContext, adoption.TurnContext)); + var pairs = _main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()); + Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single(pairs, + result => result.CallId == delivery.Input.UserMessage.ToolCallId!.Value.Value).Result)); + Assert.Equal(0, _child.CallCount); + Assert.Equal(2, _main.CallCount); + } + finally { release.TrySetResult(); } + } + + private static readonly SessionId Session = new("signalr/background-owner-adversarial"); + private StartBackgroundChildRun Retry(ChildRunAccepted admission) + => new(_start!.Prepared!, admission.Run.StartKey, admission.Run.OriginalContext, + admission.Run.SourceOperation, _start.Token); + private async Task<(IActorRef Owner, IActorRef Manager, Akka.TestKit.TestProbe Subscriber)> CreateOwnerAsync() + { + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = Session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + return (await OwnerAsync(), manager, subscriber); + } + private Task OwnerAsync() => Sys.ActorSelection( + $"/user/session-manager/{Uri.EscapeDataString(Session.Value)}").ResolveOne(Ceiling, TestContext.Current.CancellationToken); + private static Task SendOriginalAsync(IActorRef manager) => manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Delegate the neutral task.", Source = new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), TurnId = new TurnId("original"), + MessageId = "original", Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, + Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + } + }, Ceiling, TestContext.Current.CancellationToken); + private static ValueTask CompletedAsync(Akka.TestKit.TestProbe subscriber) + => subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + private async Task ReadJournalAsync() + => (await ReadJournalPositionsAsync()).Select(record => record.Event).ToArray(); + private async Task<(ISessionEvent Event, long SequenceNr)[]> ReadJournalPositionsAsync() + { + var done = new TaskCompletionSource<(ISessionEvent Event, long SequenceNr)[]>(TaskCreationOptions.RunContinuationsAsynchronously); + var reader = Sys.ActorOf(Props.Create(() => new ChildJournalReader($"session-{Session.Value}", done))); + var watcher = CreateTestProbe(); + watcher.Watch(reader); + var records = await done.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Sys.Stop(reader); + await watcher.ExpectTerminatedAsync(reader, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + return records; + } + private sealed class ChildJournalReader : ReceivePersistentActor + { + public ChildJournalReader(string id, TaskCompletionSource<(ISessionEvent Event, long SequenceNr)[]> done) + { + PersistenceId = id; + var events = new List<(ISessionEvent Event, long SequenceNr)>(); + Recover(evt => events.Add((evt, LastSequenceNr))); + Recover(_ => done.TrySetResult(events.ToArray())); + } + public override string PersistenceId { get; } + public override Recovery Recovery => new(SnapshotSelectionCriteria.None); + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class StartExecutor(SubAgentSpawner spawner, ToolRegistry registry, ToolAccessPolicy policy) : IToolExecutor + { + public PreparedChildRun? Prepared { get; private set; } + public SubAgentSpawner Spawner => spawner; + public INetclawTool? StartTool { get; set; } + public ToolInvocationContext? StartInvocation { get; private set; } + public List ProbeRequesters { get; } = []; + public List ProbeCallIds { get; } = []; + public CancellationToken Token { get; private set; } + public CancellationToken? StartInvocationCancellation { get; set; } + public CancellationToken? StartReplyWaitCancellation { get; set; } + public TaskCompletionSource StartReplyWaitCancelled { get; } = NewSignal(); + public ToolInvocationContext? ControlInvocation { get; private set; } + public CancellationToken ControlToken { get; private set; } + public TaskCompletionSource? ControlCompletionGate { get; set; } + public string? ChildMailbox { get; set; } + public TaskCompletionSource ControlReply { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public async Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) + { + if (call.Name is "neutral_probe" or "missing_receipt_probe") + { + ProbeRequesters.Add(context.RunScope.DefaultDeliveryTarget?.DestinationId); + ProbeCallIds.Add(call.CallId); + if (call.Name == "neutral_probe") + context.Outputs.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + return "The neutral result is unchanged."; + } + var factory = context.Invocation.SpawnChildActor + ?? throw new InvalidOperationException("The start test requires the owner factory."); + if (call.Name == "status_json_probe") + return await new CheckAgentRunTool().ExecuteAsync(new Dictionary + { + ["RunId"] = call.Arguments!["run_id"]!.ToString(), + ["Cancel"] = JsonSerializer.SerializeToElement(call.Arguments).GetProperty("cancel").GetBoolean() + }, context.Invocation, ct); + if (call.Name == "control_probe") + { + ControlInvocation = context.Invocation; + ControlToken = ct; + var reply = Assert.IsType(await factory(new ChildControlRequest( + new SubAgentRunId(call.Arguments!["run_id"]!.ToString()!), true), "control", ct)); + ControlReply.TrySetResult(reply); + if (ControlCompletionGate is { } completion) await completion.Task.WaitAsync(ct); + context.Outputs.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + return JsonSerializer.Serialize(new { found = reply.Run is not null, state = reply.Run?.State.ToString(), + dispatch_closed = reply.Run?.DispatchClosedAtMs is not null }); + } + var scope = context.RunScope with { SpawnChildActor = async (payload, name, token) => + { + Prepared = Assert.IsType(payload); + if (ChildMailbox is { } mailbox) Prepared = Prepared with { Props = Prepared.Props.WithMailbox(mailbox) }; + Token = token; + try { return await factory(Prepared, name, StartReplyWaitCancellation ?? token); } + catch (OperationCanceledException) when (StartReplyWaitCancellation?.IsCancellationRequested == true + || StartInvocationCancellation?.IsCancellationRequested == true) + { + StartReplyWaitCancelled.TrySetResult(); + throw; + } + }}; + StartInvocation = new ToolInvocationContext(scope, context.ExecutionTimeout, context.Outputs); + if (StartTool is SkillLoadTool routedTool) + { + registry.ReplaceCore(routedTool); + var dispatchContext = new ToolExecutionContext(scope, context.ExecutionTimeout, context.Outputs); + dispatchContext.Approval.RestoreAuthorizationAttemptId(context.Approval.AuthorizationAttemptId); + return await new DispatchingToolExecutor(registry, policy).ExecuteAsync( + call, dispatchContext, StartInvocationCancellation ?? ct); + } + if (StartTool is { } startTool) + return await startTool.ExecuteAsync(call.Arguments, StartInvocation, StartInvocationCancellation ?? ct); + return await spawner.StartRunAsync(new SubAgentProfile + { + Name = "worker", Description = "Complete the neutral task.", SystemPrompt = "Complete the assigned task.", + ModelRole = call.CallId == "start-2" ? ModelRole.Fallback : ModelRole.Compaction, ToolNames = ["file_read"], EmitStructuredFindings = false + }, "Inspect the neutral fixture.", null, + new ToolInvocationContext(scope, context.ExecutionTimeout, context.Outputs), StartInvocationCancellation ?? ct, null); + } + } + + private sealed class RoleProvider(IChatClient main, IChatClient child, IChatClient secondChild) : IChatClientProvider + { + public IChatClient Child { get; set; } = child; + public IChatClient GetClient(ModelRole role) => role switch + { + ModelRole.Main => main, + ModelRole.Fallback => secondChild, + _ => Child + }; + } + + async ValueTask IAsyncDisposable.DisposeAsync() + { + _childRelease.TrySetResult(); + _secondChildRelease.TrySetResult(); + try { await base.DisposeAsync(); } + finally { if (_directory is not null) await _directory.DisposeAsync(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Approvals.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Approvals.cs new file mode 100644 index 000000000..49375c164 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Approvals.cs @@ -0,0 +1,96 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Child_approval_retains_the_original_requester_after_a_later_parent_turn(bool cancel) + { + var session = new SessionId("signalr/background-approval"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + _child.ToolCallsOnFirstCall = [new FunctionCallContent("child-approval-1", "approval_probe", + new Dictionary { ["_rationale"] = "Request the fixture operation." })]; + MessageSource Source(string turn, string sender) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId(sender), TurnId = new TurnId(turn), + MessageId = turn, Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, + Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }; + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Start the approval fixture.", Source = Source("original", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + _childRelease.TrySetResult(); + var prompt = await subscriber.FishForMessageAsync(message => message is ToolInteractionRequest, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var request = Assert.IsType(prompt); + Assert.Equal(new SenderId("operator-a"), request.RequesterSenderId); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Answer an independent question.", Source = Source("later", "operator-b") + }, Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + ToolInteractionResponse Answer(string sender) => new() + { + SessionId = session, CallId = request.CallId, SenderId = new SenderId(sender), + SelectedKey = new ApprovalOptionKey(ApprovalOptionKeys.ApproveOnce) + }; + Assert.IsType(await manager.Ask(Answer("operator-b"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + if (cancel) + { + _main.PlannedResponses.Enqueue([new FunctionCallContent("cancel-approval", "control_probe", + new Dictionary { ["_rationale"] = "Cancel the fixture.", ["run_id"] = _start!.Prepared!.RunId.Value })]); + _main.PlannedResponses.Enqueue([new TextContent("The cancellation was admitted.")]); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Cancel the fixture.", Source = Source("cancel", "operator-a") + }, Ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => Assert.Contains(_main.ReceivedMessages.SelectMany(static messages => messages) + .SelectMany(static message => message.Contents.OfType()), + static result => result.CallId.StartsWith("child-result-", StringComparison.Ordinal)), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.IsType(await manager.Ask(Answer("operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(0, Volatile.Read(ref _approvedEffects)); + } + else + { + Assert.IsType(await manager.Ask(Answer("operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + await AwaitAssertAsync(() => Assert.Equal(1, Volatile.Read(ref _approvedEffects)), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.IsType(await manager.Ask(Answer("operator-a"), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(1, Volatile.Read(ref _approvedEffects)); + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Controls.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Controls.cs new file mode 100644 index 000000000..a063609d1 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Controls.cs @@ -0,0 +1,94 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_later_foreign_requester_cannot_read_or_cancel_the_original_child(bool cancel) + { + var session = new SessionId("signalr/background-control"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + MessageSource Source(string turn, string sender) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId(sender), TurnId = new TurnId(turn), + MessageId = turn, Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, + Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }; + await manager.Ask(new SendUserMessage + { SessionId = session, Content = "Start the control fixture.", Source = Source("original", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var runId = _start!.Prepared!.RunId; + async Task Control(string turn, string sender, bool requestCancel) + { + _main.PlannedResponses.Enqueue([new FunctionCallContent(turn, "control_probe", new Dictionary + { ["_rationale"] = "Inspect the fixture run.", ["run_id"] = runId.Value, ["cancel"] = requestCancel })]); + _main.PlannedResponses.Enqueue([new TextContent("The control request returned.")]); + await manager.Ask(new SendUserMessage + { SessionId = session, Content = "Check the fixture run.", Source = Source(turn, sender) }, + Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + } + await Control("foreign", "operator-b", cancel); + Assert.NotNull(_start.LastControlReply); + Assert.Null(_start.LastControlReply.Run); + Assert.DoesNotContain("log_path", _start.LastControlBody!, StringComparison.Ordinal); + Assert.DoesNotContain("artifact_directory", _start.LastControlBody!, StringComparison.Ordinal); + Assert.Equal(1, _child.CallCount); + await Control("original-status", "operator-a", false); + var retained = Assert.IsType(_start.LastControlReply!.Run); + Assert.Equal(BackgroundChildState.Running, retained.State); + Assert.Null(retained.CancellationRequestedAtMs); + Assert.Null(retained.DispatchClosedAtMs); + Assert.Equal(1, _child.CallCount); + using var status = JsonDocument.Parse(_start.LastControlBody!); + var logPath = status.RootElement.GetProperty("log_path").GetString()!; + var artifactDirectory = status.RootElement.GetProperty("artifact_directory").GetString()!; + Assert.True(Path.IsPathFullyQualified(logPath)); + Assert.True(Path.IsPathFullyQualified(artifactDirectory)); + var childStorage = Assert.IsType(_start.Prepared!.Execution.Scope.Authority.Session).Storage; + Assert.Equal(childStorage.LogPath.Value, logPath); + Assert.Equal(childStorage.ArtifactDirectory.Value, artifactDirectory); + Assert.True(File.Exists(logPath)); + _main.PlannedResponses.Enqueue([new FunctionCallContent("read-live-log", "live_log_read", new Dictionary + { ["_rationale"] = "Read the authorized child log.", ["Path"] = logPath })]); + _main.PlannedResponses.Enqueue([new TextContent("The live log read completed.")]); + await manager.Ask(new SendUserMessage + { SessionId = session, Content = "Read the child log.", Source = Source("read-log", "operator-a") }, + Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.IsType(_start.LastLogReceipt); + Assert.NotNull(_start.LastLogRead); + Assert.False(_childRelease.Task.IsCompleted); + Assert.Equal(1, _child.CallCount); + _childRelease.TrySetResult(); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Findings.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Findings.cs new file mode 100644 index 000000000..6133c1967 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Findings.cs @@ -0,0 +1,80 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Memory; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Configuration; +using Netclaw.Security; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerTests +{ + [Fact] + public async Task A_later_parent_input_cannot_replace_the_child_finding_authority() + { + const string findingText = "The neutral fixture contains a durable factual conclusion for the original operator."; + _start!.EmitStructuredFindings = true; + _child.PlannedResponses.Enqueue([new TextContent(findingText)]); + var session = new SessionId("signalr/background-finding-authority"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + MessageSource Source(string turn, TrustAudience audience, TrustBoundary boundary) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), + TurnId = new TurnId(turn), MessageId = turn, Audience = audience, Boundary = boundary, + Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }; + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Delegate the factual conclusion.", + Source = Source("original", TrustAudience.Personal, TrustBoundary.Personal) + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is SubAgentOutput { Phase: SubAgentPhase.Started }, + Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + + _main.PlannedResponses.Enqueue([new TextContent("The later parent request completed independently.")]); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Answer the later independent request.", + Source = Source("later", TrustAudience.Team, TrustBoundary.Team) + }, Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.False(_childRelease.Task.IsCompleted); + + _childRelease.TrySetResult(); + var completed = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is SubAgentOutput { Phase: SubAgentPhase.Completed }, Ceiling, + cancellationToken: TestContext.Current.CancellationToken)); + Assert.True(completed.Success); + Assert.Equal(1, completed.FindingsCount); + var checkpoint = await _findingCheckpoint.Finding.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.Equal("original", checkpoint.TurnId!.Value.Value); + var payload = Assert.IsType(checkpoint.Payload); + Assert.Equal(TrustAudience.Personal.ToWireValue(), payload.Audience); + Assert.Equal(TrustBoundary.Personal.Value, payload.Boundary); + Assert.Equal(findingText, payload.Content); + Assert.True(payload.HasAcceptedSubAgentFinding); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Contains(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()), + result => result.Result?.ToString()?.Contains(findingText, StringComparison.Ordinal) == true); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Recovery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Recovery.cs new file mode 100644 index 000000000..1a38cc30f --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.Recovery.cs @@ -0,0 +1,104 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Collections.Concurrent; +using System.Text.Json; +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerTests +{ + [Fact] + public async Task Recovery_commits_two_lost_terminals_at_distinct_actual_journal_positions_before_join() + { + var session = new SessionId("signalr/background-recovery"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(session.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + _main.ToolCallsOnFirstCall = new[] { "start-1", "start-2" }.Select(id => new FunctionCallContent(id, "start_probe", + new Dictionary { ["_rationale"] = "Delegate a distinct fixture." })).ToList(); + var held = NewSignal(); + var release = NewSignal(); + var starts = 0; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ChildRunEvent.Started || Interlocked.Increment(ref starts) != 2) + return false; + held.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + try + { + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Start two fixture tasks.", Source = new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), + TurnId = new TurnId("original"), MessageId = "original", Audience = TrustAudience.Personal, + Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + } + }, Ceiling, TestContext.Current.CancellationToken); + await held.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + Sys.Stop(owner); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + release.TrySetResult(); + var terminals = new ConcurrentQueue<(ChildRunEvent.TerminalRecorded Event, long Sequence)>(); + await Journal.OnWrite.FailIf(record => + { + if (record.Payload is ChildRunEvent.TerminalRecorded terminal) + terminals.Enqueue((terminal, record.SequenceNr)); + return false; + }); + var recovered = CreateTestProbe(); + manager.Tell(new JoinSession(recovered) { SessionId = session, Filter = OutputFilter.Full }, recovered.Ref); + await recovered.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(2, terminals.Count); + var committed = terminals.ToArray(); + Assert.All(committed, pair => + { + Assert.True(pair.Event.Terminal.Lost); + Assert.Equal(SubAgentOutcomeReason.OwnerRestartLost, pair.Event.Terminal.Result.OutcomeReason); + Assert.Equal(pair.Sequence, pair.Event.TerminalSequenceNr); + }); + Assert.Equal(2, committed.Select(pair => pair.Sequence).Distinct().Count()); + Assert.Equal(1, _child.CallCount); + await AwaitAssertAsync(() => + { + var results = _main.ReceivedMessages.Last().SelectMany(message => message.Contents.OfType()).ToArray(); + Assert.Single(results, result => result.CallId == "start-1"); + Assert.Single(results, result => result.CallId == "start-2"); + var deliveries = results.Where(result => result.CallId.StartsWith("child-result-", StringComparison.Ordinal)).ToArray(); + Assert.Equal(2, deliveries.Length); + Assert.All(deliveries, result => + { + using var body = JsonDocument.Parse(result.Result!.ToString()!); + Assert.Equal("Lost", body.RootElement.GetProperty("state").GetString()); + }); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, _child.CallCount); + Assert.Equal(2, terminals.Count); + } + finally { release.TrySetResult(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.cs new file mode 100644 index 000000000..f6e6dc15f --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerTests.cs @@ -0,0 +1,374 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Jobs; +using Netclaw.Actors.Memory; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Reminders; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tests.Utilities; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerTests(ITestOutputHelper output) + : PersistenceTestKit(output: output), IAsyncDisposable +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(30); + private readonly FakeChatClient _main = new(); + private readonly FakeChatClient _child = new(); + private readonly TaskCompletionSource _childRelease = NewSignal(); + private TestSessionTempDirectory? _directory; + private StartExecutor? _start; + private int _approvedEffects; + private readonly FindingCheckpointSink _findingCheckpoint = new(); + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + { + base.ConfigureAkka(builder, provider); + builder.WithNetclawSerialization() + .WithNetclawActors(provider.GetRequiredService()); + } + + protected override void ConfigureServices(HostBuilderContext context, IServiceCollection services) + { + _directory = TestSessionTempDirectory.Create("netclaw-child-owner-"); + services.AddSingleton(_directory); + services.AddSingleton(_directory.Paths); + services.AddSingleton(new FakeCapabilityResolver()); + services.AddSingleton(TimeProvider.System); + services.AddSingleton(SecurityPolicyDefaults.Resolve(null)); + services.AddSingleton(); + services.AddSingleton(new SchedulingConfig()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(NullNotificationSink.Instance); + services.AddSingleton(NullReminderChannelNotifier.Instance); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(_findingCheckpoint); + var clients = new RoleProvider(_main, _child); + services.AddSingleton(clients); + services.AddSingleton(new ModelCapabilities { ModelId = "fake-model", ContextWindowTokens = 128000 }); + services.AddSingleton(new SessionConfig + { + Tuning = new SessionTuning { SnapshotInterval = 1000, TitleGenerationInterval = 0 } + }); + var prompt = new StaticSystemPromptProvider("Use the assigned tools."); + services.AddSingleton(prompt); + var registry = new ToolRegistry(); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "start_probe"), "builtin"); + registry.RegisterCore(new FakeNetclawTool("file_read", "The neutral fixture is available.", "file", + invocation => invocation.TryComplete(new ToolInvocationReceipt.Succeeded([], null)))); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "control_probe"), "builtin"); + registry.RegisterCore(AIFunctionFactory.Create(() => "unused", "live_log_read"), "file"); + registry.RegisterCore(new FakeNetclawTool("approval_probe", "The approved fixture returned its result.", "builtin", invocation => + { + Interlocked.Increment(ref _approvedEffects); + invocation.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + })); + services.AddSingleton(registry); + var toolConfig = new ToolConfig(); + toolConfig.AudienceProfiles.Personal.ApprovalPolicy = new ToolApprovalConfig + { + ToolOverrides = new Dictionary(StringComparer.Ordinal) + { ["approval_probe"] = ToolApprovalMode.Approval } + }; + var policy = new ToolAccessPolicy(_directory.Paths, toolConfig, + new EffectivePolicyDefaults(DeploymentPosture.Personal, TrustAudience.Personal, + ShellExecutionMode.HostAllowed, UsedStrictFallback: false), + new ShellCommandPolicy(), new ToolPathPolicy([])); + var snapshots = new WorkingContextSnapshotProvider(new GitWorkingContextInspector(TimeProvider.System), + NullLogger.Instance); + var spawner = new SubAgentSpawner(clients, registry, policy, null, prompt, snapshots, + NullLogger.Instance); + _start = new StartExecutor(spawner, new FileReadTool(toolConfig, _directory.Paths, new ToolPathPolicy([]))); + services.AddSingleton(_start); + services.AddLlmSessionCompositeRecords(); + _main.ToolCallsOnFirstCall = [new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the assigned task." })]; + _child.NextResponseGate = _childRelease; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Acceptance_commit_precedes_child_provider_and_parent_result(bool failCommit) + { + _child.ToolCallsOnFirstCall = [new FunctionCallContent("child-read-1", "file_read", + new Dictionary { ["_rationale"] = "Read the assigned neutral fixture." })]; + ChildRunEvent.Checkpointed? checkpoint = null; + var entered = NewSignal(); + var release = NewSignal(); + ChildRunAccepted? accepted = null; + (ChildRunEvent.TerminalRecorded Event, long SequenceNr)? terminal = null; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is ChildRunEvent.Checkpointed childCheckpoint) + checkpoint = childCheckpoint; + if (record.Payload is ChildRunEvent.TerminalRecorded childTerminal) + terminal = (childTerminal, record.SequenceNr); + if (record.Payload is not ChildRunAccepted admission) return false; + accepted = admission; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return failCommit; + }); + var session = new SessionId("signalr/background-owner"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(session.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + async Task ActAsync() + { + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Delegate the neutral task.", Source = new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), + TurnId = new TurnId("original"), MessageId = "original", Audience = TrustAudience.Personal, + Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + } + }, Ceiling, TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.NotNull(accepted); + Assert.Equal("original", accepted.Run.OriginalContext.TurnId); + Assert.False(accepted.Run.StartBatchSettled); + Assert.Equal(0, _child.CallCount); + Assert.Equal(1, _main.CallCount); + release.TrySetResult(); + if (failCommit) + { + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(0, _child.CallCount); + Assert.Equal(1, _main.CallCount); + return; + } + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(2, _main.CallCount); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompleted); + var result = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType())); + Assert.Equal("start-1", result.CallId); + using var body = JsonDocument.Parse(result.Result!.ToString()!); + Assert.Equal("Accepted", body.RootElement.GetProperty("state").GetString()); + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("check_agent_run", body.RootElement.GetProperty("control_tool").GetString()); + Assert.DoesNotContain(_main.ReceivedMessages[^1], message => message.Text?.Contains("child final", StringComparison.Ordinal) == true); + Assert.NotNull(_start!.Prepared); + var retry = new StartBackgroundChildRun(_start.Prepared, accepted.Run.StartKey, + accepted.Run.OriginalContext, accepted.Run.SourceOperation, _start.Token); + var recorded = await owner.Ask(retry, Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(accepted.Run.RunId, Assert.IsType(recorded).Run.RunId); + Assert.Equal(1, _child.CallCount); + var conflict = retry with { Prepared = _start.Prepared with { ArgumentsDigest = new string('b', 64) } }; + Assert.IsType(await owner.Ask(conflict, Ceiling, + TestContext.Current.CancellationToken)); + var foreign = retry with { InvocationContext = retry.InvocationContext with { RequesterSenderId = new SenderId("operator-b") } }; + await Assert.ThrowsAsync(() => owner.Ask(foreign, Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(1, _child.CallCount); + _childRelease.TrySetResult(); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(3, _main.CallCount); + Assert.Equal(2, _child.CallCount); + Assert.NotNull(checkpoint); + Assert.Equal(1, checkpoint.Checkpoint.CompletedRound); + Assert.Contains("The neutral fixture is available.", checkpoint.Checkpoint.Summary, StringComparison.Ordinal); + Assert.NotNull(terminal); + Assert.Equal(terminal.Value.SequenceNr, terminal.Value.Event.TerminalSequenceNr); + var delivered = _main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()) + .Where(item => item.CallId != "start-1").ToArray(); + var childResult = Assert.Single(delivered); + using var childBody = JsonDocument.Parse(childResult.Result!.ToString()!); + Assert.Equal(accepted.Run.RunId.Value, childBody.RootElement.GetProperty("run_id").GetString()); + var childCall = Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()), + call => call.CallId == childResult.CallId); + Assert.Equal("start_probe", childCall.Name); + Assert.Single(_main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()), + item => item.CallId == "start-1"); + } + try + { + if (failCommit) + await EventFilter.Error(contains: "Failed to persist event type").ExpectOneAsync(ActAsync, + cancellationToken: TestContext.Current.CancellationToken); + else + await ActAsync(); + } + finally + { + release.TrySetResult(); + _childRelease.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Cancellation_keeps_failed_outcome_and_confines_the_atomic_report(bool linkedReport) + { + var session = new SessionId("signalr/background-cancel"); + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + MessageSource Source(string turn) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), + TurnId = new TurnId(turn), MessageId = turn, Audience = TrustAudience.Personal, + Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }; + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Delegate the neutral task.", Source = Source("original") + }, Ceiling, TestContext.Current.CancellationToken); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await subscriber.FishForMessageAsync(message => message is TurnCompleted, Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + var prepared = Assert.IsType(_start!.Prepared); + var storage = Assert.IsType(prepared.Execution.Scope.Authority.Session).Storage; + var artifact = storage.ArtifactDirectory.Value; + var report = Path.Combine(artifact, "cancelled-results.json"); + var outside = Path.Combine(_directory!.Path, "outside-marker"); + await File.WriteAllTextAsync(outside, "outside unchanged", TestContext.Current.CancellationToken); + if (linkedReport) + { + Directory.CreateDirectory(artifact); + File.CreateSymbolicLink(report, outside); + } + _main.PlannedResponses.Enqueue([new FunctionCallContent("cancel-1", "control_probe", + new Dictionary { ["_rationale"] = "Cancel the accepted child.", ["run_id"] = prepared.RunId.Value })]); + _main.PlannedResponses.Enqueue([new TextContent("The cancellation was admitted.")]); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Cancel that child.", Source = Source("cancel-turn") + }, Ceiling, TestContext.Current.CancellationToken); + FunctionResultContent? delivered = null; + await AwaitAssertAsync(() => + { + delivered = _main.ReceivedMessages.SelectMany(messages => messages) + .SelectMany(message => message.Contents.OfType()) + .LastOrDefault(result => result.CallId.StartsWith("child-result-", StringComparison.Ordinal)); + Assert.NotNull(delivered); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + using var body = JsonDocument.Parse(delivered!.Result!.ToString()!); + Assert.Equal("Cancelled", body.RootElement.GetProperty("state").GetString()); + Assert.Equal("Failed", body.RootElement.GetProperty("outcome").GetString()); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent.Value, body.RootElement.GetProperty("reason").GetString()); + Assert.Equal(1, _child.CallCount); + Assert.True(_childRelease.Task.IsCanceled); + Assert.Equal("outside unchanged", await File.ReadAllTextAsync(outside, TestContext.Current.CancellationToken)); + if (linkedReport) + Assert.Contains("No report path is confirmed", body.RootElement.GetProperty("warning").GetString(), StringComparison.Ordinal); + else + { + using var saved = JsonDocument.Parse(await File.ReadAllTextAsync(report, TestContext.Current.CancellationToken)); + Assert.Equal("Cancelled", saved.RootElement.GetProperty("state").GetString()); + Assert.Contains(report, body.RootElement.GetProperty("output").GetString(), StringComparison.Ordinal); + } + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class StartExecutor(SubAgentSpawner spawner, FileReadTool fileRead) : IToolExecutor + { + public PreparedChildRun? Prepared { get; private set; } + public CancellationToken Token { get; private set; } + public ChildControlReply? LastControlReply { get; private set; } + public bool EmitStructuredFindings { get; set; } + public string? LastControlBody { get; private set; } + public string? LastLogRead { get; private set; } + public ToolInvocationReceipt? LastLogReceipt { get; private set; } + public async Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) + { + var factory = context.Invocation.SpawnChildActor + ?? throw new InvalidOperationException("The start test requires the owner factory."); + if (call.Name == "live_log_read") + { + LastLogRead = await fileRead.ExecuteAsync(call.Arguments!, context, ct); + LastLogReceipt = context.Outputs.Receipt; + return LastLogRead; + } + if (call.Name == "control_probe") + { + var controlScope = context.RunScope with { SpawnChildActor = async (payload, name, token) => + { + var reply = Assert.IsType(await factory(payload, name, token)); + LastControlReply = reply; + return reply; + }}; + LastControlBody = await new CheckAgentRunTool().ExecuteAsync(new Dictionary + { + ["runId"] = call.Arguments!["run_id"], + ["cancel"] = !call.Arguments.TryGetValue("cancel", out var cancel) || cancel is true + }, new ToolExecutionContext(controlScope, context.ExecutionTimeout, context.Outputs), ct); + return LastControlBody; + } + var scope = context.RunScope with { SpawnChildActor = async (payload, name, token) => + { + Prepared = Assert.IsType(payload); + Token = token; + return await factory(payload, name, token); + }}; + return await spawner.StartRunAsync(new SubAgentProfile + { + Name = "worker", Description = "Complete the neutral task.", SystemPrompt = "Complete the assigned task.", + ModelRole = ModelRole.Compaction, ToolNames = ["file_read", "approval_probe"], EmitStructuredFindings = EmitStructuredFindings + }, "Inspect the neutral fixture.", null, + new ToolInvocationContext(scope, context.ExecutionTimeout, context.Outputs), ct, null); + } + } + + private sealed class RoleProvider(IChatClient main, IChatClient child) : IChatClientProvider + { + public IChatClient GetClient(ModelRole role) => role == ModelRole.Main ? main : child; + } + + private sealed class FindingCheckpointSink : IMemoryCheckpointSink + { + public TaskCompletionSource Finding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task EnqueueAsync(MemoryCheckpointRequest request, CancellationToken ct = default) + { + if (request.TriggerType == CheckpointTriggerType.SubagentFindings) + Finding.TrySetResult(request); + return Task.FromResult(new MemoryCheckpointEnqueueResult("fixture-checkpoint", 0)); + } + } + + async ValueTask IAsyncDisposable.DisposeAsync() + { + _childRelease.TrySetResult(); + try { await base.DisposeAsync(); } + finally { if (_directory is not null) await _directory.DisposeAsync(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildReportGraceAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildReportGraceAdversarialTests.cs new file mode 100644 index 000000000..303ef8a7d --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildReportGraceAdversarialTests.cs @@ -0,0 +1,83 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Sessions; +using Xunit; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed class BackgroundChildReportGraceAdversarialTests +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(30); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Independent_grace_settles_without_wait_for_an_uncooperative_report_operation(bool synchronousHold) + { + var time = new FakeTimeProvider(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var asyncRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var syncRelease = new ManualResetEventSlim(); + async Task Operation(CancellationToken token) + { + entered.TrySetResult(token); + try + { + if (synchronousHold) + syncRelease.Wait(TestContext.Current.CancellationToken); + else + await asyncRelease.Task; + } + finally { completed.TrySetResult(); } + } + var finalization = LlmSessionActor.CompleteChildReportWithinGraceAsync(Operation, time); + try + { + var token = await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.False(token.IsCancellationRequested); + time.Advance(TimeSpan.FromSeconds(4)); + Assert.False(finalization.IsCompleted); + Assert.False(completed.Task.IsCompleted); + time.Advance(TimeSpan.FromSeconds(1)); + await Assert.ThrowsAnyAsync(() => finalization.WaitAsync(Ceiling, + TestContext.Current.CancellationToken)); + Assert.True(token.IsCancellationRequested); + Assert.False(completed.Task.IsCompleted); + } + finally + { + syncRelease.Set(); + asyncRelease.TrySetResult(); + await completed.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_report_operation_preserves_its_actual_success_or_failure(bool fail) + { + var time = new FakeTimeProvider(); + var executed = false; + var expected = new IOException("The report writer failed before an asynchronous boundary."); + Task Operation(CancellationToken token) + { + Assert.False(token.IsCancellationRequested); + executed = true; + if (fail) throw expected; + return Task.CompletedTask; + } + var finalization = LlmSessionActor.CompleteChildReportWithinGraceAsync(Operation, time); + if (fail) + Assert.Same(expected, await Assert.ThrowsAsync(() => finalization.WaitAsync(Ceiling, + TestContext.Current.CancellationToken))); + else + await finalization.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + Assert.True(executed); + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs index a1b9c0d08..39ed65b7c 100644 --- a/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs @@ -2917,7 +2917,7 @@ internal sealed class ControllableWorkingContextSnapshotProvider : IWorkingConte { public Exception? Failure { get; set; } public TaskCompletionSource? Pending { get; set; } - public TaskCompletionSource InvocationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource InvocationStarted { get; set; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task CreateAsync( WorkingContext context, diff --git a/src/Netclaw.Actors.Tests/Sessions/Pipelines/BackgroundChildStampedAcceptanceTests.cs b/src/Netclaw.Actors.Tests/Sessions/Pipelines/BackgroundChildStampedAcceptanceTests.cs new file mode 100644 index 000000000..7890365bc --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/Pipelines/BackgroundChildStampedAcceptanceTests.cs @@ -0,0 +1,171 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Akka.Event; +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Pipelines; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Actors.Tests.Memory; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tests.Utilities; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions.Pipelines; + +public sealed class BackgroundChildStampedAcceptanceTests(ITestOutputHelper output) : TestKit(output: output), IAsyncDisposable +{ + private readonly TestSessionTempDirectory _directory = TestSessionTempDirectory.Create("netclaw-stamped-acceptance-"); + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(10); + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) { } + + [Theory] + [InlineData("requester")] + [InlineData("call")] + [InlineData("operation")] + [InlineData("valid")] + public async Task The_actual_pipeline_checks_the_owner_reply_against_the_stamped_start_before_returning_acceptance(string variant) + { + const string callId = "canonical-start-call"; + var session = new SessionId("signalr/stamped-start-owner"); + var context = TurnContext.FromMessageSource(session, new TurnId("original-turn"), new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("original-requester"), + Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }); + var registry = new ToolRegistry(); + registry.RegisterCore(new FakeNetclawTool("file_read", "unused", "builtin")); + var policy = new ToolAccessPolicy(_directory.Paths, new ToolConfig(), + new EffectivePolicyDefaults(DeploymentPosture.Personal, TrustAudience.Personal, + ShellExecutionMode.HostAllowed, UsedStrictFallback: false), + new ShellCommandPolicy(), new ToolPathPolicy([])); + var client = new FakeChatClient(); + var snapshots = new ControllableWorkingContextSnapshotProvider(); + var spawner = new SubAgentSpawner(new StampedClientProvider(client), registry, policy, null, + new StaticSystemPromptProvider("Use the assigned tools."), snapshots, NullLogger.Instance); + var profiles = new SubAgentDefinitionRegistry(); + profiles.Register(new SubAgentProfile + { + Name = "worker", Description = "Complete the assigned task.", SystemPrompt = "Inspect the neutral fixture.", + ModelRole = ModelRole.Compaction, ToolNames = ["file_read"], Visibility = SubAgentVisibility.UserFacing, + EmitStructuredFindings = false + }); + var executor = new StampedSpawnExecutor(new SpawnAgentTool(profiles, spawner, _directory.Paths)); + var probe = CreateTestProbe(); + BackgroundChildRun? returned = null; + var requests = new List(); + var pipeline = new SessionToolExecutionPipeline(executor, TimeProvider.System, NoLogger.Instance); + var batch = new SessionToolBatch(context, new SessionToolRunEnvironment + { + RecurrenceTaskId = new TurnId("original-task"), + Storage = SessionStoragePaths.CreateVersion2(new SessionStorageEnvelopeRoot(_directory.Path)), + InlineOutputBudget = InlineOutputBudget.Default, + SpawnChildActor = (payload, _, _) => + { + var request = Assert.IsType(payload); + requests.Add(request); + Assert.True(SessionState.SameCanonicalContext(context.ToRecord(), request.InvocationContext)); + Assert.Equal(new ChildRunStartKey.Tool(new ToolCallId(callId)) + { SessionId = session, TurnId = context.TurnId }, request.StartKey); + Assert.Equal("spawn_agent", request.SourceOperation); + var id = new SubAgentRunId("existing-recorded-run"); + Assert.NotEqual(id, request.Prepared.RunId); + var run = new BackgroundChildRun + { + RunId = id, AgentName = request.Prepared.AgentName, + ScopeId = new SubAgentScopeId($"{session.Value}/subagent/{request.Prepared.AgentName.Value}/{id.Value}"), + ArgumentsDigest = request.Prepared.ArgumentsDigest, StartKey = request.StartKey, + SourceOperation = request.SourceOperation, OriginalContext = request.InvocationContext, + OriginInputIds = [new InputId("original-input")], + InitialWorkingSnapshot = request.Prepared.Execution.Scope.InitialWorkingSnapshot, + ParentCheckpoint = new ToolLoopCheckpoint { TaskId = "original-task" }, + AcceptedAtMs = 1, StartedAtMs = 2 + }; + returned = variant switch + { + "requester" => run with { OriginalContext = run.OriginalContext with { RequesterSenderId = new SenderId("foreign-requester") } }, + "call" => run with { StartKey = new ChildRunStartKey.Tool(new ToolCallId("foreign-start-call")) + { SessionId = session, TurnId = context.TurnId } }, + "operation" => run with { SourceOperation = "skill_load" }, + "valid" => run, + _ => throw new ArgumentOutOfRangeException(nameof(variant)) + }; + // These replies are internally valid. Only the actual stamped wrapper can reject their correlation. + returned.Validate(); + return Task.FromResult(new ChildStartReply.Accepted(returned, BackgroundChildState.Running)); + } + }) + { + ToolCalls = [new FunctionCallContent(callId, "spawn_agent", new Dictionary + { ["Agent"] = "worker", ["Task"] = "Inspect the neutral fixture.", ["_rationale"] = "Delegate the assigned task." })], + DefaultTimeout = new ToolExecutionTimeout(Ceiling), ReplyTo = probe.Ref, EmitSubAgentOutput = _ => { }, + ApprovalRequests = new ToolApprovalRequests(new ApprovalChannel(), _ => { }, new ToolExecutionTimeout(Timeout.InfiniteTimeSpan)), + BackgroundJobs = new BackgroundJobDispatch.Unavailable(), CancellationToken = TestContext.Current.CancellationToken + }; + await pipeline.ExecuteAsync(batch); + var completed = await probe.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.Single(requests); + Assert.Equal(1, executor.CallCount); + Assert.Equal(0, client.CallCount); + Assert.NotNull(returned); + var result = Assert.Single(completed.ToolResults); + Assert.Equal(new ToolCallId(callId), result.ToolCallId); + var receipt = Assert.Single(completed.ToolReceipts); + Assert.Equal(callId, receipt.Key); + Assert.Same(executor.Outputs!.Receipt, receipt.Value); + Assert.Empty(completed.CompletedSubAgentRuns); + if (variant == "valid") + { + Assert.IsType(receipt.Value); + using var body = JsonDocument.Parse(result.Content); + Assert.Equal(returned.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal(returned.ScopeId.Value, body.RootElement.GetProperty("scope_id").GetString()); + Assert.Equal("Running", body.RootElement.GetProperty("state").GetString()); + Assert.Equal("check_agent_run", body.RootElement.GetProperty("control_tool").GetString()); + return; + } + Assert.IsType(receipt.Value); + Assert.Equal(ToolInvocationOutcomeCategory.TransientFailure, receipt.Value.Category); + Assert.StartsWith("Error executing tool:", result.Content, StringComparison.Ordinal); + Assert.DoesNotContain(returned.RunId.Value, result.Content, StringComparison.Ordinal); + Assert.DoesNotContain("\"run_id\"", result.Content, StringComparison.Ordinal); + } + + private sealed class StampedSpawnExecutor(SpawnAgentTool tool) : IToolExecutor + { + public int CallCount { get; private set; } + public ToolExecutionOutputs? Outputs { get; private set; } + public Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) + { + CallCount++; + Outputs = context.Outputs; + return tool.ExecuteAsync(call.Arguments, context.Invocation, ct); + } + } + + private sealed class StampedClientProvider(IChatClient client) : IChatClientProvider + { + public IChatClient GetClient(ModelRole role) => client; + } + + async ValueTask IAsyncDisposable.DisposeAsync() + { + try { await base.DisposeAsync(); } + finally { await _directory.DisposeAsync(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.AcceptanceReplies.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.AcceptanceReplies.cs new file mode 100644 index 000000000..7ce7ac992 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.AcceptanceReplies.cs @@ -0,0 +1,93 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Configuration; +using Akka.Routing; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public partial class SubAgentSpawnIntegrationTests +{ + [Theory] + [InlineData("requester")] + [InlineData("input")] + [InlineData("operation")] + public async Task The_actual_slash_adapter_rejects_a_coherent_reply_for_another_stamped_activation(string defect) + { + var ceiling = TimeSpan.FromSeconds(10); + var session = new SessionId("test-channel/slash-acceptance-correlation"); + const string mailbox = "slash-acceptance-reply-gate"; + Sys.Settings.InjectTopLevelFallback(ConfigurationFactory.ParseString( + $"{mailbox} {{ mailbox-type = \"{typeof(BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox).AssemblyQualifiedName}\" }}")); + ((ExtendedActorSystem)Sys).Provider.Deployer.SetDeploy(new Deploy( + $"/session-manager/{Uri.EscapeDataString(session.Value)}", Config.Empty, NoRouter.Instance, LocalScope.Instance, + Deploy.NoDispatcherGiven, mailbox)); + var gate = BackgroundChildApprovalGrantAdversarialTests.WaitReplyMailbox.Gates.GetOrCreateValue(Sys); + gate.MessageName = nameof(StartBackgroundChildRun); + gate.Arm(); + try + { + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + await JoinSessionAsync(manager, subscriber, session, OutputFilter.Full | OutputFilter.ProcessingState); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "/ops-route inspect the neutral fixture", Source = BuildPersonalSource() + }, ceiling, TestContext.Current.CancellationToken); + var captured = await gate.Held.Task.WaitAsync(ceiling, TestContext.Current.CancellationToken); + var request = Assert.IsType(captured.Envelope.Message); + var activation = Assert.IsType(request.StartKey); + Assert.Equal(session, activation.SessionId); + Assert.Equal("/ops-route", request.SourceOperation); + var id = new SubAgentRunId("recorded-slash-run"); + var run = new BackgroundChildRun + { + RunId = id, AgentName = request.Prepared.AgentName, + ScopeId = new SubAgentScopeId($"{session.Value}/subagent/{request.Prepared.AgentName.Value}/{id.Value}"), + ArgumentsDigest = request.Prepared.ArgumentsDigest, StartKey = request.StartKey, + SourceOperation = request.SourceOperation, OriginalContext = request.InvocationContext, + OriginInputIds = [activation.InputId], InitialWorkingSnapshot = request.Prepared.Execution.Scope.InitialWorkingSnapshot, + ParentCheckpoint = new ToolLoopCheckpoint { TaskId = "original-slash-task" }, + StartBatchSettled = true, AcceptedAtMs = 1, StartedAtMs = 2 + }; + run = defect switch + { + "requester" => run with { OriginalContext = run.OriginalContext with { RequesterSenderId = new SenderId("foreign-requester") } }, + "input" => run with { StartKey = new ChildRunStartKey.Slash(new InputId("foreign-input")) + { SessionId = session, TurnId = activation.TurnId }, OriginInputIds = [new InputId("foreign-input")] }, + "operation" => run with { SourceOperation = "/other-route" }, + _ => throw new ArgumentOutOfRangeException(nameof(defect)) + }; + // The held owner request cannot execute. Its Ask sender still exercises the actual slash reply closure. + run.Validate(); + captured.Envelope.Sender.Tell(new ChildStartReply.Accepted(run, BackgroundChildState.Running), captured.Receiver); + var outputs = new List(); + await subscriber.FishForMessageAsync(message => + { + outputs.Add(message); + return message is ProcessingStateOutput { IsProcessing: false } && outputs.OfType().Any(); + }, ceiling, cancellationToken: TestContext.Current.CancellationToken); + var failure = Assert.Single(outputs.OfType()); + Assert.Equal(ErrorCategory.ToolFailure, failure.Category); + Assert.Contains("routed to subagent", failure.Message, StringComparison.Ordinal); + Assert.Equal(TurnOutcome.Failed, Assert.Single(outputs.OfType()).Outcome); + Assert.Empty(outputs.OfType()); + Assert.Empty(outputs.OfType()); + Assert.Equal(0, _clientProvider.Main.CallCount); + Assert.Equal(0, _clientProvider.Compaction.CallCount); + var identity = await Sys.ActorSelection($"{captured.Receiver.Path}/child-run-{request.Prepared.RunId.Value}") + .Ask(new Identify("no-uncommitted-child"), ceiling, TestContext.Current.CancellationToken); + Assert.Null(identity.Subject); + } + finally { gate.Release(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.Recovery.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.Recovery.cs new file mode 100644 index 000000000..ce1ca3e17 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.Recovery.cs @@ -0,0 +1,107 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka; +using Akka.Actor; +using Akka.Configuration; +using Akka.Hosting; +using Akka.Persistence; +using Akka.Routing; +using ModelChatRole = Microsoft.Extensions.AI.ChatRole; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Reminders; +using Netclaw.Configuration; +using Netclaw.Security; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public partial class SubAgentSpawnIntegrationTests +{ + [Fact] + public async Task A_recovered_direct_slash_keeps_its_original_context_before_a_later_input() + { + var ceiling = TimeSpan.FromSeconds(10); + var session = new SessionId("test-channel/recovered-direct-slash"); + Sys.Settings.InjectTopLevelFallback(ConfigurationFactory.ParseString( + $"slash-recovery-capture {{ mailbox-type = \"{typeof(ToolRecurrenceAdversarialTests.ReplyCaptureMailbox).AssemblyQualifiedName}\" }}")); + ((ExtendedActorSystem)Sys).Provider.Deployer.SetDeploy(new Deploy( + $"/session-manager/{Uri.EscapeDataString(session.Value)}", Config.Empty, NoRouter.Instance, LocalScope.Instance, + Deploy.NoDispatcherGiven, "slash-recovery-capture")); + MessageSource Source(string turn, string requester) => new() + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId(requester), TurnId = new TurnId(turn), MessageId = turn, + Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted) + }; + InputAdmitted Input(string id, string content, string requester) => new() + { + SessionId = session, InputId = new InputId(id), ExecutableText = content, + UserMessage = new SerializableChatMessage { Role = Netclaw.Actors.Protocol.ChatRole.User, Content = content }, + TurnContext = TurnContext.FromMessageSource(session, new TurnId(id), Source(id, requester)).ToRecord() + }; + var original = Input("original-command", "/ops-route inspect the original neutral task", "operator-a"); + var later = Input("later-request", "Answer the later neutral request.", "operator-b"); + var seed = Sys.ActorOf(Props.Create(() => new PendingSlashSeed($"session-{session.Value}"))); + await seed.Ask(original, ceiling, TestContext.Current.CancellationToken); + await seed.Ask(later, ceiling, TestContext.Current.CancellationToken); + var watcher = CreateTestProbe(); + watcher.Watch(seed); + Sys.Stop(seed); + await watcher.ExpectTerminatedAsync(seed, ceiling, cancellationToken: TestContext.Current.CancellationToken); + + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _clientProvider.Compaction.NextResponseGate = gate; + var manager = ActorRegistry.Get(); + var subscriber = CreateTestProbe(); + manager.Tell(new JoinSession(subscriber) { SessionId = session, Filter = OutputFilter.Full }, subscriber.Ref); + await subscriber.ExpectMsgAsync(ceiling, cancellationToken: TestContext.Current.CancellationToken); + await manager.Ask(new SendUserMessage + { + SessionId = session, Content = "Resume the work that was interrupted by the daemon restart.", + Source = Source("restart-resume-neutral:1", "reminder-system") with + { ReminderId = new ReminderId("restart-resume-neutral:1"), Principal = PrincipalClassification.VerifiedAutomation } + }, ceiling, TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, ceiling, TestContext.Current.CancellationToken); + await _clientProvider.Compaction.FirstCallEntered.Task.WaitAsync(ceiling, TestContext.Current.CancellationToken); + var start = Assert.IsType(Assert.Single( + ToolRecurrenceAdversarialTests.ReplyCaptureMailbox.Captures.GetOrCreateValue(Sys), + envelope => envelope.Message is StartBackgroundChildRun).Message); + Assert.Equivalent(original.TurnContext, start.InvocationContext, strict: true); + Assert.Equal(original.InputId, Assert.IsType(start.StartKey).InputId); + Assert.Equal(original.TurnContext.TurnId, start.StartKey.TurnId.Value); + Assert.NotEqual(later.TurnContext.TurnId, start.InvocationContext.TurnId); + Assert.Contains(_clientProvider.Compaction.ReceivedMessages[0], message => message.Role == ModelChatRole.User + && message.Text?.EndsWith("Task:\ninspect the original neutral task", StringComparison.Ordinal) == true); + Assert.Contains(_clientProvider.Main.ReceivedMessages[0], message => message.Role == ModelChatRole.User + && message.Text?.Contains("Answer the later neutral request.", StringComparison.Ordinal) == true); + Assert.Equal(1, _clientProvider.Main.CallCount); + Assert.Equal(1, _clientProvider.Compaction.CallCount); + Assert.False(gate.Task.IsCompleted); + gate.TrySetResult(); + await ExpectTurnNumberAsync(subscriber, 3, ceiling, TestContext.Current.CancellationToken); + Assert.Equal(2, _clientProvider.Main.CallCount); + Assert.Equal(1, _clientProvider.Compaction.CallCount); + } + + private sealed class PendingSlashSeed : ReceivePersistentActor + { + public override string PersistenceId { get; } + public PendingSlashSeed(string persistenceId) + { + PersistenceId = persistenceId; + RecoverAny(_ => { }); + Command(input => + { + var reply = Sender; + Persist(input, _ => reply.Tell(Done.Instance)); + }); + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index 44fde9b90..a092122da 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -29,7 +29,7 @@ namespace Netclaw.Actors.Tests.Sessions; -public class SubAgentSpawnIntegrationTests : LlmSessionTestBase +public partial class SubAgentSpawnIntegrationTests : LlmSessionTestBase { private const string ApprovalProbeToolName = "approval_probe"; private const string HiddenSpecialtyToolName = "hidden_specialty"; @@ -216,6 +216,7 @@ You specialize in daemon health checks. registry.RegisterCore(new SpawnAgentTool(subAgentRegistry, spawner, subAgentPaths)); registry.RegisterCore(new SearchToolsTool(registry, toolAccessPolicy)); registry.RegisterCore(new LoadToolTool(registry, toolAccessPolicy)); + registry.Register(new CheckAgentRunTool()); registry.RegisterCore(new AttachFileTool(toolConfig, subAgentPaths, new ToolPathPolicy([]))); _recordingFileReadTool = new RecordingContextTool("file_read", "stub file content", "file"); registry.RegisterCore(_recordingFileReadTool); @@ -237,6 +238,8 @@ You specialize in daemon health checks. [Fact] public async Task Spawn_agent_runs_under_session_and_emits_subagent_events() { + var childRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _clientProvider.Compaction.NextResponseGate = childRelease; _clientProvider.Main.ToolCallsOnFirstCall = [ CreateToolCall( @@ -275,15 +278,9 @@ await sessionManager.Ask(new SendUserMessage Assert.Equal("summarizer", started.AgentName.Value); Assert.Equal(4, started.ToolCount); - var completed = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(SubAgentPhase.Completed, completed.Phase); - Assert.Equal("summarizer", completed.AgentName.Value); - Assert.True(completed.Success); - Assert.Equal(0, completed.FindingsCount); - Assert.Null(completed.MemoryDecision); - - // Drain the tool result output for spawn_agent emitted after tool execution - await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); + var acceptance = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); + using var accepted = JsonDocument.Parse(acceptance.Result); + Assert.Equal("Accepted", accepted.RootElement.GetProperty("state").GetString()); var text = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); Assert.Contains("fake", text.Text, StringComparison.OrdinalIgnoreCase); @@ -292,6 +289,20 @@ await sessionManager.Ask(new SendUserMessage Assert.Equal(2, _clientProvider.Main.CallCount); Assert.Equal(1, _clientProvider.Compaction.CallCount); + Assert.False(childRelease.Task.IsCompleted); + childRelease.TrySetResult(); + var completed = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is SubAgentOutput { Phase: SubAgentPhase.Completed }, TimeSpan.FromSeconds(3), + cancellationToken: TestContext.Current.CancellationToken)); + Assert.Equal("summarizer", completed.AgentName.Value); + Assert.True(completed.Success); + Assert.Equal(0, completed.FindingsCount); + Assert.Null(completed.MemoryDecision); + await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + Assert.Equal(3, _clientProvider.Main.CallCount); + var parentPairs = _clientProvider.Main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()); + Assert.Single(parentPairs, result => result.CallId == "call-spawn"); + Assert.Single(parentPairs, result => result.CallId.StartsWith("child-result-", StringComparison.Ordinal)); var subagentCall = Assert.Single(_clientProvider.Compaction.ReceivedMessages); Assert.Contains(subagentCall, m => @@ -365,8 +376,8 @@ await sessionManager.Ask(new SendUserMessage Content = "Use the summarizer.", Source = BuildPersonalSource() }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -440,8 +451,8 @@ await sessionManager.Ask(new SendUserMessage Content = "Use one deferred child tool.", Source = BuildPersonalSource() }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -559,8 +570,8 @@ await sessionManager.Ask(new SendUserMessage Content = "Use a child to check unavailable tools.", Source = BuildPersonalSource() }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -671,9 +682,9 @@ await subscriber.ExpectMsgAsync( cancellationToken: TestContext.Current.CancellationToken); for (var i = 0; i < 2; i++) { - var approval = await subscriber.ExpectMsgAsync( - TimeSpan.FromSeconds(3), - cancellationToken: TestContext.Current.CancellationToken); + var approval = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolInteractionRequest, TimeSpan.FromSeconds(3), + cancellationToken: TestContext.Current.CancellationToken)); Assert.Equal(ShellTool.ToolName, approval.ToolName.Value); var denied = await sessionManager.Ask(new ToolInteractionResponse { @@ -684,8 +695,8 @@ await subscriber.ExpectMsgAsync( }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.IsType(denied); } - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -723,6 +734,7 @@ public async Task Loaded_child_tool_does_not_transfer_to_the_next_child() ["task"] = "Use one synthetic tool." }) ]); + _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("The child start was accepted.")]); _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("First child completed.")]); _clientProvider.Main.PlannedResponses.Enqueue( [ @@ -761,6 +773,7 @@ await sessionManager.Ask(new JoinSession(subscriber) await subscriber.ExpectMsgAsync( cancellationToken: TestContext.Current.CancellationToken); + var expectedTurn = 2; foreach (var content in new[] { "Run the first child.", "Run the second child." }) { await sessionManager.Ask(new SendUserMessage @@ -769,10 +782,11 @@ await sessionManager.Ask(new SendUserMessage Content = content, Source = BuildPersonalSource() }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, expectedTurn, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + expectedTurn += 2; } Assert.True(_clientProvider.Compaction.ReceivedToolNames.Count >= 3); @@ -808,6 +822,7 @@ public async Task Child_native_correction_exposes_one_tool_only_until_child_comp ["task"] = "Inspect with the native probe." }) ]); + _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("The child start was accepted.")]); _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("First child returned.")]); _clientProvider.Main.PlannedResponses.Enqueue( [ @@ -865,9 +880,9 @@ await subscriber.ExpectMsgAsync( await subscriber.ExpectMsgAsync( TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); - var approval = await subscriber.ExpectMsgAsync( - TimeSpan.FromSeconds(3), - cancellationToken: TestContext.Current.CancellationToken); + var approval = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolInteractionRequest, TimeSpan.FromSeconds(3), + cancellationToken: TestContext.Current.CancellationToken)); Assert.Equal(deferredToolName, approval.ToolName.Value); var denied = await sessionManager.Ask(new ToolInteractionResponse { @@ -877,8 +892,8 @@ await subscriber.ExpectMsgAsync( SenderId = source.SenderId! }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.IsType(denied); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -888,8 +903,8 @@ await sessionManager.Ask(new SendUserMessage Content = "Run a fresh child.", Source = source }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, 4, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -932,6 +947,7 @@ public async Task Child_model_failure_discards_loaded_tools_before_a_fresh_child ["task"] = "Load the failure probe." }) ]); + _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("The child start was accepted.")]); _clientProvider.Main.PlannedResponses.Enqueue([new TextContent("Failure was recorded.")]); _clientProvider.Main.PlannedResponses.Enqueue( [ @@ -973,6 +989,7 @@ await sessionManager.Ask(new JoinSession(subscriber) await subscriber.ExpectMsgAsync( cancellationToken: TestContext.Current.CancellationToken); + var expectedTurn = 2; foreach (var content in new[] { "Run the failing child.", "Run the fresh child." }) { await sessionManager.Ask(new SendUserMessage @@ -981,18 +998,18 @@ await sessionManager.Ask(new SendUserMessage Content = content, Source = BuildPersonalSource() }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync( - subscriber, + await ExpectTurnNumberAsync( + subscriber, expectedTurn, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + expectedTurn += 2; } Assert.True(failureProbe.WasCalled); Assert.Contains( "synthetic child model failure", - GetToolResult( - _clientProvider.Main.ReceivedMessages[1], - "call-failure-spawn-1"), + string.Join("\n", _clientProvider.Main.ReceivedMessages[2].SelectMany(message => message.Contents.OfType()) + .Where(result => result.CallId != "call-failure-spawn-1").Select(result => result.Result?.ToString())), StringComparison.Ordinal); Assert.Equal( ["file_read", "load_tool", "search_tools"], @@ -1072,9 +1089,10 @@ await sessionManager.Ask(new SendUserMessage var started = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(SubAgentPhase.Started, started.Phase); - var request = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); + var request = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolInteractionRequest, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken)); Assert.NotEqual(childCallId, request.CallId.Value); - Assert.StartsWith($"{parentCallId}/subagent-approval/", request.CallId.Value, StringComparison.Ordinal); + Assert.True(Guid.TryParseExact(request.CallId.Value.Split("/subagent-approval/", StringSplitOptions.None)[0], "N", out _)); Assert.Contains("subagent-approval", request.CallId.Value, StringComparison.Ordinal); Assert.DoesNotContain(childCallId, request.CallId.Value, StringComparison.Ordinal); AssertApprovalButtonValuesRoundTrip(request); @@ -1092,15 +1110,16 @@ await sessionManager.Ask(new SendUserMessage }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.IsType(approvalReply); - var completed = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); + var completed = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is SubAgentOutput { Phase: SubAgentPhase.Completed }, TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken)); Assert.Equal(SubAgentPhase.Completed, completed.Phase); Assert.True(completed.Success); - - var result = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal("spawn_agent", result.ToolName.Value); - - await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); - await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + using var acceptance = JsonDocument.Parse(GetToolResult(_clientProvider.Main.ReceivedMessages[^1], parentCallId)); + Assert.Equal("Accepted", acceptance.RootElement.GetProperty("state").GetString()); + Assert.StartsWith($"{acceptance.RootElement.GetProperty("run_id").GetString()}/subagent-approval/", + request.CallId.Value, StringComparison.Ordinal); Assert.NotNull(_recordingApprovalTool); Assert.True(_recordingApprovalTool!.WasCalled); @@ -1157,7 +1176,8 @@ await sessionManager.Ask(new SendUserMessage await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); - var request = await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); + var request = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolInteractionRequest, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken)); Assert.Contains("subagent-approval", request.CallId.Value, StringComparison.Ordinal); Assert.DoesNotContain("call-subagent-approval-expire", request.CallId.Value, StringComparison.Ordinal); AssertApprovalButtonValuesRoundTrip(request); @@ -1183,7 +1203,9 @@ await sessionManager.Ask(new JoinSession(subscriberB) var nack = Assert.IsType(reply); Assert.Equal(ApprovalNackReasons.PromptExpired, nack.Reason); - var notice = await subscriberB.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); + var notice = Assert.IsType(await subscriberB.FishForMessageAsync( + message => message is TextOutput text && text.Text.Contains("expired", StringComparison.OrdinalIgnoreCase), + TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken)); Assert.Contains("expired", notice.Text, StringComparison.OrdinalIgnoreCase); Assert.False(_recordingApprovalTool.WasCalled); @@ -1197,12 +1219,16 @@ await sessionManager.Ask(new SendUserMessage await subscriberB.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); await subscriberB.ExpectMsgAsync(TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); - var resumedCall = _clientProvider.Main.ReceivedMessages[^1]; - Assert.Contains(resumedCall, message => - message.Role == Microsoft.Extensions.AI.ChatRole.Tool - && message.Contents.OfType().Any(result => - result.CallId == "call-spawn-approval-expire" - && result.Result?.ToString()?.Contains("session restarted", StringComparison.OrdinalIgnoreCase) == true)); + await AwaitAssertAsync(() => + { + Assert.Contains(_clientProvider.Main.ReceivedMessages.SelectMany(messages => messages) + .SelectMany(message => message.Contents.OfType()), result => + result.CallId != "call-spawn-approval-expire" + && result.Result?.ToString()?.Contains("session owner restarted", StringComparison.OrdinalIgnoreCase) == true); + return Task.CompletedTask; + }, TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken); + using var acceptance = JsonDocument.Parse(GetToolResult(_clientProvider.Main.ReceivedMessages[^1], "call-spawn-approval-expire")); + Assert.Equal("Accepted", acceptance.RootElement.GetProperty("state").GetString()); } [Fact] @@ -1256,10 +1282,12 @@ await sessionManager.Ask(new SendUserMessage }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); var text = await ExpectTextOutputAsync(subscriber, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Contains("fake", text.Text, StringComparison.OrdinalIgnoreCase); - await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + using var acceptance = JsonDocument.Parse(text.Text); + Assert.Equal("Accepted", acceptance.RootElement.GetProperty("state").GetString()); + Assert.Equal(4, acceptance.RootElement.EnumerateObject().Count()); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - Assert.Equal(0, _clientProvider.Main.CallCount); + Assert.Equal(1, _clientProvider.Main.CallCount); Assert.Equal(1, _clientProvider.Compaction.CallCount); var subagentCall = Assert.Single(_clientProvider.Compaction.ReceivedMessages); @@ -1320,37 +1348,36 @@ await manager.Ask(new SendUserMessage await subscriber.FishForMessageAsync(message => { initial.Add(message); - return message is ProcessingStateOutput { IsProcessing: false }; + return message is ProcessingStateOutput { IsProcessing: false } && initial.OfType().Count() == 2; }, ceiling, cancellationToken: TestContext.Current.CancellationToken); - Assert.Single(initial.OfType()); + Assert.Equal(2, initial.OfType().Count()); Assert.Contains(initial, message => message is SubAgentOutput { Phase: SubAgentPhase.Started }); Assert.Contains(initial, message => message is SubAgentOutput { Phase: SubAgentPhase.Completed }); Assert.Equal(1, _clientProvider.Compaction.CallCount); - var originalMainCalls = toolSpawn ? 2 : 0; + var originalMainCalls = toolSpawn ? 3 : 1; Assert.Equal(originalMainCalls, _clientProvider.Main.CallCount); var envelopes = ToolRecurrenceAdversarialTests.ReplyCaptureMailbox.Captures.GetOrCreateValue(Sys) - .Where(envelope => envelope.Message is SpawnChildActorRequest or ToolExecutionSubAgentActivity - || envelope.Message.GetType().Name is "RoutedSkillSubAgentActivity" or "RoutedSkillExecutionCompleted").ToArray(); - var spawn = Assert.IsType(Assert.Single(envelopes, - envelope => envelope.Message is SpawnChildActorRequest).Message); - Assert.True(spawn.ExecutionToken.IsCancellationRequested); + .Where(envelope => envelope.Message is StartBackgroundChildRun or BackgroundChildTerminal + || envelope.Message.GetType().Name == "RoutedSkillStartAccepted").ToArray(); + var start = Assert.IsType(Assert.Single(envelopes, + envelope => envelope.Message is StartBackgroundChildRun).Message); + Assert.True(start.ExecutionToken.IsCancellationRequested); + var terminal = Assert.IsType(Assert.Single(envelopes, + envelope => envelope.Message is BackgroundChildTerminal).Message); + Assert.Equal(start.Prepared.RunId, terminal.RunId); + Assert.True(terminal.Result.Success); if (toolSpawn) - { - var activity = envelopes.Select(envelope => envelope.Message).OfType().ToArray(); - var started = Assert.Single(activity); - Assert.Equal(SubAgentPhase.Started, started.Output.Phase); - Assert.Equal(spawn.ExecutionToken, started.ExecutionToken); Assert.Equal(2, envelopes.Length); - } else { - Assert.Equal(2, envelopes.Count(envelope => envelope.Message.GetType().Name == "RoutedSkillSubAgentActivity")); - Assert.Single(envelopes, envelope => envelope.Message.GetType().Name == "RoutedSkillExecutionCompleted"); - Assert.Equal(4, envelopes.Length); + var accepted = Assert.Single(envelopes, envelope => envelope.Message.GetType().Name == "RoutedSkillStartAccepted"); + var token = Assert.IsType(accepted.Message.GetType().GetProperty("ExecutionToken")!.GetValue(accepted.Message)); + Assert.Equal(start.ExecutionToken, token); + Assert.Equal(3, envelopes.Length); } var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(session.Value)}") .ResolveOne(ceiling, TestContext.Current.CancellationToken); - var childPath = $"{owner.Path}/{spawn.ActorName}"; + var childPath = $"{owner.Path}/child-run-{start.Prepared.RunId.Value}"; await AwaitAssertAsync(async () => { var identity = await Sys.ActorSelection(childPath).Ask(new Identify("original-child"), @@ -1384,7 +1411,7 @@ await AwaitAssertAsync(() => observations.Add(message); return message is SessionJoined; }, ceiling, cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(1, Assert.IsType(acknowledged).TurnCount); + Assert.Equal(2, Assert.IsType(acknowledged).TurnCount); var identity = await Sys.ActorSelection(childPath).Ask(new Identify("replayed-child"), ceiling, TestContext.Current.CancellationToken); Assert.Null(identity.Subject); @@ -1433,30 +1460,36 @@ await manager.Ask(new SendUserMessage await subscriber.FishForMessageAsync(message => { original.Add(message); - return message is ProcessingStateOutput { IsProcessing: false }; + return message is ProcessingStateOutput { IsProcessing: false } + && original.OfType().Count() == (providerResolutionFailure ? 1 : 2); }, ceiling, cancellationToken: TestContext.Current.CancellationToken); - var failure = Assert.Single(original.OfType()); - Assert.Contains("The routed provider failed.", failure.Message, StringComparison.Ordinal); - Assert.Equal(TurnOutcome.Failed, Assert.Single(original.OfType()).Outcome); - var callbackName = providerResolutionFailure ? "RoutedSkillExecutionFailed" : "RoutedSkillExecutionCompleted"; + var originalMainCalls = providerResolutionFailure ? 0 : 1; + var callbackName = providerResolutionFailure ? "RoutedSkillExecutionFailed" : nameof(BackgroundChildTerminal); var captured = Assert.Single(ToolRecurrenceAdversarialTests.ReplyCaptureMailbox.Captures.GetOrCreateValue(Sys), envelope => envelope.Message.GetType().Name == callbackName); - var tokenProperty = captured.Message.GetType().GetProperty("ExecutionToken"); - Assert.NotNull(tokenProperty); - var originalToken = Assert.IsType(tokenProperty.GetValue(captured.Message)); - Assert.True(originalToken.IsCancellationRequested); - Assert.Equal(providerResolutionFailure ? 1 : 0, _clientProvider.CompactionResolutionFailures); - Assert.Empty(_clientProvider.Compaction.PlannedExceptions); - Assert.Equal(0, _clientProvider.Main.CallCount); - if (!providerResolutionFailure) + if (providerResolutionFailure) + { + var failure = Assert.Single(original.OfType()); + Assert.Contains("The routed provider failed.", failure.Message, StringComparison.Ordinal); + Assert.Equal(TurnOutcome.Failed, Assert.Single(original.OfType()).Outcome); + var originalToken = Assert.IsType(captured.Message.GetType().GetProperty("ExecutionToken")!.GetValue(captured.Message)); + Assert.True(originalToken.IsCancellationRequested); + } + else { - var resultProperty = captured.Message.GetType().GetProperty("Result"); - Assert.NotNull(resultProperty); - var result = Assert.IsType(resultProperty.GetValue(captured.Message)); - Assert.False(result.Success); - Assert.Equal(SubAgentRunOutcome.Failed, result.Outcome); + var terminal = Assert.IsType(captured.Message); + Assert.False(terminal.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, terminal.Result.Outcome); + Assert.Contains("The routed provider failed.", terminal.Result.Output, StringComparison.Ordinal); Assert.Contains(original, message => message is SubAgentOutput { Phase: SubAgentPhase.Completed, Success: false }); + Assert.Empty(original.OfType()); + Assert.Equal(2, original.OfType().Count()); + Assert.Contains(_clientProvider.Main.ReceivedMessages[^1].SelectMany(message => message.Contents.OfType()), + result => result.Result?.ToString()?.Contains("The routed provider failed.", StringComparison.Ordinal) == true); } + Assert.Equal(providerResolutionFailure ? 1 : 0, _clientProvider.CompactionResolutionFailures); + Assert.Empty(_clientProvider.Compaction.PlannedExceptions); + Assert.Equal(originalMainCalls, _clientProvider.Main.CallCount); _clientProvider.FailCompactionResolution = false; var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -1465,7 +1498,11 @@ await manager.Ask(new SendUserMessage { SessionId = session, Content = "Complete the fresh task without a sub-agent.", Source = BuildPersonalSource() }, ceiling, TestContext.Current.CancellationToken); - await _clientProvider.Main.FirstCallEntered.Task.WaitAsync(ceiling, TestContext.Current.CancellationToken); + await AwaitAssertAsync(() => + { + Assert.Equal(originalMainCalls + 1, _clientProvider.Main.CallCount); + return Task.CompletedTask; + }, ceiling, cancellationToken: TestContext.Current.CancellationToken); var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(session.Value)}") .ResolveOne(ceiling, TestContext.Current.CancellationToken); owner.Tell(captured.Message, captured.Sender); @@ -1481,7 +1518,7 @@ await subscriber.FishForMessageAsync(message => }, ceiling, cancellationToken: TestContext.Current.CancellationToken); Assert.DoesNotContain(observed, message => message is ErrorOutput or TurnCompleted or TextOutput or SubAgentOutput); Assert.False(gate.Task.IsCompleted); - Assert.Equal(1, _clientProvider.Main.CallCount); + Assert.Equal(originalMainCalls + 1, _clientProvider.Main.CallCount); gate.TrySetResult(); var final = new List(); await subscriber.FishForMessageAsync(message => @@ -1492,7 +1529,7 @@ await subscriber.FishForMessageAsync(message => Assert.Equal(TurnOutcome.Completed, Assert.Single(final.OfType()).Outcome); Assert.Single(final.OfType()); Assert.DoesNotContain(final, message => message is ErrorOutput or SubAgentOutput); - Assert.Equal(1, _clientProvider.Main.CallCount); + Assert.Equal(originalMainCalls + 1, _clientProvider.Main.CallCount); } [Fact] @@ -1562,9 +1599,9 @@ await sessionManager.Ask(new SendUserMessage }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); await ExpectTextOutputAsync(subscriber, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - Assert.Equal(0, _clientProvider.Main.CallCount); + Assert.Equal(1, _clientProvider.Main.CallCount); Assert.Equal(1, _clientProvider.Compaction.CallCount); var subagentCall = Assert.Single(_clientProvider.Compaction.ReceivedMessages); @@ -1599,7 +1636,7 @@ await sessionManager.Ask(new JoinSession(subscriber) Assert.Equal(sessionId, firstAck.SessionId); await ExpectTextOutputAsync(subscriber, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); var callsAfterFirst = _clientProvider.Compaction.CallCount; @@ -1669,7 +1706,7 @@ await AwaitAssertAsync(() => gate.TrySetResult(); await ExpectTextOutputAsync(subscriber, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); await AwaitAssertAsync(() => { @@ -1713,10 +1750,12 @@ await sessionManager.Ask(new SendUserMessage }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); var text = await ExpectTextOutputAsync(subscriber, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await ExpectTurnCompletedAsync(subscriber, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + await ExpectTurnNumberAsync(subscriber, 2, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - Assert.Contains("fake", text.Text, StringComparison.OrdinalIgnoreCase); - Assert.Equal(0, _clientProvider.Main.CallCount); + using var acceptance = JsonDocument.Parse(text.Text); + Assert.Equal("Accepted", acceptance.RootElement.GetProperty("state").GetString()); + Assert.Equal(4, acceptance.RootElement.EnumerateObject().Count()); + Assert.Equal(1, _clientProvider.Main.CallCount); Assert.Equal(2, _clientProvider.Compaction.CallCount); Assert.NotNull(_recordingFileReadTool); Assert.True(_recordingFileReadTool!.WasCalled); @@ -1777,6 +1816,15 @@ private static async Task ExpectTurnCompletedAsync(Akka.TestKit.T throw new Xunit.Sdk.XunitException("Expected TurnCompleted but only received other session outputs."); } + private static async Task ExpectTurnNumberAsync( + Akka.TestKit.TestProbe probe, int expectedTurn, TimeSpan timeout, CancellationToken ct) + { + var message = await probe.FishForMessageAsync( + output => output is TurnCompleted completed && completed.TurnNumber.Value == expectedTurn, + timeout, cancellationToken: ct); + return Assert.IsType(message); + } + private async Task ColdRespawnAsync(SessionId sessionId) { var escapedId = Uri.EscapeDataString(sessionId.Value); diff --git a/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceAdversarialTests.cs index 000d5278d..c602e9350 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceAdversarialTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceAdversarialTests.cs @@ -764,8 +764,8 @@ public sealed class ReplyCaptureQueue(ConcurrentQueue captured) : IMes public bool HasMessages => _inner.HasMessages; public void Enqueue(IActorRef receiver, Envelope envelope) { - if (envelope.Message is LlmResponseReceived or ToolExecutionSingleCompleted or ToolExecutionBatchCompleted or ToolExecutionCompleted or ToolExecutionApprovalRequested or ToolExecutionFailed or SpawnChildActorRequest or ToolExecutionSubAgentActivity - || envelope.Message.GetType().Name is "RoutedSkillSubAgentActivity" or "RoutedSkillExecutionCompleted" or "RoutedSkillExecutionFailed") + if (envelope.Message is StartBackgroundChildRun or BackgroundChildTerminal or LlmResponseReceived or ToolExecutionSingleCompleted or ToolExecutionBatchCompleted or ToolExecutionCompleted or ToolExecutionApprovalRequested or ToolExecutionFailed or SpawnChildActorRequest or ToolExecutionSubAgentActivity + || envelope.Message.GetType().Name is "RoutedSkillSubAgentActivity" or "RoutedSkillExecutionCompleted" or "RoutedSkillExecutionFailed" or "RoutedSkillStartAccepted") captured.Enqueue(envelope); _inner.Enqueue(receiver, envelope); } diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs new file mode 100644 index 000000000..457565eae --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs @@ -0,0 +1,179 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildAcceptanceTests +{ + [Fact] + public void Atomic_child_delivery_restores_the_original_task_after_a_fresh_completed_turn() + { + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), "start"); + var accepted = Run(state, "run-delivery", "start"); + state = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Session, Run = accepted })); + state = Observe(state, "start", missingReceipt: false); + var terminal = new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Completed(new WorkingContextDelta { ReadFiles = ["/project/read.txt"] }), + Output = "The child confirmed its result.", AgentName = accepted.AgentName, + RunId = accepted.RunId, ScopeId = accepted.ScopeId + }, false, null); + state = state.Apply(RoundTrip(new ChildRunEvent.TerminalRecorded(terminal, 1) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 1 + })); + state = state.Apply(RoundTrip(new ChildRunEvent.ResultPrepared(terminal, 1) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 2 + })); + state = state.Apply(new TurnRecorded { SessionId = Session }); + state = AdmittedTask("fresh", state); + state = state.CloseInputs(state.AdoptedTaskInputIds).Apply(new TurnRecorded { SessionId = Session }); + state = state with { WorkingContext = new WorkingContext { ProjectDirectory = "/new-parent-project" } }; + var retained = state.ChildRuns[accepted.RunId]; + var input = new InputAdmitted + { + SessionId = Session, InputId = new InputId($"child-result-{accepted.RunId.Value}"), + SourceChildRunId = accepted.RunId, TurnContext = accepted.OriginalContext, + UserMessage = new SerializableChatMessage + { + Role = ChatRole.Tool, Name = "spawn_agent", ToolCallId = new ToolCallId("fresh-framework-result"), + Content = ChildRunDelivery.Body(retained) + } + }; + state = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 3 + })); + state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + var pending = Assert.Single(state.PendingInputs); + Assert.Equal(accepted.RunId, pending.SourceChildRunId); + Assert.Equal(pending.InputId, state.ChildRuns[accepted.RunId].DeliveryInputId); + state = state.Apply(RoundTrip(new ToolTaskAdopted + { + SessionId = Session, TurnContext = accepted.OriginalContext, InputIds = [pending.InputId], + ContinuedChildRunId = accepted.RunId + })); + Assert.Equal("original", state.LoopCheckpoint.TaskId); + Assert.Equal(1, Assert.Single(state.LoopCheckpoint.Entries).EqualRounds); + Assert.Equal("/new-parent-project", state.WorkingContext.ProjectDirectory); + Assert.False(state.LoopReceiptFailure); + Assert.Equal(accepted.OriginalContext, state.AdoptedTaskContext); + Assert.Throws(() => state.Apply(new ChildRunEvent.DeliveryAdmitted(input) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 4 + })); + } + + [Fact] + public void Coalesced_child_adoption_retains_both_pairs_after_input_closure_and_snapshot() + { + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), "first", "second"); + var runs = new[] { Run(state, "first-run", "first"), Run(state, "second-run", "second") }; + foreach (var run in runs) + state = state.Apply(new ChildRunAccepted { SessionId = Session, Run = run }); + state = Observe(Observe(state, "first", false), "second", false).Apply(new TurnRecorded { SessionId = Session }); + var ids = new List(); + foreach (var (run, index) in runs.Select((run, index) => (run, index))) + { + var terminal = new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Completed(WorkingContextDelta.Empty), Output = run.RunId.Value, + AgentName = run.AgentName, RunId = run.RunId, ScopeId = run.ScopeId + }, false, null); + state = state.Apply(new ChildRunEvent.TerminalRecorded(terminal, index + 1) + { SessionId = Session, RunId = run.RunId, RecordedAtMs = index + 1 }); + state = state.Apply(new ChildRunEvent.ResultPrepared(terminal, index + 1) + { SessionId = Session, RunId = run.RunId, RecordedAtMs = index + 1 }); + var input = new InputAdmitted + { + SessionId = Session, InputId = new InputId($"child-result-{run.RunId.Value}"), + SourceChildRunId = run.RunId, TurnContext = run.OriginalContext, + UserMessage = new SerializableChatMessage + { + Role = ChatRole.Tool, Name = ChildRunDelivery.ToolName(run), + ToolCallId = new ToolCallId($"result-{index}"), Content = ChildRunDelivery.Body(state.ChildRuns[run.RunId]) + } + }; + state = state.Apply(new ChildRunEvent.DeliveryAdmitted(input) { SessionId = Session, RunId = run.RunId, RecordedAtMs = index + 1 }); + ids.Add(input.InputId); + } + var adoption = RoundTrip(new ToolTaskAdopted + { + SessionId = Session, TurnContext = runs[^1].OriginalContext, + InputIds = ids.ToArray(), ContinuedChildRunId = runs[^1].RunId + }); + var retainedCount = state.History.Count; + state = state.Apply(adoption); + Assert.Equal(retainedCount + 4, state.History.Count); + state = state.Apply(adoption); + Assert.Equal(retainedCount + 4, state.History.Count); + var changed = state with { History = state.History.SetItem(retainedCount + 1, + state.History[retainedCount + 1] with { Content = "Altered result." }) }; + Assert.Throws(() => changed.Apply(adoption)); + changed = state with { History = state.History.SetItem(retainedCount, state.History[retainedCount] with + { ToolCalls = [state.History[retainedCount].ToolCalls[0] with { Name = new ToolName("changed") }] }) }; + Assert.Throws(() => changed.Apply(adoption)); + state = state.Apply(new InputClosed { SessionId = Session, InputIds = ids.ToArray(), TaskId = "original" }); + state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + Assert.Empty(state.PendingInputs); + Assert.Equal(new[] { "result-0", "result-1" }, state.History.Where(message => message.Role == ChatRole.Tool) + .Select(message => message.ToolCallId!.Value.Value)); + Assert.Equal("original", state.LoopCheckpoint.TaskId); + Assert.False(state.LoopReceiptFailure); + } + + [Fact] + public void Durable_cancel_requires_dispatch_closure_and_preserves_the_cancelled_outcome() + { + var state = AdmitBatch(AdmittedTask("cancel", SessionState.Empty), "start"); + var accepted = Run(state, "run-cancel", "start"); + state = state.Apply(new ChildRunAccepted { SessionId = Session, Run = accepted }); + state = state.Apply(RoundTrip(new ChildRunEvent.Started + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 1 + })); + state = state.Apply(RoundTrip(new ChildRunEvent.Checkpointed( + new ChildRunCheckpoint(1, "One confirmed read.", new WorkingContextDelta { ReadFiles = ["/project/read.txt"] })) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 2 + })); + state = state.Apply(RoundTrip(new ChildRunEvent.CancellationRequested + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 3 + })); + var terminal = new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Cancelled(SubAgentOutcomeReason.CancelledByParent), + Output = "The run stopped. External effects remain unconfirmed.", AgentName = accepted.AgentName, + RunId = accepted.RunId, ScopeId = accepted.ScopeId + }, false, "The report write failed. The committed checkpoint remains available."); + var receipt = new ChildRunEvent.TerminalRecorded(terminal, 1) + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 5 + }; + Assert.Throws(() => state.Apply(receipt)); + state = state.Apply(RoundTrip(new ChildRunEvent.DispatchClosed + { + SessionId = Session, RunId = accepted.RunId, RecordedAtMs = 4 + })); + state = state.Apply(RoundTrip(receipt)); + var restored = Assert.Single(SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())).ChildRuns).Value; + Assert.Equal(BackgroundChildState.Cancelled, restored.State); + Assert.False(restored.Terminal!.Result.Success); + Assert.Equal(SubAgentRunOutcome.Failed, restored.Terminal.Result.Outcome); + Assert.Equal(SubAgentOutcomeReason.CancelledByParent, restored.Terminal.Result.OutcomeReason); + Assert.Equal("/project/read.txt", Assert.Single(restored.ChildCheckpoint!.ConfirmedActivity.ReadFiles)); + Assert.Equal(terminal.EvidenceWarning, restored.Terminal.EvidenceWarning); + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs new file mode 100644 index 000000000..b0fd84afc --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs @@ -0,0 +1,91 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Tools; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildAcceptanceTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public void Run_context_is_bounded_and_confined_to_the_current_authority(bool publicAudience, bool foreignRequester) + { + var calls = Enumerable.Range(0, 10).Select(index => "call-" + index).ToArray(); + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), calls); + foreach (var call in calls) + state = state.Apply(new ChildRunAccepted { SessionId = Session, Run = Run(state, "run-" + call, call) }); + state = AdmittedTask("later", state); + if (foreignRequester) + state = state with { AdoptedTaskContext = state.AdoptedTaskContext! with { RequesterSenderId = new SenderId("other") } }; + var input = new ContextAssemblyInput(state, [], false, null, null, null, Session, + SessionStoragePaths.CreateLegacy("/tmp/neutral-child-context", "/tmp/neutral-child-context/session", Session.Value), false, null, string.Empty, + publicAudience ? TrustAudience.Public : TrustAudience.Personal); + var context = SessionMessageAssembler.BuildVolatileContextBlock(input); + if (publicAudience || foreignRequester) + { + Assert.DoesNotContain("[background-agent-runs]", context, StringComparison.Ordinal); + Assert.DoesNotContain("run-call-", context, StringComparison.Ordinal); + return; + } + Assert.Contains("[background-agent-runs]", context, StringComparison.Ordinal); + Assert.Contains("load_tool(name: \"check_agent_run\")", context, StringComparison.Ordinal); + Assert.Contains("Results arrive automatically.", context, StringComparison.Ordinal); + using var runs = JsonDocument.Parse(context.Split("[background-agent-runs]\n", StringSplitOptions.None)[1].Split('\n')[0]); + Assert.Equal(8, runs.RootElement.GetArrayLength()); + Assert.All(runs.RootElement.EnumerateArray(), run => + { + Assert.Equal("Accepted", run.GetProperty("state").GetString()); + Assert.False(run.GetProperty("dispatch_closed").GetBoolean()); + }); + Assert.DoesNotContain("/project", context, StringComparison.Ordinal); + } + + [Fact] + public async Task The_deferred_control_tool_loads_by_name_and_reports_the_real_owner_reply() + { + var registry = new ToolRegistry(); + var tool = new CheckAgentRunTool(); + registry.Register(tool); + var policy = TestToolAccessPolicy.Create(new ToolConfig()); + Assert.False(registry.IsCoreTool(CheckAgentRunTool.ToolName)); + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), "start"); + var run = Run(state, "run-control", "start"); + var requests = new List(); + var context = TestToolExecutionContext.CreateBound(Session.Value, Path.Combine(Path.GetTempPath(), "neutral-child-control"), new TestToolExecutionContextOptions + { + Audience = TrustAudience.Personal, + SpawnChildActor = (payload, _, _) => + { + requests.Add(Assert.IsType(payload)); + return Task.FromResult(new ChildControlReply(run)); + } + }); + var loaded = await new LoadToolTool(registry, policy).ExecuteAsync( + new Dictionary { ["name"] = CheckAgentRunTool.ToolName }, context, + TestContext.Current.CancellationToken); + Assert.Equal(CheckAgentRunTool.ToolName, loaded); + var result = await tool.ExecuteAsync(new Dictionary { ["runId"] = run.RunId.Value, ["cancel"] = true }, + context, TestContext.Current.CancellationToken); + Assert.Equal(new ChildControlRequest(run.RunId, true), Assert.Single(requests)); + Assert.IsType(context.Outputs.Receipt); + using var body = JsonDocument.Parse(result); + Assert.Equal("Accepted", body.RootElement.GetProperty("state").GetString()); + Assert.False(body.RootElement.GetProperty("cancellation_requested").GetBoolean()); + Assert.False(body.RootElement.GetProperty("dispatch_closed").GetBoolean()); + Assert.False(SubAgentToolPolicy.IsAllowedForSubAgent(CheckAgentRunTool.ToolName)); + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs new file mode 100644 index 000000000..d9d44f329 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs @@ -0,0 +1,211 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Akka.Serialization; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Sessions; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using StoredRole = Netclaw.Actors.Protocol.ChatRole; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildAcceptanceTests(ITestOutputHelper output) : TestKit(output: output) +{ + private static readonly SessionId Session = new("signalr/child-acceptance"); + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + => builder.WithNetclawSerialization(); + + [Fact] + public void Registered_acceptance_and_snapshot_preserve_working_facts_without_the_live_shell_host() + { + var state = AdmittedTask("original", SessionState.Empty); + state = AdmitBatch(state, "first", "second"); + var run = Run(state, "run-one", "first"); + var evt = new ChildRunAccepted { SessionId = Session, Run = run }; + var decoded = RoundTrip(evt); + Assert.Equal(Bytes(evt), Bytes(decoded)); + Assert.Null(decoded.Run.InitialWorkingSnapshot.ShellEnvironment); + Assert.NotNull(run.InitialWorkingSnapshot.ShellEnvironment); + Assert.Equal(run.InitialWorkingSnapshot.WorkingContext.ProjectDirectory, decoded.Run.InitialWorkingSnapshot.WorkingContext.ProjectDirectory); + var recovered = SessionState.FromSnapshot(RoundTrip(state.Apply(decoded).ToSnapshot())); + var restored = Assert.Single(recovered.ChildRuns).Value; + Assert.Equal(run.StartKey, restored.StartKey); + Assert.Equal(run.OriginalContext, restored.OriginalContext); + Assert.Equal(run.OriginInputIds, restored.OriginInputIds); + Assert.Equal(run.InitialWorkingSnapshot.WorkingContext.RecentFiles, restored.InitialWorkingSnapshot.WorkingContext.RecentFiles); + var git = Assert.IsType(restored.InitialWorkingSnapshot.Git).Snapshot; + Assert.Equal("main", git.Branch); + Assert.Equal("head", git.Head); + Assert.Equal("origin/main", git.Upstream); + Assert.Equal(2, git.Ahead); + Assert.Equal(1, git.Modified); + Assert.Equal(new[] { "/project/changed.txt" }, git.ChangedFiles); + Assert.Empty(recovered.ActiveBackgroundJobs); + } + + [Fact] + public void Sibling_acceptances_retain_the_settled_parent_round_across_fresh_task_and_compaction() + { + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), "first", "second"); + var first = Run(state, "run-one", "first"); + var second = Run(state, "run-two", "second"); + state = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Session, Run = first })); + state = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Session, Run = second })); + state = Observe(state, "first", missingReceipt: false); + Assert.All(state.ChildRuns.Values, run => Assert.False(run.StartBatchSettled)); + state = Observe(state, "second", missingReceipt: false); + Assert.All(state.ChildRuns.Values, run => + { + Assert.True(run.StartBatchSettled); + Assert.Equal(1, Assert.Single(run.ParentCheckpoint.Entries).EqualRounds); + }); + state = state.Apply(new TurnRecorded { SessionId = Session }); + state = AdmittedTask("fresh", state); + state = state.Apply(new SessionCompacted { SessionId = Session, Summary = "neutral summary" }); + var recovered = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + Assert.Equal("fresh", recovered.LoopCheckpoint.TaskId); + Assert.Empty(recovered.LoopCheckpoint.Entries); + Assert.Equal(2, recovered.ChildRuns.Count); + Assert.All(recovered.ChildRuns.Values, run => + { + Assert.Equal("original", run.ParentCheckpoint.TaskId); + Assert.Equal(1, Assert.Single(run.ParentCheckpoint.Entries).EqualRounds); + Assert.False(run.ParentReceiptFailure); + Assert.True(run.StartBatchSettled); + }); + } + + [Fact] + public void A_settled_missing_receipt_survives_fresh_input_in_the_child_ledger() + { + var state = AdmitBatch(AdmittedTask("original", SessionState.Empty), "start"); + var run = Run(state, "missing-receipt", "start"); + state = state.Apply(new ChildRunAccepted { SessionId = Session, Run = run }); + state = Observe(state, "start", missingReceipt: true); + state = AdmittedTask("fresh", state); + var retained = Assert.Single(SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())).ChildRuns).Value; + Assert.True(retained.StartBatchSettled); + Assert.True(retained.ParentReceiptFailure); + Assert.Equal("original", retained.ParentCheckpoint.TaskId); + Assert.Empty(retained.ParentCheckpoint.Entries); + } + + [Fact] + public void Direct_activation_round_trips_without_a_fabricated_tool_round() + { + var state = AdmittedTask("slash", SessionState.Empty); + var run = Run(state, "slash-run", "unused") with + { + StartKey = new ChildRunStartKey.Slash(state.AdoptedTaskInputIds[0]) { SessionId = Session, TurnId = new TurnId("slash") }, + SourceOperation = "/inspect", StartBatchSettled = true, + InitialWorkingSnapshot = new WorkingContextSnapshot { WorkingContext = WorkingContext.Empty, Git = new GitWorkingContextInspection.Skipped() } + }; + var recovered = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Session, Run = run })); + recovered = SessionState.FromSnapshot(RoundTrip(recovered.ToSnapshot())); + Assert.IsType(Assert.Single(recovered.ChildRuns).Value.StartKey); + Assert.Null(recovered.LoopAdmission); + Assert.Empty(recovered.LoopObservations); + Assert.Empty(recovered.LoopCheckpoint.Entries); + Assert.DoesNotContain(recovered.History, message => message.Role == StoredRole.Tool); + } + + [Fact] + public void Old_snapshot_reads_an_empty_child_ledger_without_transcript_inference() + { + var legacy = new SessionSnapshot + { + History = [new SerializableChatMessage { Role = StoredRole.Assistant, Content = "The child is still active." }] + }; + var recovered = SessionState.FromSnapshot(RoundTrip(legacy)); + Assert.Empty(recovered.ChildRuns); + Assert.Equal(legacy.History[0], Assert.Single(recovered.History)); + } + + private static SessionState AdmittedTask(string turn, SessionState prior) + { + var context = new TurnContextRecord + { + SessionId = Session, TurnId = turn, Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, + RequesterSenderId = new SenderId("operator"), RequesterPrincipal = PrincipalClassification.Operator, + TransportAuthenticity = TransportAuthenticity.LocalProcess, PayloadTaint = PayloadTaint.Trusted + }; + var input = new InputAdmitted + { + SessionId = Session, InputId = new InputId("input-" + turn), TurnContext = context, + UserMessage = new SerializableChatMessage { Content = turn } + }; + return prior.Apply(input).Apply(new ToolTaskAdopted + { + SessionId = Session, TurnContext = context, InputIds = [input.InputId] + }); + } + + private static SessionState AdmitBatch(SessionState state, params string[] ids) + { + var tracker = new TurnStateTracker(); + tracker.RestoreCheckpoint(state.LoopCheckpoint); + var prepared = ToolCycleSignatureFactory.Prepare(ids.Select(id => new FunctionCallContent(id, "spawn_agent", new Dictionary())).ToArray(), new FakeToolExecutor()); + Assert.Equal(ToolCycleDecisionKind.Execute, tracker.EvaluateBeforeDispatch(prepared).Kind); + var evt = new ToolBatchStarted + { + SessionId = Session, + LoopAdmission = new ToolLoopAdmission + { + TaskId = state.LoopCheckpoint.TaskId, ActionHash = prepared.Action.Value, + Calls = prepared.Calls.Select(call => new ToolLoopPreparedCall(call.CallId.Value, call.ToolName.Value, call.ArgumentsHash, false)).ToArray() + }, + LoopDelta = tracker.CaptureDelta(state.LoopCheckpoint.TaskId, state.LoopCheckpoint) + }; + return state.CloseInputs(state.AdoptedTaskInputIds).ApplyLoopAdmission(evt); + } + + private static BackgroundChildRun Run(SessionState state, string id, string call) => new() + { + RunId = new SubAgentRunId(id), ScopeId = new SubAgentScopeId($"{Session.Value}/subagent/worker/{id}"), + AgentName = new AgentName("worker"), SourceOperation = "spawn_agent", ArgumentsDigest = new string('A', 64), + StartKey = new ChildRunStartKey.Tool(new ToolCallId(call)) { SessionId = Session, TurnId = new TurnId(state.AdoptedTaskContext!.TurnId) }, + OriginalContext = state.AdoptedTaskContext!, OriginInputIds = state.AdoptedTaskInputIds, + ParentCheckpoint = state.LoopCheckpoint, ParentReceiptFailure = state.LoopReceiptFailure, + InitialWorkingSnapshot = new WorkingContextSnapshot + { + WorkingContext = new WorkingContext { ProjectDirectory = "/project", RecentFiles = ["/project/input.txt"] }, + Git = new GitWorkingContextInspection.Available(new GitWorkingContextSnapshot + { + Worktree = "/project", CommonDirectory = "/project/.git", Branch = "main", Head = "head", Upstream = "origin/main", + Ahead = 2, Modified = 1, ChangedFiles = ["/project/changed.txt"] + }), + ShellEnvironment = ShellExecutionEnvironment.CreateBash(ShellPlatform.Linux) + } + }; + + private static SessionState Observe(SessionState state, string call, bool missingReceipt) + => state.ApplyLoopObservation(new ToolCallRecorded + { + SessionId = Session, + LoopObservation = ToolCycleSignatureFactory.CreateObservation(call, + missingReceipt ? null : new ToolInvocationReceipt.Succeeded([], null), "accepted", false) + }); + + private byte[] Bytes(object value) => Sys.Serialization.FindSerializerFor(value).ToBinary(value); + + private T RoundTrip(T value) where T : notnull + { + var serializer = Sys.Serialization.FindSerializerFor(value); + var manifest = Assert.IsAssignableFrom(serializer).Manifest(value); + return Assert.IsType(Sys.Serialization.Deserialize(serializer.ToBinary(value), serializer.Identifier, manifest)); + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs new file mode 100644 index 000000000..ecf80b422 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs @@ -0,0 +1,250 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Jobs; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildLedgerAdversarialTests +{ + [Theory] + [InlineData("event-session")] + [InlineData("event-run")] + [InlineData("result-run")] + [InlineData("result-scope")] + [InlineData("result-agent")] + public void A_terminal_fact_cannot_replace_the_accepted_owner(string fault) + { + var (state, run) = AcceptedSlash(receiptFailure: false); + var terminal = Terminal(run); + var evt = new ChildRunEvent.TerminalRecorded(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123457 }; + evt = fault switch + { + "event-session" => evt with { SessionId = new SessionId("signalr/foreign") }, + "event-run" => evt with { RunId = new SubAgentRunId("foreign") }, + "result-run" => evt with { Terminal = terminal with { Result = terminal.Result with { RunId = new SubAgentRunId("foreign") } } }, + "result-scope" => evt with { Terminal = terminal with { Result = terminal.Result with { ScopeId = new SubAgentScopeId("foreign") } } }, + "result-agent" => evt with { Terminal = terminal with { Result = terminal.Result with { AgentName = new AgentName("foreign") } } }, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + RejectUnchanged(state, () => state.Apply(RoundTrip(evt))); + var valid = state.Apply(RoundTrip(new ChildRunEvent.TerminalRecorded(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123457 })); + Assert.Equal(terminal.Result.Output, valid.ChildRuns[run.RunId].Terminal!.Result.Output); + RejectUnchanged(valid, () => valid.Apply(new ChildRunEvent.CancellationRequested + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123458 })); + } + + [Theory] + [InlineData("body")] + [InlineData("authority")] + [InlineData("source-run")] + [InlineData("dual-lineage")] + [InlineData("input-id")] + [InlineData("used-call-id")] + public void Only_the_atomic_canonical_child_result_can_admit_a_continuation(string fault) + { + var (state, run) = PreparedSlash(receiptFailure: true); + var input = Delivery(run, "framework-result"); + if (fault == "used-call-id") + state = state with { History = state.History.Add(ChildRunDelivery.Call(run, input.UserMessage.ToolCallId!.Value)) }; + var invalid = fault switch + { + "body" => input with { UserMessage = input.UserMessage with { Content = "A fabricated successful result." } }, + "authority" => input with { TurnContext = input.TurnContext with { RequesterSenderId = new SenderId("operator-b") } }, + "source-run" => input with { SourceChildRunId = new SubAgentRunId("foreign") }, + "dual-lineage" => input with { SourceBackgroundJobId = new BackgroundJobId("foreign-job") }, + "input-id" => input with { InputId = new InputId("foreign-input") }, + "used-call-id" => input, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + RejectUnchanged(state, () => state.Apply(invalid)); + RejectUnchanged(state, () => state.Apply(new ChildRunEvent.DeliveryAdmitted(invalid) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + var validInput = Delivery(run, "unused-framework-call"); + var valid = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(validInput) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + Assert.Equal(validInput.InputId, Assert.Single(valid.PendingInputs).InputId); + Assert.Equal(validInput.InputId, valid.ChildRuns[run.RunId].DeliveryInputId); + RejectUnchanged(valid, () => valid.Apply(new ChildRunEvent.DeliveryAdmitted(validInput) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123461 })); + var restored = SessionState.FromSnapshot(RoundTrip(valid.ToSnapshot())); + Assert.Equal(validInput.UserMessage.Content, Assert.Single(restored.PendingInputs).UserMessage.Content); + Assert.True(restored.ChildRuns[run.RunId].ParentReceiptFailure); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Original_turn_continuation_restores_detector_evidence_before_the_duplicate_adoption_path(bool receiptFailure) + { + var (state, run) = PreparedSlash(receiptFailure); + var input = Delivery(run, "framework-result"); + state = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + // The authority turn stays unchanged, but the current detector state differs from the retained task. + state = state with { LoopCheckpoint = new ToolLoopCheckpoint { TaskId = "different-current-task" }, LoopReceiptFailure = !receiptFailure }; + state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + var adoption = new ToolTaskAdopted + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId }; + RejectUnchanged(state, () => state.Apply(adoption with { ContinuedChildRunId = null })); + RejectUnchanged(state, () => state.Apply(adoption with { ContinuedJobKey = "foreign-job" })); + var restored = state.Apply(RoundTrip(adoption)); + Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, restored.LoopCheckpoint)); + Assert.Equal(receiptFailure, restored.LoopReceiptFailure); + Assert.True(SessionState.SameCanonicalContext(run.OriginalContext, restored.AdoptedTaskContext!)); + Assert.Equal(new[] { input.InputId }, restored.AdoptedTaskInputIds); + Assert.Null(restored.LoopAdmission); + Assert.Empty(restored.LoopObservations); + Assert.Equal(input.UserMessage.Content, Assert.Single(restored.PendingInputs).UserMessage.Content); + } + + [Theory] + [InlineData("authority")] + [InlineData("task")] + [InlineData("checkpoint")] + [InlineData("receipt-failure")] + public void Child_results_coalesce_only_with_the_same_original_task_and_detector_evidence(string fault) + { + var (state, first) = PreparedSlash(receiptFailure: true); + var second = first with + { + RunId = new SubAgentRunId("sibling-run"), ScopeId = new SubAgentScopeId($"{Owner.Value}/subagent/worker/sibling-run"), + StartKey = new ChildRunStartKey.Slash(new InputId("origin-two")) { SessionId = Owner, TurnId = new TurnId(first.OriginalContext.TurnId) }, + Terminal = null, PreparedTerminal = null + }; + var secondTerminal = Terminal(second); + second = second with { Terminal = secondTerminal, TerminalSequenceNr = 2, PreparedTerminal = secondTerminal }; + // This snapshot premise tests the continuation consumer independently from live owner refresh. + var incompatible = fault switch + { + "authority" => second with { OriginalContext = second.OriginalContext with { RequesterSenderId = new SenderId("operator-b") } }, + "task" => second with { ParentCheckpoint = second.ParentCheckpoint with { TaskId = "other-original-task" } }, + "checkpoint" => second with { ParentCheckpoint = second.ParentCheckpoint with { LastBlockedAction = "other-correction" } }, + "receipt-failure" => second with { ParentReceiptFailure = false }, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + SessionState AdmitPair(BackgroundChildRun sibling) + { + var pair = state with { ChildRuns = state.ChildRuns.Add(sibling.RunId, sibling) }; + foreach (var run in new[] { first, sibling }) + pair = pair.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(Delivery(run, $"framework-{run.RunId.Value}")) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + return SessionState.FromSnapshot(RoundTrip(pair.ToSnapshot())); + } + ToolTaskAdopted AdoptPair(SessionState pair) => new() + { SessionId = Owner, TurnContext = first.OriginalContext, InputIds = pair.PendingInputs.Select(input => input.InputId).ToArray(), ContinuedChildRunId = second.RunId }; + var invalid = AdmitPair(incompatible); + RejectUnchanged(invalid, () => invalid.Apply(AdoptPair(invalid))); + var valid = AdmitPair(second); + var adopted = valid.Apply(RoundTrip(AdoptPair(valid))); + Assert.Equal(2, adopted.AdoptedTaskInputIds.Count); + Assert.True(BackgroundChildRun.SameCheckpoint(first.ParentCheckpoint, adopted.LoopCheckpoint)); + Assert.True(adopted.LoopReceiptFailure); + Assert.Equal(new[] { first.RunId, second.RunId }, adopted.PendingInputs.Select(input => input.SourceChildRunId!.Value)); + } + + [Theory] + [InlineData("body")] + [InlineData("role")] + [InlineData("name")] + [InlineData("missing-call")] + [InlineData("empty-call")] + [InlineData("original-start-call")] + public void A_snapshot_cannot_replace_the_canonical_pending_child_result(string fault) + { + var (state, run) = Acceptance(Owner, slash: false); + state = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Owner, Run = run })); + state = state.ApplyLoopObservation(new ToolCallRecorded + { + SessionId = Owner, + LoopObservation = ToolCycleSignatureFactory.CreateObservation("start-call", new ToolInvocationReceipt.Succeeded([], null), "accepted", false) + }).CloseInputs(run.OriginInputIds); + run = state.ChildRuns[run.RunId]; + Assert.True(run.StartBatchSettled); + var terminal = Terminal(run); + state = state.Apply(RoundTrip(new ChildRunEvent.TerminalRecorded(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123458 })); + state = state.Apply(RoundTrip(new ChildRunEvent.ResultPrepared(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123459 })); + run = state.ChildRuns[run.RunId]; + var input = Delivery(run, "framework-result"); + state = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + var snapshot = state.ToSnapshot(); + var changed = fault switch + { + "body" => input.UserMessage with { Content = "A forged success replaces the recorded result." }, + "role" => input.UserMessage with { Role = ChatRole.User }, + "name" => input.UserMessage with { Name = "shell" }, + "missing-call" => input.UserMessage with { ToolCallId = null }, + "empty-call" => input.UserMessage with { ToolCallId = new ToolCallId("") }, + "original-start-call" => input.UserMessage with { ToolCallId = new ToolCallId("start-call") }, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + var corrupted = RoundTrip(snapshot with { PendingInputs = [input with { UserMessage = changed }] }); + Assert.Throws(() => SessionState.FromSnapshot(corrupted)); + var valid = SessionState.FromSnapshot(RoundTrip(snapshot)); + var restoredMessage = Assert.Single(valid.PendingInputs).UserMessage; + Assert.Equal(input.UserMessage.Content, restoredMessage.Content); + Assert.Equal(input.UserMessage.Role, restoredMessage.Role); + Assert.Equal(input.UserMessage.Name, restoredMessage.Name); + Assert.Equal(input.UserMessage.ToolCallId, restoredMessage.ToolCallId); + var adopted = valid.Apply(RoundTrip(new ToolTaskAdopted + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId })); + Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, adopted.LoopCheckpoint)); + Assert.Equal(input.UserMessage.Content, Assert.Single(adopted.PendingInputs).UserMessage.Content); + Assert.Equal("framework-result", Assert.Single(adopted.PendingInputs).UserMessage.ToolCallId!.Value.Value); + } + + private (SessionState State, BackgroundChildRun Run) AcceptedSlash(bool receiptFailure) + { + var (state, run) = Acceptance(Owner, slash: true); + state = state with { LoopReceiptFailure = receiptFailure }; + run = run with { ParentReceiptFailure = receiptFailure }; + return (state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Owner, Run = run })).CloseInputs(run.OriginInputIds), run); + } + + private (SessionState State, BackgroundChildRun Run) PreparedSlash(bool receiptFailure) + { + var (state, run) = AcceptedSlash(receiptFailure); + var terminal = Terminal(run); + state = state.Apply(RoundTrip(new ChildRunEvent.TerminalRecorded(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123458 })); + state = state.Apply(RoundTrip(new ChildRunEvent.ResultPrepared(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123459 })); + return (state, state.ChildRuns[run.RunId]); + } + + private static ChildRunTerminal Terminal(BackgroundChildRun run) => new(new SubAgentResult + { + Completion = new ChildRunCompletion.Completed(new WorkingContextDelta { ReadFiles = ["/neutral/confirmed.txt"] }), + Output = $"The child {run.RunId.Value} confirmed its result.", RunId = run.RunId, ScopeId = run.ScopeId, AgentName = run.AgentName + }, false, null); + + private static InputAdmitted Delivery(BackgroundChildRun run, string callId) => new() + { + SessionId = Owner, InputId = new InputId($"child-result-{run.RunId.Value}"), SourceChildRunId = run.RunId, + TurnContext = run.OriginalContext, + UserMessage = new SerializableChatMessage + { Role = ChatRole.Tool, Name = ChildRunDelivery.ToolName(run), ToolCallId = new ToolCallId(callId), Content = ChildRunDelivery.Body(run) } + }; + + private void RejectUnchanged(SessionState state, Func apply) + { + var before = Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot()); + Assert.Throws(() => apply()); + Assert.Equal(before, Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot())); + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs new file mode 100644 index 000000000..9785f37e7 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs @@ -0,0 +1,150 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka; +using Akka.Actor; +using Akka.Hosting; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using StoredRole = Netclaw.Actors.Protocol.ChatRole; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildLedgerAdversarialTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Actual_recovery_checks_terminal_positions_before_any_child_result_continuation(bool snapshot, bool invalid) + { + var (premise, prototype) = Acceptance(Owner, slash: true); + var input = premise.PendingInputs[0]; + var adoption = new ToolTaskAdopted + { SessionId = Owner, TurnContext = input.TurnContext, InputIds = [input.InputId] }; + var state = SessionState.Empty.Apply(input).Apply(adoption); + var run = prototype with + { OriginInputIds = [input.InputId], ParentCheckpoint = state.LoopCheckpoint, ParentReceiptFailure = false }; + var accepted = new ChildRunAccepted { SessionId = Owner, Run = run }; + var started = new ChildRunEvent.Started { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123457 }; + var turn = new TurnRecorded + { + SessionId = Owner, UserMessage = input.UserMessage, ConsumedInputIds = [input.InputId], RecordedAtMs = 123458, + AssistantReply = new SerializableChatMessage { Role = StoredRole.Assistant, Content = "The original task completed." } + }; + var terminal = Terminal(run); + // A positive but incorrect stamp remains self-consistent until the actor checks the real store position. + var terminalSequence = !snapshot && invalid ? 7L : 6L; + var recorded = new ChildRunEvent.TerminalRecorded(terminal, terminalSequence) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123459 }; + var prepared = new ChildRunEvent.ResultPrepared(terminal, terminalSequence) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123459 }; + var title = new SessionTitleSet { SessionId = Owner, Title = "A durable position marker", SetAtMs = 123460 }; + var events = new ISessionEvent[] { input, adoption, accepted, started, turn, recorded, prepared, title }; + var seed = Sys.ActorOf(Props.Create(() => new SnapshotSeeder($"session-{Owner.Value}"))); + for (var index = 0; index < events.Length; index++) + { + var serializer = Codec(events[index]); + var decoded = Assert.IsAssignableFrom(Sys.Serialization.Deserialize( + serializer.ToBinary(events[index]), serializer.Identifier, serializer.Manifest(events[index]))); + Assert.Equal(index + 1L, await seed.Ask(decoded, Ceiling, TestContext.Current.CancellationToken)); + } + state = state.Apply(accepted).Apply(started).Apply(turn).CloseInputs([input.InputId]) + .Apply(recorded).Apply(prepared).Apply(title); + var storedRun = state.ChildRuns[run.RunId]; + Assert.Equal(terminalSequence, storedRun.TerminalSequenceNr); + if (snapshot) + { + if (invalid) + state = state with { ChildRuns = state.ChildRuns.SetItem(run.RunId, storedRun with { TerminalSequenceNr = 9 }) }; + var stored = RoundTrip(state.ToSnapshot()); + Assert.Single(SessionState.FromSnapshot(stored).ChildRuns); + long savedAt = 0; + await Snapshots.OnSave.FailIf((id, criteria) => + { + if (id == $"session-{Owner.Value}") savedAt = criteria.MaxSequenceNr; + return false; + }); + await seed.Ask(stored, Ceiling, TestContext.Current.CancellationToken); + Assert.Equal(8, savedAt); + Assert.Equal(invalid, Assert.Single(stored.ChildRuns).TerminalSequenceNr > savedAt); + if (!invalid) Assert.True(storedRun.TerminalSequenceNr < savedAt); + } + var watcher = CreateTestProbe(); + watcher.Watch(seed); + Sys.Stop(seed); + await watcher.ExpectTerminatedAsync(seed, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + + var entered = NewSignal(); + var release = NewSignal(); + var response = NewSignal(); + _model.NextResponseGate = response; + await Snapshots.OnLoad.FailIf(async (id, _) => + { + if (id != $"session-{Owner.Value}") return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + var observer = CreateTestProbe(); + var manager = ActorRegistry.Get(); + async Task RecoverAsync() + { + manager.Tell(new JoinSession(observer) { SessionId = Owner, Filter = OutputFilter.Full | OutputFilter.ProcessingState }, observer.Ref); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(Owner.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + observer.Watch(owner); + release.TrySetResult(); + var first = await observer.ReceiveOneAsync(Ceiling, TestContext.Current.CancellationToken); + if (invalid) + { + Assert.Equal(owner, Assert.IsType(first).ActorRef); + return; + } + Assert.Equal(Owner, Assert.IsType(first).SessionId); + await _model.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + response.TrySetResult(); + var completed = false; + await observer.FishForMessageAsync(message => + { + if (message is TurnCompleted) completed = true; + return completed && message is ProcessingStateOutput { IsProcessing: false }; + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + } + try + { + if (invalid) + await EventFilter.Error().ExpectOneAsync(RecoverAsync, cancellationToken: TestContext.Current.CancellationToken); + else + await RecoverAsync(); + } + finally + { + release.TrySetResult(); + response.TrySetResult(); + } + Assert.Equal(invalid ? 0 : 1, _model.CallCount); + Assert.Equal(0, _tools.CallCount); + if (!invalid) + { + var request = Assert.Single(_model.ReceivedMessages); + var call = Assert.Single(request.SelectMany(message => message.Contents.OfType())); + var result = Assert.Single(request.SelectMany(message => message.Contents.OfType())); + Assert.Equal(ChildRunDelivery.ToolName(storedRun), call.Name); + Assert.StartsWith("child-result-", call.CallId); + Assert.Equal(call.CallId, result.CallId); + Assert.Equal(ChildRunDelivery.Body(storedRun), Assert.IsType(result.Result)); + Assert.Contains(request, message => message.Text == turn.AssistantReply.Content); + } + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs new file mode 100644 index 000000000..faf5fa1d1 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs @@ -0,0 +1,334 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Security.Cryptography; +using System.Text.Json; +using Akka; +using Akka.Actor; +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Akka.Persistence; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Jobs; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Reminders; +using Netclaw.Actors.Serialization; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tests.Sessions; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tests.Utilities; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using SessionFakeChatClient = Netclaw.Actors.Tests.Sessions.FakeChatClient; +using StoredRole = Netclaw.Actors.Protocol.ChatRole; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildLedgerAdversarialTests(ITestOutputHelper output) + : PersistenceTestKit(output: output), IAsyncDisposable +{ + private static readonly TimeSpan Ceiling = TimeSpan.FromSeconds(20); + private static readonly SessionId Owner = new("signalr/child-ledger-owner"); + private readonly SessionFakeChatClient _model = new(); + private readonly FakeToolExecutor _tools = new(); + private TestSessionTempDirectory? _directory; + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + { + base.ConfigureAkka(builder, provider); + builder.WithNetclawSerialization() + .WithNetclawActors(provider.GetRequiredService()); + } + + protected override void ConfigureServices(HostBuilderContext context, IServiceCollection services) + { + _directory = TestSessionTempDirectory.Create("netclaw-child-ledger-"); + services.AddSingleton(_directory); + services.AddSingleton(_directory.Paths); + services.AddSingleton(new FakeCapabilityResolver()); + services.AddSingleton(TimeProvider.System); + services.AddSingleton(SecurityPolicyDefaults.Resolve(null)); + services.AddSingleton(); + services.AddSingleton(new SchedulingConfig()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(NullNotificationSink.Instance); + services.AddSingleton(NullReminderChannelNotifier.Instance); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(new SingleClientProvider(_model)); + services.AddSingleton(new ModelCapabilities { ModelId = "fake-model", ContextWindowTokens = 128_000 }); + services.AddSingleton(new SessionConfig { Tuning = new SessionTuning { TitleGenerationInterval = 0 } }); + services.AddSingleton(new StaticSystemPromptProvider("Inspect recorded run facts.")); + services.AddSingleton(_tools); + services.AddSingleton(new ToolRegistry()); + services.AddLlmSessionCompositeRecords(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Registered_event_and_snapshot_keep_canonical_run_facts_and_distinct_task_identity(bool slash) + { + var (state, run) = Acceptance(Owner, slash); + var evt = new ChildRunAccepted { SessionId = Owner, Run = run }; + var serializer = Codec(evt); + Assert.Equal("cra-v1", serializer.Manifest(evt)); + var decoded = RoundTrip(evt); + Assert.Equal(serializer.ToBinary(evt), serializer.ToBinary(decoded)); + var accepted = state.Apply(decoded); + var restored = SessionState.FromSnapshot(RoundTrip(accepted.ToSnapshot())); + var actual = Assert.Single(restored.ChildRuns).Value; + Assert.Equal(run.RunId, actual.RunId); + Assert.Equal(run.ScopeId, actual.ScopeId); + Assert.Equal(run.AgentName, actual.AgentName); + Assert.Equal(run.StartKey, actual.StartKey); + Assert.Equal(run.ArgumentsDigest, actual.ArgumentsDigest); + Assert.Equal(run.SourceOperation, actual.SourceOperation); + Assert.Equal(run.AcceptedAtMs, actual.AcceptedAtMs); + Assert.True(SessionState.SameCanonicalContext(run.OriginalContext, actual.OriginalContext)); + Assert.Equal(run.OriginInputIds, actual.OriginInputIds); + Assert.Equal("authority-turn", actual.OriginalContext.TurnId); + Assert.Equal("retained-detector-task", actual.ParentCheckpoint.TaskId); + Assert.NotEqual(actual.OriginalContext.TurnId, actual.ParentCheckpoint.TaskId); + Assert.Equal(Codec(accepted.ToSnapshot()).ToBinary(accepted.ToSnapshot()), Codec(restored.ToSnapshot()).ToBinary(restored.ToSnapshot())); + Assert.Equal(run.ParentReceiptFailure, actual.ParentReceiptFailure); + Assert.Equal(slash, actual.StartBatchSettled); + Assert.Equal(run.InitialWorkingSnapshot.WorkingContext.ProjectDirectory, actual.InitialWorkingSnapshot.WorkingContext.ProjectDirectory); + Assert.Equal(run.InitialWorkingSnapshot.WorkingContext.RecentFiles, actual.InitialWorkingSnapshot.WorkingContext.RecentFiles); + Assert.Equal("main", Assert.IsType(actual.InitialWorkingSnapshot.Git).Snapshot.Branch); + Assert.Equal(new[] { "/neutral/changed.txt" }, Assert.IsType(actual.InitialWorkingSnapshot.Git).Snapshot.ChangedFiles); + Assert.Empty(restored.ActiveBackgroundJobs); + Assert.Equal(state.History, restored.History); + if (slash) + { + Assert.Null(restored.LoopAdmission); + Assert.Empty(restored.LoopObservations); + } + } + + [Theory] + [InlineData("foreign-call")] + [InlineData("source-operation")] + [InlineData("source-context")] + [InlineData("requester")] + [InlineData("origin-order")] + [InlineData("event-session")] + [InlineData("checkpoint")] + [InlineData("receipt-failure")] + public void An_acceptance_cannot_replace_canonical_admitted_evidence(string fault) + { + var (state, run) = Acceptance(Owner, slash: false); + var evt = new ChildRunAccepted { SessionId = Owner, Run = run }; + evt = fault switch + { + "foreign-call" => evt with { Run = run with { StartKey = new ChildRunStartKey.Tool(new ToolCallId("unadmitted")) { SessionId = Owner, TurnId = new TurnId("authority-turn") } } }, + "source-operation" => evt with { Run = run with { SourceOperation = "skill_load" } }, + "source-context" => evt with { Run = run with { OriginalContext = run.OriginalContext with { SourceScope = "foreign-source" } } }, + "requester" => evt with { Run = run with { OriginalContext = run.OriginalContext with { RequesterSenderId = new SenderId("operator-b") } } }, + "origin-order" => evt with { Run = run with { OriginInputIds = run.OriginInputIds.Reverse().ToArray() } }, + "event-session" => evt with { SessionId = new SessionId("signalr/foreign") }, + "checkpoint" => evt with { Run = run with { ParentCheckpoint = run.ParentCheckpoint with { LastBlockedAction = "forged-correction" } } }, + "receipt-failure" => evt with { Run = run with { ParentReceiptFailure = !run.ParentReceiptFailure } }, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + var bytes = Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot()); + Assert.Throws(() => state.Apply(RoundTrip(evt))); + Assert.Equal(bytes, Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot())); + Assert.Single(state.Apply(new ChildRunAccepted { SessionId = Owner, Run = run }).ChildRuns); + } + + [Fact] + public void A_second_committed_identity_cannot_replace_an_existing_start_after_snapshot_restore() + { + var (state, run) = Acceptance(Owner, slash: false); + state = SessionState.FromSnapshot(RoundTrip(state.Apply(new ChildRunAccepted { SessionId = Owner, Run = run }).ToSnapshot())); + var duplicate = run with + { + RunId = new SubAgentRunId("another-run"), ScopeId = new SubAgentScopeId($"{Owner.Value}/subagent/worker/another-run"), + ArgumentsDigest = new string('B', 64) + }; + var before = Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot()); + Assert.Throws(() => state.Apply(new ChildRunAccepted { SessionId = Owner, Run = duplicate })); + Assert.Equal(before, Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot())); + Assert.Equal(run.RunId, Assert.Single(state.ChildRuns).Key); + Assert.Throws(() => SessionState.FromSnapshot(RoundTrip(state.ToSnapshot() with { ChildRuns = [run, duplicate] }))); + } + + [Fact] + public void Actual_pre_change_snapshot_bytes_restore_an_empty_child_ledger_without_transcript_inference() + { + var path = Path.Combine(AppContext.BaseDirectory, "LegacyPersistence", "legacy-session-v0.json"); + using var fixture = JsonDocument.Parse(File.ReadAllBytes(path)); + Assert.Equal("2e6bc4f014dc96b606566709df1bd11f90ecf34a", fixture.RootElement.GetProperty("baseline").GetString()); + var entry = fixture.RootElement.GetProperty("entries").GetProperty("completed-snapshot"); + var bytes = Convert.FromBase64String(entry.GetProperty("base64").GetString()!); + Assert.Equal(entry.GetProperty("sha256").GetString(), Convert.ToHexString(SHA256.HashData(bytes))); + var serializer = Codec(new SessionSnapshot()); + Assert.Equal(serializer.Identifier, entry.GetProperty("serializerId").GetInt32()); + Assert.Equal(serializer.Manifest(new SessionSnapshot()), entry.GetProperty("manifest").GetString()); + var snapshot = Assert.IsType(Sys.Serialization.Deserialize(bytes, serializer.Identifier, entry.GetProperty("manifest").GetString()!)); + var state = SessionState.FromSnapshot(snapshot); + Assert.Empty(snapshot.ChildRuns); + Assert.Empty(state.ChildRuns); + Assert.Equal(1, state.TurnCount); + Assert.Equal("The neutral task is complete.", state.History[1].Content); + Assert.Equal(2, state.History.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Actual_snapshot_recovery_binds_each_run_to_the_owner_before_the_join_reply(bool foreign) + { + var (state, run) = Acceptance(foreign ? new SessionId("signalr/foreign-child-owner") : Owner, slash: true); + var snapshot = state.Apply(new ChildRunAccepted { SessionId = run.StartKey.SessionId, Run = run }).ToSnapshot() with + { History = [new SerializableChatMessage { Role = StoredRole.Assistant, Content = "The stored owner marker." }] }; + Assert.Single(SessionState.FromSnapshot(RoundTrip(snapshot)).ChildRuns); + var seed = Sys.ActorOf(Props.Create(() => new SnapshotSeeder($"session-{Owner.Value}"))); + await seed.Ask(snapshot, Ceiling, TestContext.Current.CancellationToken); + var seedWatcher = CreateTestProbe(); + seedWatcher.Watch(seed); + Sys.Stop(seed); + await seedWatcher.ExpectTerminatedAsync(seed, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + await Snapshots.OnLoad.FailIf(async (id, _) => + { + if (id != $"session-{Owner.Value}") return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return false; + }); + var observer = CreateTestProbe(); + var manager = ActorRegistry.Get(); + async Task RecoverAsync() + { + manager.Tell(new JoinSession(observer) { SessionId = Owner, Filter = OutputFilter.Full }, observer.Ref); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var owner = await Sys.ActorSelection($"/user/session-manager/{Uri.EscapeDataString(Owner.Value)}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + observer.Watch(owner); + release.TrySetResult(); + var observed = await observer.ReceiveOneAsync(Ceiling, TestContext.Current.CancellationToken); + if (foreign) + Assert.Equal(owner, Assert.IsType(observed).ActorRef); + else + { + var joined = Assert.IsType(observed); + Assert.Equal(Owner, joined.SessionId); + Assert.Equal("The stored owner marker.", Assert.Single(joined.RecentMessages!).Content); + } + } + try + { + if (foreign) + await EventFilter.Error().ExpectOneAsync(RecoverAsync, + cancellationToken: TestContext.Current.CancellationToken); + else + await RecoverAsync(); + } + finally + { + release.TrySetResult(); + } + if (foreign) + { + Assert.Equal(0, _model.CallCount); + Assert.Equal(0, _tools.CallCount); + } + } + + private static (SessionState State, BackgroundChildRun Run) Acceptance(SessionId session, bool slash) + { + var context = TurnContext.FromMessageSource(session, new TurnId("authority-turn"), new MessageSource + { + ChannelType = ChannelType.SignalR, SenderId = new SenderId("operator-a"), TurnId = new TurnId("authority-turn"), + Audience = TrustAudience.Personal, Boundary = TrustBoundary.Personal, Principal = PrincipalClassification.Operator, + Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted), + DefaultDeliveryTarget = new ChannelDeliveryTargetInfo("signalr", "destination", "operator-a", "operator-a") + }).ToRecord(); + var first = new InputAdmitted { SessionId = session, InputId = new InputId("origin-one"), TurnContext = context, + UserMessage = new SerializableChatMessage { Role = StoredRole.User, Content = "Inspect a neutral task." } }; + var second = first with { InputId = new InputId("origin-two") }; + var state = SessionState.Empty.Apply(first).Apply(second).Apply(new ToolTaskAdopted + { SessionId = session, TurnContext = context, InputIds = [first.InputId, second.InputId] }); + // The ledger must retain detector identity independently from the original authority turn. + state = state with { LoopCheckpoint = new ToolLoopCheckpoint + { + TaskId = "retained-detector-task", LastBlockedAction = "retained-correction", + Entries = [new ToolLoopEntry { ToolName = "neutral_probe", ArgumentsHash = "args", OutcomeHash = "result", EqualRounds = 2, Corrected = true }], + ColdKeys = [new ToolLoopKey("old_probe", "cold-args")], AdjacentHistory = [new ToolLoopAdjacent("action", "outcome")] + } }; + if (!slash) + state = state.ApplyLoopAdmission(new ToolBatchStarted + { + SessionId = session, LoopAdmission = new ToolLoopAdmission + { TaskId = state.LoopCheckpoint.TaskId, ActionHash = "start-action", Calls = [new ToolLoopPreparedCall("start-call", "spawn_agent", "start-args", false)] }, + LoopDelta = new ToolLoopDelta { TaskId = state.LoopCheckpoint.TaskId, LastBlockedAction = state.LoopCheckpoint.LastBlockedAction, + ColdKeys = state.LoopCheckpoint.ColdKeys, AdjacentHistory = state.LoopCheckpoint.AdjacentHistory } + }); + return (state, new BackgroundChildRun + { + RunId = new SubAgentRunId("accepted-run"), ScopeId = new SubAgentScopeId($"{session.Value}/subagent/worker/accepted-run"), + AgentName = new AgentName("worker"), SourceOperation = slash ? "/inspect" : "spawn_agent", ArgumentsDigest = new string('A', 64), + StartKey = slash ? new ChildRunStartKey.Slash(first.InputId) { SessionId = session, TurnId = new TurnId(context.TurnId) } + : new ChildRunStartKey.Tool(new ToolCallId("start-call")) { SessionId = session, TurnId = new TurnId(context.TurnId) }, + OriginalContext = context, OriginInputIds = state.AdoptedTaskInputIds, ParentCheckpoint = state.LoopCheckpoint, + ParentReceiptFailure = state.LoopReceiptFailure, StartBatchSettled = slash, AcceptedAtMs = 123456, + InitialWorkingSnapshot = new WorkingContextSnapshot + { + WorkingContext = new WorkingContext { ProjectDirectory = "/neutral", RecentFiles = ["/neutral/source.txt", "/neutral/result.txt"] }, + Git = new GitWorkingContextInspection.Available(new GitWorkingContextSnapshot + { Worktree = "/neutral", CommonDirectory = "/neutral/.git", Branch = "main", Head = "head", ChangedFiles = ["/neutral/changed.txt"] }) + } + }); + } + + private NetclawProtobufSerializer Codec(object value) => Assert.IsType(Sys.Serialization.FindSerializerFor(value)); + private T RoundTrip(T value) where T : notnull + { + var serializer = Codec(value); + return Assert.IsType(Sys.Serialization.Deserialize(serializer.ToBinary(value), serializer.Identifier, serializer.Manifest(value))); + } + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + private sealed class SnapshotSeeder : ReceivePersistentActor + { + public override string PersistenceId { get; } + public SnapshotSeeder(string id) + { + PersistenceId = id; + IActorRef? reply = null; + RecoverAny(_ => { }); + Command(evt => + { + var acknowledgement = Sender; + Persist(evt, _ => acknowledgement.Tell(LastSequenceNr)); + }); + Command(snapshot => + { + reply = Sender; + var serializer = Assert.IsType(Context.System.Serialization.FindSerializerFor(snapshot)); + SaveSnapshot(Context.System.Serialization.Deserialize(serializer.ToBinary(snapshot), serializer.Identifier, serializer.Manifest(snapshot))); + }); + Command(_ => reply!.Tell(Done.Instance)); + Command(failed => reply!.Tell(new Status.Failure(failed.Cause))); + } + } + + async ValueTask IAsyncDisposable.DisposeAsync() + { + try { await base.DisposeAsync(); } + finally { if (_directory is not null) await _directory.DisposeAsync(); } + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs deleted file mode 100644 index fb35ad16a..000000000 --- a/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs +++ /dev/null @@ -1,271 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -using Akka.Actor; -using Akka.Hosting; -using Akka.Hosting.TestKit; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using Netclaw.Actors.SubAgents; -using Netclaw.Actors.Sessions; -using Netclaw.Actors.Tests.Memory; -using Netclaw.Actors.Tools; -using Netclaw.Configuration; -using Netclaw.Security; -using Netclaw.Tests.Utilities; -using Netclaw.Tools; -using Xunit; - -namespace Netclaw.Actors.Tests.SubAgents; - -/// -/// Regression guard for the default-interface-method dispatch gap that let -/// spawn_agent bypass its streaming override and run the non-streaming -/// path — emitting zero activity items. Exercises the full chain: -/// → INetclawTool dispatch → -/// → , drained the way -/// the production pipeline drains a self-monitoring tool. -/// -public class SpawnAgentStreamingTests : TestKit -{ - public SpawnAgentStreamingTests(ITestOutputHelper output) : base(output: output) { } - - protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) - { - // No persistence or hosting needed — SubAgentActor is spawned standalone. - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Spawn_agent_streams_activity_through_executor_dispatch_to_watchdog(bool fail) - { - using var dir = new DisposableTempDir(); - var paths = new NetclawPaths(dir.Path); - paths.EnsureDirectoriesExist(); - - var toolAccessPolicy = new ToolAccessPolicy(new NetclawPaths(), - new ToolConfig(), - new EffectivePolicyDefaults( - DeploymentPosture.Personal, - TrustAudience.Personal, - ShellExecutionMode.HostAllowed, - UsedStrictFallback: false), - new ShellCommandPolicy(), - new ToolPathPolicy([])); - - // The sub-agent resolves "file_read" from this registry; the fake LLM - // never calls it — it just has to resolve so the spawn proceeds. - var registry = new ToolRegistry(); - registry.Register(new FakeNetclawTool("file_read", "stub content")); - - var subAgentRegistry = new SubAgentDefinitionRegistry(); - subAgentRegistry.Register(new SubAgentProfile - { - Name = "summarizer", - Description = "Summarize content", - SystemPrompt = "You are a summarizer.", - ToolNames = ["file_read"], - Visibility = SubAgentVisibility.UserFacing - }); - - var spawner = new SubAgentSpawner( - new SingleClientProvider(new FakeChatClient { Failure = fail ? new InvalidOperationException("fixture LLM failure") : null }), - registry, - toolAccessPolicy, - approvalService: null, - new StaticSystemPromptProvider("You are a summarizer."), - new WorkingContextSnapshotProvider( - new GitWorkingContextInspector(TimeProvider.System), - NullLogger.Instance), - NullLogger.Instance); - - registry.Register(new SpawnAgentTool(subAgentRegistry, spawner, paths)); - - var executor = new DispatchingToolExecutor( - registry, toolAccessPolicy, approvalService: null, NullLogger.Instance); - - var ctx = TestToolExecutionContext.CreateBound("console/streaming-test", dir.Path, new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - SpawnChildActor = (props, name, _) => Task.FromResult(Sys.ActorOf((Props)props, name)), - }); - - var spawnCall = new FunctionCallContent( - "call-1", - "spawn_agent", - new Dictionary - { - ["agent"] = "summarizer", - ["task"] = "Summarize the project.", - ["_rationale"] = "Verify streamed sub-agent activity." - }); - - // Drain the stream the way the production pipeline drains a self-monitoring - // tool, collecting activity items along the way. - var activity = new List(); - string? result = null; - await foreach (var update in executor.ExecuteStreamAsync(spawnCall, ctx, TestContext.Current.CancellationToken)) - { - switch (update) - { - case ToolActivityUpdate a: - activity.Add(a); - break; - case ToolCompletedUpdate done: - result = done.Result; - break; - } - } - - // The DIM dispatch gap ran spawn_agent on the non-streaming path: zero - // activity items. The streaming override emits progress. - if (fail) - { - Assert.NotNull(result); - Assert.Contains("Outcome: failed", result, StringComparison.OrdinalIgnoreCase); - Assert.DoesNotContain("LogPath:", result); - Assert.DoesNotContain("ArtifactDirectory:", result); - return; - } - Assert.NotEmpty(activity); - Assert.Contains("LogPath:", result); - Assert.Contains("ArtifactDirectory:", result); - - // The terminal result still flows through — a successful sub-agent run, - // not a "Subagent '...' failed: ..." message from FormatResult. - Assert.NotNull(result); - Assert.NotEmpty(result); - Assert.Contains("Subagent run finished.", result, StringComparison.Ordinal); - Assert.Contains("Outcome: completed", result, StringComparison.OrdinalIgnoreCase); - Assert.Contains("Summary:", result, StringComparison.Ordinal); - Assert.DoesNotContain("failed", result, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task Spawn_agent_self_monitoring_survives_quiet_window_after_first_activity() - { - using var dir = new DisposableTempDir(); - var paths = new NetclawPaths(dir.Path); - paths.EnsureDirectoriesExist(); - - var toolAccessPolicy = new ToolAccessPolicy(new NetclawPaths(), - new ToolConfig(), - new EffectivePolicyDefaults( - DeploymentPosture.Personal, - TrustAudience.Personal, - ShellExecutionMode.HostAllowed, - UsedStrictFallback: false), - new ShellCommandPolicy(), - new ToolPathPolicy([])); - - var registry = new ToolRegistry(); - registry.Register(new FakeNetclawTool("file_read", "stub content")); - - var subAgentRegistry = new SubAgentDefinitionRegistry(); - subAgentRegistry.Register(new SubAgentProfile - { - Name = "summarizer", - Description = "Summarize content", - SystemPrompt = "You are a summarizer.", - ToolNames = ["file_read"], - Visibility = SubAgentVisibility.UserFacing - }); - - var fakeClient = new ControlledStreamingChatClient(); - var spawner = new SubAgentSpawner( - new SingleClientProvider(fakeClient), - registry, - toolAccessPolicy, - approvalService: null, - new StaticSystemPromptProvider("You are a summarizer."), - new WorkingContextSnapshotProvider( - new GitWorkingContextInspector(TimeProvider.System), - NullLogger.Instance), - NullLogger.Instance); - - registry.Register(new SpawnAgentTool(subAgentRegistry, spawner, paths)); - - var executor = new DispatchingToolExecutor( - registry, toolAccessPolicy, approvalService: null, NullLogger.Instance); - - var ctx = TestToolExecutionContext.CreateBound("console/self-monitoring-test", dir.Path, new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - SpawnChildActor = (props, name, _) => Task.FromResult(Sys.ActorOf((Props)props, name)), - }); - - var spawnCall = new FunctionCallContent( - "call-1", - "spawn_agent", - new Dictionary - { - ["agent"] = "summarizer", - ["task"] = "Summarize the project.", - ["_rationale"] = "Verify self-monitored sub-agent activity." - }); - - // spawn_agent is self-monitoring, so the parent drains it with no watchdog at - // all — a long quiet window can never trip a timeout; the sub-agent owns its - // own liveness and always yields a terminal item. - Assert.Equal(ToolLivenessMode.SelfMonitoring, executor.GetLivenessMode(spawnCall)); - - async Task DrainAsync() - { - string? completed = null; - await foreach (var update in executor.ExecuteStreamAsync(spawnCall, ctx, TestContext.Current.CancellationToken)) - { - if (update is ToolCompletedUpdate done) - completed = done.Result; - } - return completed; - } - - var drain = DrainAsync(); - - // The drain cannot finish until the sub-agent produces its terminal result. - Assert.False(drain.IsCompleted); - - fakeClient.Complete("summary complete"); - - var result = await drain.WaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - Assert.NotNull(result); - Assert.Contains("Subagent run finished.", result, StringComparison.Ordinal); - Assert.Contains("Outcome: completed", result, StringComparison.OrdinalIgnoreCase); - Assert.Contains("summary complete", result); - } - - private sealed class ControlledStreamingChatClient : IChatClient - { - private readonly TaskCompletionSource _response = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - => throw new NotSupportedException("ControlledStreamingChatClient is streaming-only."); - - public async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var responseText = await _response.Task.WaitAsync(cancellationToken); - var response = new ChatResponse(new ChatMessage(ChatRole.Assistant, [new TextContent(responseText)])); - - foreach (var update in response.ToChatResponseUpdates()) - { - cancellationToken.ThrowIfCancellationRequested(); - yield return update; - } - } - - public void Complete(string responseText) => _response.TrySetResult(responseText); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - public void Dispose() { } - } -} diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs index 901daa523..e5cba1fd9 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs @@ -1845,6 +1845,58 @@ public async Task LLM_failure_returns_failure() Assert.Empty(result.Findings); } + [Fact] + public async Task Background_terminal_closes_dispatch_before_the_owner_acknowledges_receipt() + { + var ceiling = TimeSpan.FromSeconds(30); + var dispatch = new ChildRunDispatch(); + var admittedEntries = 0; + Assert.Equal(1, await dispatch.Enter(() => Task.FromResult(++admittedEntries))); + var runId = new SubAgentRunId("terminal-gate-run"); + var scope = SubAgentTestScope.Create( + scopeId: "test-session/subagent/test-agent/terminal-gate-run", + sessionDirectory: TestPath("terminal", "dispatch-gate")); + var client = new FakeChatClient { ResponseText = "The assigned task is complete." }; + var agent = await ChildActorOfAsync( + SubAgentActor.CreateProps(CreateDefinition(), client, PermissivePolicy()), + cancellationToken: TestContext.Current.CancellationToken); + Watch(agent); + try + { + agent.Tell(new RunBackgroundSubAgent(runId, new RunSubAgent + { + Scope = scope, Task = "Return the neutral task result.", Timeout = ceiling + }, dispatch), TestActor); + var terminal = await ExpectMsgAsync(ceiling, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(agent, LastSender); + Assert.Equal(runId, terminal.RunId); + Assert.Equal(runId, terminal.Result.RunId); + Assert.Equal(scope.ScopeId, terminal.Result.ScopeId); + Assert.True(terminal.Result.Success); + Assert.Equal("The assigned task is complete.", terminal.Result.Output); + Assert.Equal(agent, await Sys.ActorSelection(agent.Path).ResolveOne(ceiling, + TestContext.Current.CancellationToken)); + var lateEntries = 0; + await Assert.ThrowsAnyAsync(async () => + await dispatch.Enter(() => + { + ++lateEntries; + return Task.CompletedTask; + })); + Assert.Equal(0, lateEntries); + Assert.Equal(1, admittedEntries); + Assert.Equal(1, client.CallCount); + agent.Tell(new BackgroundChildTerminalAck(runId), TestActor); + await ExpectTerminatedAsync(agent, ceiling, + cancellationToken: TestContext.Current.CancellationToken); + } + finally + { + Sys.Stop(agent); + } + } + [Fact] public async Task Actor_stops_after_completion() { diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs index 7a44c6a27..6ae25fc7b 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs @@ -59,10 +59,11 @@ public async Task Spawner_missing_session_context_logs_lifecycle_under_session_s // "subagent tried to spawn but never launched" failure shape. var context = TestToolExecutionContext.CreateBound(SessionId, null, TrustAudience.Personal); - var result = await spawner.SpawnAsync( - Profile("summarizer"), "do the work", null, context.Invocation, TestContext.Current.CancellationToken); + var result = await spawner.StartRunAsync( + Profile("summarizer"), "do the work", null, context.Invocation, TestContext.Current.CancellationToken, null); - Assert.False(result.Success); + Assert.Contains("no session context available", result, StringComparison.Ordinal); + Assert.Equal(ToolInvocationOutcomeCategory.TransientFailure, context.Receipt?.Category); // The spawn attempt and its failure are both logged under the session's id, so the // file-logger routes them to that session's session.log. var requested = Assert.Single(logger.Entries, e => e.Message.Contains("spawn requested", StringComparison.Ordinal)); diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs index cb7d105d4..ae1e5f497 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs @@ -8,9 +8,9 @@ using Akka.Hosting.TestKit; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging.Abstractions; -using System.Threading.Channels; using Netclaw.Actors.Authorization.Consent; using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; using Netclaw.Actors.SubAgents; using Netclaw.Actors.Sessions; using Netclaw.Actors.Tests.Memory; @@ -55,7 +55,7 @@ async ValueTask IAsyncDisposable.DisposeAsync() } [Fact] - public async Task Spawn_async_propagates_parent_resolved_cwd_on_run_message() + public async Task Background_preparation_propagates_parent_resolved_cwd_on_run_message() { var environment = ShellExecutionEnvironment.CreatePowerShell( @"C:\Program Files\PowerShell\7\pwsh.exe", @@ -100,42 +100,32 @@ public async Task Spawn_async_propagates_parent_resolved_cwd_on_run_message() Visibility = SubAgentVisibility.UserFacing }; - var spawnTask = spawner.SpawnAsync( + var prepared = Assert.IsType(await spawner.PrepareRunAsync( profile, "Summarize the repo.", runtimeContext: null, context.Invocation, - TestContext.Current.CancellationToken); + TestContext.Current.CancellationToken, systemPromptOverlay: null)).Run; - var run = await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + var run = prepared.Execution; var bound = Assert.IsType(run.Scope.Authority.Session); Assert.Equal(_parentSessionDir.Path, bound.SessionDirectory); Assert.Equal(_testProjectDir.Path, run.Scope.Authority.ProjectDirectory); Assert.Equal(_testProjectDir.Path, run.Scope.Authority.InheritedCwd); Assert.Same(environment, run.Scope.InitialWorkingSnapshot.ShellEnvironment); - - childProbe.Reply(new SubAgentResult - { - Completion = new ChildRunCompletion.Completed(WorkingContextDelta.Empty), - Output = "ok", - AgentName = new AgentName(profile.Name) - }); - - var result = await spawnTask; - Assert.True(result.Success); } [Theory] [InlineData(ChannelType.Headless)] [InlineData(ChannelType.Reminder)] [InlineData(ChannelType.Webhook)] - public async Task Spawn_async_does_not_bridge_approval_for_non_interactive_parent(ChannelType channelType) + public async Task Background_preparation_does_not_bridge_approval_for_non_interactive_parent(ChannelType channelType) { var childProbe = CreateTestProbe($"non-interactive-{channelType}-child"); var spawner = CreateSpawner(); var context = TestToolExecutionContext.CreateBound( "automation/subagent-parent", - Path.GetTempPath(), + _parentSessionDir.Path, new TestToolExecutionContextOptions { Audience = TrustAudience.Personal, @@ -144,30 +134,26 @@ public async Task Spawn_async_does_not_bridge_approval_for_non_interactive_paren SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref) }); - var spawnTask = spawner.SpawnAsync( + var prepared = Assert.IsType(await spawner.PrepareRunAsync( CreateProfile(), "Inspect the system.", runtimeContext: null, context.Invocation, - TestContext.Current.CancellationToken); + TestContext.Current.CancellationToken, systemPromptOverlay: null)).Run; - var run = await childProbe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); + var run = prepared.Execution; Assert.IsType(run.Scope.Authority.InteractiveApproval); - - childProbe.Reply(SuccessfulResult()); - Assert.True((await spawnTask).Success); } [Fact] - public async Task Spawn_async_preserves_approval_bridge_for_interactive_parent() + public async Task Background_preparation_preserves_approval_bridge_for_interactive_parent() { var childProbe = CreateTestProbe("interactive-approval-child"); var approvalBridge = new RecordingParentApprovalBridge(ConsentAnswer.Once.Instance); var spawner = CreateSpawner(); var context = TestToolExecutionContext.CreateBound( "interactive/subagent-parent", - Path.GetTempPath(), + _parentSessionDir.Path, new TestToolExecutionContextOptions { Audience = TrustAudience.Personal, @@ -176,25 +162,21 @@ public async Task Spawn_async_preserves_approval_bridge_for_interactive_parent() SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref) }); - var spawnTask = spawner.SpawnAsync( + var prepared = Assert.IsType(await spawner.PrepareRunAsync( CreateProfile(), "Inspect the system.", runtimeContext: null, context.Invocation, - TestContext.Current.CancellationToken); + TestContext.Current.CancellationToken, systemPromptOverlay: null)).Run; - var run = await childProbe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); + var run = prepared.Execution; var available = Assert.IsType( run.Scope.Authority.InteractiveApproval); Assert.Same(approvalBridge, available.Bridge); - - childProbe.Reply(SuccessfulResult()); - Assert.True((await spawnTask).Success); } [Fact] - public async Task Spawn_async_ignores_definition_tool_metadata_for_runtime_tool_resolution() + public async Task Background_preparation_ignores_definition_tool_metadata_for_runtime_tool_resolution() { var toolRegistry = new ToolRegistry(); toolRegistry.Register(new FakeNetclawTool("inspect_context", "ok")); @@ -236,30 +218,19 @@ public async Task Spawn_async_ignores_definition_tool_metadata_for_runtime_tool_ Visibility = SubAgentVisibility.UserFacing }; - var spawnTask = spawner.SpawnAsync( + var prepared = Assert.IsType(await spawner.PrepareRunAsync( profile, "Summarize the repo.", runtimeContext: null, context.Invocation, - TestContext.Current.CancellationToken); - - await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - childProbe.Reply(new SubAgentResult - { - Completion = new ChildRunCompletion.Completed(WorkingContextDelta.Empty), - Output = "ok", - AgentName = new AgentName(profile.Name) - }); + TestContext.Current.CancellationToken, systemPromptOverlay: null)).Run; - var result = await spawnTask; - - Assert.True(result.Success); - var started = Assert.Single(notifications, n => n.IsStarted); - Assert.Equal(1, started.ToolCount); + Assert.Equal(1, prepared.ToolCount); + Assert.Empty(notifications); } [Fact] - public async Task Spawn_async_returns_only_unconfirmed_git_changes_as_observed() + public async Task Terminal_enrichment_returns_only_unconfirmed_git_changes_as_observed() { await using var spawnerContextDir = TestSessionTempDirectory.Create("netclaw-spawner-"); var projectDirectory = spawnerContextDir.Path; @@ -279,45 +250,27 @@ public async Task Spawn_async_returns_only_unconfirmed_git_changes_as_observed() GitSnapshot(projectDirectory, "src/Confirmed.cs", "src/Observed.cs")) } ]); - var spawner = CreateSpawner(new SequenceWorkingContextSnapshotProvider(snapshots)); - var childProbe = CreateTestProbe("working-context-child"); - var context = TestToolExecutionContext.CreateBound("console/subagent-parent", _parentSessionDir.Path, new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - ProjectDirectory = projectDirectory, - SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref) - }); - - var spawnTask = spawner.SpawnAsync( - CreateProfile(), - "Update the project.", - runtimeContext: null, - context.Invocation, - TestContext.Current.CancellationToken); - - await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - childProbe.Reply(SuccessfulResult() with + var initial = snapshots.Dequeue(); + var result = await SubAgentSpawner.EnrichWorkingContextResultAsync(SuccessfulResult() with { Completion = new ChildRunCompletion.Completed(new WorkingContextDelta { ProjectDirectory = projectDirectory, ConfirmedChangedFiles = [confirmedPath] }) - }); - - var result = await spawnTask; + }, initial, TrustAudience.Personal, new SequenceWorkingContextSnapshotProvider(snapshots), + TestContext.Current.CancellationToken); Assert.Equal([confirmedPath], result.WorkingContext!.ConfirmedChangedFiles); Assert.Equal([observedPath], result.WorkingContext.ObservedChangedFiles); } [Fact] - public async Task Initial_snapshot_cancellation_completes_stream_without_starting_child() + public async Task Initial_snapshot_cancellation_rejects_before_owner_acceptance() { using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); var spawner = CreateSpawner(new CancelledWorkingContextSnapshotProvider()); - var channel = Channel.CreateUnbounded(); var childSpawned = false; var notifications = new List(); var context = TestToolExecutionContext.CreateBound( @@ -334,25 +287,23 @@ public async Task Initial_snapshot_cancellation_completes_stream_without_startin SubAgentActivitySink = notifications.Add }); - var result = await spawner.SpawnAsync( + var result = await spawner.PrepareRunAsync( CreateProfile(), "Inspect the system.", runtimeContext: null, context.Invocation, cancellation.Token, - activitySink: channel.Writer); + systemPromptOverlay: null); - Assert.IsType(result.Completion); + Assert.IsType(Assert.IsType(result).Result.Response.Completion); Assert.False(childSpawned); Assert.Empty(notifications); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); } [Fact] - public async Task Initial_snapshot_failure_completes_stream_without_starting_child() + public async Task Initial_snapshot_failure_rejects_before_owner_acceptance() { var spawner = CreateSpawner(new FailedWorkingContextSnapshotProvider()); - var channel = Channel.CreateUnbounded(); var childSpawned = false; var context = TestToolExecutionContext.CreateBound( "console/subagent-parent", @@ -367,25 +318,23 @@ public async Task Initial_snapshot_failure_completes_stream_without_starting_chi } }); - var result = await spawner.SpawnAsync( + var result = await spawner.PrepareRunAsync( CreateProfile(), "Inspect the system.", runtimeContext: null, context.Invocation, TestContext.Current.CancellationToken, - activitySink: channel.Writer); + systemPromptOverlay: null); - var failed = Assert.IsType(result.Completion); + var failed = Assert.IsType(Assert.IsType(result).Result.Response.Completion); Assert.Equal(SubAgentOutcomeReason.SpawnError, failed.FailureReason); Assert.False(childSpawned); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); } [Fact] - public async Task Fatal_initial_snapshot_failure_completes_stream_and_propagates() + public async Task Fatal_initial_snapshot_failure_propagates_before_owner_acceptance() { var spawner = CreateSpawner(new FatalWorkingContextSnapshotProvider()); - var channel = Channel.CreateUnbounded(); var childSpawned = false; var context = TestToolExecutionContext.CreateBound( "console/subagent-parent", @@ -400,141 +349,21 @@ public async Task Fatal_initial_snapshot_failure_completes_stream_and_propagates } }); - await Assert.ThrowsAsync(() => spawner.SpawnAsync( + await Assert.ThrowsAsync(() => spawner.PrepareRunAsync( CreateProfile(), "Inspect the system.", runtimeContext: null, context.Invocation, TestContext.Current.CancellationToken, - activitySink: channel.Writer)); + systemPromptOverlay: null)); Assert.False(childSpawned); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task Cancellation_while_waiting_for_child_returns_typed_cancellation_and_completes_stream() - { - using var cancellation = new CancellationTokenSource(); - var childProbe = CreateTestProbe("cancelled-child"); - var channel = Channel.CreateUnbounded(); - var notifications = new List(); - var spawner = CreateSpawner(new SequenceWorkingContextSnapshotProvider(new Queue( - [ - EmptySnapshot() - ]))); - var context = TestToolExecutionContext.CreateBound( - "console/subagent-parent", - _parentSessionDir.Path, - new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref), - SubAgentActivitySink = notifications.Add - }); - - var spawn = spawner.SpawnAsync( - CreateProfile(), - "Inspect the system.", - runtimeContext: null, - context.Invocation, - cancellation.Token, - activitySink: channel.Writer); - await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - - cancellation.Cancel(); - var result = await spawn; - - Assert.IsType(result.Completion); - Assert.Contains(notifications, notification => notification.IsStarted); - Assert.Contains(notifications, notification => - !notification.IsStarted && notification.OutcomeReason == SubAgentOutcomeReason.CancelledByParent); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task Final_snapshot_cancellation_returns_typed_cancellation_and_completes_stream() - { - using var cancellation = new CancellationTokenSource(); - var childProbe = CreateTestProbe("final-snapshot-cancelled-child"); - var channel = Channel.CreateUnbounded(); - var spawner = CreateSpawner(new CancelOnSecondWorkingContextSnapshotProvider(cancellation)); - var context = TestToolExecutionContext.CreateBound( - "console/subagent-parent", - _parentSessionDir.Path, - new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref) - }); - - var spawn = spawner.SpawnAsync( - CreateProfile(), - "Inspect the system.", - runtimeContext: null, - context.Invocation, - cancellation.Token, - activitySink: channel.Writer); - await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - childProbe.Reply(SuccessfulResult() with - { - Completion = new ChildRunCompletion.Completed(new WorkingContextDelta - { - ProjectDirectory = Path.GetTempPath() - }) - }); - - var result = await spawn; - - Assert.IsType(result.Completion); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task Fatal_final_snapshot_failure_completes_stream_and_propagates() - { - var childProbe = CreateTestProbe("final-snapshot-fatal-child"); - var channel = Channel.CreateUnbounded(); - var spawner = CreateSpawner(new FatalOnSecondWorkingContextSnapshotProvider()); - var context = TestToolExecutionContext.CreateBound( - "console/subagent-parent", - _parentSessionDir.Path, - new TestToolExecutionContextOptions - { - Audience = TrustAudience.Personal, - SpawnChildActor = (_, _, _) => Task.FromResult(childProbe.Ref) - }); - - var spawn = spawner.SpawnAsync( - CreateProfile(), - "Inspect the system.", - runtimeContext: null, - context.Invocation, - TestContext.Current.CancellationToken, - activitySink: channel.Writer); - await childProbe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - childProbe.Reply(SuccessfulResult() with - { - Completion = new ChildRunCompletion.Completed(new WorkingContextDelta - { - ProjectDirectory = Path.GetTempPath() - }) - }); - - await Assert.ThrowsAsync(() => spawn); - await channel.Reader.Completion.WaitAsync(TestContext.Current.CancellationToken); } [Fact] public async Task Spawned_sub_agent_bills_its_llm_calls_to_session_metrics() { - // Full-wiring regression guard for #1597: a sub-agent spawned through the real - // SubAgentSpawner must record its LLM-call tokens to the ISessionMetrics handed - // to the spawner. Unlike the actor-level tests, this exercises the - // spawner -> CreateProps -> actor pass-through, so dropping the metrics argument - // anywhere along that chain fails here. The SpawnChildActor factory materializes - // the spawner-built Props into a real SubAgentActor (a probe stand-in would - // bypass CreateProps entirely and hide a broken pass-through). + // The actual prepared Props must preserve the process-wide metrics sink. var toolRegistry = new ToolRegistry(); toolRegistry.Register(new FakeNetclawTool("inspect_context", "ok")); @@ -567,7 +396,7 @@ public async Task Spawned_sub_agent_bills_its_llm_calls_to_session_metrics() var context = TestToolExecutionContext.CreateBound("console/subagent-parent", _parentSessionDir.Path, new TestToolExecutionContextOptions { Audience = TrustAudience.Personal, - SpawnChildActor = (props, name, _) => Task.FromResult(Sys.ActorOf((Props)props, name)), + SpawnChildActor = (_, _, _) => Task.FromResult(TestActor), }); var profile = new SubAgentProfile @@ -579,14 +408,14 @@ public async Task Spawned_sub_agent_bills_its_llm_calls_to_session_metrics() Visibility = SubAgentVisibility.UserFacing }; - var result = await spawner.SpawnAsync( - profile, - "Summarize the repo.", - runtimeContext: null, - context.Invocation, - TestContext.Current.CancellationToken); - - Assert.True(result.Success, $"Expected success but got: {result.Output}"); + var prepared = Assert.IsType(await spawner.PrepareRunAsync( + profile, "Summarize the repo.", null, context.Invocation, + TestContext.Current.CancellationToken, null)).Run; + var owner = Sys.ActorOf(Props.Create(() => new MetricsRunOwner(prepared, TestActor))); + owner.Tell("start", TestActor); + var terminal = await ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.True(terminal.Result.Success); + owner.Tell(new BackgroundChildTerminalAck(prepared.RunId), TestActor); // One text-only LLM call → exactly one usage record, carrying the fake's tokens. var call = Assert.Single(metrics.TokenUsageCalls); Assert.Equal((175L, 60L), call); @@ -640,8 +469,9 @@ public async Task AuthorityRegression_Child_spawner_retains_team_file_authority( Name = "reader", Description = "Read a file", SystemPrompt = "Read the supplied file.", ToolNames = ["file_read"], Visibility = SubAgentVisibility.UserFacing }; - var spawn = spawner.SpawnAsync(profile, "Read the supplied file.", runtimeContext: null, parent.Invocation, TestContext.Current.CancellationToken); - var run = await probe.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + var prepared = Assert.IsType(await spawner.PrepareRunAsync( + profile, "Read the supplied file.", null, parent.Invocation, TestContext.Current.CancellationToken, null)).Run; + var run = prepared.Execution; Assert.Equal(parent.Audience, run.Scope.Authority.Audience); Assert.Equal(parent.Boundary, run.Scope.Authority.Boundary); var childStorage = Assert.IsType(run.Scope.Authority.Session).Storage; @@ -657,14 +487,15 @@ public async Task AuthorityRegression_Child_spawner_retains_team_file_authority( Directory.CreateDirectory(childStorage.ArtifactDirectory.Value); var artifact = Path.Combine(childStorage.ArtifactDirectory.Value, "result.txt"); await File.WriteAllTextAsync(artifact, "child-artifact", TestContext.Current.CancellationToken); - probe.Reply(new SubAgentResult + var actorResult = new SubAgentResult { Completion = partial ? new ChildRunCompletion.Partial(SubAgentOutcomeReason.ToolIterationBudgetExhausted, WorkingContextDelta.Empty) : new ChildRunCompletion.Completed(WorkingContextDelta.Empty), Output = "child-summary", AgentName = new AgentName("reader") - }); - var completed = await spawn; + }; + var completed = new EnrichedChildRunResult.SuccessfulRun(actorResult, + new EnrichedChildRunResult.RunLocations(childStorage.LogPath, childStorage.ArtifactDirectory)).ToProtocolResult(); Assert.True(completed.Success); Assert.Equal("child-summary", completed.Output); Assert.Equal(childStorage.LogPath.Value, completed.LogPath); @@ -708,6 +539,18 @@ private static SubAgentSpawner CreateSpawner(IWorkingContextSnapshotProvider wor ChangedFiles = [.. changedFiles] }; + private sealed class MetricsRunOwner : ReceiveActor + { + public MetricsRunOwner(PreparedChildRun prepared, IActorRef observer) + { + var child = Context.ActorOf(prepared.Props); + Receive(_ => child.Tell(new RunBackgroundSubAgent( + prepared.RunId, prepared.Execution, new ChildRunDispatch()), Self)); + Receive(terminal => observer.Tell(terminal, Self)); + Receive(ack => child.Tell(ack, Self)); + } + } + private static WorkingContextSnapshot EmptySnapshot() => new() { WorkingContext = WorkingContext.Empty, @@ -766,34 +609,4 @@ public Task CreateAsync( Task.FromException(new OutOfMemoryException("snapshot failed fatally")); } - private sealed class FatalOnSecondWorkingContextSnapshotProvider : IWorkingContextSnapshotProvider - { - private int _invocation; - - public Task CreateAsync( - WorkingContext context, - TrustAudience audience, - CancellationToken cancellationToken) => ++_invocation == 1 - ? Task.FromResult(EmptySnapshot()) - : Task.FromException( - new OutOfMemoryException("snapshot failed fatally")); - } - - private sealed class CancelOnSecondWorkingContextSnapshotProvider(CancellationTokenSource cancellation) - : IWorkingContextSnapshotProvider - { - private int _invocations; - - public Task CreateAsync( - WorkingContext context, - TrustAudience audience, - CancellationToken cancellationToken) - { - if (Interlocked.Increment(ref _invocations) == 1) - return Task.FromResult(EmptySnapshot()); - - cancellation.Cancel(); - return Task.FromCanceled(cancellationToken); - } - } } diff --git a/src/Netclaw.Actors/Protocol/SessionSnapshot.cs b/src/Netclaw.Actors/Protocol/SessionSnapshot.cs index 92314ec37..6bc563ed2 100644 --- a/src/Netclaw.Actors/Protocol/SessionSnapshot.cs +++ b/src/Netclaw.Actors/Protocol/SessionSnapshot.cs @@ -6,6 +6,7 @@ using Netclaw.Actors.Jobs; using Netclaw.Actors.Serialization; using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; using static Netclaw.Actors.Sessions.SessionProtocol; namespace Netclaw.Actors.Protocol; @@ -62,6 +63,8 @@ public sealed record AdoptedContextSnapshotMessage public Sessions.SessionProtocol.TurnContextRecord? AdoptedTaskContext { get; init; } public IReadOnlyList AdoptedTaskInputIds { get; init; } = []; + public IReadOnlyList ChildRuns { get; init; } = []; + public int TurnCount { get; init; } public string? Title { get; init; } diff --git a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildLifecycle.cs b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildLifecycle.cs new file mode 100644 index 000000000..7a4b6cd05 --- /dev/null +++ b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildLifecycle.cs @@ -0,0 +1,156 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Proto = Netclaw.Actors.Serialization.Proto; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Serialization; + +internal static partial class NetclawProtoMapper +{ + internal static Proto.ChildRunEventProto ToProto(ChildRunEvent evt) + { + var proto = new Proto.ChildRunEventProto + { + SessionId = evt.SessionId.Value, RunId = evt.RunId.Value, RecordedAtMs = evt.RecordedAtMs + }; + switch (evt) + { + case ChildRunEvent.Started: proto.Started = true; break; + case ChildRunEvent.Checkpointed checkpoint: proto.Checkpoint = ToProto(checkpoint.Checkpoint); break; + case ChildRunEvent.CancellationRequested: proto.CancellationRequested = true; break; + case ChildRunEvent.DispatchClosed: proto.DispatchClosed = true; break; + case ChildRunEvent.TerminalRecorded terminal when terminal.TerminalSequenceNr > 0: + proto.Terminal = ToProto(terminal.Terminal); proto.TerminalSequenceNr = terminal.TerminalSequenceNr; break; + case ChildRunEvent.ResultPrepared prepared when prepared.TerminalSequenceNr > 0: + proto.Prepared = ToProto(prepared.Terminal); proto.TerminalSequenceNr = prepared.TerminalSequenceNr; break; + case ChildRunEvent.DeliveryAdmitted delivery: proto.Delivery = ToProto(delivery.Input); break; + default: throw new InvalidDataException("A child lifecycle event has an unknown fact."); + } + return proto; + } + + internal static ChildRunEvent FromProto(Proto.ChildRunEventProto proto) + { + var sessionId = new SessionId(proto.SessionId); + var runId = new SubAgentRunId(proto.RunId); + return proto.FactCase switch + { + Proto.ChildRunEventProto.FactOneofCase.Started when proto.Started => new ChildRunEvent.Started + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.Checkpoint => new ChildRunEvent.Checkpointed(FromProto(proto.Checkpoint)) + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.CancellationRequested when proto.CancellationRequested => new ChildRunEvent.CancellationRequested + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.DispatchClosed when proto.DispatchClosed => new ChildRunEvent.DispatchClosed + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.Terminal when proto.TerminalSequenceNr > 0 => new ChildRunEvent.TerminalRecorded(FromProto(proto.Terminal), proto.TerminalSequenceNr) + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.Prepared when proto.TerminalSequenceNr > 0 => new ChildRunEvent.ResultPrepared(FromProto(proto.Prepared), proto.TerminalSequenceNr) + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + Proto.ChildRunEventProto.FactOneofCase.Delivery => new ChildRunEvent.DeliveryAdmitted(FromProto(proto.Delivery)) + { SessionId = sessionId, RunId = runId, RecordedAtMs = proto.RecordedAtMs }, + _ => throw new InvalidDataException("A child lifecycle event has no valid fact.") + }; + } + + private static Proto.ChildWorkingContextProto ToProto(WorkingContextDelta delta) + { + var proto = new Proto.ChildWorkingContextProto(); + if (delta.ProjectDirectory is not null) proto.ProjectDirectory = delta.ProjectDirectory; + if (delta.Worktree is not null) proto.Worktree = delta.Worktree; + if (delta.Branch is not null) proto.Branch = delta.Branch; + if (delta.Head is not null) proto.Head = delta.Head; + proto.ReadFiles.AddRange(delta.ReadFiles); + proto.ConfirmedChangedFiles.AddRange(delta.ConfirmedChangedFiles); + proto.ObservedChangedFiles.AddRange(delta.ObservedChangedFiles); + return proto; + } + + private static WorkingContextDelta FromProto(Proto.ChildWorkingContextProto proto) => new() + { + ProjectDirectory = proto.HasProjectDirectory ? proto.ProjectDirectory : null, + Worktree = proto.HasWorktree ? proto.Worktree : null, + Branch = proto.HasBranch ? proto.Branch : null, + Head = proto.HasHead ? proto.Head : null, + ReadFiles = proto.ReadFiles.ToArray(), ConfirmedChangedFiles = proto.ConfirmedChangedFiles.ToArray(), + ObservedChangedFiles = proto.ObservedChangedFiles.ToArray() + }; + + private static Proto.ChildRunCheckpointProto ToProto(ChildRunCheckpoint checkpoint) => new() + { + CompletedRound = checkpoint.CompletedRound, Summary = checkpoint.Summary, ConfirmedActivity = ToProto(checkpoint.ConfirmedActivity) + }; + + private static ChildRunCheckpoint FromProto(Proto.ChildRunCheckpointProto proto) + => new(proto.CompletedRound, proto.Summary, FromProto(proto.ConfirmedActivity + ?? throw new InvalidDataException("A child checkpoint lacks confirmed activity evidence."))); + + private static Proto.ChildRunTerminalProto ToProto(ChildRunTerminal terminal) + { + var result = terminal.Result; + var proto = new Proto.ChildRunTerminalProto + { + Lost = terminal.Lost, Output = result.Output, Outcome = (int)result.Outcome, + RunId = result.RunId?.Value ?? throw new InvalidDataException("A child terminal has no run ID."), + ScopeId = result.ScopeId?.Value ?? throw new InvalidDataException("A child terminal has no scope ID."), + AgentName = result.AgentName.Value, FindingsCount = result.FindingsCount + }; + if (terminal.EvidenceWarning is not null) proto.EvidenceWarning = terminal.EvidenceWarning; + if (result.OutcomeReason is { } reason) proto.OutcomeReason = reason.Value; + if (result.WorkingContext is { } delta) proto.WorkingContext = ToProto(delta); + if (result.LogPath is not null) proto.LogPath = result.LogPath; + if (result.ArtifactDirectory is not null) proto.ArtifactDirectory = result.ArtifactDirectory; + foreach (var finding in result.Findings) + { + var item = new Proto.ChildFindingProto + { + Shape = (int)finding.Shape, Title = finding.Title, Content = finding.Content, Kind = finding.Kind, + Sensitivity = (int)finding.Sensitivity, RecallMode = (int)finding.RecallMode, + UpdateSemantics = finding.UpdateSemantics, Confidence = finding.Confidence, + Durability = (int)finding.Durability, Reusability = (int)finding.Reusability + }; + item.Evidence.AddRange(finding.Evidence); + if (finding.FreshnessAtMs is { } fresh) item.FreshnessAtMs = fresh; + proto.Findings.Add(item); + } + return proto; + } + + private static ChildRunTerminal FromProto(Proto.ChildRunTerminalProto proto) + { + var outcome = (SubAgentRunOutcome)proto.Outcome; + if (!Enum.IsDefined(outcome)) throw new InvalidDataException("A child terminal has an unknown outcome."); + var result = new SubAgentResult + { + Completion = ChildRunCompletion.FromReportedOutcome(outcome, + proto.HasOutcomeReason ? new SubAgentOutcomeReason(proto.OutcomeReason) : null, + proto.WorkingContext is null ? null : FromProto(proto.WorkingContext)), + Output = proto.Output, AgentName = new AgentName(proto.AgentName), + RunId = new SubAgentRunId(proto.RunId), ScopeId = new SubAgentScopeId(proto.ScopeId), + FindingsCount = proto.FindingsCount, + LogPath = proto.HasLogPath ? proto.LogPath : null, + ArtifactDirectory = proto.HasArtifactDirectory ? proto.ArtifactDirectory : null, + Findings = proto.Findings.Select(static item => new SubAgentFinding + { + Shape = CheckedEnum(item.Shape), Title = item.Title, Content = item.Content, + Kind = item.Kind, Sensitivity = CheckedEnum(item.Sensitivity), + RecallMode = CheckedEnum(item.RecallMode), UpdateSemantics = item.UpdateSemantics, + Confidence = item.Confidence, Durability = CheckedEnum(item.Durability), + Reusability = CheckedEnum(item.Reusability), Evidence = item.Evidence.ToArray(), + FreshnessAtMs = item.HasFreshnessAtMs ? item.FreshnessAtMs : null + }).ToList() + }; + return new ChildRunTerminal(result, proto.Lost, proto.HasEvidenceWarning ? proto.EvidenceWarning : null); + } + + private static T CheckedEnum(int value) where T : struct, Enum + => Enum.IsDefined(typeof(T), value) ? (T)Enum.ToObject(typeof(T), value) + : throw new InvalidDataException($"A child finding has an unknown {typeof(T).Name} value."); +} diff --git a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildRuns.cs b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildRuns.cs new file mode 100644 index 000000000..aa31b28e3 --- /dev/null +++ b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.ChildRuns.cs @@ -0,0 +1,168 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Collections.Immutable; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Proto = Netclaw.Actors.Serialization.Proto; +using GitInspectionKind = Netclaw.Actors.Serialization.Proto.GitWorkingContextInspectionProto.Types.Kind; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Serialization; + +internal static partial class NetclawProtoMapper +{ + internal static Proto.ChildRunAcceptedProto ToProto(ChildRunAccepted evt) => new() + { + SessionId = evt.SessionId.Value, + Run = ToProto(evt.Run) + }; + + internal static ChildRunAccepted FromProto(Proto.ChildRunAcceptedProto proto) => new() + { + SessionId = new SessionId(proto.SessionId), + Run = FromProto(proto.Run ?? throw new InvalidDataException("A child acceptance has no run record.")) + }; + + internal static Proto.BackgroundChildRunProto ToProto(BackgroundChildRun run) + { + run.Validate(); + var proto = new Proto.BackgroundChildRunProto + { + RunId = run.RunId.Value, ScopeId = run.ScopeId.Value, + OwnerSessionId = run.StartKey.SessionId.Value, OriginalTurnId = run.StartKey.TurnId.Value, + AgentName = run.AgentName.Value, SourceOperation = run.SourceOperation, ArgumentsDigest = run.ArgumentsDigest, + OriginalContext = ToProto(run.OriginalContext), InitialWorkingContext = ToProto(run.InitialWorkingSnapshot.WorkingContext), + InitialGit = ToProto(run.InitialWorkingSnapshot.Git), ParentCheckpoint = ToProto(run.ParentCheckpoint), + ParentReceiptFailure = run.ParentReceiptFailure, StartBatchSettled = run.StartBatchSettled, AcceptedAtMs = run.AcceptedAtMs + }; + if (run.StartedAtMs is { } started) proto.StartedAtMs = started; + if (run.CancellationRequestedAtMs is { } cancelled) proto.CancellationRequestedAtMs = cancelled; + if (run.DispatchClosedAtMs is { } closed) proto.DispatchClosedAtMs = closed; + if (run.ChildCheckpoint is { } checkpoint) proto.ChildCheckpoint = ToProto(checkpoint); + if (run.Terminal is { } terminal) proto.Terminal = ToProto(terminal); + if (run.TerminalSequenceNr is { } sequence) proto.TerminalSequenceNr = sequence; + if (run.PreparedTerminal is { } prepared) proto.PreparedTerminal = ToProto(prepared); + if (run.DeliveryInputId is { } delivery) proto.DeliveryInputId = delivery.Value; + switch (run.StartKey) + { + case ChildRunStartKey.Tool tool: + proto.ToolCallId = tool.CallId.Value; + break; + case ChildRunStartKey.Slash slash: + proto.SlashInputId = slash.InputId.Value; + break; + default: + throw new InvalidDataException("A child acceptance has an unknown activation kind."); + } + proto.Approvals.AddRange(run.Approvals.Select(static approval => new Proto.ChildRunApprovalProto + { + Request = ToProto(approval.Request), + Resolution = approval.Resolution is null ? null : ToProto(approval.Resolution) + })); + proto.OriginInputIds.AddRange(run.OriginInputIds.Select(static id => id.Value)); + return proto; + } + + internal static BackgroundChildRun FromProto(Proto.BackgroundChildRunProto proto) + { + var session = new SessionId(proto.OwnerSessionId); + var turn = new TurnId(proto.OriginalTurnId); + ChildRunStartKey key = proto.ActivationCase switch + { + Proto.BackgroundChildRunProto.ActivationOneofCase.ToolCallId => new ChildRunStartKey.Tool(new ToolCallId(proto.ToolCallId)) { SessionId = session, TurnId = turn }, + Proto.BackgroundChildRunProto.ActivationOneofCase.SlashInputId => new ChildRunStartKey.Slash(new InputId(proto.SlashInputId)) { SessionId = session, TurnId = turn }, + _ => throw new InvalidDataException("A child acceptance has no activation identity.") + }; + if (proto.OriginalContext is null || proto.InitialWorkingContext is null || proto.InitialGit is null || proto.ParentCheckpoint is null) + throw new InvalidDataException("A child acceptance lacks mandatory canonical evidence."); + var run = new BackgroundChildRun + { + RunId = new SubAgentRunId(proto.RunId), ScopeId = new SubAgentScopeId(proto.ScopeId), StartKey = key, + AgentName = new AgentName(proto.AgentName), SourceOperation = proto.SourceOperation, ArgumentsDigest = proto.ArgumentsDigest, + OriginalContext = FromProto(proto.OriginalContext), OriginInputIds = proto.OriginInputIds.Select(static id => new InputId(id)).ToArray(), + // The host shell object stays live-only. Recovery never reconstructs child execution. + InitialWorkingSnapshot = new WorkingContextSnapshot + { + WorkingContext = FromProto(proto.InitialWorkingContext), Git = FromProto(proto.InitialGit) + }, + ParentCheckpoint = FromProto(proto.ParentCheckpoint), ParentReceiptFailure = proto.ParentReceiptFailure, + StartBatchSettled = proto.StartBatchSettled, AcceptedAtMs = proto.AcceptedAtMs, + StartedAtMs = proto.HasStartedAtMs ? proto.StartedAtMs : null, + CancellationRequestedAtMs = proto.HasCancellationRequestedAtMs ? proto.CancellationRequestedAtMs : null, + DispatchClosedAtMs = proto.HasDispatchClosedAtMs ? proto.DispatchClosedAtMs : null, + ChildCheckpoint = proto.ChildCheckpoint is null ? null : FromProto(proto.ChildCheckpoint), + Terminal = proto.Terminal is null ? null : FromProto(proto.Terminal), + TerminalSequenceNr = proto.HasTerminalSequenceNr ? proto.TerminalSequenceNr : null, + PreparedTerminal = proto.PreparedTerminal is null ? null : FromProto(proto.PreparedTerminal), + DeliveryInputId = proto.HasDeliveryInputId ? new InputId(proto.DeliveryInputId) : null, + Approvals = proto.Approvals.Select(static approval => new ChildRunApproval( + approval.Request is null ? throw new InvalidDataException("A child approval has no request.") : FromProto(approval.Request), + approval.Resolution is null ? null : FromProto(approval.Resolution))).ToArray() + }; + run.Validate(); + return run; + } + + private static Proto.GitWorkingContextInspectionProto ToProto(GitWorkingContextInspection inspection) + { + var proto = new Proto.GitWorkingContextInspectionProto(); + switch (inspection) + { + case GitWorkingContextInspection.Skipped: + proto.Kind = GitInspectionKind.Skipped; + break; + case GitWorkingContextInspection.NotRepository: + proto.Kind = GitInspectionKind.NotRepository; + break; + case GitWorkingContextInspection.ExecutableNotFound: + proto.Kind = GitInspectionKind.ExecutableNotFound; + break; + case GitWorkingContextInspection.Unavailable unavailable: + proto.Kind = GitInspectionKind.Unavailable; + proto.Reason = unavailable.Reason; + break; + case GitWorkingContextInspection.Available { Snapshot: var git }: + _ = git.GetHeadState(); + proto.Kind = GitInspectionKind.Available; + proto.Worktree = git.Worktree; proto.CommonDirectory = git.CommonDirectory; + proto.Detached = git.Detached; proto.Ahead = git.Ahead; proto.Behind = git.Behind; + proto.Staged = git.Staged; proto.Modified = git.Modified; proto.Untracked = git.Untracked; + if (git.Branch is not null) proto.Branch = git.Branch; + if (git.Head is not null) proto.Head = git.Head; + if (git.Upstream is not null) proto.Upstream = git.Upstream; + proto.ChangedFiles.AddRange(git.ChangedFiles.Order(StringComparer.Ordinal)); + break; + default: + throw new InvalidDataException("A child snapshot has an unknown Git inspection state."); + } + return proto; + } + + private static GitWorkingContextInspection FromProto(Proto.GitWorkingContextInspectionProto proto) + { + if (proto.Kind != GitInspectionKind.Available) + return proto.Kind switch + { + GitInspectionKind.Skipped => new GitWorkingContextInspection.Skipped(), + GitInspectionKind.NotRepository => new GitWorkingContextInspection.NotRepository(), + GitInspectionKind.ExecutableNotFound => new GitWorkingContextInspection.ExecutableNotFound(), + GitInspectionKind.Unavailable when proto.HasReason => new GitWorkingContextInspection.Unavailable(proto.Reason), + _ => throw new InvalidDataException("A child snapshot has invalid Git inspection evidence.") + }; + var git = new GitWorkingContextSnapshot + { + Worktree = proto.Worktree, CommonDirectory = proto.CommonDirectory, + Branch = proto.HasBranch ? proto.Branch : null, Detached = proto.Detached, + Head = proto.HasHead ? proto.Head : null, Upstream = proto.HasUpstream ? proto.Upstream : null, + Ahead = proto.Ahead, Behind = proto.Behind, Staged = proto.Staged, Modified = proto.Modified, Untracked = proto.Untracked, + ChangedFiles = proto.ChangedFiles.ToImmutableHashSet(StringComparer.Ordinal) + }; + _ = git.GetHeadState(); + return new GitWorkingContextInspection.Available(git); + } +} diff --git a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs index 5c1280a5e..ddc5457f9 100644 --- a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs +++ b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs @@ -18,7 +18,7 @@ namespace Netclaw.Actors.Serialization; -internal static class NetclawProtoMapper +internal static partial class NetclawProtoMapper { private const int NoAssignmentDigestWireKind = 1; private const int ExactAssignmentDigestWireKind = 2; @@ -32,6 +32,8 @@ internal static class NetclawProtoMapper SerializableToolCall v => ToProto(v), TurnRecorded v => ToProto(v), InputAdmitted v => ToProto(v), + ChildRunAccepted v => ToProto(v), + ChildRunEvent v => ToProto(v), InputClosed v => ToProto(v), ToolTaskAdopted v => ToProto(v), SessionTitleSet v => ToProto(v), @@ -198,6 +200,8 @@ internal static Proto.InputAdmittedProto ToProto(InputAdmitted evt) BackgroundJobLineageVersion = evt.BackgroundJobLineageVersion, BackgroundJobOrigin = evt.BackgroundJobOrigin is null ? null : ToProto(evt.BackgroundJobOrigin) }; + if (evt.SourceChildRunId is { } sourceChild) + proto.SourceChildRunId = sourceChild.Value; if (evt.SourceBackgroundJobId is { } sourceJobId) proto.SourceBackgroundJobId = sourceJobId.Value; if (evt.SourceMessageId is not null) @@ -220,6 +224,7 @@ internal static Proto.InputAdmittedProto ToProto(InputAdmitted evt) : FromProto(proto.TurnContext), AdmittedAtMs = proto.AdmittedAtMs, SourceBackgroundJobId = proto.HasSourceBackgroundJobId ? new BackgroundJobId(proto.SourceBackgroundJobId) : null, + SourceChildRunId = proto.HasSourceChildRunId ? new SubAgentRunId(proto.SourceChildRunId) : null, BackgroundJobLineageVersion = proto.BackgroundJobLineageVersion, BackgroundJobOrigin = proto.BackgroundJobOrigin is null ? null : FromProto(proto.BackgroundJobOrigin) }; @@ -233,6 +238,8 @@ internal static Proto.ToolTaskAdoptedProto ToProto(ToolTaskAdopted evt) proto.InputIds.AddRange(evt.InputIds.Select(static id => id.Value)); if (evt.ContinuedJobKey is not null) proto.ContinuedJobKey = evt.ContinuedJobKey; + if (evt.ContinuedChildRunId is { } child) + proto.ContinuedChildRunId = child.Value; return proto; } @@ -244,7 +251,8 @@ internal static Proto.ToolTaskAdoptedProto ToProto(ToolTaskAdopted evt) : FromProto(proto.TurnContext), InputIds = proto.InputIds.Select(static id => new InputId(id)).ToArray(), AdoptedAtMs = proto.AdoptedAtMs, - ContinuedJobKey = proto.HasContinuedJobKey ? proto.ContinuedJobKey : null + ContinuedJobKey = proto.HasContinuedJobKey ? proto.ContinuedJobKey : null, + ContinuedChildRunId = proto.HasContinuedChildRunId ? new SubAgentRunId(proto.ContinuedChildRunId) : null }; internal static Proto.InputClosedProto ToProto(InputClosed evt) @@ -376,6 +384,8 @@ internal static Proto.ToolApprovalRequestedProto ToProto(ToolApprovalRequested e Audience = (Proto.TrustAudience)(int)evt.Audience, RequestedAtMs = evt.RequestedAtMs }; + if (evt.SourceChildRunId is { } childRun) proto.SourceChildRunId = childRun.Value; + if (evt.OriginalChildCallId is { } childCall) proto.OriginalChildCallId = childCall.Value; proto.Patterns.AddRange(evt.Patterns); proto.CandidateVerbs.AddRange(evt.CandidateVerbs); if (evt.RequesterSenderId is not null) @@ -408,7 +418,9 @@ internal static Proto.ToolApprovalRequestedProto ToProto(ToolApprovalRequested e internal static ToolApprovalRequested FromProto(Proto.ToolApprovalRequestedProto proto) => new() { SessionId = FromProto(proto.SessionId), + SourceChildRunId = proto.HasSourceChildRunId ? new SubAgentRunId(proto.SourceChildRunId) : null, CallId = proto.CallId, + OriginalChildCallId = proto.HasOriginalChildCallId ? new ToolCallId(proto.OriginalChildCallId) : null, ToolName = proto.ToolName, Patterns = proto.Patterns.ToArray(), CandidateVerbs = proto.CandidateVerbs.ToArray(), @@ -452,12 +464,14 @@ internal static Proto.ToolApprovalResolvedProto ToProto(ToolApprovalResolved evt }; if (evt.AuthorizationAttemptId is not null) proto.AuthorizationAttemptId = evt.AuthorizationAttemptId; + if (evt.SourceChildRunId is { } childRun) proto.SourceChildRunId = childRun.Value; return proto; } internal static ToolApprovalResolved FromProto(Proto.ToolApprovalResolvedProto proto) => new() { SessionId = FromProto(proto.SessionId), + SourceChildRunId = proto.HasSourceChildRunId ? new SubAgentRunId(proto.SourceChildRunId) : null, CallId = proto.CallId, Decision = proto.Decision, AuthorizationAttemptId = proto.HasAuthorizationAttemptId @@ -667,6 +681,7 @@ internal static Proto.SessionSnapshotProto ToProto(SessionSnapshot snap) proto.ProcessedBackgroundJobIds.AddRange(snap.ProcessedBackgroundJobIds.Select(static id => id.Value)); proto.History.AddRange(snap.History.Select(ToProto)); proto.ActiveBackgroundJobs.AddRange(snap.ActiveBackgroundJobs.Select(ToProto)); + proto.ChildRuns.AddRange(snap.ChildRuns.Select(ToProto)); proto.AdoptedContextRecords.AddRange(snap.AdoptedContextRecords.Select(ToAdoptedContextSnapshotRecord)); proto.PendingInputs.AddRange(snap.PendingInputs.Select(ToProto)); proto.RecentSourceMessageKeys.AddRange(snap.RecentSourceMessageKeys); @@ -691,6 +706,7 @@ internal static Proto.SessionSnapshotProto ToProto(SessionSnapshot snap) WorkingContext = proto.WorkingContext is not null ? FromProto(proto.WorkingContext) : null, History = proto.History.Select(FromProto).ToArray(), ActiveBackgroundJobs = proto.ActiveBackgroundJobs.Select(FromProto).ToArray(), + ChildRuns = proto.ChildRuns.Select(FromProto).ToArray(), AdoptedContextRecords = proto.AdoptedContextRecords.Select(FromAdoptedContextSnapshotRecord).ToArray(), PendingInputs = proto.PendingInputs.Select(FromProto).ToArray(), RecentSourceMessageKeys = proto.RecentSourceMessageKeys.ToArray() diff --git a/src/Netclaw.Actors/Serialization/NetclawProtobufSerializer.cs b/src/Netclaw.Actors/Serialization/NetclawProtobufSerializer.cs index b03050c8f..14e7dcca8 100644 --- a/src/Netclaw.Actors/Serialization/NetclawProtobufSerializer.cs +++ b/src/Netclaw.Actors/Serialization/NetclawProtobufSerializer.cs @@ -28,6 +28,8 @@ public sealed class NetclawProtobufSerializer : SerializerWithStringManifest private const string SerializableToolCallManifest = "stc-v1"; private const string TurnRecordedManifest = "tr-v1"; private const string InputAdmittedManifest = "ia-v1"; + private const string ChildRunAcceptedManifest = "cra-v1"; + private const string ChildRunEventManifest = "cre-v1"; private const string InputClosedManifest = "ic-v1"; private const string ToolTaskAdoptedManifest = "tta-v1"; private const string SessionTitleSetManifest = "sts-v1"; @@ -59,6 +61,14 @@ public sealed class NetclawProtobufSerializer : SerializerWithStringManifest [typeof(SerializableToolCall)] = SerializableToolCallManifest, [typeof(TurnRecorded)] = TurnRecordedManifest, [typeof(InputAdmitted)] = InputAdmittedManifest, + [typeof(ChildRunAccepted)] = ChildRunAcceptedManifest, + [typeof(ChildRunEvent.Started)] = ChildRunEventManifest, + [typeof(ChildRunEvent.Checkpointed)] = ChildRunEventManifest, + [typeof(ChildRunEvent.CancellationRequested)] = ChildRunEventManifest, + [typeof(ChildRunEvent.DispatchClosed)] = ChildRunEventManifest, + [typeof(ChildRunEvent.TerminalRecorded)] = ChildRunEventManifest, + [typeof(ChildRunEvent.ResultPrepared)] = ChildRunEventManifest, + [typeof(ChildRunEvent.DeliveryAdmitted)] = ChildRunEventManifest, [typeof(InputClosed)] = InputClosedManifest, [typeof(ToolTaskAdopted)] = ToolTaskAdoptedManifest, [typeof(SessionTitleSet)] = SessionTitleSetManifest, @@ -120,6 +130,10 @@ public override object FromBinary(byte[] bytes, string manifest) Proto.TurnRecordedProto.Parser.ParseFrom(bytes)), InputAdmittedManifest => NetclawProtoMapper.FromProto( Proto.InputAdmittedProto.Parser.ParseFrom(bytes)), + ChildRunEventManifest => NetclawProtoMapper.FromProto( + Proto.ChildRunEventProto.Parser.ParseFrom(bytes)), + ChildRunAcceptedManifest => NetclawProtoMapper.FromProto( + Proto.ChildRunAcceptedProto.Parser.ParseFrom(bytes)), ToolTaskAdoptedManifest => NetclawProtoMapper.FromProto( Proto.ToolTaskAdoptedProto.Parser.ParseFrom(bytes)), InputClosedManifest => NetclawProtoMapper.FromProto( diff --git a/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto b/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto index ac3b8ff13..94c172e16 100644 --- a/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto +++ b/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto @@ -108,6 +108,7 @@ message InputAdmittedProto { optional string source_background_job_id = 10; int32 background_job_lineage_version = 11; BackgroundJobOriginProto background_job_origin = 12; + optional string source_child_run_id = 13; reserved 7, 8; } @@ -119,6 +120,7 @@ message ToolTaskAdoptedProto { repeated string input_ids = 3; int64 adopted_at_ms = 4; optional string continued_job_key = 5; + optional string continued_child_run_id = 6; } message InputClosedProto { @@ -254,6 +256,8 @@ message ToolApprovalRequestedProto { optional string authorization_attempt_id = 20; optional string managed_temporary_directory = 21; optional string repository_common_directory = 22; + optional string source_child_run_id = 23; + optional string original_child_call_id = 24; } message ToolApprovalResolvedProto { @@ -262,6 +266,7 @@ message ToolApprovalResolvedProto { string decision = 3; int64 resolved_at_ms = 4; optional string authorization_attempt_id = 5; + optional string source_child_run_id = 6; } message ToolBatchAbandonedProto { @@ -356,6 +361,7 @@ message SessionSnapshotProto { ToolApprovalRequestedProto.TurnContextRecordProto adopted_task_context = 16; repeated string adopted_task_input_ids = 17; repeated string processed_background_job_ids = 18; + repeated BackgroundChildRunProto child_runs = 19; } // ── Session state ── @@ -434,3 +440,129 @@ message MemoriesDistilledV2Proto { repeated ProposedMemoryContextProto proposals = 2; int64 timestamp_ms = 3; } + +// Durable child acceptance. Live authority objects remain in the owner adapter. +message ChildRunAcceptedProto { + string session_id = 1; + BackgroundChildRunProto run = 2; +} + +message BackgroundChildRunProto { + string run_id = 1; + string scope_id = 2; + string owner_session_id = 3; + string original_turn_id = 4; + oneof activation { + string tool_call_id = 5; + string slash_input_id = 6; + } + string agent_name = 7; + string source_operation = 8; + string arguments_digest = 9; + ToolApprovalRequestedProto.TurnContextRecordProto original_context = 10; + repeated string origin_input_ids = 11; + WorkingContextProto initial_working_context = 12; + GitWorkingContextInspectionProto initial_git = 13; + ToolLoopCheckpointProto parent_checkpoint = 14; + bool parent_receipt_failure = 15; + bool start_batch_settled = 16; + int64 accepted_at_ms = 17; + optional int64 started_at_ms = 18; + optional int64 cancellation_requested_at_ms = 19; + optional int64 dispatch_closed_at_ms = 20; + ChildRunCheckpointProto child_checkpoint = 21; + ChildRunTerminalProto terminal = 22; + ChildRunTerminalProto prepared_terminal = 23; + optional string delivery_input_id = 24; + optional int64 terminal_sequence_nr = 25; + repeated ChildRunApprovalProto approvals = 26; +} + +message GitWorkingContextInspectionProto { + enum Kind { + UNSET = 0; + SKIPPED = 1; + AVAILABLE = 2; + NOT_REPOSITORY = 3; + EXECUTABLE_NOT_FOUND = 4; + UNAVAILABLE = 5; + } + Kind kind = 1; + optional string reason = 2; + string worktree = 3; + string common_directory = 4; + optional string branch = 5; + bool detached = 6; + optional string head = 7; + optional string upstream = 8; + int32 ahead = 9; + int32 behind = 10; + int32 staged = 11; + int32 modified = 12; + int32 untracked = 13; + repeated string changed_files = 14; +} + +message ChildRunCheckpointProto { + int32 completed_round = 1; + string summary = 2; + ChildWorkingContextProto confirmed_activity = 3; +} +message ChildWorkingContextProto { + optional string project_directory = 1; + optional string worktree = 2; + optional string branch = 3; + optional string head = 4; + repeated string read_files = 5; + repeated string confirmed_changed_files = 6; + repeated string observed_changed_files = 7; +} +message ChildFindingProto { + int32 shape = 1; + string title = 2; + string content = 3; + string kind = 4; + int32 sensitivity = 5; + int32 recall_mode = 6; + string update_semantics = 7; + double confidence = 8; + int32 durability = 9; + int32 reusability = 10; + repeated string evidence = 11; + optional int64 freshness_at_ms = 12; +} +message ChildRunTerminalProto { + bool lost = 1; + optional string evidence_warning = 2; + string output = 3; + int32 outcome = 4; + optional string outcome_reason = 5; + string run_id = 6; + string scope_id = 7; + string agent_name = 8; + ChildWorkingContextProto working_context = 9; + repeated ChildFindingProto findings = 10; + int32 findings_count = 11; + optional string log_path = 12; + optional string artifact_directory = 13; +} +message ChildRunEventProto { + string session_id = 1; + string run_id = 2; + int64 recorded_at_ms = 3; + oneof fact { + bool started = 4; + ChildRunCheckpointProto checkpoint = 5; + bool cancellation_requested = 6; + bool dispatch_closed = 7; + ChildRunTerminalProto terminal = 8; + ChildRunTerminalProto prepared = 9; + InputAdmittedProto delivery = 10; + } + int64 terminal_sequence_nr = 11; +} + +message ChildRunApprovalProto { + ToolApprovalRequestedProto request = 1; + ToolApprovalResolvedProto resolution = 2; +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildApprovals.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildApprovals.cs new file mode 100644 index 000000000..40d45fec9 --- /dev/null +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildApprovals.cs @@ -0,0 +1,192 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Event; +using Netclaw.Actors.Authorization.Consent; +using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Sessions; + +public sealed partial class LlmSessionActor +{ + private sealed record ChildApprovalRequested(SubAgentRunId RunId, ChildRunDispatch Dispatch, + ToolInteractionRequestDispatch Prompt) : INoSerializationVerificationNeeded; + private sealed record ChildApprovalWaitFinished(SubAgentRunId RunId, AuthorizationAttemptId Attempt, + ConsentAnswer Answer) : INoSerializationVerificationNeeded; + + private sealed class ChildApprovalBridge(IParentConsentBridge inner, IActorRef owner, SubAgentRunId runId) + : IParentConsentBridge + { + public async Task RequestConsentAsync(ParentApprovalRequest request, CancellationToken ct) + { + var step = await inner.RequestConsentAsync(request, ct).ConfigureAwait(false); + owner.Tell(new ChildApprovalWaitFinished(runId, request.AuthorizationAttemptId, step.Answer)); + return step; + } + } + + private RunSubAgent WithChildApprovalOwner(BackgroundChildRun run, RunSubAgent execution, ChildRuntime runtime) + { + if (execution.Scope.Authority.InteractiveApproval is InteractiveApprovalCapability.Unavailable) + return execution; + if (!TurnContext.TryFromRecord(run.OriginalContext, out var context, out var reason) || context is null) + throw new InvalidDataException($"The child approval context is invalid: {reason}"); + var owner = Self; + var bridge = new ParentSessionApprovalBridge(_approvalChannel, + prompt => owner.Tell(new ChildApprovalRequested(run.RunId, runtime.Dispatch, prompt)), + new ToolExecutionTimeout(Timeout.InfiniteTimeSpan), _sessionId, run.RunId.Value, + context.RequesterSenderId, context.RequesterPrincipal, context.HasAdoptedContext, + context.HasThirdPartyAdoptedContext, context.AdoptedSpeakerIds); + return execution with + { + Scope = execution.Scope with + { + Authority = execution.Scope.Authority with + { + InteractiveApproval = new InteractiveApprovalCapability.Available(new ChildApprovalBridge(bridge, owner, run.RunId)) + } + } + }; + } + + private void HandleChildApprovalRequested(ChildApprovalRequested message) + { + if (!_backgroundChildRuntimes.TryGetValue(message.RunId, out var runtime) + || !ReferenceEquals(runtime.Dispatch, message.Dispatch) + || !_state.ChildRuns.TryGetValue(message.RunId, out var run) + || run.Terminal is not null || run.DispatchClosedAtMs is not null) + { + RefuseChildApprovalWait(message.Prompt.Request.CallId); + return; + } + if (!TurnContext.TryFromRecord(run.OriginalContext, out var original, out var reason) || original is null) + throw new InvalidDataException($"The child approval context is invalid: {reason}"); + var evt = CreateToolApprovalRequest(message.Prompt, original) with + { + SourceChildRunId = run.RunId, OriginalChildCallId = message.Prompt.OriginalChildCallId + }; + _ = _state.ApplyChildApproval(evt); + try + { + _ = runtime.Dispatch.Enter(() => + { + Persist(evt, committed => + { + ApplyToolApprovalRequested(committed); + EmitOutput(message.Prompt.Request); + }); + return Task.CompletedTask; + }); + } + catch (OperationCanceledException) + { + RefuseChildApprovalWait(message.Prompt.Request.CallId); + } + } + + private void RefuseChildApprovalWait(ToolCallId callId) + { + if (_approvalChannel.TryClaim(callId, out var wait)) + wait.Complete(ConsentAnswer.Denied); + } + + private async Task TryHandleChildApprovalResponseAsync(ToolInteractionResponse message) + { + var run = _state.ChildRuns.Values.SingleOrDefault(candidate => + candidate.Approvals.Any(approval => approval.Request.CallId == message.CallId.Value)); + if (run is null) + return false; + var approval = run.Approvals.Single(item => item.Request.CallId == message.CallId.Value); + if (approval.Resolution is not null || run.Terminal is not null || run.DispatchClosedAtMs is not null + || !_backgroundChildRuntimes.TryGetValue(run.RunId, out var runtime)) + { + EmitExpiredPromptNotice(); + TryReplyNack(ApprovalNackReasons.PromptExpired); + return true; + } + var pending = PendingToolInteraction.From(approval.Request, persistApprovalState: false); + var authorization = await AuthorizeApprovalResponseAsync(pending, message, persistApprovalGrant: false); + if (authorization.Answer is not { } answer) + { + TryReplyNack(authorization.NackReason ?? ApprovalNackReasons.WrongRequester); + return true; + } + if (!_approvalChannel.TryClaim(message.CallId, out var wait)) + { + EmitExpiredPromptNotice(); + TryReplyNack(ApprovalNackReasons.PromptExpired); + return true; + } + try + { + // A claimed prompt alone cannot grant after the run closes dispatch admission. + await runtime.Dispatch.Enter(() => PersistApprovalGrantIfNeededAsync(pending, answer, runtime.Lifetime.Token)); + PersistChildApprovalResolution(run.RunId, approval.Request, answer, () => + { + wait.Complete(answer); + TryReplyAck(); + }); + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + wait.Complete(ConsentAnswer.Denied); + _log.Error(error, "Child approval decision failed runId={RunId} callId={CallId}", run.RunId.Value, message.CallId.Value); + EmitExpiredPromptNotice(); + TryReplyNack(ApprovalNackReasons.PersistFailed); + } + return true; + } + + private void PersistChildApprovalResolution(SubAgentRunId runId, ToolApprovalRequested request, + ConsentAnswer answer, Action afterCommit) + { + var evt = new ToolApprovalResolved + { + SessionId = _sessionId, SourceChildRunId = runId, CallId = request.CallId, + AuthorizationAttemptId = request.AuthorizationAttemptId, + Decision = ConsentAnswerCodec.ToJournalText(answer), ResolvedAtMs = NowMs() + }; + _ = _state.ApplyChildApproval(evt); + Persist(evt, committed => + { + ApplyToolApprovalResolved(committed); + afterCommit(); + }); + } + + private void HandleChildApprovalWaitFinished(ChildApprovalWaitFinished message) + { + if (!_state.ChildRuns.TryGetValue(message.RunId, out var run)) + return; + var request = run.Approvals.SingleOrDefault(approval => + approval.Request.AuthorizationAttemptId == message.Attempt.Value && approval.Resolution is null)?.Request; + if (request is not null) + PersistChildApprovalResolution(run.RunId, request, message.Answer, () => { }); + } + + private void ExpireChildApprovals(SubAgentRunId runId, Action afterCommit) + { + var pending = _state.ChildRuns[runId].Approvals.FirstOrDefault(static approval => approval.Resolution is null); + if (pending is null) + { + afterCommit(); + return; + } + PersistChildApprovalResolution(runId, pending.Request, ConsentAnswer.Denied, () => + { + RefuseChildApprovalWait(new ToolCallId(pending.Request.CallId)); + EmitExpiredPromptNotice(); + ExpireChildApprovals(runId, afterCommit); + }); + } +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRecovery.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRecovery.cs new file mode 100644 index 000000000..cea0bdadf --- /dev/null +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRecovery.cs @@ -0,0 +1,158 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Netclaw.Actors.Channels; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Sessions; + +public sealed partial class LlmSessionActor +{ + private bool HasLiveChildRuns => _state.ChildRuns.Values.Any(static run => run.Terminal is null); + private bool HasUnsettledChildWork => _state.ChildRuns.Values.Any(static run => run.DeliveryInputId is null); + + private void RecoverBackgroundChildren() + { + if (_state.ChildRuns.Count == 0) + { + TransitionTo(SessionPhase.Ready); + return; + } + Become(AwaitRecoveredChildFacts); + if ((_state.ChildRuns.Values.Any(static run => !run.StartBatchSettled) + || _state.PendingInputs.FirstOrDefault()?.SourceChildRunId is not null) + && ParkedToolBatchHistory.FindRedrivableAssistantMessage(_state.History, null) is not null) + { + Persist(BuildInterruptedToolBatchAfterRecoveryEvent(), abandoned => + { + ApplyToolBatchAbandoned(abandoned); + RecoverNextChild(); + }); + return; + } + RecoverNextChild(); + } + + private void AwaitRecoveredChildFacts() + { + Command(message => + { + if (!_state.ChildRuns.TryGetValue(message.RunId, out var run) || run.Terminal is not null + || run.CancellationRequestedAtMs is null || run.DispatchClosedAtMs is null) + throw new InvalidDataException("A recovered cancellation has no committed closure."); + RecordChildTerminal(run, message.Terminal, ActorRefs.Nobody, _ => RecoverNextChild()); + }); + CommandAny(_ => Stash.Stash()); + } + + private void RecoverNextChild() + { + var run = _state.ChildRuns.Values.Where(static candidate => candidate.Terminal is null) + .OrderBy(static candidate => candidate.AcceptedAtMs).ThenBy(static candidate => candidate.RunId.Value, StringComparer.Ordinal) + .FirstOrDefault(); + if (run is null) + { + foreach (var recorded in _state.ChildRuns.Values.Where(static candidate => candidate.PreparedTerminal is null)) + EnrichRecordedChild(recorded); + TransitionTo(SessionPhase.Ready); + ResumeAdmittedChildContinuation(); + Stash.UnstashAll(); + return; + } + ExpireChildApprovals(run.RunId, () => + { + var current = _state.ChildRuns[run.RunId]; + if (current.CancellationRequestedAtMs is not null) + { + if (current.DispatchClosedAtMs is not null) + _ = FinalizeCancelledChildAsync(Self, current); + else + Persist(new ChildRunEvent.DispatchClosed + { + SessionId = _sessionId, RunId = current.RunId, RecordedAtMs = NowMs() + }, closed => + { + _state = _state.Apply(closed); + _ = FinalizeCancelledChildAsync(Self, _state.ChildRuns[current.RunId]); + }); + return; + } + var storage = _sessionStorage.ForChild(current.RunId, current.ScopeId); + var terminal = new ChildRunTerminal(new SubAgentResult + { + AgentName = current.AgentName, RunId = current.RunId, ScopeId = current.ScopeId, + Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.OwnerRestartLost), + Output = "The session owner restarted before a durable child terminal receipt. The recorded checkpoint retains known local evidence.", + LogPath = storage.LogPath.Value, ArtifactDirectory = storage.ArtifactDirectory.Value + }, true, "External effects can remain uncertain. The framework did not restart the child."); + // Each callback acknowledges its journal position before the next terminal stamp. + RecordChildTerminal(current, terminal, ActorRefs.Nobody, _ => RecoverNextChild()); + }); + } + + private void ResumeAdmittedChildContinuation() + { + // Input closure excludes failed or completed reviews. Ordinary inputs retain their existing restart contract. + var pending = _state.PendingInputs.TakeWhile(static input => input.SourceChildRunId is not null).ToArray(); + if (pending.Length == 0) + return; + foreach (var input in pending) + _ = _state.GetChildContinuation(input); + + var adoptedCount = 0; + if (_state.AdoptedTaskContext is { } adopted + && pending.Any(input => _state.AdoptedTaskInputIds.Contains(input.InputId))) + { + adoptedCount = _state.AdoptedTaskInputIds.Count; + if (!pending.Take(adoptedCount).Select(static input => input.InputId).SequenceEqual(_state.AdoptedTaskInputIds) + || pending.Take(adoptedCount).Any(input => !SessionState.SameCanonicalContext(input.TurnContext, adopted)) + || !TurnContext.TryFromRecord(adopted, out var context, out _) || context is null) + throw new InvalidDataException("A recovered child continuation differs from its durable adoption."); + _currentTurnContext = context; + BindTurnTelemetry(context); + _toolApprovals.StartTurn(context); + _currentTrustContext = _trustContextDeriver?.DeriveFromTurnContext(context); + _turnState.RestoreCheckpoint(_state.LoopCheckpoint); + _recallManager.ResetForNewTurn(); + SetSystemPrompt(); + } + _currentTurnSource = null; + for (var index = 0; index < pending.Length; index++) + { + var input = pending[index]; + _buffer.Add((new SendUserMessage + { + SessionId = _sessionId, Content = input.UserMessage.Content!, AdmittedInputId = input.InputId + }, index < adoptedCount)); + } + // Internal replay preserves adopted detector evidence; new child inputs use canonical adoption. + DrainBufferedUserMessages(() => + { + FireLlmCall(); + TransitionTo(SessionPhase.Processing); + }, () => TransitionTo(SessionPhase.Ready)); + } + + private void RequestChildRestartDrain() + { + var run = _state.ChildRuns.Values.FirstOrDefault(static candidate => + candidate.Terminal is null && candidate.CancellationRequestedAtMs is null); + if (run is null) + return; + Persist(new ChildRunEvent.CancellationRequested + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }, admitted => + { + _state = _state.Apply(admitted); + CloseChildDispatch(run.RunId); + RequestChildRestartDrain(); + }); + } +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs new file mode 100644 index 000000000..27f4cbc6d --- /dev/null +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs @@ -0,0 +1,595 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Netclaw.Actors.Tools; +using Netclaw.Actors.Channels; +using Netclaw.Configuration; +using Akka.Event; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Actors.Sessions.Pipelines; +using Netclaw.Actors.Memory; +using Netclaw.Actors.SubAgents; +using Netclaw.Security; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.Sessions; + +public sealed partial class LlmSessionActor +{ + // Actor references and lifetime tokens cannot enter the durable child ledger. + private readonly Dictionary _backgroundChildRuntimes = []; + private sealed record ChildRuntime(IActorRef Actor, CancellationTokenSource Lifetime, ChildRunDispatch Dispatch); + private sealed record ChildCancellationFinalized(SubAgentRunId RunId, ChildRunTerminal Terminal) : INoSerializationVerificationNeeded; + private sealed record ChildDispatchClosureCompleted(SubAgentRunId RunId, Exception? Cause) : INoSerializationVerificationNeeded; + private sealed record DeliverPreparedChildResults; + private sealed record ChildResultEnriched(SubAgentRunId RunId, ChildRunTerminal Terminal, + IReadOnlyList Findings) : INoSerializationVerificationNeeded; + + private void RegisterBackgroundChildHandlers() + { + Command(HandleChildApprovalRequested); + Command(HandleChildApprovalWaitFinished); + Command(_ => DeliverPreparedChildResultsInOrder()); + Command(HandleChildControl); + Command(message => + { + if (!_state.ChildRuns.TryGetValue(message.RunId, out var run) || run.Terminal is not null + || run.CancellationRequestedAtMs is null || run.DispatchClosedAtMs is not null) + return; + if (message.Cause is not null) + throw new InvalidOperationException("The child dispatch closure failed.", message.Cause); + Persist(new ChildRunEvent.DispatchClosed + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }, closed => + { + _state = _state.Apply(closed); + _ = FinalizeCancelledChildAsync(Self, _state.ChildRuns[run.RunId]); + }); + }); + Command(message => + { + if (!_state.ChildRuns.TryGetValue(message.RunId, out var run) || run.Terminal is not null + || run.CancellationRequestedAtMs is null || run.DispatchClosedAtMs is null) + return; + var child = _backgroundChildRuntimes.TryGetValue(run.RunId, out var runtime) ? runtime.Actor : ActorRefs.Nobody; + RecordChildTerminal(run, message.Terminal, child, EnrichRecordedChild); + }); + Command(HandleStartBackgroundChildRun); + Command(message => + { + if (!_backgroundChildRuntimes.TryGetValue(message.RunId, out var runtime) || !runtime.Actor.Equals(Sender) + || !_state.ChildRuns.TryGetValue(message.RunId, out var run) || run.Terminal is not null + || run.DispatchClosedAtMs is not null) + return; + if (message.Checkpoint.Summary.Length > _config.Tuning.MaxInlineToolResultChars) + throw new InvalidDataException("A child checkpoint exceeds the owner output bound."); + var evt = new ChildRunEvent.Checkpointed(message.Checkpoint) + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }; + _ = _state.Apply(evt); + var child = Sender; + Persist(evt, committed => + { + _state = _state.Apply(committed); + child.Tell(new BackgroundChildCheckpointAck(run.RunId, message.Checkpoint.CompletedRound), Self); + }); + }); + Command(HandleBackgroundChildTerminal); + Command(msg => + { + if (!_state.ChildRuns.TryGetValue(msg.RunId, out var run) || run.Terminal is null || run.PreparedTerminal is not null) + return; + Persist(new ChildRunEvent.ResultPrepared(msg.Terminal, run.TerminalSequenceNr!.Value) + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }, evt => + { + _state = _state.Apply(evt); + _log.Info("child_run_result_prepared owner={SessionId} runId={RunId} journalSequence={SequenceNr}", + _sessionId.Value, run.RunId.Value, LastSequenceNr); + if (!TurnContext.TryFromRecord(run.OriginalContext, out var original, out var reason) || original is null) + throw new InvalidDataException($"The child finding has invalid original authority: {reason}"); + foreach (var finding in msg.Findings) + { + if (finding.Decision != SubAgentFindingReviewDecision.Accepted) + continue; + // A later parent input cannot replace the accepted child's memory authority. + EnqueueCheckpointFireAndForget(new MemoryCheckpointRequest( + _sessionId, original.TurnId, CheckpointTriggerType.SubagentFindings, 80, + SessionMemoryCheckpointFactory.ForSubAgentFinding(_sessionId, + original.Boundary.Value, original.Audience.ToWireValue(), finding))); + } + Self.Tell(new DeliverPreparedChildResults()); + }); + }); + Command(terminated => + { + var runtime = _backgroundChildRuntimes.FirstOrDefault(pair => pair.Value.Actor.Equals(terminated.ActorRef)); + if (runtime.Value is null) + return; + _backgroundChildRuntimes.Remove(runtime.Key); + CancelChildLifetime(runtime.Value); + if (!_state.ChildRuns.TryGetValue(runtime.Key, out var run) || run.Terminal is not null + || run.CancellationRequestedAtMs is not null) + return; + RecordChildTerminal(run, new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.ActorStopped), + Output = "The child stopped before a durable terminal receipt.", + AgentName = run.AgentName, RunId = run.RunId, ScopeId = run.ScopeId + }, false, null), ActorRefs.Nobody, EnrichRecordedChild); + }); + } + + private void HandleStartBackgroundChildRun(StartBackgroundChildRun request) + { + var existing = _state.ChildRuns.Values.SingleOrDefault(run => run.StartKey == request.StartKey); + if (existing is not null) + { + if (!SessionState.SameCanonicalContext(existing.OriginalContext, request.InvocationContext) + || existing.SourceOperation != request.SourceOperation) + { + Sender.Tell(new Status.Failure(new InvalidDataException("The child retry differs from its original admitted authority."))); + return; + } + Sender.Tell(existing.ArgumentsDigest == request.Prepared.ArgumentsDigest + ? new ChildStartReply.Accepted(existing, existing.State) + : new ChildStartReply.Conflict()); + return; + } + if (!OwnsToolExecution(request.ExecutionToken)) + { + Sender.Tell(new Status.Failure(new OperationCanceledException("The start dispatch no longer owns this request."))); + return; + } + if (_state.AdoptedTaskContext is null + || !SessionState.SameCanonicalContext(_state.AdoptedTaskContext, request.InvocationContext) + || request.StartKey.SessionId != _sessionId + || request.StartKey.TurnId.Value != request.InvocationContext.TurnId) + { + Sender.Tell(new Status.Failure(new InvalidDataException("The child start differs from canonical admitted authority."))); + return; + } + var prepared = request.Prepared; + var childScope = prepared.Execution.Scope; + var expectedStorage = _sessionStorage.ForChild(prepared.RunId, childScope.ScopeId); + if (childScope.Authority.Session is not ToolSessionScope.Bound bound + || bound.SessionId != childScope.ScopeId.Value || bound.Storage != expectedStorage + || childScope.Authority.Audience != request.InvocationContext.Audience + || childScope.Authority.Boundary != request.InvocationContext.Boundary + || prepared.Execution.ActivitySink is not null) + { + Sender.Tell(new Status.Failure(new InvalidDataException("The child preparation differs from its assigned authority or storage."))); + return; + } + var accepted = new BackgroundChildRun + { + RunId = prepared.RunId, ScopeId = childScope.ScopeId, StartKey = request.StartKey, + AgentName = prepared.AgentName, ArgumentsDigest = prepared.ArgumentsDigest, SourceOperation = request.SourceOperation, + OriginalContext = request.InvocationContext, OriginInputIds = _state.AdoptedTaskInputIds.ToArray(), + InitialWorkingSnapshot = childScope.InitialWorkingSnapshot, + ParentCheckpoint = _state.LoopCheckpoint, ParentReceiptFailure = _state.LoopReceiptFailure, + StartBatchSettled = request.StartKey is ChildRunStartKey.Slash, AcceptedAtMs = NowMs() + }; + var evt = new ChildRunAccepted { SessionId = _sessionId, Run = accepted }; + // Validate the complete consumer representation before the journal accepts it. + _ = _state.Apply(evt); + var replyTo = Sender; + Persist(evt, committed => + { + _state = _state.Apply(committed); + _log.Info("child_run_accepted owner={SessionId} runId={RunId} journalSequence={SequenceNr}", + _sessionId.Value, accepted.RunId.Value, LastSequenceNr); + var response = new ChildStartReply.Accepted(accepted, BackgroundChildState.Accepted); + var lifetime = new CancellationTokenSource(); + IActorRef child; + try + { + child = Context.ActorOf(prepared.Props, $"child-run-{accepted.RunId.Value}"); + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + lifetime.Dispose(); + _log.Error(error, "Child creation failed after acceptance runId={RunId}", accepted.RunId.Value); + RecordChildTerminal(accepted, new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.SpawnError), + Output = "The child could not start after durable acceptance.", AgentName = accepted.AgentName, + RunId = accepted.RunId, ScopeId = accepted.ScopeId + }, false, null), ActorRefs.Nobody, EnrichRecordedChild); + replyTo.Tell(response); + return; + } + _backgroundChildRuntimes.Add(accepted.RunId, new ChildRuntime(child, lifetime, new ChildRunDispatch())); + Context.Watch(child); + Persist(new ChildRunEvent.Started + { + SessionId = _sessionId, RunId = accepted.RunId, RecordedAtMs = NowMs() + }, started => + { + _state = _state.Apply(started); + EmitOutput(new SubAgentOutput + { + SessionId = _sessionId, TimestampMs = NowMs(), AgentName = accepted.AgentName, + Phase = SubAgentPhase.Started, ToolCount = prepared.ToolCount + }, OutputFilter.ToolCalls); + child.Tell(new RunBackgroundSubAgent(accepted.RunId, WithChildApprovalOwner(accepted, prepared.Execution, _backgroundChildRuntimes[accepted.RunId]) with + { + Cancellation = lifetime.Token, ActivitySink = null + }, _backgroundChildRuntimes[accepted.RunId].Dispatch), Self); + replyTo.Tell(response); + }); + }); + } + + private void HandleBackgroundChildTerminal(BackgroundChildTerminal message) + { + if (!_backgroundChildRuntimes.TryGetValue(message.RunId, out var runtime) || !runtime.Actor.Equals(Sender) + || !_state.ChildRuns.TryGetValue(message.RunId, out var run)) + { + _log.Warning("Rejected a child terminal with no live owner runId={RunId}", message.RunId.Value); + return; + } + if (run.Terminal is not null) + { + Sender.Tell(new BackgroundChildTerminalAck(run.RunId), Self); + return; + } + if (run.CancellationRequestedAtMs is not null) + { + // Framework finalization retains the last durable checkpoint after cancellation. + return; + } + RecordChildTerminal(run, new ChildRunTerminal(message.Result, false, null), Sender, EnrichRecordedChild); + } + + private void RecordChildTerminal(BackgroundChildRun run, ChildRunTerminal terminal, IActorRef child, + Action afterCommit) + { + if (_state.ChildRuns[run.RunId].Approvals.Any(static approval => approval.Resolution is null)) + { + ExpireChildApprovals(run.RunId, () => RecordChildTerminal(_state.ChildRuns[run.RunId], terminal, child, afterCommit)); + return; + } + // Queue one terminal Persist until its acknowledgement. LastSequenceNr excludes queued writes. + var evt = new ChildRunEvent.TerminalRecorded(terminal, LastSequenceNr + 1) + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }; + _ = _state.Apply(evt); + Persist(evt, recorded => + { + _state = _state.Apply(recorded); + _log.Info("child_run_terminal_recorded owner={SessionId} runId={RunId} journalSequence={SequenceNr}", + _sessionId.Value, run.RunId.Value, LastSequenceNr); + EmitOutput(new SubAgentOutput + { + SessionId = _sessionId, TimestampMs = NowMs(), AgentName = run.AgentName, + Phase = SubAgentPhase.Completed, Success = terminal.Result.Success, + Outcome = terminal.Result.Outcome, OutcomeReason = terminal.Result.OutcomeReason, + Duration = TimeSpan.FromMilliseconds(Math.Max(0, NowMs() - run.AcceptedAtMs)), + FindingsCount = terminal.Result.Findings.Count + }, OutputFilter.ToolCalls); + if (!child.Equals(ActorRefs.Nobody)) + child.Tell(new BackgroundChildTerminalAck(run.RunId), Self); + if (run.CancellationRequestedAtMs is not null && !child.Equals(ActorRefs.Nobody)) + Context.Stop(child); + afterCommit(_state.ChildRuns[run.RunId]); + if (_restartDrainRequested && _phase.Current == SessionPhase.Passivating && !HasLiveChildRuns) + CompletePassivation(); + }); + } + + private void EnrichRecordedChild(BackgroundChildRun run) + { + // Terminal enrichment belongs to the committed receipt, after the child lifetime ends. + _ = EnrichRecordedChildAsync(Self, run, run.Terminal!, CancellationToken.None); + } + + private async Task EnrichRecordedChildAsync(IActorRef owner, BackgroundChildRun run, ChildRunTerminal terminal, CancellationToken token) + { + ChildRunTerminal prepared; + try + { + var result = await SubAgentSpawner.EnrichWorkingContextResultAsync( + terminal.Result, run.InitialWorkingSnapshot, run.OriginalContext.Audience, _workingContextSnapshots, token).ConfigureAwait(false); + if (result.Success) + { + var storage = _sessionStorage.ForChild(run.RunId, run.ScopeId); + result = new EnrichedChildRunResult.SuccessfulRun(result, + new EnrichedChildRunResult.RunLocations(storage.LogPath, storage.ArtifactDirectory)).ToProtocolResult(); + } + prepared = terminal with { Result = result }; + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + _log.Warning("Child result enrichment failed runId={RunId} reason={Reason}", run.RunId.Value, error.GetType().Name); + prepared = terminal with { EvidenceWarning = "Result enrichment failed. The original terminal receipt remains unchanged." }; + } + catch (Exception error) when (FatalExceptionPolicy.IsFatal(error)) + { + owner.Tell(new WorkingContextSnapshotFatal(error)); + return; + } + var findings = new List(); + if (prepared.Result.Success) + { + try + { + foreach (var finding in prepared.Result.Findings) + { + var review = SessionToolExecutionPipeline.ReviewSubAgentFinding(finding, _sessionId); + findings.Add(new AcceptedSubAgentFinding + { + RunId = run.RunId, AgentName = run.AgentName, + Duration = TimeSpan.FromMilliseconds(Math.Max(0, NowMs() - run.AcceptedAtMs)), + Shape = finding.Shape, Title = finding.Title, Content = finding.Content, Kind = finding.Kind, + Sensitivity = finding.Sensitivity, RecallMode = finding.RecallMode, + UpdateSemantics = finding.UpdateSemantics, Confidence = finding.Confidence, + Durability = finding.Durability, Reusability = finding.Reusability, + Evidence = finding.Evidence, FreshnessAtMs = finding.FreshnessAtMs, + Decision = review.Decision, DecisionReason = review.Reason + }); + } + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + findings.Clear(); + _log.Warning("Child finding review failed runId={RunId} reason={Reason}", run.RunId.Value, error.GetType().Name); + prepared = prepared with + { + EvidenceWarning = string.Join(" ", new[] { prepared.EvidenceWarning, + "Finding review failed. The factual findings remain in the terminal result." }.Where(static text => text is not null)) + }; + } + catch (Exception error) when (FatalExceptionPolicy.IsFatal(error)) + { + owner.Tell(new WorkingContextSnapshotFatal(error)); + return; + } + } + owner.Tell(new ChildResultEnriched(run.RunId, prepared, findings)); + } + + private void DeliverPreparedChildResultsInOrder() + { + if (_restartDrainRequested) + return; + if (_phase.Current is SessionPhase.Recovering or SessionPhase.Passivating) + return; + var next = _state.ChildRuns.Values.Where(static run => run.Terminal is not null && run.DeliveryInputId is null) + .OrderBy(static run => run.TerminalSequenceNr).FirstOrDefault(); + // An earlier terminal owns the next delivery slot, even when enrichment finishes later. + if (next is null || next.PreparedTerminal is null || !next.StartBatchSettled) + return; + var callId = new ToolCallId($"child-result-{Guid.NewGuid():N}"); + var input = new InputAdmitted + { + SessionId = _sessionId, InputId = new InputId($"child-result-{next.RunId.Value}"), + SourceChildRunId = next.RunId, TurnContext = next.OriginalContext, AdmittedAtMs = NowMs(), + UserMessage = new SerializableChatMessage + { + Role = Protocol.ChatRole.Tool, ToolCallId = callId, + Name = ChildRunDelivery.ToolName(next), Content = ChildRunDelivery.Body(next) + } + }; + var evt = new ChildRunEvent.DeliveryAdmitted(input) + { + SessionId = _sessionId, RunId = next.RunId, RecordedAtMs = input.AdmittedAtMs + }; + _ = _state.Apply(evt); + Persist(evt, committed => + { + _state = _state.Apply(committed); + _log.Info("child_run_delivery_admitted owner={SessionId} runId={RunId} journalSequence={SequenceNr} inputId={InputId} callId={ToolCallId}", + _sessionId.Value, next.RunId.Value, LastSequenceNr, input.InputId.Value, callId.Value); + var command = new SendUserMessage + { + SessionId = _sessionId, Content = input.UserMessage.Content!, AdmittedInputId = input.InputId + }; + if (_phase.Current == SessionPhase.Ready && _state.PendingInputs[0].InputId == input.InputId) + ContinueIncomingUserMessage(command); + else + _buffer.Add((command, false)); + Self.Tell(new DeliverPreparedChildResults()); + }); + } + + private bool CanShareChildContinuation(InputAdmitted first, InputAdmitted next) + { + if (first.SourceChildRunId is null || next.SourceChildRunId is null) + return first.SourceChildRunId is null && next.SourceChildRunId is null; + var firstRun = _state.GetChildContinuation(first); + var nextRun = _state.GetChildContinuation(next); + return SessionState.SameCanonicalContext(firstRun.OriginalContext, nextRun.OriginalContext) + && BackgroundChildRun.SameCheckpoint(firstRun.ParentCheckpoint, nextRun.ParentCheckpoint) + && firstRun.ParentReceiptFailure == nextRun.ParentReceiptFailure; + } + + private void AppendAdmittedInput(InputAdmitted input) + { + if (input.SourceChildRunId is not null) + { + var run = _state.GetChildContinuation(input); + var callId = input.UserMessage.ToolCallId!.Value; + var existing = _state.History.LastOrDefault(message => message.ToolCalls.Any(call => call.CallId == callId)); + if (existing is null || !ParkedToolBatchHistory.HasToolResult(_state.History, existing, callId.Value)) + throw new InvalidDataException("A child result lacks its durably adopted delivery pair."); + MergeSuccessfulSubAgentWorkingContext(run.PreparedTerminal!.Result.Completion); + return; + } + _state = _state with { History = _state.History.Add(input.UserMessage) }; + } + + private void ContinueAdoptedChildResult(InputAdmitted input) + { + _turnEmittedText = false; + _activeInputIds.Clear(); + _activeInputIds.Add(input.InputId); + _sessionManagedTemporaryCorrections.Clear(); + _deliveryRetry.Clear(); + _currentTurnSource = null; + SetSystemPrompt(); + AppendAdmittedInput(input); + _recallManager.ResetForNewTurn(); + _compactionOverflowRetryCount = 0; + if (_state.LoopReceiptFailure) + SettleToolLoop(ToolCycleMessages.MissingReceipt); + else + { + FireLlmCall(); + TransitionTo(SessionPhase.Processing); + } + } + + private void HandleChildControl(ControlBackgroundChildRun message) + { + if (!OwnsToolExecution(message.ExecutionToken) || _state.AdoptedTaskContext is null + || !SessionState.SameCanonicalContext(_state.AdoptedTaskContext, message.InvocationContext)) + { + Sender.Tell(new Status.Failure(new OperationCanceledException("The control dispatch no longer owns this request."))); + return; + } + if (!_state.ChildRuns.TryGetValue(message.Request.RunId, out var run) + || !TurnContext.TryFromRecord(run.OriginalContext, out var original, out _) || original is null + || !TurnContext.TryFromRecord(message.InvocationContext, out var current, out _) || current is null + || original.SessionId != current.SessionId || !TurnContext.HasSameAuthority(original, current)) + { + Sender.Tell(new ChildControlReply(null)); + return; + } + if (!message.Request.Cancel || run.Terminal is not null || run.CancellationRequestedAtMs is not null) + { + Sender.Tell(new ChildControlReply(run)); + return; + } + var replyTo = Sender; + Persist(new ChildRunEvent.CancellationRequested + { + SessionId = _sessionId, RunId = run.RunId, RecordedAtMs = NowMs() + }, admitted => + { + _state = _state.Apply(admitted); + replyTo.Tell(new ChildControlReply(_state.ChildRuns[run.RunId])); + CloseChildDispatch(run.RunId); + }); + } + + private void CloseChildDispatch(SubAgentRunId runId) + { + if (_backgroundChildRuntimes.TryGetValue(runId, out var runtime)) + { + CancelChildLifetime(runtime); + _ = ConfirmChildDispatchClosureAsync(Self, runId, runtime.Dispatch); + } + else + Self.Tell(new ChildDispatchClosureCompleted(runId, null)); + } + + private static async Task ConfirmChildDispatchClosureAsync(IActorRef owner, SubAgentRunId runId, ChildRunDispatch dispatch) + { + Exception? failure = null; + try + { + await dispatch.WaitForEnteredInvocationAsync().ConfigureAwait(false); + } + catch (Exception error) + { + failure = error; + } + owner.Tell(new ChildDispatchClosureCompleted(runId, failure)); + } + + private void CancelChildLifetime(ChildRuntime runtime) + { + runtime.Dispatch.Close(); + // The first asynchronous cancellation owns disposal, including later actor termination. + if (!runtime.Lifetime.IsCancellationRequested) + _ = ChildRunDispatch.CancelAndDisposeAsync(runtime.Lifetime, _log); + } + + private async Task FinalizeCancelledChildAsync(IActorRef owner, BackgroundChildRun run) + { + var storage = _sessionStorage.ForChild(run.RunId, run.ScopeId); + var artifact = storage.ArtifactDirectory.Value; + var anchor = storage.Binding?.EnvelopeRoot.Value ?? storage.SessionDirectory.Value; + var report = Path.Combine(artifact, "cancelled-results.json"); + string output; + string? warning = null; + try + { + await CompleteChildReportWithinGraceAsync(async finalizationToken => + { + try + { + finalizationToken.ThrowIfCancellationRequested(); + if (!ToolOutputSpillLocation.TryEnsureSessionDirectory(anchor) + || !ToolOutputSpillLocation.IsSafeForIo(anchor, artifact)) + throw new IOException("The child artifact directory crosses a filesystem link."); + Directory.CreateDirectory(artifact); + if (!ToolOutputSpillLocation.IsSafeForIo(anchor, artifact) + || !ToolOutputSpillLocation.IsSafeForIo(artifact, report)) + throw new IOException("The child report destination crosses a filesystem link."); + var body = JsonSerializer.Serialize(new + { + run_id = run.RunId.Value, state = "Cancelled", + summary = run.ChildCheckpoint?.Summary, + confirmed_activity = run.ChildCheckpoint?.ConfirmedActivity, + external_effects = "Recorded receipts describe known local results. They do not prove external effects stopped." + }); + await AtomicFile.WriteAllTextAsync(report, body, AtomicFile.HardenOwnerOnly, finalizationToken).ConfigureAwait(false); + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + _log.Warning("Child report worker failed runId={RunId} reason={Reason}", run.RunId.Value, error.GetType().Name); + throw; + } + }, _timeProvider).ConfigureAwait(false); + output = $"The child was cancelled. The last durable partial report is at {report}."; + } + catch (Exception error) when (!FatalExceptionPolicy.IsFatal(error)) + { + output = "The child was cancelled. Netclaw retained its last durable checkpoint."; + warning = error is OperationCanceledException + ? "The five-second framework report grace expired. No report path is confirmed." + : $"The framework report failed: {error.GetType().Name}. No report path is confirmed."; + _log.Warning("Child report finalization failed runId={RunId} reason={Reason}", run.RunId.Value, error.GetType().Name); + } + owner.Tell(new ChildCancellationFinalized(run.RunId, new ChildRunTerminal(new SubAgentResult + { + Completion = new ChildRunCompletion.Cancelled(SubAgentOutcomeReason.CancelledByParent) + { ConfirmedActivity = run.ChildCheckpoint?.ConfirmedActivity }, + AgentName = run.AgentName, RunId = run.RunId, ScopeId = run.ScopeId, + Output = output, ArtifactDirectory = artifact, LogPath = storage.LogPath.Value + }, false, warning))); + } + + internal static async Task CompleteChildReportWithinGraceAsync( + Func operation, TimeProvider timeProvider) + { + using var grace = new CancellationTokenSource(TimeSpan.FromSeconds(5), timeProvider); + var token = grace.Token; + // Filesystem checks and flushes can block before their first await. + var work = Task.Run(() => operation(token), token); + StreamTaskObservation.ObserveSilently(work); + await work.WaitAsync(token).ConfigureAwait(false); + } + + private void CloseBackgroundChildLifetimes() + { + foreach (var runtime in _backgroundChildRuntimes.Values) + { + CancelChildLifetime(runtime); + } + _backgroundChildRuntimes.Clear(); + } +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 2f5c597b8..d14d9419a 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -49,7 +49,7 @@ namespace Netclaw.Actors.Sessions; /// - Processing: buffers incoming messages while LLM call is in flight /// - Compacting: runs tiered context compaction when usage exceeds threshold /// -public sealed class LlmSessionActor : ReceivePersistentActor, IWithTimers +public sealed partial class LlmSessionActor : ReceivePersistentActor, IWithTimers { private readonly SessionId _sessionId; private readonly IChatClient _chatClient; @@ -316,6 +316,20 @@ public LlmSessionActor( ApplyToolBatchStarted(evt); }); Recover(evt => _state = _state.Apply(evt)); + Recover(evt => + { + if (evt.SessionId != _sessionId || evt.Run.StartKey.SessionId != _sessionId) + throw new InvalidDataException("A child acceptance belongs to another session owner."); + _state = _state.Apply(evt); + }); + Recover(evt => + { + if (evt.SessionId != _sessionId) + throw new InvalidDataException("A child lifecycle event belongs to another session owner."); + if (evt is ChildRunEvent.TerminalRecorded terminal && terminal.TerminalSequenceNr != LastSequenceNr) + throw new InvalidDataException("A child terminal differs from its journal position."); + _state = _state.Apply(evt); + }); Recover(evt => _state = _state.Apply(evt)); Recover(evt => _state = _state.Apply(evt)); Recover(ApplyToolCallRecorded); @@ -326,6 +340,10 @@ public LlmSessionActor( { if (offer.Snapshot is SessionSnapshot snapshot) { + if (snapshot.ChildRuns.Any(run => run.StartKey.SessionId != _sessionId)) + throw new InvalidDataException("A child snapshot belongs to another session owner."); + if (snapshot.ChildRuns.Any(run => run.TerminalSequenceNr > offer.Metadata.SequenceNr)) + throw new InvalidDataException("A child terminal exceeds its snapshot journal position."); _state = SessionState.FromSnapshot(snapshot); if (snapshot.EligibleDeliveryTurnNumber is { } eligibleTurn) _deliveryRetry.MarkEligible(eligibleTurn); @@ -341,8 +359,6 @@ public LlmSessionActor( // Always read fresh from disk — identity file edits take effect immediately SetSystemPrompt(); - TransitionTo(SessionPhase.Ready); - // Resolve once at recovery time. ActorRegistry returns Nobody if // the dispatcher hasn't been registered (unit-test scenarios that // skip the hosting wiring) — leave _logActor null in that case. @@ -371,6 +387,7 @@ public LlmSessionActor( _timeProvider), "memory-observer"); } + RecoverBackgroundChildren(); }); } @@ -397,6 +414,7 @@ private void TransitionTo(SessionPhase target) { case SessionPhase.Ready: Become(Ready); + Self.Tell(new DeliverPreparedChildResults()); break; case SessionPhase.Processing: Become(Processing); @@ -441,10 +459,10 @@ private void Ready() Command(_ => { - if (HasLiveBackgroundJobs) + if (HasLiveBackgroundJobs || HasUnsettledChildWork) { _log.Info( - "Session idle but active background jobs remain; deferring passivation"); + "Session idle but active background jobs or child work remain; deferring passivation"); return; } @@ -487,6 +505,7 @@ private void Ready() CommandDistillationAckNoOp(); CommandJobReapResolved(); Command(HandleSpawnChildActorRequest); + RegisterBackgroundChildHandlers(); Command(HandleDeliveryFailedWhenReady); // Approval click for a tool batch that parked while the session was @@ -767,24 +786,9 @@ private void Processing() }); Command(HandleSpawnChildActorRequest); - Command(HandleRoutedSkillExecutionCompleted); + RegisterBackgroundChildHandlers(); + Command(HandleRoutedSkillStartAccepted); Command(HandleRoutedSkillExecutionFailed); - Command(msg => - { - if (!OwnsToolExecution(msg.ExecutionToken)) - return; - EmitOutput(new SubAgentOutput - { - SessionId = _sessionId, - TimestampMs = msg.TimestampMs, - AgentName = msg.AgentName, - Phase = msg.Phase, - ToolCount = msg.ToolCount, - Success = msg.Success ?? false, - Duration = msg.Duration ?? TimeSpan.Zero, - FindingsCount = msg.FindingsCount - }, OutputFilter.ToolCalls); - }); CommandDistillationAckNoOp(); CommandJobReapResolved(); } @@ -1209,15 +1213,20 @@ private void Compacting() // Buffer an approval response during compaction — compaction rewrites // history, so the parked tool batch cannot be re-driven mid-flight. // DrainBufferOrReady replays the buffered response after compaction. - Command(msg => + CommandAsync(async msg => { + if (await TryHandleChildApprovalResponseAsync(msg)) + return; _log.Info("Buffering tool interaction response (compaction in progress)"); _deferredApprovalResponse = msg; TryReplyAck(); }); - Command(msg => + CommandAsync(async msg => { + if (TryResolveTextApprovalResponse(msg, out var response, out _) && response is not null + && await TryHandleChildApprovalResponseAsync(response)) + return; _log.Info("Buffering text tool interaction response (compaction in progress)"); _deferredApprovalResponse = msg; TryReplyAck(); @@ -1225,6 +1234,7 @@ private void Compacting() Command(HandleCompactionWatchdogExpired); Command(HandleSpawnChildActorRequest); + RegisterBackgroundChildHandlers(); Command(HandleCompactionTriggered); @@ -1617,6 +1627,7 @@ private void Passivating() Command(_ => { }); Command(_ => { }); Command(HandleSpawnChildActorRequest); + RegisterBackgroundChildHandlers(); // Timeout — stop even if distillation didn't finish Command(_ => @@ -1671,7 +1682,7 @@ private void Passivating() // and we proceed to FinalizePassivation. private void CompletePassivation() { - if (_passivationFinalStopScheduled) + if (_passivationFinalStopScheduled || HasLiveChildRuns) return; // The final snapshot must capture the reaped job marks — defer until @@ -2130,10 +2141,16 @@ private void DispatchToolBatch( // Marshal child-actor spawning back onto the session actor thread. Func> spawnChildActor = async (props, name, ct) => - await self.Ask( + { + if (props is StartBackgroundChildRun start) + return await self.Ask(start, toolExecutionTimeout, ct); + if (props is ControlBackgroundChildRun control) + return await self.Ask(control, toolExecutionTimeout, ct); + return await self.Ask( new SpawnChildActorRequest((Props)props, name, toolExecutionCt), timeout: toolExecutionTimeout, cancellationToken: ct); + }; var registry = ActorRegistry.For(Context.System); var backgroundJobs = registry.TryGet(out var manager) @@ -2313,6 +2330,8 @@ private void DrainBufferedUserMessages(Action continuation, Action afterRejected var input = _state.PendingInputs.Single(item => item.InputId == buffered.Message.AdmittedInputId); if (!TurnContext.TryFromRecord(input.TurnContext, out var candidate, out var reason) || candidate is null) throw new InvalidDataException($"A buffered input has invalid authority: {reason}"); + if (latest is not null && !CanShareChildContinuation(latest, input)) + break; if (prefixLength > 0 && (latest?.SourceBackgroundJobId is not null || input.SourceBackgroundJobId is not null)) break; if (prefixContext is not null && !TurnContext.HasSameAuthority(prefixContext, candidate)) @@ -2360,12 +2379,18 @@ private void DrainAdoptedUserMessages(int prefixLength, Action continuation) foreach (var (message, _) in _buffer.Take(prefixLength)) { var refs = message.MediaReferences.Count > 0 ? message.MediaReferences : null; - _state = _state.AddUserMessage(message.Content, refs); + if (message.AdmittedInputId is { } inputId) + AppendAdmittedInput(_state.PendingInputs.Single(input => input.InputId == inputId)); + else + _state = _state.AddUserMessage(message.Content, refs); if (message.AdmittedInputId is { } id && !_activeInputIds.Contains(id)) _activeInputIds.Add(id); } _buffer.RemoveRange(0, prefixLength); - continuation(); + if (_state.LoopReceiptFailure) + SettleToolLoop(ToolCycleMessages.MissingReceipt); + else + continuation(); } private void HandleDeliveryFailedWhenReady(DeliveryFailed msg) @@ -2569,8 +2594,13 @@ private void RestoreConsumedInputs(IReadOnlyList inputIds, Serializable if (!pendingById.TryGetValue(id, out var input)) throw new InvalidOperationException($"Journal input {id} is missing before its terminal event."); - var message = index == inputIds.Count - 1 ? lastUserMessage : input.UserMessage; - _state = _state with { History = _state.History.Add(message) }; + if (input.SourceChildRunId is not null) + AppendAdmittedInput(input); + else + { + var message = index == inputIds.Count - 1 ? lastUserMessage : input.UserMessage; + _state = _state with { History = _state.History.Add(message) }; + } } _state = _state.CloseInputs(inputIds); @@ -2632,7 +2662,13 @@ private void ContinueIncomingUserMessage(SendUserMessage cmd) CloseInvalidJobDelivery(admitted, TryReplyAck); return; } - AdoptToolTask(admitted, [admitted.InputId], () => ContinueAdoptedIncomingUserMessage(cmd)); + AdoptToolTask(admitted, [admitted.InputId], () => + { + if (admitted.SourceChildRunId is not null) + ContinueAdoptedChildResult(admitted); + else + ContinueAdoptedIncomingUserMessage(cmd); + }); } private void AdoptToolTask(InputAdmitted admitted, IReadOnlyList inputIds, Action continuation) @@ -2640,7 +2676,7 @@ private void AdoptToolTask(InputAdmitted admitted, IReadOnlyList inputI var evt = new ToolTaskAdopted { SessionId = _sessionId, TurnContext = admitted.TurnContext, InputIds = inputIds.ToArray(), AdoptedAtMs = NowMs(), - ContinuedJobKey = admitted.SourceBackgroundJobId?.Value + ContinuedJobKey = admitted.SourceBackgroundJobId?.Value, ContinuedChildRunId = admitted.SourceChildRunId }; Persist(evt, adopted => { @@ -2805,7 +2841,7 @@ private void ContinueResumedInput(SendUserMessage cmd, InputAdmitted[] pending, _activeInputIds.AddRange(pending.Select(static item => item.InputId)); foreach (var item in pending) - _state = _state with { History = _state.History.Add(item.UserMessage) }; + AppendAdmittedInput(item); _recallManager.ResetForNewTurn(); _compactionOverflowRetryCount = 0; @@ -2819,7 +2855,10 @@ private void ContinueResumedInput(SendUserMessage cmd, InputAdmitted[] pending, SettleToolLoop(ToolCycleMessages.MissingReceipt); return; } - FireInitialTurnLlmCall(recallQuery); + if (pending.Any(static input => input.SourceChildRunId is not null)) + FireLlmCall(); + else + FireInitialTurnLlmCall(recallQuery); TransitionTo(SessionPhase.Processing); } @@ -2941,24 +2980,8 @@ private void CommandCommonMessages() Command(msg => HandleDistillationResult(msg)); - Command(HandleRoutedSkillExecutionCompleted); + Command(HandleRoutedSkillStartAccepted); Command(HandleRoutedSkillExecutionFailed); - Command(msg => - { - if (!OwnsToolExecution(msg.ExecutionToken)) - return; - EmitOutput(new SubAgentOutput - { - SessionId = _sessionId, - TimestampMs = msg.TimestampMs, - AgentName = msg.AgentName, - Phase = msg.Phase, - ToolCount = msg.ToolCount, - Success = msg.Success ?? false, - Duration = msg.Duration ?? TimeSpan.Zero, - FindingsCount = msg.FindingsCount - }, OutputFilter.ToolCalls); - }); Command(cmd => { @@ -3064,6 +3087,7 @@ protected override void PreRestart(Exception reason, object message) CancelAndDisposeLlmCts(); CancelAndDisposeToolExecutionCts(); + CloseBackgroundChildLifetimes(); base.PreRestart(reason, message); } @@ -3072,6 +3096,8 @@ protected override void PostStop() CancelAndDisposeLlmCts(); CancelAndDisposeToolExecutionCts(); + CloseBackgroundChildLifetimes(); + // Safety net for non-graceful stop paths (shutdown timeout, OOM, etc.). if (!_passivationCompleted) _lifecycleObserver?.OnSessionDeactivated(_sessionId); @@ -3825,7 +3851,11 @@ private bool TryHandleRoutedSlashCommand(SkillEntry skill, string remainder, IRe CancelAndDisposeToolExecutionCts(); _activeToolExecutionCts = new CancellationTokenSource(); - _ = ExecuteRoutedSkillAsync(Self, skill, profile, effectiveTask, skillBody, routedAudience, + var original = _currentTurnContext + ?? throw new InvalidOperationException("A routed activation requires canonical admitted authority."); + var activationId = _activeInputIds.Single(); + _ = ExecuteRoutedSkillAsync(Self, skill, profile, effectiveTask, skillBody, original, + new ChildRunStartKey.Slash(activationId) { SessionId = _sessionId, TurnId = original.TurnId }, _activeToolExecutionCts.Token); TransitionTo(SessionPhase.Processing); return true; @@ -3837,45 +3867,49 @@ private async Task ExecuteRoutedSkillAsync( SubAgentProfile profile, string task, string skillBody, - TrustAudience audience, + TurnContext original, + ChildRunStartKey.Slash activation, CancellationToken executionToken) { try { - Func> spawnChildActor = async (props, name, ct) => - await self.Ask( - new SpawnChildActorRequest((Props)props, name, executionToken), - timeout: _config.ToolExecutionTimeout, - cancellationToken: ct); - var outputs = new ToolExecutionOutputs(info => + Func> spawnChildActor = async (payload, name, ct) => { - self.Tell(new RoutedSkillSubAgentActivity( - executionToken, - _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(), - new AgentName(info.AgentName), - info.IsStarted ? SubAgentPhase.Started : SubAgentPhase.Completed, - info.ToolCount, - info.Success, - info.Duration, - info.Findings.Count)); - }); + if (payload is not PreparedChildRun prepared) + throw new InvalidDataException("A routed activation has no canonical child preparation."); + var request = new StartBackgroundChildRun( + prepared, activation, original.ToRecord(), "/" + skill.Name, executionToken); + var response = await self.Ask(request, _config.ToolExecutionTimeout, ct); + if (response is ChildStartReply.Accepted accepted) + accepted.Validate(request); + return response; + }; + var outputs = new ToolExecutionOutputs(); + var approvalBridge = original.SupportsInteractiveApproval && original.HasApprovalRequester + ? new ParentSessionApprovalBridge(_approvalChannel, + prompt => self.Tell(new ToolExecutionApprovalRequested(prompt, executionToken)), + new ToolExecutionTimeout(Timeout.InfiniteTimeSpan), _sessionId, activation.InputId.Value, + original.RequesterSenderId, original.RequesterPrincipal, original.HasAdoptedContext, + original.HasThirdPartyAdoptedContext, original.AdoptedSpeakerIds) + : null; var context = new ToolExecutionContext(new ToolRunScope { Session = new ToolSessionScope.Bound(_sessionId.Value, _sessionStorage), // The caller resolved this audience from the turn authority and // refused the command when no authority existed. - Audience = audience, + Audience = original.Audience, InlineOutputBudget = new InlineOutputBudget(_config.Tuning.MaxInlineToolResultChars), - Boundary = _currentTurnContext?.Boundary ?? _currentTurnSource?.Boundary, - ChannelType = _currentTurnContext?.ChannelType?.ToWireValue() - ?? (_currentTurnSource is null ? null : _currentTurnSource.ChannelType.ToWireValue()), + Boundary = original.Boundary, + ChannelType = original.ChannelType?.ToWireValue(), ProjectDirectory = _state.WorkingContext.ProjectDirectory, RecentFiles = _state.WorkingContext.RecentFiles, - InteractiveApproval = new InteractiveApprovalCapability.Unavailable(), + InteractiveApproval = approvalBridge is null + ? new InteractiveApprovalCapability.Unavailable() + : new InteractiveApprovalCapability.Available(approvalBridge), SpawnChildActor = spawnChildActor, }, new ToolExecutionTimeout(_config.ToolExecutionTimeout), outputs); - var result = await _subAgentSpawner!.SpawnAsync( + var acceptance = await _subAgentSpawner!.StartRunAsync( profile, task, runtimeContext: null, @@ -3883,7 +3917,9 @@ await self.Ask( executionToken, systemPromptOverlay: skillBody); - self.Tell(new RoutedSkillExecutionCompleted(skill.Name, profile.Name, result, executionToken)); + if (outputs.Receipt is not ToolInvocationReceipt.Succeeded) + throw new InvalidOperationException(acceptance); + self.Tell(new RoutedSkillStartAccepted(acceptance, executionToken)); } catch (Exception ex) { @@ -3891,22 +3927,11 @@ await self.Ask( } } - private void HandleRoutedSkillExecutionCompleted(RoutedSkillExecutionCompleted msg) + private void HandleRoutedSkillStartAccepted(RoutedSkillStartAccepted msg) { if (!OwnsToolExecution(msg.ExecutionToken)) return; CancelAndDisposeToolExecutionCts(); - if (!msg.Result.Success) - { - FailCurrentTurn( - $"Skill '/{msg.SkillName}' routed to subagent '{msg.SubagentName}' failed: {msg.Result.Output}", - new InvalidOperationException(msg.Result.Output), - ErrorCategory.ToolFailure); - return; - } - - MergeSuccessfulSubAgentWorkingContext(msg.Result.Completion); - var userMsg = _state.FindLastUserMessage(); var turnEvent = new TurnRecorded { @@ -3919,7 +3944,7 @@ private void HandleRoutedSkillExecutionCompleted(RoutedSkillExecutionCompleted m AssistantReply = new SerializableChatMessage { Role = Protocol.ChatRole.Assistant, - Content = msg.Result.Output + Content = msg.Acceptance }, RecordedAtMs = NowMs(), SourceReminderId = _currentTurnSource?.ReminderId, @@ -3947,7 +3972,7 @@ private void HandleRoutedSkillExecutionCompleted(RoutedSkillExecutionCompleted m ProcessedReminderIds = processed }).CompleteTurnBackgroundJobBookkeeping(evt.SourceBackgroundJobId); - EmitOutput(new TextOutput(msg.Result.Output) + EmitOutput(new TextOutput(msg.Acceptance) { SessionId = _sessionId }, OutputFilter.Text); @@ -4158,6 +4183,8 @@ private void RestoreActiveToolBatchFrom(ToolBatchStarted evt) private void ApplyToolCallRecorded(ToolCallRecorded evt) { _state = _state.ApplyLoopObservation(evt); + if (_phase.Current != SessionPhase.Recovering) + Self.Tell(new DeliverPreparedChildResults()); var alreadyRecorded = false; if (evt.ToolResult.ToolCallId is { } toolCallId) { @@ -4185,7 +4212,30 @@ private void ApplyToolCallRecorded(ToolCallRecorded evt) private void HandleToolInteractionRequestDispatch(ToolInteractionRequestDispatch dispatch) { var msg = dispatch.Request; - var evt = new ToolApprovalRequested + var evt = CreateToolApprovalRequest(dispatch, _currentTurnContext); + + if (!dispatch.PersistApprovalState) + { + // Live sub-agent approvals need the prompt in the in-memory pending + // map so responses can be authorized, but there is no durable child + // actor/tool batch to redrive after restart. + ApplyToolApprovalRequested(evt, persistApprovalState: false); + EmitOutput(msg); + return; + } + + Persist(evt, e => + { + ApplyToolApprovalRequested(e); + EmitOutput(msg); + TryStopDurableApprovalWaits(); + }); + } + + private ToolApprovalRequested CreateToolApprovalRequest(ToolInteractionRequestDispatch dispatch, TurnContext? context) + { + var msg = dispatch.Request; + return new ToolApprovalRequested { SessionId = _sessionId, CallId = msg.CallId.Value, @@ -4193,10 +4243,10 @@ private void HandleToolInteractionRequestDispatch(ToolInteractionRequestDispatch ToolName = msg.ToolName.Value, Patterns = msg.Patterns, CandidateVerbs = msg.CandidateVerbs, - Audience = _currentTurnContext?.Audience ?? CurrentTurnAudience(), - Boundary = _currentTurnContext?.Boundary ?? _currentTurnSource?.Boundary, - ChannelType = _currentTurnContext?.ChannelType?.ToWireValue() ?? _currentTurnSource?.ChannelType.ToWireValue(), - SupportsInteractiveApproval = _currentTurnContext?.SupportsInteractiveApproval + Audience = context?.Audience ?? CurrentTurnAudience(), + Boundary = context?.Boundary ?? _currentTurnSource?.Boundary, + ChannelType = context?.ChannelType?.ToWireValue() ?? _currentTurnSource?.ChannelType.ToWireValue(), + SupportsInteractiveApproval = context?.SupportsInteractiveApproval ?? _currentTurnSource?.ChannelType.SupportsInteractiveApproval(), RequesterSenderId = msg.RequesterSenderId, RequesterPrincipal = msg.RequesterPrincipal, @@ -4206,27 +4256,10 @@ private void HandleToolInteractionRequestDispatch(ToolInteractionRequestDispatch RepositoryCommonDirectory = msg.RepositoryCommonDirectory, OptionKeys = msg.Options.Select(o => o.Key.Value).ToArray(), Candidates = msg.Candidates, - TurnContext = _currentTurnContext?.ToRecord(), + TurnContext = context?.ToRecord(), ManagedTemporaryDirectory = dispatch.ManagedTemporaryDirectory, RequestedAtMs = NowMs() }; - - if (!dispatch.PersistApprovalState) - { - // Live sub-agent approvals need the prompt in the in-memory pending - // map so responses can be authorized, but there is no durable child - // actor/tool batch to redrive after restart. - ApplyToolApprovalRequested(evt, persistApprovalState: false); - EmitOutput(msg); - return; - } - - Persist(evt, e => - { - ApplyToolApprovalRequested(e); - EmitOutput(msg); - TryStopDurableApprovalWaits(); - }); } private void ApplyToolApprovalRequested(ToolApprovalRequested evt) @@ -4234,6 +4267,11 @@ private void ApplyToolApprovalRequested(ToolApprovalRequested evt) private void ApplyToolApprovalRequested(ToolApprovalRequested evt, bool persistApprovalState) { + if (evt.SourceChildRunId is not null) + { + _state = _state.ApplyChildApproval(evt); + return; + } var pending = _toolApprovals.Request( evt, persistApprovalState, @@ -4276,6 +4314,11 @@ private void ClearApprovalTurnState() private void ApplyToolApprovalResolved(ToolApprovalResolved evt) { + if (evt.SourceChildRunId is not null) + { + _state = _state.ApplyChildApproval(evt); + return; + } _toolApprovals.Resolve(evt.CallId, ConsentAnswerCodec.FromJournalText(evt.Decision), out _); } @@ -4288,9 +4331,11 @@ private void ApplyToolBatchAbandoned(ToolBatchAbandoned evt) ToolResult = result, RecordedAtMs = evt.AbandonedAtMs }); + _state = _state.SettleAbandonedChildStartBatches(evt); _toolApprovals.ClearCalls(); ClearApprovalTurnState(); ClearActiveToolBatchTracking(); + Self.Tell(new DeliverPreparedChildResults()); } private void ClearActiveToolBatchTracking() @@ -4421,8 +4466,12 @@ private void EmitUsageOutput(UsageDetails usage) }, OutputFilter.Usage); } + private bool HasPendingChildApprovals + => _state.ChildRuns.Values.Any(static run => run.Approvals.Any(static approval => approval.Resolution is null)); + private bool HasApprovalHistory - => _toolApprovals.ResolvedCount > 0 + => _state.ChildRuns.Values.Any(static run => run.Approvals.Count > 0) + || _toolApprovals.ResolvedCount > 0 || ParkedToolBatchHistory.FindRedrivableAssistantMessage(_state.History, null) is not null; /// @@ -4561,7 +4610,7 @@ private bool TryResolveTextApprovalResponse( var pending = ResolveLatestPendingApprovalForSender(msg.SenderId); if (pending is null) { - if (_toolApprovals.PendingCount == 0) + if (_toolApprovals.PendingCount == 0 && !HasPendingChildApprovals) { if (HasApprovalHistory) { @@ -4629,7 +4678,15 @@ private bool TryResolveTextApprovalResponse( } private PendingToolInteraction? ResolveLatestPendingApprovalForSender(SenderId senderId) - => _toolApprovals.FindLatestPending(pending => CanApprovePending(pending, senderId)); + { + var parent = _toolApprovals.FindLatestPending(pending => CanApprovePending(pending, senderId)); + var child = _state.ChildRuns.Values.SelectMany(static run => run.Approvals) + .Where(static approval => approval.Resolution is null) + .Select(static approval => PendingToolInteraction.From(approval.Request, persistApprovalState: false)) + .Where(pending => CanApprovePending(pending, senderId)) + .OrderByDescending(static pending => pending.Request.RequestedAtMs).FirstOrDefault(); + return child is not null && (parent is null || child.Request.RequestedAtMs > parent.Request.RequestedAtMs) ? child : parent; + } private static bool CanApprovePending(PendingToolInteraction pending, SenderId senderId) => TryGetApprovalAuthority(pending, out var principal, out var requesterSenderId, out _) @@ -4677,6 +4734,8 @@ private static bool TryGetApprovalAuthority( private async Task HandleProcessingApprovalResponseAsync(ToolInteractionResponse msg) { + if (await TryHandleChildApprovalResponseAsync(msg)) + return; if (!_toolApprovals.TryGetPending(msg.CallId.Value, out var pending)) { _log.Warning("Ignoring tool interaction response for unknown call {CallId}", msg.CallId); @@ -4769,6 +4828,8 @@ private void PersistApprovalResolved( /// private async Task HandleToolInteractionResponseWhenIdle(ToolInteractionResponse msg) { + if (await TryHandleChildApprovalResponseAsync(msg)) + return; var callId = msg.CallId.Value; if (!_toolApprovals.TryGetPending(callId, out var pending)) @@ -5444,11 +5505,8 @@ private void PersistAdoptedContextIfNeeded(MessageSource? source) Persist(evt, e => _state = _state.Apply(e)); } - private sealed record RoutedSkillExecutionCompleted( - string SkillName, - string SubagentName, - SubAgentResult Result, - CancellationToken ExecutionToken) : INoSerializationVerificationNeeded; + private sealed record RoutedSkillStartAccepted( + string Acceptance, CancellationToken ExecutionToken) : INoSerializationVerificationNeeded; private sealed record RoutedSkillExecutionFailed( string SkillName, @@ -5456,16 +5514,6 @@ private sealed record RoutedSkillExecutionFailed( string ErrorMessage, CancellationToken ExecutionToken) : INoSerializationVerificationNeeded; - private sealed record RoutedSkillSubAgentActivity( - CancellationToken ExecutionToken, - long TimestampMs, - AgentName AgentName, - SubAgentPhase Phase, - int ToolCount, - bool? Success, - TimeSpan? Duration, - int FindingsCount) : INoSerializationVerificationNeeded; - private enum ApprovalRedriveOutcome { Started, @@ -5523,6 +5571,7 @@ private void RequestRestartDrain() { _restartDrainRequested = true; _restartDrainReplyTo = Sender; + RequestChildRestartDrain(); if (_phase.Current == SessionPhase.Ready) TransitionTo(SessionPhase.Passivating); @@ -5762,9 +5811,13 @@ private InputAdmitted[] GetRestartInputPrefix(TurnContext firstContext) var prefix = new List(); foreach (var input in _state.PendingInputs) { + if (prefix.Count > 0 && prefix[0].ExecutableText?.TrimStart().StartsWith("/", StringComparison.Ordinal) == true) + break; if (!TurnContext.TryFromRecord(input.TurnContext, out var candidate, out _) || candidate is null || !TurnContext.HasSameAuthority(firstContext, candidate)) break; + if (prefix.Count > 0 && !CanShareChildContinuation(prefix[^1], input)) + break; if (prefix.Count > 0 && (prefix[^1].SourceBackgroundJobId is not null || input.SourceBackgroundJobId is not null)) break; prefix.Add(input); diff --git a/src/Netclaw.Actors/Sessions/ParentSessionApprovalBridge.cs b/src/Netclaw.Actors/Sessions/ParentSessionApprovalBridge.cs index 87f1fca12..d5c4c16b2 100644 --- a/src/Netclaw.Actors/Sessions/ParentSessionApprovalBridge.cs +++ b/src/Netclaw.Actors/Sessions/ParentSessionApprovalBridge.cs @@ -165,6 +165,7 @@ private async Task PromptAsync( .ToList() }, persistApprovalState) { + OriginalChildCallId = persistApprovalState ? null : request.CallId, // Only a journaled wait can restore the managed temporary retry after passivation. ManagedTemporaryDirectory = persistApprovalState && approval.IsManagedTemporaryRetry ? approval.ManagedTemporaryDirectory diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs index c31fcffc9..106a4f367 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs @@ -14,6 +14,7 @@ using Netclaw.Actors.Jobs; using Netclaw.Actors.Protocol; using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; using Netclaw.Actors.Tools; using Netclaw.Configuration; using Netclaw.Media; @@ -551,6 +552,23 @@ private async Task ExecuteSingleToolAsync( : null; var callScope = batch.RunScope with { + SpawnChildActor = async (argument, name, startToken) => + { + var request = argument switch + { + PreparedChildRun prepared => new StartBackgroundChildRun(prepared, + new ChildRunStartKey.Tool(new ToolCallId(tc.CallId)) + { SessionId = batch.TurnContext.SessionId, TurnId = batch.TurnContext.TurnId }, + batch.TurnContext.ToRecord(), tc.Name, batch.CancellationToken), + ChildControlRequest control => new ControlBackgroundChildRun(control, + batch.TurnContext.ToRecord(), batch.CancellationToken), + _ => argument + }; + var response = await batch.RunScope.SpawnChildActor!(request, name, startToken).ConfigureAwait(false); + if (request is StartBackgroundChildRun start && response is ChildStartReply.Accepted accepted) + accepted.Validate(start); + return response; + }, InteractiveApproval = approvalBridge is null ? new InteractiveApprovalCapability.Unavailable() : new InteractiveApprovalCapability.Available(approvalBridge) @@ -868,16 +886,7 @@ private static async Task ExecuteToolAttemptAsync( { var stream = executor.ExecuteStreamAsync(toolCall, context, cancellationToken); - // Self-monitoring tools (spawn_agent) own their liveness end to end and - // always drive their stream to a terminal item, so the parent does not - // supervise them at all — it drains to that terminal item under caller - // (turn/user) cancellation only. For spawn_agent the terminal item is - // produced by SpawnAgentTool's stream, which completes when SpawnAsync - // returns; SpawnAsync's finally unconditionally completes the activity - // channel, and SubAgentActor.PostStop guarantees the reply that lets - // SpawnAsync return even on a crash. (Note: PostStop alone only unblocks - // the spawner Ask — the terminal stream item depends on that finally - // running.) + // A self-monitoring tool owns its terminal stream and its liveness policy. if (executor.GetLivenessMode(toolCall) == ToolLivenessMode.SelfMonitoring) return await DrainToCompletionAsync(stream, toolCall.Name, cancellationToken); diff --git a/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs b/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs index a02c2bed0..b31e1ece2 100644 --- a/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs +++ b/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs @@ -3,7 +3,9 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- +using System.Text.Json; using Microsoft.Extensions.AI; +using Netclaw.Actors.Channels; using Netclaw.Actors.Protocol; using Netclaw.Actors.Tools; using Netclaw.Configuration; @@ -247,6 +249,10 @@ internal static string BuildVolatileContextBlock(ContextAssemblyInput input) if (!input.State.ActiveBackgroundJobs.IsEmpty && input.Audience != TrustAudience.Public) parts.Add(FormatActiveBackgroundJobs(input.State)); + var childRuns = FormatBackgroundChildRuns(input); + if (childRuns.Length > 0) + parts.Add(childRuns); + if (input.SlashCommandSkillContent is not null) parts.Add(input.SlashCommandSkillContent); @@ -259,6 +265,35 @@ internal static string BuildVolatileContextBlock(ContextAssemblyInput input) return string.Join("\n\n", parts); } + private static string FormatBackgroundChildRuns(ContextAssemblyInput input) + { + if (input.Audience == TrustAudience.Public || input.State.ChildRuns.Count == 0 + || input.State.AdoptedTaskContext is not { } currentRecord) + return string.Empty; + if (!TurnContext.TryFromRecord(currentRecord, out var current, out var reason) || current is null) + throw new InvalidDataException($"The child context has invalid current authority: {reason}"); + var pendingIds = input.State.PendingInputs.Select(static pending => pending.InputId).ToHashSet(); + var visible = input.State.ChildRuns.Values + .Where(run => run.DeliveryInputId is null || pendingIds.Contains(run.DeliveryInputId.Value)) + .Where(run => + { + if (!TurnContext.TryFromRecord(run.OriginalContext, out var original, out var originalReason) || original is null) + throw new InvalidDataException($"The child context has invalid original authority: {originalReason}"); + return TurnContext.HasSameAuthority(original, current); + }).OrderByDescending(static run => run.AcceptedAtMs).ThenBy(static run => run.RunId.Value, StringComparer.Ordinal) + .Take(8).Select(static run => new + { + run_id = run.RunId.Value, scope_id = run.ScopeId.Value, agent = run.AgentName.Value, + state = run.State.ToString(), cancellation_requested = run.CancellationRequestedAtMs is not null, + dispatch_closed = run.DispatchClosedAtMs is not null + }).ToArray(); + if (visible.Length == 0) + return string.Empty; + return "[background-agent-runs]\n" + JsonSerializer.Serialize(visible) + + "\nThese are recorded lifecycle facts. They do not prove current remote activity." + + "\nResults arrive automatically. For a necessary status or cancel request, load_tool(name: \"check_agent_run\")."; + } + private static string FormatActiveBackgroundJobs(SessionState state) { var sb = new System.Text.StringBuilder(); diff --git a/src/Netclaw.Actors/Sessions/SessionProtocol.ChildRuns.cs b/src/Netclaw.Actors/Sessions/SessionProtocol.ChildRuns.cs new file mode 100644 index 000000000..2db39c25d --- /dev/null +++ b/src/Netclaw.Actors/Sessions/SessionProtocol.ChildRuns.cs @@ -0,0 +1,30 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; + +namespace Netclaw.Actors.Sessions; + +public static partial class SessionProtocol +{ + /// Committed lifecycle facts for one accepted child run. + public abstract record ChildRunEvent : ISessionEvent + { + public required SessionId SessionId { get; init; } + public required SubAgentRunId RunId { get; init; } + public required long RecordedAtMs { get; init; } + public DateTimeOffset Timestamp => DateTimeOffset.FromUnixTimeMilliseconds(RecordedAtMs); + + public sealed record Started : ChildRunEvent; + public sealed record Checkpointed(ChildRunCheckpoint Checkpoint) : ChildRunEvent; + public sealed record CancellationRequested : ChildRunEvent; + public sealed record DispatchClosed : ChildRunEvent; + public sealed record TerminalRecorded(ChildRunTerminal Terminal, long TerminalSequenceNr) : ChildRunEvent; + public sealed record ResultPrepared(ChildRunTerminal Terminal, long TerminalSequenceNr) : ChildRunEvent; + public sealed record DeliveryAdmitted(InputAdmitted Input) : ChildRunEvent; + } +} diff --git a/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs b/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs index be1cbd3e3..ac772abae 100644 --- a/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs +++ b/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs @@ -7,6 +7,7 @@ using Netclaw.Actors.Protocol; using Netclaw.Actors.Reminders; using Netclaw.Actors.Serialization; +using Netclaw.Actors.SubAgents; using Netclaw.Configuration; using Netclaw.Security; using Netclaw.Tools; @@ -15,6 +16,13 @@ namespace Netclaw.Actors.Sessions; public static partial class SessionProtocol { + public sealed record ChildRunAccepted : ISessionEvent + { + public SessionId SessionId { get; init; } + public required BackgroundChildRun Run { get; init; } + public DateTimeOffset Timestamp => DateTimeOffset.FromUnixTimeMilliseconds(Run.AcceptedAtMs); + } + // ===== Events (persisted to the session journal) ===== /// @@ -62,6 +70,7 @@ public sealed record TurnRecorded : ISessionEvent /// public sealed record InputAdmitted : ISessionEvent { + public Netclaw.Tools.SubAgentRunId? SourceChildRunId { get; init; } public BackgroundJobId? SourceBackgroundJobId { get; init; } public int BackgroundJobLineageVersion { get; init; } public BackgroundJobOrigin? BackgroundJobOrigin { get; init; } @@ -86,6 +95,7 @@ public sealed record InputAdmitted : ISessionEvent /// Commits canonical task authority before the next model request. public sealed record ToolTaskAdopted : ISessionEvent { + public Netclaw.Tools.SubAgentRunId? ContinuedChildRunId { get; init; } public string? ContinuedJobKey { get; init; } public SessionId SessionId { get; init; } public required TurnContextRecord TurnContext { get; init; } @@ -193,6 +203,10 @@ public sealed record TurnContextRecord public sealed record ToolApprovalRequested : ISessionEvent { + public SubAgentRunId? SourceChildRunId { get; init; } + + public ToolCallId? OriginalChildCallId { get; init; } + public SessionId SessionId { get; init; } public string CallId { get; init; } = string.Empty; @@ -243,6 +257,8 @@ public sealed record ToolApprovalRequested : ISessionEvent public sealed record ToolApprovalResolved : ISessionEvent { + public SubAgentRunId? SourceChildRunId { get; init; } + public SessionId SessionId { get; init; } public string CallId { get; init; } = string.Empty; diff --git a/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs b/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs new file mode 100644 index 000000000..9635296e2 --- /dev/null +++ b/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs @@ -0,0 +1,186 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Protocol; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Sessions; + +public sealed partial record SessionState +{ + internal SessionState SettleAbandonedChildStartBatches(ToolBatchAbandoned evt) + { + if (LoopAdmission is not { } admission) + return this; + var closedCalls = evt.ToolResults.Where(static result => result.Role == Protocol.ChatRole.Tool) + .Select(static result => result.ToolCallId?.Value).ToHashSet(StringComparer.Ordinal); + if (!admission.Calls.All(call => closedCalls.Contains(call.CallId) + || LoopObservations.Any(observation => observation.CallId == call.CallId))) + return this; + var runs = ChildRuns; + foreach (var (id, run) in runs) + { + if (run.ParentCheckpoint.TaskId == admission.TaskId && run.StartKey is ChildRunStartKey.Tool tool + && admission.Calls.Any(call => call.CallId == tool.CallId.Value)) + runs = runs.SetItem(id, run with { StartBatchSettled = true }); + } + // Abandonment closes unknown work. It creates no receipt or detector observation. + return this with { ChildRuns = runs }; + } + + public SessionState ApplyChildApproval(ToolApprovalRequested request) + { + if (request.SourceChildRunId is not { } runId || !ChildRuns.TryGetValue(runId, out var run) + || run.Terminal is not null || run.CancellationRequestedAtMs is not null || run.DispatchClosedAtMs is not null + || run.Approvals.Any(approval => approval.Request.CallId == request.CallId)) + throw new InvalidDataException("A child approval has no open accepted run."); + var approval = new ChildRunApproval(request, null); + run.ValidateApproval(approval); + return this with { ChildRuns = ChildRuns.SetItem(runId, run with { Approvals = [..run.Approvals, approval] }) }; + } + + public SessionState ApplyChildApproval(ToolApprovalResolved resolution) + { + if (resolution.SourceChildRunId is not { } runId || !ChildRuns.TryGetValue(runId, out var run)) + throw new InvalidDataException("A child approval resolution has no accepted run."); + var matches = run.Approvals.Where(approval => approval.Request.CallId == resolution.CallId).ToArray(); + if (matches is not [var pending] || pending.Resolution is not null) + throw new InvalidDataException("A child approval resolution has no pending request."); + var resolved = pending with { Resolution = resolution }; + run.ValidateApproval(resolved); + _ = Netclaw.Actors.Authorization.Consent.ConsentAnswerCodec.FromJournalText(resolution.Decision); + return this with { ChildRuns = ChildRuns.SetItem(runId, run with + { Approvals = run.Approvals.Select(approval => approval == pending ? resolved : approval).ToArray() }) }; + } + + public SessionState Apply(ChildRunEvent evt) + { + if (!ChildRuns.TryGetValue(evt.RunId, out var run) || run.StartKey.SessionId != evt.SessionId) + throw new InvalidDataException("A child lifecycle fact has no accepted session owner."); + var next = evt switch + { + ChildRunEvent.Started when run.StartedAtMs is null && run.Terminal is null && run.CancellationRequestedAtMs is null => + run with { StartedAtMs = evt.RecordedAtMs }, + ChildRunEvent.Checkpointed checkpoint when run.Terminal is null && run.DispatchClosedAtMs is null + && checkpoint.Checkpoint.CompletedRound > (run.ChildCheckpoint?.CompletedRound ?? 0) => + run with { ChildCheckpoint = checkpoint.Checkpoint }, + ChildRunEvent.CancellationRequested when run.Terminal is null && run.CancellationRequestedAtMs is null => + run with { CancellationRequestedAtMs = evt.RecordedAtMs }, + ChildRunEvent.DispatchClosed when run.CancellationRequestedAtMs is not null && run.DispatchClosedAtMs is null && run.Terminal is null => + run with { DispatchClosedAtMs = evt.RecordedAtMs }, + ChildRunEvent.TerminalRecorded terminal when run.Terminal is null && terminal.TerminalSequenceNr > 0 + && !ChildRuns.Values.Any(other => other.TerminalSequenceNr == terminal.TerminalSequenceNr) + && (run.CancellationRequestedAtMs is null || run.DispatchClosedAtMs is not null) => + run with { Terminal = terminal.Terminal, TerminalSequenceNr = terminal.TerminalSequenceNr }, + ChildRunEvent.ResultPrepared prepared when run.Terminal is not null && run.PreparedTerminal is null + && prepared.TerminalSequenceNr == run.TerminalSequenceNr + && SameTerminalOutcome(run.Terminal, prepared.Terminal) => + run with { PreparedTerminal = prepared.Terminal }, + ChildRunEvent.DeliveryAdmitted delivery when run.PreparedTerminal is not null && run.StartBatchSettled && run.DeliveryInputId is null => + ValidateChildDelivery(run, delivery.Input), + _ => throw new InvalidDataException("A child lifecycle fact violates its committed order.") + }; + next.Validate(); + var state = this with { ChildRuns = ChildRuns.SetItem(run.RunId, next) }; + return evt is ChildRunEvent.DeliveryAdmitted admitted ? state.AdmitInputCore(admitted.Input) : state; + } + + private SessionState AdoptChildContinuation(ToolTaskAdopted evt) + { + if (evt.ContinuedJobKey is not null || evt.InputIds.Count == 0 || evt.InputIds.Distinct().Count() != evt.InputIds.Count + || !PendingInputs.Take(evt.InputIds.Count).Select(static input => input.InputId).SequenceEqual(evt.InputIds)) + throw new InvalidDataException("A child continuation does not name the next canonical input prefix."); + var prefix = PendingInputs.Take(evt.InputIds.Count).ToArray(); + var run = GetChildContinuation(prefix[^1]); + if (evt.ContinuedChildRunId != run.RunId || !SameCanonicalContext(evt.TurnContext, run.OriginalContext)) + throw new InvalidDataException("A child continuation differs from its original authority."); + foreach (var input in prefix) + { + var sibling = GetChildContinuation(input); + if (!SameCanonicalContext(sibling.OriginalContext, run.OriginalContext) + || !BackgroundChildRun.SameCheckpoint(sibling.ParentCheckpoint, run.ParentCheckpoint) + || sibling.ParentReceiptFailure != run.ParentReceiptFailure) + throw new InvalidDataException("A child prefix merges distinct parent tasks or detector evidence."); + } + if (ParkedToolBatchHistory.FindRedrivableAssistantMessage(History, null) is not null) + throw new InvalidDataException("A child continuation requires canonical closure of the prior tool batch."); + var history = History; + foreach (var input in prefix) + { + var sibling = GetChildContinuation(input); + var callId = input.UserMessage.ToolCallId!.Value; + var canonicalCall = ChildRunDelivery.Call(sibling, callId); + var existing = history.Where(message => message.ToolCalls.Any(call => call.CallId == callId)).ToArray(); + if (existing.Length == 0) + { + if (history.Any(message => message.Role == ChatRole.Tool && message.ToolCallId == callId)) + throw new InvalidDataException("A child continuation has an orphan result correlation."); + history = history.Add(canonicalCall).Add(input.UserMessage); + continue; + } + if (existing is not [var assistant] || assistant.Role != canonicalCall.Role + || assistant.Content != canonicalCall.Content || assistant.Name != canonicalCall.Name + || assistant.ToolCallId != canonicalCall.ToolCallId || assistant.MediaReferences.Count != 0 + || !assistant.ToolCalls.SequenceEqual(canonicalCall.ToolCalls)) + throw new InvalidDataException("A repeated child continuation differs from its canonical call."); + var results = ParkedToolBatchHistory.FindToolResultsFor(history, assistant); + if (results is not [var result] || result.Role != input.UserMessage.Role + || result.ToolCallId != callId || result.Name != input.UserMessage.Name + || result.Content != input.UserMessage.Content || result.ToolCalls.Count != 0 || result.MediaReferences.Count != 0) + throw new InvalidDataException("A repeated child continuation differs from its canonical result."); + } + return this with + { + History = history, + AdoptedTaskContext = run.OriginalContext, AdoptedTaskInputIds = evt.InputIds, + LoopCheckpoint = run.ParentCheckpoint, LoopReceiptFailure = run.ParentReceiptFailure, + LoopAdmission = null, LoopObservations = [] + }; + } + + private static bool SameTerminalOutcome(ChildRunTerminal original, ChildRunTerminal prepared) + => original.Lost == prepared.Lost && original.Result.Output == prepared.Result.Output + && original.Result.Outcome == prepared.Result.Outcome && original.Result.OutcomeReason == prepared.Result.OutcomeReason + && original.Result.RunId == prepared.Result.RunId && original.Result.ScopeId == prepared.Result.ScopeId + && original.Result.AgentName == prepared.Result.AgentName; + + private BackgroundChildRun ValidateChildDelivery(BackgroundChildRun run, InputAdmitted input) + { + ValidateChildDeliveryContent(run, input); + var callId = input.UserMessage.ToolCallId!.Value; + if (History.Any(message => message.ToolCalls.Any(call => call.CallId == callId)) + || PendingInputs.Any(pending => pending.InputId == input.InputId)) + throw new InvalidDataException("A child delivery repeats a committed input or provider correlation."); + return run with { DeliveryInputId = input.InputId }; + } + + private static void ValidateChildDeliveryContent(BackgroundChildRun run, InputAdmitted input) + { + if (input.SessionId != run.StartKey.SessionId || input.SourceChildRunId != run.RunId + || input.SourceBackgroundJobId is not null || input.BackgroundJobLineageVersion != 0 || input.BackgroundJobOrigin is not null + || !SameCanonicalContext(input.TurnContext, run.OriginalContext) + || input.InputId.Value != $"child-result-{run.RunId.Value}" + || input.UserMessage.Role != ChatRole.Tool || input.UserMessage.Name != ChildRunDelivery.ToolName(run) + || input.UserMessage.Content != ChildRunDelivery.Body(run) || input.UserMessage.ToolCallId is not { } callId + || string.IsNullOrWhiteSpace(callId.Value) + || run.StartKey is ChildRunStartKey.Tool tool && callId == tool.CallId + || input.UserMessage.ToolCalls.Count != 0 || input.UserMessage.MediaReferences.Count != 0 + || input.ExecutableText is not null) + throw new InvalidDataException("A child delivery differs from its canonical result, origin, or authority."); + } + + internal BackgroundChildRun GetChildContinuation(InputAdmitted input) + { + if (input.SourceBackgroundJobId is not null || input.SourceChildRunId is not { } id + || !ChildRuns.TryGetValue(id, out var run) || run.DeliveryInputId != input.InputId + || run.PreparedTerminal is null || !run.StartBatchSettled + || run.StartKey.SessionId != input.SessionId || !SameCanonicalContext(run.OriginalContext, input.TurnContext)) + throw new InvalidDataException("A child continuation lacks canonical committed delivery evidence."); + ValidateChildDeliveryContent(run, input); + return run; + } +} diff --git a/src/Netclaw.Actors/Sessions/SessionState.cs b/src/Netclaw.Actors/Sessions/SessionState.cs index ff2f7a619..44d4e803c 100644 --- a/src/Netclaw.Actors/Sessions/SessionState.cs +++ b/src/Netclaw.Actors/Sessions/SessionState.cs @@ -9,6 +9,7 @@ using Netclaw.Actors.Protocol; using Netclaw.Actors.Sessions.Handlers; using Netclaw.Actors.Reminders; +using Netclaw.Actors.SubAgents; using static Netclaw.Actors.Sessions.SessionProtocol; namespace Netclaw.Actors.Sessions; @@ -22,7 +23,7 @@ namespace Netclaw.Actors.Sessions; /// event via the Apply methods. Transient concerns (subscribers, message /// buffer, behavior) remain on the actor. /// -public sealed record SessionState +public sealed partial record SessionState { public sealed record AdoptedContextAuditRecord( string AuthorizedMessageId, @@ -59,6 +60,8 @@ public sealed record AdoptedContextAuditMessage( public TurnContextRecord? AdoptedTaskContext { get; init; } public IReadOnlyList AdoptedTaskInputIds { get; init; } = []; + public ImmutableDictionary ChildRuns { get; init; } = []; + public int TurnCount { get; init; } public string? Title { get; init; } @@ -106,12 +109,37 @@ public sealed record AdoptedContextAuditMessage( // ── Event application (pure functions) ── + public SessionState Apply(ChildRunAccepted evt) + { + var run = evt.Run; + run.Validate(); + if (run.State != BackgroundChildState.Accepted || run.ChildCheckpoint is not null || run.DeliveryInputId is not null) + throw new InvalidDataException("A child acceptance contains facts from a later lifecycle stage."); + if (evt.SessionId != run.StartKey.SessionId || AdoptedTaskContext is null + || !SameCanonicalContext(AdoptedTaskContext, run.OriginalContext) + || !AdoptedTaskInputIds.SequenceEqual(run.OriginInputIds) + || !BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, LoopCheckpoint) + || run.ParentReceiptFailure != LoopReceiptFailure) + throw new InvalidDataException("A child acceptance differs from the canonical parent task."); + if (ChildRuns.ContainsKey(run.RunId) || ChildRuns.Values.Any(existing => existing.StartKey == run.StartKey)) + throw new InvalidDataException("A child acceptance repeats an already committed start identity."); + if (run.StartKey is ChildRunStartKey.Tool tool + && (run.StartBatchSettled || LoopAdmission is null + || !LoopAdmission.Calls.Any(call => call.CallId == tool.CallId.Value && call.ToolName == run.SourceOperation))) + throw new InvalidDataException("A child acceptance does not belong to its admitted parent tool call."); + return this with { ChildRuns = ChildRuns.Add(run.RunId, run) }; + } + public SessionState Apply(ToolTaskAdopted evt) { if (!TurnContext.TryFromRecord(evt.TurnContext, out var context, out var reason) || context is null) throw new InvalidDataException($"An adopted task has invalid authority: {reason}"); if (evt.SessionId != context.SessionId) throw new InvalidDataException("An adopted task has a different session identity."); + if (evt.ContinuedChildRunId is not null) + return AdoptChildContinuation(evt); + if (PendingInputs.Take(evt.InputIds.Count).Any(static input => input.SourceChildRunId is not null)) + throw new InvalidDataException("A child delivery requires its explicit continuation identity."); if (evt.ContinuedJobKey is not null && (evt.ContinuedJobKey != context.TurnId.Value || evt.TurnContext.SourceKind != BackgroundJobManagerActor.SourceKind)) throw new InvalidDataException("A continued job key differs from its canonical authority context."); @@ -277,7 +305,24 @@ private SessionState RefreshJobEvidence() if (job.LineageVersion == 1 && job.Origin?.TurnId.Value == LoopCheckpoint.TaskId) jobs = jobs.SetItem(key, job with { OriginCheckpoint = LoopCheckpoint, OriginReceiptFailure = LoopReceiptFailure }); } - return this with { ActiveBackgroundJobs = jobs }; + var runs = ChildRuns; + foreach (var (id, run) in runs) + { + if (run.ParentCheckpoint.TaskId != LoopCheckpoint.TaskId) + continue; + var settled = run.StartBatchSettled + || run.StartKey is ChildRunStartKey.Tool tool + && LoopAdmission is { } admission + && admission.Calls.Any(call => call.CallId == tool.CallId.Value) + && admission.Calls.All(call => LoopObservations.Any(observation => observation.CallId == call.CallId)); + runs = runs.SetItem(id, run with + { + ParentCheckpoint = LoopCheckpoint, + ParentReceiptFailure = LoopReceiptFailure, + StartBatchSettled = settled + }); + } + return this with { ActiveBackgroundJobs = jobs, ChildRuns = runs }; } internal static bool SameCanonicalContext(TurnContextRecord left, TurnContextRecord right) @@ -316,6 +361,13 @@ public SessionState Apply(InputClosed evt) } public SessionState Apply(InputAdmitted evt) + { + if (evt.SourceChildRunId is not null) + throw new InvalidDataException("A child input requires atomic delivery admission."); + return AdmitInputCore(evt); + } + + private SessionState AdmitInputCore(InputAdmitted evt) { var sourceKey = SourceMessageKey(evt); var keys = sourceKey is null || RecentSourceMessageKeys.Contains(sourceKey) @@ -687,6 +739,7 @@ public SessionSnapshot ToSnapshot() LoopCheckpoint = LoopCheckpoint, LoopAdmission = LoopAdmission, LoopObservations = LoopObservations.ToArray(), LoopReceiptFailure = LoopReceiptFailure, AdoptedTaskContext = AdoptedTaskContext, AdoptedTaskInputIds = AdoptedTaskInputIds, + ChildRuns = ChildRuns.Values.OrderBy(static run => run.AcceptedAtMs).ThenBy(static run => run.RunId.Value, StringComparer.Ordinal).ToArray(), ProcessedBackgroundJobIds = ProcessedBackgroundJobIds.ToArray(), PendingInputs = PendingInputs.ToArray(), RecentSourceMessageKeys = RecentSourceMessageKeys.ToArray(), @@ -721,6 +774,10 @@ public SessionSnapshot ToSnapshot() public static SessionState FromSnapshot(SessionSnapshot snapshot) { + foreach (var run in snapshot.ChildRuns) + run.Validate(); + if (snapshot.ChildRuns.Select(static run => run.StartKey).Distinct().Count() != snapshot.ChildRuns.Count) + throw new InvalidDataException("A child snapshot repeats an accepted start identity."); var activeJobs = snapshot.ActiveBackgroundJobs.Count > 0 ? snapshot.ActiveBackgroundJobs.ToImmutableDictionary( j => $"{Jobs.BackgroundJobManagerActor.JobDeliveryKeyPrefix}{j.JobId}", j => j) @@ -747,7 +804,11 @@ [.. record.Messages message.AuthorityAtInclusion))])) : []; - return new SessionState + var terminalSequences = snapshot.ChildRuns.Where(static run => run.TerminalSequenceNr is not null) + .Select(static run => run.TerminalSequenceNr!.Value).ToArray(); + if (terminalSequences.Distinct().Count() != terminalSequences.Length) + throw new InvalidDataException("A child snapshot repeats a terminal journal sequence."); + var state = new SessionState { History = ImmutableList.CreateRange(snapshot.History), LoopCheckpoint = snapshot.LoopCheckpoint ?? new ToolLoopCheckpoint(), @@ -755,6 +816,7 @@ [.. record.Messages LoopObservations = ImmutableList.CreateRange(snapshot.LoopObservations), LoopReceiptFailure = snapshot.LoopReceiptFailure, AdoptedTaskContext = snapshot.AdoptedTaskContext, AdoptedTaskInputIds = snapshot.AdoptedTaskInputIds, + ChildRuns = snapshot.ChildRuns.ToImmutableDictionary(static run => run.RunId), ProcessedBackgroundJobIds = ImmutableHashSet.CreateRange(snapshot.ProcessedBackgroundJobIds), PendingInputs = ImmutableList.CreateRange(snapshot.PendingInputs), RecentSourceMessageKeys = ImmutableList.CreateRange(snapshot.RecentSourceMessageKeys), @@ -764,5 +826,12 @@ [.. record.Messages ActiveBackgroundJobs = activeJobs, AdoptedContextRecords = adoptedContextRecords }; + foreach (var input in state.PendingInputs.Where(static input => input.SourceChildRunId is not null)) + _ = state.GetChildContinuation(input); + if (state.PendingInputs.Where(static input => input.SourceChildRunId is not null) + .Select(static input => input.InputId).Distinct().Count() + != state.PendingInputs.Count(static input => input.SourceChildRunId is not null)) + throw new InvalidDataException("A child snapshot repeats a pending delivery input."); + return state; } } diff --git a/src/Netclaw.Actors/Sessions/ToolApprovalState.cs b/src/Netclaw.Actors/Sessions/ToolApprovalState.cs index 134c7c18a..41216a352 100644 --- a/src/Netclaw.Actors/Sessions/ToolApprovalState.cs +++ b/src/Netclaw.Actors/Sessions/ToolApprovalState.cs @@ -51,6 +51,7 @@ internal sealed record ToolInteractionRequestDispatch( bool PersistApprovalState) : INoSerializationVerificationNeeded { internal string? ManagedTemporaryDirectory { get; init; } + internal ToolCallId? OriginalChildCallId { get; init; } } internal enum ApprovalTurnPhase diff --git a/src/Netclaw.Actors/SubAgents/BackgroundChildRun.cs b/src/Netclaw.Actors/SubAgents/BackgroundChildRun.cs new file mode 100644 index 000000000..7847d6ee8 --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/BackgroundChildRun.cs @@ -0,0 +1,159 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Channels; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.SubAgents; + +/// Framework correlation from the original admitted input and operation. +public abstract record ChildRunStartKey +{ + public required SessionId SessionId { get; init; } + public required TurnId TurnId { get; init; } + + public sealed record Tool(ToolCallId CallId) : ChildRunStartKey; + + // A direct activation has one framework slot. The model cannot choose another slot. + public sealed record Slash(InputId InputId) : ChildRunStartKey; +} + +/// Durable acceptance facts. Runtime references and child execution state remain outside this record. +public sealed record BackgroundChildRun +{ + public required SubAgentRunId RunId { get; init; } + public required SubAgentScopeId ScopeId { get; init; } + public required ChildRunStartKey StartKey { get; init; } + public required AgentName AgentName { get; init; } + public required string SourceOperation { get; init; } + public required string ArgumentsDigest { get; init; } + public required TurnContextRecord OriginalContext { get; init; } + public required IReadOnlyList OriginInputIds { get; init; } + public required WorkingContextSnapshot InitialWorkingSnapshot { get; init; } + public required ToolLoopCheckpoint ParentCheckpoint { get; init; } + public bool ParentReceiptFailure { get; init; } + public bool StartBatchSettled { get; init; } + public long AcceptedAtMs { get; init; } + public long? StartedAtMs { get; init; } + public long? CancellationRequestedAtMs { get; init; } + public long? DispatchClosedAtMs { get; init; } + public ChildRunCheckpoint? ChildCheckpoint { get; init; } + public ChildRunTerminal? Terminal { get; init; } + public long? TerminalSequenceNr { get; init; } + public ChildRunTerminal? PreparedTerminal { get; init; } + public InputId? DeliveryInputId { get; init; } + public IReadOnlyList Approvals { get; init; } = []; + + public BackgroundChildState State => Terminal switch + { + { Lost: true } => BackgroundChildState.Lost, + { Result.Completion: ChildRunCompletion.Cancelled } => BackgroundChildState.Cancelled, + { Result.Completion: ChildRunCompletion.Completed } => BackgroundChildState.Completed, + { Result.Completion: ChildRunCompletion.Partial } => BackgroundChildState.Partial, + not null => BackgroundChildState.Failed, + _ when CancellationRequestedAtMs is not null => BackgroundChildState.Cancelling, + _ when StartedAtMs is not null => BackgroundChildState.Running, + _ => BackgroundChildState.Accepted + }; + + internal static bool SameCheckpoint(ToolLoopCheckpoint left, ToolLoopCheckpoint right) + => left.TaskId == right.TaskId && left.LastBlockedAction == right.LastBlockedAction + && left.Entries.SequenceEqual(right.Entries) && left.ColdKeys.SequenceEqual(right.ColdKeys) + && left.AdjacentHistory.SequenceEqual(right.AdjacentHistory); + + public void Validate() + { + if (!TurnContext.TryFromRecord(OriginalContext, out var context, out var reason) || context is null) + throw new InvalidDataException($"A child acceptance has invalid authority: {reason}"); + if (StartKey.SessionId != context.SessionId || StartKey.TurnId != context.TurnId) + throw new InvalidDataException("A child start key differs from its original authority."); + if (string.IsNullOrWhiteSpace(RunId.Value) || string.IsNullOrWhiteSpace(AgentName.Value) + || ScopeId.Value != $"{StartKey.SessionId.Value}/subagent/{AgentName.Value}/{RunId.Value}") + throw new InvalidDataException("A child acceptance has invalid owner identifiers."); + if (ArgumentsDigest.Length != 64 || ArgumentsDigest.Any(static character => !char.IsAsciiHexDigit(character)) + || string.IsNullOrWhiteSpace(SourceOperation)) + throw new InvalidDataException("A child acceptance has invalid canonical argument evidence."); + if (OriginInputIds.Count == 0 || OriginInputIds.Any(static id => string.IsNullOrWhiteSpace(id.Value)) + || OriginInputIds.Distinct().Count() != OriginInputIds.Count) + throw new InvalidDataException("A child acceptance has invalid original input provenance."); + if (string.IsNullOrWhiteSpace(ParentCheckpoint.TaskId)) + throw new InvalidDataException("A child acceptance has no parent detector task."); + if (DispatchClosedAtMs is not null && CancellationRequestedAtMs is null + || PreparedTerminal is not null && Terminal is null + || DeliveryInputId is not null && PreparedTerminal is null) + throw new InvalidDataException("A child ledger has invalid lifecycle order."); + if ((Terminal is null) != (TerminalSequenceNr is null) || TerminalSequenceNr is <= 0) + throw new InvalidDataException("A child terminal has no valid journal sequence."); + if (Terminal is { } terminal) + ValidateTerminal(terminal); + if (PreparedTerminal is { } prepared) + ValidateTerminal(prepared); + if (Terminal is not null && Approvals.Any(static approval => approval.Resolution is null)) + throw new InvalidDataException("A terminal child retains an unresolved approval prompt."); + if (Approvals.Select(static approval => approval.Request.CallId).Distinct(StringComparer.Ordinal).Count() != Approvals.Count) + throw new InvalidDataException("A child run has duplicate approval prompts."); + foreach (var approval in Approvals) + ValidateApproval(approval); + switch (StartKey) + { + case ChildRunStartKey.Tool tool when !string.IsNullOrWhiteSpace(tool.CallId.Value): + break; + case ChildRunStartKey.Slash slash when OriginInputIds.Contains(slash.InputId) && StartBatchSettled: + break; + default: + throw new InvalidDataException("A child acceptance has invalid activation correlation."); + } + } + + internal void ValidateApproval(ChildRunApproval approval) + { + var request = approval.Request; + if (request.SourceChildRunId != RunId || request.SessionId != StartKey.SessionId + || request.TurnContext is null || !SessionState.SameCanonicalContext(OriginalContext, request.TurnContext) + || request.OriginalChildCallId is not { } childCall || string.IsNullOrWhiteSpace(childCall.Value) + || !request.CallId.StartsWith($"{RunId.Value}/subagent-approval/", StringComparison.Ordinal) + || !AuthorizationAttemptId.TryParse(request.AuthorizationAttemptId, out _) + || request.Audience != OriginalContext.Audience || request.Boundary != OriginalContext.Boundary + || request.ChannelType != OriginalContext.ChannelType + || request.RequesterSenderId != OriginalContext.RequesterSenderId + || request.SupportsInteractiveApproval != OriginalContext.SupportsInteractiveApproval + || request.RequesterPrincipal != OriginalContext.RequesterPrincipal + || request.HasThirdPartyAdoptedContext != OriginalContext.HasThirdPartyAdoptedContext + || !request.AdoptedSpeakerIds.SequenceEqual(OriginalContext.AdoptedSpeakerIds, StringComparer.Ordinal) + || request.OptionKeys.Count == 0 || string.IsNullOrWhiteSpace(request.ToolName)) + throw new InvalidDataException("A child approval differs from its accepted authority or call."); + if (approval.Resolution is { } resolution + && (resolution.SourceChildRunId != RunId || resolution.SessionId != request.SessionId + || resolution.CallId != request.CallId || resolution.AuthorizationAttemptId != request.AuthorizationAttemptId)) + throw new InvalidDataException("A child approval resolution differs from its request."); + } + + private void ValidateTerminal(ChildRunTerminal terminal) + { + if (terminal.Result.RunId != RunId || terminal.Result.ScopeId != ScopeId + || terminal.Result.AgentName != AgentName + || terminal.Lost && terminal.Result.Completion is not ChildRunCompletion.Failed + || CancellationRequestedAtMs is not null && terminal.Result.Completion is not ChildRunCompletion.Cancelled) + throw new InvalidDataException("A child terminal differs from its accepted run or durable cancellation."); + } +} + +public enum BackgroundChildState +{ + Accepted, Running, Cancelling, Completed, Partial, Failed, Cancelled, Lost +} + +/// Evidence from a completed child tool round. It grants no authority. +public sealed record ChildRunCheckpoint(int CompletedRound, string Summary, WorkingContextDelta ConfirmedActivity); + +/// A recorded terminal result. Enrichment cannot replace its outcome. +public sealed record ChildRunTerminal(SubAgentResult Result, bool Lost, string? EvidenceWarning); + +/// A run owns its approval request after the parent turn ends. +public sealed record ChildRunApproval(ToolApprovalRequested Request, ToolApprovalResolved? Resolution); diff --git a/src/Netclaw.Actors/SubAgents/CheckAgentRunTool.cs b/src/Netclaw.Actors/SubAgents/CheckAgentRunTool.cs new file mode 100644 index 000000000..b321b298f --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/CheckAgentRunTool.cs @@ -0,0 +1,55 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.ComponentModel; +using System.Text.Json; +using Netclaw.Tools; + +namespace Netclaw.Actors.SubAgents; + +[NetclawTool("check_agent_run", + "Read an accepted child run and its authorized log and artifact paths, or request cancellation. " + + "Cancellation admission and dispatch closure are separate facts.", + Grant = "builtin")] +public sealed partial class CheckAgentRunTool : NetclawTool +{ + public const string ToolName = "check_agent_run"; + + public record Params( + [property: Description("The run_id from an accepted child start.")] string RunId, + [property: Description("Set true to request cancellation.")] bool Cancel = false); + + protected override async Task ExecuteAsync(Params args, ToolInvocationContext context, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(args.RunId)) + { + context.Outputs.TryComplete(new ToolInvocationReceipt.OtherOutcome(ToolInvocationOutcomeCategory.InvalidInput)); + return "Error: run_id is required."; + } + var owner = context.SpawnChildActor + ?? throw new InvalidOperationException("Child control requires the session owner adapter."); + var response = await owner(new ChildControlRequest(new SubAgentRunId(args.RunId), args.Cancel), ToolName, ct); + if (response is not ChildControlReply reply) + throw new InvalidDataException("The child control adapter returned an invalid response."); + if (reply.Run is not { } run) + { + context.Outputs.TryComplete(new ToolInvocationReceipt.OtherOutcome(ToolInvocationOutcomeCategory.InvalidInput)); + return "Error: the child run was not found or is not accessible from this session."; + } + var parentStorage = context.SessionStorage + ?? throw new InvalidOperationException("Child status requires resolved session storage."); + var storage = parentStorage.ForChild(run.RunId, run.ScopeId); + context.Outputs.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + return JsonSerializer.Serialize(new + { + run_id = run.RunId.Value, scope_id = run.ScopeId.Value, agent = run.AgentName.Value, + log_path = storage.LogPath.Value, artifact_directory = storage.ArtifactDirectory.Value, + state = run.State.ToString(), cancellation_requested = run.CancellationRequestedAtMs is not null, + dispatch_closed = run.DispatchClosedAtMs is not null, + checkpoint = run.ChildCheckpoint, + terminal = run.Terminal is null ? null : ChildRunDelivery.TerminalBody(run, run.PreparedTerminal ?? run.Terminal) + }); + } +} diff --git a/src/Netclaw.Actors/SubAgents/ChildRunDelivery.cs b/src/Netclaw.Actors/SubAgents/ChildRunDelivery.cs new file mode 100644 index 000000000..4a573d1b9 --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/ChildRunDelivery.cs @@ -0,0 +1,47 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Netclaw.Actors.Protocol; +using Netclaw.Tools; + +namespace Netclaw.Actors.SubAgents; + +/// Materializes recorded child data without another task dispatch or authorization receipt. +internal static class ChildRunDelivery +{ + internal static string ToolName(BackgroundChildRun run) + => run.StartKey is ChildRunStartKey.Slash ? "skill_load" : run.SourceOperation; + + internal static string Body(BackgroundChildRun run) + { + var terminal = run.PreparedTerminal ?? throw new InvalidDataException("A child result is not ready for delivery."); + return TerminalBody(run, terminal); + } + + internal static string TerminalBody(BackgroundChildRun run, ChildRunTerminal terminal) + { + var result = terminal.Result; + return JsonSerializer.Serialize(new + { + run_id = run.RunId.Value, scope_id = run.ScopeId.Value, state = run.State.ToString(), + source_operation = run.SourceOperation, outcome = result.Outcome.ToString(), + reason = result.OutcomeReason?.Value, output = result.Output, + working_context = result.WorkingContext, findings = result.Findings, + log_path = result.LogPath, artifact_directory = result.ArtifactDirectory, + warning = terminal.EvidenceWarning, checkpoint = run.ChildCheckpoint + }); + } + + internal static SerializableChatMessage Call(BackgroundChildRun run, ToolCallId callId) => new() + { + Role = ChatRole.Assistant, + ToolCalls = [new SerializableToolCall + { + CallId = callId, Name = new ToolName(ToolName(run)), + ArgumentsJson = JsonSerializer.Serialize(new { run_id = run.RunId.Value, source_operation = run.SourceOperation }) + }] + }; +} diff --git a/src/Netclaw.Actors/SubAgents/ChildRunOperations.cs b/src/Netclaw.Actors/SubAgents/ChildRunOperations.cs new file mode 100644 index 000000000..9ca9f8db4 --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/ChildRunOperations.cs @@ -0,0 +1,128 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Event; +using Netclaw.Actors.Sessions; +using Netclaw.Tools; +using static Netclaw.Actors.Sessions.SessionProtocol; +using static Netclaw.Actors.SubAgents.SubAgentProtocol; + +namespace Netclaw.Actors.SubAgents; + +/// Live preparation stays inside the existing owner adapter. +internal sealed record PreparedChildRun : INoSerializationVerificationNeeded +{ + public required Props Props { get; init; } + public required SubAgentRunId RunId { get; init; } + public required AgentName AgentName { get; init; } + public required int ToolCount { get; init; } + public required string ArgumentsDigest { get; init; } + public required RunSubAgent Execution { get; init; } +} + +/// The pipeline stamps these facts before asynchronous tool setup. +internal sealed record StartBackgroundChildRun( + PreparedChildRun Prepared, + ChildRunStartKey StartKey, + TurnContextRecord InvocationContext, + string SourceOperation, + CancellationToken ExecutionToken) : INoSerializationVerificationNeeded; + +internal sealed record RunBackgroundSubAgent(SubAgentRunId RunId, RunSubAgent Execution, ChildRunDispatch Dispatch) : INoSerializationVerificationNeeded; +internal sealed record BackgroundChildTerminal(SubAgentRunId RunId, SubAgentResult Result) : INoSerializationVerificationNeeded; +internal sealed record BackgroundChildTerminalAck(SubAgentRunId RunId) : INoSerializationVerificationNeeded; + +internal abstract record ChildStartReply : INoSerializationVerificationNeeded +{ + internal sealed record Accepted(BackgroundChildRun Run, BackgroundChildState State) : ChildStartReply + { + internal void Validate(PreparedChildRun prepared) + { + Run.Validate(); + // A duplicate can retain another run ID and a later state, but its preparation must match. + if (Run.ArgumentsDigest != prepared.ArgumentsDigest || Run.AgentName != prepared.AgentName || State != Run.State) + throw new InvalidDataException("The child acceptance differs from its canonical preparation or current state."); + } + + internal void Validate(StartBackgroundChildRun request) + { + Validate(request.Prepared); + if (Run.StartKey != request.StartKey + || !SessionState.SameCanonicalContext(Run.OriginalContext, request.InvocationContext) + || Run.SourceOperation != request.SourceOperation) + throw new InvalidDataException("The child acceptance differs from its original start request."); + } + } + internal sealed record Conflict : ChildStartReply; +} + +internal sealed record BackgroundChildCheckpoint(SubAgentRunId RunId, ChildRunCheckpoint Checkpoint) : INoSerializationVerificationNeeded; +internal sealed record BackgroundChildCheckpointAck(SubAgentRunId RunId, int CompletedRound) : INoSerializationVerificationNeeded; + +/// Closes local dispatch admission. It does not schedule provider capacity. +internal sealed class ChildRunDispatch +{ + private readonly object _sync = new(); + private bool _closed; + + public Task Enter(Func> operation) + { + lock (_sync) + { + if (Volatile.Read(ref _closed)) + throw new OperationCanceledException("The child dispatch admission is closed."); + return operation(); + } + } + + public Task Enter(Func operation) + { + lock (_sync) + { + if (Volatile.Read(ref _closed)) + throw new OperationCanceledException("The child dispatch admission is closed."); + return operation(); + } + } + + public void Close() + => Volatile.Write(ref _closed, true); + + public Task WaitForEnteredInvocationAsync() + { + // Only the synchronous invocation holds this lock, not its returned remote task. + return Task.Run(() => + { + lock (_sync) + { + if (!Volatile.Read(ref _closed)) + throw new InvalidOperationException("Child admission must close before its acknowledgement."); + } + }); + } + + internal static async Task CancelAndDisposeAsync(CancellationTokenSource source, ILoggingAdapter log) + { + try + { + // CancelAsync marks the token before arbitrary callbacks run outside the actor thread. + await source.CancelAsync().ConfigureAwait(false); + } + catch (Exception error) + { + log.Error(error, "Child cancellation callbacks failed."); + } + finally + { + source.Dispose(); + } + } +} + +internal sealed record ChildControlRequest(SubAgentRunId RunId, bool Cancel) : INoSerializationVerificationNeeded; +internal sealed record ControlBackgroundChildRun(ChildControlRequest Request, TurnContextRecord InvocationContext, + CancellationToken ExecutionToken) : INoSerializationVerificationNeeded; +internal sealed record ChildControlReply(BackgroundChildRun? Run) : INoSerializationVerificationNeeded; diff --git a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs index f030e8f29..a8d505217 100644 --- a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs +++ b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs @@ -4,27 +4,23 @@ // // ----------------------------------------------------------------------- using System.ComponentModel; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Channels; using Microsoft.Extensions.Logging; using Netclaw.Configuration; using Netclaw.Tools; -using static Netclaw.Actors.SubAgents.SubAgentProtocol; namespace Netclaw.Actors.SubAgents; /// /// User-facing tool that delegates a task to a named subagent. -/// The subagent runs autonomously with its own tools and returns a result. +/// The subagent runs in the background and delivers its result to the parent. /// Only user-facing agents (not internal platform agents) are invocable. /// [NetclawTool("spawn_agent", "Delegate a task to a specialist subagent. " - + "The subagent runs autonomously with its own tools and returns a result. " + + "The subagent runs in the background and delivers its result to the parent. " + "Use the discovery context layer to see available subagents.", Grant = "builtin", - Liveness = ToolLivenessMode.SelfMonitoring)] + Liveness = ToolLivenessMode.Opaque)] public sealed partial class SpawnAgentTool : NetclawTool { private readonly SubAgentDefinitionRegistry _registry; @@ -65,73 +61,7 @@ protected override async Task ExecuteAsync(Params args, ToolInvocationCo if (error is not null) return error; - var result = await _spawner.SpawnRunAsync(profile!, args.Task, args.Context, context, ct, systemPromptOverlay: null, activitySink: null); - return FormatResult(args.Agent, result); - } - - /// - /// Streaming entry point: the sub-agent's liveness/progress activity is surfaced - /// as the tool call's stream for visibility. The parent applies NO watchdog to - /// this call — spawn_agent is self-monitoring, so the sub-agent owns its own - /// liveness and the parent simply drains the stream to its terminal item (see - /// SessionToolExecutionPipeline). - /// - public override async IAsyncEnumerable ExecuteStreamAsync( - IDictionary? arguments, - ToolInvocationContext context, - [EnumeratorCancellation] CancellationToken ct = default) - { - SubAgentProfile? profile = null; - if (TryParse(arguments, out var failure, out var args)) - (failure, profile) = Resolve(args, context); - - if (failure is not null) - { - yield return new ToolCompletedUpdate(failure); - yield break; - } - - // The spawner writes the sub-agent's activity into this channel and - // completes it (even on failure) when the run ends. - var channel = Channel.CreateUnbounded(); - var spawnTask = _spawner.SpawnRunAsync( - profile!, args!.Task, args.Context, context, ct, systemPromptOverlay: null, activitySink: channel.Writer); - try - { - await foreach (var activity in channel.Reader.ReadAllAsync(ct)) - yield return activity; - - yield return new ToolCompletedUpdate(FormatResult(args.Agent, await spawnTask)); - } - finally - { - // Observe the spawn even when enumeration is abandoned (cancellation), - // so the sub-agent is stopped before this tool call returns. - await spawnTask; - } - } - - private static string FormatResult(string agent, EnrichedChildRunResult enriched) - { - var result = enriched.Response; - var builder = new StringBuilder(); - builder.AppendLine("Subagent run finished."); - builder.AppendLine($"Agent: {agent}"); - if (result.RunId is { } runId) - builder.AppendLine($"RunId: {runId.Value}"); - builder.AppendLine($"Outcome: {result.Outcome.ToString().ToLowerInvariant()}"); - if (result.OutcomeReason is { } reason) - builder.AppendLine($"Reason: {reason.Value}"); - if (enriched is EnrichedChildRunResult.SuccessfulRun success) - { - builder.AppendLine($"LogPath: {success.Locations.LogPath.Value}"); - builder.AppendLine($"ArtifactDirectory: {success.Locations.ArtifactDirectory.Value}"); - } - - builder.AppendLine(); - builder.AppendLine(result.Success ? "Summary:" : "Error:"); - builder.Append(result.Output); - return builder.ToString(); + return await _spawner.StartRunAsync(profile!, args.Task, args.Context, context, ct, systemPromptOverlay: null); } /// diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index 15d09d217..6f05605bd 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -107,6 +107,7 @@ [Subagent Execution Contract] private readonly ManagedTemporaryCorrectionState _managedTemporaryCorrections = new(); private long _llmCallId; private IActorRef _replyTo = ActorRefs.Nobody; + private SubAgentRunId? _backgroundRunId; private CancellationTokenSource? _executionCts; // Threaded into approval-bridge waits so a slow human approver does not @@ -325,11 +326,17 @@ public void Release(ActorBase actor) /// protected override void PostStop() { - _executionCts?.Cancel(); - _externalCts?.Cancel(); + _backgroundDispatch?.Close(); + if (_executionCts is { } execution) + _ = ChildRunDispatch.CancelAndDisposeAsync(execution, _log); + if (_externalCts is { } external) + _ = ChildRunDispatch.CancelAndDisposeAsync(external, _log); + _executionCts = null; + _externalCts = null; if (!_completed) { _completed = true; + _pendingCheckpoint = null; _replyTo.Tell(new SubAgentResult { Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.ActorStopped), @@ -339,8 +346,6 @@ protected override void PostStop() FindingsCount = 0 }); } - _executionCts?.Dispose(); - _externalCts?.Dispose(); _externalCancellationRegistration.Dispose(); _toolExposure.EvictAll(); _loadedDeferredToolNames.Clear(); @@ -349,88 +354,102 @@ protected override void PostStop() private void Idle() { - Receive(msg => + Receive(msg => BeginRun(msg, Sender)); + Receive(msg => { - _replyTo = Sender; - - _approvalBridge = msg.Scope.Authority.InteractiveApproval is InteractiveApprovalCapability.Available available - ? available.Bridge - : null; - var scopeId = msg.Scope.ScopeId.Value; - var subAgentAudience = msg.Scope.Authority.Audience; - _toolExecutionContext = new ToolExecutionContext( - msg.Scope.Authority, - ToolExecutionTimeout.Default); - _fileActivity = new ChildFileActivityTracker(msg.Scope.InitialWorkingSnapshot.WorkingContext); - var coreTools = ResolveCoreAiTools(); - _toolExposure.SeedBaseTools(coreTools); - _aiTools = _toolExposure.AvailableTools; - _coreToolCount = coreTools.Count; - _executionCts = new CancellationTokenSource(); - _externalCts = new CancellationTokenSource(); - var self = Self; // Capture before callback — Self requires active actor context - _externalCancellationRegistration = msg.Cancellation.Register(() => self.Tell(SubAgentCancelled.Instance)); - - // Enrich the logger so every sub-agent log line correlates back to the - // parent session (SessionId) and to this specific sub-agent run - // (SubSessionId), and is plainly attributable to the sub-agent. scopeId is - // "{parentSessionId}/subagent/{name}/{runId}"; NormalizeSessionId strips the - // "/subagent/..." suffix to recover the parent. SessionId matches the key the - // session/channel actors already tag their loggers with, so sub-agent and - // parent logs share one filterable attribute (so OTEL groups them under the parent); - // SubSessionId isolates a single run and is what the file-logger partitions on, so the - // sub-agent's lines land in its OWN session.log rather than the parent's. - var parentSessionId = SubAgentSessionScope.NormalizeSessionId(scopeId); - _parentSessionId = parentSessionId; - _subSessionId = scopeId; - var enrichedLog = Context.GetLogger(); - if (!string.IsNullOrWhiteSpace(parentSessionId)) - enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SessionId, parentSessionId); - if (!string.IsNullOrWhiteSpace(scopeId)) - enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SubSessionId, scopeId); - _log = enrichedLog; - - // The run is bounded by a two-phase inactivity watchdog re-armed on - // every progress event (LLM response, tool batch, streaming delta, - // keepalive) plus a keepalive-immune no-progress deadline. A sub-agent - // making real progress is never killed; a stalled one (no bytes) or a - // wedged one (keepalives but no tokens) is. FireLlmCall arms both. An - // unset prefill defaults to the session budget rather than collapsing - // to the inter-delta budget; an unset no-progress deadline is - // unbounded (only direct/test callers leave it unset). - _activitySink = msg.ActivitySink; - _interDeltaBudget = msg.Timeout; - _prefillBudget = msg.PrefillTimeout > TimeSpan.Zero ? msg.PrefillTimeout : DefaultPrefillBudget; - _noProgressBudget = msg.NoProgressTimeout > TimeSpan.Zero ? msg.NoProgressTimeout : null; - - // Build initial conversation: system prompt (from file, verbatim) + task as user message. - // If the caller supplied runtime context, prefix it onto the user message so the - // system prompt stays reproducible across invocations. - _history.Add(new AiChatMessage( - Microsoft.Extensions.AI.ChatRole.System, - BuildSystemPrompt( - _definition, - _projectInstructions, - CanDeclareProjectScope()))); - _history.Add(new AiChatMessage(Microsoft.Extensions.AI.ChatRole.User, - BuildUserMessage( - msg.RuntimeContext, - BuildModelContext( - msg.Scope.InitialWorkingSnapshot, - subAgentAudience, - ToolExecutionContext.SessionStorage), - msg.Task))); - - _log.Info("SubAgent [{AgentName}] starting (tools={ToolCount}, prefill={Prefill}, interDelta={InterDelta}, noProgress={NoProgress})", - _definition.Name, _aiTools.Count, _prefillBudget, _interDeltaBudget, - _noProgressBudget?.ToString() ?? "unbounded"); - LogToolExposure(); - - FireLlmCall(); - Become(Processing); + if (!Sender.Equals(Context.Parent)) + { + Sender.Tell(new Status.Failure(new InvalidOperationException("Only the session owner can start its background child."))); + return; + } + _backgroundRunId = msg.RunId; + _backgroundDispatch = msg.Dispatch; + BeginRun(msg.Execution, Sender); }); } + private void BeginRun(RunSubAgent msg, IActorRef replyTo) + { + _replyTo = replyTo; + + _approvalBridge = msg.Scope.Authority.InteractiveApproval is InteractiveApprovalCapability.Available available + ? available.Bridge + : null; + var scopeId = msg.Scope.ScopeId.Value; + var subAgentAudience = msg.Scope.Authority.Audience; + _toolExecutionContext = new ToolExecutionContext( + msg.Scope.Authority, + ToolExecutionTimeout.Default); + _fileActivity = new ChildFileActivityTracker(msg.Scope.InitialWorkingSnapshot.WorkingContext); + var coreTools = ResolveCoreAiTools(); + _toolExposure.SeedBaseTools(coreTools); + _aiTools = _toolExposure.AvailableTools; + _coreToolCount = coreTools.Count; + _executionCts = new CancellationTokenSource(); + _externalCts = new CancellationTokenSource(); + var self = Self; // Capture before callback — Self requires active actor context + _externalCancellationRegistration = msg.Cancellation.Register(() => self.Tell(SubAgentCancelled.Instance)); + + // Enrich the logger so every sub-agent log line correlates back to the + // parent session (SessionId) and to this specific sub-agent run + // (SubSessionId), and is plainly attributable to the sub-agent. scopeId is + // "{parentSessionId}/subagent/{name}/{runId}"; NormalizeSessionId strips the + // "/subagent/..." suffix to recover the parent. SessionId matches the key the + // session/channel actors already tag their loggers with, so sub-agent and + // parent logs share one filterable attribute (so OTEL groups them under the parent); + // SubSessionId isolates a single run and is what the file-logger partitions on, so the + // sub-agent's lines land in its OWN session.log rather than the parent's. + var parentSessionId = SubAgentSessionScope.NormalizeSessionId(scopeId); + _parentSessionId = parentSessionId; + _subSessionId = scopeId; + var enrichedLog = Context.GetLogger(); + if (!string.IsNullOrWhiteSpace(parentSessionId)) + enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SessionId, parentSessionId); + if (!string.IsNullOrWhiteSpace(scopeId)) + enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SubSessionId, scopeId); + _log = enrichedLog; + + // The run is bounded by a two-phase inactivity watchdog re-armed on + // every progress event (LLM response, tool batch, streaming delta, + // keepalive) plus a keepalive-immune no-progress deadline. A sub-agent + // making real progress is never killed; a stalled one (no bytes) or a + // wedged one (keepalives but no tokens) is. FireLlmCall arms both. An + // unset prefill defaults to the session budget rather than collapsing + // to the inter-delta budget; an unset no-progress deadline is + // unbounded (only direct/test callers leave it unset). + _activitySink = msg.ActivitySink; + _interDeltaBudget = msg.Timeout; + _prefillBudget = msg.PrefillTimeout > TimeSpan.Zero ? msg.PrefillTimeout : DefaultPrefillBudget; + _noProgressBudget = msg.NoProgressTimeout > TimeSpan.Zero ? msg.NoProgressTimeout : null; + + // Build initial conversation: system prompt (from file, verbatim) + task as user message. + // If the caller supplied runtime context, prefix it onto the user message so the + // system prompt stays reproducible across invocations. + _history.Add(new AiChatMessage( + Microsoft.Extensions.AI.ChatRole.System, + BuildSystemPrompt( + _definition, + _projectInstructions, + CanDeclareProjectScope()))); + _history.Add(new AiChatMessage(Microsoft.Extensions.AI.ChatRole.User, + BuildUserMessage( + msg.RuntimeContext, + BuildModelContext( + msg.Scope.InitialWorkingSnapshot, + subAgentAudience, + ToolExecutionContext.SessionStorage), + msg.Task))); + + _log.Info("SubAgent [{AgentName}] starting (tools={ToolCount}, prefill={Prefill}, interDelta={InterDelta}, noProgress={NoProgress})", + _definition.Name, _aiTools.Count, _prefillBudget, _interDeltaBudget, + _noProgressBudget?.ToString() ?? "unbounded"); + LogToolExposure(); + + FireLlmCall(); + Become(Processing); + } + + private void Processing() { Receive(msg => @@ -541,6 +560,15 @@ private void Processing() null); }); + Receive(ack => + { + if (ack.RunId != _backgroundRunId || !Sender.Equals(_replyTo) + || _pendingCheckpoint is not { } pending || pending.Round != ack.CompletedRound) + return; + _pendingCheckpoint = null; + pending.Continue(); + }); + Receive(msg => { _toolExecutionWatchdogState = ToolExecutionWatchdogState.None; @@ -601,17 +629,29 @@ private void Processing() AddModelInputMediaNudge(msg.ModelInputMediaReferences); - if (missingReceipt) + _turnState.RecordToolCompletion(msg.ToolResults.Count); + void ContinueRound() { - Complete(true, ToolCycleMessages.MissingReceipt, SubAgentRunOutcome.Partial, - new SubAgentOutcomeReason("tool_receipt_missing")); - return; + if (missingReceipt) + Complete(true, ToolCycleMessages.MissingReceipt, SubAgentRunOutcome.Partial, + new SubAgentOutcomeReason("tool_receipt_missing")); + else + FireLlmCall(); } - - _turnState.RecordToolCompletion(msg.ToolResults.Count); - _log.Debug("SubAgent [{AgentName}] tool iteration {Count}, continuing", - _definition.Name, _turnState.ToolIterationCount); - FireLlmCall(); + if (_backgroundRunId is { } runId) + { + var summary = string.Join("\n\n", _history.Where(message => message.Role == Microsoft.Extensions.AI.ChatRole.Tool) + .SelectMany(message => message.Contents.OfType()) + .Select(result => result.Result?.ToString() ?? string.Empty)); + var bound = ToolExecutionContext.MaxInlineToolResultChars; + if (summary.Length > bound) + summary = summary[^bound..]; + var checkpoint = new ChildRunCheckpoint(_turnState.ToolIterationCount, summary, _fileActivity.BuildResult()); + _pendingCheckpoint = (checkpoint.CompletedRound, ContinueRound); + _replyTo.Tell(new BackgroundChildCheckpoint(runId, checkpoint), Self); + } + else + ContinueRound(); }); Receive(msg => @@ -632,8 +672,6 @@ private void Processing() Receive(_ => { - _executionCts?.Cancel(); - _externalCts?.Cancel(); _log.Warning("SubAgent [{AgentName}] cancelled by parent", _definition.Name); Complete(false, "Subagent cancelled by parent", SubAgentRunOutcome.Failed, SubAgentOutcomeReason.CancelledByParent); }); @@ -655,7 +693,6 @@ private void Processing() // LLM stream and is never refreshed by keepalives, so reaching it // means a genuine wedge. var noProgress = _noProgressBudget?.TotalSeconds ?? 0; - _executionCts?.Cancel(); _watchdog.Stop(Timers); _log.Warning( "SubAgent [{AgentName}] timed out: no substantive output for {Budget:F0}s after {Iterations} tool iterations", @@ -686,7 +723,6 @@ private void Processing() // Report the budget that was actually in force: the prefill budget // while still waiting for the first token, else the inter-delta budget. var activeBudget = _anyContentStreamed ? _interDeltaBudget : _prefillBudget; - _executionCts?.Cancel(); _watchdog.Stop(Timers); _log.Warning( "SubAgent [{AgentName}] timed out: no activity for {Budget}s ({Phase}) after {Iterations} tool iterations", @@ -859,7 +895,8 @@ private void HandleToolCalls(AiChatMessage assistantMessage, List toolCalls) @@ -930,7 +967,21 @@ private void FireLlmCall(bool forceNoTools = false) messages.Count, options?.Tools?.Count > 0, forceNoTools); - _ = InvokeLlmAsync(client, messages, options, self, callId, _executionCts?.Token ?? CancellationToken.None); + if (_backgroundDispatch is null) + _ = InvokeLlmAsync(client, messages, options, self, callId, _executionCts?.Token ?? CancellationToken.None); + else + { + try + { + _ = _backgroundDispatch.Enter(() => InvokeLlmAsync(client, messages, options, self, callId, + _executionCts?.Token ?? CancellationToken.None)); + } + catch (OperationCanceledException) + { + // Cancellation finalization owns the terminal result after admission closes. + _log.Debug("Child provider dispatch refused after local closure."); + } + } } private IReadOnlyList ResolveCoreAiTools() @@ -985,6 +1036,9 @@ private bool CanDeclareProjectScope() => SetWorkingDirectoryTool.ToolName, StringComparison.Ordinal)); + private ChildRunDispatch? _backgroundDispatch; + private (int Round, Action Continue)? _pendingCheckpoint; + private bool _completed; private void Complete( @@ -1000,16 +1054,18 @@ private void Complete( // caller wins; subsequent calls are no-ops. if (_completed) return; _completed = true; + _backgroundDispatch?.Close(); + _pendingCheckpoint = null; _managedTemporaryCorrections.Clear(); _toolExecutionWatchdogState = ToolExecutionWatchdogState.None; _pendingApprovalWaits = 0; - _executionCts?.Cancel(); - _externalCts?.Cancel(); + if (_executionCts is { } execution) + _ = ChildRunDispatch.CancelAndDisposeAsync(execution, _log); + if (_externalCts is { } external) + _ = ChildRunDispatch.CancelAndDisposeAsync(external, _log); _watchdog.Stop(Timers); _externalCancellationRegistration.Dispose(); - _executionCts?.Dispose(); - _externalCts?.Dispose(); _executionCts = null; _externalCts = null; _pendingApprovalWaits = 0; @@ -1037,15 +1093,26 @@ private void Complete( : []; var workingContextResult = BuildWorkingContextResult(success); - _replyTo.Tell(new SubAgentResult + var result = new SubAgentResult { Completion = ChildRunCompletion.FromReportedOutcome(resolvedOutcome, outcomeReason, workingContextResult), Output = output, AgentName = _definition.Name, Findings = findings, FindingsCount = findings.Count, - }); - + }; + if (_backgroundRunId is { } runId) + { + result = result with { RunId = runId, ScopeId = new SubAgentScopeId(_subSessionId ?? throw new InvalidOperationException("A background child has no assigned scope.")) }; + _replyTo.Tell(new BackgroundChildTerminal(runId, result), Self); + Become(() => Receive(ack => + { + if (ack.RunId == runId && Sender.Equals(_replyTo)) + Context.Stop(Self); + })); + return; + } + _replyTo.Tell(result); Context.Stop(Self); } @@ -1431,7 +1498,8 @@ private static async Task ExecuteToolsAsync( AgentName agentName, string? parentSessionId, string? subSessionId, - IReadOnlyList refusedCalls) + IReadOnlyList refusedCalls, + ChildRunDispatch? dispatch) { try { @@ -1481,7 +1549,9 @@ executor is IApprovalShellProvider shellProvider } try { - var result = await executor.ExecuteAsync(tc, toolContext, ct); + var result = await (dispatch is null + ? executor.ExecuteAsync(tc, toolContext, ct) + : dispatch.Enter(() => executor.ExecuteAsync(tc, toolContext, ct))); return BuildToolResult( cleanedTc, result, @@ -1567,7 +1637,9 @@ consumedManagedTemporaryKey is { } defectConsumed retryContext.Approval.RestoreAuthorizationAttemptId( toolContext.Approval.AuthorizationAttemptId); retryContext.Approval.SeedOneTimeConsent(retryConsent); - var result = await executor.ExecuteAsync(tc, retryContext, ct); + var result = await (dispatch is null + ? executor.ExecuteAsync(tc, retryContext, ct) + : dispatch.Enter(() => executor.ExecuteAsync(tc, retryContext, ct))); return BuildToolResult( cleanedTc, ConsentAnswerCodec.AppendResultNote(result, step.Answer), diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index 797e699f3..f8b3d222e 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -3,9 +3,9 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using System.Diagnostics; -using System.Threading.Channels; -using Akka.Actor; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; using Microsoft.Extensions.Logging; using Netclaw.Actors.Tools; using Netclaw.Actors.Sessions; @@ -17,9 +17,8 @@ namespace Netclaw.Actors.SubAgents; /// -/// Shared utility that encapsulates the subagent spawn-ask-report lifecycle. -/// Resolves audience-scoped runtime tools, spawns a as a child of the -/// owning session actor, awaits results, and emits observability notifications. +/// Prepares a child run under the parent authority and requests durable owner acceptance. +/// The session owner controls the child lifetime and terminal delivery. /// Singleton — registered in DI. /// public sealed class SubAgentSpawner @@ -65,35 +64,15 @@ public SubAgentSpawner( _sessionMetrics = sessionMetrics; } - /// - /// Spawn a subagent as a child of the owning session, execute the task, and return the result. - /// The subagent is created via which is - /// wired by LlmSessionActor to Context.ActorOf. If no spawn factory is - /// available (e.g., in tests or standalone mode), returns a failure result. - /// Reports start/complete notifications through the per-call output sink. - /// - public async Task SpawnAsync( - SubAgentProfile profile, - string task, - string? runtimeContext, - ToolInvocationContext context, - CancellationToken ct = default, - string? systemPromptOverlay = null, - ChannelWriter? activitySink = null) + internal abstract record ChildPreparation { - var result = await SpawnRunAsync(profile, task, runtimeContext, context, ct, systemPromptOverlay, activitySink) - .ConfigureAwait(false); - return result.ToProtocolResult(); + internal sealed record Ready(PreparedChildRun Run) : ChildPreparation; + internal sealed record Rejected(EnrichedChildRunResult Result) : ChildPreparation; } - internal async Task SpawnRunAsync( - SubAgentProfile profile, - string task, - string? runtimeContext, - ToolInvocationContext context, - CancellationToken ct, - string? systemPromptOverlay, - ChannelWriter? activitySink) + internal async Task PrepareRunAsync( + SubAgentProfile profile, string task, string? runtimeContext, ToolInvocationContext context, + CancellationToken ct, string? systemPromptOverlay) { // Parent-side spawn breadcrumbs — each event is fanned out to daemon.log/Seq and // the parent's session.log from one place (see SubAgentSpawnBreadcrumbs), covering @@ -103,26 +82,24 @@ internal async Task SpawnRunAsync( if (context.SpawnChildActor is null) { SubAgentSpawnBreadcrumbs.NoSessionContext(_logger, context, profile.Name); - activitySink?.TryComplete(); - return new EnrichedChildRunResult.OtherRun(new SubAgentResult + return new ChildPreparation.Rejected(new EnrichedChildRunResult.OtherRun(new SubAgentResult { Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.SpawnUnavailable), Output = $"Cannot spawn subagent '{profile.Name}': no session context available.", AgentName = new AgentName(profile.Name) - }); + })); } var exposure = ResolveTools(context); if (exposure.Tools.Count == 0) { SubAgentSpawnBreadcrumbs.NoToolsAvailable(_logger, context, profile.Name); - activitySink?.TryComplete(); - return new EnrichedChildRunResult.OtherRun(new SubAgentResult + return new ChildPreparation.Rejected(new EnrichedChildRunResult.OtherRun(new SubAgentResult { Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.NoToolsAvailable), Output = $"Cannot spawn subagent '{profile.Name}': no tools are available under the parent audience policy.", AgentName = new AgentName(profile.Name) - }); + })); } var definition = new SubAgentDefinition @@ -166,26 +143,19 @@ internal async Task SpawnRunAsync( } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - activitySink?.TryComplete(); - return new EnrichedChildRunResult.OtherRun(CancelledResult(definition.Name, runId, scopeId)); + return new ChildPreparation.Rejected(new EnrichedChildRunResult.OtherRun(CancelledResult(definition.Name, runId, scopeId))); } catch (Exception ex) when (!FatalExceptionPolicy.IsFatal(ex)) { SubAgentSpawnBreadcrumbs.RunFailed(_logger, context, profile.Name, runId, ex); - activitySink?.TryComplete(); - return new EnrichedChildRunResult.OtherRun(new SubAgentResult + return new ChildPreparation.Rejected(new EnrichedChildRunResult.OtherRun(new SubAgentResult { Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.SpawnError), Output = $"Subagent error: {ex.Message}", AgentName = definition.Name, RunId = runId, ScopeId = scopeId - }); - } - catch (Exception ex) when (FatalExceptionPolicy.IsFatal(ex)) - { - activitySink?.TryComplete(); - throw; + })); } // A transport can own an approval channel without being able to service // interactive prompts. Fork only the admitted capability, never bridge @@ -212,15 +182,6 @@ internal async Task SpawnRunAsync( InitialWorkingSnapshot = initialWorkingSnapshot }; - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = true, - ToolCount = exposure.Tools.Count - }); - - // Spawn as child of the session actor via the context factory var props = SubAgentActor.CreatePropsWithProjectInstructionProvider( definition, chatClient, @@ -230,69 +191,7 @@ internal async Task SpawnRunAsync( _sessionMetrics, exposure.CoreToolNames, _logger); - var actorName = $"subagent-{definition.Name}-{runId}"; - IActorRef subAgent; - try - { - subAgent = (IActorRef)await context.SpawnChildActor(props, actorName, ct); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = false, - Outcome = SubAgentRunOutcome.Failed, - OutcomeReason = SubAgentOutcomeReason.CancelledByParent - }); - activitySink?.TryComplete(); - return new EnrichedChildRunResult.OtherRun(CancelledResult(definition.Name, runId, scopeId)); - } - catch (Exception ex) when (!FatalExceptionPolicy.IsFatal(ex)) - { - // The child actor was never created (session actor ActorOf failed or the - // spawn ask timed out). Record it to the session transcript before the - // exception propagates to the tool pipeline. - SubAgentSpawnBreadcrumbs.ChildSpawnFailed(_logger, context, profile.Name, runId, ex); - // Balance the IsStarted=true notification above: the non-streaming path - // (activitySink is null) relies solely on the per-call output sink, so without - // a terminal event the session UI shows a sub-agent stuck in "Started". - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = false, - Outcome = SubAgentRunOutcome.Failed, - OutcomeReason = SubAgentOutcomeReason.SpawnError - }); - activitySink?.TryComplete(); - throw; - } - catch (Exception ex) when (FatalExceptionPolicy.IsFatal(ex)) - { - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = false, - Outcome = SubAgentRunOutcome.Failed, - OutcomeReason = SubAgentOutcomeReason.SpawnError - }); - activitySink?.TryComplete(); - throw; - } - - SubAgentSpawnBreadcrumbs.ChildSpawned(_logger, context, profile.Name, runId); - - var sw = Stopwatch.StartNew(); - try - { - var result = await subAgent.Ask( - new RunSubAgent + var execution = new RunSubAgent { Scope = childScope, Task = task, @@ -301,108 +200,61 @@ internal async Task SpawnRunAsync( PrefillTimeout = prefillTimeout, NoProgressTimeout = noProgressTimeout, Cancellation = ct, - // Null for non-streaming callers such as routed skills and the - // legacy ExecuteAsync path; the sub-agent surfaces its progress - // through its own session-correlated logs regardless. Streaming - // spawn_agent calls pass a real sink so the parent tool's - // liveness watchdog sees progress. - ActivitySink = activitySink - }, - // No Ask timeout: a healthy run is bounded by the sub-agent's own - // watchdogs, not by wall-clock, so any finite ceiling here could - // pre-empt a legitimately long run. The sub-agent self-completes on - // two internal budgets: the liveness watchdog (no bytes at all, - // including keepalives) and the no-progress deadline (keepalives but - // no real tokens for NoProgressTimeoutSeconds). ct — the spawning - // tool call's token — only adds parent-turn / user cancellation on - // top; once streaming spawn_agent has emitted its first activity, - // the parent no longer applies an inter-item watchdog. Keepalive - // wedges are bounded by the sub-agent's own no-progress watchdog. - timeout: Timeout.InfiniteTimeSpan, - cancellationToken: ct); - - sw.Stop(); - - result = await EnrichWorkingContextResultAsync( - result, - initialWorkingSnapshot, - context.Audience, - ct).ConfigureAwait(false); - - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = result.Success, - Outcome = result.Outcome, - OutcomeReason = result.OutcomeReason, - Duration = sw.Elapsed, - Findings = result.Findings, - WorkingContext = result.WorkingContext - }); - - SubAgentSpawnBreadcrumbs.Completed(_logger, context, profile.Name, runId, result.Success, sw.ElapsedMilliseconds); - - var response = result with { RunId = runId, ScopeId = scopeId }; - return result.Completion switch - { - ChildRunCompletion.Completed or ChildRunCompletion.Partial => - new EnrichedChildRunResult.SuccessfulRun(response, - new EnrichedChildRunResult.RunLocations(childStorage.LogPath, childStorage.ArtifactDirectory)), - ChildRunCompletion.Failed or ChildRunCompletion.Cancelled => new EnrichedChildRunResult.OtherRun(response), - _ => throw new InvalidOperationException("Unexpected child run completion.") - }; - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + ActivitySink = null + }; + var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new { - sw.Stop(); - TryStopSubAgent(subAgent); - - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = false, - Outcome = SubAgentRunOutcome.Failed, - OutcomeReason = SubAgentOutcomeReason.CancelledByParent, - Duration = sw.Elapsed - }); - - return new EnrichedChildRunResult.OtherRun(CancelledResult(definition.Name, runId, scopeId)); - } - catch (Exception ex) when (!FatalExceptionPolicy.IsFatal(ex)) + profile.Name, definition.SystemPrompt, definition.ProjectInstructions, definition.OperatingRules, + definition.ModelRole, definition.EmitStructuredFindings, task, runtimeContext, + timeout = subAgentTimeout.Ticks, prefill = prefillTimeout.Ticks, noProgress = noProgressTimeout.Ticks, + tools = exposure.Tools.OrderBy(static tool => tool.Name, StringComparer.Ordinal) + .Select(static tool => + { + var function = tool.ToAITool() as Microsoft.Extensions.AI.AIFunctionDeclaration + ?? throw new InvalidDataException("A child tool requires its canonical function definition."); + return new { tool.Name, function.Description, schema = function.JsonSchema.GetRawText() }; + }).ToArray() + })))); + return new ChildPreparation.Ready(new PreparedChildRun { - sw.Stop(); - - TryStopSubAgent(subAgent); - - context.Outputs.ReportSubAgentActivity(new SubAgentNotificationInfo - { - RunId = runId, - AgentName = definition.Name.Value, - IsStarted = false, - Success = false, - Outcome = SubAgentRunOutcome.Failed, - OutcomeReason = SubAgentOutcomeReason.SpawnError, - Duration = sw.Elapsed - }); + Props = props, RunId = runId, AgentName = definition.Name, ToolCount = exposure.Tools.Count, + ArgumentsDigest = digest, Execution = execution + }); + } - SubAgentSpawnBreadcrumbs.RunFailed(_logger, context, profile.Name, runId, ex); - return new EnrichedChildRunResult.OtherRun(new SubAgentResult - { - Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.SpawnError), - Output = $"Subagent error: {ex.Message}", - AgentName = new AgentName(profile.Name), - RunId = runId, - ScopeId = scopeId - }); + internal async Task StartRunAsync( + SubAgentProfile profile, string task, string? runtimeContext, ToolInvocationContext context, + CancellationToken ct, string? systemPromptOverlay) + { + var preparation = await PrepareRunAsync(profile, task, runtimeContext, context, ct, systemPromptOverlay).ConfigureAwait(false); + if (preparation is ChildPreparation.Rejected rejected) + { + context.Outputs.TryComplete(new ToolInvocationReceipt.OtherOutcome(ToolInvocationOutcomeCategory.TransientFailure)); + return rejected.Result.Response.Output; } - finally + var prepared = ((ChildPreparation.Ready)preparation).Run; + var factory = context.SpawnChildActor ?? throw new InvalidOperationException("A child start requires its owning session adapter."); + var response = await factory(prepared, $"child-run-{prepared.RunId.Value}", ct).ConfigureAwait(false); + switch (response) { - // Terminate the streaming caller's activity reader even on failure. - activitySink?.TryComplete(); + case ChildStartReply.Accepted accepted: + accepted.Validate(prepared); + if (accepted.Run.StartKey.SessionId.Value != context.SessionId + || accepted.Run.OriginalContext.Audience != context.Audience + || accepted.Run.OriginalContext.Boundary != context.Boundary + || accepted.Run.OriginalContext.ChannelType != context.ChannelType) + throw new InvalidDataException("The child acceptance differs from its invoking session authority."); + context.Outputs.TryComplete(new ToolInvocationReceipt.Succeeded([], null)); + return JsonSerializer.Serialize(new + { + run_id = accepted.Run.RunId.Value, scope_id = accepted.Run.ScopeId.Value, + state = accepted.State.ToString(), control_tool = "check_agent_run" + }); + case ChildStartReply.Conflict: + context.Outputs.TryComplete(new ToolInvocationReceipt.OtherOutcome(ToolInvocationOutcomeCategory.InvalidInput)); + return "Error: start_conflict. The start key already identifies different canonical arguments."; + default: + throw new InvalidDataException("The session owner returned a noncanonical child acceptance."); } } @@ -418,10 +270,11 @@ private static SubAgentResult CancelledResult( ScopeId = scopeId }; - private async Task EnrichWorkingContextResultAsync( + internal static async Task EnrichWorkingContextResultAsync( SubAgentResult result, WorkingContextSnapshot initialSnapshot, TrustAudience audience, + IWorkingContextSnapshotProvider snapshots, CancellationToken cancellationToken) { if (!result.Success || result.WorkingContext is not { } childContext) @@ -435,7 +288,7 @@ private async Task EnrichWorkingContextResultAsync( .Distinct(FilePathComparer) .Take(WorkingContext.MaxRecentFiles)] }; - var finalSnapshot = await _workingContextSnapshots.CreateAsync( + var finalSnapshot = await snapshots.CreateAsync( finalContext, audience, cancellationToken).ConfigureAwait(false); @@ -498,11 +351,6 @@ private sealed record ResolvedSubAgentTools( IReadOnlyList Tools, IReadOnlySet CoreToolNames); - private static void TryStopSubAgent(IActorRef subAgent) - { - subAgent.Tell(PoisonPill.Instance); - } - private static void CreateChildLogTarget(string logPath) { var directory = Path.GetDirectoryName(logPath) diff --git a/src/Netclaw.Actors/Tools/SkillLoadTool.cs b/src/Netclaw.Actors/Tools/SkillLoadTool.cs index 5ed0b29a7..d19c0e955 100644 --- a/src/Netclaw.Actors/Tools/SkillLoadTool.cs +++ b/src/Netclaw.Actors/Tools/SkillLoadTool.cs @@ -153,17 +153,8 @@ protected override async Task ExecuteAsync(Params args, ToolInvocationCo _sessionMetrics?.RecordSkillLoaded(skill.Name, SkillLoadMethod.SkillLoadTool); _logger?.LogInformation("turn_skill_loaded skill={SkillName} method=skill_load", skill.Name); - var routedResult = await _subAgentSpawner.SpawnAsync( - profile, - args.Task, - args.Context, - context!, - ct, - systemPromptOverlay: routedBody); - - return routedResult.Success - ? routedResult.Output - : $"Subagent '{profile.Name}' failed: {routedResult.Output}"; + return await _subAgentSpawner.StartRunAsync( + profile, args.Task, args.Context, context, ct, systemPromptOverlay: routedBody); } string body; diff --git a/src/Netclaw.Actors/Tools/ToolAccessPolicy.cs b/src/Netclaw.Actors/Tools/ToolAccessPolicy.cs index 94246ea5f..d9f2276f4 100644 --- a/src/Netclaw.Actors/Tools/ToolAccessPolicy.cs +++ b/src/Netclaw.Actors/Tools/ToolAccessPolicy.cs @@ -12,6 +12,7 @@ using Netclaw.Security; using Netclaw.Security.Authorization.Filesystem; using Netclaw.Tools; +using Netclaw.Actors.SubAgents; using ShellSyntaxTree; namespace Netclaw.Actors.Tools; @@ -1505,7 +1506,7 @@ private bool IsFeatureDisabledTool(string toolName) => !_featureGates.SearchEnabled, "skill_load" or "skill_read_resource" => !_featureGates.SkillSyncEnabled, - "spawn_agent" + "spawn_agent" or CheckAgentRunTool.ToolName => !_featureGates.SubAgentsEnabled, "set_reminder" or "cancel_reminder" or "list_reminders" or "get_reminder_history" or "run_reminder" => !_featureGates.SchedulingEnabled, diff --git a/src/Netclaw.Actors/Tools/ToolLivenessValidator.cs b/src/Netclaw.Actors/Tools/ToolLivenessValidator.cs index 8728954f4..a0d95116b 100644 --- a/src/Netclaw.Actors/Tools/ToolLivenessValidator.cs +++ b/src/Netclaw.Actors/Tools/ToolLivenessValidator.cs @@ -14,8 +14,7 @@ namespace Netclaw.Actors.Tools; /// [NetclawTool(Liveness = …)] attribute must also resolve to /// SelfMonitoring at runtime (). A silent /// downgrade to would place the tool on a -/// wall-clock watchdog — for spawn_agent that means killing a healthy, -/// self-monitoring sub-agent mid-run. The source generator emits +/// wall-clock watchdog. The source generator emits /// LivenessMode directly from the attribute, so a mismatch means stale /// generated code or a hand-rolled override; either way, fail loud at startup /// rather than mis-supervising at runtime. (No silent fallbacks.) diff --git a/src/Netclaw.Cli/Resources/identity/AGENTS.template.md b/src/Netclaw.Cli/Resources/identity/AGENTS.template.md index f7dd31bd8..247647ddf 100644 --- a/src/Netclaw.Cli/Resources/identity/AGENTS.template.md +++ b/src/Netclaw.Cli/Resources/identity/AGENTS.template.md @@ -22,7 +22,17 @@ Public conversations and the sub-agents they launch. ## Delegation - +Use children for useful independent work, including substantial code tasks, when their profiles permit that work. +Give each child a precise objective, authorized workspace, output artifact, and acceptance checks. +Assign one writer per workspace. Concurrent writers need separate authorized worktrees. +Review each returned artifact and its evidence before delivery. + +`spawn_agent` returns durable acceptance before the child finishes. Continue independent parent work. +The terminal result arrives later. Load `check_agent_run` through `load_tool` for status or cancellation. +Use its exact `log_path` with normal file tools for child diagnosis. A log does not prove current health or completion. +Cancellation acceptance does not prove dispatch closure. Read the closure facts and retained partial evidence. +Use cancellation and a revised task for new instructions. Live steering and private agent messages are unavailable. +Load `netclaw-operations`, then read `references/child-runs.md` for the full lifecycle rules. ## Review and Quality Gates diff --git a/src/Netclaw.Configuration/Resources/AGENTS.md b/src/Netclaw.Configuration/Resources/AGENTS.md index bf24b4cfa..b87f3c163 100644 --- a/src/Netclaw.Configuration/Resources/AGENTS.md +++ b/src/Netclaw.Configuration/Resources/AGENTS.md @@ -259,6 +259,24 @@ Available subagents are listed in the [available-subagents] context block. Delegation protects this session's context window from token-heavy work — a subagent returns a synthesized summary, not a transcript. +Use children for useful independent work without a separate delegation request. +Include substantial code tasks when the selected profile permits them. +Give each child a precise objective, authorized workspace, output artifact, and acceptance checks. +Assign one writer per workspace. Concurrent writers need separate authorized worktrees. + +`spawn_agent` returns a durable accepted run identifier before the child finishes. +Continue independent parent work. Acceptance does not prove task success. +The terminal result arrives later with child attribution. +Load `check_agent_run` through `load_tool` when status or cancellation is necessary. +Use its exact `log_path` with normal file tools when a live child needs diagnosis. +A log does not prove current health or completion. +Avoid tight status polls. Normal completion uses the later terminal result. +Cancellation acceptance does not prove dispatch closure or terminal completion. +Read the closure facts and partial evidence before you choose a replacement task. +An ordinary later parent message does not cancel an accepted child or its approval prompt. +Use cancellation and a revised task for new instructions. Live steering and private agent messages are unavailable. +Load `netclaw-operations`, then read `references/child-runs.md` for the complete lifecycle and authority rules. + **When to delegate:** - Research requiring 2+ sources or multiple searches - Parallelizable tasks (multiple independent queries can run concurrently) diff --git a/src/Netclaw.Configuration/SubAgentToolPolicy.cs b/src/Netclaw.Configuration/SubAgentToolPolicy.cs index ab1493fc3..d985fa943 100644 --- a/src/Netclaw.Configuration/SubAgentToolPolicy.cs +++ b/src/Netclaw.Configuration/SubAgentToolPolicy.cs @@ -16,6 +16,7 @@ public static class SubAgentToolPolicy private static readonly HashSet DeniedSubAgentToolNames = new(StringComparer.Ordinal) { "spawn_agent", + "check_agent_run", ToolAudienceProfileToolCatalog.AttachFile }; diff --git a/src/Netclaw.Daemon.Tests/Mcp/ToolIndexUpdaterTests.cs b/src/Netclaw.Daemon.Tests/Mcp/ToolIndexUpdaterTests.cs index 57b5f4091..b667d1b9a 100644 --- a/src/Netclaw.Daemon.Tests/Mcp/ToolIndexUpdaterTests.cs +++ b/src/Netclaw.Daemon.Tests/Mcp/ToolIndexUpdaterTests.cs @@ -44,6 +44,8 @@ public async Task StartAsync_with_no_user_facing_agents_sets_actionable_discover await updater.StartAsync(TestContext.Current.CancellationToken); + Assert.Contains(registry.GetAllTools(), tool => tool.Name == CheckAgentRunTool.ToolName); + Assert.False(registry.IsCoreTool(CheckAgentRunTool.ToolName)); var discovery = subAgentLayer.GetContextLayer(TrustAudience.Personal); Assert.False(string.IsNullOrWhiteSpace(discovery)); Assert.Contains("available-subagents", discovery, StringComparison.OrdinalIgnoreCase); @@ -104,5 +106,6 @@ public async Task StartAsync_keeps_public_tool_index_filtered_from_hidden_capabi Assert.Contains("file_read:", publicIndex); Assert.DoesNotContain("set_reminder", publicIndex); Assert.DoesNotContain("memorizer", publicIndex); + Assert.DoesNotContain(CheckAgentRunTool.ToolName, publicIndex); } } diff --git a/src/Netclaw.Daemon/Mcp/ToolIndexUpdater.cs b/src/Netclaw.Daemon/Mcp/ToolIndexUpdater.cs index 1c6fa522f..c0c987a5d 100644 --- a/src/Netclaw.Daemon/Mcp/ToolIndexUpdater.cs +++ b/src/Netclaw.Daemon/Mcp/ToolIndexUpdater.cs @@ -60,10 +60,9 @@ public Task StartAsync(CancellationToken cancellationToken) _toolRegistry.Register(new SpawnAgentTool( _subAgentRegistry, _subAgentSpawner, _paths, _subAgentConfig, _agentLoader, _loggerFactory.CreateLogger())); + _toolRegistry.Register(new CheckAgentRunTool()); - // Fail loud at startup if a self-monitoring tool (e.g. spawn_agent) silently - // resolved to a wall-clock-supervised mode — that would let the parent kill a - // healthy sub-agent mid-run. See ToolLivenessValidator. + // A stale generated liveness declaration must fail before tools become available. ToolLivenessValidator.AssertSelfMonitoringConsistency( _toolRegistry.GetAllRegistrations().Select(r => r.Tool)); diff --git a/src/Netclaw.Tools.Abstractions/SubAgentRunMetadata.cs b/src/Netclaw.Tools.Abstractions/SubAgentRunMetadata.cs index 406c3ede1..6c209feb6 100644 --- a/src/Netclaw.Tools.Abstractions/SubAgentRunMetadata.cs +++ b/src/Netclaw.Tools.Abstractions/SubAgentRunMetadata.cs @@ -86,6 +86,7 @@ public readonly record struct SubAgentOutcomeReason(string Value) public static readonly SubAgentOutcomeReason CancelledByParent = new("cancelled_by_parent"); public static readonly SubAgentOutcomeReason NoSubstantiveOutputTimeout = new("no_substantive_output_timeout"); public static readonly SubAgentOutcomeReason NoActivityTimeout = new("no_activity_timeout"); + public static readonly SubAgentOutcomeReason OwnerRestartLost = new("owner_restart_lost"); public static readonly SubAgentOutcomeReason ActorStopped = new("actor_stopped"); public static readonly SubAgentOutcomeReason SpawnUnavailable = new("spawn_unavailable"); public static readonly SubAgentOutcomeReason NoToolsAvailable = new("no_tools_available"); diff --git a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs index 09ae5d266..fd2e2200b 100644 --- a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs +++ b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs @@ -184,7 +184,7 @@ public static ChildRunCompletion FromReportedOutcome( SubAgentRunOutcome.Partial when delta is not null && reason is { } partialReason => new Partial(partialReason, delta), SubAgentRunOutcome.Failed when reason == SubAgentOutcomeReason.CancelledByParent => - new Cancelled(reason.Value), + new Cancelled(reason.Value) { ConfirmedActivity = delta }, SubAgentRunOutcome.Failed when reason is { } failureReason => new Failed(failureReason), _ => throw new ArgumentException( $"Invalid child completion: outcome={outcome}, reason={(reason is { } value ? value.Value : "none")}, hasDelta={delta is not null}.", @@ -219,10 +219,11 @@ public sealed record Failed(SubAgentOutcomeReason FailureReason) : ChildRunCompl public sealed record Cancelled(SubAgentOutcomeReason CancellationReason) : ChildRunCompletion { + public WorkingContextDelta? ConfirmedActivity { get; init; } public override bool Success => false; public override SubAgentRunOutcome Outcome => SubAgentRunOutcome.Failed; public override SubAgentOutcomeReason? Reason => CancellationReason; - public override WorkingContextDelta? Delta => null; + public override WorkingContextDelta? Delta => ConfirmedActivity; } } From bd1391b99be8120b8c89056542f9fd18114b1564 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 22:51:44 +0000 Subject: [PATCH 06/22] Record independently verified background lifecycle tasks --- .../changes/background-child-runs/tasks.md | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 95a86c6ca..5034181cf 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -3,7 +3,7 @@ - [x] 1.1 Rebase on PR 1 and reconcile its frozen types and deltas; verify the combined recurrence contract includes the confirmed model-owned recovery boundary. - [x] 1.2 Record the confirmed backend-owned capacity boundary. Remove provider-slot coordination and the pending approval gate. Verify independent request and local control clauses. - [x] 1.3 Have an independent test author derive lifecycle assertions from the delta; verify the author lists paired controls and deliberate faulty variants. -- [ ] 1.4 Extend existing actor fixtures with start, provider, dispatch, terminal, and persistence barriers; verify assertions use acknowledgements rather than sleeps. +- [x] 1.4 Extend existing actor fixtures with start, provider, dispatch, terminal, and persistence barriers; verify assertions use acknowledgements rather than sleeps. - [x] 1.5 Capture a pre-change journal/snapshot fixture with conversation and parent approval state; verify the fixture predates new lifecycle records. Preparation evidence: @@ -24,69 +24,69 @@ Preparation evidence: ## 2. Durable acceptance and independent lifetime -- [ ] 2.1 Extend the owner-session seam with typed accepted-run operations and reuse `ChildRunScope.Authority`; verify no parallel requester or path-root inputs exist. -- [ ] 2.2 Stamp tool and routed activation keys before asynchronous setup; verify retries preserve original call/input IDs and conflicting digests fail. -- [ ] 2.3 Add framework-owned acceptance records and snapshot conversion; verify round-trip and journal replay preserve the same key, context, and run ID. -- [ ] 2.4 Persist acceptance before execution and acknowledgement; fault the write and verify zero child requests or task effects. -- [ ] 2.5 Deduplicate concurrent and recovered starts; lose the accepted response and verify one child plus the same returned run ID. -- [ ] 2.6 Separate accepted-run lifetime from the start token; cancel that token after commit and verify the child remains controllable. -- [ ] 2.7 Report post-acceptance creation failure through the accepted run; verify one durable terminal failure and no false rejection/restart. +- [x] 2.1 Extend the owner-session seam with typed accepted-run operations and reuse `ChildRunScope.Authority`; verify no parallel requester or path-root inputs exist. +- [x] 2.2 Stamp tool and routed activation keys before asynchronous setup; verify retries preserve original call/input IDs and conflicting digests fail. +- [x] 2.3 Add framework-owned acceptance records and snapshot conversion; verify round-trip and journal replay preserve the same key, context, and run ID. +- [x] 2.4 Persist acceptance before execution and acknowledgement; fault the write and verify zero child requests or task effects. +- [x] 2.5 Deduplicate concurrent and recovered starts; lose the accepted response and verify one child plus the same returned run ID. +- [x] 2.6 Separate accepted-run lifetime from the start token; cancel that token after commit and verify the child remains controllable. +- [x] 2.7 Report post-acceptance creation failure through the accepted run; verify one durable terminal failure and no false rejection/restart. ## 3. Terminal receipt and attributed continuation -- [ ] 3.1 Add child terminal acknowledgement and close task dispatch before terminal output; verify the child retains its result until durable acknowledgement. -- [ ] 3.2 Authenticate terminal sender and generation; replay stale and duplicate results and verify one committed winner. -- [ ] 3.3 Persist terminal receipt before enrichment; fail persistence and verify no successful acknowledgement, delivery claim, or model request. -- [ ] 3.4 Move result enrichment to recorded terminal handling; fail enrichment and verify the original outcome survives with an explicit warning. -- [ ] 3.5 Atomically persist delivery dedup and continuation input; crash before/after admission and verify one pending input without child replay. -- [ ] 3.6 Materialize a fresh attributed provider tool-call/result pair; inspect actual messages and verify the original start call receives no second result. -- [ ] 3.7 Preserve original authority and compatible journal-order queues; verify another speaker and active compaction cannot change result authority. -- [ ] 3.8 Retain and refresh parent detector checkpoints in the child run ledger. Verify committed start settlement and direct activation without fabricated receipts. Verify late results after fresh input, sibling checkpoint updates, distinct-task separation, receipt failure, and recovery. Preserve current parent directory/project/branch. -- [ ] 3.9 Keep failed parent continuations visible. Verify committed input is not readmitted and the model receives retained loss facts before it chooses further work. +- [x] 3.1 Add child terminal acknowledgement and close task dispatch before terminal output; verify the child retains its result until durable acknowledgement. +- [x] 3.2 Authenticate terminal sender and generation; replay stale and duplicate results and verify one committed winner. +- [x] 3.3 Persist terminal receipt before enrichment; fail persistence and verify no successful acknowledgement, delivery claim, or model request. +- [x] 3.4 Move result enrichment to recorded terminal handling; fail enrichment and verify the original outcome survives with an explicit warning. +- [x] 3.5 Atomically persist delivery dedup and continuation input; crash before/after admission and verify one pending input without child replay. +- [x] 3.6 Materialize a fresh attributed provider tool-call/result pair; inspect actual messages and verify the original start call receives no second result. +- [x] 3.7 Preserve original authority and compatible journal-order queues; verify another speaker and active compaction cannot change result authority. +- [x] 3.8 Retain and refresh parent detector checkpoints in the child run ledger. Verify committed start settlement and direct activation without fabricated receipts. Verify late results after fresh input, sibling checkpoint updates, distinct-task separation, receipt failure, and recovery. Preserve current parent directory/project/branch. +- [x] 3.9 Keep failed parent continuations visible. Verify committed input is not readmitted and the model receives retained loss facts before it chooses further work. ## 4. Every routed start and deferred parent controls -- [ ] 4.1 Convert `spawn_agent` to the canonical acceptance JSON. Verify identical output/model bodies, valid owner identifiers, and initial `Accepted` state. Hold the child and verify parent replies before release. Reject malformed or uncommitted acceptance. -- [ ] 4.2 Convert routed `skill_load`; verify target validation, content scan, overlay semantics, and explicit failures remain intact. -- [ ] 4.3 Convert slash, scheduled slash, and reminder skill activation; verify each returns acceptance and retains its original occurrence/call correlation. -- [ ] 4.4 Preserve inline activation and no-inline-fallback failures; verify unknown/internal targets create neither a child nor inline work. -- [ ] 4.5 Add Deferred `check_agent_run(run_id, cancel=false)` through normal non-shell policy; verify explicit load, unchanged core, and direct ownership checks. -- [ ] 4.6 Deny child, cross-session, and wrong-requester controls; verify no target details and keep the valid parent control operational. -- [ ] 4.7 Add bounded recorded-run context with the exact control name; verify it cannot claim unobserved live provider state or expose hidden results. -- [ ] 4.8 Return canonical live `log_path` and `artifact_directory` after owner authorization. Hold a real child, inspect the actual status JSON, and read its exact log with ordinary file tools. Verify foreign controls reveal no paths and returned paths cannot bypass file policy. +- [x] 4.1 Convert `spawn_agent` to the canonical acceptance JSON. Verify identical output/model bodies, valid owner identifiers, and initial `Accepted` state. Hold the child and verify parent replies before release. Reject malformed or uncommitted acceptance. +- [x] 4.2 Convert routed `skill_load`; verify target validation, content scan, overlay semantics, and explicit failures remain intact. +- [x] 4.3 Convert slash, scheduled slash, and reminder skill activation; verify each returns acceptance and retains its original occurrence/call correlation. +- [x] 4.4 Preserve inline activation and no-inline-fallback failures; verify unknown/internal targets create neither a child nor inline work. +- [x] 4.5 Add Deferred `check_agent_run(run_id, cancel=false)` through normal non-shell policy; verify explicit load, unchanged core, and direct ownership checks. +- [x] 4.6 Deny child, cross-session, and wrong-requester controls; verify no target details and keep the valid parent control operational. +- [x] 4.7 Add bounded recorded-run context with the exact control name; verify it cannot claim unobserved live provider state or expose hidden results. +- [x] 4.8 Return canonical live `log_path` and `artifact_directory` after owner authorization. Hold a real child, inspect the actual status JSON, and read its exact log with ordinary file tools. Verify foreign controls reveal no paths and returned paths cannot bypass file policy. ## 5. Cancellation, partial evidence, and approval prompts -- [ ] 5.1 Persist cancellation admission and acknowledge local dispatch closure separately; hold a dispatch worker and verify the acknowledgement cannot precede closure. -- [ ] 5.2 Close queued model/tool dispatch and exact approval retries; release stale callbacks and verify no new task operation after confirmed cutover. -- [ ] 5.3 Commit bounded checkpoints after completed child rounds; verify confirmed receipts and paths survive a later cancel without a new model response. -- [ ] 5.4 Add five-second framework-only finalization with a separate token; verify report writes stay run-local and no task tool or project edit occurs. -- [ ] 5.5 Fault report writes and expire controlled finalization timers; verify retained checkpoint evidence, explicit reason, and one cancelled result. -- [ ] 5.6 Extend cancelled completion with confirmed activity; verify `Success=false`, `Failed`, and `CancelledByParent` survive wire conversion. -- [ ] 5.7 Exercise both durable completion/cancel orders; verify the first permitted terminal winner and accurate unconfirmed external-effect reports. -- [ ] 5.8 Retain original approval prompt correlation beyond the start call; verify an original eligible answer resumes only the exact live child retry. -- [ ] 5.9 Persist child prompt lifecycle facts and settle them on cancel/loss; verify late answers create no tool execution or authorization grant. -- [ ] 5.10 Keep child prompt waits independent of later ordinary parent input; verify that input neither adopts requester authority nor abandons the child prompt. +- [x] 5.1 Persist cancellation admission and acknowledge local dispatch closure separately; hold a dispatch worker and verify the acknowledgement cannot precede closure. +- [x] 5.2 Close queued model/tool dispatch and exact approval retries; release stale callbacks and verify no new task operation after confirmed cutover. +- [x] 5.3 Commit bounded checkpoints after completed child rounds; verify confirmed receipts and paths survive a later cancel without a new model response. +- [x] 5.4 Add five-second framework-only finalization with a separate token; verify report writes stay run-local and no task tool or project edit occurs. +- [x] 5.5 Fault report writes and expire controlled finalization timers; verify retained checkpoint evidence, explicit reason, and one cancelled result. +- [x] 5.6 Extend cancelled completion with confirmed activity; verify `Success=false`, `Failed`, and `CancelledByParent` survive wire conversion. +- [x] 5.7 Exercise both durable completion/cancel orders; verify the first permitted terminal winner and accurate unconfirmed external-effect reports. +- [x] 5.8 Retain original approval prompt correlation beyond the start call; verify an original eligible answer resumes only the exact live child retry. +- [x] 5.9 Persist child prompt lifecycle facts and settle them on cancel/loss; verify late answers create no tool execution or authorization grant. +- [x] 5.10 Keep child prompt waits independent of later ordinary parent input; verify that input neither adopts requester authority nor abandons the child prompt. ## 6. Passivation, restart, and compatibility -- [ ] 6.1 Defer idle passivation for live children and pending result admission; verify normal idle behavior resumes when those obligations end. -- [ ] 6.2 Retain shell-job reap behavior as a nearby control; run its existing integration cases and verify no child state enters shell-job records. -- [ ] 6.3 Cancel children during coordinated drain with bounded finalization; verify parent snapshot retains terminal and prompt disposition. -- [ ] 6.4 Recover accepted children as explicit `Lost` without relaunch; verify committed cancellation remains cancelled and stale success cannot win. -- [ ] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child, committed result loss, or framework tool execution during recovery. -- [ ] 6.6 Load the captured pre-change fixture. Verify retained history, parent prompts, and an explicit empty child ledger. Capture a baseline SQLite storage binding and marker through the real resolver. Verify the candidate preserves that row and marker. The journal fixture alone supplies no storage-binding proof. -- [ ] 6.7 Probe prior-reader compatibility against new events; verify rollback instructions require backup restoration when compatibility fails. +- [x] 6.1 Defer idle passivation for live children and pending result admission; verify normal idle behavior resumes when those obligations end. +- [x] 6.2 Retain shell-job reap behavior as a nearby control; run its existing integration cases and verify no child state enters shell-job records. +- [x] 6.3 Cancel children during coordinated drain with bounded finalization; verify parent snapshot retains terminal and prompt disposition. +- [x] 6.4 Recover accepted children as explicit `Lost` without relaunch; verify committed cancellation remains cancelled and stale success cannot win. +- [x] 6.5 Crash at acceptance, terminal persistence, enrichment, delivery preparation, and admission; verify no duplicate child, committed result loss, or framework tool execution during recovery. +- [x] 6.6 Load the captured pre-change fixture. Verify retained history, parent prompts, and an explicit empty child ledger. Capture a baseline SQLite storage binding and marker through the real resolver. Verify the candidate preserves that row and marker. The journal fixture alone supplies no storage-binding proof. +- [x] 6.7 Probe prior-reader compatibility against new events; verify rollback instructions require backup restoration when compatibility fails. ## 7. Integrated health, evals, and sensitive fault controls - [ ] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. - [ ] 7.2 Add targeted Subagents and background cases for acceptance, routed starts, controls, cancellation, prompts, and recovery; verify actual dispatch and artifact evidence. - [ ] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. -- [ ] 7.4 Hold several accepted child requests. Verify independent parent dispatch and locally responsive status/cancel without provider-capacity coordination. +- [x] 7.4 Hold several accepted child requests. Verify independent parent dispatch and locally responsive status/cancel without provider-capacity coordination. - [ ] 7.5 Hold a child on the selected real backend that accepts concurrent requests. Verify a parent reply before child release in five critical-case trials. -- [ ] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. -- [ ] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. +- [x] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. +- [x] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. - [ ] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. - [ ] 7.9 Require an independent verifier to inspect integrated results and assertions; verify no skipped cases, weakened gates, or author-only pass claims remain. @@ -94,8 +94,8 @@ Preparation evidence: - [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. - [ ] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. -- [ ] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. -- [ ] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. +- [x] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. +- [x] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. - [ ] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. - [ ] 8.6 Run applicable builds, required tests, Slopwatch, and copyright-header verification; retain exact candidate and meaningful test counts. - [ ] 8.7 Validate the OpenSpec change strictly and sync implemented deltas through the appropriate skill; verify the PRD and active plan trace to the final contract. From 3c7a27c13cb8932c882e6bbf7d1e6a7918ab790b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 23:08:02 +0000 Subject: [PATCH 07/22] Align background child contracts with durable owner behavior --- docs/architecture/tool-authorization.md | 9 +++ ...SPEC-002-session-lifecycle-and-protocol.md | 9 ++- ...C-016-tool-liveness-and-stall-detection.md | 6 ++ .../changes/background-child-runs/design.md | 6 +- .../specs/background-subagent-runs/spec.md | 39 ++++++++- .../specs/netclaw-subagents/spec.md | 81 +++++++++++++++++++ .../specs/session-state-machine/spec.md | 8 ++ 7 files changed, 154 insertions(+), 4 deletions(-) diff --git a/docs/architecture/tool-authorization.md b/docs/architecture/tool-authorization.md index 195762cf1..66ed9cc6b 100644 --- a/docs/architecture/tool-authorization.md +++ b/docs/architecture/tool-authorization.md @@ -863,6 +863,15 @@ Normal tool policy, the owner session, and the original eligible requester defin Children cannot discover, load, or dispatch that control. A foreign caller receives no target details. The owner uses the original authority record. It does not copy requester or path authority into a second policy structure. Cancellation creates no authorization grant. A report path still requires ordinary file access policy. + +The owner commits cancellation admission before it reports `Cancelling`. +It closes the live dispatch gate, requests token cancellation, and waits for any synchronous dispatch prefix to exit. +It commits `DispatchClosed` before framework-only report finalization. +It commits any unresolved child prompt disposition before the terminal receipt. + +Token cancellation can resolve a prompt earlier. Durable closure does not require prompt resolution to commit first. +See [the operation-health specification](../spec/SPEC-016-tool-liveness-and-stall-detection.md#cancellation-and-partial-evidence) for the full cancellation sequence. + After authorization, the control adapter obtains `log_path` and `artifact_directory` from the existing canonical child storage binding. These call-local output paths are data. They create no grant or separate durable authority. The parent uses normal file tools for log access. A denied control reveals neither path. diff --git a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md index 6023b2ed5..ba50a7319 100644 --- a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md +++ b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md @@ -160,6 +160,13 @@ The live dispatch gate also applies to that grant operation and the child retry. Cancellation, terminal settlement, and recovery expire unresolved child prompts. Cancellation admission, confirmed local dispatch closure, and terminal settlement are separate lifecycle facts. +The owner commits cancellation admission before it reports `Cancelling`. +It closes the live dispatch gate, requests token cancellation, and waits for any synchronous dispatch prefix to exit. +It then commits `DispatchClosed` before it starts framework-only finalization. + +The owner commits any unresolved child prompt disposition before it commits the terminal receipt. +Token cancellation can resolve a prompt earlier. Prompt resolution need not commit before `DispatchClosed`. + After closure, framework-only finalization has a separate five-second deadline. It retains checkpoints and permits atomic report writes only inside the assigned run artifact directory. The complete report operation runs outside the actor thread, including path checks, directory creation, serialization, and file flushes. @@ -269,7 +276,7 @@ A valid summary permits a zero-message retention window. A nonempty extractive window remains valid when the observer fails. The observer prompt directs the model to preserve the current response format and each field's definition, conditions, and exceptions. The observer call uses the existing provider intent to suppress extra model analysis. -The provider adapter selects the supported wire representation. The sidecar timeout and task budgets do not change. +The provider adapter selects the supported wire representation. The sidecar operation timeout does not change. During compaction, tool call/result pairs must remain atomic. Never orphan a tool call from its result. Tool interactions older than the retention window diff --git a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md index b6dc00b7c..40dca749a 100644 --- a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md +++ b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md @@ -90,7 +90,9 @@ A start-tool timeout after acceptance does not cancel the accepted run. The parent explicitly loads `check_agent_run` for authorized status or cancellation. The owner commits cancellation admission before it reports that fact. The owner closes the run's local model, tool, and approval-retry admission gate. +It requests token cancellation and waits for any synchronous dispatch prefix to exit. The owner then commits the dispatch closure fact. + That committed fact proves that no new local task work can start. It does not prove that an earlier external effect stopped or that its outcome is known. @@ -100,6 +102,10 @@ It cannot request model output, run task tools, edit the project, call an extern A failed write or grace expiry preserves the last confirmed checkpoint and an explicit reason. Cancellation remains a failure outcome even when useful partial evidence exists. +The owner commits any unresolved child prompt disposition before the terminal receipt. +Token cancellation can resolve a prompt earlier. Durable closure does not require prompt resolution to commit first. +A late child terminal payload cannot replace the last committed checkpoint or supply a successful parent activity merge. + Positive example: cancellation returns confirmed file activity and an existing partial artifact without another model call. Negative example: late provider output or an approval answer cannot start task work after dispatch closure. diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md index 892585c54..122e3c7a8 100644 --- a/openspec/changes/background-child-runs/design.md +++ b/openspec/changes/background-child-runs/design.md @@ -208,7 +208,9 @@ Do not replace the owner with a probe that emits the desired acceptance or termi The parent accepts terminal messages only from the current recorded child generation. The first committed terminal receipt wins over later cancellation requests. If cancellation admission commits first, later child success cannot become the terminal outcome. -The parent can retain its confirmed evidence inside the cancelled result. +The cancelled result retains the last committed child checkpoint as partial evidence. +The owner ignores a late child terminal payload after cancellation admission. +That payload cannot replace the cancelled outcome or supply a successful parent activity merge. Actor mailbox arrival alone is insufficient: journal commit determines the winner. The child closes normal dispatch, sends one terminal result, and waits for the owner acknowledgement. @@ -227,6 +229,8 @@ Then it closes child work admission, cancels active calls, and settles run-owned Closure must cover pending thread-pool tool dispatch and queued provider requests, not just the actor mailbox. The child/runtime adapter sends an explicit dispatch-closure acknowledgement after no new task dispatch can start. The control response reports `dispatch_closed = false` until that acknowledgement exists. +The owner commits any unresolved child prompt disposition before it commits the terminal receipt. +Token cancellation can resolve a prompt earlier. Durable closure does not require prompt resolution to commit first. It must never describe cancellation admission alone as proof of stopped dispatch. Once closure is acknowledged, no fresh model request, tool launch, or approval retry can start for that run. diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md index 01ab42bf1..7b0d533a6 100644 --- a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -56,6 +56,10 @@ Direct slash adapters SHALL use the same committed owner values for their accept Their human prose may differ. Their identifiers, state, and control name SHALL NOT differ. An acceptance body SHALL NOT contain a child's final output or claim successful task completion. The later terminal result SHALL remain a distinct owner delivery under its fresh result correlation. +The adapter SHALL validate the typed owner reply before it records success or exposes acceptance. +That validation SHALL match the prepared digest and agent, coherent recorded state, and valid owner identifiers. +The stamped start wrapper SHALL also match the original full context, start key, and source operation. +A valid duplicate MAY retain a different run ID from a new preparation proposal. #### Scenario: A held child produces one accepted tool body @@ -73,6 +77,27 @@ The later terminal result SHALL remain a distinct owner delivery under its fresh - **AND** it cannot infer acceptance or launch another child from human prose or child output - **AND** a failed acceptance write still creates no child model request or task effect +#### Scenario: A typed reply cannot replace the original requester + +- **GIVEN** a structurally valid owner reply with a foreign requester in its original context +- **WHEN** the actual pipeline or slash wrapper validates that reply against its stamped start +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** no child text or later parent context supplies replacement authority + +#### Scenario: A typed reply cannot replace the stamped start key + +- **GIVEN** a structurally valid owner reply with a different original tool call or slash input key +- **WHEN** the actual pipeline or slash wrapper validates that reply +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** it does not adopt the reply as the requested run + +#### Scenario: A typed reply cannot replace the source operation + +- **GIVEN** a structurally valid owner reply that identifies another source operation +- **WHEN** the actual pipeline or slash wrapper validates that reply +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** it retains the original activation attribution + ### Requirement: Start retries preserve one durable run identity The runtime SHALL stamp start correlation from the original admitted input and call context. @@ -104,6 +129,14 @@ A conflicting digest under an accepted key SHALL fail explicitly and SHALL NOT a - **THEN** both callers receive the same run ID - **AND** exactly one child starts +#### Scenario: A valid current-state duplicate differs from the new proposal + +- **GIVEN** the original start key and canonical digest identify a recorded `Running` run +- **AND** preparation proposes a new run ID for an equivalent retry +- **WHEN** the owner returns the existing coherent run +- **THEN** the adapter accepts the existing run ID and `Running` state +- **AND** it creates no second child or requirement that both run IDs match + ### Requirement: The run lifetime is independent of its start call After acceptance commits, the child SHALL use a run-owned lifetime. @@ -236,7 +269,9 @@ The result SHALL distinguish useful partial evidence from successful task comple The owner SHALL accept terminal messages only from the recorded child generation. The first committed terminal receipt SHALL win over later cancellation admission. Cancellation admission that commits first SHALL prevent later success from becoming the terminal outcome. -The owner SHALL preserve safe evidence from a late child response without changing the cancelled outcome. +Cancellation SHALL retain the last committed child checkpoint as partial evidence. +The owner SHALL reject a late child terminal response as a replacement outcome after cancellation admission. +It SHALL NOT merge that late response as successful parent activity. The child SHALL close task dispatch and retain its result until the owner acknowledges durable terminal receipt. Duplicate terminal messages SHALL receive the same acknowledgement without another delivery admission. Persistence failure SHALL NOT produce a successful terminal acknowledgement. @@ -253,7 +288,7 @@ Persistence failure SHALL NOT produce a successful terminal acknowledgement. - **GIVEN** cancellation admission commits before a success receipt - **WHEN** the late success arrives - **THEN** the terminal outcome remains cancelled -- **AND** safe recorded evidence can remain available as partial evidence +- **AND** the last committed child checkpoint remains available as partial evidence #### Scenario: Terminal receipt persistence fails diff --git a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md index 2191107ee..7e9df778d 100644 --- a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md +++ b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md @@ -118,3 +118,84 @@ Healthy task duration alone SHALL NOT cause terminal timeout. - **WHEN** operation inactivity would otherwise expire - **THEN** the legitimate approval wait remains open - **AND** explicit run cancellation can still settle it + +### Requirement: Configurable subagent timeouts + +The system SHALL use resolved subagent profile values for operation-health deadlines. +`SubAgentProfile.TimeoutSeconds` SHALL control inactivity between model deltas after substantive output starts. +`SubAgentProfile.PrefillTimeoutSeconds` SHALL override the shared wait for initial output when present. +When absent, it SHALL inherit `SubAgentConfig.PrefillTimeoutSeconds` from the `SubAgents` section of `netclaw.json`. +`SubAgentConfig.NoProgressTimeoutSeconds` SHALL bound time without substantive model output. +Content-free keepalives SHALL NOT refresh that no-progress deadline. + +These values SHALL NOT define a wall-clock task-lifetime limit. +The file definition loader SHALL reject profile `timeoutSeconds` outside 5–600 seconds. +It SHALL reject profile `prefillTimeoutSeconds` outside 5–3600 seconds. +This change SHALL NOT migrate configuration fields or alter their current defaults. + +#### Scenario: Custom timeout from configuration + +- **GIVEN** a valid agent profile has `timeoutSeconds: 300` +- **WHEN** the spawner prepares that profile for the actual child actor +- **THEN** the child uses 300 seconds for model inactivity after substantive output starts +- **AND** healthy operations can continue beyond 300 seconds of total task duration + +#### Scenario: Missing config section uses defaults + +- **GIVEN** configuration omits the `SubAgents` section and the profile omits timeout overrides +- **WHEN** the runtime resolves the child operation-health values +- **THEN** it uses a 60-second profile inactivity value and a 1800-second initial-output value +- **AND** its no-progress value is 1200 seconds +- **AND** none of those values defines the child's total task duration + +#### Scenario: Invalid profile timeout is rejected before execution + +- **GIVEN** a file agent definition has `timeoutSeconds: -1` +- **WHEN** the definition loader validates that profile +- **THEN** it rejects that definition before a child can execute +- **AND** it does not replace the invalid value with a default + +#### Scenario: Invalid timeout rejected by doctor + +- **GIVEN** `netclaw.json` contains `"SubAgents": { "PrefillTimeoutSeconds": -1 }` +- **WHEN** the operator runs `netclaw doctor` +- **THEN** doctor reports a validation error for the timeout value + +### Requirement: Subagent observability events + +`LlmSessionActor` SHALL emit structured `SubAgentOutput` events to session subscribers for accepted child runs. +It SHALL emit `Started` only after the durable `Started` record commits. +It SHALL emit `Completed` only after the durable terminal receipt commits. +These events SHALL retain the `OutputFilter.ToolCalls` category and their existing payload fields. +Start-tool acceptance SHALL NOT substitute for the child terminal event. + +The owner SHALL NOT require the completed start tool's activity callback to deliver later child events. +Existing headless CLI and channel render rules SHALL remain unchanged. + +#### Scenario: Subagent start event emitted + +- **GIVEN** an accepted child start within the session owner +- **WHEN** the owner commits `Started` +- **THEN** it emits one `SubAgentOutput` with `Phase = Started` +- **AND** that event includes the agent name and tool count +- **AND** subscribers receive it under `OutputFilter.ToolCalls` + +#### Scenario: Subagent completion event emitted + +- **GIVEN** a child finishes after its start tool returns acceptance +- **WHEN** the owner commits its terminal receipt +- **THEN** it emits one `SubAgentOutput` with `Phase = Completed` +- **AND** that event includes the existing success status, duration, and outcome fields + +#### Scenario: Headless CLI renders subagent events + +- **GIVEN** the headless CLI subscribes with `OutputFilter.Full` +- **WHEN** a subagent starts and completes +- **THEN** the CLI renders `[subagent:start] ( tools)` +- **AND** it renders `[subagent:done] (, )` + +#### Scenario: Slack adapter suppresses subagent events + +- **GIVEN** the Slack adapter subscribes to session output +- **WHEN** a subagent starts and completes +- **THEN** no subagent-specific messages are posted to Slack diff --git a/openspec/changes/background-child-runs/specs/session-state-machine/spec.md b/openspec/changes/background-child-runs/specs/session-state-machine/spec.md index d4f0766f8..68992b47e 100644 --- a/openspec/changes/background-child-runs/specs/session-state-machine/spec.md +++ b/openspec/changes/background-child-runs/specs/session-state-machine/spec.md @@ -110,3 +110,11 @@ Once restart-drain reaches `Passivating`, new user messages and approval prompt - **WHEN** the owner closes dispatch and finalization reaches its bounded terminal result - **THEN** that cancelled result and prompt disposition remain durable - **AND** recovery cannot automatically relaunch the child + +#### Scenario: Coordinated restart settles an unanswered child approval prompt + +- **GIVEN** a real accepted child waits for its original requester's unanswered approval prompt +- **WHEN** coordinated drain cancels the child and commits its terminal receipt +- **THEN** the final snapshot retains that run's `Cancelled` outcome and exact prompt correlation +- **AND** the prompt resolution is durable `Denied` before the terminal receipt +- **AND** no child retry or reusable authorization grant results From 821b6d90823de13f7428ca2d962835315183b9cc Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 23:14:49 +0000 Subject: [PATCH 08/22] Sync background child run specifications --- docs/architecture/tool-authorization.md | 2 +- .../specs/netclaw-subagents/spec.md | 24 + .../specs/background-subagent-runs/spec.md | 534 ++++++++++++++++++ openspec/specs/netclaw-session/spec.md | 34 ++ openspec/specs/netclaw-subagents/spec.md | 220 ++++---- .../specs/progressive-tool-disclosure/spec.md | 30 + openspec/specs/session-state-machine/spec.md | 92 ++- .../specs/skill-execution-routing/spec.md | 45 ++ openspec/specs/tool-authorization/spec.md | 88 ++- 9 files changed, 921 insertions(+), 148 deletions(-) create mode 100644 openspec/specs/background-subagent-runs/spec.md diff --git a/docs/architecture/tool-authorization.md b/docs/architecture/tool-authorization.md index 66ed9cc6b..859d099d4 100644 --- a/docs/architecture/tool-authorization.md +++ b/docs/architecture/tool-authorization.md @@ -733,7 +733,7 @@ Leaks today: | Published contract | Output `ToolInteractionRequest` and input `ToolInteractionResponse` ([`SessionProtocol.Outputs.cs`](../../src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs)). Journal events `ToolApprovalRequested` and `ToolApprovalResolved`. Option key strings such as `approve_once`. | | Must not know | Policy rules. It renders the options that Consent offers. | | Data | Durable session journal for prompt lifecycle and original requester facts. A child execution waiter remains run-local. | -| Rules | [TA-10](../../openspec/specs/tool-authorization/spec.md#requirement-ta-10-consent-prompts-offer-only-safe-options), [TA-11](../../openspec/specs/tool-authorization/spec.md#requirement-ta-11-an-unanswered-consent-request-survives-restart), [TA-12](../../openspec/specs/tool-authorization/spec.md#requirement-ta-12-subagent-consent-goes-through-the-parent-session) | +| Rules | [TA-10](../../openspec/specs/tool-authorization/spec.md#requirement-ta-10-consent-prompts-offer-only-safe-options), [TA-11](../../openspec/specs/tool-authorization/spec.md#requirement-ta-11-an-unanswered-consent-request-survives-restart), [TA-12](../../openspec/specs/tool-authorization/spec.md#requirement-ta-12-subagent-approval-prompts-go-through-the-parent-session) | Leaks today: diff --git a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md index 7e9df778d..08500946f 100644 --- a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md +++ b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md @@ -199,3 +199,27 @@ Existing headless CLI and channel render rules SHALL remain unchanged. - **GIVEN** the Slack adapter subscribes to session output - **WHEN** a subagent starts and completes - **THEN** no subagent-specific messages are posted to Slack + +### Requirement: Sub-agent watchdog pauses during human approval + +The sub-agent inactivity watchdog SHALL treat parent approval waits as intentional suspension. While one or more approval waits are active, watchdog timeout ticks SHALL NOT complete the sub-agent as inactive. When the last approval wait settles, the watchdog SHALL be re-baselined so future inactivity is still bounded. + +#### Scenario: Slow approval does not trigger inactivity timeout +- **GIVEN** a sub-agent with an active approval wait +- **AND** the human approval decision takes longer than the sub-agent inactivity budget +- **WHEN** the approval eventually arrives +- **THEN** the sub-agent applies the approval outcome +- **AND** the sub-agent is not failed for inactivity during the wait + +#### Scenario: Parent start completes before child approval +- **GIVEN** the parent start call returns durable child acceptance +- **AND** the child waits for human approval +- **WHEN** the parent performs independent authorized work +- **THEN** the child approval wait remains open and cancellable +- **AND** the completed parent start call has no watchdog dependency on that wait + +#### Scenario: Parallel approval waits keep watchdog paused until all settle +- **GIVEN** a sub-agent tool batch with two approval-gated calls +- **WHEN** both calls are waiting for parent approval +- **THEN** the watchdog remains paused until both approval waits have settled +- **AND** the watchdog is re-armed only after the final wait completes diff --git a/openspec/specs/background-subagent-runs/spec.md b/openspec/specs/background-subagent-runs/spec.md new file mode 100644 index 000000000..7905150fc --- /dev/null +++ b/openspec/specs/background-subagent-runs/spec.md @@ -0,0 +1,534 @@ +# background-subagent-runs Specification + +## Purpose + +Define durable acceptance, parent controls, later child results, cancellation, and explicit restart loss for background child tasks. +Use the [engineering glossary](../../../docs/spec/GLOSSARY.md) for shared terms. +Source requirements: PRD-001 FR-001, FR-002, FR-003, FR-005, FR-007, FR-011, and FR-017; PRD-008 scheduled skill execution. + +## Requirements + +### Requirement: All child starts return durable acceptance + +Every explicit or routed child start SHALL return a machine-actionable accepted run identifier before the child completes. +Acceptance SHALL commit before child execution or the accepted response. +The response SHALL include the run ID, child scope ID, lifecycle state, and the deferred control tool name. +An accepted response SHALL NOT claim that the task succeeded or expose a final result before receipt. +Pre-admission failure SHALL return an explicit rejected start and create no child. +Post-admission failure SHALL retain the accepted identifier and produce a terminal failure for that run. + +#### Scenario: Parent receives acceptance while the child remains held + +- **GIVEN** an authorized child start and a child provider barrier +- **WHEN** the owner commits acceptance and the child reaches that barrier +- **THEN** the start call returns its accepted identifier before the barrier releases +- **AND** the parent can leave its tool phase and process later input + +#### Scenario: Acceptance persistence fails + +- **GIVEN** a prepared child start +- **WHEN** acceptance persistence fails +- **THEN** no child model or task tool executes +- **AND** the caller receives an explicit failure rather than a false accepted identifier + +#### Scenario: Child creation fails after acceptance + +- **GIVEN** acceptance commits before child creation +- **WHEN** child creation fails +- **THEN** the accepted run remains queryable +- **AND** one later terminal result describes the start failure + +### Requirement: Tool acceptance has one canonical JSON representation + +The tool acceptance body SHALL be one JSON object without a prose prefix, Markdown fence, or suffix. +The object SHALL contain `run_id`, `scope_id`, `state`, and `control_tool` as JSON strings. +Each required field SHALL occur exactly once. +The owner SHALL supply nonempty canonical run and scope identifier values from its committed acceptance record. +The identifier strings SHALL use `SubAgentRunId.Value` and `SubAgentScopeId.Value` from that record. +The first accepted response SHALL use `state: "Accepted"` and `control_tool: "check_agent_run"`. +That initial state SHALL describe the committed admission fact, rather than claim that the current live state cannot advance. +The adapter SHALL NOT derive identifiers or acceptance from child text, actor names, or an uncommitted proposal. +An equivalent start retry SHALL retain those identifiers and may report the owner's current recorded lifecycle state. +JSON field order and insignificant whitespace SHALL NOT change the contract. + +`spawn_agent` and routed `skill_load` SHALL expose this body as their tool result string. +`ToolResultOutput.Result` and the corresponding model `FunctionResultContent.Result` string SHALL carry the same JSON object. +The adapter SHALL format that body once and reuse it for both consumers. +Direct slash adapters SHALL use the same committed owner values for their acceptance acknowledgement. +Their human prose may differ. Their identifiers, state, and control name SHALL NOT differ. +An acceptance body SHALL NOT contain a child's final output or claim successful task completion. +The later terminal result SHALL remain a distinct owner delivery under its fresh result correlation. +The adapter SHALL validate the typed owner reply before it records success or exposes acceptance. +That validation SHALL match the prepared digest and agent, coherent recorded state, and valid owner identifiers. +The stamped start wrapper SHALL also match the original full context, start key, and source operation. +A valid duplicate MAY retain a different run ID from a new preparation proposal. + +#### Scenario: A held child produces one accepted tool body + +- **GIVEN** the owner commits acceptance and the child provider remains held +- **WHEN** the start adapter returns its tool result +- **THEN** the output event and model result contain the same JSON object +- **AND** its first state is `Accepted` with canonical nonempty owner identifiers and the exact control name +- **AND** the parent does not receive the child's final output as that acceptance + +#### Scenario: A malformed or uncommitted body is not acceptance + +- **GIVEN** a result has a missing, duplicate, non-string, or empty required field, or no committed owner acceptance +- **WHEN** a start consumer validates that result +- **THEN** it reports an invalid acceptance rather than a live accepted run +- **AND** it cannot infer acceptance or launch another child from human prose or child output +- **AND** a failed acceptance write still creates no child model request or task effect + +#### Scenario: A typed reply cannot replace the original requester + +- **GIVEN** a structurally valid owner reply with a foreign requester in its original context +- **WHEN** the actual pipeline or slash wrapper validates that reply against its stamped start +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** no child text or later parent context supplies replacement authority + +#### Scenario: A typed reply cannot replace the stamped start key + +- **GIVEN** a structurally valid owner reply with a different original tool call or slash input key +- **WHEN** the actual pipeline or slash wrapper validates that reply +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** it does not adopt the reply as the requested run + +#### Scenario: A typed reply cannot replace the source operation + +- **GIVEN** a structurally valid owner reply that identifies another source operation +- **WHEN** the actual pipeline or slash wrapper validates that reply +- **THEN** it rejects the reply before successful receipt or visible acceptance +- **AND** it retains the original activation attribution + +### Requirement: Start retries preserve one durable run identity + +The runtime SHALL stamp start correlation from the original admitted input and call context. +A tool start key SHALL contain the owner session ID, original admitted turn ID, and original tool call ID. +A slash start key SHALL contain the owner session ID, admitted input ID, original turn ID, and framework activation slot zero. +A scheduled slash start SHALL reuse the admitted scheduled input and its persisted occurrence correlation. +The model SHALL NOT supply these keys or mint a replacement key during retry. +Concurrent or recovered duplicates with the same key and canonical start digest SHALL return the same run. +A conflicting digest under an accepted key SHALL fail explicitly and SHALL NOT alter that run. + +#### Scenario: A lost accepted response does not duplicate execution + +- **GIVEN** the owner commits acceptance but the response never reaches the caller +- **WHEN** the runtime retries the same stamped start +- **THEN** it returns the existing run ID +- **AND** no second child or external task effect starts + +#### Scenario: Conflicting retry fails without mutation + +- **GIVEN** one key belongs to an accepted task +- **WHEN** a retry reuses that key with different task or resolved profile content +- **THEN** the runtime returns `start_conflict` +- **AND** the accepted task and its authority remain unchanged + +#### Scenario: Two concurrent duplicates share acceptance + +- **GIVEN** two equivalent start requests arrive before the acceptance write completes +- **WHEN** the write commits +- **THEN** both callers receive the same run ID +- **AND** exactly one child starts + +#### Scenario: A valid current-state duplicate differs from the new proposal + +- **GIVEN** the original start key and canonical digest identify a recorded `Running` run +- **AND** preparation proposes a new run ID for an equivalent retry +- **WHEN** the owner returns the existing coherent run +- **THEN** the adapter accepts the existing run ID and `Running` state +- **AND** it creates no second child or requirement that both run IDs match + +### Requirement: The run lifetime is independent of its start call + +After acceptance commits, the child SHALL use a run-owned lifetime. +Start-call completion, cancellation, or timeout SHALL NOT cancel that accepted run. +A later parent turn SHALL NOT replace the run's original authority or lifetime. +Explicit run cancellation, explicit parent stop, and owner loss SHALL end that lifetime through the defined terminal contract. +No task-lifetime time, token, or tool-count ceiling SHALL be introduced by this capability. +Existing per-operation health checks and the shared recurrence contract SHALL remain effective. + +#### Scenario: Start-tool cancellation after acceptance leaves the child live + +- **GIVEN** the owner committed a child acceptance +- **WHEN** the original start token cancels or its tool response times out +- **THEN** the child remains live until a run-level terminal boundary +- **AND** parent status and cancellation remain available + +#### Scenario: A later user turn does not cancel a child + +- **GIVEN** a live child that belongs to an earlier parent turn +- **WHEN** the parent accepts a later authorized user turn +- **THEN** the child retains its original context and lifetime +- **AND** the parent does not wait for that child's completion merely because it exists + +### Requirement: Parent controls describe recorded state without widening authority + +`check_agent_run(run_id, cancel=false)` SHALL return the owner's recorded run state. +The parent caller SHALL pass normal tool policy and match the owner session and original eligible requester context. +A child or mismatched session/requester SHALL receive a denial without target details. +The control SHALL NOT route through the shell-job manager or use shell authority merely because it resembles shell-job cancellation. +Status SHALL expose cancellation admission separately from confirmed local dispatch closure. +Repeated status or cancellation requests SHALL NOT create another child, terminal result, or model continuation. +After authorization, status SHALL include the canonical `log_path` and `artifact_directory` before terminal completion. +The adapter SHALL obtain these paths from the existing child storage binding without a separate durable path authority. +The parent SHALL use ordinary file tools under normal file policy for diagnostic log access. +A returned path SHALL NOT grant file authority or prove current health, dispatch closure, or completion. + +#### Scenario: Original parent inspects its live run + +- **GIVEN** the original authorized parent requester and its accepted run +- **WHEN** the parent loads and calls the control +- **THEN** it receives the run ID, recorded state, and cancellation/dispatch-closure facts +- **AND** the call does not wait for task completion + +#### Scenario: Parent reads a live child's exact log path + +- **GIVEN** an accepted child remains held before terminal completion +- **WHEN** the original authorized requester calls the deferred status tool +- **THEN** the response includes the exact canonical child log path and artifact directory +- **AND** the parent can pass that path to ordinary file tools under normal file policy +- **AND** the response does not claim a terminal outcome from log contents + +#### Scenario: A returned path does not bypass file policy + +- **GIVEN** an authorized status response supplies a child log path +- **AND** current file policy denies the requested read +- **WHEN** the parent calls a file tool with that path +- **THEN** the file tool denies the read +- **AND** the status response creates no file grant + +#### Scenario: Another session cannot inspect or cancel the run + +- **GIVEN** another session supplies a valid run identifier +- **WHEN** it attempts status or cancellation +- **THEN** the runtime reveals no target state, paths, or requester details +- **AND** the target remains live + +### Requirement: Cancellation has a durable admission and a confirmed local cutover + +The owner SHALL persist cancellation admission before it reports `Cancelling`. +It SHALL then close normal task admission, cancel active requests, and settle run-owned approval prompts. +The closure SHALL include pending tool-dispatch workers and queued provider requests. +The runtime SHALL acknowledge dispatch closure only after no new model request, task tool, or approval retry can start. +An admission acknowledgement SHALL NOT claim that dispatch closure already occurred. +Already launched remote work that lacks a confirmed outcome SHALL remain explicitly unknown. + +#### Scenario: Cancellation admission precedes closure + +- **GIVEN** a live child with a held local dispatch worker +- **WHEN** cancellation admission commits before that worker acknowledges closure +- **THEN** status reports `Cancelling` with dispatch closure unconfirmed +- **AND** the runtime does not claim complete task termination + +#### Scenario: No fresh dispatch follows closure acknowledgement + +- **GIVEN** the runtime acknowledges local dispatch closure +- **WHEN** stale provider output or a late approval answer attempts more task work +- **THEN** no new tool, provider request, retry, or grant is created +- **AND** the recorded cancellation outcome remains effective + +#### Scenario: Remote work ignores cancellation + +- **GIVEN** a remote operation started before cancellation cutover +- **WHEN** that operation ignores cancellation and provides no final receipt +- **THEN** local settlement completes without waiting for remote cooperation +- **AND** the result identifies the unconfirmed external effect + +### Requirement: Cancellation preserves recorded partial evidence through framework-only finalization + +The runtime SHALL retain confirmed results and file activity during normal execution after each completed feedback round. +After local dispatch closure, finalization SHALL have a separate five-second grace deadline by default. +Finalization SHALL use a token independent of the cancelled task token. +It SHALL permit only framework-owned checkpoint preservation and atomic writes inside the assigned run artifact directory. +It SHALL NOT invoke a model, task tool, external request, project edit, or approval prompt action. +Grace expiry or report-write failure SHALL retain the last committed checkpoint and expose an explicit finalization reason. +The result SHALL distinguish useful partial evidence from successful task completion. + +#### Scenario: Cancellation returns useful recorded evidence + +- **GIVEN** a child completed useful file operations and recorded a checkpoint +- **WHEN** the parent cancels it +- **THEN** one cancelled terminal result retains that confirmed activity and valid artifact references +- **AND** no new model request occurs to produce the report + +#### Scenario: Report write fails + +- **GIVEN** cancellation finalization attempts an atomic run-local report write +- **WHEN** the write fails +- **THEN** the runtime returns retained checkpoint evidence with an explicit write-failure reason +- **AND** it does not expose a nonexistent report path + +#### Scenario: Finalization does not finish within grace + +- **GIVEN** a held framework finalization operation +- **WHEN** the five-second grace deadline expires +- **THEN** one cancelled terminal result uses the retained checkpoint +- **AND** late finalization cannot replace the terminal winner or start task work + +### Requirement: Terminal precedence follows durable owner order + +The owner SHALL accept terminal messages only from the recorded child generation. +The first committed terminal receipt SHALL win over later cancellation admission. +Cancellation admission that commits first SHALL prevent later success from becoming the terminal outcome. +Cancellation SHALL retain the last committed child checkpoint as partial evidence. +The owner SHALL reject a late child terminal response as a replacement outcome after cancellation admission. +It SHALL NOT merge that late response as successful parent activity. +The child SHALL close task dispatch and retain its result until the owner acknowledges durable terminal receipt. +Duplicate terminal messages SHALL receive the same acknowledgement without another delivery admission. +Persistence failure SHALL NOT produce a successful terminal acknowledgement. + +#### Scenario: Completion wins the durable race + +- **GIVEN** a valid completion receipt commits before cancellation admission +- **WHEN** a later cancel request arrives +- **THEN** the completed outcome remains the terminal winner +- **AND** the cancel request does not create a second terminal result + +#### Scenario: Cancellation wins the durable race + +- **GIVEN** cancellation admission commits before a success receipt +- **WHEN** the late success arrives +- **THEN** the terminal outcome remains cancelled +- **AND** the last committed child checkpoint remains available as partial evidence + +#### Scenario: Terminal receipt persistence fails + +- **GIVEN** a child waits for its terminal acknowledgement +- **WHEN** the owner's terminal persistence fails +- **THEN** no successful acknowledgement is emitted +- **AND** recovery or owner loss remains explicit rather than false delivery success + +### Requirement: Cancelled results preserve wire compatibility without false success + +A cancelled result SHALL keep `Success = false`, wire outcome `Failed`, and reason `CancelledByParent`. +Its recorded lifecycle state SHALL be `Cancelled`. +Its typed completion SHALL carry confirmed file activity without claiming successful completion. +The owner SHALL expose only validated run locations and safe result paths under ordinary path policy. +Child-authored path claims SHALL NOT supply access authority, an approval prompt answer, or unverified file activity. +A lost run SHALL have a failed wire outcome and an explicit owner-loss reason. + +#### Scenario: Cancelled evidence does not become successful output + +- **GIVEN** a cancelled child wrote a valid artifact before cutover +- **WHEN** the result is serialized and decoded +- **THEN** its cancellation state, failed wire outcome, and confirmed artifact survive +- **AND** no consumer interprets it as successful task completion + +#### Scenario: A forged result path supplies no authority + +- **GIVEN** a child result claims an unrelated protected path +- **WHEN** the owner validates the result and the parent attempts a read +- **THEN** the claim creates neither confirmed activity nor a path grant +- **AND** the ordinary read policy rejects unauthorized access + +### Requirement: One durable terminal delivery admits one attributed continuation + +Each run SHALL have one deterministic terminal delivery identifier. +The owner SHALL persist the terminal receipt before asynchronous enrichment. +It SHALL retain that receipt if enrichment fails and expose an explicit delivery warning. +Continuation admission SHALL atomically persist both its dedup marker and its input/transcript item. +Recovery SHALL restore an admitted unfinished continuation without admitting it again. +Completed continuation input SHALL NOT be reintroduced by duplicate result or recovery replay. +An absent persistence acknowledgement SHALL remain an explicit undelivered state. + +#### Scenario: Crash before continuation admission preserves the result + +- **GIVEN** the terminal receipt is durable but continuation admission is absent +- **WHEN** the owner recovers +- **THEN** it uses the recorded result and the same delivery identifier +- **AND** it admits one continuation without restarting the child + +#### Scenario: Crash after admission does not duplicate input + +- **GIVEN** continuation admission commits and its model turn remains unfinished +- **WHEN** the owner recovers +- **THEN** it restores that pending continuation +- **AND** it creates neither another admission nor another child start + +#### Scenario: Enrichment failure leaves a truthful result + +- **GIVEN** a durable terminal receipt +- **WHEN** result enrichment fails +- **THEN** run status retains the original outcome and reports the delivery warning +- **AND** any delivered framework report uses only validated recorded facts + +### Requirement: Later results remain tool-origin content under original authority + +The later result SHALL use the original accepted run context and fresh framework delivery correlation. +It SHALL NOT attach a second result to the completed original tool call ID. +Provider messages SHALL contain a valid attributed synthetic tool-call/result pair without redispatch of the start or control operation. +The pair SHALL represent framework result delivery, not a new tool request or authorization receipt. +Child text SHALL NOT become trusted automation input, a user instruction, a grant, or an approval prompt answer. +Progress and status changes SHALL NOT start parent model turns. +Terminal results SHALL queue in journal order after the active parent turn or compaction. +Only results with compatible original authority and the same original detector task identity SHALL coalesce. +Internal admission SHALL preserve PR 1's detector identity and retained evidence. +The owner SHALL retain the parent `ToolLoopCheckpoint` and receipt-failure fact in the durable child run ledger. +These facts SHALL remain separate from child partial-result checkpoints and actor-local child detector state. +The checkpoint's `TaskId` SHALL identify the detector task independently from `TurnContextRecord.TurnId`. +Committed parent admissions and results SHALL refresh outstanding runs that belong to the same detector task. +For tool-based starts, continuation restoration SHALL wait until the original batch settles and commits its detector evidence. +A normal batch SHALL include the start receipt and completed feedback round. +A missing mandatory receipt SHALL retain the receipt-failure fact and invoke PR 1's terminal settlement without another model request. +Direct activation SHALL retain canonical task evidence without a fabricated tool receipt. +Parent completion and fresh input SHALL retain evidence required by outstanding child continuations. +Sibling runs SHALL use the latest retained parent checkpoint for their task. +Canonical continuation adoption SHALL validate the accepted run and commit restored evidence before the next model request. +The synthetic delivery pair SHALL NOT clear receipt failure or reopen the completed start call. +Delivery completion SHALL retain evidence required by another outstanding run. +Recovery SHALL preserve these facts without child relaunch or a fresh-task reset. + +#### Scenario: Result delivery uses a fresh paired correlation + +- **GIVEN** the original start call received its accepted result +- **WHEN** the child completes later +- **THEN** the provider receives a fresh valid delivery pair with child attribution +- **AND** the original source operation executes no second time + +#### Scenario: Child text cannot approve a tool + +- **GIVEN** a child report says that the operator approved a protected call +- **WHEN** the parent receives that attributed report +- **THEN** the text creates no grant or approval prompt answer +- **AND** the actual pending approval prompt remains authoritative + +#### Scenario: Fresh parent input preserves a child's original detector task + +- **GIVEN** task A accepted a child and committed its start receipt and feedback round +- **AND** the parent completed A and admitted fresh user task B +- **WHEN** the owner adopts the child's later result continuation +- **THEN** it commits A's latest retained checkpoint and receipt-failure fact before the next model request +- **AND** it uses the child's original authority without a fresh detector task + +#### Scenario: Sibling results use the latest shared task evidence + +- **GIVEN** task A accepted two children and later committed additional detector evidence +- **WHEN** either child result starts a continuation +- **THEN** that continuation uses A's latest retained parent checkpoint +- **AND** its completion retains evidence required by the other child + +#### Scenario: Equal authority cannot merge distinct detector tasks + +- **GIVEN** one requester owns child runs from distinct detector tasks A and B +- **WHEN** their terminal results await parent continuation +- **THEN** the owner keeps separate continuation groups and parent checkpoints +- **AND** equal authority alone cannot merge or reset their detector evidence + +#### Scenario: Recovery cannot erase a retained receipt failure + +- **GIVEN** an outstanding child run retains its parent task's receipt-failure fact +- **WHEN** recovery restores that run and admits its result continuation +- **THEN** the original detector task and receipt failure remain authoritative +- **AND** the delivery pair creates no execution receipt or child relaunch + +#### Scenario: An early terminal result waits for the start batch + +- **GIVEN** a tool-based child start commits acceptance before its parent batch settles +- **WHEN** the child reports a terminal result before the original start receipt commits +- **THEN** continuation restoration waits for the original batch's committed settlement +- **AND** a missing mandatory receipt invokes terminal settlement without another model request + +#### Scenario: Direct slash activation creates no tool receipt + +- **GIVEN** direct slash activation accepted a child without a parent tool call +- **WHEN** its terminal result admits a continuation +- **THEN** the owner retains the canonical original detector evidence +- **AND** it fabricates no start receipt or completed tool feedback round + +#### Scenario: A different speaker does not replace original authority + +- **GIVEN** a child belongs to one requester and another speaker later joins the parent session +- **WHEN** the child result starts its continuation +- **THEN** the continuation uses the original run context +- **AND** the later speaker supplies no inherited authority for that run + +### Requirement: Child approval prompt lifetime belongs to the run + +The owner SHALL retain each child approval prompt's original requester, exact call, authorization attempt, and run correlation. +It SHALL persist prompt lifecycle facts before display without persisting live waiters or run-local retry authorization objects. +Start-call completion or a later ordinary parent message SHALL NOT abandon the child prompt. +An accepted answer SHALL still pass exact retry authorization and run-liveness checks. +Cancellation, terminal loss, or recovery of an interrupted child SHALL visibly settle or expire its prompt. +A stale answer SHALL execute no tool and create no grant. + +#### Scenario: Original requester answers after the parent start turn ends + +- **GIVEN** a live child owns an unanswered approval prompt +- **WHEN** its original eligible requester answers during a later parent turn +- **THEN** the exact child call can resume under its original context +- **AND** unrelated parent input remains independent of that wait + +#### Scenario: Answer after cancellation cannot grant access + +- **GIVEN** cancellation admission settled the child prompt +- **WHEN** an approval answer arrives late +- **THEN** the operator sees an expired or cancelled disposition +- **AND** no task tool, one-time retry, or persistent grant is created + +### Requirement: Recovery reports loss without child replay + +Journal and snapshot recovery SHALL preserve start keys, original contexts, checkpoints, terminal receipts, and delivery admission markers. +An accepted run without a durable terminal receipt SHALL become `Lost` once after owner recovery. +A committed cancellation SHALL remain cancelled with its retained evidence. +Recorded terminal delivery SHALL resume from its recorded phase without child relaunch. +Recovered child prompts SHALL expire visibly. +Recovery SHALL NOT replay child tools, infer completion from text, or reconstruct child execution with the current requester. + +#### Scenario: Accepted child becomes lost after abrupt restart + +- **GIVEN** an accepted live child without a durable terminal receipt +- **WHEN** the owner recovers after abrupt restart +- **THEN** it records one explicit lost terminal outcome +- **AND** no replacement child or task side effect starts automatically + +#### Scenario: Cancellation remains effective after restart + +- **GIVEN** cancellation admission committed before restart +- **WHEN** the owner recovers without a child finalization response +- **THEN** it settles a cancelled result from retained evidence +- **AND** it cannot turn a stale success into the terminal outcome + +### Requirement: Upgrade and rollback preserve explicit lifecycle limits + +The candidate SHALL load captured pre-change journals and snapshots without fabricating accepted background runs. +It SHALL retain existing conversations, parent approval state, and session storage bindings. +Interrupted legacy synchronous calls SHALL use explicit loss/healing behavior rather than background relaunch. +Rollback SHALL either prove prior-reader compatibility for new records or require the pre-upgrade database backup. +No rollback procedure SHALL claim reversal of completed external effects. + +#### Scenario: Captured old state remains readable + +- **GIVEN** a pre-change snapshot and journal with parent approval and conversation state +- **WHEN** the candidate loads that captured state +- **THEN** it preserves those existing records and uses an explicit empty child ledger +- **AND** it infers no live child from old transcript text + +#### Scenario: The previous binary cannot read a new record + +- **GIVEN** a compatibility probe demonstrates that the previous binary rejects a new record +- **WHEN** rollback instructions are reviewed +- **THEN** they require the pre-upgrade database backup and active-run settlement +- **AND** they retain the warning that external effects remain + +### Requirement: Parent and child model requests remain independent + +The parent SHALL issue its model request without waiting for a live child's completion or a Netclaw provider-slot gate. +The inference backend SHALL own its request queues and capacity. +Netclaw SHALL NOT add provider-slot limits, capacity reservations, or parent/child inference preemption. +Status and cancellation SHALL remain locally usable while a child request remains pending. +An actor acknowledgement SHALL NOT substitute for required real-model evidence. + +#### Scenario: A held child does not block the parent request + +- **GIVEN** a live child with a held model request +- **WHEN** the user submits a parent question +- **THEN** the parent's model request reaches the backend before child release +- **AND** Netclaw does not wait for or allocate a provider slot + +#### Scenario: The backend owns a queued request + +- **GIVEN** the backend queues the parent's request behind a child request +- **WHEN** Netclaw awaits the parent response +- **THEN** Netclaw does not reserve capacity or preempt the child to change that order +- **AND** local status and cancellation remain available diff --git a/openspec/specs/netclaw-session/spec.md b/openspec/specs/netclaw-session/spec.md index fd28ffb7d..46f8ecf0f 100644 --- a/openspec/specs/netclaw-session/spec.md +++ b/openspec/specs/netclaw-session/spec.md @@ -1062,3 +1062,37 @@ request. - **WHEN** it learns a session path - **THEN** the storage layout does not claim to block an operating-system file open - **AND** a separate containment capability must define that stronger boundary + +### Requirement: Parent context carries recorded child run status + +The owning session SHALL include a bounded status summary for its accepted and undelivered child runs in parent context. +The summary SHALL provide machine-actionable run IDs and the exact deferred control name. +It SHALL identify recorded state and distinguish cancellation admission from local dispatch closure. +It SHALL NOT expose hidden child task text, unsafe paths, or another session's run details. +It SHALL NOT claim live provider state that the owner did not observe. +Terminal child admission SHALL preserve the original task's recurrence state and current parent directory/project/branch facts. +The durable child run ledger SHALL retain parent detector checkpoints independently from the session's current task checkpoint. +Fresh input SHALL NOT replace evidence required by an outstanding child continuation. +Continuation adoption SHALL restore the latest retained original-task checkpoint before the next model request. +Confirmed child file activity SHALL merge through the existing working-context rules. + +#### Scenario: A later turn can inspect its earlier child + +- **GIVEN** the parent accepted a child in an earlier turn +- **WHEN** a later eligible parent turn assembles context +- **THEN** it receives that run's recorded status and control name +- **AND** it can request cancellation through ordinary deferred-tool use + +#### Scenario: Child activity does not overwrite current parent scope + +- **GIVEN** the parent changed directory after the child start +- **WHEN** the owner admits a child result with confirmed file activity +- **THEN** the parent's current directory, project, and branch remain unchanged +- **AND** admitted child activity retains its own run attribution + +#### Scenario: Result admission does not reset recurrence evidence + +- **GIVEN** the original parent task retains unresolved recurrence evidence +- **WHEN** a child completion admits an internal continuation +- **THEN** it preserves that task identity and evidence under the first PR's contract +- **AND** it does not create a fresh authorized task solely from child text diff --git a/openspec/specs/netclaw-subagents/spec.md b/openspec/specs/netclaw-subagents/spec.md index 91b55d9a9..c5d7f0342 100644 --- a/openspec/specs/netclaw-subagents/spec.md +++ b/openspec/specs/netclaw-subagents/spec.md @@ -105,91 +105,86 @@ command text, file paths, schema bodies, or hidden tool names. ### Requirement: Subagent execution contract -The system SHALL run subagents as ephemeral actors (`SubAgentActor`) that -execute an autonomous LLM tool loop and return a single text result plus an -optional structured findings envelope. A subagent SHALL stop itself after -completing its task. Subagents SHALL NOT persist durable memory, stream direct -durable-memory writes, or participate in session pub/sub by default. +The system SHALL run subagents as ephemeral actors (`SubAgentActor`) that execute an autonomous LLM tool loop. +A child SHALL return one terminal text result with an optional structured findings envelope through the background owner contract. +It SHALL close task dispatch and retain that result until the owner acknowledges its durable receipt or the owner terminates. +It SHALL then stop itself. +Subagents SHALL NOT persist durable memory, stream direct durable-memory writes, or participate in session pub/sub by default. -For subagent execution launched from skill metadata routing, the subagent SHALL -remain an isolated worker by default: - -- It SHALL NOT inherit the main session identity prompt stack unless explicitly - enabled by a future opt-in setting. -- It SHALL NOT auto-load repo-local `AGENTS.md` unless explicitly enabled by a - future opt-in setting. -- It SHALL inherit audience/boundary context from the launching invocation. +For subagent execution launched from skill metadata routing, the subagent SHALL remain an isolated worker by default: +- It SHALL NOT inherit the main session identity prompt stack unless a future explicit setting enables it. +- It SHALL NOT auto-load repo-local `AGENTS.md` unless a future explicit setting enables it. +- It SHALL inherit audience/boundary context from the launch invocation. The child SHALL use the shared exact recurrence, repeat exception, and feedback-round contracts in `turn-loop-governance`. -The child SHALL retain its detector evidence for its actor-local task lifetime. +It SHALL retain its detector evidence for its actor-local task lifetime. It SHALL NOT substitute success for an absent receipt or create a durable child-resumption contract. +All explicit and routed starts SHALL use `background-subagent-runs` acceptance, lifetime, and later-result rules. #### Scenario: Subagent completes with text response and findings - **GIVEN** a `SubAgentDefinition` with a name, system prompt, and tool list -- **WHEN** the subagent receives a `RunSubAgent` message -- **THEN** the subagent executes its LLM/tool loop and returns a `SubAgentResult` -- **AND** the result MAY include structured findings for the parent session to - review -- **AND** stops itself +- **WHEN** the child finishes its task +- **THEN** it returns a `SubAgentResult` to its owner +- **AND** optional findings remain available for parent review +- **AND** the child stops after durable receipt acknowledgement or owner termination #### Scenario: Subagent executes tool calls in a loop -- **GIVEN** the LLM returns `FunctionCallContent` tool calls -- **WHEN** the subagent processes the response -- **THEN** it executes the tool calls via `DispatchingToolExecutor` -- **AND** sends tool results back to the LLM -- **AND** continues until a text response or an explicit runtime terminal decision +- **GIVEN** the model returns `FunctionCallContent` tool calls +- **WHEN** the child processes that response +- **THEN** eligible calls use the normal dispatch and authorization contracts +- **AND** their paired results return to the model +- **AND** the loop continues until text completion or an explicit runtime terminal decision #### Scenario: Subagent hits maximum tool iterations -- **GIVEN** a child completed more than the former tool iteration limit with useful work -- **WHEN** the model requests another eligible tool call -- **THEN** the removed iteration ceiling does not force a final model request -- **AND** the child continues under the shared recurrence and operation contracts +- **GIVEN** a child exceeds the former tool iteration limit through useful work +- **WHEN** it requests another eligible call +- **THEN** no removed iteration ceiling forces a final model request +- **AND** operation health and recurrence rules remain effective #### Scenario: Useful child work exceeds the former ceiling -- **GIVEN** a child task requires more than 30 useful tool rounds -- **WHEN** a deterministic provider drives that task through the real child loop -- **THEN** no tool count or iteration count stops the task -- **AND** independent artifact checks prove the required result +- **GIVEN** a child task needs more than 30 useful feedback rounds +- **WHEN** a deterministic provider drives that task through real dispatch +- **THEN** no count-based stop ends the task +- **AND** independent artifact checks prove its required result #### Scenario: Child recurrence settles without another model response -- **GIVEN** two equal completed call outcomes and a subsequent paired correction -- **WHEN** the child requests the same prohibited identity again -- **THEN** the child returns a Partial result with `ToolCycleStopped` -- **AND** it retains confirmed file activity and accurate partial evidence +- **GIVEN** the shared recurrence contract requires terminal settlement +- **WHEN** the child settles that run +- **THEN** it returns `Partial` with `ToolCycleStopped` and confirmed activity - **AND** it requests no final model response +- **AND** the owner persists and delivers that terminal result through the background contract #### Scenario: Default subagent cannot write durable memory directly -- **GIVEN** a default subagent is executing within a user-facing session -- **WHEN** it attempts to persist durable cross-session memory directly -- **THEN** the durable write path is unavailable or denied to that subagent -- **AND** the subagent must return findings to the parent session instead +- **GIVEN** a default child executes within a user-facing session +- **WHEN** it attempts a direct durable cross-session memory write +- **THEN** that path remains unavailable or denied +- **AND** the child returns findings for parent policy review instead #### Scenario: Routed subagent does not inherit main identity prompt stack -- **GIVEN** a slash-invoked skill routes execution via `metadata.subagent` -- **WHEN** the routed subagent prompt is assembled -- **THEN** the main session identity prompt stack is not included by default +- **GIVEN** a slash skill routes through `metadata.subagent` +- **WHEN** its isolated child prompt is assembled +- **THEN** the main session identity stack remains absent by default #### Scenario: Routed subagent does not auto-load repo AGENTS -- **GIVEN** a slash-invoked skill routes execution via `metadata.subagent` -- **WHEN** the routed subagent prompt is assembled -- **THEN** repo-local `AGENTS.md` is not auto-loaded by default +- **GIVEN** a slash skill routes through `metadata.subagent` +- **WHEN** its isolated child prompt is assembled +- **THEN** repo-local `AGENTS.md` remains absent by default #### Scenario: Routed subagent inherits launch audience -- **GIVEN** a routed subagent activation launched from a parent invocation with - audience `team` -- **WHEN** the subagent executes tool calls -- **THEN** tool execution context audience is `team` -- **AND** routed execution does not widen audience to a broader default +- **GIVEN** a routed launch has audience `Team` +- **WHEN** the child executes a task tool +- **THEN** that tool uses inherited `Team` authority +- **AND** background lifetime does not widen its audience ### Requirement: User-facing target validation for routed skill execution @@ -238,79 +233,114 @@ into checkpoints, and remain the default durable-memory owner. ### Requirement: Subagent timeout enforcement -The system SHALL enforce a wall-clock timeout on subagent execution. When the -timeout fires, the subagent SHALL return a failure result and stop itself. +The system SHALL enforce child operation health checks, not a wall-clock task-lifetime timeout. +The checks SHALL distinguish first substantive output, inter-delta inactivity, and content-free keepalives. +A child stall SHALL return an explicit failed terminal result through the background owner contract. +An authorized human approval wait SHALL remain independent of operation inactivity and remain cancellable. +Healthy task duration alone SHALL NOT cause terminal timeout. #### Scenario: Subagent times out -- **GIVEN** a `RunSubAgent` message with a `Timeout` of 30 seconds -- **WHEN** 30 seconds elapse without completion -- **THEN** the subagent returns `SubAgentResult` with `Success = false` -- **AND** the output contains "timed out" -- **AND** the subagent stops itself +- **GIVEN** a child provider operation has an applicable inactivity deadline +- **WHEN** the operation exceeds that deadline without the required progress +- **THEN** the child returns a failed terminal result with the corresponding timeout reason +- **AND** the owner records it and the child stops after receipt acknowledgement #### Scenario: LLM call failure returns failure result -- **GIVEN** the LLM throws an exception during a subagent call -- **WHEN** the subagent processes the error -- **THEN** it returns `SubAgentResult` with `Success = false` -- **AND** the output contains the error message -- **AND** the subagent stops itself +- **GIVEN** a child model call throws an exception +- **WHEN** the child processes that failure +- **THEN** it returns an explicit failed result +- **AND** background delivery preserves the failure rather than success + +#### Scenario: Healthy child outlives one operation deadline + +- **GIVEN** a child repeatedly completes healthy model and tool operations +- **WHEN** total task duration exceeds any individual operation deadline +- **THEN** task age alone does not stop the child +- **AND** individual stalled operations still trigger their health checks + +#### Scenario: A live approval wait does not become a stall + +- **GIVEN** a child waits on its run-owned approval prompt +- **WHEN** operation inactivity would otherwise expire +- **THEN** the legitimate approval wait remains open +- **AND** explicit run cancellation can still settle it ### Requirement: Configurable subagent timeouts -The system SHALL read subagent timeout values from the `SubAgents` section of -`netclaw.json`. When the section is absent, the system SHALL use built-in -defaults that match the current hardcoded values (180s for store, 30s for -search, 60s general default). Timeout values MUST be positive integers -between 5 and 600 seconds. +The system SHALL use resolved subagent profile values for operation-health deadlines. +`SubAgentProfile.TimeoutSeconds` SHALL control inactivity between model deltas after substantive output starts. +`SubAgentProfile.PrefillTimeoutSeconds` SHALL override the shared wait for initial output when present. +When absent, it SHALL inherit `SubAgentConfig.PrefillTimeoutSeconds` from the `SubAgents` section of `netclaw.json`. +`SubAgentConfig.NoProgressTimeoutSeconds` SHALL bound time without substantive model output. +Content-free keepalives SHALL NOT refresh that no-progress deadline. + +These values SHALL NOT define a wall-clock task-lifetime limit. +The file definition loader SHALL reject profile `timeoutSeconds` outside 5–600 seconds. +It SHALL reject profile `prefillTimeoutSeconds` outside 5–3600 seconds. +This change SHALL NOT migrate configuration fields or alter their current defaults. #### Scenario: Custom timeout from configuration -- **GIVEN** `netclaw.json` contains `"SubAgents": { "StoreMemoryTimeoutSeconds": 300 }` -- **WHEN** the `store_memory` tool spawns a subagent -- **THEN** the subagent uses a 300-second timeout +- **GIVEN** a valid agent profile has `timeoutSeconds: 300` +- **WHEN** the spawner prepares that profile for the actual child actor +- **THEN** the child uses 300 seconds for model inactivity after substantive output starts +- **AND** healthy operations can continue beyond 300 seconds of total task duration #### Scenario: Missing config section uses defaults -- **GIVEN** `netclaw.json` does not contain a `SubAgents` section -- **WHEN** the `store_memory` tool spawns a subagent -- **THEN** the subagent uses the default 180-second timeout +- **GIVEN** configuration omits the `SubAgents` section and the profile omits timeout overrides +- **WHEN** the runtime resolves the child operation-health values +- **THEN** it uses a 60-second profile inactivity value and a 1800-second initial-output value +- **AND** its no-progress value is 1200 seconds +- **AND** none of those values defines the child's total task duration #### Scenario: Invalid timeout rejected by doctor -- **GIVEN** `netclaw.json` contains `"SubAgents": { "DefaultTimeoutSeconds": -1 }` +- **GIVEN** `netclaw.json` contains `"SubAgents": { "PrefillTimeoutSeconds": -1 }` - **WHEN** the operator runs `netclaw doctor` - **THEN** doctor reports a validation error for the timeout value +#### Scenario: Invalid profile timeout is rejected before execution + +- **GIVEN** a file agent definition has `timeoutSeconds: -1` +- **WHEN** the definition loader validates that profile +- **THEN** it rejects that definition before a child can execute +- **AND** it does not replace the invalid value with a default + ### Requirement: Subagent observability events -The system SHALL emit structured `SubAgentOutput` events to session subscribers -when a subagent starts and completes. These events SHALL be filtered under the -`OutputFilter.ToolCalls` category. Tools that spawn subagents SHALL notify the -session via `ToolExecutionContext.OnSubAgentActivity`. +`LlmSessionActor` SHALL emit structured `SubAgentOutput` events to session subscribers for accepted child runs. +It SHALL emit `Started` only after the durable `Started` record commits. +It SHALL emit `Completed` only after the durable terminal receipt commits. +These events SHALL retain the `OutputFilter.ToolCalls` category and their existing payload fields. +Start-tool acceptance SHALL NOT substitute for the child terminal event. + +The owner SHALL NOT require the completed start tool's activity callback to deliver later child events. +Existing headless CLI and channel render rules SHALL remain unchanged. #### Scenario: Subagent start event emitted -- **GIVEN** a tool spawns a subagent within a session's tool execution pipeline -- **WHEN** the subagent begins execution -- **THEN** a `SubAgentOutput` event with `Phase = Started` is emitted -- **AND** the event includes the agent name and tool count -- **AND** the event is delivered to subscribers with `ToolCalls` in their filter +- **GIVEN** an accepted child start within the session owner +- **WHEN** the owner commits `Started` +- **THEN** it emits one `SubAgentOutput` with `Phase = Started` +- **AND** that event includes the agent name and tool count +- **AND** subscribers receive it under `OutputFilter.ToolCalls` #### Scenario: Subagent completion event emitted -- **GIVEN** a subagent completes (success or failure) -- **WHEN** the result is received by the calling tool -- **THEN** a `SubAgentOutput` event with `Phase = Completed` is emitted -- **AND** the event includes success status and duration +- **GIVEN** a child finishes after its start tool returns acceptance +- **WHEN** the owner commits its terminal receipt +- **THEN** it emits one `SubAgentOutput` with `Phase = Completed` +- **AND** that event includes the existing success status, duration, and outcome fields #### Scenario: Headless CLI renders subagent events - **GIVEN** the headless CLI subscribes with `OutputFilter.Full` - **WHEN** a subagent starts and completes - **THEN** the CLI renders `[subagent:start] ( tools)` -- **AND** renders `[subagent:done] (, )` +- **AND** it renders `[subagent:done] (, )` #### Scenario: Slack adapter suppresses subagent events @@ -422,12 +452,12 @@ The sub-agent inactivity watchdog SHALL treat parent approval waits as intention - **THEN** the sub-agent applies the approval outcome - **AND** the sub-agent is not failed for inactivity during the wait -#### Scenario: Parent spawn-agent watchdog pauses during approval -- **GIVEN** a parent session is consuming a streaming `spawn_agent` tool call -- **AND** the child sub-agent is waiting for human approval longer than the parent tool inactivity budget -- **WHEN** the approval wait is still active -- **THEN** the parent `spawn_agent` tool call is not timed out for inactivity -- **AND** the parent tool watchdog resumes after the approval wait settles +#### Scenario: Parent start completes before child approval +- **GIVEN** the parent start call returns durable child acceptance +- **AND** the child waits for human approval +- **WHEN** the parent performs independent authorized work +- **THEN** the child approval wait remains open and cancellable +- **AND** the completed parent start call has no watchdog dependency on that wait #### Scenario: Parallel approval waits keep watchdog paused until all settle - **GIVEN** a sub-agent tool batch with two approval-gated calls diff --git a/openspec/specs/progressive-tool-disclosure/spec.md b/openspec/specs/progressive-tool-disclosure/spec.md index 4bb79b400..6eb5d981b 100644 --- a/openspec/specs/progressive-tool-disclosure/spec.md +++ b/openspec/specs/progressive-tool-disclosure/spec.md @@ -235,3 +235,33 @@ entered durable chat history. - **WHEN** that child completes, the model call fails and evicts leases, or the session recovers - **THEN** the later exposure set is rebuilt from the policy-filtered core - **AND** the correction does not create durable exposure state + +### Requirement: Parent child control remains deferred and absent from child exposure + +`check_agent_run` SHALL use Deferred registration and normal audience policy. +Accepted-run context SHALL name it so the parent can call `load_tool` by its exact name. +Start acceptance SHALL NOT automatically expose its schema or change the bounded core set. +Children SHALL NOT discover, load, inherit, or directly dispatch this parent control. +Loading SHALL NOT grant run ownership, requester eligibility, or path access. +This change SHALL NOT expose agent message, child question, steering, or peer tools. + +#### Scenario: Parent loads one known control + +- **GIVEN** a parent receives accepted-run context with `check_agent_run` +- **WHEN** it loads that exact permitted name +- **THEN** the next parent request can use that control schema +- **AND** unrelated deferred schemas remain absent + +#### Scenario: Acceptance does not expand core exposure + +- **GIVEN** a parent starts a child +- **WHEN** acceptance commits +- **THEN** the initial core name snapshot remains unchanged +- **AND** the control stays absent until explicit load under normal policy + +#### Scenario: Child cannot discover or dispatch parent control + +- **GIVEN** a child searches, loads, or directly calls `check_agent_run` +- **WHEN** exposure or dispatch evaluates the child policy +- **THEN** search/load reveals no hidden schema or existence detail +- **AND** direct invocation cannot inspect or cancel any run diff --git a/openspec/specs/session-state-machine/spec.md b/openspec/specs/session-state-machine/spec.md index fd4eea496..e9588b494 100644 --- a/openspec/specs/session-state-machine/spec.md +++ b/openspec/specs/session-state-machine/spec.md @@ -75,12 +75,19 @@ Any transition not in this set SHALL throw `InvalidOperationException`. The session actor SHALL enter `Passivating` when its idle timeout fires in phase `Ready` and its active-work check returns false. The default idle timeout SHALL be one hour. Subscriber count and journaled approval state SHALL NOT -change idle eligibility. The `background-job-execution` capability defines the -active-job check. In `Passivating`, the actor SHALL request final memory -distillation from the observer actor, if present, wait up to five seconds, save -a snapshot, notify the lifecycle observer, and stop itself. Idle passivation -SHALL retain the post-snapshot grace window. The actor SHALL emit -`SessionDeactivated` only when it commits to stop. +change idle eligibility. The active-work check SHALL be false only when no live +background job, no live child, and no unadmitted child result requires the owner. +A live child includes accepted, running, cancelling, and terminal-acknowledgement +states that still need their owner. An admitted pending result SHALL remain +protected through the ordinary pending-input lifecycle. The +`background-job-execution` capability defines the active-job check. After those +obligations end, normal idle passivation SHALL resume. Child ownership SHALL NOT +alter shell-job reap semantics. In `Passivating`, the actor SHALL request final +memory distillation from the observer actor, if present, wait up to five seconds, +save a snapshot, notify the lifecycle observer, and stop itself. Idle passivation +SHALL retain the post-snapshot grace window that lets racing input abort the stop +and return to `Ready`. The actor SHALL emit `SessionDeactivated` only when it +commits to stop. #### Scenario: Session passivates with no active work and a live subscriber @@ -133,6 +140,20 @@ SHALL retain the post-snapshot grace window. The actor SHALL emit - **AND** it does not emit `SessionDeactivated` - **AND** it handles the message +#### Scenario: Live child prevents idle owner loss + +- **GIVEN** the session has no subscribers but owns a live background child +- **WHEN** idle timeout fires +- **THEN** the owner remains available for terminal receipt, prompts, status, and cancellation +- **AND** it cannot stop merely because the original start turn ended + +#### Scenario: Undelivered result prevents idle loss + +- **GIVEN** a terminal child receipt is durable but continuation admission remains pending +- **WHEN** idle timeout fires +- **THEN** the owner retains its delivery obligation +- **AND** normal idle passivation resumes after admission and ordinary pending-input settlement + ### Requirement: Phase transition logging and observability Each phase transition SHALL be logged at Info level with the source and target @@ -181,41 +202,54 @@ the legal phase-transition rules: re-driving a tool batch transitions ### Requirement: Restart-drain passivation is non-interruptible -When a coordinated daemon restart requests session drain, the actor SHALL reject -new work, allow the current in-flight turn or compaction to finish, and then -enter `Passivating`. Once restart-drain mode reaches `Passivating`, shutdown is -non-interruptible: inbound user messages and tool-interaction responses SHALL -NOT abort passivation. +When coordinated restart requests drain, the actor SHALL reject new work and complete its current parent turn or compaction. +It SHALL also cancel accepted live children through the run cancellation contract and bounded framework-only finalization. +It SHALL retain child terminal and prompt-settlement records before passivation whenever the owner remains operational. +Abrupt owner loss SHALL use the explicit recovery-loss contract instead. +Once restart-drain reaches `Passivating`, new user messages and approval prompt answers SHALL NOT abort shutdown. #### Scenario: Restart drain finishes current turn before passivating -- **GIVEN** the session actor is in `Processing` or `Compacting` -- **WHEN** restart drain is requested -- **THEN** the actor rejects new work -- **AND** allows the current in-flight work to finish -- **AND** only then transitions to `Passivating` +- **GIVEN** the parent is `Processing` or `Compacting` +- **WHEN** coordinated restart requests drain +- **THEN** it rejects new work and completes its current parent phase +- **AND** accepted children settle through cancellation rather than an unbounded lifetime wait #### Scenario: Restart-drain passivation rejects racing inbound work -- **GIVEN** the session actor is in `Passivating` because restart drain is active -- **WHEN** a `SendUserMessage` or `ToolInteractionResponse` arrives -- **THEN** the actor does NOT abort passivation -- **AND** the shutdown continues to completion +- **GIVEN** restart-drain reached `Passivating` +- **WHEN** new user input or an approval prompt answer arrives +- **THEN** it does not abort shutdown +- **AND** a settled child prompt cannot resume task work #### Scenario: Response in Compacting is buffered and replayed -- **GIVEN** the session actor is in phase `Compacting` -- **WHEN** a `ToolInteractionResponse` arrives -- **THEN** the actor SHALL buffer the response rather than re-driving mid-compaction -- **AND** SHALL replay the buffered response to itself after compaction completes +- **GIVEN** an ordinary live approval response arrives during `Compacting` +- **WHEN** the response is admitted under the existing run or parent prompt authority +- **THEN** its safe response transition respects the existing compaction boundary +- **AND** it does not redispatch a parent batch or cancelled child mid-compaction #### Scenario: Unknown call id does not transition phase -- **GIVEN** the session actor is in phase `Ready` with no matching pending - interaction and no reconstructable call in history -- **WHEN** a `ToolInteractionResponse` arrives -- **THEN** the actor SHALL remain in phase `Ready` -- **AND** SHALL emit a user-visible "approval prompt expired" message +- **GIVEN** the session has no live or reconstructable prompt for a call ID +- **WHEN** an answer arrives +- **THEN** the actor retains its phase +- **AND** it reports an expired approval prompt without tool execution + +#### Scenario: Coordinated restart settles a live child + +- **GIVEN** an accepted child still runs when drain begins +- **WHEN** the owner closes dispatch and finalization reaches its bounded terminal result +- **THEN** that cancelled result and prompt disposition remain durable +- **AND** recovery cannot automatically relaunch the child + +#### Scenario: Coordinated restart settles an unanswered child approval prompt + +- **GIVEN** a real accepted child waits for its original requester's unanswered approval prompt +- **WHEN** coordinated drain cancels the child and commits its terminal receipt +- **THEN** the final snapshot retains that run's `Cancelled` outcome and exact prompt correlation +- **AND** the prompt resolution is durable `Denied` before the terminal receipt +- **AND** no child retry or reusable authorization grant results ### Requirement: Approval turn state is explicit diff --git a/openspec/specs/skill-execution-routing/spec.md b/openspec/specs/skill-execution-routing/spec.md index 13cb86f90..a6a2a2696 100644 --- a/openspec/specs/skill-execution-routing/spec.md +++ b/openspec/specs/skill-execution-routing/spec.md @@ -167,3 +167,48 @@ snapshot contract as explicit `spawn_agent` execution. - **THEN** routing fails deterministically against the reloaded registry - **AND** inline fallback is not attempted - **AND** the stale prior definition is not used + +### Requirement: Routed activations use background acceptance and later result delivery + +Every first-party routed activation SHALL use the `background-subagent-runs` start contract. +This includes `skill_load`, slash commands, scheduled slash payloads, and reminder skill activation. +The activation SHALL validate metadata, reload and validate the target, and scan the skill before acceptance. +It SHALL return an accepted run identifier without waiting for final child output. +The later result SHALL preserve skill, profile, original input, and original authority attribution. +The skill body SHALL remain an additive child system overlay on the routed path. +Inline activation and deterministic failure without inline fallback SHALL remain unchanged. + +#### Scenario: Tool-driven routed skill returns acceptance + +- **GIVEN** an authorized routed skill and a non-empty task +- **WHEN** `skill_load` activates that skill +- **THEN** it returns the accepted run identifier before child completion +- **AND** the later child result arrives through the owner's attributed continuation + +#### Scenario: Slash activation frees the parent phase + +- **GIVEN** a valid slash skill with `metadata.subagent` +- **WHEN** its child acceptance commits +- **THEN** the parent emits an acceptance reply and completes the slash input +- **AND** a later user input does not wait for that child merely because the skill routed + +#### Scenario: Scheduled activation preserves its original correlation + +- **GIVEN** an admitted scheduled slash payload +- **WHEN** activation retries after its accepted response disappears +- **THEN** it uses the original admitted occurrence and activation key +- **AND** it creates no second child + +#### Scenario: Invalid route cannot become inline work + +- **GIVEN** an unknown target, invalid metadata, or rejected skill content +- **WHEN** routed activation runs its pre-admission checks +- **THEN** it returns the existing explicit failure with useful remediation +- **AND** neither a child nor inline execution starts + +#### Scenario: A loaded routed skill retains its overlay + +- **GIVEN** a scanned routed skill body +- **WHEN** its background child starts +- **THEN** the body remains an additive system overlay +- **AND** acceptance does not inject that body into the parent's inline path diff --git a/openspec/specs/tool-authorization/spec.md b/openspec/specs/tool-authorization/spec.md index 1d4dc1136..051fdd8fd 100644 --- a/openspec/specs/tool-authorization/spec.md +++ b/openspec/specs/tool-authorization/spec.md @@ -1560,35 +1560,47 @@ calls in the batch SHALL continue. - **THEN** Netclaw posts an expired-prompt notice - **AND** the tool does not run again -### Requirement: TA-12 Subagent consent goes through the parent session - -A subagent SHALL ask for consent through its parent session with the same -consent request contract, a parent-scoped correlation identifier, the parent -requester, and the inherited parent working directory. A subagent SHALL use -chat grants of its parent session, and its own chat grants SHALL NOT reach the -parent. - -- A subagent consent request SHALL be live-only. After a daemon or parent - restart, Netclaw SHALL reject the old prompt as expired and SHALL close the - interrupted `spawn_agent` call. -- An answer that arrives after the subagent wait ends SHALL NOT run the tool. -- A subagent without a parent bridge SHALL NOT ask. Its run SHALL fail with - `ToolExecutionFailed`. -- A one-time consent in a subagent SHALL cover only that exact call. +### Requirement: TA-12 Subagent approval prompts go through the parent session + +A subagent SHALL request an approval prompt through its owner session with the existing computed policy request contract. +The prompt SHALL retain the original parent requester, inherited working scope, exact child call, authorization attempt, and accepted run correlation. +The child SHALL use the parent's chat authorization grants. Its private grants SHALL NOT reach the parent. + +- Prompt lifecycle facts SHALL persist before display; the execution waiter remains run-local. +- Start-tool completion or a later ordinary parent message SHALL NOT abandon a live child prompt. +- A restart cannot resume an interrupted child. Its old prompt SHALL expire visibly under the loss contract. +- Cancellation or terminal loss SHALL settle the prompt. A late answer SHALL create no retry or authorization grant. +- A child without a safe parent bridge SHALL fail with `ToolExecutionFailed` rather than synthesize a requester. +- A one-time authorization grant SHALL cover only the exact prompted retry and remain subject to launch and run-liveness checks. +- Background acceptance and child text SHALL create no approval authority. #### Scenario: Subagent prompt reaches the parent requester -- **GIVEN** a subagent spawned in an interactive Personal session -- **WHEN** the subagent calls `shell_execute` with an uncovered command -- **THEN** the parent channel shows a consent request for the parent requester -- **AND** the subagent resumes after the answer +- **GIVEN** a live child belongs to an interactive Personal parent run +- **WHEN** it requests a protected shell call +- **THEN** the parent channel displays an approval prompt for the original eligible requester +- **AND** a valid answer can resume the exact child call under normal authorization #### Scenario: Subagent prompt expires after restart -- **GIVEN** an unanswered subagent consent request and a daemon restart -- **WHEN** the requester answers the old prompt -- **THEN** Netclaw rejects the answer as expired -- **AND** the tool does not run +- **GIVEN** an unanswered child prompt and an interrupted child after restart +- **WHEN** the original requester answers the old prompt +- **THEN** the system reports an expired prompt +- **AND** no child call or new authorization grant results + +#### Scenario: Another speaker cannot answer the original prompt + +- **GIVEN** a child prompt belongs to one original requester +- **WHEN** another speaker joins a later parent turn and answers that prompt +- **THEN** the answer cannot replace the recorded requester +- **AND** the protected child call remains unexecuted + +#### Scenario: Start completion does not end a live prompt + +- **GIVEN** the start call returned accepted and the child later requests approval +- **WHEN** the eligible requester answers through the parent +- **THEN** the run-owned wait remains available for exact retry authorization +- **AND** its lifetime does not depend on the completed start tool ### Requirement: TA-13 The persistent grant store fails closed @@ -1730,3 +1742,33 @@ SHALL report an error for an unrestricted Public or Team profile. - **GIVEN** `Tools.ShellMode` `HostAllowed` and a Personal `shell_execute` override of `Auto` - **WHEN** the operator runs `netclaw doctor` - **THEN** the audience profile check reports a warning + +### Requirement: TA-17 Child controls enforce parent session and original requester ownership + +The deferred child control SHALL require an authorized parent invocation in the owning session and original eligible requester context. +It SHALL use the existing original authority record, rather than parallel audience, requester, or path-root fields. +It SHALL be unavailable to children through discovery, schema load, and direct dispatch. +Unauthorized requests SHALL reveal no target existence, state, task text, result paths, or requester details. +Cancellation SHALL acknowledge local cutover separately from admission and SHALL create no tool or path grant. +Child result locations SHALL remain data for ordinary file access, not access authority. + +#### Scenario: Eligible parent cancels its child + +- **GIVEN** an authorized parent invocation under the original eligible requester context +- **WHEN** it cancels its own accepted run +- **THEN** the owner admits cancellation and later reports local dispatch closure +- **AND** no broader shell or path authority results + +#### Scenario: Child cannot cancel a sibling + +- **GIVEN** a child knows a sibling run identifier +- **WHEN** it attempts direct child-control dispatch +- **THEN** the runtime denies the operation without target details +- **AND** the sibling remains live + +#### Scenario: Known identifier does not authorize cross-session status + +- **GIVEN** another session obtains a child identifier from text +- **WHEN** it requests status or cancellation +- **THEN** the runtime returns a non-disclosing denial +- **AND** no state mutation or result access occurs From 644e2cb8d97edca431a3141f1505b4e7f60d917a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 9 Oct 2026 23:24:51 +0000 Subject: [PATCH 09/22] Use portable paths in the background context fixture --- .../BackgroundChildAcceptanceTests.ToolsAndContext.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs index b0fd84afc..56222f8db 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.ToolsAndContext.cs @@ -31,8 +31,9 @@ public void Run_context_is_bounded_and_confined_to_the_current_authority(bool pu state = AdmittedTask("later", state); if (foreignRequester) state = state with { AdoptedTaskContext = state.AdoptedTaskContext! with { RequesterSenderId = new SenderId("other") } }; + var sessionDirectory = Path.Combine(Path.GetTempPath(), "neutral-child-context"); var input = new ContextAssemblyInput(state, [], false, null, null, null, Session, - SessionStoragePaths.CreateLegacy("/tmp/neutral-child-context", "/tmp/neutral-child-context/session", Session.Value), false, null, string.Empty, + SessionStoragePaths.CreateLegacy(sessionDirectory, Path.Combine(sessionDirectory, "session"), Session.Value), false, null, string.Empty, publicAudience ? TrustAudience.Public : TrustAudience.Personal); var context = SessionMessageAssembler.BuildVolatileContextBlock(input); if (publicAudience || foreignRequester) From d38480b969789f5841500394bbe4125424a06308 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 00:25:06 +0000 Subject: [PATCH 10/22] Wait for provider cancellation in the child lifetime test --- .../BackgroundChildOwnerAdversarialTests.StartLifetime.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs index 3f18c69f6..322cfc1a3 100644 --- a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.StartLifetime.cs @@ -204,7 +204,11 @@ await AwaitAssertAsync(async () => [Fact] public async Task A_child_provider_that_ignores_cancellation_cannot_delay_owner_terminal_settlement() { - var provider = new UncooperativeChildClient(_child); + var providerCancelled = NewSignal(); + var provider = new UncooperativeChildClient(_child) + { + CancellationCallback = () => providerCancelled.TrySetResult() + }; Assert.IsType(Host.Services.GetRequiredService()).Child = provider; var path = Path.Combine(_directory!.Path, "confirmed-before-uncooperative-provider.txt"); var calls = 0; @@ -254,6 +258,7 @@ await AwaitAssertAsync(async () => Assert.Equal(SubAgentOutcomeReason.CancelledByParent, terminal.Terminal.Result.OutcomeReason); var completion = Assert.IsType(terminal.Terminal.Result.Completion); Assert.Equal(path, Assert.Single(completion.ConfirmedActivity!.ReadFiles)); + await providerCancelled.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); Assert.True(provider.Token.IsCancellationRequested); Assert.False(provider.Pending.Task.IsCompleted); Assert.Equal(2, provider.Count); From 219a4789224785fbeeefb9d7042b2c2e04c3def6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 01:42:41 +0000 Subject: [PATCH 11/22] test(subagents): preserve child death and parent status handling --- ...ndChildOwnerAdversarialTests.ActorDeath.cs | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ActorDeath.cs diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ActorDeath.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ActorDeath.cs new file mode 100644 index 000000000..fe22992d1 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ActorDeath.cs @@ -0,0 +1,88 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text.Json; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task Real_child_actor_death_records_one_failure_and_preserves_parent_status_tools() + { + var (owner, manager, subscriber) = await CreateObligationOwnerAsync(); + await SendObligationInputAsync(manager, "original", "operator-a"); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var before = await ReadJournalAsync(); + var accepted = Assert.Single(before.OfType()); + Assert.Equal(accepted.Run.RunId, Assert.Single(before.OfType()).RunId); + Assert.Empty(before.OfType()); + Assert.False(_childRelease.Task.IsCompleted); + var child = await Sys.ActorSelection($"{owner.Path}/child-run-{accepted.Run.RunId.Value}") + .ResolveOne(Ceiling, TestContext.Current.CancellationToken); + var watcher = CreateTestProbe(); + watcher.Watch(child); + Sys.Stop(child); + await watcher.ExpectTerminatedAsync(child, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await AwaitAssertAsync(async () => + { + var recorded = Assert.Single((await ReadJournalAsync()).OfType()); + Assert.Equal(accepted.Run.RunId, recorded.RunId); + Assert.Equal(SubAgentOutcomeReason.ActorStopped, recorded.Terminal.Result.OutcomeReason); + }, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + await CompletedObligationTurnAsync(subscriber); + var settled = await ReadJournalPositionsAsync(); + var terminalRow = Assert.Single(settled, record => record.Event is ChildRunEvent.TerminalRecorded); + var terminal = Assert.IsType(terminalRow.Event); + Assert.Equal(accepted.Run.RunId, terminal.RunId); + Assert.Equal(terminalRow.SequenceNr, terminal.TerminalSequenceNr); + Assert.False(terminal.Terminal.Lost); + Assert.False(terminal.Terminal.Result.Success); + Assert.Equal(SubAgentOutcomeReason.ActorStopped, terminal.Terminal.Result.OutcomeReason); + Assert.IsType(terminal.Terminal.Result.Completion); + var delivery = Assert.Single(settled.Select(record => record.Event).OfType()); + Assert.Equal(accepted.Run.RunId, delivery.RunId); + Assert.Single(settled.Select(record => record.Event).OfType(), + turn => turn.ConsumedInputIds.Contains(delivery.Input.InputId)); + + const string statusCallId = "status-after-child-actor-death"; + _main.PlannedResponses.Enqueue([new FunctionCallContent(statusCallId, "status_json_probe", + new Dictionary + { + ["run_id"] = accepted.Run.RunId.Value, ["cancel"] = false, + ["_rationale"] = "Inspect the child after its actor stops." + })]); + await SendObligationInputAsync(manager, statusCallId, "operator-a"); + var output = Assert.IsType(await subscriber.FishForMessageAsync( + message => message is ToolResultOutput result && result.CallId.Value == statusCallId, + Ceiling, cancellationToken: TestContext.Current.CancellationToken)); + using (var body = JsonDocument.Parse(output.Result)) + { + Assert.Equal(accepted.Run.RunId.Value, body.RootElement.GetProperty("run_id").GetString()); + Assert.Equal("Failed", body.RootElement.GetProperty("state").GetString()); + Assert.False(body.RootElement.GetProperty("cancellation_requested").GetBoolean()); + } + await CompletedObligationTurnAsync(subscriber); + var final = await ReadJournalAsync(); + var status = Assert.Single(final.OfType(), + record => record.ToolResult.ToolCallId == new ToolCallId(statusCallId)); + Assert.Equal(output.Result, status.ToolResult.Content); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Single(final.OfType()); + Assert.Equal(owner, await OwnerAsync()); + Assert.Equal(1, _child.CallCount); + Assert.False(_childRelease.Task.IsCompletedSuccessfully); + } +} From b37aa7e54417f8f87e7a27cbaf3b5066ab6f96d2 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 03:30:34 +0000 Subject: [PATCH 12/22] test: await child model entry before the count assertion --- .../Sessions/SubAgentSpawnIntegrationTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index a092122da..dc1b7ad9c 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -288,6 +288,8 @@ await sessionManager.Ask(new SendUserMessage await subscriber.ExpectMsgAsync(TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); Assert.Equal(2, _clientProvider.Main.CallCount); + await _clientProvider.Compaction.FirstCallEntered.Task.WaitAsync( + TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); Assert.Equal(1, _clientProvider.Compaction.CallCount); Assert.False(childRelease.Task.IsCompleted); childRelease.TrySetResult(); From 6bcd13f08453b3ffa4840ee61d9b004370a1d616 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 04:17:00 +0000 Subject: [PATCH 13/22] Rebalance the focused mutation gate groups --- .github/workflows/pr_validation.yml | 4 ++-- TOOLING.md | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index a5fe4d191..0a81a71a6 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -171,9 +171,9 @@ jobs: matrix: include: - group: shell-analysis - gates: shell-command-analysis tool-authorizer-order mcp-artifact-admission tool-task-adoption + gates: shell-command-analysis tool-authorizer-order mcp-artifact-admission - group: shell-assignment-and-paths - gates: shell-assignment shell-config-read path-access + gates: shell-assignment shell-config-read path-access tool-task-adoption - group: verb-chain-and-reminder gates: exact-verb-chain reminder-execution literal-twin stream-commit-gate - group: approval-and-authorization diff --git a/TOOLING.md b/TOOLING.md index a26da8065..5349e65de 100644 --- a/TOOLING.md +++ b/TOOLING.md @@ -32,7 +32,16 @@ All focused gates run in one job definition, `mutation-gates` in `pr_validation.yml`, on each pull request and merge group. They do not run on `dev` pushes. The job has four Linux matrix groups that run in parallel with the normal test matrix. The baseline run at `2e6bc4f01` took 12m24s, 12m29s, 18m56s, and 20m14s across the four groups. -The new task-adoption gate uses the lightest group, `shell-analysis`. The job timeout remains 25 minutes. +The task-adoption gate uses `shell-assignment-and-paths`. The job timeout remains 25 minutes. +The prior `shell-analysis` job reached its deadline at `fa6aad73` after 342 reported mutants were killed. +The final child-control target produced no report. +The same mutation inputs passed the earlier `832ebff8` job in 20m25s. + +The destination group took 13m13s to 16m49s across four retained hosted runs. +It took 14m53s in the `fa6aad73` run. The complete gate moves without target or test changes. +Hosted verification of the new distribution remains pending. +See [the cancelled job](https://github.com/netclaw-dev/netclaw/actions/runs/38020934971/job/114121548608) +and [the prior successful job](https://github.com/netclaw-dev/netclaw/actions/runs/38014833252/job/114102687127). Each group runs its gates in sequence after one checkout and tool restore, and it reports every failed gate. To add a gate, add its script name (`scripts/run--mutations.sh`) to the lightest group. Do not add a new job. @@ -83,7 +92,7 @@ The gate correctly rejected those timeouts. The separate-filter run killed all six targets in 339.85 seconds. Its reports contain no tested timeout or survivor. The separate build adds 152.286 seconds over the earlier five-target run. -CI runs this gate in `shell-analysis` and uploads `artifacts/stryker/tool-task-adoption` with the group reports. +CI runs this gate in `shell-assignment-and-paths` and uploads `artifacts/stryker/tool-task-adoption` with the group reports. The `adoption` and `child-control` directories each retain their own mutation report. The local duration does not establish the combined CI duration. From 1c71e96583ef6a9586d27c4f388fdf697525a3bc Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 10:42:04 +0000 Subject: [PATCH 14/22] Clarify child finalization during daemon drain --- docs/spec/SPEC-002-session-lifecycle-and-protocol.md | 3 ++- .../SPEC-016-tool-liveness-and-stall-detection.md | 3 ++- openspec/changes/background-child-runs/tasks.md | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md index ba50a7319..0a684cac6 100644 --- a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md +++ b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md @@ -183,7 +183,8 @@ Owner restart marks unresolved accepted children `Lost` and never recreates them The model decides its next action from the recorded evidence. The framework does not automatically retry uncertain effects. Idle passivation defers live children and pending delivery. -Explicit stop and coordinated drain cancel children through the bounded closure path. +`PrepareForDaemonRestart` admits child cancellation and waits for bounded framework finalization before the drain acknowledgement. +An abrupt actor stop closes live dispatch and requests token cancellation without a durable finalization guarantee. The inference backend owns request queues and capacity. Netclaw adds no parent or child slot coordinator. Positive example: a held child stays live while the parent answers later input, then delivers one attributed result. diff --git a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md index 40dca749a..d1f2f872e 100644 --- a/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md +++ b/docs/spec/SPEC-016-tool-liveness-and-stall-detection.md @@ -116,7 +116,8 @@ Terminal persistence failure produces no successful receipt acknowledgement. ## Recovery, Passivation, And Drain Idle passivation defers live children and pending delivery. -Explicit parent stop and coordinated drain cancel children through the bounded closure path. +`PrepareForDaemonRestart` admits child cancellation and waits for bounded framework finalization before the drain acknowledgement. +An abrupt actor stop closes live dispatch and requests token cancellation without a durable finalization guarantee. Owner restart marks unresolved accepted runs `Lost`; it does not recreate or resume child execution. A prior durable cancellation retains its cancelled outcome. Committed terminal facts and pending delivery remain available through journal and snapshot recovery. diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 5034181cf..67d3ed3ce 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -93,10 +93,19 @@ Preparation evidence: ## 8. Docs, operational guidance, and handoff - [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. -- [ ] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. +- [x] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. - [x] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. - [x] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. - [ ] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. - [ ] 8.6 Run applicable builds, required tests, Slopwatch, and copyright-header verification; retain exact candidate and meaningful test counts. - [ ] 8.7 Validate the OpenSpec change strictly and sync implemented deltas through the appropriate skill; verify the PRD and active plan trace to the final contract. - [ ] 8.8 Prepare rollback and evidence handoff for the combined PR; verify no merge, rollout, publication, or database replacement is implied by local passes. + +Documentation scope review at `2357991d`: + +- `SPEC-002`, `SPEC-016`, and the authorization architecture describe the implemented owner, original requester, prompt lifetime, and cancellation order. +- `PrepareForDaemonRestart` admits child cancellation and waits for bounded framework finalization before the drain acknowledgement. +- An abrupt actor stop closes live dispatch and requests token cancellation without a durable finalization guarantee. +- Recovery retains committed terminal or cancellation facts. An unresolved accepted run becomes `Lost`. +- Independent source review approves the two-document clarification. It changes no runtime or detector policy. +- This closes task 8.2. The remaining integrated health, model, specification-sync, and handoff gates stay open. From b052736964f69cdfcd41a23713e568554d23c31b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 11:10:15 +0000 Subject: [PATCH 15/22] Record relay and parent response verification --- openspec/changes/background-child-runs/tasks.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 67d3ed3ce..b692145ef 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -80,16 +80,26 @@ Preparation evidence: ## 7. Integrated health, evals, and sensitive fault controls -- [ ] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. +- [x] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. - [ ] 7.2 Add targeted Subagents and background cases for acceptance, routed starts, controls, cancellation, prompts, and recovery; verify actual dispatch and artifact evidence. - [ ] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. - [x] 7.4 Hold several accepted child requests. Verify independent parent dispatch and locally responsive status/cancel without provider-capacity coordination. -- [ ] 7.5 Hold a child on the selected real backend that accepts concurrent requests. Verify a parent reply before child release in five critical-case trials. +- [x] 7.5 Hold a child on the selected real backend that accepts concurrent requests. Verify a parent reply before child release in five critical-case trials. - [x] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. - [x] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. - [ ] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. - [ ] 7.9 Require an independent verifier to inspect integrated results and assertions; verify no skipped cases, weakened gates, or author-only pass claims remain. +Bounded relay and response-order evidence: + +- Five hosted behavioral controls pass at `8aba7cb1`. They verify canonical request attribution, sidecar isolation, setup acknowledgements, real HTTP barriers, and response bytes. +- Independent review verifies the relay assertions. The existing controls require no relay repair or duplicate test. +- Five fixed `2c043a93` observations prove parent replies before exact child release on the selected real backend. +- Actual parent and child SSE requests overlap in recorder evidence. Independent review verifies all five bindings and archived request bytes. +- Full strict results remain pass, fail, pass, pass, pass. The second trial fails the later artifact-response check. +- Recorder completion follows downstream writes. These observations prove no independent upstream EOF time or simultaneous GPU work. +- This closes tasks 7.1 and 7.5 only. Full critical-case acceptance remains open, and every original verdict stays unchanged. + ## 8. Docs, operational guidance, and handoff - [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. From 73a415f23badf7022806dcb7bb96b39350d6f786 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 11:28:30 +0000 Subject: [PATCH 16/22] Record hosted boundary and mutation proof --- TOOLING.md | 5 ++++- openspec/changes/background-child-runs/tasks.md | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/TOOLING.md b/TOOLING.md index 5349e65de..a27a37e62 100644 --- a/TOOLING.md +++ b/TOOLING.md @@ -39,7 +39,10 @@ The same mutation inputs passed the earlier `832ebff8` job in 20m25s. The destination group took 13m13s to 16m49s across four retained hosted runs. It took 14m53s in the `fa6aad73` run. The complete gate moves without target or test changes. -Hosted verification of the new distribution remains pending. +The [combined `153d9223` hosted run](https://github.com/netclaw-dev/netclaw/actions/runs/38045919693) verifies the new distribution. +Its four groups pass in 16m51s, 23m14s, 15m11s, and 21m01s. +Each group stays below the 25-minute limit. All six expected task-adoption and child-authority targets have `Killed` status. +The `8aba7cb1` assignment group passes in 24m52s, with eight seconds of margin. See [the cancelled job](https://github.com/netclaw-dev/netclaw/actions/runs/38020934971/job/114121548608) and [the prior successful job](https://github.com/netclaw-dev/netclaw/actions/runs/38014833252/job/114102687127). Each group runs its gates in sequence after one checkout and tool restore, and it reports every failed gate. diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index b692145ef..44322400e 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -82,12 +82,12 @@ Preparation evidence: - [x] 7.1 Extend the existing background relay with explicit parent/child request IDs and barriers; verify setup/sidecar calls cannot consume child script stages. - [ ] 7.2 Add targeted Subagents and background cases for acceptance, routed starts, controls, cancellation, prompts, and recovery; verify actual dispatch and artifact evidence. -- [ ] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. +- [x] 7.3 Preserve queued-grant revocation, shell lifecycle, and PR 1 long-task controls; verify the combined candidate passes those unchanged boundaries. - [x] 7.4 Hold several accepted child requests. Verify independent parent dispatch and locally responsive status/cancel without provider-capacity coordination. - [x] 7.5 Hold a child on the selected real backend that accepts concurrent requests. Verify a parent reply before child release in five critical-case trials. - [x] 7.6 Make provider output ignore cancellation or never finish; verify framework settlement without another model response and report remote-computation limits. - [x] 7.7 Demonstrate rejection of duplicate start, early terminal acknowledgement, late dispatch, stale approval prompt, and delivery-reset mutants in isolation. -- [ ] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. +- [x] 7.8 Keep any new focused mutation target narrow and inside the documented CI budget; verify expected mutants execute and none survives. - [ ] 7.9 Require an independent verifier to inspect integrated results and assertions; verify no skipped cases, weakened gates, or author-only pass claims remain. Bounded relay and response-order evidence: @@ -100,6 +100,19 @@ Bounded relay and response-order evidence: - Recorder completion follows downstream writes. These observations prove no independent upstream EOF time or simultaneous GPU work. - This closes tasks 7.1 and 7.5 only. Full critical-case acceptance remains open, and every original verdict stays unchanged. +Combined boundary and mutation evidence at `153d9223`: + +- All three hosted platform test jobs pass. Their actual merge checkout has the exact combined source tree. +- The included controls preserve revoked queued grants, shell approval lifecycle, session-owned reap, and the prior long-task assertions. +- The long-task controls require actual file effects after 65 parent steps and 35 child steps. +- Hosted logs report assembly totals and skips. They supply no separate named pass counts for these controls. +- All fourteen mutation gates execute once. All six expected task-adoption and child-authority mutants have `Killed` status. +- The narrow gate discovers thirteen adoption tests and one child-authority test. No expected target survives, times out, or disappears. +- The four groups finish in 16m51s, 23m14s, 15m11s, and 21m01s. Each stays below the 25-minute limit. +- The `8aba7cb1` assignment group also passes in 24m52s. Its margin is eight seconds. +- Two unrelated shell-config-read mutations retain their timeout dispositions. They supply no assertion-kill claim for this gate. +- Independent review approves tasks 7.3 and 7.8. These checks prove no detector activation, days-long productivity, or full model acceptance. + ## 8. Docs, operational guidance, and handoff - [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. From d00a6918cfe5f5cb1bbf3a94ac50bb75ac9bc782 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 12:44:45 +0000 Subject: [PATCH 17/22] Record targeted background instruction eval evidence --- openspec/changes/background-child-runs/tasks.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 44322400e..c0f17a86d 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -119,7 +119,7 @@ Combined boundary and mutation evidence at `153d9223`: - [x] 8.2 Update `SPEC-002`, `SPEC-016`, and the authorization architecture document; verify they describe implemented ownership and cancellation order. - [x] 8.3 Add minimum runtime `AGENTS.md` guidance; verify it describes background acceptance, deferred control, partial result review, and cancel/recreate only. - [x] 8.4 Update `subagent-authoring` and `netclaw-operations` versions; verify instructions advertise no deferred messages or restored tool ceilings. -- [ ] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. +- [x] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. - [ ] 8.6 Run applicable builds, required tests, Slopwatch, and copyright-header verification; retain exact candidate and meaningful test counts. - [ ] 8.7 Validate the OpenSpec change strictly and sync implemented deltas through the appropriate skill; verify the PRD and active plan trace to the final contract. - [ ] 8.8 Prepare rollback and evidence handoff for the combined PR; verify no merge, rollout, publication, or database replacement is implied by local passes. @@ -132,3 +132,17 @@ Documentation scope review at `2357991d`: - Recovery retains committed terminal or cancellation facts. An unresolved accepted run becomes `Lost`. - Independent source review approves the two-document clarification. It changes no runtime or detector policy. - This closes task 8.2. The remaining integrated health, model, specification-sync, and handoff gates stay open. + +Targeted instruction eval evidence at `8a57fbf5`: + +- The eight changed instruction files match the prior discovery and cancellation observations. +- Five discovery observations load the canonical coordination skill and the complete selected resource. +- Their original results remain pass, pass, fail, pass, pass. A separate corrected-oracle assessment accepts all five archives. +- Five cancellation observations pass with unchanged instruction bytes. They prove deferred controls, terminal attribution, complete partial-report reads, and final status closure. +- The operations case reads the complete selected child-run resource. Its activation preview proves no complete operations-guide read. +- One fresh author-guide observation passes after the narrow assertion repair. It loads the complete version `1.5.0` guide and returns the required contract answer. +- The original author-guide failure remains false. All 54 files remain unchanged; the separate offline assessment adds no replacement verdict. +- Independent review verifies actual provider pairs, database verdicts, source equivalence, fresh identities, and cleanup. +- These observations exercise the private identity. Public identity and CLI templates retain evidence of equal source. +- This closes task 8.5 only. The explanation proves no executed restart, late approval, framework finalization restriction, or task that lasts days. +- Integrated health, full critical-case acceptance, specification sync, and final handoff remain open. From c7a73d35e5c81a44e5bd13b88da3a3eb3d98dae6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 14:07:46 +0000 Subject: [PATCH 18/22] Reset recurrence evidence at durable child result adoption --- .../.system/files/netclaw-operations/SKILL.md | 8 +- .../changes/background-child-runs/design.md | 15 +- .../changes/background-child-runs/proposal.md | 4 +- .../specs/background-subagent-runs/spec.md | 30 +- .../specs/netclaw-session/spec.md | 7 +- .../specs/turn-loop-governance/spec.md | 64 ++++ .../specs/background-subagent-runs/spec.md | 30 +- openspec/specs/netclaw-session/spec.md | 7 +- openspec/specs/turn-loop-governance/spec.md | 10 +- .../ToolTaskAdoptionMutationTests.cs | 2 +- ...dOwnerAdversarialTests.AdmittedRecovery.cs | 57 +++- ...oundChildOwnerAdversarialTests.Delivery.cs | 14 +- ...ildOwnerAdversarialTests.ProductiveRead.cs | 160 ++++++++++ ...dChildOwnerAdversarialTests.WindowCrash.cs | 107 +++++++ .../BackgroundChildOwnerAdversarialTests.cs | 2 + .../Sessions/BackgroundJobLineageTests.cs | 6 +- .../ToolLoopReplayAdversarialTests.cs | 2 +- .../Sessions/ToolRecurrenceCheckpointTests.cs | 6 +- .../ToolTaskRecoveryAdversarialTests.cs | 2 +- ...ackgroundChildAcceptanceTests.Lifecycle.cs | 4 +- .../BackgroundChildAcceptanceTests.cs | 2 +- ...ldLedgerAdversarialTests.DetectorWindow.cs | 302 ++++++++++++++++++ ...ndChildLedgerAdversarialTests.Lifecycle.cs | 6 +- ...edgerAdversarialTests.TerminalPositions.cs | 2 +- .../BackgroundChildLedgerAdversarialTests.cs | 2 +- .../Serialization/NetclawProtoMapper.cs | 5 +- .../Protos/netclaw_messages.proto | 1 + .../Sessions/LlmSessionActor.cs | 2 +- .../Sessions/SessionProtocol.Events.cs | 2 +- .../Sessions/SessionState.ChildRuns.cs | 87 ++++- src/Netclaw.Actors/Sessions/SessionState.cs | 2 + 31 files changed, 880 insertions(+), 70 deletions(-) create mode 100644 openspec/changes/background-child-runs/specs/turn-loop-governance/spec.md create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ProductiveRead.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.WindowCrash.cs create mode 100644 src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.DetectorWindow.cs diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 491bbbdca..050a91d90 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about schedules, reminders, cron jobs, timers, background jobs, or child status and cancellation. Also use for diagnostics, troubleshooting, MCP tools, daemon health, identity updates, Netclaw capabilities, and maintenance." metadata: author: netclaw - version: "2.109.0" + version: "2.111.0" --- # Netclaw Operations @@ -1014,7 +1014,11 @@ An accessible Pending or Running job permits its exact noncancel status query. T Prefer completion notices or useful independent work over repeated status queries. Exact protection can miss changed arguments or noisy results. Operation timeouts and cancellation remain active. -A background result resumes its original exact recurrence evidence after a fresh user task or restart. +A shell-job result resumes its original exact recurrence evidence after a fresh user task or restart. +The first durable consumption of a canonical child result starts a new recurrence window under the original task and authority. +This permits review of the child's new artifact after an earlier absent-artifact correction. +Acceptance, status, delivery admission, restart, and duplicate consumption do not reset that window. +A missing execution receipt remains a failure. The child result does not authorize another tool or path. A missing or invalid new-format job origin causes an explicit partial failure. The runtime records the known job result as data and preserves unrelated tasks. Do not treat that report as approval for another tool call. diff --git a/openspec/changes/background-child-runs/design.md b/openspec/changes/background-child-runs/design.md index 122e3c7a8..d0e2f3b55 100644 --- a/openspec/changes/background-child-runs/design.md +++ b/openspec/changes/background-child-runs/design.md @@ -303,7 +303,14 @@ A missing mandatory receipt retains the receipt-failure fact and invokes PR 1's Direct activation retains canonical task evidence without a fabricated tool receipt. Parent completion and fresh input retain evidence that an outstanding child continuation needs. Sibling runs use the latest retained parent checkpoint for their task. -Canonical continuation adoption validates the accepted run and commits restored evidence before the next model request. +The first durable canonical child consumption starts a fresh recurrence window under the retained detector task identity. +The owner preserves original authority and sticky receipt failure. +It clears exact entries, adjacent history, cold keys, and the last blocked action. +It refreshes same-task child and job checkpoint copies through `RefreshJobEvidence`. +Acceptance, status, terminal preparation, and delivery admission cannot reset the window. +Durable adopted context and input IDs reject a second reset after duplicate adoption or compaction. +Recovery restores the current committed window. +An additive `ToolTaskAdopted` Boolean records this decision. Missing legacy values remain false and retain old replay semantics. The synthetic delivery pair supplies no execution receipt. It cannot clear receipt failure or reopen the completed start call. Schematic; authorization, admission, and persistence failure paths are abbreviated: @@ -321,7 +328,7 @@ Parent completes A; fresh user task B starts -> retain A's evidence in X; keep B's current checkpoint separate Child X terminal receipt -> admit one attributed continuation through durable delivery - -> validate X; durably restore the latest retained A checkpoint + -> validate X; first durable consumption starts a new window under A with sticky receipt failure -> if evidence permits, request the model under X's original authority Continuation completes -> settle only this delivery; retain evidence for outstanding siblings @@ -329,7 +336,7 @@ Continuation completes Positive example: two children from A share its latest committed checkpoint before either continuation requests the model. Negative example: children from A and B cannot coalesce solely because the same requester owns both tasks. -Recovery preserves parent detector evidence without child relaunch or a fresh-task reset. +Recovery preserves the current committed window without child relaunch or another reset. Result admission preserves the parent's current directory, project, and branch. ### 8. Original approval prompts remain run-owned @@ -378,7 +385,7 @@ Reuse actor persistence fixtures, provider barriers, session subscribers, and th Add child-request correlation to the existing relay before it claims background-agent coverage. Use actual dispatch counts, persisted state, path contents, and prompt settlement as assertions. Keep scripted protocol evidence separate from real-model evidence. -Keep the inherited PR 1 recurrence gates. This change cannot reset their state through child-result admission. +Keep the inherited PR 1 recurrence gates. Only first durable canonical child consumption starts the approved new window; admission cannot. Use paired controls and fault injection at acceptance, closure, terminal receipt, enrichment, continuation admission, and restart. Require five successful trials per critical real-model case on the owner-authorized target. diff --git a/openspec/changes/background-child-runs/proposal.md b/openspec/changes/background-child-runs/proposal.md index 84e56204d..9a9e97988 100644 --- a/openspec/changes/background-child-runs/proposal.md +++ b/openspec/changes/background-child-runs/proposal.md @@ -18,7 +18,7 @@ Use the [engineering glossary](../../../docs/spec/GLOSSARY.md) for shared terms. - Cancel task execution before a five-second framework-only grace period preserves recorded partial evidence. - Defer idle passivation while live children or undelivered terminal results require their owner. - Report restart loss explicitly. Do not replay interrupted child tools or relaunch lost children. -- Preserve the first PR's recurrence contract and removed tool ceilings. +- Preserve removed tool ceilings. Permit a new recurrence window only at first durable canonical child-result consumption. - Use **approval prompt** as the glossary's displayed term. Keep existing code identifiers and the legacy anchor unchanged. In scope: background execution, parent status/cancel, minimum runtime instructions, lifecycle tests, and targeted subagent evals. @@ -45,7 +45,7 @@ The third PR owns the coordination skill and general worker profile. Reuse `ChildRunScope.Authority`, `ToolRunScope`, `TurnContextRecord`, the owner-session seam, and existing session storage. Update the spawner, child actor, parent actor, routed skill consumers, approval bridge, session state, snapshots, and framework wire schema. Update runtime `AGENTS.md`, `subagent-authoring`, `netclaw-operations`, and the authorization architecture document in the runtime PR. -The first PR's frozen `per-call-tool-recurrence` contract is a dependency. This package does not redefine its detector. +The first PR's `per-call-tool-recurrence` contract remains a dependency. This package adds only the approved canonical child-consumption window exception. ### Security and operational impact diff --git a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md index 7b0d533a6..e06141137 100644 --- a/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md +++ b/openspec/changes/background-child-runs/specs/background-subagent-runs/spec.md @@ -372,7 +372,14 @@ A missing mandatory receipt SHALL retain the receipt-failure fact and invoke PR Direct activation SHALL retain canonical task evidence without a fabricated tool receipt. Parent completion and fresh input SHALL retain evidence required by outstanding child continuations. Sibling runs SHALL use the latest retained parent checkpoint for their task. -Canonical continuation adoption SHALL validate the accepted run and commit restored evidence before the next model request. +Canonical continuation adoption SHALL validate the accepted run and commit its recurrence window before the next model request. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window with the retained detector `TaskId`. +It SHALL clear exact entries, adjacent history, cold keys, and the last blocked action. +It SHALL preserve original authority, receipt failure, and current parent directory/project/branch facts. +It SHALL refresh every same-task child and job checkpoint copy through the existing ledger update. +Acceptance, status, terminal preparation, delivery admission, recovery, and duplicate consumption SHALL NOT start that window. +A duplicate adoption SHALL use durable adopted context and input IDs, even when compaction removes the delivery pair. +Legacy adoption events without the new window marker SHALL retain their original checkpoint restoration semantics. The synthetic delivery pair SHALL NOT clear receipt failure or reopen the completed start call. Delivery completion SHALL retain evidence required by another outstanding run. Recovery SHALL preserve these facts without child relaunch or a fresh-task reset. @@ -396,14 +403,14 @@ Recovery SHALL preserve these facts without child relaunch or a fresh-task reset - **GIVEN** task A accepted a child and committed its start receipt and feedback round - **AND** the parent completed A and admitted fresh user task B - **WHEN** the owner adopts the child's later result continuation -- **THEN** it commits A's latest retained checkpoint and receipt-failure fact before the next model request +- **THEN** its first durable consumption commits a fresh window with A's retained task ID and receipt-failure fact - **AND** it uses the child's original authority without a fresh detector task #### Scenario: Sibling results use the latest shared task evidence - **GIVEN** task A accepted two children and later committed additional detector evidence - **WHEN** either child result starts a continuation -- **THEN** that continuation uses A's latest retained parent checkpoint +- **THEN** its first durable consumption starts a fresh window with A's retained detector task identity - **AND** its completion retains evidence required by the other child #### Scenario: Equal authority cannot merge distinct detector tasks @@ -411,7 +418,22 @@ Recovery SHALL preserve these facts without child relaunch or a fresh-task reset - **GIVEN** one requester owns child runs from distinct detector tasks A and B - **WHEN** their terminal results await parent continuation - **THEN** the owner keeps separate continuation groups and parent checkpoints -- **AND** equal authority alone cannot merge or reset their detector evidence +- **AND** equal authority alone cannot merge their detector evidence or authorize a new window + +#### Scenario: A child result permits a new artifact read + +- **GIVEN** the parent received recurrence feedback for an absent child artifact +- **WHEN** it first durably consumes the canonical child result after the child writes that artifact +- **THEN** its next authorized read uses a fresh recurrence window +- **AND** the same task ID, original authority, and receipt failure remain authoritative + +#### Scenario: Duplicate adoption after compaction preserves the current window + +- **GIVEN** durable child adoption and later recurrence evidence +- **AND** compaction removes the original delivery pair from the transcript +- **WHEN** the same canonical adoption appears again or the owner recovers +- **THEN** durable adopted context and input IDs preserve the current checkpoint +- **AND** the runtime grants no second window #### Scenario: Recovery cannot erase a retained receipt failure diff --git a/openspec/changes/background-child-runs/specs/netclaw-session/spec.md b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md index 401b7cf38..afedffbb1 100644 --- a/openspec/changes/background-child-runs/specs/netclaw-session/spec.md +++ b/openspec/changes/background-child-runs/specs/netclaw-session/spec.md @@ -10,7 +10,10 @@ It SHALL NOT claim live provider state that the owner did not observe. Terminal child admission SHALL preserve the original task's recurrence state and current parent directory/project/branch facts. The durable child run ledger SHALL retain parent detector checkpoints independently from the session's current task checkpoint. Fresh input SHALL NOT replace evidence required by an outstanding child continuation. -Continuation adoption SHALL restore the latest retained original-task checkpoint before the next model request. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window with the retained original-task ID. +It SHALL preserve original authority and sticky receipt failure before the next model request. +Recovery and duplicate adoption SHALL restore the current committed window without another reset. +Legacy child adoption records SHALL retain their prior checkpoint restoration semantics. Confirmed child file activity SHALL merge through the existing working-context rules. #### Scenario: A later turn can inspect its earlier child @@ -32,4 +35,4 @@ Confirmed child file activity SHALL merge through the existing working-context r - **GIVEN** the original parent task retains unresolved recurrence evidence - **WHEN** a child completion admits an internal continuation - **THEN** it preserves that task identity and evidence under the first PR's contract -- **AND** it does not create a fresh authorized task solely from child text +- **AND** admission creates no fresh window or authorized task solely from child text diff --git a/openspec/changes/background-child-runs/specs/turn-loop-governance/spec.md b/openspec/changes/background-child-runs/specs/turn-loop-governance/spec.md new file mode 100644 index 000000000..e432d3972 --- /dev/null +++ b/openspec/changes/background-child-runs/specs/turn-loop-governance/spec.md @@ -0,0 +1,64 @@ +## MODIFIED Requirements + +### Requirement: Detector evidence retains its task lifetime + +The system SHALL retain established recurrence and unresolved corrections through compaction and automatic task continuation. +It SHALL retain parent evidence through framework-owned persistence and recovery. +It SHALL never infer trusted success from result text or an absent receipt. +A final actual result without its mandatory receipt SHALL cause an explicit runtime contract failure. +The system SHALL preserve known results and prior evidence, complete required call-result pairs, and settle partial/failure without another model request. +In-flight calls, approval waits, and external cancellation SHALL retain their separate lifecycle contracts. +Fresh authorized user input or a fresh scheduled task SHALL reset prior task evidence. +Overflow replay, automatic restart continuation, result admission, and approval retries SHALL NOT perform that reset. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window within its original detector task. +The owner SHALL retain task identity, original authority, and sticky receipt failure. +It SHALL clear exact entries, adjacent history, cold keys, and the last blocked action, then refresh same-task ledger copies. +Acceptance, status, terminal preparation, admission, recovery, and duplicate adoption SHALL NOT grant this exception. +Shell-job result continuation and legacy child adoption events SHALL retain their prior checkpoint restoration semantics. + +The recent cold-key horizon SHALL contain at most 256 keys with one observation. +The system SHALL retain pinned suspicion outside the canonical child-window exception. +A changed actual outcome or a fresh authorized task SHALL invalidate that suspicion. +It SHALL NOT evict pinned evidence through age, cold-key pressure, compaction, or a status query. +It SHALL surface resource and persistence failures without silently discarding evidence. +Usage count alone SHALL NOT terminate a useful task. + +#### Scenario: Compaction retains an unresolved correction + +- **GIVEN** the model received an exact recurrence correction +- **WHEN** the session compacts and the model repeats that identity +- **THEN** the runtime reaches terminal settlement + +#### Scenario: Restart continuation retains the original episode + +- **GIVEN** durable correction evidence for an interrupted authorized task +- **WHEN** the parent recovers and continues that same task +- **THEN** a repeated prohibited identity reaches terminal settlement +- **AND** recovery does not redispatch calls with committed results + +#### Scenario: A new scheduled task starts cleanly + +- **GIVEN** a previous task produced a repeated status result +- **WHEN** a distinct authorized schedule occurrence starts a new task +- **THEN** the previous task's detector state does not penalize the new task + +#### Scenario: Cold-key pressure cannot erase established suspicion + +- **GIVEN** an identity has two equal completed observations and a correction +- **WHEN** the task produces more than 256 other distinct identities +- **THEN** the unresolved correction remains effective + +#### Scenario: A missing receipt is not success + +- **GIVEN** a correlated actual result lacks its trusted receipt +- **WHEN** the group completes +- **THEN** the runtime reports a missing-receipt contract failure +- **AND** it preserves result pairs and prior evidence without a fabricated success +- **AND** it settles a truthful partial/failure result without another model request + +#### Scenario: Useful work exceeds the former parent ceiling + +- **GIVEN** a deterministic task requires more than 60 distinct useful tool rounds +- **WHEN** the parent executes the task +- **THEN** no iteration count stops the task +- **AND** independent artifact checks prove its required result diff --git a/openspec/specs/background-subagent-runs/spec.md b/openspec/specs/background-subagent-runs/spec.md index 7905150fc..29ea01729 100644 --- a/openspec/specs/background-subagent-runs/spec.md +++ b/openspec/specs/background-subagent-runs/spec.md @@ -374,7 +374,14 @@ A missing mandatory receipt SHALL retain the receipt-failure fact and invoke PR Direct activation SHALL retain canonical task evidence without a fabricated tool receipt. Parent completion and fresh input SHALL retain evidence required by outstanding child continuations. Sibling runs SHALL use the latest retained parent checkpoint for their task. -Canonical continuation adoption SHALL validate the accepted run and commit restored evidence before the next model request. +Canonical continuation adoption SHALL validate the accepted run and commit its recurrence window before the next model request. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window with the retained detector `TaskId`. +It SHALL clear exact entries, adjacent history, cold keys, and the last blocked action. +It SHALL preserve original authority, receipt failure, and current parent directory/project/branch facts. +It SHALL refresh every same-task child and job checkpoint copy through the existing ledger update. +Acceptance, status, terminal preparation, delivery admission, recovery, and duplicate consumption SHALL NOT start that window. +A duplicate adoption SHALL use durable adopted context and input IDs, even when compaction removes the delivery pair. +Legacy adoption events without the new window marker SHALL retain their original checkpoint restoration semantics. The synthetic delivery pair SHALL NOT clear receipt failure or reopen the completed start call. Delivery completion SHALL retain evidence required by another outstanding run. Recovery SHALL preserve these facts without child relaunch or a fresh-task reset. @@ -398,14 +405,14 @@ Recovery SHALL preserve these facts without child relaunch or a fresh-task reset - **GIVEN** task A accepted a child and committed its start receipt and feedback round - **AND** the parent completed A and admitted fresh user task B - **WHEN** the owner adopts the child's later result continuation -- **THEN** it commits A's latest retained checkpoint and receipt-failure fact before the next model request +- **THEN** its first durable consumption commits a fresh window with A's retained task ID and receipt-failure fact - **AND** it uses the child's original authority without a fresh detector task #### Scenario: Sibling results use the latest shared task evidence - **GIVEN** task A accepted two children and later committed additional detector evidence - **WHEN** either child result starts a continuation -- **THEN** that continuation uses A's latest retained parent checkpoint +- **THEN** its first durable consumption starts a fresh window with A's retained detector task identity - **AND** its completion retains evidence required by the other child #### Scenario: Equal authority cannot merge distinct detector tasks @@ -413,7 +420,22 @@ Recovery SHALL preserve these facts without child relaunch or a fresh-task reset - **GIVEN** one requester owns child runs from distinct detector tasks A and B - **WHEN** their terminal results await parent continuation - **THEN** the owner keeps separate continuation groups and parent checkpoints -- **AND** equal authority alone cannot merge or reset their detector evidence +- **AND** equal authority alone cannot merge their detector evidence or authorize a new window + +#### Scenario: A child result permits a new artifact read + +- **GIVEN** the parent received recurrence feedback for an absent child artifact +- **WHEN** it first durably consumes the canonical child result after the child writes that artifact +- **THEN** its next authorized read uses a fresh recurrence window +- **AND** the same task ID, original authority, and receipt failure remain authoritative + +#### Scenario: Duplicate adoption after compaction preserves the current window + +- **GIVEN** durable child adoption and later recurrence evidence +- **AND** compaction removes the original delivery pair from the transcript +- **WHEN** the same canonical adoption appears again or the owner recovers +- **THEN** durable adopted context and input IDs preserve the current checkpoint +- **AND** the runtime grants no second window #### Scenario: Recovery cannot erase a retained receipt failure diff --git a/openspec/specs/netclaw-session/spec.md b/openspec/specs/netclaw-session/spec.md index 46f8ecf0f..72345b5cb 100644 --- a/openspec/specs/netclaw-session/spec.md +++ b/openspec/specs/netclaw-session/spec.md @@ -1073,7 +1073,10 @@ It SHALL NOT claim live provider state that the owner did not observe. Terminal child admission SHALL preserve the original task's recurrence state and current parent directory/project/branch facts. The durable child run ledger SHALL retain parent detector checkpoints independently from the session's current task checkpoint. Fresh input SHALL NOT replace evidence required by an outstanding child continuation. -Continuation adoption SHALL restore the latest retained original-task checkpoint before the next model request. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window with the retained original-task ID. +It SHALL preserve original authority and sticky receipt failure before the next model request. +Recovery and duplicate adoption SHALL restore the current committed window without another reset. +Legacy child adoption records SHALL retain their prior checkpoint restoration semantics. Confirmed child file activity SHALL merge through the existing working-context rules. #### Scenario: A later turn can inspect its earlier child @@ -1095,4 +1098,4 @@ Confirmed child file activity SHALL merge through the existing working-context r - **GIVEN** the original parent task retains unresolved recurrence evidence - **WHEN** a child completion admits an internal continuation - **THEN** it preserves that task identity and evidence under the first PR's contract -- **AND** it does not create a fresh authorized task solely from child text +- **AND** admission creates no fresh window or authorized task solely from child text diff --git a/openspec/specs/turn-loop-governance/spec.md b/openspec/specs/turn-loop-governance/spec.md index dea4f7234..96cce5a77 100644 --- a/openspec/specs/turn-loop-governance/spec.md +++ b/openspec/specs/turn-loop-governance/spec.md @@ -192,10 +192,16 @@ A final actual result without its mandatory receipt SHALL cause an explicit runt The system SHALL preserve known results and prior evidence, complete required call-result pairs, and settle partial/failure without another model request. In-flight calls, approval waits, and external cancellation SHALL retain their separate lifecycle contracts. Fresh authorized user input or a fresh scheduled task SHALL reset prior task evidence. -Overflow replay, automatic restart continuation, result delivery, and approval retries SHALL NOT perform that reset. +Overflow replay, automatic restart continuation, result admission, and approval retries SHALL NOT perform that reset. +The first durable consumption of a canonical child result SHALL start a fresh recurrence window within its original detector task. +The owner SHALL retain task identity, original authority, and sticky receipt failure. +It SHALL clear exact entries, adjacent history, cold keys, and the last blocked action, then refresh same-task ledger copies. +Acceptance, status, terminal preparation, admission, recovery, and duplicate adoption SHALL NOT grant this exception. +Shell-job result continuation and legacy child adoption events SHALL retain their prior checkpoint restoration semantics. The recent cold-key horizon SHALL contain at most 256 keys with one observation. -The system SHALL pin established suspicion until an actual changed outcome or a fresh authorized task invalidates it. +The system SHALL retain pinned suspicion outside the canonical child-window exception. +A changed actual outcome or a fresh authorized task SHALL invalidate that suspicion. It SHALL NOT evict pinned evidence through age, cold-key pressure, compaction, or a status query. It SHALL surface resource and persistence failures without silently discarding evidence. Usage count alone SHALL NOT terminate a useful task. diff --git a/src/Netclaw.Actors.MutationTests/ToolTaskAdoptionMutationTests.cs b/src/Netclaw.Actors.MutationTests/ToolTaskAdoptionMutationTests.cs index b2d0afac4..4c62e4101 100644 --- a/src/Netclaw.Actors.MutationTests/ToolTaskAdoptionMutationTests.cs +++ b/src/Netclaw.Actors.MutationTests/ToolTaskAdoptionMutationTests.cs @@ -81,6 +81,6 @@ public void The_canonical_compatible_prefix_adopts_its_latest_context_once() }).ToRecord() }; - private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new() + private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new(false) { SessionId = Session, TurnContext = input.TurnContext, InputIds = ids }; } diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs index 88a2e0085..7dd198da9 100644 --- a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.AdmittedRecovery.cs @@ -20,20 +20,27 @@ namespace Netclaw.Actors.Tests.Sessions; public sealed partial class BackgroundChildOwnerAdversarialTests { [Fact] - public async Task Cold_recovery_resumes_an_already_adopted_child_input_without_duplicate_adoption_or_detector_reset() + public async Task A_trusted_restart_after_child_input_consumption_preserves_post_adoption_detector_evidence() { PrepareAdmittedRecoveryRounds(); var (owner, manager, subscriber) = await CreateOwnerAsync(); await manager.Ask(new SendUserMessage { - SessionId = Session, Content = "Delegate after the two confirmed rounds.", + SessionId = Session, Content = "Delegate the assigned task.", Source = DeliverySource("original", "operator-a") }, Ceiling, TestContext.Current.CancellationToken); await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); await CompletedAsync(subscriber); var oldResponse = NewSignal(); var recoveredResponse = NewSignal(); - _main.NextResponseGate = oldResponse; + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-1")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-2")]); + await Journal.OnWrite.FailIf(record => + { + if (record.Payload is ToolCallRecorded { ToolResult.ToolCallId: { Value: "prefix-probe-2" } }) + _main.NextResponseGate = oldResponse; + return false; + }); try { _childRelease.TrySetResult(); @@ -45,11 +52,22 @@ await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, var adopted = Assert.Single(prefix.Select(row => row.Event).OfType(), evt => evt.ContinuedChildRunId == accepted.Run.RunId); Assert.Equal(new[] { delivery.Input.InputId }, adopted.InputIds); + Assert.True(adopted.StartsChildContinuationWindow); Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, adopted.TurnContext)); Assert.Empty(prefix.Select(row => row.Event).OfType()); Assert.Single(prefix.Select(row => row.Event).OfType()); AssertAdmittedRecoveryPair(_main.ReceivedMessages[^1], delivery, accepted.Run); Assert.Equal(new[] { "prefix-probe-1", "prefix-probe-2" }, _start!.ProbeCallIds); + var postAdoption = prefix.Where(row => row.Event is ToolCallRecorded call + && call.ToolResult.ToolCallId is { } id && id.Value.StartsWith("prefix-probe-", StringComparison.Ordinal)).ToArray(); + Assert.Equal(2, postAdoption.Length); + var adoptionPosition = Assert.Single(prefix, row => ReferenceEquals(row.Event, adopted)).SequenceNr; + Assert.All(postAdoption, row => Assert.True(row.SequenceNr > adoptionPosition)); + Assert.All(postAdoption, row => Assert.False(Assert.IsType(row.Event).LoopObservation!.Synthetic)); + var confirmed = Assert.IsType(await owner.Ask(Retry(accepted), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(2, Assert.Single(confirmed.Run.ParentCheckpoint.Entries, + entry => entry.ToolName == "neutral_probe").EqualRounds); var watcher = CreateTestProbe(); watcher.Watch(owner); @@ -63,7 +81,19 @@ await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, recoveredSubscriber.Ref); await recoveredSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); - Assert.NotEqual(owner, await OwnerAsync()); + var recovered = await OwnerAsync(); + Assert.NotEqual(owner, recovered); + await AssertTerminalCutOwnerBarrierAsync(recovered, "consumed-child-before-trusted-resume"); + Assert.Equal(5, _main.CallCount); + await manager.Ask(new SendUserMessage + { + SessionId = Session, Content = "Resume the work that was interrupted by the daemon restart.", + Source = DeliverySource("restart-resume-consumed-child:1", "reminder-system") with + { + ReminderId = new ReminderId("restart-resume-consumed-child:1"), + Principal = PrincipalClassification.VerifiedAutomation + } + }, Ceiling, TestContext.Current.CancellationToken); await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, cancellationToken: TestContext.Current.CancellationToken); var resumed = await ReadJournalPositionsAsync(); @@ -71,6 +101,9 @@ await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, AssertAdmittedRecoveryPair(_main.ReceivedMessages[^1], delivery, accepted.Run); Assert.Single(resumed.Select(row => row.Event).OfType(), evt => evt.ContinuedChildRunId == accepted.Run.RunId); + var restored = Assert.IsType(await recovered.Ask(Retry(accepted), Ceiling, + TestContext.Current.CancellationToken)); + Assert.True(BackgroundChildRun.SameCheckpoint(confirmed.Run.ParentCheckpoint, restored.Run.ParentCheckpoint)); Assert.Equal(1, _child.CallCount); recoveredResponse.TrySetResult(); await CompletedAsync(recoveredSubscriber); @@ -92,7 +125,7 @@ public async Task Cold_recovery_keeps_an_ordinary_head_before_its_pending_child_ var (owner, manager, subscriber) = await CreateOwnerAsync(); await manager.Ask(new SendUserMessage { - SessionId = Session, Content = "Delegate after the two confirmed rounds.", + SessionId = Session, Content = "Delegate the assigned task.", Source = DeliverySource("original", "operator-a") }, Ceiling, TestContext.Current.CancellationToken); await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); @@ -107,7 +140,7 @@ await manager.Ask(new SendUserMessage SessionId = Session, Content = "Complete the ordinary head first.", Source = DeliverySource("ordinary-head", "operator-b") }, Ceiling, TestContext.Current.CancellationToken); - await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, + await AwaitAssertAsync(() => Assert.Equal(3, _main.CallCount), Ceiling, cancellationToken: TestContext.Current.CancellationToken); _childRelease.TrySetResult(); ChildRunEvent.DeliveryAdmitted? delivery = null; @@ -141,12 +174,14 @@ await recoveredSubscriber.ExpectMsgAsync(Ceiling, var recovered = await OwnerAsync(); Assert.NotEqual(owner, recovered); await AssertTerminalCutOwnerBarrierAsync(recovered, "ordinary-prefix-before-resume"); - Assert.Equal(5, _main.CallCount); + Assert.Equal(3, _main.CallCount); var waiting = await ReadJournalPositionsAsync(); Assert.Equal(prefix.Select(row => row.SequenceNr), waiting.Select(row => row.SequenceNr)); _main.NextResponseGate = recoveredResponse; _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("ordinary-probe")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-1")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-2")]); _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-refused-3")]); _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-stopped-4")]); await manager.Ask(new SendUserMessage @@ -158,7 +193,7 @@ await manager.Ask(new SendUserMessage Principal = PrincipalClassification.VerifiedAutomation } }, Ceiling, TestContext.Current.CancellationToken); - await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, + await AwaitAssertAsync(() => Assert.Equal(4, _main.CallCount), Ceiling, cancellationToken: TestContext.Current.CancellationToken); var ordinaryRequest = _main.ReceivedMessages[^1]; Assert.Contains(ordinaryRequest, message => message.Role == Microsoft.Extensions.AI.ChatRole.User @@ -173,8 +208,8 @@ await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, await CompletedAsync(recoveredSubscriber); var final = await ReadJournalPositionsAsync(); AssertAdmittedRecoverySettlement(final, delivery, accepted.Run, - ["prefix-probe-1", "prefix-probe-2", "ordinary-probe"]); - Assert.Equal(new[] { "operator-a", "operator-a", "operator-b" }, _start!.ProbeRequesters); + ["ordinary-probe", "prefix-probe-1", "prefix-probe-2"]); + Assert.Equal(new[] { "operator-b", "operator-a", "operator-a" }, _start!.ProbeRequesters); Assert.Single(final.Select(row => row.Event).OfType(), evt => evt.TurnContext.TurnId == "ordinary-head"); var childAdoption = Assert.Single(final, @@ -202,8 +237,6 @@ await AwaitAssertAsync(() => Assert.Equal(6, _main.CallCount), Ceiling, private void PrepareAdmittedRecoveryRounds() { _main.ToolCallsOnFirstCall = null; - _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-1")]); - _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-2")]); _main.PlannedResponses.Enqueue([new FunctionCallContent("start-1", "start_probe", new Dictionary { ["_rationale"] = "Inspect the neutral fixture." })]); _main.PlannedResponses.Enqueue([new TextContent("The original child run was accepted.")]); diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs index c1d081bc6..f469f91af 100644 --- a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.Delivery.cs @@ -39,6 +39,8 @@ public async Task Actual_terminals_keep_journal_order_and_original_evidence_afte _main.PlannedResponses.Enqueue(starts); if (!receiptFailure) _main.PlannedResponses.Enqueue([new TextContent("The original task has its two child runs.")]); _main.PlannedResponses.Enqueue([new TextContent("The fresh task has no tool effect.")]); + _main.PlannedResponses.Enqueue([Probe("window-probe-1")]); + _main.PlannedResponses.Enqueue([Probe("window-probe-2")]); _main.PlannedResponses.Enqueue([Probe("probe-3")]); _main.PlannedResponses.Enqueue([Probe("probe-4")]); var (owner, manager, subscriber) = await CreateOwnerAsync(); @@ -133,7 +135,7 @@ await AwaitAssertAsync(async () => }).Where(deliveryIds.Contains); Assert.Equal(deliveryIds, consumed); Assert.Equal(receiptFailure ? ToolCycleMessages.MissingReceipt : ToolCycleMessages.Final, final.AssistantReply.Content); - Assert.Equal(oldModelCount + (receiptFailure ? 1 : 3), _main.CallCount); + Assert.Equal(oldModelCount + (receiptFailure ? 1 : 5), _main.CallCount); Assert.Equal(new[] { "probe-1", "probe-2" }, _start.ProbeCallIds.Where(id => id.StartsWith("probe-", StringComparison.Ordinal))); Assert.Equal(1, _child.CallCount); Assert.Equal(1, _secondChild.CallCount); @@ -154,6 +156,16 @@ await AwaitAssertAsync(async () => Assert.Equal(delivery.Input.UserMessage.Content, Assert.IsType(Assert.Single(results, result => result.CallId == id).Result)); } + Assert.True(childAdoption.StartsChildContinuationWindow); + Assert.Equal(new[] { "window-probe-1", "window-probe-2" }, + _start.ProbeCallIds.Where(id => id.StartsWith("window-probe-", StringComparison.Ordinal))); + foreach (var id in new[] { "window-probe-1", "window-probe-2" }) + { + var actual = Assert.Single(after.OfType(), + evt => evt.ToolResult.ToolCallId == new ToolCallId(id)); + Assert.False(actual.LoopObservation!.Synthetic); + Assert.True(Array.IndexOf(after, childAdoption) < Array.IndexOf(after, actual)); + } var correction = Assert.Single(after.OfType(), evt => evt.ToolResult.ToolCallId == new ToolCallId("probe-3")); Assert.True(correction.LoopObservation!.Synthetic); Assert.StartsWith(ToolCycleMessages.Correction, correction.ToolResult.Content); diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ProductiveRead.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ProductiveRead.cs new file mode 100644 index 000000000..17ec9d0bd --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.ProductiveRead.cs @@ -0,0 +1,160 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Text; +using Akka.Actor; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Security; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_report_created_by_the_accepted_child_remains_readable_after_its_canonical_delivery(bool priorMissingReads) + { + const string report = """ + {"completed":true,"artifact":"implementation.json","changed_files":["source/catalog.py"],"checks":{"duplicate_rejection":true,"prior_state_preserved":true,"valid_order_preserved":true},"limits":"This deterministic child fixture proves an actual artifact write and an authorized parent read. It does not prove Git integration, release, deployment."} + """ + "\n"; + const string source = "The catalog must preserve its prior records after an invalid refresh.\n"; + const string finalReply = "I read the complete child report after its canonical delivery."; + Assert.Equal(349, Encoding.UTF8.GetByteCount(report)); + var root = Path.Combine(_directory!.Paths.WorkspacesDirectory, "productive-child-read"); + Directory.CreateDirectory(root); + var reportPath = Path.Combine(root, "implementation.json"); + var sourcePath = Path.Combine(root, "requirements.md"); + await File.WriteAllTextAsync(sourcePath, source, TestContext.Current.CancellationToken); + Assert.False(File.Exists(reportPath)); + + var config = new ToolConfig(); + var registry = Host.Services.GetRequiredService(); + registry.ReplaceCore(new FileReadTool(config, _directory.Paths, new ToolPathPolicy([]))); + registry.RegisterCore(new FileWriteTool(config, _directory.Paths, new ToolPathPolicy([]))); + var profiles = new SubAgentDefinitionRegistry(); + profiles.Register(new SubAgentProfile + { + Name = "worker", Description = "Write the assigned complete report.", + SystemPrompt = "Write the complete report with file_write.", + ModelRole = ModelRole.Compaction, ToolNames = [FileWriteTool.ToolName], + EmitStructuredFindings = false, Visibility = SubAgentVisibility.UserFacing + }); + _start!.StartTool = new SpawnAgentTool(profiles, _start.Spawner, _directory.Paths); + FunctionCallContent Read(string id, string path) => new(id, FileReadTool.ToolName, + new Dictionary { ["Path"] = path, ["_rationale"] = "Read the complete assigned file." }); + _main.ToolCallsOnFirstCall = null; + _main.PlannedResponses.Enqueue([new FunctionCallContent("start-1", "spawn_agent", new Dictionary + { + ["Agent"] = "worker", ["Task"] = "Write the complete report at " + reportPath, + ["_rationale"] = "Delegate the complete report to the child." + })]); + if (priorMissingReads) + { + _main.PlannedResponses.Enqueue([Read("report-before-1", reportPath)]); + _main.PlannedResponses.Enqueue([Read("report-before-2", reportPath)]); + } + else + _main.PlannedResponses.Enqueue([Read("independent-source", sourcePath)]); + _main.PlannedResponses.Enqueue([new TextContent("The child owns its report task. The parent completed its available work.")]); + _main.PlannedResponses.Enqueue([Read("report-after-3", reportPath)]); + _main.PlannedResponses.Enqueue([Read("report-after-4", reportPath)]); + _main.PlannedResponses.Enqueue([new TextContent(finalReply)]); + _child.ToolCallsOnFirstCall = [new FunctionCallContent("child-report-write", FileWriteTool.ToolName, + new Dictionary + { + ["Path"] = reportPath, ["Content"] = report, + ["_rationale"] = "Write the complete assigned report." + })]; + _child.PlannedResponses.Enqueue([new TextContent("The complete report exists at " + reportPath)]); + + var (owner, manager, subscriber) = await CreateOwnerAsync(); + try + { + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var before = await ReadJournalPositionsAsync(); + var accepted = Assert.Single(before.Select(row => row.Event).OfType()); + Assert.Equal("original", accepted.Run.OriginalContext.TurnId); + Assert.Equal(new SenderId("operator-a"), accepted.Run.OriginalContext.RequesterSenderId); + Assert.False(_childRelease.Task.IsCompleted); + Assert.False(File.Exists(reportPath)); + Assert.Empty(before.Select(row => row.Event).OfType()); + var initialReads = before.Where(row => row.Event is ToolCallRecorded call + && call.ToolResult.ToolCallId is { } id + && (id.Value.StartsWith("report-before-", StringComparison.Ordinal) || id.Value == "independent-source")).ToArray(); + Assert.Equal(priorMissingReads ? 2 : 1, initialReads.Length); + foreach (var row in initialReads) + { + var read = Assert.IsType(row.Event); + Assert.False(read.LoopObservation!.Synthetic); + Assert.False(read.LoopObservation.MissingReceipt); + Assert.Equal(priorMissingReads ? (int)ToolInvocationOutcomeCategory.NotFound + : (int)ToolInvocationOutcomeCategory.Success, read.LoopObservation.Category); + Assert.Equal(priorMissingReads ? "Error: File not found: " + reportPath : source, read.ToolResult.Content); + Assert.True(before.Single(value => ReferenceEquals(value.Event, accepted)).SequenceNr < row.SequenceNr); + } + var current = Assert.IsType(await owner.Ask(Retry(accepted), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(BackgroundChildState.Running, current.State); + var readEvidence = Assert.Single(current.Run.ParentCheckpoint.Entries, + entry => entry.ToolName == FileReadTool.ToolName); + Assert.Equal(priorMissingReads ? 2 : 1, readEvidence.EqualRounds); + Assert.Equal("original", current.Run.ParentCheckpoint.TaskId); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, current.Run.OriginalContext)); + + _childRelease.TrySetResult(); + await CompletedAsync(subscriber); + var rows = await ReadJournalPositionsAsync(); + var events = rows.Select(row => row.Event).ToArray(); + Assert.Single(events.OfType()); + Assert.Single(events.OfType()); + var terminal = Assert.Single(events.OfType()); + var prepared = Assert.Single(events.OfType()); + var delivery = Assert.Single(events.OfType()); + var adoption = Assert.Single(events.OfType(), item => item.ContinuedChildRunId is not null); + Assert.Equal(accepted.Run.RunId, terminal.RunId); + Assert.Equal(accepted.Run.RunId, prepared.RunId); + Assert.Equal(accepted.Run.RunId, delivery.RunId); + Assert.Equal(accepted.Run.RunId, adoption.ContinuedChildRunId); + Assert.Equal(new[] { delivery.Input.InputId }, adoption.InputIds); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, adoption.TurnContext)); + Assert.Equal(SubAgentRunOutcome.Completed, terminal.Terminal.Result.Outcome); + Assert.Contains(reportPath, terminal.Terminal.Result.WorkingContext!.ConfirmedChangedFiles); + Assert.Equal(report, await File.ReadAllTextAsync(reportPath, TestContext.Current.CancellationToken)); + Assert.Equal(349, new FileInfo(reportPath).Length); + var childWrite = Assert.Single(_child.ReceivedMessages[^1] + .SelectMany(message => message.Contents.OfType()), + item => item.CallId == "child-report-write"); + Assert.Equal("Successfully wrote 349 bytes to " + reportPath, Assert.IsType(childWrite.Result)); + long Position(ISessionEvent value) => rows.Single(row => ReferenceEquals(row.Event, value)).SequenceNr; + Assert.True(Position(terminal) < Position(prepared)); + Assert.True(Position(prepared) < Position(delivery)); + Assert.True(Position(delivery) < Position(adoption)); + Assert.Single(events.OfType(), item => item.ContinuedChildRunId is null); + foreach (var id in new[] { "report-after-3", "report-after-4" }) + { + var read = Assert.Single(events.OfType(), + item => item.ToolResult.ToolCallId == new ToolCallId(id)); + Assert.True(Position(adoption) < Position(read)); + Assert.False(read.LoopObservation!.Synthetic); + Assert.Equal((int)ToolInvocationOutcomeCategory.Success, read.LoopObservation.Category); + Assert.Equal(report, read.ToolResult.Content); + } + Assert.Equal(finalReply, events.OfType().Last().AssistantReply.Content); + } + finally { _childRelease.TrySetResult(); } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.WindowCrash.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.WindowCrash.cs new file mode 100644 index 000000000..0cfc252c3 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.WindowCrash.cs @@ -0,0 +1,107 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.SubAgents; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed partial class BackgroundChildOwnerAdversarialTests +{ + [Fact] + public async Task A_failed_child_adoption_write_cannot_dispatch_before_recovery_commits_the_new_window() + { + _main.ToolCallsOnFirstCall = null; + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-1")]); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("prefix-probe-2")]); + _main.PlannedResponses.Enqueue([new FunctionCallContent("start-1", "start_probe", + new Dictionary { ["_rationale"] = "Delegate the assigned task." })]); + _main.PlannedResponses.Enqueue([new TextContent("The child owns its accepted task.")]); + var entered = NewSignal(); + var release = NewSignal(); + var recoveredResponse = NewSignal(); + var adoptionWrites = 0; + await Journal.OnWrite.FailIf(async record => + { + if (record.Payload is not ToolTaskAdopted { ContinuedChildRunId: not null } + || Interlocked.Increment(ref adoptionWrites) != 1) + return false; + entered.TrySetResult(); + await release.Task.WaitAsync(TestContext.Current.CancellationToken); + return true; + }); + var (owner, manager, subscriber) = await CreateOwnerAsync(); + var watcher = CreateTestProbe(); + watcher.Watch(owner); + try + { + await SendOriginalAsync(manager); + await _child.FirstCallEntered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + await CompletedAsync(subscriber); + var accepted = Assert.Single((await ReadJournalAsync()).OfType()); + var retained = Assert.IsType(await owner.Ask(Retry(accepted), Ceiling, + TestContext.Current.CancellationToken)); + Assert.Equal(2, Assert.Single(retained.Run.ParentCheckpoint.Entries, + entry => entry.ToolName == "neutral_probe").EqualRounds); + await EventFilter.Error(contains: "Failed to persist event type").ExpectOneAsync(async () => + { + _childRelease.TrySetResult(); + await entered.Task.WaitAsync(Ceiling, TestContext.Current.CancellationToken); + var held = await ReadJournalAsync(); + Assert.Single(held.OfType()); + Assert.Single(held.OfType()); + Assert.Single(held.OfType()); + Assert.DoesNotContain(held.OfType(), evt => evt.ContinuedChildRunId is not null); + Assert.Equal(4, _main.CallCount); + Assert.Equal(new[] { "prefix-probe-1", "prefix-probe-2" }, _start!.ProbeCallIds); + release.TrySetResult(); + await watcher.ExpectTerminatedAsync(owner, Ceiling, cancellationToken: TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + _main.PlannedResponses.Enqueue([AdmittedRecoveryProbe("recovered-fresh-window")]); + _main.PlannedResponses.Enqueue([new TextContent("The recovered parent completed its authorized probe.")]); + _main.NextResponseGate = recoveredResponse; + var recoveredSubscriber = CreateTestProbe(); + manager.Tell(new JoinSession(recoveredSubscriber) { SessionId = Session, Filter = OutputFilter.Full }, recoveredSubscriber.Ref); + await recoveredSubscriber.ExpectMsgAsync(Ceiling, cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(owner, await OwnerAsync()); + await AwaitAssertAsync(() => Assert.Equal(5, _main.CallCount), Ceiling, + cancellationToken: TestContext.Current.CancellationToken); + recoveredResponse.TrySetResult(); + await CompletedAsync(recoveredSubscriber); + var final = await ReadJournalPositionsAsync(); + Assert.Single(final, row => row.Event is ChildRunAccepted); + Assert.Single(final, row => row.Event is ChildRunEvent.TerminalRecorded); + Assert.Single(final, row => row.Event is ChildRunEvent.ResultPrepared); + var delivery = Assert.IsType(Assert.Single(final, + row => row.Event is ChildRunEvent.DeliveryAdmitted).Event); + var adoption = Assert.Single(final, row => row.Event is ToolTaskAdopted { ContinuedChildRunId: not null }); + var committed = Assert.IsType(adoption.Event); + Assert.True(committed.StartsChildContinuationWindow); + Assert.Equal(accepted.Run.RunId, committed.ContinuedChildRunId); + Assert.Equal(new[] { delivery.Input.InputId }, committed.InputIds); + Assert.True(SessionState.SameCanonicalContext(accepted.Run.OriginalContext, committed.TurnContext)); + var observation = Assert.Single(final, row => row.Event is ToolCallRecorded call + && call.ToolResult.ToolCallId == new ToolCallId("recovered-fresh-window")); + Assert.True(adoption.SequenceNr < observation.SequenceNr); + Assert.False(Assert.IsType(observation.Event).LoopObservation!.Synthetic); + Assert.Equal(new[] { "prefix-probe-1", "prefix-probe-2", "recovered-fresh-window" }, _start!.ProbeCallIds); + Assert.Equal(2, adoptionWrites); + Assert.Equal(6, _main.CallCount); + Assert.Equal(1, _child.CallCount); + } + finally + { + release.TrySetResult(); + _childRelease.TrySetResult(); + recoveredResponse.TrySetResult(); + } + } +} diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs index f80e0192d..e2f0c56fe 100644 --- a/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundChildOwnerAdversarialTests.cs @@ -386,6 +386,8 @@ private sealed class StartExecutor(SubAgentSpawner spawner, ToolRegistry registr public TaskCompletionSource ControlReply { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public async Task ExecuteAsync(FunctionCallContent call, ToolExecutionContext context, CancellationToken ct = default) { + if (call.Name is FileReadTool.ToolName or FileWriteTool.ToolName) + return await new DispatchingToolExecutor(registry, policy).ExecuteAsync(call, context, ct); if (call.Name is "neutral_probe" or "missing_receipt_probe") { ProbeRequesters.Add(context.RunScope.DefaultDeliveryTarget?.DestinationId); diff --git a/src/Netclaw.Actors.Tests/Sessions/BackgroundJobLineageTests.cs b/src/Netclaw.Actors.Tests/Sessions/BackgroundJobLineageTests.cs index b07808f95..5a1be2d46 100644 --- a/src/Netclaw.Actors.Tests/Sessions/BackgroundJobLineageTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/BackgroundJobLineageTests.cs @@ -28,7 +28,7 @@ public void Job_continuation_restores_original_evidence_after_a_fresh_task_and_s var state = SessionState.Empty.TrackBackgroundJob("bg-job:first", Job("first", checkpoint)) .TrackBackgroundJob("bg-job:second", Job("second", checkpoint)); var fresh = UserInput("fresh-task"); - state = state.Apply(fresh).Apply(new ToolTaskAdopted + state = state.Apply(fresh).Apply(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = fresh.TurnContext, InputIds = new[] { fresh.InputId } }); Assert.Equal("fresh-task", state.LoopCheckpoint.TaskId); state = state.CloseInputs(new[] { fresh.InputId }); @@ -36,7 +36,7 @@ public void Job_continuation_restores_original_evidence_after_a_fresh_task_and_s var delivery = Delivery("first"); state = state.Apply(delivery); Assert.True(state.TryGetBackgroundContinuation(delivery, out _, out _)); - state = state.Apply(new ToolTaskAdopted + state = state.Apply(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = delivery.TurnContext, InputIds = new[] { delivery.InputId }, ContinuedJobKey = "bg-job:first" }); Assert.Equal(Origin.TurnId.Value, state.LoopCheckpoint.TaskId); Assert.Equal("original-correction", state.LoopCheckpoint.LastBlockedAction); @@ -90,7 +90,7 @@ public void Rejected_delivery_report_is_durable_and_does_not_close_another_task( var user = UserInput("active-user"); var delivery = Delivery("orphan"); var later = UserInput("later-user"); - var state = SessionState.Empty.Apply(user).Apply(new ToolTaskAdopted + var state = SessionState.Empty.Apply(user).Apply(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = user.TurnContext, InputIds = new[] { user.InputId } }) .CloseInputs(new[] { user.InputId }); state = state with { LoopCheckpoint = state.LoopCheckpoint with { LastBlockedAction = "keep-correction" }, LoopReceiptFailure = true }; diff --git a/src/Netclaw.Actors.Tests/Sessions/ToolLoopReplayAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/ToolLoopReplayAdversarialTests.cs index 1c9ab9fbc..983a17213 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ToolLoopReplayAdversarialTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ToolLoopReplayAdversarialTests.cs @@ -290,7 +290,7 @@ private SessionState CompleteTwoRounds(SessionState state, string task) }).ToRecord() }; - private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new() + private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new(false) { SessionId = Session, TurnContext = input.TurnContext, InputIds = ids }; private SessionState Admit(SessionState state, PreparedToolCycleBatch batch, string taskId, IReadOnlyList refused) diff --git a/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceCheckpointTests.cs b/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceCheckpointTests.cs index 2cbb4be17..29dd52ed4 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceCheckpointTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ToolRecurrenceCheckpointTests.cs @@ -117,7 +117,7 @@ public void Adoption_survives_input_consumption_and_terminal_events_close_the_ta SessionId = session, InputId = InputId.New(), TurnContext = canonical, UserMessage = new SerializableChatMessage { Role = ChatRole.User, Content = "Read one file." } }; - var state = SessionState.Empty.Apply(input).Apply(new ToolTaskAdopted + var state = SessionState.Empty.Apply(input).Apply(new ToolTaskAdopted(false) { SessionId = session, TurnContext = canonical, InputIds = [input.InputId] }); @@ -153,14 +153,14 @@ public void Fresh_admission_preserves_active_evidence_until_durable_adoption() UserMessage = new SerializableChatMessage { Role = ChatRole.User, Content = "Read the old file." } }; var freshInput = oldInput with { InputId = InputId.New(), TurnContext = next }; - var state = SessionState.Empty.Apply(oldInput).Apply(new ToolTaskAdopted + var state = SessionState.Empty.Apply(oldInput).Apply(new ToolTaskAdopted(false) { SessionId = session, TurnContext = first, InputIds = [oldInput.InputId] }); state = state with { LoopReceiptFailure = true }; state = state.Apply(freshInput); Assert.Equal("first-task", state.LoopCheckpoint.TaskId); Assert.True(state.LoopReceiptFailure); Assert.Equal("first-task", state.AdoptedTaskContext!.TurnId); - var adopted = NetclawProtoMapper.FromProto(NetclawProtoMapper.ToProto(new ToolTaskAdopted + var adopted = NetclawProtoMapper.FromProto(NetclawProtoMapper.ToProto(new ToolTaskAdopted(false) { SessionId = session, TurnContext = next, InputIds = [freshInput.InputId] })); state = state.CloseInputs([oldInput.InputId]).Apply(adopted); Assert.Equal("next-task", state.LoopCheckpoint.TaskId); diff --git a/src/Netclaw.Actors.Tests/Sessions/ToolTaskRecoveryAdversarialTests.cs b/src/Netclaw.Actors.Tests/Sessions/ToolTaskRecoveryAdversarialTests.cs index 79bbb96d2..637350120 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ToolTaskRecoveryAdversarialTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ToolTaskRecoveryAdversarialTests.cs @@ -596,7 +596,7 @@ private async Task SeedAsync(SessionId session, IReadOnlyList eve Provenance = new SourceProvenance(TransportAuthenticity.Verified, PayloadTaint.Trusted), DefaultDeliveryTarget = new ChannelDeliveryTargetInfo("signalr", "destination", requester, requester) }; - private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new() + private static ToolTaskAdopted Adoption(InputAdmitted input, IReadOnlyList ids) => new(false) { SessionId = input.SessionId, TurnContext = input.TurnContext, InputIds = ids }; private static SendUserMessage Restart(SessionId session) => new() { diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs index 457565eae..d621f8ad2 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.Lifecycle.cs @@ -59,7 +59,7 @@ public void Atomic_child_delivery_restores_the_original_task_after_a_fresh_compl var pending = Assert.Single(state.PendingInputs); Assert.Equal(accepted.RunId, pending.SourceChildRunId); Assert.Equal(pending.InputId, state.ChildRuns[accepted.RunId].DeliveryInputId); - state = state.Apply(RoundTrip(new ToolTaskAdopted + state = state.Apply(RoundTrip(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = accepted.OriginalContext, InputIds = [pending.InputId], ContinuedChildRunId = accepted.RunId @@ -108,7 +108,7 @@ public void Coalesced_child_adoption_retains_both_pairs_after_input_closure_and_ state = state.Apply(new ChildRunEvent.DeliveryAdmitted(input) { SessionId = Session, RunId = run.RunId, RecordedAtMs = index + 1 }); ids.Add(input.InputId); } - var adoption = RoundTrip(new ToolTaskAdopted + var adoption = RoundTrip(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = runs[^1].OriginalContext, InputIds = ids.ToArray(), ContinuedChildRunId = runs[^1].RunId diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs index d9d44f329..d329fd1ad 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildAcceptanceTests.cs @@ -148,7 +148,7 @@ private static SessionState AdmittedTask(string turn, SessionState prior) SessionId = Session, InputId = new InputId("input-" + turn), TurnContext = context, UserMessage = new SerializableChatMessage { Content = turn } }; - return prior.Apply(input).Apply(new ToolTaskAdopted + return prior.Apply(input).Apply(new ToolTaskAdopted(false) { SessionId = Session, TurnContext = context, InputIds = [input.InputId] }); diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.DetectorWindow.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.DetectorWindow.cs new file mode 100644 index 000000000..2ea3ea3e0 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.DetectorWindow.cs @@ -0,0 +1,302 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Google.Protobuf; +using System.Collections.Immutable; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Jobs; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Serialization; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Handlers; +using Netclaw.Actors.SubAgents; +using Netclaw.Actors.Tools; +using Netclaw.Configuration; +using Netclaw.Tools; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; +using StoredRole = Netclaw.Actors.Protocol.ChatRole; + +namespace Netclaw.Actors.Tests.SubAgents; + +public sealed partial class BackgroundChildLedgerAdversarialTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_child_window_marker_changes_detector_evidence_without_new_authority(bool startsWindow) + { + var (state, run, adoption) = WindowDelivery(startsWindow, false); + Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, state.LoopCheckpoint)); + var wire = NetclawProtoMapper.ToProto(adoption).ToByteArray(); + var fields = new List(); + using (var reader = new CodedInputStream(wire)) + while (reader.ReadTag() is var tag && tag != 0) + { + fields.Add(WireFormat.GetTagFieldNumber(tag)); + reader.SkipLastField(); + } + Assert.Equal(startsWindow, fields.Contains(7)); + var decoded = RoundTrip(adoption); + Assert.Equal(startsWindow, decoded.StartsChildContinuationWindow); + var adopted = SessionState.FromSnapshot(RoundTrip(state.Apply(decoded).ToSnapshot())); + Assert.True(SessionState.SameCanonicalContext(run.OriginalContext, adopted.AdoptedTaskContext!)); + Assert.Equal(run.ParentCheckpoint.TaskId, adopted.LoopCheckpoint.TaskId); + Assert.NotEqual(run.OriginalContext.TurnId, adopted.LoopCheckpoint.TaskId); + Assert.Equal(adoption.InputIds, adopted.AdoptedTaskInputIds); + Assert.Equal(ChildRunDelivery.Body(run), Assert.Single(adopted.PendingInputs).UserMessage.Content); + if (!startsWindow) + { + Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, adopted.LoopCheckpoint)); + return; + } + Assert.Empty(adopted.LoopCheckpoint.Entries); + Assert.Empty(adopted.LoopCheckpoint.ColdKeys); + Assert.Empty(adopted.LoopCheckpoint.AdjacentHistory); + Assert.Null(adopted.LoopCheckpoint.LastBlockedAction); + Assert.True(BackgroundChildRun.SameCheckpoint(adopted.LoopCheckpoint, adopted.ChildRuns[run.RunId].ParentCheckpoint)); + } + + [Theory] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void A_new_child_window_keeps_every_existing_missing_receipt_failure(bool retained, bool current) + { + var (state, _, adoption) = WindowDelivery(true, retained); + state = state with { LoopReceiptFailure = current }; + var adopted = SessionState.FromSnapshot(RoundTrip(state.Apply(RoundTrip(adoption)).ToSnapshot())); + Assert.True(adopted.LoopReceiptFailure); + Assert.All(adopted.ChildRuns.Values, child => Assert.True(child.ParentReceiptFailure)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void Duplicate_child_adoption_keeps_new_evidence_after_consumption_and_compaction(bool consume, bool compact) + { + var (state, _, adoption) = WindowDelivery(true, false); + state = state.Apply(RoundTrip(adoption)); + state = ObserveWindowProbe(state, "new-window-1"); + state = ObserveWindowProbe(state, "new-window-2"); + if (consume) + state = state.CloseInputs(adoption.InputIds); + if (compact) + state = state.Apply(RoundTrip(new SessionCompacted + { + SessionId = Owner, Summary = "The child result remains in durable session state.", + CompactedMessages = [new SerializableChatMessage { Role = StoredRole.User, Content = "The retained task summary." }] + })); + state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + var before = Codec(state.ToSnapshot()).ToBinary(state.ToSnapshot()); + var duplicate = state.Apply(RoundTrip(adoption)); + Assert.Equal(before, Codec(duplicate.ToSnapshot()).ToBinary(duplicate.ToSnapshot())); + var tracker = new TurnStateTracker(); + tracker.RestoreCheckpoint(duplicate.LoopCheckpoint); + Assert.Equal(ToolCycleDecisionKind.Correct, tracker.EvaluateBeforeDispatch(WindowProbe("next-window-3")).Kind); + RejectUnchanged(state, () => state.Apply(adoption with { TurnContext = adoption.TurnContext with { RequesterSenderId = new SenderId("other") } })); + RejectUnchanged(state, () => state.Apply(adoption with { InputIds = [new InputId("other-result")] })); + } + + [Fact] + public void An_unrelated_adoption_cannot_request_a_child_window() + { + var (state, run) = Acceptance(Owner, slash: true); + RejectUnchanged(state, () => state.Apply(new ToolTaskAdopted(true) + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = run.OriginInputIds })); + Assert.Equal(state, state.Apply(new ToolTaskAdopted(false) + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = run.OriginInputIds })); + } + + [Theory] + [InlineData("name")] + [InlineData("arguments")] + [InlineData("body")] + [InlineData("result-name")] + [InlineData("orphan-result")] + [InlineData("assistant-role")] + public void A_duplicate_after_consumption_rejects_corruption_of_its_retained_attributed_pair(string fault) + { + var (state, _, adoption) = WindowDelivery(true, false); + state = state.Apply(RoundTrip(adoption)).CloseInputs(adoption.InputIds); + var call = Assert.Single(state.History, message => message.ToolCalls.Any(item => item.CallId.Value == "window-framework-result")); + var result = Assert.Single(state.History, message => message.ToolCallId?.Value == "window-framework-result"); + Assert.Equal(state, state.Apply(RoundTrip(adoption))); + var changed = fault switch + { + "name" => call with { ToolCalls = [call.ToolCalls[0] with { Name = new ToolName("shell") }] }, + "arguments" => call with { ToolCalls = [call.ToolCalls[0] with { ArgumentsJson = "{}" }] }, + "body" => result with { Content = "The result replaces the actual durable outcome." }, + "result-name" => result with { Name = "unrelated_tool" }, + "assistant-role" => call with { Role = StoredRole.User }, + "orphan-result" => call, + _ => throw new ArgumentOutOfRangeException(nameof(fault)) + }; + state = state with + { + History = fault == "orphan-result" ? state.History.Remove(call) + : state.History.Select(message => ReferenceEquals(message, fault is "body" or "result-name" ? result : call) ? changed : message).ToImmutableList() + }; + RejectUnchanged(state, () => state.Apply(RoundTrip(adoption))); + } + + [Fact] + public void An_unrelated_tool_pair_with_the_same_arguments_and_body_cannot_replace_delivery_identity() + { + var (state, run, adoption) = WindowDelivery(true, false); + state = state.Apply(RoundTrip(adoption)).CloseInputs(adoption.InputIds); + var call = ChildRunDelivery.Call(run, new ToolCallId("unrelated-call")); + call = call with { ToolCalls = [call.ToolCalls[0] with { Name = new ToolName("unrelated_tool") }] }; + state = state with { History = state.History.Add(call).Add(new SerializableChatMessage + { Role = StoredRole.Tool, ToolCallId = new ToolCallId("unrelated-call"), Name = "unrelated_tool", Content = ChildRunDelivery.Body(run) }) }; + Assert.Equal(state, state.Apply(RoundTrip(adoption))); + } + + [Fact] + public void A_canonical_shell_job_continuation_cannot_request_a_child_window() + { + var (_, run) = Acceptance(Owner, slash: true); + const string key = "bg-job:canonical-job"; + var origin = new BackgroundJobOrigin(new TurnId(run.ParentCheckpoint.TaskId), new ToolCallId("job-start")); + var job = new ActiveJobInfo + { + JobId = new BackgroundJobId("canonical-job"), Command = "echo result", Rationale = "Read the assigned job result.", + StartedAtMs = 0, Audience = run.OriginalContext.Audience, Boundary = run.OriginalContext.Boundary!.Value, + LineageVersion = 1, Origin = origin, OriginCheckpoint = run.ParentCheckpoint + }; + var input = new InputAdmitted + { + SessionId = Owner, InputId = new InputId("job-result"), SourceMessageId = key, + SourceBackgroundJobId = new BackgroundJobId(key), BackgroundJobLineageVersion = 1, BackgroundJobOrigin = origin, + TurnContext = run.OriginalContext with + { + TurnId = key, SourceKind = BackgroundJobManagerActor.SourceKind, + RequesterPrincipal = PrincipalClassification.VerifiedAutomation, TransportAuthenticity = TransportAuthenticity.LocalProcess + }, + UserMessage = new SerializableChatMessage { Role = StoredRole.User, Content = "The job returned its result." } + }; + var state = SessionState.Empty.TrackBackgroundJob(key, job).Apply(input); + Assert.True(state.TryGetBackgroundContinuation(input, out _, out _)); + var ordinary = new ToolTaskAdopted(false) + { SessionId = Owner, TurnContext = input.TurnContext, InputIds = [input.InputId], ContinuedJobKey = key }; + Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, state.Apply(RoundTrip(ordinary)).LoopCheckpoint)); + RejectUnchanged(state, () => state.Apply(RoundTrip(ordinary with { StartsChildContinuationWindow = true }))); + } + + [Fact] + public void A_later_sibling_uses_current_evidence_and_starts_its_own_first_window() + { + var (state, first, firstAdoption) = WindowDelivery(true, false); + var second = first with + { + RunId = new SubAgentRunId("later-sibling"), ScopeId = new SubAgentScopeId($"{Owner.Value}/subagent/worker/later-sibling"), + StartKey = new ChildRunStartKey.Slash(new InputId("origin-two")) { SessionId = Owner, TurnId = new TurnId(first.OriginalContext.TurnId) }, + Terminal = null, PreparedTerminal = null, TerminalSequenceNr = null, DeliveryInputId = null + }; + var job = new ActiveJobInfo + { + JobId = new BackgroundJobId("same-parent-job"), Command = "echo result", Rationale = "Read the assigned result.", + StartedAtMs = 0, Audience = first.OriginalContext.Audience, Boundary = first.OriginalContext.Boundary!.Value, + LineageVersion = 1, Origin = new BackgroundJobOrigin(new TurnId(first.ParentCheckpoint.TaskId), new ToolCallId("job-start")), + OriginCheckpoint = first.ParentCheckpoint + }; + state = state.TrackBackgroundJob("bg-job:same-parent-job", job); + state = state.Apply(RoundTrip(new ChildRunAccepted { SessionId = Owner, Run = second })); + state = state.Apply(RoundTrip(firstAdoption)); + Assert.Empty(state.ChildRuns[second.RunId].ParentCheckpoint.Entries); + Assert.True(BackgroundChildRun.SameCheckpoint(state.LoopCheckpoint, + state.ActiveBackgroundJobs["bg-job:same-parent-job"].OriginCheckpoint!)); + state = ObserveWindowProbe(state, "between-siblings-1"); + state = ObserveWindowProbe(state, "between-siblings-2"); + state = state.CloseInputs(firstAdoption.InputIds); + second = state.ChildRuns[second.RunId]; + Assert.Equal(2, Assert.Single(second.ParentCheckpoint.Entries).EqualRounds); + Assert.Equal(2, Assert.Single(state.ActiveBackgroundJobs["bg-job:same-parent-job"].OriginCheckpoint!.Entries).EqualRounds); + var terminal = Terminal(second); + state = state.Apply(RoundTrip(new ChildRunEvent.TerminalRecorded(terminal, 2) + { SessionId = Owner, RunId = second.RunId, RecordedAtMs = 123461 })); + state = state.Apply(RoundTrip(new ChildRunEvent.ResultPrepared(terminal, 2) + { SessionId = Owner, RunId = second.RunId, RecordedAtMs = 123462 })); + second = state.ChildRuns[second.RunId]; + var input = Delivery(second, "second-framework-result"); + state = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { SessionId = Owner, RunId = second.RunId, RecordedAtMs = 123463 })); + Assert.Equal(2, Assert.Single(state.LoopCheckpoint.Entries).EqualRounds); + var next = state.Apply(RoundTrip(new ToolTaskAdopted(true) + { SessionId = Owner, TurnContext = second.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = second.RunId })); + Assert.Empty(next.LoopCheckpoint.Entries); + Assert.Empty(next.ActiveBackgroundJobs["bg-job:same-parent-job"].OriginCheckpoint!.Entries); + Assert.Equal(first.ParentCheckpoint.TaskId, next.LoopCheckpoint.TaskId); + Assert.True(SessionState.SameCanonicalContext(first.OriginalContext, next.AdoptedTaskContext!)); + } + + [Fact] + public void Only_the_committed_adoption_changes_the_retained_detector_window() + { + var (state, run) = AcceptedSlash(false); + var retained = state.LoopCheckpoint; + var terminal = Terminal(run); + var recorded = new ChildRunEvent.TerminalRecorded(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123458 }; + var prepared = new ChildRunEvent.ResultPrepared(terminal, 1) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123459 }; + state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); + Assert.True(BackgroundChildRun.SameCheckpoint(retained, state.LoopCheckpoint)); + state = SessionState.FromSnapshot(RoundTrip(state.Apply(RoundTrip(recorded)).ToSnapshot())); + Assert.True(BackgroundChildRun.SameCheckpoint(retained, state.LoopCheckpoint)); + state = SessionState.FromSnapshot(RoundTrip(state.Apply(RoundTrip(prepared)).ToSnapshot())); + Assert.True(BackgroundChildRun.SameCheckpoint(retained, state.LoopCheckpoint)); + run = state.ChildRuns[run.RunId]; + var input = Delivery(run, "committed-window-result"); + state = SessionState.FromSnapshot(RoundTrip(state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })).ToSnapshot())); + Assert.True(BackgroundChildRun.SameCheckpoint(retained, state.LoopCheckpoint)); + var adoption = new ToolTaskAdopted(true) + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId }; + var adopted = SessionState.FromSnapshot(RoundTrip(state.Apply(RoundTrip(adoption)).ToSnapshot())); + Assert.Empty(adopted.LoopCheckpoint.Entries); + Assert.Equal(retained.TaskId, adopted.LoopCheckpoint.TaskId); + Assert.True(SessionState.SameCanonicalContext(run.OriginalContext, adopted.AdoptedTaskContext!)); + } + + private (SessionState State, BackgroundChildRun Run, ToolTaskAdopted Adoption) WindowDelivery(bool marker, bool missing) + { + var (state, run) = PreparedSlash(missing); + var input = Delivery(run, "window-framework-result"); + state = state.Apply(RoundTrip(new ChildRunEvent.DeliveryAdmitted(input) + { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); + return (state, run, new ToolTaskAdopted(marker) + { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId }); + } + + private PreparedToolCycleBatch WindowProbe(string id) => ToolCycleSignatureFactory.Prepare( + [new FunctionCallContent(id, "window_probe", new Dictionary + { ["value"] = "neutral", ["_rationale"] = "Read the unchanged neutral value." })], _tools); + + private SessionState ObserveWindowProbe(SessionState state, string id) + { + var tracker = new TurnStateTracker(); + tracker.RestoreCheckpoint(state.LoopCheckpoint); + var batch = WindowProbe(id); + Assert.Equal(ToolCycleDecisionKind.Execute, tracker.EvaluateBeforeDispatch(batch).Kind); + state = state.ApplyLoopAdmission(RoundTrip(new ToolBatchStarted + { + SessionId = Owner, LoopDelta = tracker.CaptureDelta(state.LoopCheckpoint.TaskId, state.LoopCheckpoint), + LoopAdmission = new ToolLoopAdmission + { + TaskId = state.LoopCheckpoint.TaskId, ActionHash = batch.Action.Value, + Calls = batch.Calls.Select(call => new ToolLoopPreparedCall(call.CallId.Value, call.ToolName.Value, + call.ArgumentsHash, call.AllowsPendingJob)).ToArray() + } + })); + return state.ApplyLoopObservation(RoundTrip(new ToolCallRecorded + { + SessionId = Owner, LoopObservation = ToolCycleSignatureFactory.CreateObservation(id, + new ToolInvocationReceipt.OtherOutcome(ToolInvocationOutcomeCategory.TransientFailure), "The exact neutral outcome.", false) + })); + } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs index ecf80b422..2def99677 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.Lifecycle.cs @@ -96,7 +96,7 @@ public void Original_turn_continuation_restores_detector_evidence_before_the_dup // The authority turn stays unchanged, but the current detector state differs from the retained task. state = state with { LoopCheckpoint = new ToolLoopCheckpoint { TaskId = "different-current-task" }, LoopReceiptFailure = !receiptFailure }; state = SessionState.FromSnapshot(RoundTrip(state.ToSnapshot())); - var adoption = new ToolTaskAdopted + var adoption = new ToolTaskAdopted(false) { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId }; RejectUnchanged(state, () => state.Apply(adoption with { ContinuedChildRunId = null })); RejectUnchanged(state, () => state.Apply(adoption with { ContinuedJobKey = "foreign-job" })); @@ -143,7 +143,7 @@ SessionState AdmitPair(BackgroundChildRun sibling) { SessionId = Owner, RunId = run.RunId, RecordedAtMs = 123460 })); return SessionState.FromSnapshot(RoundTrip(pair.ToSnapshot())); } - ToolTaskAdopted AdoptPair(SessionState pair) => new() + ToolTaskAdopted AdoptPair(SessionState pair) => new(false) { SessionId = Owner, TurnContext = first.OriginalContext, InputIds = pair.PendingInputs.Select(input => input.InputId).ToArray(), ContinuedChildRunId = second.RunId }; var invalid = AdmitPair(incompatible); RejectUnchanged(invalid, () => invalid.Apply(AdoptPair(invalid))); @@ -201,7 +201,7 @@ public void A_snapshot_cannot_replace_the_canonical_pending_child_result(string Assert.Equal(input.UserMessage.Role, restoredMessage.Role); Assert.Equal(input.UserMessage.Name, restoredMessage.Name); Assert.Equal(input.UserMessage.ToolCallId, restoredMessage.ToolCallId); - var adopted = valid.Apply(RoundTrip(new ToolTaskAdopted + var adopted = valid.Apply(RoundTrip(new ToolTaskAdopted(false) { SessionId = Owner, TurnContext = run.OriginalContext, InputIds = [input.InputId], ContinuedChildRunId = run.RunId })); Assert.True(BackgroundChildRun.SameCheckpoint(run.ParentCheckpoint, adopted.LoopCheckpoint)); Assert.Equal(input.UserMessage.Content, Assert.Single(adopted.PendingInputs).UserMessage.Content); diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs index 9785f37e7..f77b65d0d 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.TerminalPositions.cs @@ -29,7 +29,7 @@ public async Task Actual_recovery_checks_terminal_positions_before_any_child_res { var (premise, prototype) = Acceptance(Owner, slash: true); var input = premise.PendingInputs[0]; - var adoption = new ToolTaskAdopted + var adoption = new ToolTaskAdopted(false) { SessionId = Owner, TurnContext = input.TurnContext, InputIds = [input.InputId] }; var state = SessionState.Empty.Apply(input).Apply(adoption); var run = prototype with diff --git a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs index faf5fa1d1..1084df34d 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/BackgroundChildLedgerAdversarialTests.cs @@ -261,7 +261,7 @@ private static (SessionState State, BackgroundChildRun Run) Acceptance(SessionId var first = new InputAdmitted { SessionId = session, InputId = new InputId("origin-one"), TurnContext = context, UserMessage = new SerializableChatMessage { Role = StoredRole.User, Content = "Inspect a neutral task." } }; var second = first with { InputId = new InputId("origin-two") }; - var state = SessionState.Empty.Apply(first).Apply(second).Apply(new ToolTaskAdopted + var state = SessionState.Empty.Apply(first).Apply(second).Apply(new ToolTaskAdopted(false) { SessionId = session, TurnContext = context, InputIds = [first.InputId, second.InputId] }); // The ledger must retain detector identity independently from the original authority turn. state = state with { LoopCheckpoint = new ToolLoopCheckpoint diff --git a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs index ddc5457f9..5778f4ac1 100644 --- a/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs +++ b/src/Netclaw.Actors/Serialization/NetclawProtoMapper.cs @@ -233,7 +233,8 @@ internal static Proto.ToolTaskAdoptedProto ToProto(ToolTaskAdopted evt) { var proto = new Proto.ToolTaskAdoptedProto { - SessionId = ToProto(evt.SessionId), TurnContext = ToProto(evt.TurnContext), AdoptedAtMs = evt.AdoptedAtMs + SessionId = ToProto(evt.SessionId), TurnContext = ToProto(evt.TurnContext), AdoptedAtMs = evt.AdoptedAtMs, + StartsChildContinuationWindow = evt.StartsChildContinuationWindow }; proto.InputIds.AddRange(evt.InputIds.Select(static id => id.Value)); if (evt.ContinuedJobKey is not null) @@ -243,7 +244,7 @@ internal static Proto.ToolTaskAdoptedProto ToProto(ToolTaskAdopted evt) return proto; } - internal static ToolTaskAdopted FromProto(Proto.ToolTaskAdoptedProto proto) => new() + internal static ToolTaskAdopted FromProto(Proto.ToolTaskAdoptedProto proto) => new(proto.StartsChildContinuationWindow) { SessionId = FromProto(proto.SessionId), TurnContext = proto.TurnContext is null diff --git a/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto b/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto index 94c172e16..c2e0af405 100644 --- a/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto +++ b/src/Netclaw.Actors/Serialization/Protos/netclaw_messages.proto @@ -121,6 +121,7 @@ message ToolTaskAdoptedProto { int64 adopted_at_ms = 4; optional string continued_job_key = 5; optional string continued_child_run_id = 6; + bool starts_child_continuation_window = 7; } message InputClosedProto { diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index d14d9419a..bd76cbe13 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2673,7 +2673,7 @@ private void ContinueIncomingUserMessage(SendUserMessage cmd) private void AdoptToolTask(InputAdmitted admitted, IReadOnlyList inputIds, Action continuation) { - var evt = new ToolTaskAdopted + var evt = new ToolTaskAdopted(admitted.SourceChildRunId is not null) { SessionId = _sessionId, TurnContext = admitted.TurnContext, InputIds = inputIds.ToArray(), AdoptedAtMs = NowMs(), ContinuedJobKey = admitted.SourceBackgroundJobId?.Value, ContinuedChildRunId = admitted.SourceChildRunId diff --git a/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs b/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs index ac772abae..969f3ed53 100644 --- a/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs +++ b/src/Netclaw.Actors/Sessions/SessionProtocol.Events.cs @@ -93,7 +93,7 @@ public sealed record InputAdmitted : ISessionEvent } /// Commits canonical task authority before the next model request. - public sealed record ToolTaskAdopted : ISessionEvent + public sealed record ToolTaskAdopted(bool StartsChildContinuationWindow) : ISessionEvent { public Netclaw.Tools.SubAgentRunId? ContinuedChildRunId { get; init; } public string? ContinuedJobKey { get; init; } diff --git a/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs b/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs index 9635296e2..0eb630cab 100644 --- a/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs +++ b/src/Netclaw.Actors/Sessions/SessionState.ChildRuns.cs @@ -91,8 +91,25 @@ ChildRunEvent.ResultPrepared prepared when run.Terminal is not null && run.Prepa private SessionState AdoptChildContinuation(ToolTaskAdopted evt) { - if (evt.ContinuedJobKey is not null || evt.InputIds.Count == 0 || evt.InputIds.Distinct().Count() != evt.InputIds.Count - || !PendingInputs.Take(evt.InputIds.Count).Select(static input => input.InputId).SequenceEqual(evt.InputIds)) + if (evt.ContinuedJobKey is not null || evt.InputIds.Count == 0 || evt.InputIds.Distinct().Count() != evt.InputIds.Count) + throw new InvalidDataException("A child continuation does not name the next canonical input prefix."); + // Consumed inputs and compacted pairs can leave the transcript. Durable adoption remains authoritative. + var alreadyAdopted = AdoptedTaskContext is { } && AdoptedTaskInputIds.SequenceEqual(evt.InputIds); + if (alreadyAdopted) + { + if (evt.ContinuedChildRunId is not { } id || !ChildRuns.TryGetValue(id, out var duplicate) + || duplicate.DeliveryInputId != evt.InputIds[^1] || duplicate.PreparedTerminal is null + || !duplicate.StartBatchSettled || duplicate.StartKey.SessionId != evt.SessionId + || !SameCanonicalContext(duplicate.OriginalContext, evt.TurnContext) + || !SameCanonicalContext(AdoptedTaskContext!, evt.TurnContext)) + throw new InvalidDataException("A repeated child adoption differs from its canonical authority."); + if (!PendingInputs.Take(evt.InputIds.Count).Select(static input => input.InputId).SequenceEqual(evt.InputIds)) + { + ValidateConsumedChildPairs(evt); + return this; + } + } + if (!PendingInputs.Take(evt.InputIds.Count).Select(static input => input.InputId).SequenceEqual(evt.InputIds)) throw new InvalidDataException("A child continuation does not name the next canonical input prefix."); var prefix = PendingInputs.Take(evt.InputIds.Count).ToArray(); var run = GetChildContinuation(prefix[^1]); @@ -122,24 +139,66 @@ private SessionState AdoptChildContinuation(ToolTaskAdopted evt) history = history.Add(canonicalCall).Add(input.UserMessage); continue; } - if (existing is not [var assistant] || assistant.Role != canonicalCall.Role - || assistant.Content != canonicalCall.Content || assistant.Name != canonicalCall.Name - || assistant.ToolCallId != canonicalCall.ToolCallId || assistant.MediaReferences.Count != 0 - || !assistant.ToolCalls.SequenceEqual(canonicalCall.ToolCalls)) - throw new InvalidDataException("A repeated child continuation differs from its canonical call."); - var results = ParkedToolBatchHistory.FindToolResultsFor(history, assistant); - if (results is not [var result] || result.Role != input.UserMessage.Role - || result.ToolCallId != callId || result.Name != input.UserMessage.Name - || result.Content != input.UserMessage.Content || result.ToolCalls.Count != 0 || result.MediaReferences.Count != 0) - throw new InvalidDataException("A repeated child continuation differs from its canonical result."); + ValidateChildPair(history, canonicalCall, input.UserMessage); } - return this with + if (alreadyAdopted) + return this; + var next = this with { History = history, AdoptedTaskContext = run.OriginalContext, AdoptedTaskInputIds = evt.InputIds, - LoopCheckpoint = run.ParentCheckpoint, LoopReceiptFailure = run.ParentReceiptFailure, + LoopCheckpoint = evt.StartsChildContinuationWindow + ? new ToolLoopCheckpoint { TaskId = run.ParentCheckpoint.TaskId } + : run.ParentCheckpoint, + LoopReceiptFailure = evt.StartsChildContinuationWindow + ? LoopReceiptFailure || run.ParentReceiptFailure + : run.ParentReceiptFailure, LoopAdmission = null, LoopObservations = [] }; + return evt.StartsChildContinuationWindow ? next.RefreshJobEvidence() : next; + } + + private void ValidateConsumedChildPairs(ToolTaskAdopted evt) + { + var runs = ChildRuns.Values.Where(run => run.DeliveryInputId is { } input && evt.InputIds.Contains(input)).ToArray(); + if (runs.Length != evt.InputIds.Count) + throw new InvalidDataException("A repeated child adoption lacks its canonical delivered runs."); + foreach (var run in runs) + { + if (run.PreparedTerminal is null || !run.StartBatchSettled || run.StartKey.SessionId != evt.SessionId + || !SameCanonicalContext(run.OriginalContext, evt.TurnContext)) + throw new InvalidDataException("A repeated child adoption differs from its canonical authority."); + var body = ChildRunDelivery.Body(run); + var toolName = ChildRunDelivery.ToolName(run); + // Retained content can identify a pair. Its absence after compaction does not identify a new adoption. + var ids = History.SelectMany(message => message.ToolCalls) + .Where(call => call.Name.Value == toolName + && call.ArgumentsJson == ChildRunDelivery.Call(run, call.CallId).ToolCalls[0].ArgumentsJson) + .Select(call => call.CallId) + .Concat(History.Where(message => message.Role == ChatRole.Tool && message.Name == toolName + && message.Content == body && message.ToolCallId is not null).Select(message => message.ToolCallId!.Value)) + .Distinct(); + foreach (var id in ids) + ValidateChildPair(History, ChildRunDelivery.Call(run, id), new SerializableChatMessage + { Role = ChatRole.Tool, ToolCallId = id, Name = toolName, Content = body }); + } + } + + private static void ValidateChildPair(IReadOnlyList history, + SerializableChatMessage canonicalCall, SerializableChatMessage canonicalResult) + { + var callId = canonicalResult.ToolCallId!.Value; + var existing = history.Where(message => message.ToolCalls.Any(call => call.CallId == callId)).ToArray(); + if (existing is not [var assistant] || assistant.Role != canonicalCall.Role + || assistant.Content != canonicalCall.Content || assistant.Name != canonicalCall.Name + || assistant.ToolCallId != canonicalCall.ToolCallId || assistant.MediaReferences.Count != 0 + || !assistant.ToolCalls.SequenceEqual(canonicalCall.ToolCalls)) + throw new InvalidDataException("A repeated child continuation differs from its canonical call."); + var results = ParkedToolBatchHistory.FindToolResultsFor(history, assistant); + if (results is not [var result] || result.Role != canonicalResult.Role + || result.ToolCallId != callId || result.Name != canonicalResult.Name + || result.Content != canonicalResult.Content || result.ToolCalls.Count != 0 || result.MediaReferences.Count != 0) + throw new InvalidDataException("A repeated child continuation differs from its canonical result."); } private static bool SameTerminalOutcome(ChildRunTerminal original, ChildRunTerminal prepared) diff --git a/src/Netclaw.Actors/Sessions/SessionState.cs b/src/Netclaw.Actors/Sessions/SessionState.cs index 44d4e803c..9f80a4180 100644 --- a/src/Netclaw.Actors/Sessions/SessionState.cs +++ b/src/Netclaw.Actors/Sessions/SessionState.cs @@ -136,6 +136,8 @@ public SessionState Apply(ToolTaskAdopted evt) throw new InvalidDataException($"An adopted task has invalid authority: {reason}"); if (evt.SessionId != context.SessionId) throw new InvalidDataException("An adopted task has a different session identity."); + if (evt.StartsChildContinuationWindow && evt.ContinuedChildRunId is null) + throw new InvalidDataException("Only a canonical child continuation can start a new recurrence window."); if (evt.ContinuedChildRunId is not null) return AdoptChildContinuation(evt); if (PendingInputs.Take(evt.InputIds.Count).Any(static input => input.SourceChildRunId is not null)) From 45a9f831b65618a5a035d619b2b1034407f448bd Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 15:26:13 +0000 Subject: [PATCH 19/22] Align child prompt specifications with scoped instructions --- .../specs/netclaw-subagents/spec.md | 29 +++++++++++++------ .../changes/background-child-runs/tasks.md | 23 ++++++++++++++- openspec/specs/netclaw-subagents/spec.md | 29 +++++++++++++------ 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md index 08500946f..73f0dd184 100644 --- a/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md +++ b/openspec/changes/background-child-runs/specs/netclaw-subagents/spec.md @@ -10,9 +10,11 @@ Subagents SHALL NOT persist durable memory, stream direct durable-memory writes, For subagent execution launched from skill metadata routing, the subagent SHALL remain an isolated worker by default: -- It SHALL NOT inherit the main session identity prompt stack unless a future explicit setting enables it. -- It SHALL NOT auto-load repo-local `AGENTS.md` unless a future explicit setting enables it. -- It SHALL inherit audience/boundary context from the launch invocation. +- It SHALL receive the embedded operating core for its audience, followed by the operator's deployment `AGENTS.md`. +- It SHALL NOT inherit `SOUL.md` or `TOOLING.md`. +- A child with a non-Public audience SHALL receive available project instructions from the inherited project directory. +- A Public child SHALL NOT receive project-local instructions. +- It SHALL inherit audience/boundary context from the launch invocation. Prompt content SHALL NOT widen that authority. The child SHALL use the shared exact recurrence, repeat exception, and feedback-round contracts in `turn-loop-governance`. It SHALL retain its detector evidence for its actor-local task lifetime. @@ -64,17 +66,26 @@ All explicit and routed starts SHALL use `background-subagent-runs` acceptance, - **THEN** that path remains unavailable or denied - **AND** the child returns findings for parent policy review instead -#### Scenario: Routed subagent does not inherit main identity prompt stack +#### Scenario: Routed subagent receives scoped operating guidance -- **GIVEN** a slash skill routes through `metadata.subagent` +- **GIVEN** a slash skill routes through `metadata.subagent` and a deployment playbook exists - **WHEN** its isolated child prompt is assembled -- **THEN** the main session identity stack remains absent by default +- **THEN** the embedded operating core for its audience appears before the deployment `AGENTS.md` +- **AND** neither `SOUL.md` nor `TOOLING.md` is included -#### Scenario: Routed subagent does not auto-load repo AGENTS +#### Scenario: Routed subagent receives inherited project instructions -- **GIVEN** a slash skill routes through `metadata.subagent` +- **GIVEN** a Personal or Team slash activation inherits a project directory with an `AGENTS.md` file - **WHEN** its isolated child prompt is assembled -- **THEN** repo-local `AGENTS.md` remains absent by default +- **THEN** available project instructions appear after the operating guidance and before the child role prompt +- **AND** the project instructions come from the inherited project directory + +#### Scenario: Public routed subagent excludes project instructions + +- **GIVEN** a Public slash activation inherits a project directory with an `AGENTS.md` file +- **WHEN** its isolated child prompt is assembled +- **THEN** the stripped Public operating core and available deployment playbook remain present +- **AND** project-local instructions remain absent #### Scenario: Routed subagent inherits launch audience diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index c0f17a86d..1b6e1cd1e 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -113,6 +113,18 @@ Combined boundary and mutation evidence at `153d9223`: - Two unrelated shell-config-read mutations retain their timeout dispositions. They supply no assertion-kill claim for this gate. - Independent review approves tasks 7.3 and 7.8. These checks prove no detector activation, days-long productivity, or full model acceptance. +Independent contract review at `a3488e95` and combined `f48e9fbe`: + +- The actor runtime bytes match across both revisions. The audit finds no new runtime defect in the inspected background paths. +- The actual guard TRX records show 177 passes. The separate authority TRX records show six passes. Neither run skips a case. +- Independent review `80ceb1b3` retains the fault controls and their valid pairs. All eleven listed source and test pins match the combined candidate. +- The retained PR 2 snapshot reports 29 successful checks. Both native logs report 29 tapes and 10 scenarios passed. +- Current observation review `7742d477` proves one held-child reply, canonical result consumption, and a complete artifact read. It supplies no five-pass acceptance claim. +- The delta now distinguishes the audience operating core and deployment playbook from separately scoped project instructions. It excludes `SOUL.md` and `TOOLING.md`. +- Public children retain the existing exclusion of project-local instructions. Prompt content grants no additional authority. +- This specification repair changes no runtime, prefill behavior, prompt bytes, or test assertion. It adds no fresh model evidence for guide version `1.0.3`. +- Task 7.9 remains open for independent review of the integrated result. Tasks 7.2, 8.6, and 8.7 also remain open. + ## 8. Docs, operational guidance, and handoff - [x] 8.1 Rename the glossary entry to approval prompt with a legacy-anchor alias; verify old links still resolve and exact code identifiers remain unchanged. @@ -122,7 +134,16 @@ Combined boundary and mutation evidence at `153d9223`: - [x] 8.5 Run targeted identity/tool/skill evals for those changed instructions; verify selected resource and control receipts satisfy the actual task. - [ ] 8.6 Run applicable builds, required tests, Slopwatch, and copyright-header verification; retain exact candidate and meaningful test counts. - [ ] 8.7 Validate the OpenSpec change strictly and sync implemented deltas through the appropriate skill; verify the PRD and active plan trace to the final contract. -- [ ] 8.8 Prepare rollback and evidence handoff for the combined PR; verify no merge, rollout, publication, or database replacement is implied by local passes. +- [x] 8.8 Prepare rollback and evidence handoff for the combined PR; verify no merge, rollout, publication, or database replacement is implied by local passes. + +Rollback instructions and evidence handoff at `a3488e95`: + +- The [runbook](../../../docs/runbooks/subagents.md#background-child-upgrade-and-rollback) requires stopped ingress, coordinated drain, and a recorded pre-upgrade backup. +- The [canonical plan](../../../.systematize/plans/background-subagents/plan.html#q-rollout) retains separate activation and rollback authority. +- The retained prior reader accepts its `sid-v1` control and rejects `cra-v1` and `cre-v1`. Direct binary downgrade has no compatibility proof. +- Rollback therefore requires the recorded pre-upgrade backup. Reader and restoration receipts prove only their stated storage scope. +- The evidence notes retain exact source revisions, local test counts, hosted checks, original failed observations, and open model gates. +- This closes task 8.8 for instructions and handoff preparation only. No live rollback, merge, rollout, publication, or database replacement receives authority. Documentation scope review at `2357991d`: diff --git a/openspec/specs/netclaw-subagents/spec.md b/openspec/specs/netclaw-subagents/spec.md index c5d7f0342..a74f44e96 100644 --- a/openspec/specs/netclaw-subagents/spec.md +++ b/openspec/specs/netclaw-subagents/spec.md @@ -113,9 +113,11 @@ Subagents SHALL NOT persist durable memory, stream direct durable-memory writes, For subagent execution launched from skill metadata routing, the subagent SHALL remain an isolated worker by default: -- It SHALL NOT inherit the main session identity prompt stack unless a future explicit setting enables it. -- It SHALL NOT auto-load repo-local `AGENTS.md` unless a future explicit setting enables it. -- It SHALL inherit audience/boundary context from the launch invocation. +- It SHALL receive the embedded operating core for its audience, followed by the operator's deployment `AGENTS.md`. +- It SHALL NOT inherit `SOUL.md` or `TOOLING.md`. +- A child with a non-Public audience SHALL receive available project instructions from the inherited project directory. +- A Public child SHALL NOT receive project-local instructions. +- It SHALL inherit audience/boundary context from the launch invocation. Prompt content SHALL NOT widen that authority. The child SHALL use the shared exact recurrence, repeat exception, and feedback-round contracts in `turn-loop-governance`. It SHALL retain its detector evidence for its actor-local task lifetime. @@ -167,17 +169,26 @@ All explicit and routed starts SHALL use `background-subagent-runs` acceptance, - **THEN** that path remains unavailable or denied - **AND** the child returns findings for parent policy review instead -#### Scenario: Routed subagent does not inherit main identity prompt stack +#### Scenario: Routed subagent receives scoped operating guidance -- **GIVEN** a slash skill routes through `metadata.subagent` +- **GIVEN** a slash skill routes through `metadata.subagent` and a deployment playbook exists - **WHEN** its isolated child prompt is assembled -- **THEN** the main session identity stack remains absent by default +- **THEN** the embedded operating core for its audience appears before the deployment `AGENTS.md` +- **AND** neither `SOUL.md` nor `TOOLING.md` is included + +#### Scenario: Routed subagent receives inherited project instructions + +- **GIVEN** a Personal or Team slash activation inherits a project directory with an `AGENTS.md` file +- **WHEN** its isolated child prompt is assembled +- **THEN** available project instructions appear after the operating guidance and before the child role prompt +- **AND** the project instructions come from the inherited project directory -#### Scenario: Routed subagent does not auto-load repo AGENTS +#### Scenario: Public routed subagent excludes project instructions -- **GIVEN** a slash skill routes through `metadata.subagent` +- **GIVEN** a Public slash activation inherits a project directory with an `AGENTS.md` file - **WHEN** its isolated child prompt is assembled -- **THEN** repo-local `AGENTS.md` remains absent by default +- **THEN** the stripped Public operating core and available deployment playbook remain present +- **AND** project-local instructions remain absent #### Scenario: Routed subagent inherits launch audience From 00064d4ecf879027066cd1161e0bd197b9bd59b1 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 10 Oct 2026 15:38:33 +0000 Subject: [PATCH 20/22] Preserve canonical child prompt scenarios across the stack --- .../changes/background-child-runs/tasks.md | 1 + .../specs/netclaw-subagents/spec.md | 35 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/openspec/changes/background-child-runs/tasks.md b/openspec/changes/background-child-runs/tasks.md index 1b6e1cd1e..9adc2b991 100644 --- a/openspec/changes/background-child-runs/tasks.md +++ b/openspec/changes/background-child-runs/tasks.md @@ -123,6 +123,7 @@ Independent contract review at `a3488e95` and combined `f48e9fbe`: - The delta now distinguishes the audience operating core and deployment playbook from separately scoped project instructions. It excludes `SOUL.md` and `TOOLING.md`. - Public children retain the existing exclusion of project-local instructions. Prompt content grants no additional authority. - This specification repair changes no runtime, prefill behavior, prompt bytes, or test assertion. It adds no fresh model evidence for guide version `1.0.3`. +- The inherited `per-call-tool-recurrence` delta now matches the canonical prompt clauses and three routed prompt scenarios. Its recurrence and authority clauses remain unchanged. - Task 7.9 remains open for independent review of the integrated result. Tasks 7.2, 8.6, and 8.7 also remain open. ## 8. Docs, operational guidance, and handoff diff --git a/openspec/changes/per-call-tool-recurrence/specs/netclaw-subagents/spec.md b/openspec/changes/per-call-tool-recurrence/specs/netclaw-subagents/spec.md index 60a36851e..94539bfcb 100644 --- a/openspec/changes/per-call-tool-recurrence/specs/netclaw-subagents/spec.md +++ b/openspec/changes/per-call-tool-recurrence/specs/netclaw-subagents/spec.md @@ -11,11 +11,11 @@ durable-memory writes, or participate in session pub/sub by default. For subagent execution launched from skill metadata routing, the subagent SHALL remain an isolated worker by default: -- It SHALL NOT inherit the main session identity prompt stack unless explicitly - enabled by a future opt-in setting. -- It SHALL NOT auto-load repo-local `AGENTS.md` unless explicitly enabled by a - future opt-in setting. -- It SHALL inherit audience/boundary context from the launching invocation. +- It SHALL receive the embedded operating core for its audience, followed by the operator's deployment `AGENTS.md`. +- It SHALL NOT inherit `SOUL.md` or `TOOLING.md`. +- A child with a non-Public audience SHALL receive available project instructions from the inherited project directory. +- A Public child SHALL NOT receive project-local instructions. +- It SHALL inherit audience/boundary context from the launch invocation. Prompt content SHALL NOT widen that authority. The child SHALL use the shared exact recurrence, repeat exception, and feedback-round contracts in `turn-loop-governance`. @@ -68,17 +68,26 @@ It SHALL NOT substitute success for an absent receipt or create a durable child- - **THEN** the durable write path is unavailable or denied to that subagent - **AND** the subagent must return findings to the parent session instead -#### Scenario: Routed subagent does not inherit main identity prompt stack +#### Scenario: Routed subagent receives scoped operating guidance -- **GIVEN** a slash-invoked skill routes execution via `metadata.subagent` -- **WHEN** the routed subagent prompt is assembled -- **THEN** the main session identity prompt stack is not included by default +- **GIVEN** a slash skill routes through `metadata.subagent` and a deployment playbook exists +- **WHEN** its isolated child prompt is assembled +- **THEN** the embedded operating core for its audience appears before the deployment `AGENTS.md` +- **AND** neither `SOUL.md` nor `TOOLING.md` is included -#### Scenario: Routed subagent does not auto-load repo AGENTS +#### Scenario: Routed subagent receives inherited project instructions -- **GIVEN** a slash-invoked skill routes execution via `metadata.subagent` -- **WHEN** the routed subagent prompt is assembled -- **THEN** repo-local `AGENTS.md` is not auto-loaded by default +- **GIVEN** a Personal or Team slash activation inherits a project directory with an `AGENTS.md` file +- **WHEN** its isolated child prompt is assembled +- **THEN** available project instructions appear after the operating guidance and before the child role prompt +- **AND** the project instructions come from the inherited project directory + +#### Scenario: Public routed subagent excludes project instructions + +- **GIVEN** a Public slash activation inherits a project directory with an `AGENTS.md` file +- **WHEN** its isolated child prompt is assembled +- **THEN** the stripped Public operating core and available deployment playbook remain present +- **AND** project-local instructions remain absent #### Scenario: Routed subagent inherits launch audience From 4ff4a44527369fb1eaa7127ff875a2cf4073f8b9 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sun, 11 Oct 2026 01:25:14 +0000 Subject: [PATCH 21/22] fix(sessions): adapt child report to the renamed session-workspace spill API dev renamed ToolOutputSpillLocation.TryEnsureSessionDirectory to EnsureSessionWorkspaceDirectory, which now returns an error string instead of a bool. Update the cancelled-child report writer to match. --- src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs index 27f4cbc6d..f7ff6149c 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.ChildRuns.cs @@ -532,7 +532,7 @@ await CompleteChildReportWithinGraceAsync(async finalizationToken => try { finalizationToken.ThrowIfCancellationRequested(); - if (!ToolOutputSpillLocation.TryEnsureSessionDirectory(anchor) + if (ToolOutputSpillLocation.EnsureSessionWorkspaceDirectory(anchor) is not null || !ToolOutputSpillLocation.IsSafeForIo(anchor, artifact)) throw new IOException("The child artifact directory crosses a filesystem link."); Directory.CreateDirectory(artifact); From 2494a01eac5eb9310dee0c53577b695ca386de93 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sun, 11 Oct 2026 17:54:20 +0000 Subject: [PATCH 22/22] fix(subagents): deliver background child death as a BackgroundChildTerminal The child's PostStop death path emitted SubAgentResult (the inline-path message), but a background child's owner only handles BackgroundChildTerminal. Route the ActorStopped result through BackgroundChildTerminal with the run and scope identities when the child runs in the background, so the owner records the terminal instead of dead-lettering it. --- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index 6f05605bd..822bba9c6 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -337,14 +337,27 @@ protected override void PostStop() { _completed = true; _pendingCheckpoint = null; - _replyTo.Tell(new SubAgentResult + var result = new SubAgentResult { Completion = new ChildRunCompletion.Failed(SubAgentOutcomeReason.ActorStopped), Output = "Subagent stopped before completion.", AgentName = _definition.Name, Findings = [], FindingsCount = 0 - }); + }; + if (_backgroundRunId is { } runId) + { + result = result with + { + RunId = runId, + ScopeId = new SubAgentScopeId(_subSessionId ?? throw new InvalidOperationException("A background child has no assigned scope.")) + }; + _replyTo.Tell(new BackgroundChildTerminal(runId, result), Self); + } + else + { + _replyTo.Tell(result); + } } _externalCancellationRegistration.Dispose(); _toolExposure.EvictAll();