From 4d9db7af1578ff791708c4a48b5580e967eb12c2 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:00:42 +1100 Subject: [PATCH 01/54] Add durable agent mailbox library and focused hosted test target --- .github/validation-named-tests.json | 8 + codex-rs/Cargo.lock | 18 + codex-rs/Cargo.toml | 2 + codex-rs/agent-mailbox/Cargo.toml | 29 + codex-rs/agent-mailbox/src/activation.rs | 122 ++ codex-rs/agent-mailbox/src/error.rs | 33 + codex-rs/agent-mailbox/src/lib.rs | 34 + codex-rs/agent-mailbox/src/mailbox_tests.rs | 1301 +++++++++++++++++ .../agent-mailbox/src/message_delivery.rs | 412 ++++++ codex-rs/agent-mailbox/src/message_rules.rs | 86 ++ codex-rs/agent-mailbox/src/messages.rs | 409 ++++++ codex-rs/agent-mailbox/src/outcome.rs | 747 ++++++++++ codex-rs/agent-mailbox/src/schema.rs | 380 +++++ codex-rs/agent-mailbox/src/snapshot.rs | 361 +++++ codex-rs/agent-mailbox/src/store.rs | 37 + codex-rs/agent-mailbox/src/types.rs | 283 ++++ codex-rs/agent-mailbox/src/work.rs | 483 ++++++ codex-rs/core/Cargo.toml | 1 + 18 files changed, 4746 insertions(+) create mode 100644 codex-rs/agent-mailbox/Cargo.toml create mode 100644 codex-rs/agent-mailbox/src/activation.rs create mode 100644 codex-rs/agent-mailbox/src/error.rs create mode 100644 codex-rs/agent-mailbox/src/lib.rs create mode 100644 codex-rs/agent-mailbox/src/mailbox_tests.rs create mode 100644 codex-rs/agent-mailbox/src/message_delivery.rs create mode 100644 codex-rs/agent-mailbox/src/message_rules.rs create mode 100644 codex-rs/agent-mailbox/src/messages.rs create mode 100644 codex-rs/agent-mailbox/src/outcome.rs create mode 100644 codex-rs/agent-mailbox/src/schema.rs create mode 100644 codex-rs/agent-mailbox/src/snapshot.rs create mode 100644 codex-rs/agent-mailbox/src/store.rs create mode 100644 codex-rs/agent-mailbox/src/types.rs create mode 100644 codex-rs/agent-mailbox/src/work.rs diff --git a/.github/validation-named-tests.json b/.github/validation-named-tests.json index c7c3c6994511..5910cdc11d71 100644 --- a/.github/validation-named-tests.json +++ b/.github/validation-named-tests.json @@ -1,6 +1,14 @@ { "schema_version": "rust-tests-command-manifest-v1", "targets": [ + { + "package": "codex-agent-mailbox", + "target_kind": "lib", + "target": "", + "profiles": ["rust_minimal", "rust_integration"], + "inventory_argv": ["cargo", "test", "--locked", "-p", "codex-agent-mailbox", "--lib", "--", "--list"], + "execution_argv": ["cargo", "test", "--locked", "-p", "codex-agent-mailbox", "--lib", "--", "--test-threads=1"] + }, { "package": "codex-app-server", "target_kind": "lib", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 24e45a7c4262..13f5b0684b34 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1996,6 +1996,23 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "codex-agent-mailbox" +version = "0.0.0" +dependencies = [ + "codex-protocol", + "codex-state", + "codex-utils-absolute-path", + "pretty_assertions", + "serde", + "serde_json", + "sqlx", + "tempfile", + "thiserror 2.0.19", + "tokio", + "uuid", +] + [[package]] name = "codex-analytics" version = "0.0.0" @@ -2800,6 +2817,7 @@ dependencies = [ "chrono", "clap", "codex-agent-graph-store", + "codex-agent-mailbox", "codex-analytics", "codex-api", "codex-app-server-protocol", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 1780b9d60317..f9d5dc234301 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -4,6 +4,7 @@ members = [ "analytics", "agent-graph-store", "agent-identity", + "agent-mailbox", "android-computer-use", "backend-client", "browser-computer-use", @@ -147,6 +148,7 @@ app_test_support = { path = "app-server/tests/common" } codex-analytics = { path = "analytics" } codex-agent-graph-store = { path = "agent-graph-store" } codex-agent-identity = { path = "agent-identity" } +codex-agent-mailbox = { path = "agent-mailbox" } codex-ansi-escape = { path = "ansi-escape" } codex-android-computer-use = { path = "android-computer-use" } codex-api = { path = "codex-api" } diff --git a/codex-rs/agent-mailbox/Cargo.toml b/codex-rs/agent-mailbox/Cargo.toml new file mode 100644 index 000000000000..86ef1684025c --- /dev/null +++ b/codex-rs/agent-mailbox/Cargo.toml @@ -0,0 +1,29 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-agent-mailbox" +version.workspace = true + +[lib] +name = "codex_agent_mailbox" +path = "src/lib.rs" +doctest = false + +[lints] +workspace = true + +[dependencies] +codex-protocol = { workspace = true } +codex-state = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sqlx = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +uuid = { workspace = true } + +[dev-dependencies] +codex-utils-absolute-path = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/agent-mailbox/src/activation.rs b/codex-rs/agent-mailbox/src/activation.rs new file mode 100644 index 000000000000..fc37d3aba8fa --- /dev/null +++ b/codex-rs/agent-mailbox/src/activation.rs @@ -0,0 +1,122 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::store::Mailbox; +use crate::types::{ActivationDecision, ActivationDisposition}; +use codex_protocol::ThreadId; +use sqlx::Row; + +impl Mailbox { + /// Claim an optional native activation exactly once for an accepted message. + /// An unresolved claim is recovery-needed, never a retry authorization. + pub async fn begin_activation( + &self, + sender: ThreadId, + message_id: &str, + ) -> MailboxResult { + let mut transaction = self.pool.begin().await?; + let message = sqlx::query( + "SELECT id FROM mailbox_messages WHERE id = ? AND sender = ? LIMIT 1", + ) + .bind(message_id) + .bind(sender.to_string()) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + let exact_id: String = message.try_get("id")?; + let claimed = sqlx::query( + "UPDATE mailbox_activations SET state = 'activating' + WHERE message_id = ? AND sender = ? AND state = 'pending'", + ) + .bind(&exact_id) + .bind(sender.to_string()) + .execute(&mut *transaction) + .await?; + if claimed.rows_affected() == 1 { + transaction.commit().await?; + self.notify_changed(); + return Ok(ActivationDecision::Activate); + } + let existing = sqlx::query( + "SELECT sender, state, disposition FROM mailbox_activations WHERE message_id = ?", + ) + .bind(message_id) + .fetch_one(&mut *transaction) + .await?; + if existing.try_get::("sender")? != sender.to_string() { + return Err(MailboxError::WrongRecipient); + } + let decision = match existing.try_get::("state")?.as_str() { + "activating" => ActivationDecision::RecoveryNeeded, + "decided" => ActivationDecision::AlreadyDecided(parse_activation_disposition( + existing.try_get::("disposition")?.as_str(), + )?), + _ => return Err(MailboxError::InvalidInput("stored activation state".into())), + }; + transaction.commit().await?; + Ok(decision) + } + + /// Persist the native adapter's final activation disposition. A lost result + /// can be retried exactly, but a conflicting disposition is rejected. + pub async fn record_activation( + &self, + sender: ThreadId, + message_id: &str, + disposition: ActivationDisposition, + ) -> MailboxResult<()> { + let mut transaction = self.pool.begin().await?; + let current = sqlx::query( + "SELECT sender, state, disposition FROM mailbox_activations WHERE message_id = ?", + ) + .bind(message_id) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if current.try_get::("sender")? != sender.to_string() { + return Err(MailboxError::WrongRecipient); + } + let disposition_string = activation_disposition_string(disposition); + let state: String = current.try_get("state")?; + if state == "decided" { + let old: String = current.try_get("disposition")?; + if old != disposition_string { + return Err(MailboxError::IdempotencyConflict); + } + transaction.commit().await?; + return Ok(()); + } + if state != "activating" { + return Err(MailboxError::InvalidInput("stored activation state".into())); + } + sqlx::query( + "UPDATE mailbox_activations SET state = 'decided', disposition = ? + WHERE message_id = ? AND sender = ? AND state = 'activating'", + ) + .bind(disposition_string) + .bind(message_id) + .bind(sender.to_string()) + .execute(&mut *transaction) + .await?; + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } +} + +fn activation_disposition_string(disposition: ActivationDisposition) -> &'static str { + match disposition { + ActivationDisposition::NotRequested => "not_requested", + ActivationDisposition::AlreadyActive => "already_active", + ActivationDisposition::Activated => "activated", + ActivationDisposition::Denied => "denied", + } +} + +fn parse_activation_disposition(value: &str) -> MailboxResult { + match value { + "not_requested" => Ok(ActivationDisposition::NotRequested), + "already_active" => Ok(ActivationDisposition::AlreadyActive), + "activated" => Ok(ActivationDisposition::Activated), + "denied" => Ok(ActivationDisposition::Denied), + _ => Err(MailboxError::InvalidInput("stored activation disposition".into())), + } +} diff --git a/codex-rs/agent-mailbox/src/error.rs b/codex-rs/agent-mailbox/src/error.rs new file mode 100644 index 000000000000..d2003f83ff3d --- /dev/null +++ b/codex-rs/agent-mailbox/src/error.rs @@ -0,0 +1,33 @@ +use thiserror::Error; + +pub type MailboxResult = Result; + +#[derive(Debug, Error)] +pub enum MailboxError { + #[error("mailbox SQLite operation failed: {0}")] + Sqlite(#[from] sqlx::Error), + #[error("mailbox JSON encoding failed: {0}")] + Json(#[from] serde_json::Error), + #[error("invalid mailbox input: {0}")] + InvalidInput(String), + #[error("mailbox idempotency key was reused with different input")] + IdempotencyConflict, + #[error("mailbox reference does not identify the exact stored work or message")] + ReferenceMismatch, + #[error("the addressed request already has a durable reply")] + RequestAlreadyResolved, + #[error("mailbox item is not canonically admitted yet")] + NotCanonical, + #[error("mailbox item is not owned by the requested recipient")] + WrongRecipient, + #[error("mailbox item cannot be replaced after admission or canonicalization")] + ReplacementNotAllowed, + #[error("a terminal work outcome has already been recorded")] + OutcomeAlreadyRecorded, + #[error("mailbox backpressure limit reached: {0}")] + Backpressure(&'static str), + #[error("mailbox schema version {found} is newer than supported version {supported}")] + NewerSchema { found: i64, supported: i64 }, + #[error("mailbox cursor is malformed")] + InvalidCursor, +} diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs new file mode 100644 index 000000000000..d6c2cff68750 --- /dev/null +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -0,0 +1,34 @@ +//! Durable, mailbox-owned capture and delivery state for native agent work. +//! +//! This crate deliberately has no dependency on `codex-core`. Adapters validate +//! trusted native-tree identity and perform canonical history writes; this crate +//! persists the accepted work, delivery intent, and exact observation state. + +mod error; +mod activation; +mod messages; +mod message_delivery; +mod message_rules; +mod outcome; +mod schema; +mod snapshot; +mod store; +mod work; +mod types; + +pub use error::{MailboxError, MailboxResult}; +pub use store::Mailbox; +pub use snapshot::is_provably_closed_peer_cycle; +pub use types::{ + ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, + CanonicalConfirmation, Continuation, ContinuationReceipt, ContinuationState, Dependency, + DependencyMode, DispatchDecision, + DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, + NativeWork, ObservationTarget, Outcome, OutcomeReceipt, + PendingPage, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, + SnapshotRequest, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, +}; + +#[cfg(test)] +#[path = "mailbox_tests.rs"] +mod tests; diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs new file mode 100644 index 000000000000..95cc7a71ad7f --- /dev/null +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -0,0 +1,1301 @@ +use crate::{ + is_provably_closed_peer_cycle, ActivationDecision, ActivationDisposition, Dependency, + DependencyMode, DispatchDecision, DispatchState, Mailbox, MailboxError, MessageDraft, + MessageKind, NativeWork, ObservationTarget, RequestDisposition, RequestSnapshot, ReturnWhen, + Selection, Selector, Snapshot, SnapshotItem, SnapshotState, WorkSnapshot, Continuation, + ContinuationState, GoalBinding, Outcome, +}; +use codex_protocol::{ + protocol::{AgentPath, InterAgentCommunication}, + ThreadId, +}; +use codex_state::SqliteConfig; +use codex_utils_absolute_path::test_support::PathExt; +use pretty_assertions::assert_eq; +use tempfile::TempDir; + +struct TestMailbox { + _home: TempDir, + config: SqliteConfig, + mailbox: Mailbox, +} + +async fn test_mailbox() -> TestMailbox { + let home = tempfile::tempdir().expect("temporary mailbox home"); + let config = SqliteConfig::new_for_testing(home.path().abs()); + let mailbox = Mailbox::open(&config).await.expect("open test mailbox"); + TestMailbox { + _home: home, + config, + mailbox, + } +} + +fn thread_id(index: u64) -> ThreadId { + ThreadId::from_string(&format!("00000000-0000-0000-0000-{index:012x}")) + .expect("fixture thread ID is a UUID") +} + +fn communication(content: &str) -> InterAgentCommunication { + InterAgentCommunication::new( + AgentPath::root(), + AgentPath::root(), + Vec::new(), + content.to_string(), + false, + ) +} + +fn native_work( + issuer: ThreadId, + agent: ThreadId, + native_key: &str, + submission_id: &str, + content: &str, +) -> NativeWork { + NativeWork { + native_key: native_key.to_string(), + issuer, + agent, + submission_id: submission_id.to_string(), + input: communication(content), + } +} + +fn message_draft( + invocation_key: &str, + sender: ThreadId, + recipient: ThreadId, + kind: MessageKind, + content: &str, + legacy_attention: bool, +) -> MessageDraft { + MessageDraft { + invocation_key: invocation_key.to_string(), + sender, + recipient, + communication: communication(content), + kind, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention, + } +} + +fn selection( + caller: ThreadId, + agent: Option, + selectors: Vec, + return_when: ReturnWhen, +) -> Selection { + Selection { + caller, + agent, + selectors, + return_when, + limit: 100, + } +} + +fn work_item<'a>(snapshot: &'a Snapshot, work_id: &str) -> &'a WorkSnapshot { + snapshot + .items + .iter() + .find_map(|item| match item { + SnapshotItem::Work(work) if work.work_ref.id == work_id => Some(work), + _ => None, + }) + .expect("selected work is present in the snapshot") +} + +fn request_item<'a>(snapshot: &'a Snapshot, request_id: &str) -> &'a RequestSnapshot { + snapshot + .items + .iter() + .find_map(|item| match item { + SnapshotItem::Request(request) if request.request_id == request_id => Some(request), + _ => None, + }) + .expect("selected request is present in the snapshot") +} + +fn goal_binding(parent: ThreadId, child: ThreadId, generation: u64, turn: &str) -> GoalBinding { + GoalBinding { + goal_id: "goal-1".to_string(), + parent_thread_id: parent, + child_thread_id: child, + control_generation: generation, + revision: 1, + source_turn_id: turn.to_string(), + } +} + +#[tokio::test] +async fn accepted_native_key_retry_preserves_work_and_provider_message_identity() { + let fixture = test_mailbox().await; + let issuer = thread_id(1); + let agent = thread_id(2); + let original = native_work(issuer, agent, "native-1", "submission-1", "do the work"); + + let accepted = fixture + .mailbox + .accept_work(original.clone()) + .await + .expect("accept native work"); + let captured_input = fixture + .mailbox + .accepted_input(&accepted) + .await + .expect("read captured native input"); + let captured_id = captured_input + .id + .as_ref() + .expect("accepted input receives a stable ID"); + assert!(captured_id.as_str().starts_with("amsg_")); + + // A retry after a lost publication reuses both the immutable work reference + // and the already-assigned provider item ID. + assert_eq!( + fixture.mailbox.accept_work(original).await.unwrap(), + accepted + ); + assert_eq!( + fixture.mailbox.accepted_input(&accepted).await.unwrap(), + captured_input + ); + + let mut conflicting_retry = native_work(issuer, agent, "native-1", "submission-1", "changed"); + conflicting_retry.input.trigger_turn = true; + assert!(matches!( + fixture.mailbox.accept_work(conflicting_retry).await, + Err(MailboxError::IdempotencyConflict) + )); + + assert_eq!( + fixture.mailbox.begin_dispatch(&accepted).await.unwrap(), + DispatchDecision::Dispatch + ); + assert_eq!( + fixture.mailbox.begin_dispatch(&accepted).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + + // Reopening the durable store does not turn an unresolved claim into + // permission to send again. + let reopened = Mailbox::open(&fixture.config).await.unwrap(); + assert_eq!( + reopened.dispatch_state(&accepted).await.unwrap(), + DispatchState::Dispatching + ); + assert_eq!( + reopened.begin_dispatch(&accepted).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + reopened.mark_dispatched(&accepted).await.unwrap(); + assert_eq!( + reopened.begin_dispatch(&accepted).await.unwrap(), + DispatchDecision::AlreadyDispatched + ); +} + +#[tokio::test] +async fn admission_is_bound_to_the_exact_input_ids_and_turn() { + let fixture = test_mailbox().await; + let issuer = thread_id(3); + let agent = thread_id(4); + let work_a = fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-a", "submission-a", "a")) + .await + .unwrap(); + let work_b = fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-b", "submission-b", "b")) + .await + .unwrap(); + fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-c", "submission-c", "c")) + .await + .unwrap(); + let id_a = fixture + .mailbox + .accepted_input(&work_a) + .await + .unwrap() + .id + .expect("accepted input has an ID") + .to_string(); + let id_b = fixture + .mailbox + .accepted_input(&work_b) + .await + .unwrap() + .id + .expect("accepted input has an ID") + .to_string(); + let work_c = fixture + .mailbox + .work_for_native_key(issuer, "native-c") + .await + .unwrap() + .expect("third native work was accepted"); + let id_c = fixture + .mailbox + .accepted_input(&work_c) + .await + .unwrap() + .id + .unwrap() + .to_string(); + + let receipt = fixture + .mailbox + .record_admission(agent, "turn-one", &[id_a.clone()]) + .await + .unwrap(); + assert_eq!(receipt.work_refs, vec![work_a.clone()]); + assert!(!receipt.already_recorded); + + let retry = fixture + .mailbox + .record_admission(agent, "turn-one", &[id_a.clone()]) + .await + .unwrap(); + assert_eq!(retry.work_refs, receipt.work_refs); + assert!(retry.already_recorded); + + let appended = fixture + .mailbox + .record_admission(agent, "turn-one", &[id_b.clone()]) + .await + .unwrap(); + assert_eq!(appended.work_refs, vec![work_b.clone()]); + assert!(!appended.already_recorded); + let covered = fixture + .mailbox + .record_turn_outcome( + agent, + "turn-one", + Outcome::Completed, + None, + Continuation::None, + ) + .await + .unwrap(); + assert_eq!(covered.outcomes.len(), 2); + assert!(covered.outcomes.iter().any(|item| item.work_ref == work_a)); + assert!(covered.outcomes.iter().any(|item| item.work_ref == work_b)); + assert!(covered.outcomes.iter().all(|item| item.work_ref != work_c)); + + assert!(matches!( + fixture + .mailbox + .record_admission(agent, "turn-two", &[id_a]) + .await, + Err(MailboxError::IdempotencyConflict) + )); + assert!(matches!( + fixture + .mailbox + .record_admission(agent, "turn-one", &[id_c, "amsg_not-admitted".to_string()]) + .await, + Err(MailboxError::ReferenceMismatch) + )); + + let too_many = (0..101) + .map(|index| format!("amsg-over-limit-{index}")) + .collect::>(); + assert!(matches!( + fixture + .mailbox + .record_admission(agent, "oversized-turn", &too_many) + .await, + Err(MailboxError::Backpressure(_)) + )); +} + +#[tokio::test] +async fn turn_outcome_covers_only_admitted_refs_and_snapshot_is_caller_scoped() { + let fixture = test_mailbox().await; + let issuer = thread_id(5); + let other_issuer = thread_id(6); + let agent = thread_id(7); + let work_a = fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-a", "submission-a", "a")) + .await + .unwrap(); + let work_b = fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-b", "submission-b", "b")) + .await + .unwrap(); + let id_a = fixture + .mailbox + .accepted_input(&work_a) + .await + .unwrap() + .id + .unwrap() + .to_string(); + let id_b = fixture + .mailbox + .accepted_input(&work_b) + .await + .unwrap() + .id + .unwrap() + .to_string(); + fixture + .mailbox + .record_admission(agent, "turn-covered", &[id_a, id_b]) + .await + .unwrap(); + + // This accepted input arrives after the native execution admitted its + // fixed set and must remain pending for a later turn. + let unadmitted = fixture + .mailbox + .accept_work(native_work(issuer, agent, "native-late", "submission-late", "late")) + .await + .unwrap(); + let unrelated = fixture + .mailbox + .accept_work(native_work( + other_issuer, + agent, + "native-other-issuer", + "submission-other-issuer", + "other issuer history", + )) + .await + .unwrap(); + fixture + .mailbox + .record_outcome( + &unrelated, + Outcome::Completed, + None, + Continuation::None, + ) + .await + .unwrap(); + + let outcome = fixture + .mailbox + .record_turn_outcome( + agent, + "turn-covered", + Outcome::Completed, + None, + Continuation::None, + ) + .await + .unwrap(); + assert_eq!(outcome.outcomes.len(), 2); + assert!(outcome.outcomes.iter().any(|item| item.work_ref == work_a)); + assert!(outcome.outcomes.iter().any(|item| item.work_ref == work_b)); + assert!(outcome.outcomes.iter().all(|item| item.work_ref != unadmitted)); + + let snapshot = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![ + Selector::Work(work_a.id.clone()), + Selector::Work(work_b.id.clone()), + Selector::Work(unadmitted.id.clone()), + Selector::Work(unrelated.id.clone()), + ], + ReturnWhen::All, + )) + .await + .unwrap(); + assert_eq!(snapshot.state, SnapshotState::Pending); + assert_eq!(snapshot.items.len(), 3); + assert!(work_item(&snapshot, &work_a.id).outcome.is_some()); + assert!(work_item(&snapshot, &work_b.id).outcome.is_some()); + assert!(work_item(&snapshot, &unadmitted.id).outcome.is_none()); + assert!(!snapshot + .items + .iter() + .any(|item| matches!(item, SnapshotItem::Work(work) if work.work_ref.id == unrelated.id))); +} + +#[tokio::test] +async fn one_turn_returns_separate_handbacks_to_each_issuer() { + let fixture = test_mailbox().await; + let issuer_a = thread_id(8); + let issuer_b = thread_id(9); + let agent = thread_id(12); + let work_a = fixture + .mailbox + .accept_work(native_work(issuer_a, agent, "native-a", "submission-a", "a")) + .await + .unwrap(); + let work_b = fixture + .mailbox + .accept_work(native_work(issuer_b, agent, "native-b", "submission-b", "b")) + .await + .unwrap(); + let id_a = fixture + .mailbox + .accepted_input(&work_a) + .await + .unwrap() + .id + .unwrap() + .to_string(); + let id_b = fixture + .mailbox + .accepted_input(&work_b) + .await + .unwrap() + .id + .unwrap() + .to_string(); + fixture + .mailbox + .record_admission(agent, "shared-turn", &[id_a, id_b]) + .await + .unwrap(); + + let result = fixture + .mailbox + .record_turn_outcome( + agent, + "shared-turn", + Outcome::Completed, + Some(communication("result for both issuers")), + Continuation::None, + ) + .await + .unwrap(); + assert_eq!(result.outcomes.len(), 2); + let response_a = result + .outcomes + .iter() + .find(|item| item.work_ref == work_a) + .and_then(|item| item.response_message_id.as_deref()) + .expect("issuer A receives its response"); + let response_b = result + .outcomes + .iter() + .find(|item| item.work_ref == work_b) + .and_then(|item| item.response_message_id.as_deref()) + .expect("issuer B receives its response"); + assert_ne!(response_a, response_b); + + let handback_a = fixture + .mailbox + .get_message(issuer_a, response_a) + .await + .unwrap() + .expect("issuer A handback is addressable"); + let handback_b = fixture + .mailbox + .get_message(issuer_b, response_b) + .await + .unwrap() + .expect("issuer B handback is addressable"); + assert_eq!(handback_a.sender, agent); + assert_eq!(handback_a.recipient, issuer_a); + assert_eq!(handback_b.sender, agent); + assert_eq!(handback_b.recipient, issuer_b); +} + +#[tokio::test] +async fn pending_and_deferred_goal_bindings_settle_only_for_the_exact_generation() { + let fixture = test_mailbox().await; + let issuer = thread_id(13); + let agent = thread_id(14); + let work_pending = fixture + .mailbox + .accept_work(native_work( + issuer, + agent, + "native-pending", + "submission-pending", + "pending goal work", + )) + .await + .unwrap(); + let work_deferred = fixture + .mailbox + .accept_work(native_work( + issuer, + agent, + "native-deferred", + "submission-deferred", + "deferred goal work", + )) + .await + .unwrap(); + let binding_pending = goal_binding(issuer, agent, 4, "pending-turn"); + let binding_deferred = goal_binding(issuer, agent, 5, "deferred-turn"); + for (work_ref, turn_id, continuation) in [ + ( + &work_pending, + "pending-turn", + Continuation::PendingGoalBinding(binding_pending.clone()), + ), + ( + &work_deferred, + "deferred-turn", + Continuation::Deferred(binding_deferred.clone()), + ), + ] { + let input_id = fixture + .mailbox + .accepted_input(work_ref) + .await + .unwrap() + .id + .unwrap() + .to_string(); + fixture + .mailbox + .record_admission(agent, turn_id, &[input_id]) + .await + .unwrap(); + let result = fixture + .mailbox + .record_turn_outcome( + agent, + turn_id, + Outcome::Completed, + None, + continuation, + ) + .await + .unwrap(); + assert!(result.outcomes.is_empty()); + assert_eq!(result.continuation_work, vec![(*work_ref).clone()]); + } + + for (work_ref, expected_state) in [ + (&work_pending, ContinuationState::PendingGoalBinding), + (&work_deferred, ContinuationState::Deferred), + ] { + let snapshot = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![Selector::Work(work_ref.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + let item = work_item(&snapshot, &work_ref.id); + assert!(item.outcome.is_none()); + assert_eq!(item.continuations.len(), 1); + assert_eq!(item.continuations[0].state, expected_state); + } + + let mut stale_generation = binding_pending.clone(); + stale_generation.control_generation += 1; + let stale = fixture + .mailbox + .record_turn_outcome( + agent, + "later-turn-stale-generation", + Outcome::Completed, + None, + Continuation::Terminal(stale_generation), + ) + .await + .unwrap(); + assert!(stale.no_matching_work); + assert!(fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![Selector::Work(work_pending.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap() + .items + .iter() + .any(|item| matches!(item, SnapshotItem::Work(work) if work.outcome.is_none()))); + + for (work_ref, binding, turn_id) in [ + (&work_pending, binding_pending, "terminal-pending"), + (&work_deferred, binding_deferred, "terminal-deferred"), + ] { + let terminal = fixture + .mailbox + .record_turn_outcome( + agent, + turn_id, + Outcome::Completed, + None, + Continuation::Terminal(binding.clone()), + ) + .await + .unwrap(); + assert_eq!(terminal.outcomes.len(), 1); + let snapshot = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![Selector::Work(work_ref.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + let receipt = work_item(&snapshot, &work_ref.id) + .outcome + .as_ref() + .expect("exact terminal binding settles the old covered work"); + assert_eq!(receipt.continuation, Continuation::Terminal(binding)); + } +} + +#[tokio::test] +async fn quiet_reply_satisfies_its_exact_request_selector() { + let fixture = test_mailbox().await; + let issuer = thread_id(15); + let agent = thread_id(16); + let request = fixture + .mailbox + .accept_message(message_draft( + "decision-request", + issuer, + agent, + MessageKind::Request, + "please decide", + false, + )) + .await + .unwrap(); + fixture + .mailbox + .mark_canonical(agent, &request.id) + .await + .unwrap(); + + let mut reply_draft = message_draft( + "decision-reply", + agent, + issuer, + MessageKind::Note, + "decision made", + false, + ); + reply_draft.reply_to = Some(request.id.clone()); + let reply = fixture + .mailbox + .accept_message(reply_draft) + .await + .unwrap(); + + let snapshot = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + assert_eq!(snapshot.state, SnapshotState::AlreadySatisfied); + let selected_request = request_item(&snapshot, &request.id); + assert_eq!(selected_request.disposition, RequestDisposition::Replied); + assert_eq!( + selected_request.reply.as_ref().map(|message| message.id.as_str()), + Some(reply.id.as_str()) + ); + assert!(!reply.legacy_attention); +} + +#[tokio::test] +async fn unrelated_successful_work_does_not_resolve_an_open_request() { + let fixture = test_mailbox().await; + let issuer = thread_id(17); + let agent = thread_id(18); + let request = fixture + .mailbox + .accept_message(message_draft( + "open-request", + issuer, + agent, + MessageKind::Request, + "await a separate decision", + false, + )) + .await + .unwrap(); + let work_ref = fixture + .mailbox + .accept_work(native_work( + issuer, + agent, + "unrelated-success", + "unrelated-submission", + "complete different work", + )) + .await + .unwrap(); + let input_id = fixture + .mailbox + .accepted_input(&work_ref) + .await + .unwrap() + .id + .unwrap() + .to_string(); + fixture + .mailbox + .record_admission(agent, "unrelated-turn", &[input_id]) + .await + .unwrap(); + fixture + .mailbox + .record_turn_outcome( + agent, + "unrelated-turn", + Outcome::Completed, + None, + Continuation::None, + ) + .await + .unwrap(); + + let snapshot = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + assert_eq!(snapshot.state, SnapshotState::Pending); + assert_eq!(request_item(&snapshot, &request.id).disposition, RequestDisposition::Open); +} + +#[tokio::test] +async fn only_the_request_author_can_supersede_or_withdraw_and_old_selectors_settle() { + let fixture = test_mailbox().await; + let author = thread_id(19); + let recipient = thread_id(22); + let stranger = thread_id(23); + let request = fixture + .mailbox + .accept_message(message_draft( + "replace-me", + author, + recipient, + MessageKind::Request, + "first request", + false, + )) + .await + .unwrap(); + + let mut unauthorized_replacement = message_draft( + "stranger-replacement", + stranger, + recipient, + MessageKind::Request, + "not yours to replace", + false, + ); + unauthorized_replacement.supersedes = Some(request.id.clone()); + assert!(matches!( + fixture + .mailbox + .accept_message(unauthorized_replacement) + .await, + Err(MailboxError::WrongRecipient) + )); + + let mut replacement_draft = message_draft( + "author-replacement", + author, + recipient, + MessageKind::Request, + "updated request", + false, + ); + replacement_draft.supersedes = Some(request.id.clone()); + let replacement = fixture + .mailbox + .accept_message(replacement_draft) + .await + .unwrap(); + let replaced_snapshot = fixture + .mailbox + .snapshot(&selection( + author, + Some(recipient), + vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + assert_eq!(replaced_snapshot.state, SnapshotState::AlreadySatisfied); + let old_request = request_item(&replaced_snapshot, &request.id); + assert_eq!(old_request.disposition, RequestDisposition::Superseded); + assert_eq!(old_request.replacement.as_deref(), Some(replacement.id.as_str())); + + let withdrawal = fixture + .mailbox + .accept_message(message_draft( + "withdraw-me", + author, + recipient, + MessageKind::Request, + "cancel this request", + false, + )) + .await + .unwrap(); + assert!(matches!( + fixture + .mailbox + .withdraw_message(stranger, &withdrawal.id) + .await, + Err(MailboxError::WrongRecipient) + )); + fixture + .mailbox + .withdraw_message(author, &withdrawal.id) + .await + .unwrap(); + let withdrawn_snapshot = fixture + .mailbox + .snapshot(&selection( + author, + Some(recipient), + vec![Selector::Reply(withdrawal.id.clone())], + ReturnWhen::Any, + )) + .await + .unwrap(); + assert_eq!(withdrawn_snapshot.state, SnapshotState::AlreadySatisfied); + assert_eq!( + request_item(&withdrawn_snapshot, &withdrawal.id).disposition, + RequestDisposition::Withdrawn + ); +} + +#[tokio::test] +async fn progress_coalesces_only_for_the_exact_unadmitted_author_and_work() { + let fixture = test_mailbox().await; + let issuer = thread_id(24); + let agent = thread_id(25); + let work_a = fixture + .mailbox + .accept_work(native_work(issuer, agent, "progress-a", "submission-a", "a")) + .await + .unwrap(); + let work_b = fixture + .mailbox + .accept_work(native_work(issuer, agent, "progress-b", "submission-b", "b")) + .await + .unwrap(); + + let mut first_draft = message_draft( + "progress-first", + issuer, + agent, + MessageKind::Progress, + "initial status", + false, + ); + first_draft.work_ref = Some(work_a.clone()); + let first = fixture + .mailbox + .accept_message(first_draft) + .await + .unwrap(); + let mut update_draft = message_draft( + "progress-update", + issuer, + agent, + MessageKind::Progress, + "latest status", + false, + ); + update_draft.work_ref = Some(work_a.clone()); + let update = fixture + .mailbox + .accept_message(update_draft) + .await + .unwrap(); + assert_eq!(update.id, first.id); + assert_eq!( + fixture + .mailbox + .get_message(agent, &first.id) + .await + .unwrap() + .unwrap() + .communication + .content, + "latest status" + ); + + let mut different_author = message_draft( + "progress-other-author", + issuer, + agent, + MessageKind::Progress, + "different path author", + false, + ); + different_author.work_ref = Some(work_a.clone()); + different_author.communication.author = + AgentPath::from_string("/root/child".to_string()).expect("valid author path"); + let other_author = fixture + .mailbox + .accept_message(different_author) + .await + .unwrap(); + assert_ne!(other_author.id, first.id); + + let mut different_work = message_draft( + "progress-other-work", + issuer, + agent, + MessageKind::Progress, + "different work", + false, + ); + different_work.work_ref = Some(work_b); + let other_work = fixture + .mailbox + .accept_message(different_work) + .await + .unwrap(); + assert_ne!(other_work.id, first.id); + + // No admission is necessary to coalesce quiet progress, but once its + // exact work is terminal the same writer cannot append late progress. + fixture + .mailbox + .record_outcome(&work_a, Outcome::Completed, None, Continuation::None) + .await + .unwrap(); + let mut late_progress = message_draft( + "progress-after-terminal", + issuer, + agent, + MessageKind::Progress, + "too late", + false, + ); + late_progress.work_ref = Some(work_a); + assert!(matches!( + fixture.mailbox.accept_message(late_progress).await, + Err(MailboxError::OutcomeAlreadyRecorded) + )); +} + +#[tokio::test] +async fn out_of_order_observation_is_per_item_after_canonical_flush() { + let fixture = test_mailbox().await; + let issuer = thread_id(26); + let agent = thread_id(27); + let work_a = fixture + .mailbox + .accept_work(native_work(issuer, agent, "observed-a", "submission-a", "a")) + .await + .unwrap(); + let work_b = fixture + .mailbox + .accept_work(native_work(issuer, agent, "observed-b", "submission-b", "b")) + .await + .unwrap(); + + for (work_ref, turn_id) in [(&work_a, "turn-a"), (&work_b, "turn-b")] { + let input_id = fixture + .mailbox + .accepted_input(work_ref) + .await + .unwrap() + .id + .unwrap() + .to_string(); + fixture + .mailbox + .record_admission(agent, turn_id, &[input_id]) + .await + .unwrap(); + let result = fixture + .mailbox + .record_turn_outcome( + agent, + turn_id, + Outcome::Completed, + None, + Continuation::None, + ) + .await + .unwrap(); + let receipt = result.outcomes.first().expect("one exact outcome"); + fixture + .mailbox + .mark_outcome_canonical(agent, &receipt.id) + .await + .unwrap(); + } + + fixture + .mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(work_b.clone())]) + .await + .unwrap(); + let partially_observed = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![ + Selector::Work(work_a.id.clone()), + Selector::Work(work_b.id.clone()), + ], + ReturnWhen::All, + )) + .await + .unwrap(); + assert_eq!(partially_observed.state, SnapshotState::Pending); + assert!(!work_item(&partially_observed, &work_a.id).observed); + assert!(work_item(&partially_observed, &work_b.id).observed); + assert!(work_item(&partially_observed, &work_a.id).outcome.is_some()); + assert!(work_item(&partially_observed, &work_b.id).outcome.is_some()); + + fixture + .mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(work_a.clone())]) + .await + .unwrap(); + let fully_observed = fixture + .mailbox + .snapshot(&selection( + issuer, + Some(agent), + vec![ + Selector::Work(work_a.id.clone()), + Selector::Work(work_b.id.clone()), + ], + ReturnWhen::All, + )) + .await + .unwrap(); + assert_eq!(fully_observed.state, SnapshotState::AlreadySatisfied); +} + +#[tokio::test] +async fn default_and_legacy_attention_are_separate_from_quiet_message_delivery() { + let fixture = test_mailbox().await; + let sender = thread_id(10); + let recipient = thread_id(11); + let kinds = [ + MessageKind::Note, + MessageKind::Progress, + MessageKind::Request, + MessageKind::Alert, + ]; + assert_eq!(MessageKind::default(), MessageKind::Note); + + for kind in kinds { + fixture + .mailbox + .accept_message(message_draft( + &format!("quiet-{kind:?}"), + sender, + recipient, + kind, + "quiet message", + false, + )) + .await + .unwrap(); + fixture + .mailbox + .accept_message(message_draft( + &format!("legacy-{kind:?}"), + sender, + recipient, + kind, + "legacy attention message", + true, + )) + .await + .unwrap(); + } + + let pending = fixture.mailbox.pending(recipient, 20).await.unwrap(); + assert_eq!(pending.len(), 8); + for kind in kinds { + assert_eq!(pending.iter().filter(|message| message.kind == kind).count(), 2); + } + assert_eq!( + pending.iter().filter(|message| message.legacy_attention).count(), + 4 + ); + assert_eq!(fixture.mailbox.quiet_count(recipient).await.unwrap(), 4); + assert_eq!(fixture.mailbox.attention(recipient, 20).await.unwrap().len(), 4); +} + +#[tokio::test] +async fn activation_claim_is_not_reissued_after_a_lost_result() { + let fixture = test_mailbox().await; + let sender = thread_id(20); + let recipient = thread_id(21); + let message = fixture + .mailbox + .accept_message(message_draft( + "activation-request", + sender, + recipient, + MessageKind::Alert, + "activate once", + false, + )) + .await + .unwrap(); + + assert_eq!( + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::Activate + ); + assert_eq!( + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::RecoveryNeeded + ); + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Activated) + .await + .unwrap(); + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Activated) + .await + .unwrap(); + assert_eq!( + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::AlreadyDecided(ActivationDisposition::Activated) + ); + assert!(matches!( + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Denied) + .await, + Err(MailboxError::IdempotencyConflict) + )); +} + +#[tokio::test] +async fn oversized_native_capture_is_rejected_without_creating_dispatchable_work() { + let fixture = test_mailbox().await; + let issuer = thread_id(30); + let agent = thread_id(31); + let mut too_large = native_work(issuer, agent, "too-large", "large-submission", "x"); + too_large.input.content = "x".repeat(1_000_001); + + assert!(matches!( + fixture.mailbox.accept_work(too_large).await, + Err(MailboxError::Backpressure(_)) + )); + assert!(fixture + .mailbox + .work_for_native_key(issuer, "too-large") + .await + .unwrap() + .is_none()); +} + +#[test] +fn closed_peer_cycles_require_every_join_choice_to_remain_inside_the_cycle() { + let any_cycle = [ + Dependency { + owner: "a".to_string(), + peers: vec!["b".to_string(), "c".to_string()], + mode: DependencyMode::Any, + }, + Dependency { + owner: "b".to_string(), + peers: vec!["a".to_string()], + mode: DependencyMode::All, + }, + Dependency { + owner: "c".to_string(), + peers: vec!["a".to_string()], + mode: DependencyMode::All, + }, + ]; + assert!(is_provably_closed_peer_cycle("a", &any_cycle)); + + let all_cycle = [ + Dependency { + owner: "x".to_string(), + peers: vec!["y".to_string(), "z".to_string()], + mode: DependencyMode::All, + }, + Dependency { + owner: "y".to_string(), + peers: vec!["x".to_string()], + mode: DependencyMode::Any, + }, + Dependency { + owner: "z".to_string(), + peers: vec!["x".to_string()], + mode: DependencyMode::Any, + }, + ]; + assert!(is_provably_closed_peer_cycle("x", &all_cycle)); + + let with_open_exit = [ + Dependency { + owner: "a".to_string(), + peers: vec!["b".to_string(), "not-in-cycle".to_string()], + mode: DependencyMode::Any, + }, + Dependency { + owner: "b".to_string(), + peers: vec!["a".to_string()], + mode: DependencyMode::All, + }, + ]; + assert!(!is_provably_closed_peer_cycle("a", &with_open_exit)); + + let all_with_open_exit = [ + Dependency { + owner: "x".to_string(), + peers: vec!["y".to_string(), "not-in-cycle".to_string()], + mode: DependencyMode::All, + }, + Dependency { + owner: "y".to_string(), + peers: vec!["x".to_string()], + mode: DependencyMode::Any, + }, + ]; + assert!(!is_provably_closed_peer_cycle("x", &all_with_open_exit)); +} diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs new file mode 100644 index 000000000000..e2d406c0052e --- /dev/null +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -0,0 +1,412 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::messages::decrement_pending_count; +use crate::store::Mailbox; +use crate::types::{CanonicalConfirmation, Envelope, PendingPage}; +use codex_protocol::ThreadId; +use std::future::Future; +use sqlx::Row; + +const MAX_PAGE_SIZE: usize = 100; + +impl Mailbox { + /// Return a bounded page of uncanonical messages without consuming them. + pub async fn pending_page( + &self, + recipient: ThreadId, + after_seq: Option, + limit: usize, + ) -> MailboxResult { + validate_page(limit)?; + let rows = sqlx::query( + "SELECT * FROM mailbox_messages + WHERE recipient = ? AND (canonical = 0 OR input_admitted = 0) + AND superseded_by IS NULL AND withdrawn = 0 + AND (? IS NULL OR seq > ?) + ORDER BY seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind(after_seq) + .bind(after_seq) + .bind((limit + 1) as i64) + .fetch_all(&self.pool) + .await?; + page_from_rows(rows, limit) + } + + /// Convenience first page; use `pending_page` to continue pagination. + pub async fn pending(&self, recipient: ThreadId, limit: usize) -> MailboxResult> { + Ok(self.pending_page(recipient, None, limit).await?.items) + } + + /// Read a bounded page including canonical records, without consuming them. + pub async fn read_page( + &self, + recipient: ThreadId, + after_seq: Option, + limit: usize, + ) -> MailboxResult { + validate_page(limit)?; + let rows = sqlx::query( + "SELECT * FROM mailbox_messages + WHERE recipient = ? AND superseded_by IS NULL + AND (? IS NULL OR seq > ?) + ORDER BY seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind(after_seq) + .bind(after_seq) + .bind((limit + 1) as i64) + .fetch_all(&self.pool) + .await?; + page_from_rows(rows, limit) + } + + pub async fn get_message( + &self, + recipient: ThreadId, + id: &str, + ) -> MailboxResult> { + let row = sqlx::query( + "SELECT * FROM mailbox_messages WHERE id = ? AND recipient = ? LIMIT 1", + ) + .bind(id) + .bind(recipient.to_string()) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(envelope_from_row).transpose() + } + + /// Withdraw an unresolved request authored by this exact native thread. + /// Withdrawal is durable and idempotent; replies cannot target it afterward. + pub async fn withdraw_message(&self, sender: ThreadId, id: &str) -> MailboxResult<()> { + let mut transaction = self.pool.begin().await?; + let row = sqlx::query( + "SELECT sender, kind, canonical, input_admitted, legacy_attention, superseded_by, withdrawn + FROM mailbox_messages WHERE id = ?", + ) + .bind(id) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if row.try_get::("sender")? != sender.to_string() { + return Err(MailboxError::WrongRecipient); + } + if row.try_get::("kind")? != "\"request\"" { + return Err(MailboxError::InvalidInput("only requests may be withdrawn".into())); + } + if row.try_get::("withdrawn")? != 0 { + transaction.commit().await?; + return Ok(()); + } + let superseded_by: Option = row.try_get("superseded_by")?; + if superseded_by.is_some() { + return Err(MailboxError::ReplacementNotAllowed); + } + let reply_exists: Option = sqlx::query_scalar( + "SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1", + ) + .bind(id) + .fetch_optional(&mut *transaction) + .await?; + if reply_exists.is_some() { + return Err(MailboxError::RequestAlreadyResolved); + } + let canonical = row.try_get::("canonical")? != 0; + let input_admitted = row.try_get::("input_admitted")? != 0; + let quiet = row.try_get::("legacy_attention")? == 0; + let updated = sqlx::query( + "UPDATE mailbox_messages SET withdrawn = 1 + WHERE id = ? AND sender = ? AND withdrawn = 0", + ) + .bind(id) + .bind(sender.to_string()) + .execute(&mut *transaction) + .await?; + if updated.rows_affected() == 1 && (!canonical || !input_admitted) { + let recipient: String = sqlx::query_scalar( + "SELECT recipient FROM mailbox_messages WHERE id = ?", + ) + .bind(id) + .fetch_one(&mut *transaction) + .await?; + decrement_pending_count(&mut transaction, &recipient, quiet).await?; + } + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } + + /// Count quiet, uncanonical message delivery without reading message bodies. + pub async fn quiet_count(&self, recipient: ThreadId) -> MailboxResult { + let count: i64 = sqlx::query_scalar( + "SELECT quiet_count FROM mailbox_pending_counts WHERE recipient = ?", + ) + .bind(recipient.to_string()) + .fetch_optional(&self.pool) + .await? + .unwrap_or(0); + Ok(count.max(0) as usize) + } + + /// Return stable IDs for canonical current bodies not yet admitted to a + /// provider request. Callers join these IDs to their existing raw history. + pub async fn unadmitted_canonical_ids( + &self, + recipient: ThreadId, + limit: usize, + ) -> MailboxResult> { + if limit == 0 || limit > 4_096 { + return Err(MailboxError::InvalidInput( + "unadmitted body limit must be between 1 and 4096".into(), + )); + } + let rows = sqlx::query( + "SELECT id FROM mailbox_messages + WHERE recipient = ? AND canonical = 1 AND input_admitted = 0 + AND superseded_by IS NULL AND withdrawn = 0 + ORDER BY seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind((limit + 1) as i64) + .fetch_all(&self.pool) + .await?; + if rows.len() > limit { + return Err(MailboxError::Backpressure("unadmitted canonical body limit")); + } + rows.iter() + .map(|row| row.try_get("id").map_err(Into::into)) + .collect() + } + + /// Filter provider-supplied AgentMessage IDs through exact mailbox ownership. + /// Unrelated native history IDs are ignored; only unadmitted IDs are returned. + pub async fn unadmitted_input_ids( + &self, + recipient: ThreadId, + ids: &[String], + ) -> MailboxResult> { + if ids.len() > 100 { + return Err(MailboxError::Backpressure("provider input ID chunk limit")); + } + let mut selected = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for id in ids { + if !seen.insert(id.as_str()) { + continue; + } + let message: Option = sqlx::query_scalar( + "SELECT id FROM mailbox_messages + WHERE id = ? AND recipient = ? AND input_admitted = 0 LIMIT 1", + ) + .bind(id) + .bind(recipient.to_string()) + .fetch_optional(&self.pool) + .await?; + let native_work: Option = sqlx::query_scalar( + "SELECT work.input_message_id FROM mailbox_work AS work + WHERE work.input_message_id = ? AND work.agent = ? + AND NOT EXISTS ( + SELECT 1 FROM mailbox_input_admissions AS admitted + WHERE admitted.recipient = work.agent + AND admitted.input_message_id = work.input_message_id + ) LIMIT 1", + ) + .bind(id) + .bind(recipient.to_string()) + .fetch_optional(&self.pool) + .await?; + if message.is_some() || native_work.is_some() { + selected.push(id.clone()); + } + } + Ok(selected) + } + + /// Exact bounded list of accepted native-work input IDs not yet admitted. + pub async fn unadmitted_native_input_ids( + &self, + recipient: ThreadId, + limit: usize, + ) -> MailboxResult> { + if limit == 0 || limit > 4_096 { + return Err(MailboxError::InvalidInput( + "unadmitted native input limit must be between 1 and 4096".into(), + )); + } + let rows = sqlx::query( + "SELECT work.input_message_id FROM mailbox_work AS work + WHERE work.agent = ? AND NOT EXISTS ( + SELECT 1 FROM mailbox_input_admissions AS admitted + WHERE admitted.recipient = work.agent + AND admitted.input_message_id = work.input_message_id + ) + ORDER BY work.seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind((limit + 1) as i64) + .fetch_all(&self.pool) + .await?; + if rows.len() > limit { + return Err(MailboxError::Backpressure("unadmitted native input limit")); + } + rows.iter() + .map(|row| row.try_get("input_message_id").map_err(Into::into)) + .collect() + } + + /// Mark canonical admission only after the adapter has appended and flushed + /// the stable ID. This operation is idempotent and recipient-bound. + pub async fn mark_canonical(&self, recipient: ThreadId, id: &str) -> MailboxResult<()> { + let mut transaction = self.pool.begin().await?; + let current = sqlx::query( + "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn FROM mailbox_messages + WHERE id = ? AND recipient = ?", + ) + .bind(id) + .bind(recipient.to_string()) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::WrongRecipient)?; + let was_canonical = current.try_get::("canonical")? != 0; + let input_admitted = current.try_get::("input_admitted")? != 0; + let quiet = current.try_get::("legacy_attention")? == 0; + let is_current = current.try_get::, _>("superseded_by")?.is_none() + && current.try_get::("withdrawn")? == 0; + let result = sqlx::query( + "UPDATE mailbox_messages SET canonical = 1 + WHERE id = ? AND recipient = ? AND canonical = 0", + ) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + if result.rows_affected() > 0 && !was_canonical && input_admitted && is_current { + decrement_pending_count(&mut transaction, &recipient.to_string(), quiet).await?; + } + sqlx::query("UPDATE mailbox_attention SET canonical = 1 WHERE id = ?") + .bind(format!("attention:{id}")) + .execute(&mut *transaction) + .await?; + sqlx::query( + "UPDATE mailbox_work SET outcome_canonical = 1 + WHERE outcome_id = ? AND issuer = ?", + ) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } + + /// Mark a no-response or typed outcome after the adapter has appended its + /// exact `OutcomeReceipt.id` and flushed canonical history. + pub async fn mark_outcome_canonical( + &self, + recipient: ThreadId, + id: &str, + ) -> MailboxResult<()> { + let mut transaction = self.pool.begin().await?; + let result = sqlx::query( + "UPDATE mailbox_work SET outcome_canonical = 1 + WHERE outcome_id = ? AND issuer = ? AND outcome_json IS NOT NULL", + ) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + if result.rows_affected() == 0 { + return Err(MailboxError::WrongRecipient); + } + sqlx::query( + "UPDATE mailbox_attention SET canonical = 1 + WHERE id IN ( + SELECT 'attention:' || id FROM mailbox_work + WHERE outcome_id = ? AND issuer = ? + )", + ) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } + + /// Reconcile one bounded page using a stable-ID-aware adapter callback. + /// The callback must return only after append+flush or exact-ID readback. + pub async fn reconcile_pending( + &self, + recipient: ThreadId, + limit: usize, + mut append_and_flush: F, + ) -> MailboxResult + where + F: FnMut(Envelope) -> Fut, + Fut: Future>, + { + let page = self.pending_page(recipient, None, limit).await?; + let mut reconciled = 0; + for envelope in page.items { + let id = envelope.id.clone(); + let _confirmation = append_and_flush(envelope).await?; + self.mark_canonical(recipient, &id).await?; + reconciled += 1; + } + Ok(reconciled) + } +} + +pub(crate) fn page_from_rows( + rows: Vec, + limit: usize, +) -> MailboxResult { + let has_more = rows.len() > limit; + let visible = rows.into_iter().take(limit).collect::>(); + let next_cursor = if has_more { + visible + .last() + .map(|row| row.try_get::("seq")) + .transpose()? + } else { + None + }; + let items = visible + .iter() + .map(envelope_from_row) + .collect::>>()?; + Ok(PendingPage { items, next_cursor }) +} + +pub(crate) fn envelope_from_row(row: &sqlx::sqlite::SqliteRow) -> MailboxResult { + let sender = ThreadId::from_string(row.try_get::("sender")?.as_str()) + .map_err(|_| MailboxError::InvalidInput("stored sender thread ID".into()))?; + let recipient = ThreadId::from_string(row.try_get::("recipient")?.as_str()) + .map_err(|_| MailboxError::InvalidInput("stored recipient thread ID".into()))?; + let work_ref: Option = row.try_get("work_ref_json")?; + Ok(Envelope { + id: row.try_get("id")?, + invocation_key: row.try_get("invocation_key")?, + sender, + recipient, + communication: serde_json::from_str(&row.try_get::("communication_json")?)?, + kind: serde_json::from_str(&row.try_get::("kind")?)?, + activate_if_idle: row.try_get::("activate_if_idle")? != 0, + work_ref: work_ref.map(|value| serde_json::from_str(&value)).transpose()?, + reply_to: row.try_get("reply_to")?, + supersedes: row.try_get("supersedes")?, + legacy_attention: row.try_get::("legacy_attention")? != 0, + canonical: row.try_get::("canonical")? != 0, + }) +} + +fn validate_page(limit: usize) -> MailboxResult<()> { + if limit == 0 || limit > MAX_PAGE_SIZE { + return Err(MailboxError::InvalidInput(format!( + "page size must be between 1 and {MAX_PAGE_SIZE}" + ))); + } + Ok(()) +} diff --git a/codex-rs/agent-mailbox/src/message_rules.rs b/codex-rs/agent-mailbox/src/message_rules.rs new file mode 100644 index 000000000000..7432f086cb9d --- /dev/null +++ b/codex-rs/agent-mailbox/src/message_rules.rs @@ -0,0 +1,86 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::types::{MessageDraft, MessageKind}; +use sqlx::{Row, Sqlite, Transaction}; + +pub(crate) async fn validate_reply_tx( + transaction: &mut Transaction<'_, Sqlite>, + request_id: &str, + draft: &MessageDraft, +) -> MailboxResult<()> { + let request = sqlx::query( + "SELECT sender, recipient, kind, canonical FROM mailbox_messages WHERE id = ?", + ) + .bind(request_id) + .fetch_optional(&mut **transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if request.try_get::("kind")? != "\"request\"" { + return Err(MailboxError::InvalidInput("reply_to must name a request".into())); + } + if request.try_get::("recipient")? != draft.sender.to_string() + || request.try_get::("sender")? != draft.recipient.to_string() + { + return Err(MailboxError::WrongRecipient); + } + if request.try_get::("canonical")? == 0 { + return Err(MailboxError::NotCanonical); + } + let resolved: Option = sqlx::query_scalar( + "SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1", + ) + .bind(request_id) + .fetch_optional(&mut **transaction) + .await?; + if resolved.is_some() { + return Err(MailboxError::RequestAlreadyResolved); + } + Ok(()) +} + +pub(crate) async fn validate_replacement_tx( + transaction: &mut Transaction<'_, Sqlite>, + old_id: &str, + draft: &MessageDraft, +) -> MailboxResult<()> { + let old = sqlx::query( + "SELECT sender, recipient, work_ref_json, kind, canonical, superseded_by, withdrawn + FROM mailbox_messages WHERE id = ?", + ) + .bind(old_id) + .fetch_optional(&mut **transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + let old_work: Option = old.try_get("work_ref_json")?; + let new_work = draft.work_ref.as_ref().map(serde_json::to_string).transpose()?; + if old.try_get::("sender")? != draft.sender.to_string() + || old.try_get::("recipient")? != draft.recipient.to_string() + || old_work != new_work + { + return Err(MailboxError::WrongRecipient); + } + let already_replaced: Option = old.try_get("superseded_by")?; + let old_kind: String = old.try_get("kind")?; + let valid_pair = match old_kind.as_str() { + "\"progress\"" => draft.kind == MessageKind::Progress, + "\"request\"" => matches!(draft.kind, MessageKind::Request | MessageKind::Note), + _ => false, + }; + if !valid_pair + || old.try_get::("canonical")? != 0 + || already_replaced.is_some() + || old.try_get::("withdrawn")? != 0 + { + return Err(MailboxError::ReplacementNotAllowed); + } + if old_kind == "\"request\"" { + let resolved: Option = + sqlx::query_scalar("SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1") + .bind(old_id) + .fetch_optional(&mut **transaction) + .await?; + if resolved.is_some() { + return Err(MailboxError::RequestAlreadyResolved); + } + } + Ok(()) +} diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs new file mode 100644 index 000000000000..db32d51a9ee6 --- /dev/null +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -0,0 +1,409 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::store::Mailbox; +use crate::types::{Envelope, MessageDraft, MessageKind, WorkRef}; +use crate::message_rules::{validate_reply_tx, validate_replacement_tx}; +use crate::work::exact_work; +use codex_protocol::{ResponseItemId, ThreadId}; +use serde::Serialize; +use sqlx::{Row, Sqlite, Transaction}; + +const MAX_MESSAGE_BYTES: usize = 8_192; +const MAX_RECORD_BYTES: usize = 16_384; +const MAX_KEY_BYTES: usize = 256; +const MAX_TOTAL_MESSAGES: i64 = 100_000; + +#[derive(Serialize)] +struct DraftRequest<'a> { + invocation_key: &'a str, + sender: ThreadId, + recipient: ThreadId, + communication: &'a codex_protocol::protocol::InterAgentCommunication, + kind: MessageKind, + activate_if_idle: bool, + work_ref: &'a Option, + reply_to: &'a Option, + supersedes: &'a Option, + legacy_attention: bool, +} + +impl Mailbox { + /// Durably accept a message intent. Retries with the same invocation key + /// return its stable ID; reusing a key for a different request is rejected. + pub async fn accept_message(&self, draft: MessageDraft) -> MailboxResult { + let id = match draft.communication.id.as_ref() { + Some(id) if id.as_str().starts_with("amsg_") => id.to_string(), + Some(_) => { + return Err(MailboxError::InvalidInput( + "expected stable amsg_* ID".into(), + )); + } + None => new_mailbox_id(), + }; + let inserted_id = id.clone(); + let envelope = { + let mut transaction = self.pool.begin().await?; + let envelope = insert_message_tx(&mut transaction, draft, Some(inserted_id)).await?; + transaction.commit().await?; + envelope + }; + self.notify_changed(); + Ok(envelope) + } + +} + +pub(crate) async fn insert_message_tx( + transaction: &mut Transaction<'_, Sqlite>, + mut draft: MessageDraft, + explicit_id: Option, +) -> MailboxResult { + validate_key(&draft.invocation_key)?; + if draft.kind == MessageKind::Progress && draft.reply_to.is_some() { + return Err(MailboxError::InvalidInput( + "progress messages cannot be replies".into(), + )); + } + let id = explicit_id + .or_else(|| draft.communication.id.as_ref().map(ToString::to_string)) + .unwrap_or_else(new_mailbox_id); + validate_id(&id)?; + draft.communication.id = Some(ResponseItemId::from_server(id.clone())); + + let mut fingerprint = draft.clone(); + fingerprint.communication.id = None; + let request_json = serde_json::to_string(&DraftRequest { + invocation_key: &fingerprint.invocation_key, + sender: &fingerprint.sender, + recipient: &fingerprint.recipient, + communication: &fingerprint.communication, + kind: fingerprint.kind, + activate_if_idle: fingerprint.activate_if_idle, + work_ref: &fingerprint.work_ref, + reply_to: &fingerprint.reply_to, + supersedes: &fingerprint.supersedes, + legacy_attention: fingerprint.legacy_attention, + })?; + let communication_json = serde_json::to_string(&draft.communication)?; + let body_bytes = draft.communication.content.len() + + draft.communication.encrypted_content.as_ref().map_or(0, String::len); + if body_bytes > MAX_MESSAGE_BYTES + || request_json.len() > MAX_RECORD_BYTES + || communication_json.len() > MAX_RECORD_BYTES + { + return Err(MailboxError::Backpressure("message byte limit")); + } + if let Some(row) = sqlx::query( + "SELECT message_id, request_json FROM mailbox_invocations WHERE invocation_key = ?", + ) + .bind(&draft.invocation_key) + .fetch_optional(&mut **transaction) + .await? + { + if row.try_get::("request_json")? != request_json { + return Err(MailboxError::IdempotencyConflict); + } + let message_id: String = row.try_get("message_id")?; + let message = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") + .bind(message_id) + .fetch_one(&mut **transaction) + .await?; + return envelope_from_row(&message); + } + + if let Some(work_ref) = draft.work_ref.as_ref() { + let work = exact_work(&mut **transaction, work_ref).await?; + let sender = draft.sender.to_string(); + let recipient = draft.recipient.to_string(); + let issuer = work.try_get::("issuer")?; + let agent = work.try_get::("agent")?; + if !((sender == issuer && recipient == agent) || (sender == agent && recipient == issuer)) { + return Err(MailboxError::WrongRecipient); + } + if draft.kind == MessageKind::Progress + && work.try_get::, _>("outcome_json")?.is_some() + { + return Err(MailboxError::OutcomeAlreadyRecorded); + } + } + if let Some(reply_to) = draft.reply_to.as_deref() { + validate_reply_tx(transaction, reply_to, &draft).await?; + } + if let Some(supersedes) = draft.supersedes.as_deref() { + validate_replacement_tx(transaction, supersedes, &draft).await?; + } + + if draft.supersedes.is_none() && draft.kind == MessageKind::Progress && !draft.activate_if_idle { + if let Some(work_ref) = draft.work_ref.as_ref() { + if let Some(existing) = sqlx::query( + "SELECT * FROM mailbox_messages + WHERE sender = ? AND recipient = ? AND work_ref_json = ? + AND author_path = ? AND recipient_path = ? + AND kind = 'progress' AND canonical = 0 AND input_admitted = 0 + AND superseded_by IS NULL + AND legacy_attention = ? + AND NOT EXISTS ( + SELECT 1 FROM mailbox_messages AS newer + WHERE newer.supersedes = mailbox_messages.id + ) + ORDER BY seq DESC LIMIT 1", + ) + .bind(draft.sender.to_string()) + .bind(draft.recipient.to_string()) + .bind(serde_json::to_string(work_ref)?) + .bind(serde_json::to_string(&draft.communication.author)?) + .bind(serde_json::to_string(&draft.communication.recipient)?) + .bind(i64::from(draft.legacy_attention)) + .fetch_optional(&mut **transaction) + .await? + { + let existing_id: String = existing.try_get("id")?; + draft.communication.id = Some(ResponseItemId::from_server(existing_id.clone())); + let coalesced_communication = serde_json::to_string(&draft.communication)?; + update_coalesced_tx( + transaction, + &existing_id, + &draft, + &request_json, + &coalesced_communication, + ) + .await?; + add_invocation_tx(transaction, &draft.invocation_key, &request_json, &existing_id).await?; + if draft.legacy_attention { + insert_attention_tx(transaction, &existing_id, draft.recipient.to_string().as_str()).await?; + } + let row = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") + .bind(existing_id) + .fetch_one(&mut **transaction) + .await?; + return envelope_from_row(&row); + } + } + } + + let collision: Option = sqlx::query_scalar( + "SELECT id FROM mailbox_messages WHERE id = ? LIMIT 1", + ) + .bind(&id) + .fetch_optional(&mut **transaction) + .await?; + if collision.is_some() { + return Err(MailboxError::IdempotencyConflict); + } + let pending_count = pending_count_tx(transaction, &draft.recipient.to_string()).await?; + if pending_count >= crate::messages::MAX_PENDING_PER_RECIPIENT { + return Err(MailboxError::Backpressure("recipient pending-message count")); + } + let total_count: i64 = sqlx::query_scalar( + "SELECT item_count FROM mailbox_counts WHERE name = 'message'", + ) + .fetch_one(&mut **transaction) + .await?; + if total_count >= MAX_TOTAL_MESSAGES { + return Err(MailboxError::Backpressure("retained message count")); + } + + let kind_json = serde_json::to_string(&draft.kind)?; + sqlx::query( + "INSERT INTO mailbox_messages ( + id, invocation_key, request_json, sender, recipient, author_path, recipient_path, + communication_json, + kind, work_ref_json, reply_to, supersedes, activate_if_idle, legacy_attention + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&id) + .bind(&draft.invocation_key) + .bind(&request_json) + .bind(draft.sender.to_string()) + .bind(draft.recipient.to_string()) + .bind(serde_json::to_string(&draft.communication.author)?) + .bind(serde_json::to_string(&draft.communication.recipient)?) + .bind(&communication_json) + .bind(kind_json) + .bind(draft.work_ref.as_ref().map(serde_json::to_string).transpose()?) + .bind(&draft.reply_to) + .bind(&draft.supersedes) + .bind(i64::from(draft.activate_if_idle)) + .bind(i64::from(draft.legacy_attention)) + .execute(&mut **transaction) + .await?; + sqlx::query( + "INSERT INTO mailbox_activations(message_id, sender, state, disposition) + VALUES (?, ?, ?, ?)", + ) + .bind(&id) + .bind(draft.sender.to_string()) + .bind(if draft.activate_if_idle { "pending" } else { "decided" }) + .bind(if draft.activate_if_idle { None } else { Some("not_requested") }) + .execute(&mut **transaction) + .await?; + add_invocation_tx(transaction, &draft.invocation_key, &request_json, &id).await?; + if let Some(old_id) = draft.supersedes.as_deref() { + let old_quiet = sqlx::query_scalar::<_, i64>( + "SELECT CASE WHEN legacy_attention = 0 THEN 1 ELSE 0 END + FROM mailbox_messages WHERE id = ? AND canonical = 0", + ) + .bind(old_id) + .fetch_optional(&mut **transaction) + .await? + .unwrap_or(0) + != 0; + let replaced = sqlx::query("UPDATE mailbox_messages SET superseded_by = ? WHERE id = ? AND canonical = 0") + .bind(&id) + .bind(old_id) + .execute(&mut **transaction) + .await?; + if replaced.rows_affected() == 1 { + decrement_pending_count(transaction, &draft.recipient.to_string(), old_quiet).await?; + } + } + sqlx::query("UPDATE mailbox_counts SET item_count = item_count + 1 WHERE name = 'message'") + .execute(&mut **transaction) + .await?; + increment_pending_count( + transaction, + &draft.recipient.to_string(), + !draft.legacy_attention, + ) + .await?; + if draft.legacy_attention { + insert_attention_tx(transaction, &id, draft.recipient.to_string().as_str()).await?; + } + let row = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") + .bind(&id) + .fetch_one(&mut **transaction) + .await?; + envelope_from_row(&row) +} + +async fn update_coalesced_tx( + transaction: &mut Transaction<'_, Sqlite>, + id: &str, + draft: &MessageDraft, + request_json: &str, + communication_json: &str, +) -> MailboxResult<()> { + sqlx::query( + "UPDATE mailbox_messages SET invocation_key = ?, request_json = ?, + communication_json = ?, author_path = ?, recipient_path = ?, kind = ?, + reply_to = ?, supersedes = ?, legacy_attention = ? + WHERE id = ? AND canonical = 0", + ) + .bind(&draft.invocation_key) + .bind(request_json) + .bind(communication_json) + .bind(serde_json::to_string(&draft.communication.author)?) + .bind(serde_json::to_string(&draft.communication.recipient)?) + .bind(serde_json::to_string(&draft.kind)?) + .bind(&draft.reply_to) + .bind(&draft.supersedes) + .bind(i64::from(draft.legacy_attention)) + .bind(id) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +async fn add_invocation_tx( + transaction: &mut Transaction<'_, Sqlite>, + invocation_key: &str, + request_json: &str, + message_id: &str, +) -> MailboxResult<()> { + sqlx::query( + "INSERT INTO mailbox_invocations(invocation_key, request_json, message_id) + VALUES (?, ?, ?)", + ) + .bind(invocation_key) + .bind(request_json) + .bind(message_id) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +async fn insert_attention_tx( + transaction: &mut Transaction<'_, Sqlite>, + message_id: &str, + recipient: &str, +) -> MailboxResult<()> { + sqlx::query( + "INSERT INTO mailbox_attention(id, recipient) VALUES (?, ?) + ON CONFLICT(id) DO NOTHING", + ) + .bind(format!("attention:{message_id}")) + .bind(recipient) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +async fn pending_count_tx( + transaction: &mut Transaction<'_, Sqlite>, + recipient: &str, +) -> MailboxResult { + Ok(sqlx::query_scalar( + "SELECT item_count FROM mailbox_pending_counts WHERE recipient = ?", + ) + .bind(recipient) + .fetch_optional(&mut **transaction) + .await? + .unwrap_or(0)) +} + +async fn increment_pending_count( + transaction: &mut Transaction<'_, Sqlite>, + recipient: &str, + quiet: bool, +) -> MailboxResult<()> { + sqlx::query( + "INSERT INTO mailbox_pending_counts(recipient, item_count, quiet_count) + VALUES (?, 1, ?) + ON CONFLICT(recipient) DO UPDATE SET + item_count = item_count + 1, + quiet_count = quiet_count + excluded.quiet_count", + ) + .bind(recipient) + .bind(i64::from(quiet)) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +pub(crate) async fn decrement_pending_count( + transaction: &mut Transaction<'_, Sqlite>, + recipient: &str, + quiet: bool, +) -> MailboxResult<()> { + sqlx::query( + "UPDATE mailbox_pending_counts SET + item_count = MAX(item_count - 1, 0), + quiet_count = MAX(quiet_count - ?, 0) + WHERE recipient = ?", + ) + .bind(i64::from(quiet)) + .bind(recipient) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +fn validate_key(key: &str) -> MailboxResult<()> { + if key.is_empty() || key.len() > MAX_KEY_BYTES { + return Err(MailboxError::InvalidInput("invalid invocation key".into())); + } + Ok(()) +} + +fn validate_id(id: &str) -> MailboxResult<()> { + if id.starts_with("amsg_") && id.len() > 5 && id.len() <= 256 { + Ok(()) + } else { + Err(MailboxError::InvalidInput("expected stable amsg_* ID".into())) + } +} + +pub(crate) const MAX_PENDING_PER_RECIPIENT: i64 = 4_096; + +fn new_mailbox_id() -> String { + ResponseItemId::with_suffix("amsg", uuid::Uuid::now_v7()).to_string() +} diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs new file mode 100644 index 000000000000..c59b2f7beb13 --- /dev/null +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -0,0 +1,747 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::messages::insert_message_tx; +use crate::store::Mailbox; +use crate::types::{ + Continuation, ContinuationReceipt, ContinuationState, GoalBinding, MessageDraft, + MessageKind, Outcome, OutcomeReceipt, TurnOutcomeReceipt, WorkRef, +}; +use crate::work::{ + ensure_mailbox_id, exact_work, exact_work_tx, new_mailbox_id, work_ref_from_row, +}; +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::{AgentPath, ThreadId}; +use serde::{Deserialize, Serialize}; +use sqlx::{Row, Sqlite, SqlitePool, Transaction}; + +const MAX_TURN_INPUTS: usize = 100; +const MAX_KEY_BYTES: usize = 256; +const MAX_AUTO_RESPONSE_BYTES: usize = 16_384; +const MAX_RESPONSE_BYTES: usize = 8_192; +const MAX_CONTINUATIONS_PER_WORK: i64 = 32; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct OutcomeData { + outcome: Outcome, + continuation: Continuation, + response: Option, +} + +impl OutcomeData { + fn new( + outcome: Outcome, + continuation: Continuation, + mut response: Option, + ) -> Self { + if let Some(response) = response.as_mut() { + response.id = None; + } + Self { + outcome, + continuation, + response, + } + } +} + +struct ResponseGroup { + issuer: ThreadId, + agent: ThreadId, + author: AgentPath, + recipient: AgentPath, + refs: Vec, + response: Option, + output_id: Option, +} + +impl Mailbox { + /// Freeze and record only the work admitted to this native turn. A pending + /// or deferred goal binding is stored separately and is not a terminal outcome. + pub async fn record_turn_outcome( + &self, + recipient: ThreadId, + source_turn_id: &str, + outcome: Outcome, + response: Option, + continuation: Continuation, + ) -> MailboxResult { + validate_key(source_turn_id)?; + let mut transaction = self.pool.begin().await?; + let admitted_rows = sqlx::query( + "SELECT work.* FROM mailbox_work AS work + JOIN mailbox_admissions AS admission ON admission.work_id = work.id + WHERE admission.recipient = ? AND admission.source_turn_id = ? + ORDER BY work.seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind(source_turn_id) + .bind((MAX_TURN_INPUTS + 1) as i64) + .fetch_all(&mut *transaction) + .await?; + if admitted_rows.len() > MAX_TURN_INPUTS { + return Err(MailboxError::Backpressure("turn outcome input count")); + } + let mut refs = admitted_rows + .iter() + .map(work_ref_from_row) + .collect::>>()?; + + if matches!(&continuation, Continuation::None | Continuation::Terminal(_)) { + let binding_json = match &continuation { + Continuation::Terminal(binding) => Some(serde_json::to_string(binding)?), + _ => None, + }; + let mut eligible = Vec::with_capacity(refs.len()); + for work_ref in refs { + let unresolved = sqlx::query_scalar::<_, String>( + "SELECT binding_json FROM mailbox_continuations + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') + ORDER BY binding_json LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(&mut *transaction) + .await?; + if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation count")); + } + if unresolved.is_empty() + || binding_json.as_ref().is_some_and(|binding| unresolved.contains(binding)) + { + eligible.push(work_ref); + } + } + refs = eligible; + } + + if let Continuation::Terminal(binding) = &continuation { + validate_binding_for_agent(binding, recipient)?; + let binding_json = serde_json::to_string(binding)?; + let linked = sqlx::query( + "SELECT work.* FROM mailbox_continuations AS continuation + JOIN mailbox_work AS work ON work.id = continuation.work_id + WHERE continuation.binding_json = ? + AND continuation.state IN ('pending_goal_binding', 'deferred') + AND work.agent = ? AND work.outcome_json IS NULL + ORDER BY work.seq LIMIT ?", + ) + .bind(&binding_json) + .bind(recipient.to_string()) + .bind((MAX_TURN_INPUTS + 1) as i64) + .fetch_all(&mut *transaction) + .await?; + if linked.len() > MAX_TURN_INPUTS { + return Err(MailboxError::Backpressure("continuation settlement count")); + } + for row in linked { + let work_ref = work_ref_from_row(&row)?; + if !refs.iter().any(|existing| existing.id == work_ref.id) { + refs.push(work_ref); + } + } + if refs.len() > MAX_TURN_INPUTS { + return Err(MailboxError::Backpressure("turn outcome input count")); + } + } + + if refs.is_empty() { + transaction.commit().await?; + return Ok(TurnOutcomeReceipt { + outcomes: Vec::new(), + continuation_work: Vec::new(), + no_matching_work: true, + already_recorded: false, + }); + } + for work_ref in &refs { + if work_ref.agent != recipient { + return Err(MailboxError::WrongRecipient); + } + if let Some(binding) = continuation_binding(&continuation) { + validate_binding_for_work(binding, work_ref)?; + } + } + + let mut groups = response_groups(&mut transaction, &refs, response.as_ref()).await?; + validate_auto_response_size(&groups)?; + if is_nonterminal(&continuation) { + let binding = continuation_binding(&continuation) + .expect("nonterminal continuation always carries a binding"); + let binding_json = serde_json::to_string(binding)?; + for work_ref in &refs { + let unresolved = sqlx::query_scalar::<_, String>( + "SELECT binding_json FROM mailbox_continuations + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') + ORDER BY binding_json LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(&mut *transaction) + .await?; + if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation count")); + } + if unresolved.iter().any(|old_binding| old_binding != &binding_json) { + return Err(MailboxError::ReferenceMismatch); + } + } + for group in &mut groups { + let _response_id = insert_response_group( + &mut transaction, + group, + format!("continuation:{}:{}:{}", recipient, source_turn_id, group.issuer), + outcome, + ) + .await?; + } + let state = continuation_state(&continuation); + for work_ref in &refs { + let row = exact_work_tx(&mut transaction, work_ref).await?; + if row.try_get::, _>("outcome_json")?.is_some() { + return Err(MailboxError::OutcomeAlreadyRecorded); + } + let response_id = groups + .iter() + .find(|group| group.issuer == work_ref.issuer) + .and_then(|group| group.output_id.as_deref()); + persist_continuation( + &mut transaction, + work_ref, + &binding_json, + state, + outcome, + source_turn_id, + response_id, + ) + .await?; + } + transaction.commit().await?; + self.notify_changed(); + return Ok(TurnOutcomeReceipt { + outcomes: Vec::new(), + continuation_work: refs, + no_matching_work: false, + already_recorded: false, + }); + } + + let binding_json = match &continuation { + Continuation::Terminal(binding) => Some(serde_json::to_string(binding)?), + _ => None, + }; + let mut payloads = Vec::with_capacity(refs.len()); + let mut existing_ids = Vec::with_capacity(refs.len()); + let mut already_recorded = true; + for work_ref in &refs { + let response = groups + .iter() + .find(|group| group.issuer == work_ref.issuer) + .and_then(|group| group.response.clone()); + let data = OutcomeData::new(outcome, continuation.clone(), response); + let payload = serde_json::to_string(&data)?; + let row = exact_work_tx(&mut transaction, work_ref).await?; + let old: Option = row.try_get("outcome_json")?; + let old_id: Option = row.try_get("outcome_id")?; + if let Some(old) = old { + if old != payload { + return Err(MailboxError::OutcomeAlreadyRecorded); + } + existing_ids.push(old_id); + } else { + already_recorded = false; + existing_ids.push(None); + } + payloads.push(payload); + } + + for group in &mut groups { + let group_has_existing = refs.iter().enumerate().any(|(index, work_ref)| { + work_ref.issuer == group.issuer && existing_ids[index].is_some() + }); + let distinct_existing = refs + .iter() + .enumerate() + .filter(|(_, work_ref)| work_ref.issuer == group.issuer) + .filter_map(|(index, _)| existing_ids[index].as_deref()) + .collect::>(); + if distinct_existing.len() > 1 { + return Err(MailboxError::ReferenceMismatch); + } + group.output_id = distinct_existing.iter().next().map(|id| (*id).to_string()); + if group.output_id.is_none() { + group.output_id = insert_response_group( + &mut transaction, + group, + format!("turn-outcome:{}:{}:{}", recipient, source_turn_id, group.issuer), + outcome, + ) + .await?; + if group.output_id.is_none() { + group.output_id = Some(new_mailbox_id()); + } + } else if group.response.is_some() && group_has_existing { + // Existing work outcome data owns the ID; its response message was + // inserted in the same transaction as the original outcome. + } + ensure_mailbox_id( + group + .output_id + .as_deref() + .ok_or(MailboxError::ReferenceMismatch)?, + )?; + } + + for (index, work_ref) in refs.iter().enumerate() { + let output_id = groups + .iter() + .find(|group| group.issuer == work_ref.issuer) + .and_then(|group| group.output_id.as_deref()) + .ok_or(MailboxError::ReferenceMismatch)?; + if existing_ids[index].is_none() { + save_outcome_tx( + &mut transaction, + work_ref, + &payloads[index], + output_id, + outcome, + ) + .await?; + } + } + if let Some(binding_json) = binding_json.as_deref() { + for work_ref in &refs { + sqlx::query( + "UPDATE mailbox_continuations SET state = 'terminal' + WHERE work_id = ? AND binding_json = ? + AND state IN ('pending_goal_binding', 'deferred')", + ) + .bind(&work_ref.id) + .bind(binding_json) + .execute(&mut *transaction) + .await?; + } + } + transaction.commit().await?; + if !already_recorded || binding_json.is_some() { + self.notify_changed(); + } + let mut receipts = Vec::with_capacity(refs.len()); + for work_ref in refs { + let group_id = groups + .iter() + .find(|group| group.issuer == work_ref.issuer) + .and_then(|group| group.output_id.clone()); + let response_id = groups + .iter() + .find(|group| group.issuer == work_ref.issuer) + .and_then(|group| group.response.as_ref().map(|_| group.output_id.clone())) + .flatten(); + receipts.push( + outcome_receipt_for(&self.pool, work_ref, group_id, response_id).await?, + ); + } + Ok(TurnOutcomeReceipt { + outcomes: receipts, + continuation_work: Vec::new(), + no_matching_work: false, + already_recorded, + }) + } + + /// Record one exact terminal result. Pending/deferred work must use a turn + /// admission so its continuation link is associated with the accepted turn. + pub async fn record_outcome( + &self, + work_ref: &WorkRef, + outcome: Outcome, + response: Option, + continuation: Continuation, + ) -> MailboxResult { + if is_nonterminal(&continuation) { + return Err(MailboxError::InvalidInput( + "pending continuation requires admitted turn linkage".into(), + )); + } + if let Continuation::Terminal(binding) = &continuation { + validate_binding_for_work(binding, work_ref)?; + } + let mut transaction = self.pool.begin().await?; + let mut groups = response_groups(&mut transaction, &[work_ref.clone()], response.as_ref()).await?; + let mut group = groups.pop().ok_or(MailboxError::ReferenceMismatch)?; + validate_auto_response_size(std::slice::from_ref(&group))?; + let data = OutcomeData::new(outcome, continuation.clone(), group.response.clone()); + let payload = serde_json::to_string(&data)?; + let row = exact_work_tx(&mut transaction, work_ref).await?; + if let Some(old) = row.try_get::, _>("outcome_json")? { + if old != payload { + return Err(MailboxError::OutcomeAlreadyRecorded); + } + let id: String = row.try_get("outcome_id")?; + transaction.commit().await?; + return outcome_receipt_for(&self.pool, work_ref.clone(), Some(id), response.is_some()).await; + } + if let Continuation::Terminal(binding) = &continuation { + let binding_json = serde_json::to_string(binding)?; + let unresolved: Option = sqlx::query_scalar( + "SELECT binding_json FROM mailbox_continuations + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') LIMIT 1", + ) + .bind(&work_ref.id) + .fetch_optional(&mut *transaction) + .await?; + if unresolved.as_deref().is_some_and(|stored| stored != binding_json) { + return Err(MailboxError::ReferenceMismatch); + } + } + group.output_id = insert_response_group( + &mut transaction, + &group, + format!("work-outcome:{}", work_ref.id), + outcome, + ) + .await?; + let output_id = group.output_id.as_deref().unwrap_or_else(|| new_mailbox_id()); + ensure_mailbox_id(output_id)?; + save_outcome_tx(&mut transaction, work_ref, &payload, output_id, outcome).await?; + if let Continuation::Terminal(binding) = &continuation { + sqlx::query( + "UPDATE mailbox_continuations SET state = 'terminal' + WHERE work_id = ? AND binding_json = ? + AND state IN ('pending_goal_binding', 'deferred')", + ) + .bind(&work_ref.id) + .bind(serde_json::to_string(binding)?) + .execute(&mut *transaction) + .await?; + } + transaction.commit().await?; + self.notify_changed(); + outcome_receipt_for(&self.pool, work_ref.clone(), Some(output_id.to_string()), response.is_some()).await + } +} + +async fn response_groups( + transaction: &mut Transaction<'_, Sqlite>, + refs: &[WorkRef], + response: Option<&InterAgentCommunication>, +) -> MailboxResult> { + let mut groups: Vec = Vec::new(); + for work_ref in refs { + let row = exact_work_tx(transaction, work_ref).await?; + let input: InterAgentCommunication = + serde_json::from_str(&row.try_get::("input_json")?)?; + let group = groups + .iter_mut() + .find(|group| group.issuer == work_ref.issuer); + if let Some(group) = group { + if group.agent != work_ref.agent + || group.author != input.recipient + || group.recipient != input.author + { + return Err(MailboxError::InvalidInput( + "issuer has inconsistent accepted communication paths".into(), + )); + } + group.refs.push(work_ref.clone()); + } else { + let mut response = response.cloned(); + if let Some(response) = response.as_mut() { + response.author = input.recipient.clone(); + response.recipient = input.author.clone(); + response.id = None; + } + groups.push(ResponseGroup { + issuer: work_ref.issuer, + agent: work_ref.agent, + author: input.recipient, + recipient: input.author, + refs: vec![work_ref.clone()], + response, + output_id: None, + }); + } + } + Ok(groups) +} + +fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxResult<()> { + let mut total = 0_usize; + for group in groups { + if let Some(response) = group.response.as_ref() { + let bytes = response.content.len() + response.encrypted_content.as_ref().map_or(0, String::len); + if bytes > MAX_RESPONSE_BYTES { + return Err(MailboxError::Backpressure("response message byte limit")); + } + total = total.saturating_add(bytes); + } + } + if total > MAX_AUTO_RESPONSE_BYTES { + return Err(MailboxError::Backpressure("automatic response aggregate byte limit")); + } + Ok(()) +} + +async fn insert_response_group( + transaction: &mut Transaction<'_, Sqlite>, + group: &mut ResponseGroup, + invocation_key: String, + outcome: Outcome, +) -> MailboxResult> { + let Some(mut communication) = group.response.clone() else { + return Ok(None); + }; + communication.author = group.author.clone(); + communication.recipient = group.recipient.clone(); + communication.id = None; + let envelope = insert_message_tx( + transaction, + MessageDraft { + invocation_key, + sender: group.agent, + recipient: group.issuer, + communication, + kind: MessageKind::Note, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention: outcome == Outcome::NeedsAttention, + }, + None, + ) + .await?; + group.output_id = Some(envelope.id.clone()); + Ok(group.output_id.clone()) +} + +async fn persist_continuation( + transaction: &mut Transaction<'_, Sqlite>, + work_ref: &WorkRef, + binding_json: &str, + state: &str, + outcome: Outcome, + source_turn_id: &str, + response_message_id: Option<&str>, +) -> MailboxResult<()> { + let exists: Option = sqlx::query_scalar( + "SELECT 1 FROM mailbox_continuations WHERE work_id = ? AND binding_json = ? LIMIT 1", + ) + .bind(&work_ref.id) + .bind(binding_json) + .fetch_optional(&mut **transaction) + .await?; + if exists.is_none() { + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM mailbox_continuations WHERE work_id = ?", + ) + .bind(&work_ref.id) + .fetch_one(&mut **transaction) + .await?; + if count >= MAX_CONTINUATIONS_PER_WORK { + return Err(MailboxError::Backpressure("work continuation count")); + } + } + sqlx::query( + "INSERT INTO mailbox_continuations( + work_id, binding_json, state, outcome, source_turn_id, response_message_id + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(work_id, binding_json) DO UPDATE SET + state = CASE + WHEN mailbox_continuations.state = 'terminal' THEN 'terminal' + WHEN excluded.state = 'deferred' THEN 'deferred' + ELSE mailbox_continuations.state + END, + outcome = excluded.outcome, + response_message_id = COALESCE(excluded.response_message_id, + mailbox_continuations.response_message_id)", + ) + .bind(&work_ref.id) + .bind(binding_json) + .bind(state) + .bind(outcome_label(outcome)) + .bind(source_turn_id) + .bind(response_message_id) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +pub(crate) async fn save_outcome_tx( + transaction: &mut Transaction<'_, Sqlite>, + work_ref: &WorkRef, + payload: &str, + outcome_id: &str, + outcome: Outcome, +) -> MailboxResult<()> { + let result = sqlx::query( + "UPDATE mailbox_work SET outcome_json = ?, outcome_id = ? + WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ? + AND outcome_json IS NULL", + ) + .bind(payload) + .bind(outcome_id) + .bind(&work_ref.id) + .bind(work_ref.issuer.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .execute(&mut **transaction) + .await?; + if result.rows_affected() == 0 { + return Err(MailboxError::ReferenceMismatch); + } + if outcome == Outcome::NeedsAttention { + sqlx::query( + "INSERT INTO mailbox_attention(id, recipient) VALUES (?, ?) + ON CONFLICT(id) DO NOTHING", + ) + .bind(format!("attention:{}", work_ref.id)) + .bind(work_ref.issuer.to_string()) + .execute(&mut **transaction) + .await?; + } + Ok(()) +} + +pub(crate) async fn outcome_receipt_for( + pool: &SqlitePool, + work_ref: WorkRef, + output_id: Option, + has_response: bool, +) -> MailboxResult { + let row = exact_work(pool, &work_ref).await?; + let stored: String = row + .try_get::, _>("outcome_json")? + .ok_or(MailboxError::ReferenceMismatch)?; + let mut data: OutcomeData = serde_json::from_str(&stored)?; + let response_message_id = if has_response { + output_id.or_else(|| row.try_get::, _>("outcome_id").ok().flatten()) + } else { + None + }; + if let (Some(response), Some(id)) = (data.response.as_mut(), response_message_id.as_ref()) { + response.id = Some(codex_protocol::ResponseItemId::from_server(id.clone())); + } + Ok(OutcomeReceipt { + work_ref, + id: row.try_get("outcome_id")?, + outcome: data.outcome, + continuation: data.continuation, + response: data.response, + response_message_id, + canonical: row.try_get::("outcome_canonical")? != 0, + observed: row.try_get::("outcome_observed")? != 0, + }) +} + +pub(crate) async fn stored_outcome_receipt_for( + pool: &SqlitePool, + work_ref: WorkRef, +) -> MailboxResult { + let row = exact_work(pool, &work_ref).await?; + let has_response = serde_json::from_str::( + &row.try_get::("outcome_json")?, + )? + .response + .is_some(); + outcome_receipt_for(pool, work_ref, None, has_response).await +} + +pub(crate) async fn continuation_receipts_for( + pool: &SqlitePool, + work_ref: &WorkRef, +) -> MailboxResult> { + let rows = sqlx::query( + "SELECT binding_json, state, outcome, source_turn_id, response_message_id + FROM mailbox_continuations + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') + ORDER BY source_turn_id LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(pool) + .await?; + if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation view limit")); + } + rows.iter() + .map(|row| { + let state = match row.try_get::("state")?.as_str() { + "pending_goal_binding" => ContinuationState::PendingGoalBinding, + "deferred" => ContinuationState::Deferred, + _ => return Err(MailboxError::InvalidInput("stored continuation state".into())), + }; + Ok(ContinuationReceipt { + binding: serde_json::from_str(&row.try_get::("binding_json")?)?, + state, + outcome: parse_outcome(row.try_get::("outcome")?.as_str())?, + source_turn_id: row.try_get("source_turn_id")?, + response_message_id: row.try_get("response_message_id")?, + }) + }) + .collect() +} + +fn is_nonterminal(continuation: &Continuation) -> bool { + matches!( + continuation, + Continuation::PendingGoalBinding(_) | Continuation::Deferred(_) + ) +} + +fn continuation_binding(continuation: &Continuation) -> Option<&GoalBinding> { + match continuation { + Continuation::None => None, + Continuation::PendingGoalBinding(binding) + | Continuation::Deferred(binding) + | Continuation::Terminal(binding) => Some(binding), + } +} + +fn continuation_state(continuation: &Continuation) -> &'static str { + match continuation { + Continuation::PendingGoalBinding(_) => "pending_goal_binding", + Continuation::Deferred(_) => "deferred", + _ => "terminal", + } +} + +fn outcome_label(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Completed => "completed", + Outcome::Failed => "failed", + Outcome::Cancelled => "cancelled", + Outcome::NeedsAttention => "needs_attention", + } +} + +fn parse_outcome(value: &str) -> MailboxResult { + match value { + "completed" => Ok(Outcome::Completed), + "failed" => Ok(Outcome::Failed), + "cancelled" => Ok(Outcome::Cancelled), + "needs_attention" => Ok(Outcome::NeedsAttention), + _ => Err(MailboxError::InvalidInput("stored continuation outcome".into())), + } +} + +fn validate_binding_for_agent(binding: &GoalBinding, agent: ThreadId) -> MailboxResult<()> { + if binding.child_thread_id != agent { + return Err(MailboxError::WrongRecipient); + } + validate_key(&binding.source_turn_id) +} + +fn validate_binding_for_work(binding: &GoalBinding, work_ref: &WorkRef) -> MailboxResult<()> { + if binding.parent_thread_id != work_ref.issuer || binding.child_thread_id != work_ref.agent { + return Err(MailboxError::WrongRecipient); + } + validate_key(&binding.goal_id)?; + validate_key(&binding.source_turn_id) +} + +fn validate_key(key: &str) -> MailboxResult<()> { + if key.is_empty() || key.len() > MAX_KEY_BYTES { + Err(MailboxError::InvalidInput("invalid native turn or binding key".into())) + } else { + Ok(()) + } +} diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs new file mode 100644 index 000000000000..d4504da38bc4 --- /dev/null +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -0,0 +1,380 @@ +use crate::error::MailboxResult; +use sqlx::{Row, SqlitePool}; + +const SCHEMA_VERSION: i64 = 5; + +pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { + let mut transaction = pool.begin().await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_schema ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + version INTEGER NOT NULL + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_counts ( + name TEXT PRIMARY KEY, + item_count INTEGER NOT NULL + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "INSERT INTO mailbox_counts(name, item_count) VALUES ('work', 0), ('message', 0) + ON CONFLICT(name) DO NOTHING", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_pending_counts ( + recipient TEXT PRIMARY KEY, + item_count INTEGER NOT NULL, + quiet_count INTEGER NOT NULL DEFAULT 0 + )", + ) + .execute(&mut *transaction) + .await?; + + let version: Option = sqlx::query_scalar( + "SELECT version FROM mailbox_schema WHERE singleton = 1", + ) + .fetch_optional(&mut *transaction) + .await?; + if let Some(version) = version { + if version > SCHEMA_VERSION { + return Err(crate::MailboxError::NewerSchema { + found: version, + supported: SCHEMA_VERSION, + }); + } + } + + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_work ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + issuer TEXT NOT NULL, + agent TEXT NOT NULL, + submission_id TEXT NOT NULL, + native_key TEXT NOT NULL, + request_json TEXT NOT NULL, + input_json TEXT NOT NULL, + input_message_id TEXT NOT NULL, + dispatch_state TEXT NOT NULL, + outcome_json TEXT, + outcome_id TEXT, + outcome_canonical INTEGER NOT NULL DEFAULT 0, + outcome_observed INTEGER NOT NULL DEFAULT 0 + )", + ) + .execute(&mut *transaction) + .await?; + + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS mailbox_work_submission_unique + ON mailbox_work(agent, submission_id)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS mailbox_work_input_id + ON mailbox_work(input_message_id)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS mailbox_work_native_key + ON mailbox_work(issuer, native_key)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_work_submission + ON mailbox_work(agent, submission_id, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_work_agent_input + ON mailbox_work(agent, input_message_id, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_work_issuer_agent + ON mailbox_work(issuer, agent, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_admissions ( + recipient TEXT NOT NULL, + source_turn_id TEXT NOT NULL, + work_id TEXT NOT NULL REFERENCES mailbox_work(id), + input_message_id TEXT NOT NULL, + PRIMARY KEY (recipient, source_turn_id, input_message_id), + UNIQUE (work_id) + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_input_admissions ( + recipient TEXT NOT NULL, + source_turn_id TEXT NOT NULL, + input_message_id TEXT NOT NULL, + work_id TEXT REFERENCES mailbox_work(id), + PRIMARY KEY (recipient, input_message_id), + UNIQUE (work_id) + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "INSERT OR IGNORE INTO mailbox_input_admissions( + recipient, source_turn_id, input_message_id, work_id + ) SELECT recipient, source_turn_id, input_message_id, work_id + FROM mailbox_admissions", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_input_admissions_turn + ON mailbox_input_admissions(recipient, source_turn_id, input_message_id)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_continuations ( + work_id TEXT NOT NULL REFERENCES mailbox_work(id), + binding_json TEXT NOT NULL, + state TEXT NOT NULL, + outcome TEXT NOT NULL DEFAULT 'needs_attention', + source_turn_id TEXT NOT NULL, + response_message_id TEXT, + PRIMARY KEY (work_id, binding_json) + )", + ) + .execute(&mut *transaction) + .await?; + ensure_column( + &mut transaction, + "mailbox_continuations", + "outcome", + "TEXT NOT NULL DEFAULT 'needs_attention'", + ) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_continuations_binding + ON mailbox_continuations(binding_json, state, work_id)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_admissions_turn + ON mailbox_admissions(recipient, source_turn_id, work_id)", + ) + .execute(&mut *transaction) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_messages ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + invocation_key TEXT NOT NULL, + request_json TEXT NOT NULL, + sender TEXT NOT NULL, + recipient TEXT NOT NULL, + author_path TEXT NOT NULL DEFAULT '', + recipient_path TEXT NOT NULL DEFAULT '', + communication_json TEXT NOT NULL, + kind TEXT NOT NULL, + work_ref_json TEXT, + reply_to TEXT, + supersedes TEXT, + superseded_by TEXT, + withdrawn INTEGER NOT NULL DEFAULT 0, + activate_if_idle INTEGER NOT NULL DEFAULT 0, + input_admitted INTEGER NOT NULL DEFAULT 0, + legacy_attention INTEGER NOT NULL DEFAULT 0, + canonical INTEGER NOT NULL DEFAULT 0, + observed INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (reply_to) REFERENCES mailbox_messages(id), + FOREIGN KEY (supersedes) REFERENCES mailbox_messages(id), + FOREIGN KEY (superseded_by) REFERENCES mailbox_messages(id) + )", + ) + .execute(&mut *transaction) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "withdrawn", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "activate_if_idle", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "input_admitted", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "author_path", + "TEXT NOT NULL DEFAULT ''", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "recipient_path", + "TEXT NOT NULL DEFAULT ''", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_pending_counts", + "quiet_count", + "INTEGER NOT NULL DEFAULT 0", + ) + .await?; + if version.is_some_and(|version| version < SCHEMA_VERSION) { + sqlx::query( + "UPDATE mailbox_pending_counts SET + item_count = ( + SELECT COUNT(*) FROM mailbox_messages AS message + WHERE message.recipient = mailbox_pending_counts.recipient + AND (message.canonical = 0 OR message.input_admitted = 0) + AND message.superseded_by IS NULL + AND message.withdrawn = 0 + ), + quiet_count = ( + SELECT COUNT(*) FROM mailbox_messages AS message + WHERE message.recipient = mailbox_pending_counts.recipient + AND (message.canonical = 0 OR message.input_admitted = 0) + AND message.superseded_by IS NULL + AND message.withdrawn = 0 AND message.legacy_attention = 0 + )", + ) + .execute(&mut *transaction) + .await?; + } + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_invocations ( + invocation_key TEXT PRIMARY KEY, + request_json TEXT NOT NULL, + message_id TEXT NOT NULL REFERENCES mailbox_messages(id) + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_pending + ON mailbox_messages(recipient, canonical, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_unadmitted + ON mailbox_messages(recipient, canonical, input_admitted, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_work + ON mailbox_messages(recipient, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_reply_lookup + ON mailbox_messages(reply_to, recipient, seq)", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS mailbox_messages_reply_once + ON mailbox_messages(reply_to) WHERE reply_to IS NOT NULL", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_attention ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + recipient TEXT NOT NULL, + canonical INTEGER NOT NULL DEFAULT 0, + observed INTEGER NOT NULL DEFAULT 0 + )", + ) + .execute(&mut *transaction) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS mailbox_activations ( + message_id TEXT PRIMARY KEY REFERENCES mailbox_messages(id), + sender TEXT NOT NULL, + state TEXT NOT NULL, + disposition TEXT + )", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "INSERT OR IGNORE INTO mailbox_activations(message_id, sender, state, disposition) + SELECT id, sender, 'decided', 'not_requested' FROM mailbox_messages + WHERE activate_if_idle = 0", + ) + .execute(&mut *transaction) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_attention_pending + ON mailbox_attention(recipient, canonical, observed, seq)", + ) + .execute(&mut *transaction) + .await?; + + sqlx::query( + "INSERT INTO mailbox_schema(singleton, version) VALUES (1, ?) + ON CONFLICT(singleton) DO UPDATE SET version = excluded.version", + ) + .bind(SCHEMA_VERSION) + .execute(&mut *transaction) + .await?; + transaction.commit().await?; + Ok(()) +} + +async fn ensure_column( + transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table: &str, + column: &str, + declaration: &str, +) -> MailboxResult<()> { + let pragma = format!("PRAGMA table_info({table})"); + let rows = sqlx::query(&pragma).fetch_all(&mut **transaction).await?; + let exists = rows + .iter() + .map(|row| row.try_get::("name")) + .collect::, _>>()? + .iter() + .any(|name| name == column); + if !exists { + let alter = format!("ALTER TABLE {table} ADD COLUMN {column} {declaration}"); + sqlx::query(&alter).execute(&mut **transaction).await?; + } + Ok(()) +} + diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs new file mode 100644 index 000000000000..f8d8661edcb2 --- /dev/null +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -0,0 +1,361 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::message_delivery::envelope_from_row; +use crate::store::Mailbox; +use crate::types::{ + Attention, Dependency, DependencyMode, ObservationTarget, ReturnWhen, Selection, + RequestDisposition, RequestSnapshot, Selector, Snapshot, SnapshotItem, SnapshotState, + WorkSnapshot, +}; +use crate::outcome::{continuation_receipts_for, stored_outcome_receipt_for}; +use crate::work::{parse_dispatch_state, work_ref_from_row}; +use codex_protocol::ThreadId; +use sqlx::Row; +use std::collections::{HashMap, HashSet}; + +const MAX_SELECTIONS: usize = 100; +const MAX_OBSERVATIONS: usize = 100; +const MAX_PAGE_SIZE: usize = 100; + +impl Mailbox { + /// Resolve caller-issued work and exact reply IDs without consuming them. + /// Completed work remains visible until its exact outcome is acknowledged. + pub async fn snapshot(&self, selection: &Selection) -> MailboxResult { + if selection.selectors.is_empty() || selection.selectors.len() > MAX_SELECTIONS { + return Err(MailboxError::InvalidInput( + "snapshot selector count must be between 1 and 100".into(), + )); + } + if selection.limit == 0 || selection.limit > MAX_PAGE_SIZE { + return Err(MailboxError::InvalidInput( + "snapshot limit must be between 1 and 100".into(), + )); + } + let mut items = Vec::new(); + let mut matched = 0; + let mut satisfied = 0; + let mut unique = HashSet::new(); + for selector in &selection.selectors { + let key = match selector { + Selector::Work(id) => format!("work:{id}"), + Selector::Reply(id) => format!("reply:{id}"), + }; + if !unique.insert(key) { + continue; + } + match selector { + Selector::Work(id) => { + let row = sqlx::query( + "SELECT * FROM mailbox_work + WHERE id = ? AND issuer = ? AND (? IS NULL OR agent = ?) LIMIT 1", + ) + .bind(id) + .bind(selection.caller.to_string()) + .bind(selection.agent.map(|agent| agent.to_string())) + .bind(selection.agent.map(|agent| agent.to_string())) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { + continue; + }; + matched += 1; + let work_ref = work_ref_from_row(&row)?; + let observed = row.try_get::("outcome_observed")? != 0; + if observed { + satisfied += 1; + } + let input: String = row.try_get("input_json")?; + let outcome = if row.try_get::, _>("outcome_json")?.is_some() { + Some(stored_outcome_receipt_for(&self.pool, work_ref.clone()).await?) + } else { + None + }; + let continuations = continuation_receipts_for(&self.pool, &work_ref).await?; + if items.len() < selection.limit { + items.push(SnapshotItem::Work(WorkSnapshot { + work_ref, + input: serde_json::from_str(&input)?, + dispatch_state: parse_dispatch_state( + row.try_get::("dispatch_state")?.as_str(), + )?, + outcome, + continuations, + observed, + })); + } + } + Selector::Reply(id) => { + let row = sqlx::query( + "SELECT * FROM mailbox_messages + WHERE id = ? AND sender = ? AND kind = '\"request\"' LIMIT 1", + ) + .bind(id) + .bind(selection.caller.to_string()) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { + continue; + }; + matched += 1; + let reply_row = sqlx::query( + "SELECT * FROM mailbox_messages + WHERE reply_to = ? AND recipient = ? ORDER BY seq LIMIT 1", + ) + .bind(id) + .bind(selection.caller.to_string()) + .fetch_optional(&self.pool) + .await?; + let reply = reply_row.as_ref().map(envelope_from_row).transpose()?; + let replacement: Option = row.try_get("superseded_by")?; + let withdrawn = row.try_get::("withdrawn")? != 0; + let disposition = if withdrawn { + RequestDisposition::Withdrawn + } else if replacement.is_some() { + RequestDisposition::Superseded + } else if reply.is_some() { + RequestDisposition::Replied + } else { + RequestDisposition::Open + }; + let observed = reply_row + .as_ref() + .map(|reply| reply.try_get::("observed").unwrap_or(0) != 0) + .unwrap_or(false); + if disposition != RequestDisposition::Open { + satisfied += 1; + } + if items.len() < selection.limit { + items.push(SnapshotItem::Request(RequestSnapshot { + request_id: id.clone(), + request: envelope_from_row(&row)?, + reply, + disposition, + replacement, + observed, + })); + } + } + } + } + let state = if matched == 0 { + SnapshotState::NoMatchingWork + } else if match selection.return_when { + ReturnWhen::Any => satisfied > 0, + ReturnWhen::All => satisfied == unique.len() && matched == unique.len(), + } { + SnapshotState::AlreadySatisfied + } else { + SnapshotState::Pending + }; + Ok(Snapshot { state, items }) + } + + /// Acknowledge exact canonical outcomes, replies, or one-shot attention. + /// Work acknowledgements are issuer-bound; replies/attention are recipient-bound. + pub async fn acknowledge_observed( + &self, + recipient: ThreadId, + targets: &[ObservationTarget], + ) -> MailboxResult<()> { + if targets.is_empty() || targets.len() > MAX_OBSERVATIONS { + return Err(MailboxError::InvalidInput( + "observation count must be between 1 and 100".into(), + )); + } + let mut transaction = self.pool.begin().await?; + for target in targets { + match target { + ObservationTarget::Work(work_ref) => { + if work_ref.issuer != recipient { + return Err(MailboxError::WrongRecipient); + } + let row = sqlx::query( + "SELECT outcome_canonical, outcome_json FROM mailbox_work + WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ?", + ) + .bind(&work_ref.id) + .bind(recipient.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if row.try_get::, _>("outcome_json")?.is_none() { + return Err(MailboxError::ReferenceMismatch); + } + if row.try_get::("outcome_canonical")? == 0 { + return Err(MailboxError::NotCanonical); + } + sqlx::query( + "UPDATE mailbox_work SET outcome_observed = 1 + WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ?", + ) + .bind(&work_ref.id) + .bind(recipient.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .execute(&mut *transaction) + .await?; + } + ObservationTarget::Reply(id) => { + let result = sqlx::query( + "UPDATE mailbox_messages SET observed = 1 + WHERE id = ? AND recipient = ? AND reply_to IS NOT NULL AND canonical = 1", + ) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + ensure_observation_target( + &mut transaction, + result.rows_affected(), + id, + recipient.to_string().as_str(), + ) + .await?; + } + ObservationTarget::Attention(id) => { + acknowledge_attention(&mut transaction, id, recipient.to_string().as_str()).await?; + } + } + } + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } + + /// Non-destructive bounded view of separate one-shot attention obligations. + pub async fn pending_attention( + &self, + recipient: ThreadId, + after_seq: Option, + limit: usize, + ) -> MailboxResult> { + if limit == 0 || limit > MAX_PAGE_SIZE { + return Err(MailboxError::InvalidInput( + "attention limit must be between 1 and 100".into(), + )); + } + let rows = sqlx::query( + "SELECT attention.* FROM mailbox_attention AS attention + LEFT JOIN mailbox_messages AS message + ON attention.id = 'attention:' || message.id + WHERE attention.recipient = ? AND attention.observed = 0 + AND (message.id IS NULL OR (message.superseded_by IS NULL AND message.withdrawn = 0)) + AND (? IS NULL OR attention.seq > ?) + ORDER BY attention.seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind(after_seq) + .bind(after_seq) + .bind(limit as i64) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|row| { + Ok(Attention { + id: row.try_get("id")?, + recipient: ThreadId::from_string(row.try_get::("recipient")?.as_str()) + .map_err(|_| MailboxError::InvalidInput("stored attention thread ID".into()))?, + canonical: row.try_get::("canonical")? != 0, + observed: row.try_get::("observed")? != 0, + }) + }) + .collect() + } + + /// First-page attention wake view; obligations remain until exact ack. + pub async fn attention( + &self, + recipient: ThreadId, + limit: usize, + ) -> MailboxResult> { + self.pending_attention(recipient, None, limit).await + } +} + +async fn acknowledge_attention( + transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + id: &str, + recipient: &str, +) -> MailboxResult<()> { + let result = sqlx::query( + "UPDATE mailbox_attention SET observed = 1 + WHERE id = ? AND recipient = ? AND canonical = 1", + ) + .bind(id) + .bind(recipient) + .execute(&mut **transaction) + .await?; + if result.rows_affected() == 0 { + let row = sqlx::query( + "SELECT canonical FROM mailbox_attention WHERE id = ? AND recipient = ?", + ) + .bind(id) + .bind(recipient) + .fetch_optional(&mut **transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if row.try_get::("canonical")? == 0 { + return Err(MailboxError::NotCanonical); + } + } + Ok(()) +} + +async fn ensure_observation_target( + transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + rows_affected: u64, + id: &str, + recipient: &str, +) -> MailboxResult<()> { + if rows_affected > 0 { + return Ok(()); + } + let row = sqlx::query( + "SELECT canonical, reply_to FROM mailbox_messages WHERE id = ? AND recipient = ?", + ) + .bind(id) + .bind(recipient) + .fetch_optional(&mut **transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if row.try_get::, _>("reply_to")?.is_none() { + return Err(MailboxError::ReferenceMismatch); + } + if row.try_get::("canonical")? == 0 { + return Err(MailboxError::NotCanonical); + } + Ok(()) +} + +/// Return true only when every choice at every reachable peer is trapped in a +/// closed waiting component. Unknown peers and open exits make the result false. +pub fn is_provably_closed_peer_cycle(start: &str, dependencies: &[Dependency]) -> bool { + let mut by_owner = HashMap::new(); + for dependency in dependencies { + if dependency.peers.is_empty() || by_owner.contains_key(&dependency.owner) { + return false; + } + by_owner.insert(dependency.owner.as_str(), dependency); + } + let mut closed: HashSet<&str> = by_owner.keys().copied().collect(); + loop { + let previous_len = closed.len(); + let previous = closed.clone(); + closed.retain(|owner| { + let Some(dependency) = by_owner.get(owner) else { + return false; + }; + match dependency.mode { + DependencyMode::All | DependencyMode::Any => dependency + .peers + .iter() + .all(|peer| previous.contains(peer.as_str())), + } + }); + if closed.len() == previous_len { + break; + } + } + closed.contains(start) +} diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs new file mode 100644 index 000000000000..df6a5a0e6191 --- /dev/null +++ b/codex-rs/agent-mailbox/src/store.rs @@ -0,0 +1,37 @@ +use crate::error::MailboxResult; +use crate::schema; +use sqlx::SqlitePool; +use tokio::sync::watch; + +#[derive(Clone)] +pub struct Mailbox { + pub(crate) pool: SqlitePool, + change_tx: watch::Sender, +} + +impl Mailbox { + /// Open the mailbox's independent `mailbox_1.sqlite` database. + pub async fn open(config: &codex_state::SqliteConfig) -> MailboxResult { + let path = config.home().join("mailbox_1.sqlite"); + let pool = config.open_read_write_pool(&path).await?; + crate::schema::initialize(&pool).await?; + let (change_tx, _) = watch::channel(0_u64); + Ok(Self { pool, change_tx }) + } + + /// Subscribe before querying a snapshot to avoid missing local mailbox writes. + /// A receiver wake is only a reason to rescan; it is not delivery proof. + pub fn subscribe(&self) -> watch::Receiver { + self.change_tx.subscribe() + } + + pub(crate) fn notify_changed(&self) { + self.change_tx.send_modify(|generation| { + *generation = generation.wrapping_add(1); + }); + } + + pub(crate) fn pool(&self) -> &SqlitePool { + &self.pool + } +} diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs new file mode 100644 index 000000000000..02b64ebcad27 --- /dev/null +++ b/codex-rs/agent-mailbox/src/types.rs @@ -0,0 +1,283 @@ +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::ThreadId; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkRef { + pub id: String, + pub issuer: ThreadId, + pub agent: ThreadId, + pub submission_id: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NativeWork { + pub native_key: String, + pub issuer: ThreadId, + pub agent: ThreadId, + pub submission_id: String, + pub input: InterAgentCommunication, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MessageKind { + Note, + Progress, + Request, + Alert, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActivationDisposition { + NotRequested, + AlreadyActive, + Activated, + Denied, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ActivationDecision { + Activate, + AlreadyDecided(ActivationDisposition), + RecoveryNeeded, +} + +impl Default for MessageKind { + fn default() -> Self { + Self::Note + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MessageDraft { + pub invocation_key: String, + pub sender: ThreadId, + pub recipient: ThreadId, + pub communication: InterAgentCommunication, + pub kind: MessageKind, + pub activate_if_idle: bool, + pub work_ref: Option, + pub reply_to: Option, + pub supersedes: Option, + pub legacy_attention: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Envelope { + pub id: String, + pub invocation_key: String, + pub sender: ThreadId, + pub recipient: ThreadId, + pub communication: InterAgentCommunication, + pub kind: MessageKind, + pub activate_if_idle: bool, + pub work_ref: Option, + pub reply_to: Option, + pub supersedes: Option, + pub legacy_attention: bool, + pub canonical: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + Completed, + Failed, + Cancelled, + NeedsAttention, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", content = "goal_id", rename_all = "snake_case")] +pub enum Continuation { + None, + PendingGoalBinding(GoalBinding), + Deferred(GoalBinding), + Terminal(GoalBinding), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GoalBinding { + pub goal_id: String, + pub parent_thread_id: ThreadId, + pub child_thread_id: ThreadId, + pub control_generation: u64, + pub revision: u64, + pub source_turn_id: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DispatchState { + Accepted, + Dispatching, + Dispatched, + RecoveryNeeded, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DispatchDecision { + Dispatch, + AlreadyDispatched, + RecoveryNeeded, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionReceipt { + pub work_refs: Vec, + pub already_recorded: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutcomeReceipt { + pub work_ref: WorkRef, + pub id: String, + pub outcome: Outcome, + pub continuation: Continuation, + pub response: Option, + pub response_message_id: Option, + pub canonical: bool, + pub observed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TurnOutcomeReceipt { + pub outcomes: Vec, + pub continuation_work: Vec, + pub no_matching_work: bool, + pub already_recorded: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "id", rename_all = "snake_case")] +pub enum ObservationTarget { + Work(WorkRef), + Reply(String), + Attention(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Attention { + pub id: String, + pub recipient: ThreadId, + pub canonical: bool, + pub observed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PendingPage { + pub items: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CanonicalConfirmation { + /// The adapter appended and flushed this stable ID to canonical history. + AppendedAndFlushed, + /// The adapter found this stable ID already present and verified the flush. + AlreadyPresentAndFlushed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReturnWhen { + Any, + All, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Selector { + Work(String), + Reply(String), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RequestDisposition { + Open, + Replied, + Superseded, + Withdrawn, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RequestSnapshot { + pub request_id: String, + pub request: Envelope, + pub reply: Option, + pub disposition: RequestDisposition, + pub replacement: Option, + pub observed: bool, +} + +#[derive(Clone, Debug)] +pub struct Selection { + pub caller: ThreadId, + pub agent: Option, + pub selectors: Vec, + pub return_when: ReturnWhen, + pub limit: usize, +} + +pub type SnapshotRequest = Selection; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SnapshotState { + Pending, + AlreadySatisfied, + NoMatchingWork, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SnapshotItem { + Work(WorkSnapshot), + Request(RequestSnapshot), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkSnapshot { + pub work_ref: WorkRef, + pub input: InterAgentCommunication, + pub dispatch_state: DispatchState, + pub outcome: Option, + pub continuations: Vec, + pub observed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContinuationState { + PendingGoalBinding, + Deferred, + Terminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContinuationReceipt { + pub binding: GoalBinding, + pub state: ContinuationState, + pub outcome: Outcome, + pub source_turn_id: String, + pub response_message_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Snapshot { + pub state: SnapshotState, + pub items: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Dependency { + pub owner: String, + pub peers: Vec, + pub mode: DependencyMode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DependencyMode { + Any, + All, +} diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs new file mode 100644 index 000000000000..f3ce61130d78 --- /dev/null +++ b/codex-rs/agent-mailbox/src/work.rs @@ -0,0 +1,483 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::messages::decrement_pending_count; +use crate::store::Mailbox; +use crate::types::{ + AdmissionReceipt, DispatchDecision, DispatchState, NativeWork, WorkRef, +}; +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::ThreadId; +use sqlx::{Row, Sqlite, Transaction}; + +const MAX_WORK_ITEMS: i64 = 50_000; +const MAX_INPUT_BYTES: usize = 1_000_000; +const MAX_NATIVE_BODY_BYTES: usize = 8_192; +const MAX_KEY_BYTES: usize = 256; +const MAX_TURN_INPUTS: usize = 100; + +impl Mailbox { + /// Capture accepted native work and its input before the adapter activates it. + /// Retries of the exact issuer-scoped native key return the same immutable ref. + pub async fn accept_work(&self, mut work: NativeWork) -> MailboxResult { + validate_key(&work.native_key, "native key")?; + validate_key(&work.submission_id, "submission id")?; + + let mut request = work.clone(); + request.input.id = None; + let body_bytes = work.input.content.len() + + work.input.encrypted_content.as_ref().map_or(0, String::len); + if body_bytes > MAX_NATIVE_BODY_BYTES { + return Err(MailboxError::Backpressure("accepted native body byte limit")); + } + let request_json = serde_json::to_string(&request)?; + if request_json.len() > MAX_INPUT_BYTES { + return Err(MailboxError::Backpressure("accepted input byte limit")); + } + if !work.input.id.as_ref().is_some_and(is_mailbox_id) { + work.input.id = Some(codex_protocol::ResponseItemId::from_server(new_mailbox_id())); + } + let input_message_id = work.input.id.as_ref().expect("assigned above").to_string(); + let input_json = serde_json::to_string(&work.input)?; + + let mut transaction = self.pool.begin().await?; + if let Some(row) = sqlx::query( + "SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ?", + ) + .bind(work.issuer.to_string()) + .bind(&work.native_key) + .fetch_optional(&mut *transaction) + .await? + { + if row.try_get::("request_json")? != request_json { + return Err(MailboxError::IdempotencyConflict); + } + let work_ref = work_ref_from_row(&row)?; + transaction.commit().await?; + return Ok(work_ref); + } + + let duplicate_submission = sqlx::query_scalar::<_, String>( + "SELECT id FROM mailbox_work WHERE agent = ? AND submission_id = ? LIMIT 1", + ) + .bind(work.agent.to_string()) + .bind(&work.submission_id) + .fetch_optional(&mut *transaction) + .await?; + if duplicate_submission.is_some() { + return Err(MailboxError::IdempotencyConflict); + } + let duplicate_input = sqlx::query_scalar::<_, String>( + "SELECT id FROM mailbox_work WHERE input_message_id = ? LIMIT 1", + ) + .bind(&input_message_id) + .fetch_optional(&mut *transaction) + .await?; + if duplicate_input.is_some() { + return Err(MailboxError::IdempotencyConflict); + } + let count: i64 = sqlx::query_scalar( + "SELECT item_count FROM mailbox_counts WHERE name = 'work'", + ) + .fetch_one(&mut *transaction) + .await?; + if count >= MAX_WORK_ITEMS { + return Err(MailboxError::Backpressure("retained work-item count")); + } + + let work_ref = WorkRef { + id: format!("work_{}", uuid::Uuid::now_v7()), + issuer: work.issuer, + agent: work.agent, + submission_id: work.submission_id, + }; + let inserted = sqlx::query( + "INSERT INTO mailbox_work ( + id, issuer, agent, submission_id, native_key, request_json, + input_json, input_message_id, dispatch_state + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'accepted')", + ) + .bind(&work_ref.id) + .bind(work_ref.issuer.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .bind(&work.native_key) + .bind(request_json) + .bind(input_json) + .bind(input_message_id) + .execute(&mut *transaction) + .await?; + if inserted.rows_affected() == 1 { + sqlx::query("UPDATE mailbox_counts SET item_count = item_count + 1 WHERE name = 'work'") + .execute(&mut *transaction) + .await?; + } + transaction.commit().await?; + self.notify_changed(); + Ok(work_ref) + } + + /// Return the exact captured native input, including its assigned stable ID. + pub async fn accepted_input(&self, work_ref: &WorkRef) -> MailboxResult { + let row = exact_work(&self.pool, work_ref).await?; + Ok(serde_json::from_str(&row.try_get::("input_json")?)?) + } + + pub async fn work_for_native_key( + &self, + issuer: ThreadId, + native_key: &str, + ) -> MailboxResult> { + let row = sqlx::query( + "SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ? LIMIT 1", + ) + .bind(issuer.to_string()) + .bind(native_key) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(work_ref_from_row).transpose() + } + + pub async fn work_for_submission( + &self, + agent: ThreadId, + submission_id: &str, + ) -> MailboxResult> { + let row = sqlx::query( + "SELECT * FROM mailbox_work WHERE agent = ? AND submission_id = ? LIMIT 1", + ) + .bind(agent.to_string()) + .bind(submission_id) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(work_ref_from_row).transpose() + } + + /// Resolve a work reference only when the supplied thread pair is exactly + /// its issuer/agent pair, in either direction. + pub async fn work_for_message_scope( + &self, + sender: ThreadId, + recipient: ThreadId, + id: &str, + ) -> MailboxResult> { + let row = sqlx::query("SELECT * FROM mailbox_work WHERE id = ? LIMIT 1") + .bind(id) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { + return Ok(None); + }; + let work_ref = work_ref_from_row(&row)?; + let pair_matches = (work_ref.issuer == sender && work_ref.agent == recipient) + || (work_ref.agent == sender && work_ref.issuer == recipient); + Ok(pair_matches.then_some(work_ref)) + } + + /// Freeze the caller's currently unsurfaced work list at wait entry. + /// Completed outcomes remain included until their exact observation ack. + pub async fn issued_work_refs( + &self, + issuer: ThreadId, + agent: Option, + limit: usize, + ) -> MailboxResult> { + if limit == 0 || limit > 100 { + return Err(MailboxError::InvalidInput( + "issued work limit must be between 1 and 100".into(), + )); + } + let rows = sqlx::query( + "SELECT * FROM mailbox_work + WHERE issuer = ? AND outcome_observed = 0 + AND (? IS NULL OR agent = ?) + ORDER BY seq LIMIT ?", + ) + .bind(issuer.to_string()) + .bind(agent.map(|agent| agent.to_string())) + .bind(agent.map(|agent| agent.to_string())) + .bind((limit + 1) as i64) + .fetch_all(&self.pool) + .await?; + if rows.len() > limit { + return Err(MailboxError::Backpressure("issued work selection limit")); + } + rows.iter().map(work_ref_from_row).collect() + } + + /// Atomically claim native activation. Only the first accepted caller may + /// send; dispatching work requires exact adapter recovery, never a retry send. + pub async fn begin_dispatch(&self, work_ref: &WorkRef) -> MailboxResult { + let mut transaction = self.pool.begin().await?; + let row = exact_work_tx(&mut transaction, work_ref).await?; + let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; + if state != DispatchState::Accepted { + transaction.commit().await?; + return Ok(match state { + DispatchState::Dispatched => DispatchDecision::AlreadyDispatched, + DispatchState::Dispatching | DispatchState::RecoveryNeeded => { + DispatchDecision::RecoveryNeeded + } + DispatchState::Accepted => unreachable!("handled above"), + }); + } + sqlx::query("UPDATE mailbox_work SET dispatch_state = 'dispatching' WHERE id = ?") + .bind(&work_ref.id) + .execute(&mut *transaction) + .await?; + transaction.commit().await?; + self.notify_changed(); + Ok(DispatchDecision::Dispatch) + } + + /// Record adapter-confirmed stale in-flight custody for this exact work. + pub async fn mark_recovery_needed(&self, work_ref: &WorkRef) -> MailboxResult<()> { + let mut transaction = self.pool.begin().await?; + let row = exact_work_tx(&mut transaction, work_ref).await?; + let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; + match state { + DispatchState::Dispatching => { + sqlx::query( + "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' WHERE id = ?", + ) + .bind(&work_ref.id) + .execute(&mut *transaction) + .await?; + } + DispatchState::RecoveryNeeded => {} + DispatchState::Accepted | DispatchState::Dispatched => { + return Err(MailboxError::ReferenceMismatch); + } + } + transaction.commit().await?; + self.notify_changed(); + Ok(()) + } + + pub async fn mark_dispatched(&self, work_ref: &WorkRef) -> MailboxResult<()> { + let result = sqlx::query( + "UPDATE mailbox_work SET dispatch_state = 'dispatched' + WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ? + AND dispatch_state = 'dispatching'", + ) + .bind(&work_ref.id) + .bind(work_ref.issuer.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + let row = exact_work(&self.pool, work_ref).await?; + if parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())? + != DispatchState::Dispatched + { + return Err(MailboxError::ReferenceMismatch); + } + } + self.notify_changed(); + Ok(()) + } + + pub async fn dispatch_state(&self, work_ref: &WorkRef) -> MailboxResult { + let row = exact_work(&self.pool, work_ref).await?; + parse_dispatch_state(row.try_get::("dispatch_state")?.as_str()) + } + + /// Link exactly the native input IDs admitted to this recipient turn. + pub async fn record_admission( + &self, + recipient: ThreadId, + source_turn_id: &str, + input_message_ids: &[String], + ) -> MailboxResult { + validate_key(source_turn_id, "source turn id")?; + if input_message_ids.len() > MAX_TURN_INPUTS { + return Err(MailboxError::Backpressure("turn input count")); + } + let mut ids = input_message_ids.to_vec(); + ids.sort_unstable(); + if ids.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(MailboxError::InvalidInput("duplicate input message ID".into())); + } + + let mut transaction = self.pool.begin().await?; + let mut work_refs = Vec::with_capacity(ids.len()); + let mut added = false; + for input_id in &ids { + let prior = sqlx::query( + "SELECT source_turn_id, work_id FROM mailbox_input_admissions + WHERE recipient = ? AND input_message_id = ? LIMIT 1", + ) + .bind(recipient.to_string()) + .bind(input_id) + .fetch_optional(&mut *transaction) + .await?; + if let Some(prior) = prior { + if prior.try_get::("source_turn_id")? != source_turn_id { + return Err(MailboxError::IdempotencyConflict); + } + if let Some(work_id) = prior.try_get::, _>("work_id")? { + let work = sqlx::query("SELECT * FROM mailbox_work WHERE id = ?") + .bind(work_id) + .fetch_one(&mut *transaction) + .await?; + work_refs.push(work_ref_from_row(&work)?); + } + continue; + } + + let native_work = sqlx::query( + "SELECT * FROM mailbox_work WHERE agent = ? AND input_message_id = ? LIMIT 1", + ) + .bind(recipient.to_string()) + .bind(input_id) + .fetch_optional(&mut *transaction) + .await?; + if let Some(row) = native_work { + let work_ref = work_ref_from_row(&row)?; + sqlx::query( + "INSERT INTO mailbox_input_admissions( + recipient, source_turn_id, input_message_id, work_id + ) VALUES (?, ?, ?, ?)", + ) + .bind(recipient.to_string()) + .bind(source_turn_id) + .bind(input_id) + .bind(&work_ref.id) + .execute(&mut *transaction) + .await?; + sqlx::query( + "INSERT INTO mailbox_admissions(recipient, source_turn_id, work_id, input_message_id) + VALUES (?, ?, ?, ?)", + ) + .bind(recipient.to_string()) + .bind(source_turn_id) + .bind(&work_ref.id) + .bind(input_id) + .execute(&mut *transaction) + .await?; + work_refs.push(work_ref); + added = true; + continue; + } + + let message = sqlx::query( + "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn + FROM mailbox_messages WHERE id = ? AND recipient = ? LIMIT 1", + ) + .bind(input_id) + .bind(recipient.to_string()) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; + if message.try_get::("input_admitted")? != 0 { + return Err(MailboxError::IdempotencyConflict); + } + let canonical = message.try_get::("canonical")? != 0; + let active = message.try_get::, _>("superseded_by")?.is_none() + && message.try_get::("withdrawn")? == 0; + let quiet = message.try_get::("legacy_attention")? == 0; + sqlx::query( + "UPDATE mailbox_messages SET input_admitted = 1 + WHERE id = ? AND recipient = ? AND input_admitted = 0", + ) + .bind(input_id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; + sqlx::query( + "INSERT INTO mailbox_input_admissions( + recipient, source_turn_id, input_message_id, work_id + ) VALUES (?, ?, ?, NULL)", + ) + .bind(recipient.to_string()) + .bind(source_turn_id) + .bind(input_id) + .execute(&mut *transaction) + .await?; + if canonical && active { + decrement_pending_count(&mut transaction, &recipient.to_string(), quiet).await?; + } + added = true; + } + transaction.commit().await?; + if added { + self.notify_changed(); + } + Ok(AdmissionReceipt { + work_refs, + already_recorded: !added, + }) + } +} + +pub(crate) async fn exact_work<'c, E>( + executor: E, + work_ref: &WorkRef, +) -> MailboxResult +where + E: sqlx::Executor<'c, Database = Sqlite>, +{ + sqlx::query( + "SELECT * FROM mailbox_work WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ?", + ) + .bind(&work_ref.id) + .bind(work_ref.issuer.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .fetch_optional(executor) + .await? + .ok_or(MailboxError::ReferenceMismatch) +} + +pub(crate) async fn exact_work_tx( + transaction: &mut Transaction<'_, Sqlite>, + work_ref: &WorkRef, +) -> MailboxResult { + exact_work(&mut **transaction, work_ref).await +} + +pub(crate) fn work_ref_from_row(row: &sqlx::sqlite::SqliteRow) -> MailboxResult { + Ok(WorkRef { + id: row.try_get("id")?, + issuer: parse_thread_id(row.try_get::("issuer")?.as_str())?, + agent: parse_thread_id(row.try_get::("agent")?.as_str())?, + submission_id: row.try_get("submission_id")?, + }) +} + +fn parse_thread_id(value: &str) -> MailboxResult { + ThreadId::from_string(value).map_err(|_| MailboxError::InvalidInput("stored thread ID".into())) +} + +pub(crate) fn parse_dispatch_state(value: &str) -> MailboxResult { + match value { + "accepted" => Ok(DispatchState::Accepted), + "dispatching" => Ok(DispatchState::Dispatching), + "dispatched" => Ok(DispatchState::Dispatched), + "recovery_needed" => Ok(DispatchState::RecoveryNeeded), + _ => Err(MailboxError::InvalidInput("stored dispatch state".into())), + } +} + +fn validate_key(value: &str, label: &'static str) -> MailboxResult<()> { + if value.is_empty() || value.len() > MAX_KEY_BYTES { + return Err(MailboxError::InvalidInput(format!("invalid {label}"))); + } + Ok(()) +} + +pub(crate) fn new_mailbox_id() -> String { + codex_protocol::ResponseItemId::with_suffix("amsg", uuid::Uuid::now_v7()).to_string() +} + +fn is_mailbox_id(id: &codex_protocol::ResponseItemId) -> bool { + id.as_str().starts_with("amsg_") && id.as_str().len() > 5 +} + +pub(crate) fn ensure_mailbox_id(id: &str) -> MailboxResult<()> { + if id.starts_with("amsg_") && id.len() > 5 { + Ok(()) + } else { + Err(MailboxError::InvalidInput("expected stable amsg_* ID".into())) + } +} + diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 5358866ae005..4bbebf815bf9 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -25,6 +25,7 @@ chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive"] } codex-analytics = { workspace = true } codex-agent-graph-store = { workspace = true } +codex-agent-mailbox = { workspace = true } codex-api = { workspace = true } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } From 7c2f9b30d821d1942618ac3c77f67f43e61100f1 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:15:08 +1100 Subject: [PATCH 02/54] Integrate opt-in native mailbox acceptance, settlement and wait reports --- .../workflows/mailbox-source-preparation.yml | 82 + codex-rs/agent-mailbox/BUILD.bazel | 6 + codex-rs/agent-mailbox/src/error.rs | 2 + codex-rs/agent-mailbox/src/lib.rs | 5 + codex-rs/agent-mailbox/src/mailbox_tests.rs | 1572 +++++------------ .../agent-mailbox/src/message_delivery.rs | 51 + codex-rs/agent-mailbox/src/message_rules.rs | 14 +- codex-rs/agent-mailbox/src/messages.rs | 31 +- codex-rs/agent-mailbox/src/outcome.rs | 348 +--- .../agent-mailbox/src/outcome_messages.rs | 113 ++ codex-rs/agent-mailbox/src/outcome_store.rs | 237 +++ codex-rs/agent-mailbox/src/runtime_fault.rs | 61 + codex-rs/agent-mailbox/src/schema.rs | 60 +- codex-rs/agent-mailbox/src/snapshot.rs | 26 +- codex-rs/agent-mailbox/src/store.rs | 11 +- codex-rs/agent-mailbox/src/types.rs | 2 +- codex-rs/agent-mailbox/src/wait_graph.rs | 133 ++ codex-rs/app-server-protocol/src/export.rs | 1 + .../protocol/collab_agent_lifecycle_tests.rs | 1 + .../src/protocol/event_mapping.rs | 110 ++ .../src/protocol/thread_history.rs | 90 + .../src/protocol/v2/item.rs | 6 + .../src/protocol/v2/tests.rs | 8 + codex-rs/core/src/agent/control.rs | 8 + codex-rs/core/src/agent/control/lifecycle.rs | 81 +- codex-rs/core/src/agent/control/mailbox.rs | 326 ++++ codex-rs/core/src/agent/control/spawn.rs | 30 + codex-rs/core/src/agent_communication.rs | 11 +- codex-rs/core/src/compact.rs | 2 +- codex-rs/core/src/compact_remote.rs | 2 +- codex-rs/core/src/compact_remote_v2.rs | 2 +- codex-rs/core/src/compact_token_budget.rs | 2 +- codex-rs/core/src/hook_runtime.rs | 6 +- codex-rs/core/src/session/mod.rs | 400 ++++- codex-rs/core/src/session/session.rs | 18 + codex-rs/core/src/session/tests.rs | 53 +- codex-rs/core/src/session/turn.rs | 45 +- codex-rs/core/src/tasks/regular.rs | 2 +- codex-rs/core/src/thread_manager.rs | 83 +- .../handlers/multi_agents/close_agent.rs | 3 + .../handlers/multi_agents/resume_agent.rs | 2 + .../tools/handlers/multi_agents/send_input.rs | 2 + .../src/tools/handlers/multi_agents/spawn.rs | 2 + .../src/tools/handlers/multi_agents/wait.rs | 3 + .../src/tools/handlers/multi_agents_spec.rs | 335 +++- .../tools/handlers/multi_agents_spec_tests.rs | 279 ++- .../src/tools/handlers/multi_agents_v2.rs | 5 + .../multi_agents_v2/mailbox_message.rs | 255 +++ .../handlers/multi_agents_v2/mailbox_wait.rs | 408 +++++ .../handlers/multi_agents_v2/message_tool.rs | 55 +- .../tools/handlers/multi_agents_v2/spawn.rs | 14 +- .../tools/handlers/multi_agents_v2/wait.rs | 2 + codex-rs/core/src/tools/spec_plan.rs | 40 +- codex-rs/core/src/tools/spec_plan_tests.rs | 262 +++ codex-rs/ext/goal/src/api.rs | 15 +- codex-rs/ext/goal/src/runtime.rs | 88 +- codex-rs/ext/goal/src/tool.rs | 10 +- codex-rs/features/src/lib.rs | 16 + codex-rs/protocol/src/items.rs | 93 + codex-rs/protocol/src/legacy_events.rs | 2 + codex-rs/protocol/src/protocol.rs | 14 + codex-rs/tui/src/app/tests.rs | 4 + codex-rs/tui/src/chatwidget.rs | 6 + codex-rs/tui/src/chatwidget/constructor.rs | 1 + codex-rs/tui/src/chatwidget/protocol.rs | 2 + codex-rs/tui/src/chatwidget/replay.rs | 2 + .../tui/src/chatwidget/tests/app_server.rs | 183 ++ .../src/chatwidget/tests/history_replay.rs | 74 + codex-rs/tui/src/chatwidget/tool_lifecycle.rs | 66 +- codex-rs/tui/src/multi_agents.rs | 18 +- codex-rs/tui/src/multi_agents/mailbox_wait.rs | 258 +++ docs/carry-divergence-ledger.md | 27 + docs/divergences/index.yaml | 22 +- docs/downstream-regression-matrix.md | 12 + docs/downstream-tool-surface-matrix.md | 13 + justfile | 1 + 76 files changed, 4900 insertions(+), 1735 deletions(-) create mode 100644 .github/workflows/mailbox-source-preparation.yml create mode 100644 codex-rs/agent-mailbox/BUILD.bazel create mode 100644 codex-rs/agent-mailbox/src/outcome_messages.rs create mode 100644 codex-rs/agent-mailbox/src/outcome_store.rs create mode 100644 codex-rs/agent-mailbox/src/runtime_fault.rs create mode 100644 codex-rs/agent-mailbox/src/wait_graph.rs create mode 100644 codex-rs/core/src/agent/control/mailbox.rs create mode 100644 codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs create mode 100644 codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs create mode 100644 codex-rs/tui/src/multi_agents/mailbox_wait.rs diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml new file mode 100644 index 000000000000..389096620876 --- /dev/null +++ b/.github/workflows/mailbox-source-preparation.yml @@ -0,0 +1,82 @@ +name: Mailbox focused hosted source preparation + +on: + push: + branches: [feat/quiet-durable-mailbox] + paths: [.github/workflows/mailbox-source-preparation.yml] + +permissions: + contents: read + +jobs: + prepare-and-compile: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + CARGO_INCREMENTAL: "0" + RUST_MIN_STACK: "8388608" + PREPARATION_ATTEMPT: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: ./.github/actions/setup-bazel-ci + with: + target: x86_64-unknown-linux-gnu + - uses: dtolnay/rust-toolchain@ebb3d1676050bfd0971c36c1e215b5751473994d + with: + toolchain: 1.96.0 + components: rustfmt, clippy + - uses: ./.github/actions/setup-rusty-v8 + with: + target: x86_64-unknown-linux-gnu + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + with: + node-version: 22 + - name: Install standard Linux build dependencies + run: | + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + - name: Prepare only this branch's changed Rust files and lock metadata + shell: bash + run: | + set -euo pipefail + while IFS= read -r source; do + rustfmt --edition 2024 --config skip_children=true "$source" + done < <(git diff --name-only 2d91cb7437de6e146c8ec4eb654755dac2fb5ddd HEAD -- '*.rs') + cd codex-rs + cargo metadata --no-deps --format-version 1 > /dev/null + - name: Refresh the Bazel dependency lock + run: just bazel-lock-update + - name: Run the owned library contracts + working-directory: codex-rs + run: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 + - name: Generate the affected native schemas + run: | + just write-config-schema + just write-app-server-schema + - name: Compile focused core mailbox and schema consumers + working-directory: codex-rs + run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 + - name: Compile focused UI mailbox consumers + working-directory: codex-rs + env: + INSTA_UPDATE: always + run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 + - name: Capture the exact generated source delta + if: always() + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/mailbox-source-preparation" + git diff --binary > "$RUNNER_TEMP/mailbox-source-preparation/source.patch" + git rev-parse HEAD > "$RUNNER_TEMP/mailbox-source-preparation/source-sha.txt" + git status --short > "$RUNNER_TEMP/mailbox-source-preparation/status.txt" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: mailbox-source-preparation-${{ github.sha }} + path: ${{ runner.temp }}/mailbox-source-preparation/ + retention-days: 3 diff --git a/codex-rs/agent-mailbox/BUILD.bazel b/codex-rs/agent-mailbox/BUILD.bazel new file mode 100644 index 000000000000..9900e3934013 --- /dev/null +++ b/codex-rs/agent-mailbox/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "agent-mailbox", + crate_name = "codex_agent_mailbox", +) diff --git a/codex-rs/agent-mailbox/src/error.rs b/codex-rs/agent-mailbox/src/error.rs index d2003f83ff3d..1d47d5376aad 100644 --- a/codex-rs/agent-mailbox/src/error.rs +++ b/codex-rs/agent-mailbox/src/error.rs @@ -30,4 +30,6 @@ pub enum MailboxError { NewerSchema { found: i64, supported: i64 }, #[error("mailbox cursor is malformed")] InvalidCursor, + #[error("wait depends on a provably closed peer cycle")] + ClosedPeerCycle, } diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index d6c2cff68750..31b9e2742e5e 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -10,15 +10,20 @@ mod messages; mod message_delivery; mod message_rules; mod outcome; +mod outcome_messages; +mod outcome_store; +mod runtime_fault; mod schema; mod snapshot; mod store; +mod wait_graph; mod work; mod types; pub use error::{MailboxError, MailboxResult}; pub use store::Mailbox; pub use snapshot::is_provably_closed_peer_cycle; +pub use wait_graph::WaitRegistration; pub use types::{ ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, CanonicalConfirmation, Continuation, ContinuationReceipt, ContinuationState, Dependency, diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 95cc7a71ad7f..340770e8a351 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1,9 +1,9 @@ use crate::{ - is_provably_closed_peer_cycle, ActivationDecision, ActivationDisposition, Dependency, - DependencyMode, DispatchDecision, DispatchState, Mailbox, MailboxError, MessageDraft, - MessageKind, NativeWork, ObservationTarget, RequestDisposition, RequestSnapshot, ReturnWhen, - Selection, Selector, Snapshot, SnapshotItem, SnapshotState, WorkSnapshot, Continuation, - ContinuationState, GoalBinding, Outcome, + is_provably_closed_peer_cycle, ActivationDecision, ActivationDisposition, AdmissionReceipt, + Continuation, ContinuationState, Dependency, DependencyMode, DispatchDecision, DispatchState, + Envelope, GoalBinding, Mailbox, MailboxError, MessageDraft, MessageKind, NativeWork, + ObservationTarget, Outcome, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, + Selector, Snapshot, SnapshotItem, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, }; use codex_protocol::{ protocol::{AgentPath, InterAgentCommunication}, @@ -23,12 +23,8 @@ struct TestMailbox { async fn test_mailbox() -> TestMailbox { let home = tempfile::tempdir().expect("temporary mailbox home"); let config = SqliteConfig::new_for_testing(home.path().abs()); - let mailbox = Mailbox::open(&config).await.expect("open test mailbox"); - TestMailbox { - _home: home, - config, - mailbox, - } + let mailbox = Mailbox::open(&config).await.expect("open mailbox"); + TestMailbox { _home: home, config, mailbox } } fn thread_id(index: u64) -> ThreadId { @@ -38,49 +34,25 @@ fn thread_id(index: u64) -> ThreadId { fn communication(content: &str) -> InterAgentCommunication { InterAgentCommunication::new( - AgentPath::root(), - AgentPath::root(), - Vec::new(), - content.to_string(), - false, + AgentPath::root(), AgentPath::root(), Vec::new(), content.to_string(), false, ) } -fn native_work( - issuer: ThreadId, - agent: ThreadId, - native_key: &str, - submission_id: &str, - content: &str, -) -> NativeWork { +fn native_work(issuer: ThreadId, agent: ThreadId, key: &str, content: &str) -> NativeWork { NativeWork { - native_key: native_key.to_string(), - issuer, - agent, - submission_id: submission_id.to_string(), - input: communication(content), + native_key: key.to_string(), issuer, agent, + submission_id: format!("submission-{key}"), input: communication(content), } } -fn message_draft( - invocation_key: &str, - sender: ThreadId, - recipient: ThreadId, - kind: MessageKind, - content: &str, +fn draft( + key: &str, sender: ThreadId, recipient: ThreadId, kind: MessageKind, content: &str, legacy_attention: bool, ) -> MessageDraft { MessageDraft { - invocation_key: invocation_key.to_string(), - sender, - recipient, - communication: communication(content), - kind, - activate_if_idle: false, - work_ref: None, - reply_to: None, - supersedes: None, - legacy_attention, + invocation_key: key.to_string(), sender, recipient, + communication: communication(content), kind, activate_if_idle: false, + work_ref: None, reply_to: None, supersedes: None, legacy_attention, } } @@ -88,1214 +60,512 @@ fn selection( caller: ThreadId, agent: Option, selectors: Vec, - return_when: ReturnWhen, + when: ReturnWhen, ) -> Selection { - Selection { - caller, - agent, - selectors, - return_when, - limit: 100, - } + Selection { caller, agent, selectors, return_when: when, limit: 100 } } -fn work_item<'a>(snapshot: &'a Snapshot, work_id: &str) -> &'a WorkSnapshot { - snapshot - .items - .iter() - .find_map(|item| match item { - SnapshotItem::Work(work) if work.work_ref.id == work_id => Some(work), - _ => None, - }) - .expect("selected work is present in the snapshot") +async fn take_snapshot( + mailbox: &Mailbox, + caller: ThreadId, + agent: Option, + selectors: Vec, + when: ReturnWhen, +) -> Snapshot { + mailbox.snapshot(&selection(caller, agent, selectors, when)).await.unwrap() } -fn request_item<'a>(snapshot: &'a Snapshot, request_id: &str) -> &'a RequestSnapshot { - snapshot - .items - .iter() - .find_map(|item| match item { - SnapshotItem::Request(request) if request.request_id == request_id => Some(request), - _ => None, - }) - .expect("selected request is present in the snapshot") +fn work_item<'a>(snapshot: &'a Snapshot, id: &str) -> &'a WorkSnapshot { + snapshot.items.iter().find_map(|item| match item { + SnapshotItem::Work(work) if work.work_ref.id == id => Some(work), _ => None, + }).expect("selected work is in snapshot") } -fn goal_binding(parent: ThreadId, child: ThreadId, generation: u64, turn: &str) -> GoalBinding { - GoalBinding { - goal_id: "goal-1".to_string(), - parent_thread_id: parent, - child_thread_id: child, - control_generation: generation, - revision: 1, - source_turn_id: turn.to_string(), - } +fn request_item<'a>(snapshot: &'a Snapshot, id: &str) -> &'a RequestSnapshot { + snapshot.items.iter().find_map(|item| match item { + SnapshotItem::Request(request) if request.request_id == id => Some(request), _ => None, + }).expect("selected request is in snapshot") } -#[tokio::test] -async fn accepted_native_key_retry_preserves_work_and_provider_message_identity() { - let fixture = test_mailbox().await; - let issuer = thread_id(1); - let agent = thread_id(2); - let original = native_work(issuer, agent, "native-1", "submission-1", "do the work"); - - let accepted = fixture - .mailbox - .accept_work(original.clone()) +async fn accept_work( + mailbox: &Mailbox, + issuer: ThreadId, + agent: ThreadId, + key: &str, + content: &str, +) -> (WorkRef, String) { + let work = mailbox + .accept_work(native_work(issuer, agent, key, content)) .await - .expect("accept native work"); - let captured_input = fixture - .mailbox - .accepted_input(&accepted) + .expect("accept work"); + let input_id = mailbox + .accepted_input(&work) .await - .expect("read captured native input"); - let captured_id = captured_input - .id - .as_ref() - .expect("accepted input receives a stable ID"); - assert!(captured_id.as_str().starts_with("amsg_")); + .expect("read accepted input") + .id.expect("accepted input has provider ID").to_string(); + (work, input_id) +} - // A retry after a lost publication reuses both the immutable work reference - // and the already-assigned provider item ID. - assert_eq!( - fixture.mailbox.accept_work(original).await.unwrap(), - accepted - ); - assert_eq!( - fixture.mailbox.accepted_input(&accepted).await.unwrap(), - captured_input - ); +async fn admit(mailbox: &Mailbox, agent: ThreadId, turn: &str, ids: &[String]) -> AdmissionReceipt { + mailbox.record_admission(agent, turn, ids).await.expect("record admission") +} - let mut conflicting_retry = native_work(issuer, agent, "native-1", "submission-1", "changed"); - conflicting_retry.input.trigger_turn = true; - assert!(matches!( - fixture.mailbox.accept_work(conflicting_retry).await, - Err(MailboxError::IdempotencyConflict) - )); +async fn finish( + mailbox: &Mailbox, agent: ThreadId, turn: &str, outcome: Outcome, + response: Option, continuation: Continuation, +) -> TurnOutcomeReceipt { + mailbox.record_turn_outcome(agent, turn, outcome, response, continuation).await + .expect("record turn outcome") +} - assert_eq!( - fixture.mailbox.begin_dispatch(&accepted).await.unwrap(), - DispatchDecision::Dispatch - ); - assert_eq!( - fixture.mailbox.begin_dispatch(&accepted).await.unwrap(), - DispatchDecision::RecoveryNeeded - ); +async fn send(mailbox: &Mailbox, message: MessageDraft) -> Envelope { + mailbox.accept_message(message).await.expect("accept message") +} - // Reopening the durable store does not turn an unresolved claim into - // permission to send again. - let reopened = Mailbox::open(&fixture.config).await.unwrap(); - assert_eq!( - reopened.dispatch_state(&accepted).await.unwrap(), - DispatchState::Dispatching - ); - assert_eq!( - reopened.begin_dispatch(&accepted).await.unwrap(), - DispatchDecision::RecoveryNeeded - ); - reopened.mark_dispatched(&accepted).await.unwrap(); - assert_eq!( - reopened.begin_dispatch(&accepted).await.unwrap(), - DispatchDecision::AlreadyDispatched - ); +fn goal_binding( + parent: ThreadId, + child: ThreadId, + generation: u64, + source_turn: &str, +) -> GoalBinding { + GoalBinding { + goal_id: "goal-1".to_string(), parent_thread_id: parent, child_thread_id: child, + control_generation: generation, revision: 1, source_turn_id: source_turn.to_string(), + } +} + +fn dependency(owner: &str, peers: &[&str], mode: DependencyMode) -> Dependency { + Dependency { + owner: owner.to_string(), + peers: peers.iter().map(|peer| (*peer).to_string()).collect(), + mode, + } } #[tokio::test] -async fn admission_is_bound_to_the_exact_input_ids_and_turn() { +async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { let fixture = test_mailbox().await; - let issuer = thread_id(3); - let agent = thread_id(4); - let work_a = fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-a", "submission-a", "a")) - .await - .unwrap(); - let work_b = fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-b", "submission-b", "b")) - .await - .unwrap(); - fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-c", "submission-c", "c")) - .await - .unwrap(); - let id_a = fixture - .mailbox - .accepted_input(&work_a) - .await - .unwrap() - .id - .expect("accepted input has an ID") - .to_string(); - let id_b = fixture - .mailbox - .accepted_input(&work_b) - .await - .unwrap() - .id - .expect("accepted input has an ID") - .to_string(); - let work_c = fixture - .mailbox - .work_for_native_key(issuer, "native-c") - .await - .unwrap() - .expect("third native work was accepted"); - let id_c = fixture - .mailbox - .accepted_input(&work_c) - .await - .unwrap() - .id - .unwrap() - .to_string(); - - let receipt = fixture - .mailbox - .record_admission(agent, "turn-one", &[id_a.clone()]) - .await - .unwrap(); - assert_eq!(receipt.work_refs, vec![work_a.clone()]); - assert!(!receipt.already_recorded); - - let retry = fixture - .mailbox - .record_admission(agent, "turn-one", &[id_a.clone()]) - .await - .unwrap(); - assert_eq!(retry.work_refs, receipt.work_refs); - assert!(retry.already_recorded); - - let appended = fixture - .mailbox - .record_admission(agent, "turn-one", &[id_b.clone()]) - .await - .unwrap(); - assert_eq!(appended.work_refs, vec![work_b.clone()]); - assert!(!appended.already_recorded); - let covered = fixture - .mailbox - .record_turn_outcome( - agent, - "turn-one", - Outcome::Completed, - None, - Continuation::None, - ) - .await - .unwrap(); - assert_eq!(covered.outcomes.len(), 2); - assert!(covered.outcomes.iter().any(|item| item.work_ref == work_a)); - assert!(covered.outcomes.iter().any(|item| item.work_ref == work_b)); - assert!(covered.outcomes.iter().all(|item| item.work_ref != work_c)); - + let (issuer, agent) = (thread_id(1), thread_id(2)); + let original = native_work(issuer, agent, "native-1", "do the work"); + let work = fixture.mailbox.accept_work(original.clone()).await.unwrap(); + let captured = fixture.mailbox.accepted_input(&work).await.unwrap(); + assert!(captured.id.as_ref().unwrap().as_str().starts_with("amsg_")); + assert_eq!(fixture.mailbox.accept_work(original).await.unwrap(), work); + assert_eq!(fixture.mailbox.accepted_input(&work).await.unwrap(), captured); + + let mut conflict = native_work(issuer, agent, "native-1", "changed input"); + conflict.input.trigger_turn = true; assert!(matches!( - fixture - .mailbox - .record_admission(agent, "turn-two", &[id_a]) - .await, + fixture.mailbox.accept_work(conflict).await, Err(MailboxError::IdempotencyConflict) )); - assert!(matches!( - fixture - .mailbox - .record_admission(agent, "turn-one", &[id_c, "amsg_not-admitted".to_string()]) - .await, - Err(MailboxError::ReferenceMismatch) - )); + assert_eq!(fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + assert_eq!( + fixture.mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + + let reopened = Mailbox::open(&fixture.config).await.unwrap(); + assert_eq!(reopened.dispatch_state(&work).await.unwrap(), DispatchState::Dispatching); + assert_eq!(reopened.begin_dispatch(&work).await.unwrap(), DispatchDecision::RecoveryNeeded); + reopened.mark_dispatched(&work).await.unwrap(); + assert_eq!(reopened.begin_dispatch(&work).await.unwrap(), DispatchDecision::AlreadyDispatched); - let too_many = (0..101) - .map(|index| format!("amsg-over-limit-{index}")) - .collect::>(); + let mut too_large = native_work(issuer, agent, "too-large", "x"); + too_large.input.content = "x".repeat(1_000_001); assert!(matches!( - fixture - .mailbox - .record_admission(agent, "oversized-turn", &too_many) - .await, + fixture.mailbox.accept_work(too_large).await, Err(MailboxError::Backpressure(_)) )); + assert!(fixture.mailbox.work_for_native_key(issuer, "too-large").await.unwrap().is_none()); } #[tokio::test] -async fn turn_outcome_covers_only_admitted_refs_and_snapshot_is_caller_scoped() { +async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped() { let fixture = test_mailbox().await; - let issuer = thread_id(5); - let other_issuer = thread_id(6); - let agent = thread_id(7); - let work_a = fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-a", "submission-a", "a")) - .await - .unwrap(); - let work_b = fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-b", "submission-b", "b")) - .await - .unwrap(); - let id_a = fixture - .mailbox - .accepted_input(&work_a) - .await - .unwrap() - .id - .unwrap() - .to_string(); - let id_b = fixture - .mailbox - .accepted_input(&work_b) - .await - .unwrap() - .id - .unwrap() - .to_string(); - fixture - .mailbox - .record_admission(agent, "turn-covered", &[id_a, id_b]) - .await - .unwrap(); - - // This accepted input arrives after the native execution admitted its - // fixed set and must remain pending for a later turn. - let unadmitted = fixture - .mailbox - .accept_work(native_work(issuer, agent, "native-late", "submission-late", "late")) - .await - .unwrap(); - let unrelated = fixture - .mailbox - .accept_work(native_work( - other_issuer, - agent, - "native-other-issuer", - "submission-other-issuer", - "other issuer history", - )) - .await - .unwrap(); - fixture - .mailbox - .record_outcome( - &unrelated, - Outcome::Completed, - None, - Continuation::None, - ) - .await - .unwrap(); - - let outcome = fixture - .mailbox - .record_turn_outcome( - agent, - "turn-covered", - Outcome::Completed, - None, - Continuation::None, - ) - .await - .unwrap(); - assert_eq!(outcome.outcomes.len(), 2); - assert!(outcome.outcomes.iter().any(|item| item.work_ref == work_a)); - assert!(outcome.outcomes.iter().any(|item| item.work_ref == work_b)); - assert!(outcome.outcomes.iter().all(|item| item.work_ref != unadmitted)); - - let snapshot = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![ - Selector::Work(work_a.id.clone()), - Selector::Work(work_b.id.clone()), - Selector::Work(unadmitted.id.clone()), - Selector::Work(unrelated.id.clone()), - ], - ReturnWhen::All, - )) - .await - .unwrap(); + let (issuer, other_issuer, agent) = (thread_id(3), thread_id(4), thread_id(5)); + let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "a", "A").await; + let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "b", "B").await; + let (c, id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; + assert_eq!( + admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]).await.work_refs, + vec![a.clone()] + ); + assert!(admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]).await.already_recorded); + assert_eq!( + admit(&fixture.mailbox, agent, "turn", &[id_b.clone()]).await.work_refs, + vec![b.clone()] + ); + assert!(matches!(fixture.mailbox.record_admission(agent, "other-turn", &[id_a.clone()]).await, + Err(MailboxError::IdempotencyConflict))); + + // New work after admission remains outside the outcome; unrelated issuer history is hidden. + let (late, _) = accept_work(&fixture.mailbox, issuer, agent, "late", "late").await; + let (foreign, _) = accept_work( + &fixture.mailbox, other_issuer, agent, "foreign", "foreign", + ).await; + fixture.mailbox.record_outcome( + &foreign, Outcome::Completed, None, Continuation::None, + ).await.unwrap(); + let outcomes = finish( + &fixture.mailbox, agent, "turn", Outcome::Completed, None, Continuation::None, + ).await; + assert_eq!(outcomes.outcomes.len(), 2); + assert!(outcomes.outcomes.iter().any(|item| item.work_ref == a)); + assert!(outcomes.outcomes.iter().any(|item| item.work_ref == b)); + assert!(outcomes.outcomes.iter().all(|item| item.work_ref != c && item.work_ref != late)); + + let snapshot = take_snapshot(&fixture.mailbox, issuer, Some(agent), vec![ + Selector::Work(a.id.clone()), Selector::Work(b.id.clone()), Selector::Work(c.id.clone()), + Selector::Work(late.id.clone()), Selector::Work(foreign.id.clone()), + ], ReturnWhen::All).await; assert_eq!(snapshot.state, SnapshotState::Pending); - assert_eq!(snapshot.items.len(), 3); - assert!(work_item(&snapshot, &work_a.id).outcome.is_some()); - assert!(work_item(&snapshot, &work_b.id).outcome.is_some()); - assert!(work_item(&snapshot, &unadmitted.id).outcome.is_none()); - assert!(!snapshot - .items - .iter() - .any(|item| matches!(item, SnapshotItem::Work(work) if work.work_ref.id == unrelated.id))); + assert_eq!(snapshot.items.len(), 4); + assert!(work_item(&snapshot, &a.id).outcome.is_some()); + assert!(work_item(&snapshot, &b.id).outcome.is_some()); + assert!(work_item(&snapshot, &c.id).outcome.is_none()); + assert!(work_item(&snapshot, &late.id).outcome.is_none()); + assert!(!snapshot.items.iter().any(|item| matches!( + item, SnapshotItem::Work(work) if work.work_ref.id == foreign.id + ))); + + let too_many = (0..101).map(|i| format!("amsg-limit-{i}")).collect::>(); + assert!(matches!(fixture.mailbox.record_admission(agent, "over-limit", &too_many).await, + Err(MailboxError::Backpressure(_)))); } #[tokio::test] -async fn one_turn_returns_separate_handbacks_to_each_issuer() { +async fn mixed_issuer_turn_creates_distinct_handbacks() { let fixture = test_mailbox().await; - let issuer_a = thread_id(8); - let issuer_b = thread_id(9); - let agent = thread_id(12); - let work_a = fixture - .mailbox - .accept_work(native_work(issuer_a, agent, "native-a", "submission-a", "a")) - .await - .unwrap(); - let work_b = fixture - .mailbox - .accept_work(native_work(issuer_b, agent, "native-b", "submission-b", "b")) - .await - .unwrap(); - let id_a = fixture - .mailbox - .accepted_input(&work_a) - .await - .unwrap() - .id - .unwrap() - .to_string(); - let id_b = fixture - .mailbox - .accepted_input(&work_b) - .await - .unwrap() - .id - .unwrap() - .to_string(); - fixture - .mailbox - .record_admission(agent, "shared-turn", &[id_a, id_b]) - .await - .unwrap(); - - let result = fixture - .mailbox - .record_turn_outcome( - agent, - "shared-turn", - Outcome::Completed, - Some(communication("result for both issuers")), - Continuation::None, - ) - .await - .unwrap(); - assert_eq!(result.outcomes.len(), 2); - let response_a = result - .outcomes - .iter() - .find(|item| item.work_ref == work_a) - .and_then(|item| item.response_message_id.as_deref()) - .expect("issuer A receives its response"); - let response_b = result - .outcomes - .iter() - .find(|item| item.work_ref == work_b) - .and_then(|item| item.response_message_id.as_deref()) - .expect("issuer B receives its response"); - assert_ne!(response_a, response_b); - - let handback_a = fixture - .mailbox - .get_message(issuer_a, response_a) - .await - .unwrap() - .expect("issuer A handback is addressable"); - let handback_b = fixture - .mailbox - .get_message(issuer_b, response_b) - .await - .unwrap() - .expect("issuer B handback is addressable"); - assert_eq!(handback_a.sender, agent); - assert_eq!(handback_a.recipient, issuer_a); - assert_eq!(handback_b.sender, agent); - assert_eq!(handback_b.recipient, issuer_b); + let (issuer_a, issuer_b, agent) = (thread_id(6), thread_id(7), thread_id(8)); + let (a, id_a) = accept_work(&fixture.mailbox, issuer_a, agent, "multi-a", "A").await; + let (b, id_b) = accept_work(&fixture.mailbox, issuer_b, agent, "multi-b", "B").await; + admit(&fixture.mailbox, agent, "shared-turn", &[id_a, id_b]).await; + let receipt = finish(&fixture.mailbox, agent, "shared-turn", Outcome::Completed, + Some(communication("one result")), Continuation::None).await; + assert_eq!(receipt.outcomes.len(), 2); + let id_a = receipt.outcomes.iter().find(|r| r.work_ref == a) + .and_then(|r| r.response_message_id.clone()).unwrap(); + let id_b = receipt.outcomes.iter().find(|r| r.work_ref == b) + .and_then(|r| r.response_message_id.clone()).unwrap(); + assert_ne!(id_a, id_b); + for (issuer, id) in [(issuer_a, id_a), (issuer_b, id_b)] { + let message = fixture.mailbox.get_message(issuer, &id).await.unwrap().unwrap(); + assert_eq!((message.sender, message.recipient), (agent, issuer)); + } } #[tokio::test] -async fn pending_and_deferred_goal_bindings_settle_only_for_the_exact_generation() { +async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() { let fixture = test_mailbox().await; - let issuer = thread_id(13); - let agent = thread_id(14); - let work_pending = fixture - .mailbox - .accept_work(native_work( - issuer, - agent, - "native-pending", - "submission-pending", - "pending goal work", - )) - .await - .unwrap(); - let work_deferred = fixture - .mailbox - .accept_work(native_work( - issuer, - agent, - "native-deferred", - "submission-deferred", - "deferred goal work", - )) - .await - .unwrap(); - let binding_pending = goal_binding(issuer, agent, 4, "pending-turn"); - let binding_deferred = goal_binding(issuer, agent, 5, "deferred-turn"); - for (work_ref, turn_id, continuation) in [ + let (issuer, agent) = (thread_id(9), thread_id(10)); + let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "pending", "A").await; + let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "deferred", "B").await; + let bindings = [ + goal_binding(issuer, agent, 4, "turn-p"), + goal_binding(issuer, agent, 5, "turn-d"), + ]; + for (work, id, turn, continuation, state) in [ ( - &work_pending, - "pending-turn", - Continuation::PendingGoalBinding(binding_pending.clone()), + &a, id_a, "turn-p", Continuation::PendingGoalBinding(bindings[0].clone()), + ContinuationState::PendingGoalBinding, ), ( - &work_deferred, - "deferred-turn", - Continuation::Deferred(binding_deferred.clone()), + &b, id_b, "turn-d", Continuation::Deferred(bindings[1].clone()), + ContinuationState::Deferred, ), ] { - let input_id = fixture - .mailbox - .accepted_input(work_ref) - .await - .unwrap() - .id - .unwrap() - .to_string(); - fixture - .mailbox - .record_admission(agent, turn_id, &[input_id]) - .await - .unwrap(); - let result = fixture - .mailbox - .record_turn_outcome( - agent, - turn_id, - Outcome::Completed, - None, - continuation, - ) - .await - .unwrap(); - assert!(result.outcomes.is_empty()); - assert_eq!(result.continuation_work, vec![(*work_ref).clone()]); - } - - for (work_ref, expected_state) in [ - (&work_pending, ContinuationState::PendingGoalBinding), - (&work_deferred, ContinuationState::Deferred), - ] { - let snapshot = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![Selector::Work(work_ref.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - let item = work_item(&snapshot, &work_ref.id); + admit(&fixture.mailbox, agent, turn, &[id]).await; + let receipt = finish( + &fixture.mailbox, agent, turn, Outcome::Completed, None, continuation, + ).await; + assert!(receipt.outcomes.is_empty()); + assert_eq!(receipt.continuation_work, vec![(*work).clone()]); + let snapshot = take_snapshot( + &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(work.id.clone())], + ReturnWhen::Any, + ).await; + let item = work_item(&snapshot, &work.id); assert!(item.outcome.is_none()); - assert_eq!(item.continuations.len(), 1); - assert_eq!(item.continuations[0].state, expected_state); + assert_eq!(item.continuations[0].state, state); } - - let mut stale_generation = binding_pending.clone(); - stale_generation.control_generation += 1; - let stale = fixture - .mailbox - .record_turn_outcome( - agent, - "later-turn-stale-generation", - Outcome::Completed, - None, - Continuation::Terminal(stale_generation), - ) - .await - .unwrap(); - assert!(stale.no_matching_work); - assert!(fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![Selector::Work(work_pending.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap() - .items - .iter() - .any(|item| matches!(item, SnapshotItem::Work(work) if work.outcome.is_none()))); - - for (work_ref, binding, turn_id) in [ - (&work_pending, binding_pending, "terminal-pending"), - (&work_deferred, binding_deferred, "terminal-deferred"), + let mut stale = bindings[0].clone(); + stale.control_generation += 1; + assert!(finish(&fixture.mailbox, agent, "stale-generation", Outcome::Completed, None, + Continuation::Terminal(stale)).await.no_matching_work); + let still_deferred = take_snapshot( + &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(a.id.clone())], + ReturnWhen::Any, + ).await; + assert!(work_item(&still_deferred, &a.id).outcome.is_none()); + + for (work, binding, turn) in [ + (&a, bindings[0].clone(), "terminal-p"), + (&b, bindings[1].clone(), "terminal-d"), ] { - let terminal = fixture - .mailbox - .record_turn_outcome( - agent, - turn_id, - Outcome::Completed, - None, - Continuation::Terminal(binding.clone()), - ) - .await - .unwrap(); - assert_eq!(terminal.outcomes.len(), 1); - let snapshot = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![Selector::Work(work_ref.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - let receipt = work_item(&snapshot, &work_ref.id) - .outcome - .as_ref() - .expect("exact terminal binding settles the old covered work"); - assert_eq!(receipt.continuation, Continuation::Terminal(binding)); + let result = finish(&fixture.mailbox, agent, turn, Outcome::Completed, None, + Continuation::Terminal(binding.clone())).await; + assert_eq!(result.outcomes.len(), 1); + let snapshot = take_snapshot( + &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(work.id.clone())], + ReturnWhen::Any, + ).await; + assert_eq!(work_item(&snapshot, &work.id).outcome.as_ref().unwrap().continuation, + Continuation::Terminal(binding)); } } #[tokio::test] -async fn quiet_reply_satisfies_its_exact_request_selector() { +async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let fixture = test_mailbox().await; - let issuer = thread_id(15); - let agent = thread_id(16); - let request = fixture - .mailbox - .accept_message(message_draft( - "decision-request", - issuer, - agent, - MessageKind::Request, - "please decide", - false, - )) - .await - .unwrap(); - fixture - .mailbox - .mark_canonical(agent, &request.id) - .await - .unwrap(); - - let mut reply_draft = message_draft( - "decision-reply", - agent, - issuer, - MessageKind::Note, - "decision made", - false, - ); + let (author, recipient, stranger) = (thread_id(11), thread_id(12), thread_id(13)); + let request = send( + &fixture.mailbox, + draft("request", author, recipient, MessageKind::Request, "decide", false), + ).await; + let (_, input_id) = accept_work( + &fixture.mailbox, author, recipient, "unrelated", "other task", + ).await; + admit(&fixture.mailbox, recipient, "unrelated-turn", &[input_id]).await; + finish( + &fixture.mailbox, recipient, "unrelated-turn", Outcome::Completed, None, + Continuation::None, + ).await; + let open = take_snapshot( + &fixture.mailbox, author, Some(recipient), vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + ).await; + assert_eq!(open.state, SnapshotState::Pending); + assert_eq!(request_item(&open, &request.id).disposition, RequestDisposition::Open); + fixture.mailbox.mark_canonical(recipient, &request.id).await.unwrap(); + let mut reply_draft = draft("reply", recipient, author, MessageKind::Note, "done", false); reply_draft.reply_to = Some(request.id.clone()); - let reply = fixture - .mailbox - .accept_message(reply_draft) - .await - .unwrap(); - - let snapshot = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![Selector::Reply(request.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - assert_eq!(snapshot.state, SnapshotState::AlreadySatisfied); - let selected_request = request_item(&snapshot, &request.id); - assert_eq!(selected_request.disposition, RequestDisposition::Replied); - assert_eq!( - selected_request.reply.as_ref().map(|message| message.id.as_str()), - Some(reply.id.as_str()) - ); - assert!(!reply.legacy_attention); -} - -#[tokio::test] -async fn unrelated_successful_work_does_not_resolve_an_open_request() { - let fixture = test_mailbox().await; - let issuer = thread_id(17); - let agent = thread_id(18); - let request = fixture - .mailbox - .accept_message(message_draft( - "open-request", - issuer, - agent, - MessageKind::Request, - "await a separate decision", - false, - )) - .await - .unwrap(); - let work_ref = fixture - .mailbox - .accept_work(native_work( - issuer, - agent, - "unrelated-success", - "unrelated-submission", - "complete different work", - )) - .await - .unwrap(); - let input_id = fixture - .mailbox - .accepted_input(&work_ref) - .await - .unwrap() - .id - .unwrap() - .to_string(); - fixture - .mailbox - .record_admission(agent, "unrelated-turn", &[input_id]) - .await - .unwrap(); - fixture - .mailbox - .record_turn_outcome( - agent, - "unrelated-turn", - Outcome::Completed, - None, - Continuation::None, - ) - .await - .unwrap(); - - let snapshot = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![Selector::Reply(request.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - assert_eq!(snapshot.state, SnapshotState::Pending); - assert_eq!(request_item(&snapshot, &request.id).disposition, RequestDisposition::Open); -} - -#[tokio::test] -async fn only_the_request_author_can_supersede_or_withdraw_and_old_selectors_settle() { - let fixture = test_mailbox().await; - let author = thread_id(19); - let recipient = thread_id(22); - let stranger = thread_id(23); - let request = fixture - .mailbox - .accept_message(message_draft( - "replace-me", - author, - recipient, - MessageKind::Request, - "first request", - false, - )) - .await - .unwrap(); - - let mut unauthorized_replacement = message_draft( - "stranger-replacement", - stranger, - recipient, - MessageKind::Request, - "not yours to replace", - false, + let reply = send(&fixture.mailbox, reply_draft).await; + let replied = take_snapshot( + &fixture.mailbox, author, Some(recipient), vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + ).await; + assert_eq!(replied.state, SnapshotState::AlreadySatisfied); + assert_eq!(request_item(&replied, &request.id).disposition, RequestDisposition::Replied); + assert_eq!(request_item(&replied, &request.id).reply.as_ref().unwrap().id, reply.id); + + let open_request = send( + &fixture.mailbox, + draft("replace-me", author, recipient, MessageKind::Request, "old", false), + ).await; + let mut unauthorized = draft( + "stranger-replace", stranger, recipient, MessageKind::Request, "no", false, ); - unauthorized_replacement.supersedes = Some(request.id.clone()); + unauthorized.supersedes = Some(open_request.id.clone()); assert!(matches!( - fixture - .mailbox - .accept_message(unauthorized_replacement) - .await, + fixture.mailbox.accept_message(unauthorized).await, Err(MailboxError::WrongRecipient) )); - - let mut replacement_draft = message_draft( - "author-replacement", - author, - recipient, - MessageKind::Request, - "updated request", - false, - ); - replacement_draft.supersedes = Some(request.id.clone()); - let replacement = fixture - .mailbox - .accept_message(replacement_draft) - .await - .unwrap(); - let replaced_snapshot = fixture - .mailbox - .snapshot(&selection( - author, - Some(recipient), - vec![Selector::Reply(request.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - assert_eq!(replaced_snapshot.state, SnapshotState::AlreadySatisfied); - let old_request = request_item(&replaced_snapshot, &request.id); - assert_eq!(old_request.disposition, RequestDisposition::Superseded); - assert_eq!(old_request.replacement.as_deref(), Some(replacement.id.as_str())); - - let withdrawal = fixture - .mailbox - .accept_message(message_draft( - "withdraw-me", - author, - recipient, - MessageKind::Request, - "cancel this request", - false, - )) - .await - .unwrap(); - assert!(matches!( - fixture - .mailbox - .withdraw_message(stranger, &withdrawal.id) - .await, - Err(MailboxError::WrongRecipient) - )); - fixture - .mailbox - .withdraw_message(author, &withdrawal.id) - .await - .unwrap(); - let withdrawn_snapshot = fixture - .mailbox - .snapshot(&selection( - author, - Some(recipient), - vec![Selector::Reply(withdrawal.id.clone())], - ReturnWhen::Any, - )) - .await - .unwrap(); - assert_eq!(withdrawn_snapshot.state, SnapshotState::AlreadySatisfied); - assert_eq!( - request_item(&withdrawn_snapshot, &withdrawal.id).disposition, - RequestDisposition::Withdrawn + let mut replacement_draft = draft( + "replacement", author, recipient, MessageKind::Request, "new", false, ); + replacement_draft.supersedes = Some(open_request.id.clone()); + let replacement = send(&fixture.mailbox, replacement_draft).await; + let superseded = take_snapshot( + &fixture.mailbox, author, None, vec![Selector::Reply(open_request.id.clone())], + ReturnWhen::Any, + ).await; + assert_eq!(superseded.state, SnapshotState::AlreadySatisfied); + let old = request_item(&superseded, &open_request.id); + assert_eq!(old.disposition, RequestDisposition::Superseded); + assert_eq!(old.replacement.as_deref(), Some(replacement.id.as_str())); + + let withdrawal = send( + &fixture.mailbox, + draft("withdraw-me", author, recipient, MessageKind::Request, "cancel", false), + ).await; + assert!(matches!(fixture.mailbox.withdraw_message(stranger, &withdrawal.id).await, + Err(MailboxError::WrongRecipient))); + fixture.mailbox.withdraw_message(author, &withdrawal.id).await.unwrap(); + let withdrawn = take_snapshot( + &fixture.mailbox, author, None, vec![Selector::Reply(withdrawal.id.clone())], + ReturnWhen::Any, + ).await; + assert_eq!(withdrawn.state, SnapshotState::AlreadySatisfied); + assert_eq!(request_item(&withdrawn, &withdrawal.id).disposition, RequestDisposition::Withdrawn); } #[tokio::test] -async fn progress_coalesces_only_for_the_exact_unadmitted_author_and_work() { +async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcome() { let fixture = test_mailbox().await; - let issuer = thread_id(24); - let agent = thread_id(25); - let work_a = fixture - .mailbox - .accept_work(native_work(issuer, agent, "progress-a", "submission-a", "a")) - .await - .unwrap(); - let work_b = fixture - .mailbox - .accept_work(native_work(issuer, agent, "progress-b", "submission-b", "b")) - .await - .unwrap(); - - let mut first_draft = message_draft( - "progress-first", - issuer, - agent, - MessageKind::Progress, - "initial status", - false, - ); + let (author, agent) = (thread_id(14), thread_id(15)); + let (work_a, _) = accept_work(&fixture.mailbox, author, agent, "work-a", "A").await; + let (work_b, _) = accept_work(&fixture.mailbox, author, agent, "work-b", "B").await; + let mut first_draft = draft("p1", author, agent, MessageKind::Progress, "first", false); first_draft.work_ref = Some(work_a.clone()); - let first = fixture - .mailbox - .accept_message(first_draft) - .await - .unwrap(); - let mut update_draft = message_draft( - "progress-update", - issuer, - agent, - MessageKind::Progress, - "latest status", - false, - ); + let first = send(&fixture.mailbox, first_draft).await; + let mut update_draft = draft("p2", author, agent, MessageKind::Progress, "latest", false); update_draft.work_ref = Some(work_a.clone()); - let update = fixture - .mailbox - .accept_message(update_draft) - .await - .unwrap(); + let update = send(&fixture.mailbox, update_draft).await; assert_eq!(update.id, first.id); assert_eq!( - fixture - .mailbox - .get_message(agent, &first.id) - .await - .unwrap() - .unwrap() - .communication - .content, - "latest status" - ); - - let mut different_author = message_draft( - "progress-other-author", - issuer, - agent, - MessageKind::Progress, - "different path author", - false, + fixture.mailbox.get_message(agent, &first.id).await.unwrap().unwrap().communication.content, + "latest" ); - different_author.work_ref = Some(work_a.clone()); - different_author.communication.author = - AgentPath::from_string("/root/child".to_string()).expect("valid author path"); - let other_author = fixture - .mailbox - .accept_message(different_author) - .await - .unwrap(); - assert_ne!(other_author.id, first.id); - let mut different_work = message_draft( - "progress-other-work", - issuer, - agent, - MessageKind::Progress, - "different work", - false, - ); - different_work.work_ref = Some(work_b); - let other_work = fixture - .mailbox - .accept_message(different_work) - .await - .unwrap(); - assert_ne!(other_work.id, first.id); - - // No admission is necessary to coalesce quiet progress, but once its - // exact work is terminal the same writer cannot append late progress. - fixture - .mailbox - .record_outcome(&work_a, Outcome::Completed, None, Continuation::None) - .await - .unwrap(); - let mut late_progress = message_draft( - "progress-after-terminal", - issuer, - agent, - MessageKind::Progress, - "too late", - false, - ); - late_progress.work_ref = Some(work_a); + let mut other_author = draft("p3", author, agent, MessageKind::Progress, "other path", false); + other_author.work_ref = Some(work_a.clone()); + other_author.communication.author = AgentPath::from_string("/root/child".to_string()).unwrap(); + assert_ne!(send(&fixture.mailbox, other_author).await.id, first.id); + let mut other_path = draft("p-path", author, agent, MessageKind::Progress, "other path", false); + other_path.work_ref = Some(work_a.clone()); + other_path.communication.recipient = AgentPath::from_string("/root/other".to_string()).unwrap(); + assert_ne!(send(&fixture.mailbox, other_path).await.id, first.id); + let mut other_work = draft("p4", author, agent, MessageKind::Progress, "work B", false); + other_work.work_ref = Some(work_b); + assert_ne!(send(&fixture.mailbox, other_work).await.id, first.id); + + fixture.mailbox.record_outcome( + &work_a, Outcome::Completed, None, Continuation::None, + ).await.unwrap(); + let mut late = draft("p-late", author, agent, MessageKind::Progress, "late", false); + late.work_ref = Some(work_a); assert!(matches!( - fixture.mailbox.accept_message(late_progress).await, + fixture.mailbox.accept_message(late).await, Err(MailboxError::OutcomeAlreadyRecorded) )); } #[tokio::test] -async fn out_of_order_observation_is_per_item_after_canonical_flush() { +async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { let fixture = test_mailbox().await; - let issuer = thread_id(26); - let agent = thread_id(27); - let work_a = fixture - .mailbox - .accept_work(native_work(issuer, agent, "observed-a", "submission-a", "a")) - .await - .unwrap(); - let work_b = fixture - .mailbox - .accept_work(native_work(issuer, agent, "observed-b", "submission-b", "b")) - .await - .unwrap(); - - for (work_ref, turn_id) in [(&work_a, "turn-a"), (&work_b, "turn-b")] { - let input_id = fixture - .mailbox - .accepted_input(work_ref) - .await - .unwrap() - .id - .unwrap() - .to_string(); - fixture - .mailbox - .record_admission(agent, turn_id, &[input_id]) - .await - .unwrap(); - let result = fixture - .mailbox - .record_turn_outcome( - agent, - turn_id, - Outcome::Completed, - None, - Continuation::None, - ) - .await - .unwrap(); - let receipt = result.outcomes.first().expect("one exact outcome"); - fixture - .mailbox - .mark_outcome_canonical(agent, &receipt.id) - .await - .unwrap(); + let (issuer, agent) = (thread_id(16), thread_id(17)); + let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "observe-a", "A").await; + let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "observe-b", "B").await; + for (work, id, turn) in [(&a, id_a, "turn-a"), (&b, id_b, "turn-b")] { + admit(&fixture.mailbox, agent, turn, &[id]).await; + let outcome = finish( + &fixture.mailbox, agent, turn, Outcome::Completed, None, Continuation::None, + ).await; + fixture.mailbox.mark_outcome_canonical(agent, &outcome.outcomes[0].id).await.unwrap(); + assert_eq!(&outcome.outcomes[0].work_ref, work); } - - fixture - .mailbox - .acknowledge_observed(issuer, &[ObservationTarget::Work(work_b.clone())]) - .await - .unwrap(); - let partially_observed = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![ - Selector::Work(work_a.id.clone()), - Selector::Work(work_b.id.clone()), - ], - ReturnWhen::All, - )) - .await - .unwrap(); - assert_eq!(partially_observed.state, SnapshotState::Pending); - assert!(!work_item(&partially_observed, &work_a.id).observed); - assert!(work_item(&partially_observed, &work_b.id).observed); - assert!(work_item(&partially_observed, &work_a.id).outcome.is_some()); - assert!(work_item(&partially_observed, &work_b.id).outcome.is_some()); - - fixture - .mailbox - .acknowledge_observed(issuer, &[ObservationTarget::Work(work_a.clone())]) - .await - .unwrap(); - let fully_observed = fixture - .mailbox - .snapshot(&selection( - issuer, - Some(agent), - vec![ - Selector::Work(work_a.id.clone()), - Selector::Work(work_b.id.clone()), - ], - ReturnWhen::All, - )) - .await - .unwrap(); - assert_eq!(fully_observed.state, SnapshotState::AlreadySatisfied); + fixture.mailbox.acknowledge_observed( + issuer, &[ObservationTarget::Work(b.clone())], + ).await.unwrap(); + let selection = selection( + issuer, Some(agent), + vec![Selector::Work(a.id.clone()), Selector::Work(b.id.clone())], + ReturnWhen::All, + ); + let partial = fixture.mailbox.snapshot(&selection).await.unwrap(); + assert_eq!(partial.state, SnapshotState::Pending); + assert!(!work_item(&partial, &a.id).observed); + assert!(work_item(&partial, &b.id).observed); + assert!( + work_item(&partial, &a.id).outcome.is_some() + && work_item(&partial, &b.id).outcome.is_some() + ); + fixture.mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(a)]) + .await.unwrap(); + assert_eq!( + fixture.mailbox.snapshot(&selection).await.unwrap().state, + SnapshotState::AlreadySatisfied + ); } #[tokio::test] -async fn default_and_legacy_attention_are_separate_from_quiet_message_delivery() { +async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate() { let fixture = test_mailbox().await; - let sender = thread_id(10); - let recipient = thread_id(11); + let (sender, recipient) = (thread_id(18), thread_id(19)); let kinds = [ - MessageKind::Note, - MessageKind::Progress, - MessageKind::Request, - MessageKind::Alert, + MessageKind::Note, MessageKind::Progress, MessageKind::Request, MessageKind::Alert, ]; assert_eq!(MessageKind::default(), MessageKind::Note); - for kind in kinds { - fixture - .mailbox - .accept_message(message_draft( - &format!("quiet-{kind:?}"), - sender, - recipient, - kind, - "quiet message", - false, - )) - .await - .unwrap(); - fixture - .mailbox - .accept_message(message_draft( - &format!("legacy-{kind:?}"), - sender, - recipient, - kind, - "legacy attention message", - true, - )) - .await - .unwrap(); + send( + &fixture.mailbox, + draft(&format!("quiet-{kind:?}"), sender, recipient, kind, "quiet", false), + ).await; + send( + &fixture.mailbox, + draft(&format!("legacy-{kind:?}"), sender, recipient, kind, "legacy", true), + ).await; } - let pending = fixture.mailbox.pending(recipient, 20).await.unwrap(); assert_eq!(pending.len(), 8); - for kind in kinds { - assert_eq!(pending.iter().filter(|message| message.kind == kind).count(), 2); - } - assert_eq!( - pending.iter().filter(|message| message.legacy_attention).count(), - 4 - ); + for kind in kinds { assert_eq!(pending.iter().filter(|m| m.kind == kind).count(), 2); } + assert_eq!(pending.iter().filter(|m| m.legacy_attention).count(), 4); assert_eq!(fixture.mailbox.quiet_count(recipient).await.unwrap(), 4); assert_eq!(fixture.mailbox.attention(recipient, 20).await.unwrap().len(), 4); } #[tokio::test] -async fn activation_claim_is_not_reissued_after_a_lost_result() { +async fn activation_claim_is_single_use_across_a_lost_result() { let fixture = test_mailbox().await; - let sender = thread_id(20); - let recipient = thread_id(21); - let message = fixture - .mailbox - .accept_message(message_draft( - "activation-request", - sender, - recipient, - MessageKind::Alert, - "activate once", - false, - )) - .await - .unwrap(); - - assert_eq!( - fixture - .mailbox - .begin_activation(sender, &message.id) - .await - .unwrap(), - ActivationDecision::Activate - ); - assert_eq!( - fixture - .mailbox - .begin_activation(sender, &message.id) - .await - .unwrap(), - ActivationDecision::RecoveryNeeded - ); - fixture - .mailbox - .record_activation(sender, &message.id, ActivationDisposition::Activated) - .await - .unwrap(); - fixture - .mailbox - .record_activation(sender, &message.id, ActivationDisposition::Activated) - .await - .unwrap(); - assert_eq!( - fixture - .mailbox - .begin_activation(sender, &message.id) - .await - .unwrap(), - ActivationDecision::AlreadyDecided(ActivationDisposition::Activated) - ); + let (sender, recipient) = (thread_id(20), thread_id(21)); + let message = send( + &fixture.mailbox, + draft("activation", sender, recipient, MessageKind::Alert, "activate", false), + ).await; + assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + ActivationDecision::Activate); + assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + ActivationDecision::RecoveryNeeded); + for _ in 0..2 { + fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Activated) + .await.unwrap(); + } + assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + ActivationDecision::AlreadyDecided(ActivationDisposition::Activated)); assert!(matches!( - fixture - .mailbox - .record_activation(sender, &message.id, ActivationDisposition::Denied) - .await, + fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Denied).await, Err(MailboxError::IdempotencyConflict) )); } -#[tokio::test] -async fn oversized_native_capture_is_rejected_without_creating_dispatchable_work() { - let fixture = test_mailbox().await; - let issuer = thread_id(30); - let agent = thread_id(31); - let mut too_large = native_work(issuer, agent, "too-large", "large-submission", "x"); - too_large.input.content = "x".repeat(1_000_001); - - assert!(matches!( - fixture.mailbox.accept_work(too_large).await, - Err(MailboxError::Backpressure(_)) - )); - assert!(fixture - .mailbox - .work_for_native_key(issuer, "too-large") - .await - .unwrap() - .is_none()); -} - #[test] -fn closed_peer_cycles_require_every_join_choice_to_remain_inside_the_cycle() { - let any_cycle = [ - Dependency { - owner: "a".to_string(), - peers: vec!["b".to_string(), "c".to_string()], - mode: DependencyMode::Any, - }, - Dependency { - owner: "b".to_string(), - peers: vec!["a".to_string()], - mode: DependencyMode::All, - }, - Dependency { - owner: "c".to_string(), - peers: vec!["a".to_string()], - mode: DependencyMode::All, - }, +fn only_closed_any_and_all_peer_cycles_are_provable() { + let closed = [ + dependency("a", &["b", "c"], DependencyMode::Any), + dependency("b", &["a"], DependencyMode::All), + dependency("c", &["a"], DependencyMode::All), + dependency("x", &["y", "z"], DependencyMode::All), + dependency("y", &["x"], DependencyMode::Any), + dependency("z", &["x"], DependencyMode::Any), ]; - assert!(is_provably_closed_peer_cycle("a", &any_cycle)); - - let all_cycle = [ - Dependency { - owner: "x".to_string(), - peers: vec!["y".to_string(), "z".to_string()], - mode: DependencyMode::All, - }, - Dependency { - owner: "y".to_string(), - peers: vec!["x".to_string()], - mode: DependencyMode::Any, - }, - Dependency { - owner: "z".to_string(), - peers: vec!["x".to_string()], - mode: DependencyMode::Any, - }, + assert!(is_provably_closed_peer_cycle("a", &closed)); + assert!(is_provably_closed_peer_cycle("x", &closed)); + let open = [ + dependency("a", &["b", "outside"], DependencyMode::Any), + dependency("b", &["a"], DependencyMode::All), ]; - assert!(is_provably_closed_peer_cycle("x", &all_cycle)); - - let with_open_exit = [ - Dependency { - owner: "a".to_string(), - peers: vec!["b".to_string(), "not-in-cycle".to_string()], - mode: DependencyMode::Any, - }, - Dependency { - owner: "b".to_string(), - peers: vec!["a".to_string()], - mode: DependencyMode::All, - }, + assert!(!is_provably_closed_peer_cycle("a", &open)); + let open_all = [ + dependency("x", &["y", "outside"], DependencyMode::All), + dependency("y", &["x"], DependencyMode::Any), ]; - assert!(!is_provably_closed_peer_cycle("a", &with_open_exit)); + assert!(!is_provably_closed_peer_cycle("x", &open_all)); +} - let all_with_open_exit = [ - Dependency { - owner: "x".to_string(), - peers: vec!["y".to_string(), "not-in-cycle".to_string()], - mode: DependencyMode::All, - }, - Dependency { - owner: "y".to_string(), - peers: vec!["x".to_string()], - mode: DependencyMode::Any, - }, - ]; - assert!(!is_provably_closed_peer_cycle("x", &all_with_open_exit)); +#[tokio::test] +async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { + let fixture = test_mailbox().await; + let (owner_a, owner_b) = (thread_id(22), thread_id(23)); + let wait_a = fixture.mailbox.begin_wait( + "wait-a".to_string(), owner_a, vec![owner_b], ReturnWhen::Any, false, + ).unwrap(); + assert_eq!(wait_a.wait_id(), "wait-a"); + assert!(matches!(fixture.mailbox.begin_wait( + "closed-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, + ), Err(MailboxError::ClosedPeerCycle))); + assert!(matches!(fixture.mailbox.begin_wait( + "still-closed-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, + ), Err(MailboxError::ClosedPeerCycle))); + let escape = fixture.mailbox.begin_wait( + "escape-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, true, + ).unwrap(); + drop(escape); + assert!(matches!(fixture.mailbox.begin_wait( + "closed-again-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, + ), Err(MailboxError::ClosedPeerCycle))); + drop(wait_a); + assert!(fixture.mailbox.begin_wait( + "open-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, + ).is_ok()); } diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index e2d406c0052e..b856639f1957 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -222,6 +222,57 @@ impl Mailbox { Ok(selected) } + /// Load only causal envelopes attached to work admitted in this exact + /// native turn. Unscoped quiet notes are intentionally excluded. + pub async fn automatic_inputs( + &self, + recipient: ThreadId, + source_turn_id: &str, + limit: usize, + ) -> MailboxResult> { + if limit == 0 || limit > 2 { + return Err(MailboxError::InvalidInput( + "automatic input limit must be between 1 and 2".into(), + )); + } + let rows = sqlx::query( + "SELECT message.* FROM mailbox_messages AS message + JOIN mailbox_work AS work ON work.id = message.work_id + JOIN mailbox_admissions AS admission ON admission.work_id = work.id + WHERE admission.recipient = ? AND admission.source_turn_id = ? + AND work.agent = ? AND message.recipient = ? + AND message.input_admitted = 0 AND message.superseded_by IS NULL + AND message.withdrawn = 0 + ORDER BY message.seq LIMIT ?", + ) + .bind(recipient.to_string()) + .bind(source_turn_id) + .bind(recipient.to_string()) + .bind(recipient.to_string()) + .bind(limit as i64) + .fetch_all(&self.pool) + .await?; + let envelopes = rows + .iter() + .map(envelope_from_row) + .collect::>>()?; + let bytes = envelopes.iter().fold(0_usize, |total, envelope| { + total + .saturating_add(envelope.communication.content.len()) + .saturating_add( + envelope + .communication + .encrypted_content + .as_ref() + .map_or(0, String::len), + ) + }); + if bytes > 16_384 { + return Err(MailboxError::Backpressure("automatic input byte limit")); + } + Ok(envelopes) + } + /// Exact bounded list of accepted native-work input IDs not yet admitted. pub async fn unadmitted_native_input_ids( &self, diff --git a/codex-rs/agent-mailbox/src/message_rules.rs b/codex-rs/agent-mailbox/src/message_rules.rs index 7432f086cb9d..e276ec7a5d9b 100644 --- a/codex-rs/agent-mailbox/src/message_rules.rs +++ b/codex-rs/agent-mailbox/src/message_rules.rs @@ -8,7 +8,8 @@ pub(crate) async fn validate_reply_tx( draft: &MessageDraft, ) -> MailboxResult<()> { let request = sqlx::query( - "SELECT sender, recipient, kind, canonical FROM mailbox_messages WHERE id = ?", + "SELECT sender, recipient, kind, canonical, superseded_by, withdrawn + FROM mailbox_messages WHERE id = ?", ) .bind(request_id) .fetch_optional(&mut **transaction) @@ -25,6 +26,11 @@ pub(crate) async fn validate_reply_tx( if request.try_get::("canonical")? == 0 { return Err(MailboxError::NotCanonical); } + if request.try_get::, _>("superseded_by")?.is_some() + || request.try_get::("withdrawn")? != 0 + { + return Err(MailboxError::RequestAlreadyResolved); + } let resolved: Option = sqlx::query_scalar( "SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1", ) @@ -61,12 +67,14 @@ pub(crate) async fn validate_replacement_tx( let already_replaced: Option = old.try_get("superseded_by")?; let old_kind: String = old.try_get("kind")?; let valid_pair = match old_kind.as_str() { - "\"progress\"" => draft.kind == MessageKind::Progress, + "\"progress\"" => { + draft.kind == MessageKind::Progress + && old.try_get::("canonical")? == 0 + } "\"request\"" => matches!(draft.kind, MessageKind::Request | MessageKind::Note), _ => false, }; if !valid_pair - || old.try_get::("canonical")? != 0 || already_replaced.is_some() || old.try_get::("withdrawn")? != 0 { diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index db32d51a9ee6..a0d7f95c4006 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -207,8 +207,8 @@ pub(crate) async fn insert_message_tx( "INSERT INTO mailbox_messages ( id, invocation_key, request_json, sender, recipient, author_path, recipient_path, communication_json, - kind, work_ref_json, reply_to, supersedes, activate_if_idle, legacy_attention - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + kind, work_ref_json, work_id, reply_to, supersedes, activate_if_idle, legacy_attention + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ) .bind(&id) .bind(&draft.invocation_key) @@ -220,6 +220,7 @@ pub(crate) async fn insert_message_tx( .bind(&communication_json) .bind(kind_json) .bind(draft.work_ref.as_ref().map(serde_json::to_string).transpose()?) + .bind(draft.work_ref.as_ref().map(|work_ref| work_ref.id.as_str())) .bind(&draft.reply_to) .bind(&draft.supersedes) .bind(i64::from(draft.activate_if_idle)) @@ -238,22 +239,32 @@ pub(crate) async fn insert_message_tx( .await?; add_invocation_tx(transaction, &draft.invocation_key, &request_json, &id).await?; if let Some(old_id) = draft.supersedes.as_deref() { - let old_quiet = sqlx::query_scalar::<_, i64>( - "SELECT CASE WHEN legacy_attention = 0 THEN 1 ELSE 0 END - FROM mailbox_messages WHERE id = ? AND canonical = 0", + let old = sqlx::query( + "SELECT recipient, canonical, input_admitted, legacy_attention + FROM mailbox_messages + WHERE id = ? AND superseded_by IS NULL AND withdrawn = 0", ) .bind(old_id) .fetch_optional(&mut **transaction) - .await? - .unwrap_or(0) - != 0; - let replaced = sqlx::query("UPDATE mailbox_messages SET superseded_by = ? WHERE id = ? AND canonical = 0") + .await?; + let replaced = sqlx::query( + "UPDATE mailbox_messages SET superseded_by = ? + WHERE id = ? AND superseded_by IS NULL AND withdrawn = 0", + ) .bind(&id) .bind(old_id) .execute(&mut **transaction) .await?; if replaced.rows_affected() == 1 { - decrement_pending_count(transaction, &draft.recipient.to_string(), old_quiet).await?; + if let Some(old) = old { + let unread = old.try_get::("canonical")? == 0 + || old.try_get::("input_admitted")? == 0; + if unread { + let recipient: String = old.try_get("recipient")?; + let quiet = old.try_get::("legacy_attention")? == 0; + decrement_pending_count(transaction, &recipient, quiet).await?; + } + } } } sqlx::query("UPDATE mailbox_counts SET item_count = item_count + 1 WHERE name = 'message'") diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index c59b2f7beb13..b6248194f9eb 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -1,57 +1,24 @@ use crate::error::{MailboxError, MailboxResult}; -use crate::messages::insert_message_tx; +use crate::outcome_messages::{ + insert_response_group, response_groups, validate_auto_response_size, ResponseGroup, +}; +use crate::outcome_store::{ + outcome_receipt_for, persist_continuation, save_outcome_tx, OutcomeData, + MAX_CONTINUATIONS_PER_WORK, +}; use crate::store::Mailbox; use crate::types::{ - Continuation, ContinuationReceipt, ContinuationState, GoalBinding, MessageDraft, - MessageKind, Outcome, OutcomeReceipt, TurnOutcomeReceipt, WorkRef, + Continuation, GoalBinding, Outcome, OutcomeReceipt, TurnOutcomeReceipt, WorkRef, }; use crate::work::{ - ensure_mailbox_id, exact_work, exact_work_tx, new_mailbox_id, work_ref_from_row, + ensure_mailbox_id, exact_work_tx, new_mailbox_id, work_ref_from_row, }; use codex_protocol::protocol::InterAgentCommunication; -use codex_protocol::{AgentPath, ThreadId}; -use serde::{Deserialize, Serialize}; -use sqlx::{Row, Sqlite, SqlitePool, Transaction}; +use codex_protocol::ThreadId; +use sqlx::Row; const MAX_TURN_INPUTS: usize = 100; const MAX_KEY_BYTES: usize = 256; -const MAX_AUTO_RESPONSE_BYTES: usize = 16_384; -const MAX_RESPONSE_BYTES: usize = 8_192; -const MAX_CONTINUATIONS_PER_WORK: i64 = 32; - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct OutcomeData { - outcome: Outcome, - continuation: Continuation, - response: Option, -} - -impl OutcomeData { - fn new( - outcome: Outcome, - continuation: Continuation, - mut response: Option, - ) -> Self { - if let Some(response) = response.as_mut() { - response.id = None; - } - Self { - outcome, - continuation, - response, - } - } -} - -struct ResponseGroup { - issuer: ThreadId, - agent: ThreadId, - author: AgentPath, - recipient: AgentPath, - refs: Vec, - response: Option, - output_id: Option, -} impl Mailbox { /// Freeze and record only the work admitted to this native turn. A pending @@ -381,14 +348,19 @@ impl Mailbox { } if let Continuation::Terminal(binding) = &continuation { let binding_json = serde_json::to_string(binding)?; - let unresolved: Option = sqlx::query_scalar( + let unresolved = sqlx::query_scalar::<_, String>( "SELECT binding_json FROM mailbox_continuations - WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') LIMIT 1", + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') + ORDER BY binding_json LIMIT ?", ) .bind(&work_ref.id) - .fetch_optional(&mut *transaction) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(&mut *transaction) .await?; - if unresolved.as_deref().is_some_and(|stored| stored != binding_json) { + if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation count")); + } + if !unresolved.is_empty() && !unresolved.contains(&binding_json) { return Err(MailboxError::ReferenceMismatch); } } @@ -419,267 +391,6 @@ impl Mailbox { } } -async fn response_groups( - transaction: &mut Transaction<'_, Sqlite>, - refs: &[WorkRef], - response: Option<&InterAgentCommunication>, -) -> MailboxResult> { - let mut groups: Vec = Vec::new(); - for work_ref in refs { - let row = exact_work_tx(transaction, work_ref).await?; - let input: InterAgentCommunication = - serde_json::from_str(&row.try_get::("input_json")?)?; - let group = groups - .iter_mut() - .find(|group| group.issuer == work_ref.issuer); - if let Some(group) = group { - if group.agent != work_ref.agent - || group.author != input.recipient - || group.recipient != input.author - { - return Err(MailboxError::InvalidInput( - "issuer has inconsistent accepted communication paths".into(), - )); - } - group.refs.push(work_ref.clone()); - } else { - let mut response = response.cloned(); - if let Some(response) = response.as_mut() { - response.author = input.recipient.clone(); - response.recipient = input.author.clone(); - response.id = None; - } - groups.push(ResponseGroup { - issuer: work_ref.issuer, - agent: work_ref.agent, - author: input.recipient, - recipient: input.author, - refs: vec![work_ref.clone()], - response, - output_id: None, - }); - } - } - Ok(groups) -} - -fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxResult<()> { - let mut total = 0_usize; - for group in groups { - if let Some(response) = group.response.as_ref() { - let bytes = response.content.len() + response.encrypted_content.as_ref().map_or(0, String::len); - if bytes > MAX_RESPONSE_BYTES { - return Err(MailboxError::Backpressure("response message byte limit")); - } - total = total.saturating_add(bytes); - } - } - if total > MAX_AUTO_RESPONSE_BYTES { - return Err(MailboxError::Backpressure("automatic response aggregate byte limit")); - } - Ok(()) -} - -async fn insert_response_group( - transaction: &mut Transaction<'_, Sqlite>, - group: &mut ResponseGroup, - invocation_key: String, - outcome: Outcome, -) -> MailboxResult> { - let Some(mut communication) = group.response.clone() else { - return Ok(None); - }; - communication.author = group.author.clone(); - communication.recipient = group.recipient.clone(); - communication.id = None; - let envelope = insert_message_tx( - transaction, - MessageDraft { - invocation_key, - sender: group.agent, - recipient: group.issuer, - communication, - kind: MessageKind::Note, - activate_if_idle: false, - work_ref: None, - reply_to: None, - supersedes: None, - legacy_attention: outcome == Outcome::NeedsAttention, - }, - None, - ) - .await?; - group.output_id = Some(envelope.id.clone()); - Ok(group.output_id.clone()) -} - -async fn persist_continuation( - transaction: &mut Transaction<'_, Sqlite>, - work_ref: &WorkRef, - binding_json: &str, - state: &str, - outcome: Outcome, - source_turn_id: &str, - response_message_id: Option<&str>, -) -> MailboxResult<()> { - let exists: Option = sqlx::query_scalar( - "SELECT 1 FROM mailbox_continuations WHERE work_id = ? AND binding_json = ? LIMIT 1", - ) - .bind(&work_ref.id) - .bind(binding_json) - .fetch_optional(&mut **transaction) - .await?; - if exists.is_none() { - let count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM mailbox_continuations WHERE work_id = ?", - ) - .bind(&work_ref.id) - .fetch_one(&mut **transaction) - .await?; - if count >= MAX_CONTINUATIONS_PER_WORK { - return Err(MailboxError::Backpressure("work continuation count")); - } - } - sqlx::query( - "INSERT INTO mailbox_continuations( - work_id, binding_json, state, outcome, source_turn_id, response_message_id - ) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(work_id, binding_json) DO UPDATE SET - state = CASE - WHEN mailbox_continuations.state = 'terminal' THEN 'terminal' - WHEN excluded.state = 'deferred' THEN 'deferred' - ELSE mailbox_continuations.state - END, - outcome = excluded.outcome, - response_message_id = COALESCE(excluded.response_message_id, - mailbox_continuations.response_message_id)", - ) - .bind(&work_ref.id) - .bind(binding_json) - .bind(state) - .bind(outcome_label(outcome)) - .bind(source_turn_id) - .bind(response_message_id) - .execute(&mut **transaction) - .await?; - Ok(()) -} - -pub(crate) async fn save_outcome_tx( - transaction: &mut Transaction<'_, Sqlite>, - work_ref: &WorkRef, - payload: &str, - outcome_id: &str, - outcome: Outcome, -) -> MailboxResult<()> { - let result = sqlx::query( - "UPDATE mailbox_work SET outcome_json = ?, outcome_id = ? - WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ? - AND outcome_json IS NULL", - ) - .bind(payload) - .bind(outcome_id) - .bind(&work_ref.id) - .bind(work_ref.issuer.to_string()) - .bind(work_ref.agent.to_string()) - .bind(&work_ref.submission_id) - .execute(&mut **transaction) - .await?; - if result.rows_affected() == 0 { - return Err(MailboxError::ReferenceMismatch); - } - if outcome == Outcome::NeedsAttention { - sqlx::query( - "INSERT INTO mailbox_attention(id, recipient) VALUES (?, ?) - ON CONFLICT(id) DO NOTHING", - ) - .bind(format!("attention:{}", work_ref.id)) - .bind(work_ref.issuer.to_string()) - .execute(&mut **transaction) - .await?; - } - Ok(()) -} - -pub(crate) async fn outcome_receipt_for( - pool: &SqlitePool, - work_ref: WorkRef, - output_id: Option, - has_response: bool, -) -> MailboxResult { - let row = exact_work(pool, &work_ref).await?; - let stored: String = row - .try_get::, _>("outcome_json")? - .ok_or(MailboxError::ReferenceMismatch)?; - let mut data: OutcomeData = serde_json::from_str(&stored)?; - let response_message_id = if has_response { - output_id.or_else(|| row.try_get::, _>("outcome_id").ok().flatten()) - } else { - None - }; - if let (Some(response), Some(id)) = (data.response.as_mut(), response_message_id.as_ref()) { - response.id = Some(codex_protocol::ResponseItemId::from_server(id.clone())); - } - Ok(OutcomeReceipt { - work_ref, - id: row.try_get("outcome_id")?, - outcome: data.outcome, - continuation: data.continuation, - response: data.response, - response_message_id, - canonical: row.try_get::("outcome_canonical")? != 0, - observed: row.try_get::("outcome_observed")? != 0, - }) -} - -pub(crate) async fn stored_outcome_receipt_for( - pool: &SqlitePool, - work_ref: WorkRef, -) -> MailboxResult { - let row = exact_work(pool, &work_ref).await?; - let has_response = serde_json::from_str::( - &row.try_get::("outcome_json")?, - )? - .response - .is_some(); - outcome_receipt_for(pool, work_ref, None, has_response).await -} - -pub(crate) async fn continuation_receipts_for( - pool: &SqlitePool, - work_ref: &WorkRef, -) -> MailboxResult> { - let rows = sqlx::query( - "SELECT binding_json, state, outcome, source_turn_id, response_message_id - FROM mailbox_continuations - WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') - ORDER BY source_turn_id LIMIT ?", - ) - .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) - .fetch_all(pool) - .await?; - if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { - return Err(MailboxError::Backpressure("work continuation view limit")); - } - rows.iter() - .map(|row| { - let state = match row.try_get::("state")?.as_str() { - "pending_goal_binding" => ContinuationState::PendingGoalBinding, - "deferred" => ContinuationState::Deferred, - _ => return Err(MailboxError::InvalidInput("stored continuation state".into())), - }; - Ok(ContinuationReceipt { - binding: serde_json::from_str(&row.try_get::("binding_json")?)?, - state, - outcome: parse_outcome(row.try_get::("outcome")?.as_str())?, - source_turn_id: row.try_get("source_turn_id")?, - response_message_id: row.try_get("response_message_id")?, - }) - }) - .collect() -} - fn is_nonterminal(continuation: &Continuation) -> bool { matches!( continuation, @@ -704,25 +415,6 @@ fn continuation_state(continuation: &Continuation) -> &'static str { } } -fn outcome_label(outcome: Outcome) -> &'static str { - match outcome { - Outcome::Completed => "completed", - Outcome::Failed => "failed", - Outcome::Cancelled => "cancelled", - Outcome::NeedsAttention => "needs_attention", - } -} - -fn parse_outcome(value: &str) -> MailboxResult { - match value { - "completed" => Ok(Outcome::Completed), - "failed" => Ok(Outcome::Failed), - "cancelled" => Ok(Outcome::Cancelled), - "needs_attention" => Ok(Outcome::NeedsAttention), - _ => Err(MailboxError::InvalidInput("stored continuation outcome".into())), - } -} - fn validate_binding_for_agent(binding: &GoalBinding, agent: ThreadId) -> MailboxResult<()> { if binding.child_thread_id != agent { return Err(MailboxError::WrongRecipient); diff --git a/codex-rs/agent-mailbox/src/outcome_messages.rs b/codex-rs/agent-mailbox/src/outcome_messages.rs new file mode 100644 index 000000000000..4653ad62ef32 --- /dev/null +++ b/codex-rs/agent-mailbox/src/outcome_messages.rs @@ -0,0 +1,113 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::messages::insert_message_tx; +use crate::types::{MessageDraft, MessageKind, Outcome, WorkRef}; +use crate::work::exact_work_tx; +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::{AgentPath, ThreadId}; +use sqlx::{Row, Sqlite, Transaction}; + +const MAX_AUTO_RESPONSE_BYTES: usize = 16_384; +const MAX_RESPONSE_BYTES: usize = 8_192; + +pub(crate) struct ResponseGroup { + pub(crate) issuer: ThreadId, + agent: ThreadId, + author: AgentPath, + recipient: AgentPath, + pub(crate) response: Option, + pub(crate) output_id: Option, +} + +pub(crate) async fn response_groups( + transaction: &mut Transaction<'_, Sqlite>, + refs: &[WorkRef], + response: Option<&InterAgentCommunication>, +) -> MailboxResult> { + let mut groups: Vec = Vec::new(); + for work_ref in refs { + let row = exact_work_tx(transaction, work_ref).await?; + let input: InterAgentCommunication = + serde_json::from_str(&row.try_get::("input_json")?)?; + let group = groups + .iter_mut() + .find(|group| group.issuer == work_ref.issuer); + if let Some(group) = group { + if group.agent != work_ref.agent + || group.author != input.recipient + || group.recipient != input.author + { + return Err(MailboxError::InvalidInput( + "issuer has inconsistent accepted communication paths".into(), + )); + } + } else { + let mut response = response.cloned(); + if let Some(response) = response.as_mut() { + response.author = input.recipient.clone(); + response.recipient = input.author.clone(); + response.id = None; + } + groups.push(ResponseGroup { + issuer: work_ref.issuer, + agent: work_ref.agent, + author: input.recipient, + recipient: input.author, + response, + output_id: None, + }); + } + } + Ok(groups) +} + +pub(crate) fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxResult<()> { + let mut total = 0_usize; + for group in groups { + if let Some(response) = group.response.as_ref() { + let bytes = response.content.len() + + response.encrypted_content.as_ref().map_or(0, String::len); + if bytes > MAX_RESPONSE_BYTES { + return Err(MailboxError::Backpressure("response message byte limit")); + } + total = total.saturating_add(bytes); + } + } + if total > MAX_AUTO_RESPONSE_BYTES { + return Err(MailboxError::Backpressure("automatic response aggregate byte limit")); + } + Ok(()) +} + +pub(crate) async fn insert_response_group( + transaction: &mut Transaction<'_, Sqlite>, + group: &mut ResponseGroup, + invocation_key: String, + outcome: Outcome, +) -> MailboxResult> { + let Some(mut communication) = group.response.clone() else { + return Ok(None); + }; + communication.author = group.author.clone(); + communication.recipient = group.recipient.clone(); + communication.id = None; + let envelope = insert_message_tx( + transaction, + MessageDraft { + invocation_key, + sender: group.agent, + recipient: group.issuer, + communication, + kind: MessageKind::Note, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention: outcome == Outcome::NeedsAttention, + }, + None, + ) + .await?; + group.output_id = Some(envelope.id.clone()); + Ok(group.output_id.clone()) +} + diff --git a/codex-rs/agent-mailbox/src/outcome_store.rs b/codex-rs/agent-mailbox/src/outcome_store.rs new file mode 100644 index 000000000000..4e396234270a --- /dev/null +++ b/codex-rs/agent-mailbox/src/outcome_store.rs @@ -0,0 +1,237 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::types::{ + Continuation, ContinuationReceipt, ContinuationState, Outcome, OutcomeReceipt, WorkRef, +}; +use crate::work::exact_work; +use codex_protocol::protocol::InterAgentCommunication; +use serde::{Deserialize, Serialize}; +use sqlx::{Row, Sqlite, SqlitePool, Transaction}; + +pub(crate) const MAX_CONTINUATIONS_PER_WORK: i64 = 32; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct OutcomeData { + pub(crate) outcome: Outcome, + pub(crate) continuation: Continuation, + pub(crate) response: Option, +} + +impl OutcomeData { + pub(crate) fn new( + outcome: Outcome, + continuation: Continuation, + mut response: Option, + ) -> Self { + if let Some(response) = response.as_mut() { + response.id = None; + } + Self { + outcome, + continuation, + response, + } + } +} + +pub(crate) async fn persist_continuation( + transaction: &mut Transaction<'_, Sqlite>, + work_ref: &WorkRef, + binding_json: &str, + state: &str, + outcome: Outcome, + source_turn_id: &str, + response_message_id: Option<&str>, +) -> MailboxResult<()> { + let existing = sqlx::query( + "SELECT state, outcome, source_turn_id, response_message_id + FROM mailbox_continuations WHERE work_id = ? AND binding_json = ? LIMIT 1", + ) + .bind(&work_ref.id) + .bind(binding_json) + .fetch_optional(&mut **transaction) + .await?; + let outcome_text = outcome_label(outcome); + if let Some(existing) = existing { + let old_state: String = existing.try_get("state")?; + let old_outcome: String = existing.try_get("outcome")?; + let old_source_turn: String = existing.try_get("source_turn_id")?; + let old_response: Option = existing.try_get("response_message_id")?; + if old_state == "terminal" + || old_outcome != outcome_text + || old_source_turn != source_turn_id + || old_response.as_deref() != response_message_id + { + return Err(MailboxError::IdempotencyConflict); + } + if old_state == "pending_goal_binding" && state == "deferred" { + sqlx::query( + "UPDATE mailbox_continuations SET state = 'deferred' + WHERE work_id = ? AND binding_json = ? AND state = 'pending_goal_binding'", + ) + .bind(&work_ref.id) + .bind(binding_json) + .execute(&mut **transaction) + .await?; + } + return Ok(()); + } + let count_rows = sqlx::query( + "SELECT 1 FROM mailbox_continuations WHERE work_id = ? LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(&mut **transaction) + .await?; + if count_rows.len() >= MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation count")); + } + sqlx::query( + "INSERT INTO mailbox_continuations( + work_id, binding_json, state, outcome, source_turn_id, response_message_id + ) VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(&work_ref.id) + .bind(binding_json) + .bind(state) + .bind(outcome_text) + .bind(source_turn_id) + .bind(response_message_id) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +pub(crate) async fn save_outcome_tx( + transaction: &mut Transaction<'_, Sqlite>, + work_ref: &WorkRef, + payload: &str, + outcome_id: &str, + outcome: Outcome, +) -> MailboxResult<()> { + let result = sqlx::query( + "UPDATE mailbox_work SET outcome_json = ?, outcome_id = ? + WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ? + AND outcome_json IS NULL", + ) + .bind(payload) + .bind(outcome_id) + .bind(&work_ref.id) + .bind(work_ref.issuer.to_string()) + .bind(work_ref.agent.to_string()) + .bind(&work_ref.submission_id) + .execute(&mut **transaction) + .await?; + if result.rows_affected() == 0 { + return Err(MailboxError::ReferenceMismatch); + } + if outcome == Outcome::NeedsAttention { + sqlx::query( + "INSERT INTO mailbox_attention(id, recipient) VALUES (?, ?) + ON CONFLICT(id) DO NOTHING", + ) + .bind(format!("attention:{}", work_ref.id)) + .bind(work_ref.issuer.to_string()) + .execute(&mut **transaction) + .await?; + } + Ok(()) +} + +pub(crate) async fn outcome_receipt_for( + pool: &SqlitePool, + work_ref: WorkRef, + output_id: Option, + has_response: bool, +) -> MailboxResult { + let row = exact_work(pool, &work_ref).await?; + let stored: String = row + .try_get::, _>("outcome_json")? + .ok_or(MailboxError::ReferenceMismatch)?; + let mut data: OutcomeData = serde_json::from_str(&stored)?; + let response_message_id = if has_response { + output_id.or_else(|| row.try_get::, _>("outcome_id").ok().flatten()) + } else { + None + }; + if let (Some(response), Some(id)) = (data.response.as_mut(), response_message_id.as_ref()) { + response.id = Some(codex_protocol::ResponseItemId::from_server(id.clone())); + } + Ok(OutcomeReceipt { + work_ref, + id: row.try_get("outcome_id")?, + outcome: data.outcome, + continuation: data.continuation, + response: data.response, + response_message_id, + canonical: row.try_get::("outcome_canonical")? != 0, + observed: row.try_get::("outcome_observed")? != 0, + }) +} + +pub(crate) async fn stored_outcome_receipt_for( + pool: &SqlitePool, + work_ref: WorkRef, +) -> MailboxResult { + let row = exact_work(pool, &work_ref).await?; + let has_response = serde_json::from_str::( + &row.try_get::("outcome_json")?, + )? + .response + .is_some(); + outcome_receipt_for(pool, work_ref, None, has_response).await +} + +pub(crate) async fn continuation_receipts_for( + pool: &SqlitePool, + work_ref: &WorkRef, +) -> MailboxResult> { + let rows = sqlx::query( + "SELECT binding_json, state, outcome, source_turn_id, response_message_id + FROM mailbox_continuations + WHERE work_id = ? + ORDER BY source_turn_id, binding_json LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(pool) + .await?; + if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation view limit")); + } + rows.iter() + .map(|row| { + let state = match row.try_get::("state")?.as_str() { + "pending_goal_binding" => ContinuationState::PendingGoalBinding, + "deferred" => ContinuationState::Deferred, + "terminal" => ContinuationState::Terminal, + _ => return Err(MailboxError::InvalidInput("stored continuation state".into())), + }; + Ok(ContinuationReceipt { + binding: serde_json::from_str(&row.try_get::("binding_json")?)?, + state, + outcome: parse_outcome(row.try_get::("outcome")?.as_str())?, + source_turn_id: row.try_get("source_turn_id")?, + response_message_id: row.try_get("response_message_id")?, + }) + }) + .collect() +} + +fn outcome_label(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Completed => "completed", + Outcome::Failed => "failed", + Outcome::Cancelled => "cancelled", + Outcome::NeedsAttention => "needs_attention", + } +} + +fn parse_outcome(value: &str) -> MailboxResult { + match value { + "completed" => Ok(Outcome::Completed), + "failed" => Ok(Outcome::Failed), + "cancelled" => Ok(Outcome::Cancelled), + "needs_attention" => Ok(Outcome::NeedsAttention), + _ => Err(MailboxError::InvalidInput("stored continuation outcome".into())), + } +} diff --git a/codex-rs/agent-mailbox/src/runtime_fault.rs b/codex-rs/agent-mailbox/src/runtime_fault.rs new file mode 100644 index 000000000000..330c62d5e3a4 --- /dev/null +++ b/codex-rs/agent-mailbox/src/runtime_fault.rs @@ -0,0 +1,61 @@ +use crate::store::Mailbox; +use codex_protocol::ThreadId; +use std::collections::HashSet; +use std::sync::Mutex; + +const MAX_RUNTIME_FAULT_AGENTS: usize = 4_096; + +#[derive(Default)] +pub(crate) struct RuntimeFaults { + agents: HashSet, + overflowed: bool, +} + +impl Mailbox { + /// Signal an in-memory observation that terminal custody could not settle. + /// This does not create or fabricate a durable work outcome. + pub fn notify_runtime_fault(&self, agent: ThreadId) { + let mut faults = lock(&self.runtime_faults); + let agent = agent.to_string(); + let changed = if faults.agents.contains(&agent) { + false + } else if faults.agents.len() < MAX_RUNTIME_FAULT_AGENTS { + faults.agents.insert(agent) + } else if !faults.overflowed { + // Preserve an honest wake if the bounded exact set fills. The + // conservative overflow indication remains until process restart. + faults.overflowed = true; + true + } else { + false + }; + drop(faults); + if changed { + self.notify_changed(); + } + } + + /// Check only the caller's bounded selected agent set for a reported fault. + /// Oversized selections conservatively wake rather than risk a silent hang. + pub fn has_runtime_fault(&self, agents: &[ThreadId]) -> bool { + if agents.len() > 100 { + return true; + } + let faults = lock(&self.runtime_faults); + faults.overflowed || agents.iter().any(|agent| faults.agents.contains(&agent.to_string())) + } + + /// Clear one exact agent's fault only after explicit adapter recovery. + pub fn clear_runtime_fault(&self, agent: ThreadId) -> bool { + let cleared = lock(&self.runtime_faults).agents.remove(&agent.to_string()); + if cleared { + self.notify_changed(); + } + cleared + } +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index d4504da38bc4..99a7f2a6a5ed 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -163,7 +163,6 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { &mut transaction, "mailbox_continuations", "outcome", - "TEXT NOT NULL DEFAULT 'needs_attention'", ) .await?; sqlx::query( @@ -192,6 +191,7 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { communication_json TEXT NOT NULL, kind TEXT NOT NULL, work_ref_json TEXT, + work_id TEXT REFERENCES mailbox_work(id), reply_to TEXT, supersedes TEXT, superseded_by TEXT, @@ -212,42 +212,42 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { &mut transaction, "mailbox_messages", "withdrawn", - "INTEGER NOT NULL DEFAULT 0", ) .await?; ensure_column( &mut transaction, "mailbox_messages", "activate_if_idle", - "INTEGER NOT NULL DEFAULT 0", ) .await?; ensure_column( &mut transaction, "mailbox_messages", "input_admitted", - "INTEGER NOT NULL DEFAULT 0", ) .await?; ensure_column( &mut transaction, "mailbox_messages", "author_path", - "TEXT NOT NULL DEFAULT ''", ) .await?; ensure_column( &mut transaction, "mailbox_messages", "recipient_path", - "TEXT NOT NULL DEFAULT ''", + ) + .await?; + ensure_column( + &mut transaction, + "mailbox_messages", + "work_id", ) .await?; ensure_column( &mut transaction, "mailbox_pending_counts", "quiet_count", - "INTEGER NOT NULL DEFAULT 0", ) .await?; if version.is_some_and(|version| version < SCHEMA_VERSION) { @@ -298,6 +298,12 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_admitted_work + ON mailbox_messages(recipient, work_id, input_admitted, seq)", + ) + .execute(&mut *transaction) + .await?; sqlx::query( "CREATE INDEX IF NOT EXISTS mailbox_messages_reply_lookup ON mailbox_messages(reply_to, recipient, seq)", @@ -361,10 +367,14 @@ async fn ensure_column( transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table: &str, column: &str, - declaration: &str, ) -> MailboxResult<()> { - let pragma = format!("PRAGMA table_info({table})"); - let rows = sqlx::query(&pragma).fetch_all(&mut **transaction).await?; + let pragma = match table { + "mailbox_continuations" => "PRAGMA table_info(mailbox_continuations)", + "mailbox_messages" => "PRAGMA table_info(mailbox_messages)", + "mailbox_pending_counts" => "PRAGMA table_info(mailbox_pending_counts)", + _ => return Err(crate::MailboxError::InvalidInput("unsupported mailbox schema table".into())), + }; + let rows = sqlx::query(pragma).fetch_all(&mut **transaction).await?; let exists = rows .iter() .map(|row| row.try_get::("name")) @@ -372,8 +382,34 @@ async fn ensure_column( .iter() .any(|name| name == column); if !exists { - let alter = format!("ALTER TABLE {table} ADD COLUMN {column} {declaration}"); - sqlx::query(&alter).execute(&mut **transaction).await?; + let alter = match (table, column) { + ("mailbox_continuations", "outcome") => { + "ALTER TABLE mailbox_continuations ADD COLUMN outcome TEXT NOT NULL DEFAULT 'needs_attention'" + } + ("mailbox_messages", "withdrawn") => { + "ALTER TABLE mailbox_messages ADD COLUMN withdrawn INTEGER NOT NULL DEFAULT 0" + } + ("mailbox_messages", "activate_if_idle") => { + "ALTER TABLE mailbox_messages ADD COLUMN activate_if_idle INTEGER NOT NULL DEFAULT 0" + } + ("mailbox_messages", "input_admitted") => { + "ALTER TABLE mailbox_messages ADD COLUMN input_admitted INTEGER NOT NULL DEFAULT 0" + } + ("mailbox_messages", "author_path") => { + "ALTER TABLE mailbox_messages ADD COLUMN author_path TEXT NOT NULL DEFAULT ''" + } + ("mailbox_messages", "recipient_path") => { + "ALTER TABLE mailbox_messages ADD COLUMN recipient_path TEXT NOT NULL DEFAULT ''" + } + ("mailbox_messages", "work_id") => { + "ALTER TABLE mailbox_messages ADD COLUMN work_id TEXT REFERENCES mailbox_work(id)" + } + ("mailbox_pending_counts", "quiet_count") => { + "ALTER TABLE mailbox_pending_counts ADD COLUMN quiet_count INTEGER NOT NULL DEFAULT 0" + } + _ => return Err(crate::MailboxError::InvalidInput("unsupported mailbox schema column".into())), + }; + sqlx::query(alter).execute(&mut **transaction).await?; } Ok(()) } diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs index f8d8661edcb2..e1038e56f7b7 100644 --- a/codex-rs/agent-mailbox/src/snapshot.rs +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -6,7 +6,7 @@ use crate::types::{ RequestDisposition, RequestSnapshot, Selector, Snapshot, SnapshotItem, SnapshotState, WorkSnapshot, }; -use crate::outcome::{continuation_receipts_for, stored_outcome_receipt_for}; +use crate::outcome_store::{continuation_receipts_for, stored_outcome_receipt_for}; use crate::work::{parse_dispatch_state, work_ref_from_row}; use codex_protocol::ThreadId; use sqlx::Row; @@ -30,6 +30,11 @@ impl Mailbox { "snapshot limit must be between 1 and 100".into(), )); } + if selection.limit < selection.selectors.len() { + return Err(MailboxError::InvalidInput( + "snapshot limit must cover every selected item".into(), + )); + } let mut items = Vec::new(); let mut matched = 0; let mut satisfied = 0; @@ -60,15 +65,15 @@ impl Mailbox { matched += 1; let work_ref = work_ref_from_row(&row)?; let observed = row.try_get::("outcome_observed")? != 0; - if observed { - satisfied += 1; - } let input: String = row.try_get("input_json")?; let outcome = if row.try_get::, _>("outcome_json")?.is_some() { Some(stored_outcome_receipt_for(&self.pool, work_ref.clone()).await?) } else { None }; + if outcome.is_some() { + satisfied += 1; + } let continuations = continuation_receipts_for(&self.pool, &work_ref).await?; if items.len() < selection.limit { items.push(SnapshotItem::Work(WorkSnapshot { @@ -86,10 +91,13 @@ impl Mailbox { Selector::Reply(id) => { let row = sqlx::query( "SELECT * FROM mailbox_messages - WHERE id = ? AND sender = ? AND kind = '\"request\"' LIMIT 1", + WHERE id = ? AND sender = ? AND kind = '\"request\"' + AND (? IS NULL OR recipient = ?) LIMIT 1", ) .bind(id) .bind(selection.caller.to_string()) + .bind(selection.agent.map(|agent| agent.to_string())) + .bind(selection.agent.map(|agent| agent.to_string())) .fetch_optional(&self.pool) .await?; let Some(row) = row else { @@ -333,7 +341,7 @@ async fn ensure_observation_target( pub fn is_provably_closed_peer_cycle(start: &str, dependencies: &[Dependency]) -> bool { let mut by_owner = HashMap::new(); for dependency in dependencies { - if dependency.peers.is_empty() || by_owner.contains_key(&dependency.owner) { + if dependency.peers.is_empty() || by_owner.contains_key(dependency.owner.as_str()) { return false; } by_owner.insert(dependency.owner.as_str(), dependency); @@ -347,10 +355,14 @@ pub fn is_provably_closed_peer_cycle(start: &str, dependencies: &[Dependency]) - return false; }; match dependency.mode { - DependencyMode::All | DependencyMode::Any => dependency + DependencyMode::Any => dependency .peers .iter() .all(|peer| previous.contains(peer.as_str())), + DependencyMode::All => dependency + .peers + .iter() + .any(|peer| previous.contains(peer.as_str())), } }); if closed.len() == previous_len { diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index df6a5a0e6191..fe9f0ac59426 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -1,5 +1,7 @@ use crate::error::MailboxResult; use crate::schema; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use sqlx::SqlitePool; use tokio::sync::watch; @@ -7,6 +9,8 @@ use tokio::sync::watch; pub struct Mailbox { pub(crate) pool: SqlitePool, change_tx: watch::Sender, + pub(crate) waits: Arc>>, + pub(crate) runtime_faults: Arc>, } impl Mailbox { @@ -16,7 +20,12 @@ impl Mailbox { let pool = config.open_read_write_pool(&path).await?; crate::schema::initialize(&pool).await?; let (change_tx, _) = watch::channel(0_u64); - Ok(Self { pool, change_tx }) + Ok(Self { + pool, + change_tx, + waits: Arc::new(Mutex::new(HashMap::new())), + runtime_faults: Arc::new(Mutex::new(crate::runtime_fault::RuntimeFaults::default())), + }) } /// Subscribe before querying a snapshot to avoid missing local mailbox writes. diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 02b64ebcad27..72377ce17f42 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -193,7 +193,7 @@ pub enum Selector { Reply(String), } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RequestDisposition { Open, diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs new file mode 100644 index 000000000000..3eb183441808 --- /dev/null +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -0,0 +1,133 @@ +use crate::error::{MailboxError, MailboxResult}; +use crate::store::Mailbox; +use crate::types::ReturnWhen; +use codex_protocol::ThreadId; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, Weak}; + +const MAX_ACTIVE_WAITS: usize = 4_096; +const MAX_PEERS_PER_WAIT: usize = 100; +const MAX_WAIT_ID_BYTES: usize = 256; + +#[derive(Clone)] +pub(crate) struct ActiveWait { + owner: String, + peers: Vec, + mode: ReturnWhen, + has_escape: bool, +} + +/// RAII handle for one transient wait dependency. Dropping it removes only the +/// exact registration; restarting the process never re-arms persisted waits. +pub struct WaitRegistration { + wait_id: String, + waits: Weak>>, +} + +impl WaitRegistration { + pub fn wait_id(&self) -> &str { + &self.wait_id + } +} + +impl Drop for WaitRegistration { + fn drop(&mut self) { + if let Some(waits) = self.waits.upgrade() { + let mut waits = lock(&waits); + waits.remove(&self.wait_id); + } + } +} + +impl Mailbox { + /// Register only currently active peer dependencies. The caller sets + /// `has_escape` for a finite deadline or any already-eligible input. + /// A new registration is rejected only when the graph proves it closed. + pub fn begin_wait( + &self, + wait_id: String, + owner: ThreadId, + peers: Vec, + mode: ReturnWhen, + has_escape: bool, + ) -> MailboxResult { + if wait_id.is_empty() || wait_id.len() > MAX_WAIT_ID_BYTES { + return Err(MailboxError::InvalidInput("invalid wait registration ID".into())); + } + if peers.len() > MAX_PEERS_PER_WAIT { + return Err(MailboxError::Backpressure("wait dependency count")); + } + let peers = peers + .into_iter() + .map(|peer| peer.to_string()) + .collect::>(); + if peers.iter().collect::>().len() != peers.len() { + return Err(MailboxError::InvalidInput("duplicate wait dependency".into())); + } + let mut waits = lock(&self.waits); + if waits.contains_key(&wait_id) { + return Err(MailboxError::IdempotencyConflict); + } + if waits.len() >= MAX_ACTIVE_WAITS { + return Err(MailboxError::Backpressure("active wait registration count")); + } + waits.insert( + wait_id.clone(), + ActiveWait { + owner: owner.to_string(), + peers, + mode, + has_escape, + }, + ); + if is_closed_owner(&waits, &owner.to_string()) { + waits.remove(&wait_id); + return Err(MailboxError::ClosedPeerCycle); + } + Ok(WaitRegistration { + wait_id, + waits: Arc::downgrade(&self.waits), + }) + } +} + +fn is_closed_owner(waits: &HashMap, owner: &str) -> bool { + let mut by_owner: HashMap<&str, Vec<&ActiveWait>> = HashMap::new(); + for wait in waits.values() { + by_owner.entry(wait.owner.as_str()).or_default().push(wait); + } + let mut closed: HashSet<&str> = by_owner + .iter() + .filter(|(_, owner_waits)| { + owner_waits + .iter() + .all(|wait| !wait.has_escape && !wait.peers.is_empty()) + }) + .map(|(owner, _)| *owner) + .collect(); + loop { + let previous = closed.clone(); + let previous_len = closed.len(); + closed.retain(|candidate| { + let Some(owner_waits) = by_owner.get(candidate) else { + return false; + }; + owner_waits.iter().all(|wait| match wait.mode { + // Any completes when one alternative is ready, so every peer + // must itself be blocked for this waiter to remain trapped. + ReturnWhen::Any => wait.peers.iter().all(|peer| previous.contains(peer.as_str())), + // All remains blocked while at least one required peer is + // blocked; other completed peers do not break the dependency. + ReturnWhen::All => wait.peers.iter().any(|peer| previous.contains(peer.as_str())), + }) + }); + if closed.len() == previous_len { + break; + } + } + closed.contains(owner) +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index a52f3c598c89..3ea661c4a8f2 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -2456,6 +2456,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; let wire = serde_json::to_value(item)?; assert_eq!(wire["wakeNotifications"], Value::Null); diff --git a/codex-rs/app-server-protocol/src/protocol/collab_agent_lifecycle_tests.rs b/codex-rs/app-server-protocol/src/protocol/collab_agent_lifecycle_tests.rs index dd4d8fbad656..b409075c6fc1 100644 --- a/codex-rs/app-server-protocol/src/protocol/collab_agent_lifecycle_tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/collab_agent_lifecycle_tests.rs @@ -537,6 +537,7 @@ fn collab_item( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } } diff --git a/codex-rs/app-server-protocol/src/protocol/event_mapping.rs b/codex-rs/app-server-protocol/src/protocol/event_mapping.rs index a0f39862cb7d..83e949229fdc 100644 --- a/codex-rs/app-server-protocol/src/protocol/event_mapping.rs +++ b/codex-rs/app-server-protocol/src/protocol/event_mapping.rs @@ -98,6 +98,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemStarted(ItemStartedNotification { thread_id, @@ -149,6 +150,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemCompleted(ItemCompletedNotification { thread_id, @@ -176,6 +178,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemStarted(ItemStartedNotification { thread_id, @@ -211,6 +214,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemCompleted(ItemCompletedNotification { thread_id, @@ -258,6 +262,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: begin_event.mailbox_wait_report.clone(), }; ServerNotification::ItemStarted(ItemStartedNotification { thread_id, @@ -306,6 +311,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: end_event.mailbox_wait_report.clone(), }; ServerNotification::ItemCompleted(ItemCompletedNotification { thread_id, @@ -332,6 +338,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemStarted(ItemStartedNotification { thread_id, @@ -372,6 +379,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemCompleted(ItemCompletedNotification { thread_id, @@ -398,6 +406,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemStarted(ItemStartedNotification { thread_id, @@ -438,6 +447,7 @@ pub fn item_event_to_server_notification( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; ServerNotification::ItemCompleted(ItemCompletedNotification { thread_id, @@ -629,6 +639,97 @@ mod tests { ); } + #[test] + fn legacy_collab_wait_events_keep_mailbox_report_in_live_items() { + let sender_thread_id = ThreadId::new(); + let receiver_thread_id = ThreadId::new(); + let active_report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["work_9".to_string()], + requested_replies_to: vec!["msg-request".to_string()], + return_when: codex_protocol::items::MailboxWaitReturnWhen::All, + completed: Vec::new(), + pending: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_9".to_string()), + message_id: None, + agent: Some("worker-A".to_string()), + status: "running".to_string(), + }], + completion_reason: None, + condition_satisfied: false, + quiet_count: 0, + wake_sender: None, + wake_ref: None, + }; + let notification = item_event_to_server_notification( + EventMsg::CollabWaitingBegin(codex_protocol::protocol::CollabWaitingBeginEvent { + started_at_ms: 1, + sender_thread_id, + receiver_thread_ids: vec![receiver_thread_id], + receiver_agents: Vec::new(), + call_id: "wait-call".to_string(), + mailbox_wait_report: Some(active_report.clone()), + }), + "thread-live", + "turn-live", + ) + .expect("legacy waiting begin maps to a notification"); + let ServerNotification::ItemStarted(notification) = notification else { + panic!("legacy waiting begin should start an item"); + }; + let ThreadItem::CollabAgentToolCall { + mailbox_wait_report, + .. + } = notification.item + else { + panic!("legacy waiting begin should map to a collab wait item"); + }; + assert_eq!(mailbox_wait_report, Some(active_report.clone())); + + let terminal_report = codex_protocol::items::MailboxWaitReport { + requested_targets: active_report.requested_targets, + requested_replies_to: active_report.requested_replies_to, + return_when: active_report.return_when, + completed: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_9".to_string()), + message_id: Some("msg-reply".to_string()), + agent: Some("worker-A".to_string()), + status: "replied".to_string(), + }], + pending: Vec::new(), + completion_reason: Some(codex_protocol::items::MailboxWakeCause::Reply), + condition_satisfied: true, + quiet_count: 0, + wake_sender: Some("worker-A".to_string()), + wake_ref: Some("msg-reply".to_string()), + }; + let notification = item_event_to_server_notification( + EventMsg::CollabWaitingEnd(codex_protocol::protocol::CollabWaitingEndEvent { + sender_thread_id, + call_id: "wait-call".to_string(), + completed_at_ms: 2, + agent_statuses: Vec::new(), + statuses: [(receiver_thread_id, AgentStatus::Completed(None))] + .into_iter() + .collect(), + mailbox_wait_report: Some(terminal_report.clone()), + }), + "thread-live", + "turn-live", + ) + .expect("legacy waiting end maps to a notification"); + let ServerNotification::ItemCompleted(notification) = notification else { + panic!("legacy waiting end should complete an item"); + }; + let ThreadItem::CollabAgentToolCall { + mailbox_wait_report, + .. + } = notification.item + else { + panic!("legacy waiting end should map to a collab wait item"); + }; + assert_eq!(mailbox_wait_report, Some(terminal_report)); + } + #[test] fn collab_resume_begin_maps_to_item_started_resume_agent() { let event = CollabResumeBeginEvent { @@ -668,6 +769,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }, ); @@ -704,6 +806,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }), "thread-phase-compatible", @@ -758,6 +861,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }), "thread-phase-compatible", @@ -864,6 +968,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }), "thread-unknown-terminal", @@ -943,6 +1048,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }, ); @@ -976,6 +1082,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; @@ -1028,6 +1135,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }, ); @@ -1064,6 +1172,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; @@ -1116,6 +1225,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }, ); diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 68d0da8935be..31a079719164 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -901,6 +901,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; self.upsert_item_in_current_turn(item); } @@ -947,6 +948,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); } @@ -971,6 +973,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; self.upsert_item_in_current_turn(item); } @@ -1002,6 +1005,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); } @@ -1044,6 +1048,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: payload.mailbox_wait_report.clone(), }; self.upsert_item_in_current_turn(item); } @@ -1086,6 +1091,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: payload.mailbox_wait_report.clone(), }); } @@ -1110,6 +1116,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; self.upsert_item_in_current_turn(item); } @@ -1143,6 +1150,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); } @@ -1167,6 +1175,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; self.upsert_item_in_current_turn(item); } @@ -1203,6 +1212,7 @@ impl ThreadHistoryBuilder { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); } @@ -3970,6 +3980,40 @@ mod tests { let turn_id = "turn-collab"; let spawn_request_model = "gpt-requested"; let spawn_effective_model = "gpt-effective"; + let active_wait_report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["work_42".to_string()], + requested_replies_to: vec!["msg-request".to_string()], + return_when: codex_protocol::items::MailboxWaitReturnWhen::All, + completed: Vec::new(), + pending: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_42".to_string()), + message_id: None, + agent: Some("worker-A".to_string()), + status: "running".to_string(), + }], + completion_reason: None, + condition_satisfied: false, + quiet_count: 1, + wake_sender: None, + wake_ref: None, + }; + let completed_wait_report = codex_protocol::items::MailboxWaitReport { + requested_targets: active_wait_report.requested_targets.clone(), + requested_replies_to: active_wait_report.requested_replies_to.clone(), + return_when: active_wait_report.return_when, + completed: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_42".to_string()), + message_id: Some("msg-reply".to_string()), + agent: Some("worker-A".to_string()), + status: "replied".to_string(), + }], + pending: Vec::new(), + completion_reason: Some(codex_protocol::items::MailboxWakeCause::Reply), + condition_satisfied: true, + quiet_count: 1, + wake_sender: Some("worker-A".to_string()), + wake_ref: Some("msg-reply".to_string()), + }; let legacy = vec![ RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { @@ -4009,6 +4053,7 @@ mod tests { receiver_thread_ids: vec![child, waiting_child], receiver_agents: Vec::new(), call_id: "wait-1".to_string(), + mailbox_wait_report: Some(active_wait_report.clone()), })), RolloutItem::EventMsg(EventMsg::CollabWaitingEnd(CollabWaitingEndEvent { sender_thread_id: sender, @@ -4018,6 +4063,7 @@ mod tests { statuses: [(child, AgentStatus::Completed(Some("waited".to_string())))] .into_iter() .collect(), + mailbox_wait_report: Some(completed_wait_report.clone()), })), RolloutItem::EventMsg(EventMsg::CollabResumeBegin(CollabResumeBeginEvent { call_id: "resume-1".to_string(), @@ -4074,6 +4120,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: Some(active_wait_report.clone()), }, /*started*/ true, ), @@ -4096,6 +4143,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: Some(completed_wait_report.clone()), }, /*started*/ false, ), @@ -4118,6 +4166,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ true, ), @@ -4142,6 +4191,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ false, ), @@ -4164,6 +4214,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ true, ), @@ -4188,6 +4239,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ false, ), @@ -4201,6 +4253,28 @@ mod tests { let canonical_turns = build_turns_from_rollout_items(&canonical); assert_eq!(legacy_turns, canonical_turns); + let wait = legacy_turns[0] + .items + .iter() + .find(|item| { + matches!( + item, + ThreadItem::CollabAgentToolCall { + id, + tool: CollabAgentTool::Wait, + .. + } if id == "wait-1" + ) + }) + .expect("replay materializes the mailbox wait report"); + let ThreadItem::CollabAgentToolCall { + mailbox_wait_report, + .. + } = wait + else { + unreachable!("wait lookup must return the wait lifecycle item"); + }; + assert_eq!(mailbox_wait_report.as_ref(), Some(&completed_wait_report)); let spawn = legacy_turns[0] .items .iter() @@ -4273,6 +4347,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; let legacy_started = current_started @@ -4387,6 +4462,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; let legacy_started = pre_additive_started @@ -4426,6 +4502,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }; let Some(ServerNotification::ItemStarted(mapped_started)) = crate::protocol::event_mapping::item_event_to_server_notification( @@ -4487,6 +4564,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ false, ), @@ -4511,6 +4589,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ true, ), @@ -4562,6 +4641,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }] ); } @@ -4608,6 +4688,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }] ); } @@ -4652,6 +4733,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }] ); } @@ -4715,6 +4797,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -4796,6 +4879,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -4833,6 +4917,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; @@ -4891,6 +4976,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -4956,6 +5042,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -5036,6 +5123,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -5703,6 +5791,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ true, ), @@ -5738,6 +5827,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*started*/ false, ), diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index f89b5d73c0a7..0b8efbe77572 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -28,6 +28,7 @@ use codex_protocol::items::AgentNotificationContent as CoreAgentNotificationCont use codex_protocol::items::AgentNotificationOrigin as CoreAgentNotificationOrigin; use codex_protocol::items::AgentNotificationSummary as CoreAgentNotificationSummary; use codex_protocol::items::AgentWakeCause as CoreAgentWakeCause; +use codex_protocol::items::MailboxWaitReport as CoreMailboxWaitReport; use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool; use codex_protocol::items::CollabAgentToolCallStatus as CoreCollabAgentToolCallStatus; use codex_protocol::items::CommandExecutionStatus as CoreCommandExecutionStatus; @@ -497,6 +498,10 @@ pub enum ThreadItem { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] wake_cause: Option, + /// Selection-aware mailbox wait state/result, when reported by the producer. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + mailbox_wait_report: Option, }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] @@ -1165,6 +1170,7 @@ impl From for ThreadItem { .map(|notifications| notifications.into_iter().map(Into::into).collect()), completion_reason: call.completion_reason, wake_cause: call.wake_cause.map(Into::into), + mailbox_wait_report: call.mailbox_wait_report, } } CoreTurnItem::SubAgentActivity(activity) => ThreadItem::SubAgentActivity { diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 7731b5285f43..e084c26b9b37 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -2876,6 +2876,7 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); assert_eq!( @@ -2905,6 +2906,7 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); @@ -2924,6 +2926,7 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); assert_eq!( @@ -2947,6 +2950,7 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); @@ -3157,6 +3161,7 @@ fn v1_omitted_spawn_identity_sentinels_do_not_become_requested_identity() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); assert_eq!( @@ -3178,6 +3183,7 @@ fn v1_omitted_spawn_identity_sentinels_do_not_become_requested_identity() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } @@ -3201,6 +3207,7 @@ fn v1_model_only_spawn_does_not_invent_requested_effort() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }); assert_eq!( @@ -3222,6 +3229,7 @@ fn v1_model_only_spawn_does_not_invent_requested_effort() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); } diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index 9dd7a59c8c87..3eb2543da576 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -79,6 +79,7 @@ const INSPECT_AGENT_TREE_STATE_DB_UNAVAILABLE_MESSAGE: &str = concat!( mod execution; mod legacy; mod lifecycle; +mod mailbox; mod residency; mod spawn; @@ -224,6 +225,8 @@ pub(crate) struct AgentControl { state: Arc, v2_residency: Arc, agent_execution_limiter: Arc, + /// One durable mailbox per native agent tree; membership remains in `AgentRegistry`. + mailbox: Arc>>, /// Session-scoped state shared by the root thread and every cloned sub-agent control handle. rollout_budget: Arc, #[cfg(test)] @@ -1172,6 +1175,11 @@ impl AgentControl { child_reference: String, child_agent_path: Option, ) { + // The mailbox settles at the producer's authoritative terminal boundary. A status + // observer must not fabricate a second completion or lose typed goal disposition. + if self.mailbox_enabled() { + return; + } let Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id, .. })) = session_source diff --git a/codex-rs/core/src/agent/control/lifecycle.rs b/codex-rs/core/src/agent/control/lifecycle.rs index 1d02fa4b6c38..0d97e2891859 100644 --- a/codex-rs/core/src/agent/control/lifecycle.rs +++ b/codex-rs/core/src/agent/control/lifecycle.rs @@ -7,6 +7,8 @@ use crate::codex_thread::ThreadConfigSnapshot; use crate::config::Config; use crate::session::new_submission_id; use crate::thread_manager::ThreadManagerState; +use codex_agent_mailbox::DispatchDecision; +use codex_agent_mailbox::WorkRef; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; @@ -28,6 +30,12 @@ pub(crate) struct PreparedV2AgentDelivery { metadata_is_published: bool, } +struct RecordedSubmission { + id: String, + work: Option, + dispatch: DispatchDecision, +} + impl AgentControl { pub(crate) async fn prepare_v2_agent_delivery( &self, @@ -215,12 +223,31 @@ impl PreparedV2AgentDelivery { .await } - fn record_submission( + async fn record_submission( &self, - communication: &InterAgentCommunication, + communication: &mut InterAgentCommunication, context: &AgentCommunicationContext, - ) -> String { - let submission_id = new_submission_id(); + ) -> CodexResult { + let mut submission_id = new_submission_id(); + let accepted = self + .control + .mailbox_accept_native_submission( + self.agent_id, + context, + communication, + &mut submission_id, + ) + .await?; + let (work, dispatch) = accepted.map_or((None, DispatchDecision::Dispatch), |(work, decision)| { + (Some(work), decision) + }); + if dispatch != DispatchDecision::Dispatch { + return Ok(RecordedSubmission { + id: submission_id, + work, + dispatch, + }); + } let op = Op::InterAgentCommunication { communication: communication.clone(), }; @@ -229,7 +256,11 @@ impl PreparedV2AgentDelivery { let emit = crate::agent_communication::emit_agent_communication_send; emit(&submission_id, context, communication, self.agent_id); } - submission_id + Ok(RecordedSubmission { + id: submission_id, + work, + dispatch, + }) } pub(crate) async fn config_snapshot(&self) -> CodexResult { @@ -256,7 +287,7 @@ impl PreparedV2AgentDelivery { pub(super) fn send_after_capacity_check( mut self, - communication: InterAgentCommunication, + mut communication: InterAgentCommunication, context: AgentCommunicationContext, interrupt: bool, ) -> futures::future::BoxFuture<'static, CodexResult> { @@ -289,24 +320,50 @@ impl PreparedV2AgentDelivery { .record_submitted_op(self.agent_id, &Op::Interrupt); thread.session.interrupt_task().await; } - let submission_id = self.record_submission(&communication, &context); + let submission = self.record_submission(&mut communication, &context).await?; + match submission.dispatch { + DispatchDecision::AlreadyDispatched => return Ok(submission.id), + DispatchDecision::RecoveryNeeded => { + return Err(CodexErr::UnsupportedOperation(format!( + "accepted work {} requires native recovery; its original input is retained and was not resent", + submission.work.as_ref().map(|work| work.id.as_str()).unwrap_or("unknown") + ))); + } + DispatchDecision::Dispatch => {} + } let send = crate::session::inter_agent_communication; - send(&thread.session, submission_id.clone(), communication).await; - return Ok(submission_id); + send(&thread.session, submission.id.clone(), communication).await; + if let Some(work) = &submission.work { + self.control.mailbox_dispatched(work).await?; + } + return Ok(submission.id); } } if communication.trigger_turn { return Err(CodexErr::ThreadNotFound(self.agent_id)); } - let submission_id = self.record_submission(&communication, &context); + let submission = self.record_submission(&mut communication, &context).await?; + match submission.dispatch { + DispatchDecision::AlreadyDispatched => return Ok(submission.id), + DispatchDecision::RecoveryNeeded => { + return Err(CodexErr::UnsupportedOperation(format!( + "accepted work {} requires native recovery; its original input is retained and was not resent", + submission.work.as_ref().map(|work| work.id.as_str()).unwrap_or("unknown") + ))); + } + DispatchDecision::Dispatch => {} + } let sequence = self.lifecycle.next_cold_mailbox_sequence(); self.lifecycle.push_cold_mail(ColdMailboxItem { - receive_id: Some(submission_id.clone()), + receive_id: Some(submission.id.clone()), communication, sequence: Some(sequence), enqueued_at_ms: Some(crate::session::input_queue::current_time_ms()), }); - Ok(submission_id) + if let Some(work) = &submission.work { + self.control.mailbox_dispatched(work).await?; + } + Ok(submission.id) }) } } diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs new file mode 100644 index 000000000000..6f35471815a4 --- /dev/null +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -0,0 +1,326 @@ +//! The narrow native-runtime adapter for the independently stored V2 mailbox. +//! +//! The mailbox never owns agent membership, execution, or approval policy. Native lifecycle +//! boundaries supply accepted inputs and outcomes; canonical history owns observation custody. + +use super::AgentControl; +use crate::agent::AgentStatus; +use crate::agent::goal_notifications::GoalNotificationPhase; +use crate::agent::goal_notifications::GoalNotificationSnapshot; +use crate::agent_communication::AgentCommunicationContext; +use crate::agent_communication::AgentCommunicationKind; +use crate::config::Config; +use codex_agent_mailbox::Continuation; +use codex_agent_mailbox::ActivationDisposition; +use codex_agent_mailbox::DispatchDecision; +use codex_agent_mailbox::GoalBinding; +use codex_agent_mailbox::Envelope; +use codex_agent_mailbox::Mailbox; +use codex_agent_mailbox::NativeWork; +use codex_agent_mailbox::Outcome; +use codex_agent_mailbox::WorkRef; +use codex_features::Feature; +use codex_protocol::ResponseItemId; +use codex_protocol::ThreadId; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::ThreadGoalStatus; +use codex_protocol::models::ResponseItem; +use std::sync::Arc; +use std::collections::HashSet; + +fn storage_error(error: impl std::fmt::Display) -> CodexErr { + // The store does not log payloads. Keep its typed diagnostic in the error chain presented + // to the native caller rather than treating failed acceptance as successful delivery. + CodexErr::Fatal(format!("agent mailbox storage failed: {error}")) +} + +impl AgentControl { + pub(crate) async fn mailbox_initialize(&self, config: &Config) -> CodexResult<()> { + if config.multi_agent_version_from_features() != MultiAgentVersion::V2 + || !config.features.enabled(Feature::AgentMailbox) + { + return Ok(()); + } + self.mailbox + .get_or_try_init(|| async { + Mailbox::open(config.sqlite_config()) + .await + .map(Arc::new) + .map_err(storage_error) + }) + .await?; + Ok(()) + } + + pub(crate) fn mailbox_enabled(&self) -> bool { + self.mailbox.get().is_some() + } + + pub(crate) fn mailbox(&self) -> Option> { + self.mailbox.get().cloned() + } + + pub(crate) async fn mailbox_work_for_key( + &self, + issuer: ThreadId, + native_key: &str, + ) -> CodexResult> { + let Some(mailbox) = self.mailbox() else { + return Ok(None); + }; + mailbox + .work_for_native_key(issuer, native_key) + .await + .map_err(storage_error) + } + + pub(super) async fn mailbox_accept_native_submission( + &self, + agent: ThreadId, + context: &AgentCommunicationContext, + communication: &mut InterAgentCommunication, + submission_id: &mut String, + ) -> CodexResult> { + let Some(mailbox) = self.mailbox() else { + return Ok(None); + }; + if !matches!( + context.kind, + AgentCommunicationKind::Spawn | AgentCommunicationKind::Followup + ) { + return Ok(None); + } + if let Some(native_key) = context.native_key.as_deref() + && let Some(existing) = mailbox.work_for_native_key(context.sender_thread_id, native_key) + .await.map_err(storage_error)? + { + *submission_id = existing.submission_id; + } + let work = mailbox + .accept_work(NativeWork { + native_key: context + .native_key + .clone() + .unwrap_or_else(|| submission_id.clone()), + issuer: context.sender_thread_id, + agent, + submission_id: submission_id.clone(), + input: communication.clone(), + }) + .await + .map_err(storage_error)?; + *communication = mailbox.accepted_input(&work).await.map_err(storage_error)?; + *submission_id = work.submission_id.clone(); + let dispatch = mailbox.begin_dispatch(&work).await.map_err(storage_error)?; + Ok(Some((work, dispatch))) + } + + pub(super) async fn mailbox_dispatched(&self, work: &WorkRef) -> CodexResult<()> { + if let Some(mailbox) = self.mailbox() { + mailbox.mark_dispatched(work).await.map_err(storage_error)?; + } + Ok(()) + } + + pub(crate) async fn mailbox_activate_if_idle( + &self, + recipient: ThreadId, + envelope: &Envelope, + ) -> CodexResult<(ActivationDisposition, Option)> { + let denied = |reason: &str| (ActivationDisposition::Denied, Some(reason.to_string())); + let metadata = self.ensure_agent_known(recipient)?; + if metadata.agent_path.as_ref().is_none_or(|path| path.is_root()) { + return Ok(denied("root_activation_is_not_allowed")); + } + let state = self.upgrade()?; + // Reloading is not a side effect of accepting mail. An unloaded recipient keeps durable + // custody until it is explicitly resumed through its normal native lifecycle. + let Ok(thread) = state.get_thread(recipient).await else { + return Ok(denied("recipient_is_not_live")); + }; + let config = thread.session.get_config().await; + if !config.features.enabled(Feature::AgentMailboxIdleActivation) { + return Ok(denied("recipient_idle_activation_not_enabled")); + } + if !thread.is_running() { + return Ok(denied("recipient_runtime_is_not_accepting_input")); + } + if thread.session.active_turn.lock().await.is_some() { + return Ok((ActivationDisposition::AlreadyActive, None)); + } + if thread.session.services.thread_extension_data + .get::() + .and_then(|store| store.snapshot()) + .is_some_and(|goal| goal.status != ThreadGoalStatus::Active) + { + return Ok(denied("recipient_goal_requires_native_control")); + } + let mut communication = envelope.communication.clone(); + communication.trigger_turn = true; + self.send_inter_agent_communication( + recipient, + communication, + AgentCommunicationContext::new(AgentCommunicationKind::Message, envelope.sender) + .with_native_key(envelope.id.clone()), + ) + .await?; + Ok((ActivationDisposition::Activated, None)) + } + + pub(crate) async fn mailbox_mark_canonical( + &self, + recipient: ThreadId, + message_ids: &[ResponseItemId], + ) -> CodexResult<()> { + let Some(mailbox) = self.mailbox() else { + return Ok(()); + }; + for id in message_ids { + if mailbox.get_message(recipient, id.as_str()).await.map_err(storage_error)?.is_some() { + mailbox.mark_canonical(recipient, id.as_str()).await.map_err(storage_error)?; + } + } + Ok(()) + } + + pub(crate) async fn mailbox_reconcile_canonical_ids( + &self, + recipient: ThreadId, + canonical_ids: &HashSet, + ) -> CodexResult<()> { + let Some(mailbox) = self.mailbox() else { + return Ok(()); + }; + let mut cursor = None; + loop { + let page = mailbox.pending_page(recipient, cursor, 100).await.map_err(storage_error)?; + for envelope in &page.items { + if envelope.communication.id.as_ref().is_some_and(|id| canonical_ids.contains(id)) { + mailbox.mark_canonical(recipient, &envelope.id).await.map_err(storage_error)?; + } + } + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + Ok(()) + } + + pub(crate) async fn mailbox_preserve_unadmitted( + &self, + recipient: ThreadId, + current_active_history: &[ResponseItem], + ) -> CodexResult> { + let Some(mailbox) = self.mailbox() else { + return Ok(Vec::new()); + }; + let unread = mailbox.unadmitted_canonical_ids(recipient, 4096) + .await.map_err(storage_error)?.into_iter().collect::>(); + Ok(current_active_history.iter().filter(|item| { + matches!(item, ResponseItem::AgentMessage { id: Some(id), .. } if unread.contains(id.as_str())) + }).cloned().collect()) + } + + pub(crate) async fn mailbox_automatic_inputs( + &self, + recipient: ThreadId, + source_turn_id: &str, + ) -> CodexResult> { + let Some(mailbox) = self.mailbox() else { + return Ok(Vec::new()); + }; + Ok(mailbox.automatic_inputs(recipient, source_turn_id, 2) + .await.map_err(storage_error)?.into_iter() + .map(|envelope| envelope.communication).collect()) + } + + pub(crate) async fn mailbox_record_admission( + &self, + recipient: ThreadId, + source_turn_id: &str, + message_ids: &[ResponseItemId], + ) -> CodexResult<()> { + let Some(mailbox) = self.mailbox() else { + return Ok(()); + }; + let ids = message_ids.iter().map(ToString::to_string).collect::>(); + for candidates in ids.chunks(100) { + let admitted = mailbox.unadmitted_input_ids(recipient, candidates) + .await.map_err(storage_error)?; + if !admitted.is_empty() { + mailbox.record_admission(recipient, source_turn_id, &admitted) + .await.map_err(storage_error)?; + } + } + Ok(()) + } + + pub(crate) async fn mailbox_record_terminal( + &self, + recipient: ThreadId, + source_turn_id: &str, + status: AgentStatus, + goal: Option, + mut completion: Option, + ) -> CodexResult<()> { + let Some(mailbox) = self.mailbox() else { + return Ok(()); + }; + if let Some(communication) = completion.as_mut() + && communication.encrypted_content.is_none() + && communication.content.len() > 7600 + { + let mut boundary = 7600; + while !communication.content.is_char_boundary(boundary) { + boundary -= 1; + } + communication.content.truncate(boundary); + communication.content.push_str(&format!( + "\n[Result preview truncated. Canonical source: agent={recipient}, turn={source_turn_id}.]" + )); + } + let mut outcome = match status { + AgentStatus::Completed(_) => Outcome::Completed, + AgentStatus::Errored(_) => Outcome::Failed, + AgentStatus::Interrupted | AgentStatus::Shutdown => Outcome::Cancelled, + AgentStatus::PendingInit | AgentStatus::Running | AgentStatus::NotFound => { + Outcome::NeedsAttention + } + }; + let continuation = match goal { + Some(snapshot) => { + let binding = GoalBinding { + goal_id: snapshot.binding.goal_id, + parent_thread_id: snapshot.binding.parent_thread_id, + child_thread_id: snapshot.binding.child_thread_id, + control_generation: snapshot.binding.control_generation, + revision: snapshot.revision, + source_turn_id: source_turn_id.to_string(), + }; + match snapshot.phase { + GoalNotificationPhase::Running | GoalNotificationPhase::ContinuationPending => { + Continuation::PendingGoalBinding(binding) + } + GoalNotificationPhase::DeferredWithOwner => Continuation::Deferred(binding), + GoalNotificationPhase::ActionRequired => { + outcome = Outcome::NeedsAttention; + Continuation::Deferred(binding) + } + GoalNotificationPhase::ResultReady => Continuation::Terminal(binding), + } + } + None => Continuation::None, + }; + if let Err(error) = mailbox.record_turn_outcome( + recipient, source_turn_id, outcome, completion, continuation, + ).await { + mailbox.notify_runtime_fault(recipient); + return Err(storage_error(error)); + } + Ok(()) + } +} diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index 70e3af2e8ff8..cbfa5658401c 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -403,6 +403,36 @@ impl AgentControl { session_source: Option, options: SpawnAgentOptions, ) -> CodexResult { + if let Some(native_key) = context.native_key.as_deref() + && let Some(work) = self + .mailbox_work_for_key(context.sender_thread_id, native_key) + .await? + && let Some(mailbox) = self.mailbox() + { + // A repeated native invocation retains its original child, including when the + // original tool publication was lost. Never provision a replacement for a retry. + mailbox + .accept_work(codex_agent_mailbox::NativeWork { + native_key: native_key.to_string(), + issuer: context.sender_thread_id, + agent: work.agent, + submission_id: work.submission_id.clone(), + input: communication, + }) + .await + .map_err(|err| CodexErr::Fatal(format!("agent mailbox acceptance failed: {err}")))?; + let metadata = self.get_agent_metadata(work.agent).ok_or_else(|| { + CodexErr::UnsupportedOperation(format!( + "accepted work {} retains original child {}; native recovery is required and no replacement was spawned", + work.id, work.agent + )) + })?; + return Ok(LiveAgent { + thread_id: work.agent, + metadata, + status: self.get_status(work.agent).await, + }); + } Box::pin(self.spawn_agent_internal( config, SpawnInitialInput::InterAgentCommunication(communication, context), diff --git a/codex-rs/core/src/agent_communication.rs b/codex-rs/core/src/agent_communication.rs index 66d69ad41817..90ad3147e865 100644 --- a/codex-rs/core/src/agent_communication.rs +++ b/codex-rs/core/src/agent_communication.rs @@ -24,8 +24,9 @@ impl AgentCommunicationKind { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AgentCommunicationContext { - kind: AgentCommunicationKind, - sender_thread_id: ThreadId, + pub(crate) kind: AgentCommunicationKind, + pub(crate) sender_thread_id: ThreadId, + pub(crate) native_key: Option, } impl AgentCommunicationContext { @@ -33,8 +34,14 @@ impl AgentCommunicationContext { Self { kind, sender_thread_id, + native_key: None, } } + + pub(crate) fn with_native_key(mut self, native_key: String) -> Self { + self.native_key = Some(native_key); + self + } } pub(crate) fn logging_enabled() -> bool { diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index a67211e5d4c1..7bafa7299c43 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -413,7 +413,7 @@ async fn run_compact_task_inner_impl( window_ids, }, ) - .await; + .await?; sess.recompute_token_usage(&turn_context).await; sess.emit_turn_item_completed(&turn_context, compaction_item) diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 12338d3cd94d..eff1a08e9b5d 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -316,7 +316,7 @@ async fn run_remote_compact_task_inner_impl( window_ids: new_window_ids, }, ) - .await; + .await?; sess.recompute_token_usage(compaction_turn_context).await; sess.emit_turn_item_completed(compaction_turn_context, compaction_item) diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 99a553680acb..953bfca124b5 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -340,7 +340,7 @@ async fn run_remote_compact_task_inner_impl( window_ids: new_window_ids, }, ) - .await; + .await?; sess.recompute_token_usage(compaction_turn_context).await; sess.emit_turn_item_completed(compaction_turn_context, compaction_item) diff --git a/codex-rs/core/src/compact_token_budget.rs b/codex-rs/core/src/compact_token_budget.rs index 1e4749663809..5e643212bbfa 100644 --- a/codex-rs/core/src/compact_token_budget.rs +++ b/codex-rs/core/src/compact_token_budget.rs @@ -80,7 +80,7 @@ async fn run_compact_task_inner( sess.emit_turn_item_started(turn_context, &compaction_item) .await; sess.start_new_context_window(step_context, world_state) - .await; + .await?; sess.emit_turn_item_completed(turn_context, compaction_item) .await; diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index c9fe099332b7..42782e230e0b 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -35,6 +35,7 @@ use codex_protocol::protocol::HookSource; use codex_protocol::protocol::HookStartedEvent; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::error::Result as CodexResult; use codex_thread_store::ReadThreadParams; use serde_json::Value; use tracing::instrument; @@ -573,7 +574,7 @@ pub(crate) async fn record_pending_input( turn_context: &Arc, pending_input: TurnInput, additional_contexts: Vec, -) { +) -> CodexResult<()> { match pending_input { TurnInput::UserInput { content, client_id } => { sess.record_user_prompt_and_emit_turn_item( @@ -589,10 +590,11 @@ pub(crate) async fn record_pending_input( } TurnInput::InterAgentCommunication(communication) => { sess.record_inter_agent_communication(turn_context, communication) - .await; + .await?; } } record_additional_contexts(sess, turn_context, additional_contexts).await; + Ok(()) } async fn run_context_injecting_hook( diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 83ea71812802..faa8d7e5f325 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2014,6 +2014,27 @@ impl Session { .await .replace(error.clone()); } + let mailbox_terminal_before_publish = self.services.agent_control.mailbox_enabled() + && turn_context.multi_agent_version == MultiAgentVersion::V2 + && matches!( + &legacy_source, + EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_) + ) + && matches!( + &turn_context.session_source, + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + agent_path: Some(_), + .. + }) + ); + let mailbox_status_override = if mailbox_terminal_before_publish { + // Persist producer outcome and delivery intent before exposing a terminal status + // that can end the child session's custody. + self.maybe_notify_parent_of_terminal_turn(turn_context, &legacy_source) + .await + } else { + None + }; self.services .rollout_thread_trace .record_codex_turn_event(&turn_context.sub_id, &legacy_source); @@ -2024,9 +2045,17 @@ impl Session { id: turn_context.sub_id.clone(), msg, }; - self.send_event_raw(event).await; - self.maybe_notify_parent_of_terminal_turn(turn_context, &legacy_source) - .await; + if mailbox_terminal_before_publish { + self.send_event_raw_with_status_override(event, mailbox_status_override) + .await; + } else { + self.send_event_raw(event).await; + } + if !mailbox_terminal_before_publish { + let _ = self + .maybe_notify_parent_of_terminal_turn(turn_context, &legacy_source) + .await; + } self.maybe_mirror_event_text_to_realtime(&legacy_source) .await; self.maybe_clear_realtime_handoff_for_event(&legacy_source) @@ -2050,13 +2079,13 @@ impl Session { &self, turn_context: &TurnContext, msg: &EventMsg, - ) { + ) -> Option { if turn_context.multi_agent_version != MultiAgentVersion::V2 { - return; + return None; } if !matches!(msg, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_)) { - return; + return None; } let SessionSource::SubAgent(SubAgentSource::ThreadSpawn { @@ -2065,25 +2094,26 @@ impl Session { .. }) = &turn_context.session_source else { - return; + return None; }; - let status = match turn_context.terminal_error.lock().await.take() { + let mailbox_enabled = self.services.agent_control.mailbox_enabled(); + let terminal_error = turn_context.terminal_error.lock().await.take(); + let status = match terminal_error.as_ref() { Some(error) => { - let status = AgentStatus::Errored(error.message); - self.agent_status.send_replace(status.clone()); + let status = AgentStatus::Errored(error.message.clone()); + if !mailbox_enabled { + self.agent_status.send_replace(status.clone()); + } status } None => { let Some(status) = agent_status_from_event(msg) else { - return; + return None; }; status } }; - if !is_final(&status) { - return; - } if let EventMsg::TurnComplete(event) = msg && matches!(status, AgentStatus::Completed(_)) @@ -2099,6 +2129,43 @@ impl Session { let _ = store.publish_turn_complete(&token, event.last_agent_message.clone()); } + if mailbox_enabled { + let completion = Self::child_completion_communication(child_agent_path, &status); + let goal_snapshot = self + .services + .thread_extension_data + .get::() + .and_then(|store| store.snapshot()); + if let Err(error) = self + .services + .agent_control + .mailbox_record_terminal( + self.thread_id, + &turn_context.sub_id, + status.clone(), + goal_snapshot, + completion, + ) + .await + { + warn!( + thread_id = %self.thread_id, + turn_id = %turn_context.sub_id, + error = %error, + "failed to durably record mailbox turn outcome" + ); + return Some(AgentStatus::Errored(format!( + "Mailbox turn outcome could not be durably recorded: {error}" + ))); + } + } + + // An interrupted turn is terminal for mailbox accounting but not for native agent + // residency, so record it above and leave the existing native completion behavior alone. + if !is_final(&status) { + return mailbox_enabled.then_some(status); + } + // Only a normal, error-free TurnComplete may be deferred. Aborts and // errors always hand back to the parent. The turn token and store must // agree on incarnation, goal, and generation before suppression. @@ -2115,7 +2182,16 @@ impl Session { }) == Some(false) { - return; + if mailbox_enabled { + return Some(status); + } + return None; + } + + // The mailbox adapter owns result delivery to every exact issuer. If its persistence + // fails, report the error above and do not publish a success-shaped native fallback. + if mailbox_enabled { + return Some(status); } self.forward_child_completion_to_parent( @@ -2125,6 +2201,7 @@ impl Session { status, ) .await; + None } /// Sends the standard completion envelope from a spawned MultiAgentV2 child to its parent. @@ -2135,36 +2212,15 @@ impl Session { child_agent_path: &codex_protocol::AgentPath, status: AgentStatus, ) { - let Some(parent_agent_path) = child_agent_path - .as_str() - .rsplit_once('/') - .and_then(|(parent, _)| codex_protocol::AgentPath::try_from(parent).ok()) + let Some(communication) = Self::child_completion_communication(child_agent_path, &status) else { return; }; - - let Some(message) = format_inter_agent_completion_message( - parent_agent_path.clone(), - child_agent_path.clone(), - &status, - ) else { - return; - }; - // `communication` owns the message. Keep a second copy only when the - // recorder will actually need it after parent delivery succeeds. let trace_message = self .services .rollout_thread_trace .is_enabled() - .then(|| message.clone()); - let mut communication = InterAgentCommunication::new( - child_agent_path.clone(), - parent_agent_path, - Vec::new(), - message, - /*trigger_turn*/ false, - ); - communication.origin = Some(codex_protocol::protocol::AgentCommunicationOrigin::Result); + .then(|| communication.content.clone()); let context = AgentCommunicationContext::new(AgentCommunicationKind::Result, self.thread_id); if let Err(err) = self @@ -2191,6 +2247,30 @@ impl Session { } } + fn child_completion_communication( + child_agent_path: &codex_protocol::AgentPath, + status: &AgentStatus, + ) -> Option { + let parent_agent_path = child_agent_path + .as_str() + .rsplit_once('/') + .and_then(|(parent, _)| codex_protocol::AgentPath::try_from(parent).ok())?; + let message = format_inter_agent_completion_message( + parent_agent_path.clone(), + child_agent_path.clone(), + status, + )?; + let mut communication = InterAgentCommunication::new( + child_agent_path.clone(), + parent_agent_path, + Vec::new(), + message, + /*trigger_turn*/ false, + ); + communication.origin = Some(codex_protocol::protocol::AgentCommunicationOrigin::Result); + Some(communication) + } + async fn maybe_mirror_event_text_to_realtime(&self, msg: &EventMsg) { if self.conversation.running_state().await.is_none() { return; @@ -2250,6 +2330,19 @@ impl Session { .await; } + async fn send_event_raw_with_status_override( + &self, + event: Event, + status_override: Option, + ) { + self.send_event_raw_with_persistence_and_status_override( + event, + /*persist*/ true, + status_override, + ) + .await; + } + /// Delivers an event without creating a local rollout for a thread that has not materialized. pub(crate) async fn send_event_raw_without_materializing_rollout(&self, event: Event) { let persist = match self.current_rollout_path().await { @@ -2264,6 +2357,16 @@ impl Session { } async fn send_event_raw_with_persistence(&self, event: Event, persist: bool) { + self.send_event_raw_with_persistence_and_status_override(event, persist, None) + .await; + } + + async fn send_event_raw_with_persistence_and_status_override( + &self, + event: Event, + persist: bool, + status_override: Option, + ) { // Persist the event into rollout storage; the store applies its persistence policy. if persist { let rollout_items = vec![RolloutItem::EventMsg(event.msg.clone())]; @@ -2273,12 +2376,12 @@ impl Session { .rollout_thread_trace .record_protocol_event(&event.msg); self.services.log_usage_event(self.thread_id, &event).await; - self.deliver_event_raw(event).await; + self.deliver_event_raw(event, status_override).await; } - async fn deliver_event_raw(&self, event: Event) { + async fn deliver_event_raw(&self, event: Event, status_override: Option) { // Record the last known agent status. - if let Some(status) = agent_status_from_event(&event.msg) { + if let Some(status) = status_override.or_else(|| agent_status_from_event(&event.msg)) { self.agent_status.send_replace(status); } if let Err(e) = self.tx_event.send(event).await { @@ -3152,6 +3255,31 @@ impl Session { } } + fn canonical_agent_message_ids(initial_history: &InitialHistory) -> HashSet { + let items: &[RolloutItem] = match initial_history { + InitialHistory::New | InitialHistory::Cleared => return HashSet::new(), + InitialHistory::Resumed(resumed) => resumed.history.as_slice(), + InitialHistory::Forked(items) => items, + }; + let mut ids = HashSet::new(); + for item in items { + let response_items: &[ResponseItem] = match item { + RolloutItem::ResponseItem(item) => std::slice::from_ref(item), + RolloutItem::Compacted(compacted) => { + compacted.replacement_history.as_deref().unwrap_or_default() + } + _ => continue, + }; + ids.extend(response_items.iter().filter_map(|item| match item { + ResponseItem::AgentMessage { id: Some(id), .. } if !id.is_empty() => { + Some(id.clone()) + } + _ => None, + })); + } + ids + } + pub(crate) fn response_item_from_user_input(&self, input: Vec) -> ResponseItem { ResponseItem::from(ResponseInputItem::from_user_input( input, @@ -3273,7 +3401,7 @@ impl Session { &self, turn_context: &TurnContext, mut communication: InterAgentCommunication, - ) { + ) -> CodexResult { communication.set_turn_id_if_missing(&turn_context.sub_id); let response_item = communication.to_model_input_item(); let items = self.prepare_conversation_items_for_history( @@ -3282,7 +3410,92 @@ impl Session { ); let items = items.as_ref(); let response_item = items[0].clone(); - { + let response_item_id = response_item + .id() + .cloned() + .expect("prepared agent message should have a stable ID"); + let rollout_items = [ + RolloutItem::InterAgentCommunicationMetadata { + trigger_turn: communication.trigger_turn, + }, + RolloutItem::ResponseItem(response_item), + ]; + let mailbox_enabled = self.services.agent_control.mailbox_enabled(); + let custody_gate = if mailbox_enabled { + Some(self.mailbox_custody_gate.lock().await) + } else { + None + }; + if mailbox_enabled { + self.live_thread().ok_or_else(|| { + std::io::Error::other( + "mailbox canonical admission requires a durable session rollout", + ) + })?; + if self + .mailbox_unsettled_response_item_ids + .lock() + .await + .contains(&response_item_id) + { + return Err(std::io::Error::other( + "mailbox message has an indeterminate canonical append; resume the session to reconcile it", + ) + .into()); + } + let mut canonical_ids = self.mailbox_canonical_response_item_ids.lock().await; + let already_canonical = canonical_ids.contains(&response_item_id); + if already_canonical { + // InitialHistory and this set are the canonical source across compaction. Flush + // before acknowledging even when the exact message was already present. + self.flush_rollout().await?; + // Re-delivery after a crash between canonical flush and mailbox acknowledgement + // must not duplicate the body in in-memory history or emit it a second time. + self.services + .agent_control + .mailbox_mark_canonical( + self.thread_id, + std::slice::from_ref(&response_item_id), + ) + .await?; + drop(canonical_ids); + drop(custody_gate); + return Ok(response_item_id); + } + + // Do not retain the ID as canonical until the append has passed its durability + // barrier and the in-memory history has accepted the same item. Failed writes are + // separately marked indeterminate to prevent retries from duplicating partial appends. + if let Err(error) = self.try_persist_rollout_items(&rollout_items).await { + self.mailbox_unsettled_response_item_ids + .lock() + .await + .insert(response_item_id.clone()); + return Err(error.into()); + } + if let Err(error) = self.flush_rollout().await { + self.mailbox_unsettled_response_item_ids + .lock() + .await + .insert(response_item_id.clone()); + return Err(error.into()); + } + { + let mut state = self.state.lock().await; + state.current_time_reminder.note_recorded_items(items); + state.record_items( + items.iter(), + turn_context.model_info.truncation_policy.into(), + ); + } + canonical_ids.insert(response_item_id.clone()); + self.services + .agent_control + .mailbox_mark_canonical(self.thread_id, std::slice::from_ref(&response_item_id)) + .await?; + drop(canonical_ids); + } else { + self.persist_rollout_items(&rollout_items).await; let mut state = self.state.lock().await; state.current_time_reminder.note_recorded_items(items); state.record_items( @@ -3290,14 +3503,9 @@ impl Session { turn_context.model_info.truncation_policy.into(), ); } - self.persist_rollout_items(&[ - RolloutItem::InterAgentCommunicationMetadata { - trigger_turn: communication.trigger_turn, - }, - RolloutItem::ResponseItem(response_item), - ]) - .await; + drop(custody_gate); self.send_raw_response_items(turn_context, items).await; + Ok(response_item_id) } async fn maybe_warn_on_server_model_mismatch( @@ -3376,9 +3584,9 @@ impl Session { reference_context_item: Option, world_state_baseline: Option>, metadata: CompactedHistoryMetadata, - ) { - let items = Self::assign_missing_response_item_ids(Cow::Owned(items)).into_owned(); - let compacted_item = CompactedItem { + ) -> CodexResult<()> { + let mut items = Self::assign_missing_response_item_ids(Cow::Owned(items)).into_owned(); + let mut compacted_item = CompactedItem { message: metadata.message, replacement_history: Some(items.clone()), window_number: Some(metadata.window_number), @@ -3391,7 +3599,55 @@ impl Session { }; // Compaction starts a new history window, so its WorldState baseline must be full. let mut world_state_item = None; - { + if self.services.agent_control.mailbox_enabled() { + let _custody_gate = self.mailbox_custody_gate.lock().await; + self.live_thread().ok_or_else(|| { + std::io::Error::other( + "mailbox history checkpoint requires a durable session rollout", + ) + })?; + if !self + .mailbox_unsettled_response_item_ids + .lock() + .await + .is_empty() + { + return Err(std::io::Error::other( + "mailbox history checkpoint is blocked by an indeterminate canonical append", + ) + .into()); + } + let canonical_ids = self.mailbox_canonical_response_item_ids.lock().await.clone(); + self.services + .agent_control + .mailbox_reconcile_canonical_ids(self.thread_id, &canonical_ids) + .await?; + // A canonical message that has not entered a provider request is still unread. Keep + // its exact body in the replacement context so a checkpoint cannot discard custody. + let active_history = self.clone_history().await.into_raw_items(); + let unadmitted_items = self + .services + .agent_control + .mailbox_preserve_unadmitted(self.thread_id, &active_history) + .await?; + let mut replacement_message_ids = items + .iter() + .filter_map(|item| match item { + ResponseItem::AgentMessage { id: Some(id), .. } => Some(id.clone()), + _ => None, + }) + .collect::>(); + for item in unadmitted_items { + if let ResponseItem::AgentMessage { id: Some(id), .. } = &item + && replacement_message_ids.insert(id.clone()) + { + items.push(item); + } + } + compacted_item.replacement_history = Some(items.clone()); + let checkpoint = [RolloutItem::Compacted(compacted_item)]; + self.try_persist_rollout_items(&checkpoint).await?; + self.flush_rollout().await?; let mut state = self.state.lock().await; state.replace_history(items, reference_context_item.clone()); if let Some(world_state) = world_state_baseline { @@ -3399,10 +3655,19 @@ impl Session { world_state_item = Some(WorldStateItem::full(snapshot.clone().into_value())); state.history.set_world_state_baseline(snapshot); } + } else { + { + let mut state = self.state.lock().await; + state.replace_history(items, reference_context_item.clone()); + if let Some(world_state) = world_state_baseline { + let snapshot = world_state.snapshot(); + world_state_item = Some(WorldStateItem::full(snapshot.clone().into_value())); + state.history.set_world_state_baseline(snapshot); + } + } + self.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) + .await; } - - self.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) - .await; // Persist the baseline after the replacement history that established it. if let Some(world_state_item) = world_state_item { self.persist_rollout_items(&[RolloutItem::WorldState(world_state_item)]) @@ -3416,6 +3681,7 @@ impl Session { let mut state = self.state.lock().await; state.queue_pending_session_start_source(codex_hooks::SessionStartSource::Compact); } + Ok(()) } async fn persist_rollout_response_items(&self, items: &[ResponseItem]) { @@ -3794,10 +4060,18 @@ impl Session { items } + async fn try_persist_rollout_items(&self, items: &[RolloutItem]) -> std::io::Result<()> { + if let Some(live_thread) = self.live_thread() { + live_thread + .append_items(items) + .await + .map_err(std::io::Error::other)?; + } + Ok(()) + } + pub(crate) async fn persist_rollout_items(&self, items: &[RolloutItem]) { - if let Some(live_thread) = self.live_thread() - && let Err(e) = live_thread.append_items(items).await - { + if let Err(e) = self.try_persist_rollout_items(items).await { error!("failed to record rollout items: {e:#}"); } } @@ -3833,7 +4107,7 @@ impl Session { &self, step_context: &StepContext, world_state: Arc, - ) -> u64 { + ) -> CodexResult { let turn_context = step_context.turn.as_ref(); let window = { let mut state = self.state.lock().await; @@ -3854,9 +4128,9 @@ impl Session { window_ids, }, ) - .await; + .await?; self.recompute_token_usage(turn_context).await; - window_number + Ok(window_number) } pub(crate) async fn reference_context_item(&self) -> Option { diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index a2305bf75837..7ea1b57b3ba3 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -59,6 +59,13 @@ pub(crate) struct Session { pub(crate) input_queue: InputQueue, pub(crate) guardian_review_session: GuardianReviewSessionManager, pub(crate) services: SessionServices, + /// Stable IDs of canonical native agent messages, retained even if model history is compacted. + pub(super) mailbox_canonical_response_item_ids: Mutex>, + /// IDs whose append/flush had an indeterminate persistence result. They are never acked or + /// retried in-place, because the canonical writer may have committed a partial append. + pub(super) mailbox_unsettled_response_item_ids: Mutex>, + /// Serializes mailbox history settlement with compacted-history checkpoints. + pub(super) mailbox_custody_gate: Mutex<()>, pub(super) git_enrichment_policy: GitEnrichmentPolicy, pub(super) fork_persistence: ForkPersistence, pub(super) next_internal_sub_id: AtomicU64, @@ -619,6 +626,12 @@ impl Session { .effective_agent_max_threads(MultiAgentVersion::V2) .unwrap_or(usize::MAX), ); + agent_control.mailbox_initialize(&config).await?; + let mailbox_canonical_response_item_ids = if agent_control.mailbox_enabled() { + Session::canonical_agent_message_ids(&initial_history) + } else { + HashSet::new() + }; let time_provider = crate::current_time::resolve_time_provider( config.current_time_reminder.as_ref(), external_time_provider, @@ -1226,6 +1239,11 @@ impl Session { input_queue: InputQueue::new(), guardian_review_session: GuardianReviewSessionManager::default(), services, + mailbox_canonical_response_item_ids: Mutex::new( + mailbox_canonical_response_item_ids, + ), + mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), + mailbox_custody_gate: Mutex::new(()), git_enrichment_policy, fork_persistence, next_internal_sub_id: AtomicU64::new(0), diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 9967cf6866b6..b745e09738c5 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2133,6 +2133,34 @@ async fn record_response_item_and_emit_turn_item_emits_hook_prompt_lifecycle() { assert!(rx.try_recv().is_err(), "no extra events expected"); } +#[test] +fn canonical_agent_message_ids_include_compacted_replacement_history() { + let id = ResponseItemId::with_suffix("amsg", "stable-native-id"); + let mut communication = InterAgentCommunication::new( + AgentPath::root().join("worker").expect("worker path"), + AgentPath::root(), + Vec::new(), + "mailbox body".to_string(), + /*trigger_turn*/ true, + ); + communication.id = Some(id.clone()); + let replacement_history = vec![communication.to_model_input_item()]; + let history = InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: Arc::new(vec![RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(replacement_history), + window_number: None, + first_window_id: None, + previous_window_id: None, + window_id: None, + })]), + rollout_path: None, + }); + + assert!(Session::canonical_agent_message_ids(&history).contains(&id)); +} + #[tokio::test] async fn record_inter_agent_communication_sets_turn_id_in_rollout_and_resume() { let (mut session, turn_context) = make_session_and_context().await; @@ -2149,7 +2177,8 @@ async fn record_inter_agent_communication_sets_turn_id_in_rollout_and_resume() { session .record_inter_agent_communication(&turn_context, communication) - .await; + .await + .expect("inter-agent communication should be recorded"); assert_eq!( strip_response_item_ids(session.clone_history().await.raw_items()), @@ -2207,17 +2236,20 @@ async fn record_inter_agent_communication_preserves_item_id_in_rollout_and_resum .await; let rollout_path = attach_thread_persistence(Arc::get_mut(&mut session).expect("unique session")).await; - let communication = InterAgentCommunication::new( + let mut communication = InterAgentCommunication::new( AgentPath::root().join("worker").expect("worker path"), AgentPath::root(), Vec::new(), "child done".to_string(), /*trigger_turn*/ false, ); + let supplied_id = ResponseItemId::with_suffix("amsg", "stable-native-id"); + communication.id = Some(supplied_id.clone()); session .record_inter_agent_communication(&turn_context, communication) - .await; + .await + .expect("inter-agent communication should be recorded"); let live_history = session.clone_history().await; let [live_item] = live_history.raw_items() else { @@ -2226,8 +2258,8 @@ async fn record_inter_agent_communication_preserves_item_id_in_rollout_and_resum let live_item_id = live_item .id() .expect("live agent message should have an item id") - .to_string(); - assert!(live_item_id.starts_with("amsg_")); + .clone(); + assert_eq!(live_item_id, supplied_id); session.flush_rollout().await.expect("rollout should flush"); let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) @@ -2242,7 +2274,7 @@ async fn record_inter_agent_communication_preserves_item_id_in_rollout_and_resum }); assert_eq!( persisted_item_id.map(ResponseItemId::as_str), - Some(live_item_id.as_str()) + Some(supplied_id.as_str()) ); let (resumed_session, _resumed_turn_context, _rx) = @@ -3215,7 +3247,8 @@ async fn start_new_context_window_assigns_and_persists_item_ids() { session .start_new_context_window(&step_context, world_state) - .await; + .await + .expect("context window checkpoint should be recorded"); let live_history = session.clone_history().await; assert!(!live_history.raw_items().is_empty()); @@ -5895,6 +5928,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { input_queue: super::input_queue::InputQueue::new(), guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(), services, + mailbox_canonical_response_item_ids: Mutex::new(HashSet::new()), + mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), + mailbox_custody_gate: Mutex::new(()), git_enrichment_policy: GitEnrichmentPolicy::Fresh, fork_persistence: ForkPersistence::Copied, next_internal_sub_id: AtomicU64::new(0), @@ -8132,6 +8168,9 @@ where input_queue: super::input_queue::InputQueue::new(), guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(), services, + mailbox_canonical_response_item_ids: Mutex::new(HashSet::new()), + mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), + mailbox_custody_gate: Mutex::new(()), git_enrichment_policy: GitEnrichmentPolicy::Fresh, fork_persistence: ForkPersistence::Copied, next_internal_sub_id: AtomicU64::new(0), diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index c2091b392eb0..3ea30876c85e 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -181,7 +181,7 @@ pub(crate) async fn run_turn( return Err(err); } if matches!(err.details(), CodexErrorDetails::TurnAborted) { - run_hooks_and_record_inputs(&sess, &turn_context, &input).await; + run_hooks_and_record_inputs(&sess, &turn_context, &input).await?; return Err(err); } let error = err.to_codex_protocol_error(); @@ -198,7 +198,7 @@ pub(crate) async fn run_turn( { Ok(step_context) => step_context, Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => { - run_hooks_and_record_inputs(&sess, &turn_context, &input).await; + run_hooks_and_record_inputs(&sess, &turn_context, &input).await?; return Err(err); } Err(err) => return Err(err), @@ -224,7 +224,8 @@ pub(crate) async fn run_turn( return Ok(None); } let mut can_drain_pending_input = input.is_empty(); - if run_hooks_and_record_inputs(&sess, &turn_context, &input).await { + let should_stop = run_hooks_and_record_inputs(&sess, &turn_context, &input).await?; + if should_stop { return Ok(None); } @@ -269,7 +270,8 @@ pub(crate) async fn run_turn( Vec::new() }; - if run_hooks_and_record_inputs(&sess, &turn_context, &pending_input).await { + let should_stop = run_hooks_and_record_inputs(&sess, &turn_context, &pending_input).await?; + if should_stop { break; } @@ -301,6 +303,13 @@ pub(crate) async fn run_turn( .record_step_world_state_if_changed(&world_state, step_context.as_ref()) .await; + for communication in sess.services.agent_control + .mailbox_automatic_inputs(sess.thread_id, &turn_context.sub_id) + .await? + { + sess.record_inter_agent_communication(&turn_context, communication).await?; + } + // Construct the input that we will send to the model. let sampling_request_input: Vec = async { sess.clone_history() @@ -310,6 +319,28 @@ pub(crate) async fn run_turn( .instrument(trace_span!("run_turn.prepare_sampling_request_input")) .await; + let mailbox_message_ids = sampling_request_input + .iter() + .filter_map(|item| match item { + ResponseItem::AgentMessage { id: Some(id), .. } if !id.is_empty() => { + Some(id.clone()) + } + _ => None, + }) + .collect::>() + .into_iter() + .collect::>(); + if !mailbox_message_ids.is_empty() { + sess.services + .agent_control + .mailbox_record_admission( + sess.thread_id, + &turn_context.sub_id, + &mailbox_message_ids, + ) + .await?; + } + let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( sess.installation_id.clone(), window_id, @@ -592,7 +623,7 @@ pub(crate) async fn run_hooks_and_record_inputs( sess: &Arc, turn_context: &Arc, input: &[TurnInput], -) -> bool { +) -> CodexResult { let mut blocked_input = false; let mut accepted_user_input = false; for input_item in input { @@ -610,10 +641,10 @@ pub(crate) async fn run_hooks_and_record_inputs( input_item.clone(), hook_outcome.additional_contexts, ) - .await; + .await?; } } - blocked_input && !accepted_user_input + Ok(blocked_input && !accepted_user_input) } #[instrument(level = "trace", skip_all)] diff --git a/codex-rs/core/src/tasks/regular.rs b/codex-rs/core/src/tasks/regular.rs index 38e51deb6a8c..3fee3181df2a 100644 --- a/codex-rs/core/src/tasks/regular.rs +++ b/codex-rs/core/src/tasks/regular.rs @@ -65,7 +65,7 @@ impl SessionTask for RegularTask { .await; let prewarmed_client_session = match prewarmed_client_session { SessionStartupPrewarmResolution::Cancelled => { - run_hooks_and_record_inputs(&sess, &ctx, &input).await; + run_hooks_and_record_inputs(&sess, &ctx, &input).await?; return Ok(None); } SessionStartupPrewarmResolution::Unavailable { .. } => None, diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 1a3eed724621..13b34479a8bb 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1,6 +1,7 @@ use crate::CodexAppsToolsCache; use crate::SkillsService; use crate::agent::AgentControl; +use crate::agent::goal_notifications::GoalNotificationSnapshot; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; @@ -1367,45 +1368,77 @@ impl ThreadManager { AgentControl::new(Arc::downgrade(&self.state), /*rollout_budget*/ None) } - /// Delivers a queue-only V2 child completion to its immediate parent. - /// Extensions use this when a completion was intentionally deferred during - /// a child turn and a later external state change makes it terminal while - /// the child is idle. + /// Settles a deferred V2 child completion after a later external state + /// change makes it terminal while the child is idle. Mailbox-enabled + /// sessions persist the outcome and delivery intent instead of queueing a + /// separate native completion. A false return means the legacy native + /// enqueue was rejected; mailbox persistence failures return an error. pub async fn notify_v2_child_completion( &self, child_thread_id: ThreadId, parent_thread_id: ThreadId, child_agent_path: AgentPath, + source_turn_id: &str, status: AgentStatus, - ) -> Result<(), String> { - let Some(parent_agent_path) = child_agent_path + goal: Option, + ) -> Result { + let parent_agent_path = child_agent_path .as_str() .rsplit_once('/') - .and_then(|(parent, _)| AgentPath::try_from(parent).ok()) - else { - return Ok(()); + .and_then(|(parent, _)| AgentPath::try_from(parent).ok()); + let communication = match parent_agent_path.as_ref() { + Some(parent_agent_path) => format_inter_agent_completion_message( + parent_agent_path.clone(), + child_agent_path.clone(), + &status, + ) + .map(|message| { + codex_protocol::protocol::InterAgentCommunication::new( + child_agent_path.clone(), + parent_agent_path.clone(), + Vec::new(), + message, + /*trigger_turn*/ false, + ) + }), + None => None, }; - let Some(message) = format_inter_agent_completion_message( - parent_agent_path.clone(), - child_agent_path.clone(), - &status, - ) else { - return Ok(()); + if let Ok(child_thread) = self.get_thread(child_thread_id).await { + let child_agent_control = child_thread.session.services.agent_control.clone(); + if child_agent_control.mailbox_enabled() { + child_agent_control + .mailbox_record_terminal( + child_thread_id, + source_turn_id, + status, + goal, + communication, + ) + .await + .map_err(|err| err.to_string())?; + return Ok(true); + } + } + + let Some(parent_agent_path) = parent_agent_path else { + return Ok(true); + }; + let Some(communication) = communication else { + return Ok(true); }; - let communication = codex_protocol::protocol::InterAgentCommunication::new( - child_agent_path, - parent_agent_path, - Vec::new(), - message, - /*trigger_turn*/ false, - ); let context = AgentCommunicationContext::new(AgentCommunicationKind::Result, child_thread_id); - self.agent_control() + match self + .agent_control() .send_inter_agent_communication(parent_thread_id, communication, context) .await - .map(|_| ()) - .map_err(|err| err.to_string()) + { + Ok(_) => Ok(true), + Err(err) => { + tracing::warn!("failed to queue deferred child completion: {err}"); + Ok(false) + } + } } fn agent_control_for_config(&self, config: &Config) -> AgentControl { diff --git a/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs b/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs index 4a54aa765704..1a0821a33583 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs @@ -61,6 +61,7 @@ async fn handle_close_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -101,6 +102,7 @@ async fn handle_close_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -134,6 +136,7 @@ async fn handle_close_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs b/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs index 2f083d5e4800..73e93ab48c0c 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs @@ -80,6 +80,7 @@ async fn handle_resume_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -148,6 +149,7 @@ async fn handle_resume_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs index ae52b88146f2..3745128b3a80 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs @@ -85,6 +85,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -121,6 +122,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs index dcbe291286db..565fd6088411 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs @@ -123,6 +123,7 @@ async fn handle_spawn_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -217,6 +218,7 @@ async fn handle_spawn_agent( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs index bdbe7da53ea6..f6ec4712ca1a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs @@ -115,6 +115,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -155,6 +156,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -229,6 +231,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 33c6bd708951..25040cc1b916 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -188,6 +188,85 @@ pub fn create_send_input_tool_v1() -> ToolSpec { }) } +pub fn create_mailbox_send_message_tool() -> ToolSpec { + let properties = BTreeMap::from([ + ( + "target".to_string(), + JsonSchema::string(Some( + "Relative or canonical task name to message (from spawn_agent).".to_string(), + )), + ), + ( + "message".to_string(), + JsonSchema::string(Some( + "Message text to queue on the target agent.".to_string(), + )) + .with_encrypted(), + ), + ( + "kind".to_string(), + JsonSchema::string_enum( + vec![ + Value::String("note".to_string()), + Value::String("progress".to_string()), + Value::String("request".to_string()), + Value::String("alert".to_string()), + ], + Some("Message kind. Defaults to `note`. When supplied, this field is authoritative over legacy `wake_wait`.".to_string()), + ), + ), + ( + "work_ref".to_string(), + JsonSchema::string(Some( + "Exact immutable accepted work reference (a `work_*` id) associated with this message, when applicable." + .to_string(), + )), + ), + ( + "reply_to".to_string(), + JsonSchema::string(Some( + "Message id this intentionally final response replies to. Progress messages cannot set `reply_to`." + .to_string(), + )), + ), + ( + "supersedes".to_string(), + JsonSchema::string(Some( + "Exact message id authored by this sender that this message supersedes." + .to_string(), + )), + ), + ( + "activate_if_idle".to_string(), + JsonSchema::boolean(Some( + "Opt in to activating the target only if it is idle. This never interrupts active work." + .to_string(), + )), + ), + ( + "wake_wait".to_string(), + JsonSchema::boolean(Some( + "Legacy one-shot attention request when `kind` is omitted. Ignored when `kind` is supplied; does not force an interrupt." + .to_string(), + )), + ), + ]); + + ToolSpec::Function(ResponsesApiTool { + name: "send_message".to_string(), + description: "Queue a durable message for an existing agent in this agent tree. Messages are accepted independently of whether the target is active; `activate_if_idle` opts into starting an idle target and never interrupts active work. Use `reply_to` only for an intentional final response, not progress. The receipt confirms durable acceptance, not reading, acknowledgement, or comprehension." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object( + properties, + Some(vec!["target".to_string(), "message".to_string()]), + Some(false.into()), + ), + output_schema: Some(mailbox_send_message_output_schema()), + }) +} + pub fn create_send_message_tool() -> ToolSpec { let properties = BTreeMap::from([ ( @@ -227,6 +306,35 @@ pub fn create_send_message_tool() -> ToolSpec { }) } +pub fn create_mailbox_read_tool() -> ToolSpec { + let properties = BTreeMap::from([ + ( + "cursor".to_string(), + JsonSchema::string(Some( + "Opaque cursor returned by the previous page. Omit to read the first page." + .to_string(), + )), + ), + ( + "limit".to_string(), + JsonSchema::integer(Some( + "Maximum messages to return. Defaults to 20 and is capped at 100." + .to_string(), + )), + ), + ]); + + ToolSpec::Function(ResponsesApiTool { + name: "mailbox_read".to_string(), + description: "Read a bounded page of message metadata addressed to the current agent. Message bodies are appended through canonical conversation history rather than duplicated in this JSON result. Continue with `next_cursor`." + .to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object(properties, /*required*/ None, Some(false.into())), + output_schema: Some(mailbox_read_output_schema()), + }) +} + pub fn create_followup_task_tool() -> ToolSpec { let properties = BTreeMap::from([ ( @@ -301,6 +409,17 @@ pub fn create_wait_agent_tool_v1(options: WaitAgentTimeoutOptions) -> ToolSpec { }) } +pub fn create_mailbox_wait_agent_tool() -> ToolSpec { + ToolSpec::Function(ResponsesApiTool { + name: "wait_agent".to_string(), + description: mailbox_wait_agent_description(), + strict: false, + defer_loading: None, + parameters: mailbox_wait_agent_tool_parameters(), + output_schema: Some(mailbox_wait_output_schema()), + }) +} + pub fn create_wait_agent_tool_v2(options: WaitAgentTimeoutOptions) -> ToolSpec { create_wait_agent_tool_v2_with_capabilities(options, registered_tool_runtime_capabilities()) } @@ -502,6 +621,13 @@ fn spawn_agent_output_schema_v2(hide_agent_metadata: bool) -> Value { "description": "Canonical task name for the spawned agent." }), ), + ( + "work_ref".to_string(), + json!({ + "type": "string", + "description": "Immutable accepted work reference assigned to the spawned agent." + }), + ), ( "requested_model".to_string(), json!({ @@ -572,6 +698,10 @@ fn followup_task_output_schema() -> Value { "type": "string", "description": "Canonical task name of the agent receiving the follow-up." }, + "work_ref": { + "type": "string", + "description": "Immutable accepted work reference assigned to the follow-up." + }, "recipient_task_name": { "type": "string", "description": "Explicit canonical task name of the recipient; the effective_* fields below describe this recipient, not the sending agent." @@ -582,11 +712,11 @@ fn followup_task_output_schema() -> Value { "description": "Identity scope for the effective_* fields in this receipt." }, "effective_model": { - "type": "string", + "type": ["string", "null"], "description": "Effective model retained by the recipient agent for the follow-up turn." }, "effective_model_provider_id": { - "type": "string", + "type": ["string", "null"], "description": "Effective model provider retained by the recipient agent for the follow-up turn." }, "effective_reasoning_effort": { @@ -625,6 +755,34 @@ fn send_input_output_schema() -> Value { }) } +fn mailbox_send_message_output_schema() -> Value { + json!({ + "type": "object", + "properties": { + "message_id": { + "type": "string", + "description": "Stable id for the durably accepted mailbox message." + }, + "durable_state": { + "type": "string", + "enum": ["accepted"], + "description": "The message was durably accepted for delivery. This does not establish that the recipient read or understood it." + }, + "activation_disposition": { + "type": "string", + "enum": ["not_requested", "already_active", "activated", "denied"], + "description": "Disposition of the separately requested idle-target activation. Durable message acceptance is unaffected by a denied activation." + }, + "activation_reason": { + "type": ["string", "null"], + "description": "Safe reason when activation was denied, such as native pause, budget, cancellation, or root protection." + } + }, + "required": ["message_id", "durable_state", "activation_disposition"], + "additionalProperties": false + }) +} + fn send_message_output_schema() -> Value { json!({ "type": "object", @@ -678,6 +836,42 @@ fn send_message_output_schema() -> Value { }) } +fn mailbox_read_output_schema() -> Value { + json!({ + "type": "object", + "properties": { + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "message_id": { "type": "string" }, + "sender": { "type": "string" }, + "kind": { "type": "string", "enum": ["note", "progress", "request", "alert"] }, + "work_ref": { "type": ["string", "null"] }, + "reply_to": { "type": ["string", "null"] }, + "supersedes": { "type": ["string", "null"] }, + "canonical_delivered": { "type": "boolean" } + }, + "required": ["message_id", "sender", "kind", "canonical_delivered"], + "additionalProperties": false + }, + "description": "Messages in this page; body content is not duplicated in the JSON result." + }, + "next_cursor": { + "type": ["string", "null"], + "description": "Opaque cursor for the next page, or null when there are no more messages." + }, + "quiet_count": { + "type": "integer", + "description": "Number of routine messages safely quieted rather than duplicated into the model-visible result." + } + }, + "required": ["messages", "next_cursor", "quiet_count"], + "additionalProperties": false + }) +} + fn list_agents_output_schema(capabilities: ToolRuntimeCapabilities) -> Value { let include_active_descendants = capabilities .subagent_inventory @@ -876,6 +1070,83 @@ fn wait_output_schema_v1() -> Value { }) } +fn mailbox_wait_output_schema() -> Value { + json!({ + "type": "object", + "properties": { + "requested_targets": { + "type": "array", + "items": { "type": "string" }, + "description": "Agent names or exact native work references requested by the wait call." + }, + "requested_replies_to": { + "type": "array", + "items": { "type": "string" }, + "description": "Request message ids requested by the wait call." + }, + "return_when": { + "type": "string", + "enum": ["any", "all"], + "description": "Whether any one selected condition or every selected condition must complete." + }, + "completed": { + "type": "array", + "items": { + "type": "object", + "properties": { + "work_ref": { "type": ["string", "null"] }, + "message_id": { "type": ["string", "null"] }, + "agent": { "type": ["string", "null"] }, + "status": { "type": "string" } + }, + "required": ["status"], + "description": "Structured selected result for a completed native work or reply join.", + "additionalProperties": false + }, + "description": "Requested targets or replies whose join condition completed, with their result records." + }, + "pending": { + "type": "array", + "items": { + "type": "object", + "properties": { + "work_ref": { "type": ["string", "null"] }, + "message_id": { "type": ["string", "null"] }, + "agent": { "type": ["string", "null"] }, + "status": { "type": "string" } + }, + "required": ["status"], + "additionalProperties": false + }, + "description": "Requested targets or replies still pending when the wait returned." + }, + "completion_reason": { + "type": "string", + "enum": ["result", "reply", "request", "human", "tool", "deadline", "cancel", "subscription_loss", "runtime_fault", "no_matching_work", "already_satisfied", "cycle", "needs_attention"], + "description": "Actual reason the wait returned. Attention and interruption do not themselves satisfy an outstanding join." + }, + "condition_satisfied": { + "type": "boolean", + "description": "Whether the requested any/all join condition was satisfied." + }, + "quiet_count": { + "type": "integer", + "description": "Count of messages safely quieted for this wait, if applicable." + }, + "wake_sender": { + "type": ["string", "null"], + "description": "Safe sender identity metadata for a wake, when applicable." + }, + "wake_ref": { + "type": ["string", "null"], + "description": "Safe reference metadata for a wake, when applicable." + } + }, + "required": ["requested_targets", "requested_replies_to", "return_when", "completed", "pending", "completion_reason", "condition_satisfied", "quiet_count"], + "additionalProperties": false + }) +} + fn wait_output_schema_v2(capabilities: ToolRuntimeCapabilities) -> Value { let wait_capability = capabilities.wait_agent; let include_pending_ids = wait_capability.is_some_and(|capability| capability.pending_ids); @@ -1313,6 +1584,62 @@ fn wait_agent_tool_parameters_v1(options: WaitAgentTimeoutOptions) -> JsonSchema ) } +fn mailbox_wait_agent_tool_parameters() -> JsonSchema { + let properties = BTreeMap::from([ + ( + "targets".to_string(), + JsonSchema::array( + JsonSchema::string(/*description*/ None), + Some( + "Optional names of agents in this tree or exact native work references (for example, `work_...`). Pass multiple targets to wait for whichever completes first unless `return_when` is `all`. Omit to wait only for matching mailbox attention/replies or until `timeout_ms`; without a timeout this wait is unbounded." + .to_string(), + ), + ), + ), + ( + "replies_to".to_string(), + JsonSchema::array( + JsonSchema::string(Some("Request message id.".to_string())), + Some( + "Optional request message ids whose intentional final replies should satisfy this wait." + .to_string(), + ), + ), + ), + ( + "timeout_ms".to_string(), + JsonSchema::integer(Some( + "Optional timeout in milliseconds. Omit for an unbounded event-only wait; no short timeout is applied implicitly." + .to_string(), + )), + ), + ( + "return_when".to_string(), + JsonSchema::string_enum( + vec![ + serde_json::Value::String("any".to_string()), + serde_json::Value::String("all".to_string()), + ], + Some("Whether all requested joins must complete or any one may complete the wait. Defaults to `any`.".to_string()), + ), + ), + ( + "native_event_wait".to_string(), + JsonSchema::boolean(Some( + "Legacy compatibility field; accepted but has no effect. Omit it." + .to_string(), + )), + ), + ]); + + JsonSchema::object(properties, /*required*/ None, Some(false.into())) +} + +fn mailbox_wait_agent_description() -> String { + "Use this for blocking coordination within the current agent tree while awaiting agent, native work, or request-reply completion. Omit targets and `replies_to` to wait for caller-issued unsurfaced work or new attention; such attention, human/tool interrupts, and ordinary request notifications do not satisfy a selected join. `return_when` defaults to `any`; `all` requires every selected join to complete. A selected failure, cancellation, or needs-attention state may return early and is reported as the actual cause. Omit `timeout_ms` for an unbounded event-only wait. Results report requested, completed, and pending joins; no matching work returns an empty result rather than polling." + .to_string() +} + fn wait_agent_tool_parameters_v2( options: WaitAgentTimeoutOptions, capabilities: ToolRuntimeCapabilities, @@ -1345,8 +1672,8 @@ fn wait_agent_tool_parameters_v2( "return_when".to_string(), JsonSchema::string_enum( vec![ - serde_json::Value::String("any".to_string()), - serde_json::Value::String("all".to_string()), + Value::String("any".to_string()), + Value::String("all".to_string()), ], Some("Whether the wait completes when any requested agent reaches terminal status or only after all requested agents are terminal.".to_string()), ), diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index b098dfb5d417..18f26afa34f2 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -362,22 +362,31 @@ fn spawn_agent_tool_hides_model_controls_without_override_exposure() { let output_schema = output_schema.expect("spawn_agent output schema"); assert_eq!( output_schema["required"], - json!(["task_name", "effective_model", "effective_reasoning_effort"]) + json!([ + "task_name", + "effective_model", + "effective_reasoning_effort" + ]) ); let output_properties = output_schema["properties"] .as_object() .expect("spawn result properties"); + assert_eq!(output_properties["work_ref"]["type"], json!("string")); + assert!(output_properties["work_ref"]["description"] + .as_str() + .expect("work_ref description") + .contains("Immutable accepted work reference")); assert!(!output_properties.contains_key("agent_id")); assert!(!output_properties.contains_key("nickname")); } #[test] -fn send_message_tool_requires_encrypted_scalar_message_and_interrupt_schema() { +fn mailbox_send_message_tool_exposes_encrypted_mailbox_contract() { let ToolSpec::Function(ResponsesApiTool { parameters, output_schema, .. - }) = create_send_message_tool() + }) = create_mailbox_send_message_tool() else { panic!("send_message should be a function tool"); }; @@ -397,7 +406,37 @@ fn send_message_tool_requires_encrypted_scalar_message_and_interrupt_schema() { .and_then(|schema| schema.encrypted), Some(true) ); - assert!(properties.contains_key("interrupt")); + for property in [ + "kind", + "work_ref", + "reply_to", + "supersedes", + "activate_if_idle", + "wake_wait", + ] { + assert!(properties.contains_key(property), "missing `{property}`"); + } + assert!(!properties.contains_key("interrupt")); + assert_eq!( + properties["kind"].enum_values.as_ref(), + Some(&vec![ + json!("note"), + json!("progress"), + json!("request"), + json!("alert") + ]) + ); + assert!(properties["kind"].description.as_deref().unwrap().contains("Defaults to `note`")); + assert!(properties["reply_to"] + .description + .as_deref() + .unwrap() + .contains("Progress messages cannot set `reply_to`")); + assert!(properties["wake_wait"] + .description + .as_deref() + .unwrap() + .contains("Ignored when `kind` is supplied")); assert!(!properties.contains_key("items")); assert_eq!( properties @@ -412,6 +451,83 @@ fn send_message_tool_requires_encrypted_scalar_message_and_interrupt_schema() { assert!(output_schema.is_some()); } +#[test] +fn send_message_legacy_tool_remains_unchanged_without_mailbox_contract() { + let ToolSpec::Function(ResponsesApiTool { + parameters, + output_schema, + .. + }) = create_send_message_tool() + else { + panic!("send_message should be a function tool"); + }; + let properties = parameters + .properties + .as_ref() + .expect("send_message should use object params"); + assert!(properties.contains_key("interrupt")); + assert!(!properties.contains_key("kind")); + assert!(!properties.contains_key("activate_if_idle")); + assert_eq!( + properties + .get("message") + .and_then(|schema| schema.encrypted), + Some(true) + ); + assert_eq!( + output_schema.expect("legacy send_message output schema")["required"], + json!([ + "task_name", + "recipient_task_name", + "effective_identity_scope", + "handoff_state", + "effective_model", + "effective_model_provider_id", + "effective_reasoning_effort", + "effective_service_tier" + ]) + ); +} + +#[test] +fn mailbox_read_tool_exposes_bounded_cursor_pagination() { + let ToolSpec::Function(ResponsesApiTool { + parameters, + output_schema, + .. + }) = create_mailbox_read_tool() + else { + panic!("mailbox_read should be a function tool"); + }; + let properties = parameters + .properties + .as_ref() + .expect("mailbox_read should use object params"); + assert!(properties.contains_key("cursor")); + assert_eq!( + properties["limit"].schema_type, + Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Integer)) + ); + assert!(properties["limit"] + .description + .as_deref() + .unwrap() + .contains("capped at 100")); + assert_eq!(parameters.required, None); + let output_schema = output_schema.expect("mailbox_read output schema"); + assert_eq!( + output_schema["required"], + json!(["messages", "next_cursor", "quiet_count"]) + ); + assert_eq!( + output_schema["properties"]["messages"]["items"]["required"], + json!(["message_id", "sender", "kind", "canonical_delivered"]) + ); + assert!(output_schema["properties"]["messages"]["items"]["properties"] + .get("message") + .is_none()); +} + #[test] fn followup_task_tool_requires_message_and_describes_model_receipt() { let ToolSpec::Function(ResponsesApiTool { @@ -461,6 +577,10 @@ fn followup_task_tool_requires_message_and_describes_model_receipt() { "type": "string", "description": "Canonical task name of the agent receiving the follow-up." }, + "work_ref": { + "type": "string", + "description": "Immutable accepted work reference assigned to the follow-up." + }, "recipient_task_name": { "type": "string", "description": "Explicit canonical task name of the recipient; the effective_* fields below describe this recipient, not the sending agent." @@ -502,17 +622,13 @@ fn followup_task_tool_requires_message_and_describes_model_receipt() { } #[test] -fn wait_agent_tool_v2_uses_timeout_only_summary_output() { +fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result() { let ToolSpec::Function(ResponsesApiTool { description, parameters, output_schema, .. - }) = create_wait_agent_tool_v2(WaitAgentTimeoutOptions { - default_timeout_ms: 30_000, - min_timeout_ms: 10_000, - max_timeout_ms: 7_200_000, - }) + }) = create_mailbox_wait_agent_tool() else { panic!("wait_agent should be a function tool"); }; @@ -525,29 +641,66 @@ fn wait_agent_tool_v2_uses_timeout_only_summary_output() { .as_ref() .expect("wait_agent should use object params"); assert!(properties.contains_key("targets")); + assert!(properties.contains_key("replies_to")); assert!(properties.contains_key("timeout_ms")); assert!(properties.contains_key("return_when")); assert!(properties.contains_key("native_event_wait")); - let completion_reason = output_schema - .as_ref() - .expect("wait output schema")["properties"]["completion_reason"]["enum"] - .as_array() - .expect("completion reason enum"); - assert!(completion_reason.contains(&json!("subscription_loss"))); - assert!(description.contains("When `return_when` is `all`")); + assert_eq!(parameters.required, None); + assert_eq!( + properties["timeout_ms"].schema_type, + Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Integer)) + ); assert_eq!( properties .get("timeout_ms") .and_then(|schema| schema.description.as_deref()), - Some( - "Optional timeout in milliseconds. Defaults to 30000, min 10000, max 7200000. Prefer longer waits to avoid busy polling." - ) + Some("Optional timeout in milliseconds. Omit for an unbounded event-only wait; no short timeout is applied implicitly.") ); - assert_eq!(parameters.required.as_ref(), None); assert_eq!( - output_schema.expect("wait output schema")["properties"]["message"]["description"], - json!("Brief wait summary without the agent's final content.") + properties["return_when"].enum_values.as_ref(), + Some(&vec![json!("any"), json!("all")]) ); + assert!(properties["return_when"].description.as_deref().unwrap().contains("Defaults to `any`")); + assert!(properties["native_event_wait"].description.as_deref().unwrap().contains("has no effect")); + assert!(description.contains("unbounded event-only wait")); + assert!(description.contains("does not satisfy a selected join")); + let output_schema = output_schema.expect("wait output schema"); + assert_eq!( + output_schema["required"], + json!([ + "requested_targets", + "requested_replies_to", + "return_when", + "completed", + "pending", + "completion_reason", + "condition_satisfied", + "quiet_count" + ]) + ); + for cause in [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention", + ] { + assert!( + output_schema["properties"]["completion_reason"]["enum"] + .as_array() + .expect("completion reason enum") + .contains(&json!(cause)), + "missing completion cause `{cause}`" + ); + } } #[test] @@ -573,24 +726,45 @@ fn wait_agent_tool_v2_omits_runtime_fields_without_capability_provider() { assert!(!description.contains("return_when")); let output_schema = output_schema.expect("wait output schema"); - assert!( - !output_schema["properties"] - .as_object() - .expect("properties should be object") - .contains_key("pending_ids") - ); - assert!( - !output_schema["properties"] - .as_object() - .expect("properties should be object") - .contains_key("completion_reason") - ); + assert!(!output_schema["properties"] + .as_object() + .expect("properties should be object") + .contains_key("pending_ids")); + assert!(!output_schema["properties"] + .as_object() + .expect("properties should be object") + .contains_key("completion_reason")); assert_eq!( output_schema["required"], json!(["message", "requested_ids", "timed_out"]) ); } +#[test] +fn mailbox_wait_agent_tool_has_no_native_capability_prerequisite() { + let ToolSpec::Function(ResponsesApiTool { + parameters, + output_schema, + .. + }) = create_mailbox_wait_agent_tool() + else { + panic!("wait_agent should be a function tool"); + }; + let properties = parameters + .properties + .as_ref() + .expect("wait_agent should use object params"); + assert!(properties.contains_key("return_when")); + assert!(properties.contains_key("native_event_wait")); + let output_schema = output_schema.expect("mailbox wait output schema"); + assert!(output_schema["properties"].get("condition_satisfied").is_some()); + assert!(output_schema["properties"].get("quiet_count").is_some()); + assert_eq!( + output_schema["properties"]["completed"]["items"]["type"], + json!("object") + ); +} + #[test] fn list_agents_tool_includes_path_prefix_and_agent_fields() { let ToolSpec::Function(ResponsesApiTool { @@ -703,8 +877,8 @@ fn inspect_agent_tree_tool_exposes_scope_and_compact_tree_fields() { } #[test] -fn send_message_tool_declares_non_acknowledgement_handoff_receipt() { - let ToolSpec::Function(ResponsesApiTool { output_schema, .. }) = create_send_message_tool() +fn mailbox_send_message_tool_declares_durable_receipt_without_claiming_comprehension() { + let ToolSpec::Function(ResponsesApiTool { output_schema, .. }) = create_mailbox_send_message_tool() else { panic!("send_message should be a function tool"); }; @@ -712,31 +886,18 @@ fn send_message_tool_declares_non_acknowledgement_handoff_receipt() { let output_schema = output_schema.expect("send_message output schema"); assert_eq!( output_schema["required"], - json!([ - "task_name", - "recipient_task_name", - "effective_identity_scope", - "handoff_state", - "effective_model", - "effective_model_provider_id", - "effective_reasoning_effort", - "effective_service_tier" - ]) - ); - assert_eq!( - output_schema["properties"]["handoff_state"]["enum"], - json!(["queued"]) + json!(["message_id", "durable_state", "activation_disposition"]) ); assert_eq!( - output_schema["properties"]["effective_model"]["type"], - json!(["string", "null"]) - ); - assert_eq!( - output_schema["properties"]["effective_model_provider_id"]["type"], - json!(["string", "null"]) + output_schema["properties"]["durable_state"]["enum"], + json!(["accepted"]) ); assert_eq!( - output_schema["properties"]["effective_identity_scope"]["enum"], - json!(["recipient"]) + output_schema["properties"]["activation_disposition"]["enum"], + json!(["not_requested", "already_active", "activated", "denied"]) ); + assert!(output_schema["properties"]["durable_state"]["description"] + .as_str() + .expect("durable state description") + .contains("does not establish that the recipient read or understood it")); } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs index d1446ec6218f..5646269b5c5c 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs @@ -29,6 +29,9 @@ use serde_json::Value as JsonValue; pub(crate) use followup_task::Handler as FollowupTaskHandler; pub(crate) use interrupt_agent::Handler as InterruptAgentHandler; pub(crate) use list_agents::Handler as ListAgentsHandler; +pub(crate) use mailbox_message::ReadHandler as MailboxReadHandler; +pub(crate) use mailbox_message::SendHandler as MailboxSendMessageHandler; +pub(crate) use mailbox_wait::Handler as MailboxWaitAgentHandler; pub(crate) use send_message::Handler as SendMessageHandler; pub(crate) use spawn::Handler as SpawnAgentHandler; pub(crate) use wait::Handler as WaitAgentHandler; @@ -36,6 +39,8 @@ pub(crate) use wait::Handler as WaitAgentHandler; mod followup_task; mod interrupt_agent; mod list_agents; +mod mailbox_message; +mod mailbox_wait; mod message_tool; mod send_message; mod spawn; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs new file mode 100644 index 000000000000..4f9ccbc8cd91 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs @@ -0,0 +1,255 @@ +//! Thin V2 facades: persistence and selection belong to `codex-agent-mailbox`. + +use super::*; +use crate::session::session::Session; +use crate::session::turn_context::TurnContext; +use crate::tools::context::FunctionToolOutput; +use crate::tools::handlers::multi_agents_spec::create_mailbox_read_tool; +use crate::tools::handlers::multi_agents_spec::create_mailbox_send_message_tool; +use codex_agent_mailbox::ActivationDecision; +use codex_agent_mailbox::ActivationDisposition; +use codex_agent_mailbox::Envelope; +use codex_agent_mailbox::Mailbox; +use codex_agent_mailbox::MessageDraft; +use codex_agent_mailbox::MessageKind; +use codex_agent_mailbox::ObservationTarget; +use codex_tools::ToolSpec; +use std::sync::Arc; + +pub(crate) struct SendHandler; +pub(crate) struct ReadHandler; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SendArgs { + target: String, + message: String, + kind: Option, + work_ref: Option, + reply_to: Option, + supersedes: Option, + #[serde(default)] + activate_if_idle: bool, + #[serde(default)] + wake_wait: bool, +} + +#[derive(Serialize)] +struct SendReceipt { + message_id: String, + durable_state: &'static str, + activation_disposition: ActivationDisposition, + #[serde(skip_serializing_if = "Option::is_none")] + activation_reason: Option, + recipient_task_name: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReadArgs { + cursor: Option, + limit: Option, +} + +#[derive(Serialize)] +pub(super) struct MessageMetadata { + pub(super) message_id: String, + sender: String, + kind: MessageKind, + #[serde(skip_serializing_if = "Option::is_none")] + work_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reply_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] + supersedes: Option, + canonical_delivered: bool, +} + +#[derive(Serialize)] +struct ReadReceipt { + messages: Vec, + next_cursor: Option, + quiet_count: usize, +} + +pub(super) fn mailbox_error(error: impl std::fmt::Display) -> FunctionCallError { + FunctionCallError::RespondToModel(format!("mailbox operation failed: {error}")) +} + +pub(super) fn mailbox_for(session: &Session) -> Result, FunctionCallError> { + session.services.agent_control.mailbox().ok_or_else(|| { + FunctionCallError::RespondToModel("this session has no enabled V2 mailbox".to_string()) + }) +} + +/// Canonical history carries the body (including opaque encrypted content). The tool output +/// carries only retrieval/selection metadata, never a second copy or a comprehension claim. +pub(super) async fn deliver_envelope( + session: &Session, + turn: &TurnContext, + envelope: &Envelope, +) -> Result { + if envelope.recipient != session.thread_id { + return Err(mailbox_error("message is not addressed to this native thread")); + } + session + .record_inter_agent_communication(turn, envelope.communication.clone()) + .await + .map_err(mailbox_error)?; + Ok(MessageMetadata { + message_id: envelope.id.clone(), + sender: envelope.communication.author.to_string(), + kind: envelope.kind, + work_ref: envelope.work_ref.as_ref().map(|work| work.id.clone()), + reply_to: envelope.reply_to.clone(), + supersedes: envelope.supersedes.clone(), + canonical_delivered: true, + }) +} + +impl ToolExecutor for SendHandler { + fn tool_name(&self) -> ToolName { + ToolName::plain("send_message") + } + + fn spec(&self) -> ToolSpec { + create_mailbox_send_message_tool() + } + + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + let ToolInvocation { session, turn, payload, call_id, .. } = invocation; + let args: SendArgs = parse_arguments(&function_arguments(payload)?)?; + if args.message.trim().is_empty() { + return Err(mailbox_error("message cannot be empty")); + } + let mailbox = mailbox_for(&session)?; + let recipient = resolve_agent_target(&session, &turn, &args.target).await?; + let metadata = session.services.agent_control.ensure_agent_known(recipient) + .map_err(|err| collab_agent_error(recipient, err))?; + let recipient_path = metadata.agent_path.ok_or_else(|| mailbox_error("recipient has no native agent path"))?; + let sender_path = turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root); + let work_ref = match args.work_ref.as_deref() { + Some(id) => Some(mailbox.work_for_message_scope(session.thread_id, recipient, id) + .await.map_err(mailbox_error)? + .ok_or_else(|| mailbox_error("work_ref does not belong to this exact author/recipient scope"))?), + None => None, + }; + let legacy_attention = args.kind.is_none() && args.wake_wait; + let envelope = mailbox.accept_message(MessageDraft { + invocation_key: call_id, + sender: session.thread_id, + recipient, + communication: InterAgentCommunication::new_encrypted( + sender_path, + recipient_path.clone(), + Vec::new(), + args.message, + /*trigger_turn*/ false, + ), + kind: args.kind.unwrap_or_default(), + work_ref, + reply_to: args.reply_to, + supersedes: args.supersedes, + legacy_attention, + activate_if_idle: args.activate_if_idle, + }).await.map_err(mailbox_error)?; + let (activation_disposition, activation_reason) = if args.activate_if_idle { + match mailbox.begin_activation(session.thread_id, &envelope.id).await.map_err(mailbox_error)? { + ActivationDecision::Activate => { + let result = session.services.agent_control + .mailbox_activate_if_idle(recipient, &envelope).await; + let (disposition, reason) = match result { + Ok(result) => result, + Err(_) => (ActivationDisposition::Denied, Some("native_activation_not_accepted".to_string())), + }; + mailbox.record_activation(session.thread_id, &envelope.id, disposition) + .await.map_err(mailbox_error)?; + (disposition, reason) + } + ActivationDecision::AlreadyDecided(disposition) => (disposition, None), + ActivationDecision::RecoveryNeeded => ( + ActivationDisposition::Denied, + Some("activation_outcome_unknown_original_message_retained".to_string()), + ), + } + } else { + mailbox.record_activation(session.thread_id, &envelope.id, ActivationDisposition::NotRequested) + .await.map_err(mailbox_error)?; + (ActivationDisposition::NotRequested, None) + }; + Ok(boxed_tool_output(FunctionToolOutput::from_text( + tool_output_json_text(&SendReceipt { + message_id: envelope.id, + durable_state: "accepted", + activation_disposition, + activation_reason, + recipient_task_name: recipient_path.to_string(), + }, "send_message"), + Some(true), + ))) + }) + } +} + +impl CoreToolRuntime for SendHandler { + fn matches_kind(&self, payload: &ToolPayload) -> bool { + matches!(payload, ToolPayload::Function { .. }) + } +} + +impl ToolExecutor for ReadHandler { + fn tool_name(&self) -> ToolName { + ToolName::plain("mailbox_read") + } + + fn spec(&self) -> ToolSpec { + create_mailbox_read_tool() + } + + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + let ToolInvocation { session, turn, payload, .. } = invocation; + let args: ReadArgs = parse_arguments(&function_arguments(payload)?)?; + let cursor = args.cursor.as_deref().map(str::parse::).transpose() + .map_err(|_| mailbox_error("cursor must be a returned mailbox cursor"))?; + let mailbox = mailbox_for(&session)?; + let page = mailbox.pending_page(session.thread_id, cursor, args.limit.unwrap_or(20)) + .await.map_err(mailbox_error)?; + let mut messages = Vec::with_capacity(page.items.len()); + for envelope in &page.items { + messages.push(deliver_envelope(&session, &turn, envelope).await?); + let mut observations = Vec::new(); + if matches!(envelope.kind, MessageKind::Request | MessageKind::Alert) + || envelope.legacy_attention + { + observations.push(ObservationTarget::Attention(format!("attention:{}", envelope.id))); + } + if envelope.reply_to.is_some() { + observations.push(ObservationTarget::Reply(envelope.id.clone())); + } + // Reading a body can observe its own attention/reply only; it never observes + // unrelated work outcomes or unresolved request obligations. + if !observations.is_empty() { + mailbox.acknowledge_observed(session.thread_id, &observations) + .await.map_err(mailbox_error)?; + } + } + let quiet_count = mailbox.quiet_count(session.thread_id).await.map_err(mailbox_error)?; + Ok(boxed_tool_output(FunctionToolOutput::from_text( + tool_output_json_text(&ReadReceipt { + messages, + next_cursor: page.next_cursor.map(|cursor| cursor.to_string()), + quiet_count, + }, "mailbox_read"), + Some(true), + ))) + }) + } +} + +impl CoreToolRuntime for ReadHandler { + fn matches_kind(&self, payload: &ToolPayload) -> bool { + matches!(payload, ToolPayload::Function { .. }) + } +} diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs new file mode 100644 index 000000000000..336c2a86d52e --- /dev/null +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -0,0 +1,408 @@ +//! Selection-aware waits over immutable native work and explicit request replies. + +use super::*; +use super::mailbox_message::deliver_envelope; +use super::mailbox_message::mailbox_error; +use super::mailbox_message::mailbox_for; +use crate::session::input_queue::InputQueueActivity; +use crate::session::session::Session; +use crate::session::turn_context::TurnContext; +use crate::tools::context::FunctionToolOutput; +use crate::tools::handlers::multi_agents_spec::create_mailbox_wait_agent_tool; +use codex_agent_mailbox::DispatchState; +use codex_agent_mailbox::Mailbox; +use codex_agent_mailbox::ObservationTarget; +use codex_agent_mailbox::Outcome; +use codex_agent_mailbox::RequestDisposition; +use codex_agent_mailbox::ReturnWhen; +use codex_agent_mailbox::Selection; +use codex_agent_mailbox::Selector; +use codex_agent_mailbox::SnapshotItem; +use codex_protocol::ResponseItemId; +use codex_protocol::ThreadId; +use codex_protocol::items::MailboxWaitReport; +use codex_protocol::items::MailboxWaitReturnWhen; +use codex_protocol::items::MailboxWaitSelectionItem; +use codex_protocol::items::MailboxWakeCause; +use codex_protocol::protocol::AgentCommunicationOrigin; +use codex_tools::ToolSpec; +use std::collections::HashSet; +use std::time::Duration; +use tokio::time::Instant; + +pub(crate) struct Handler; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WaitArgs { + #[serde(alias = "ids")] + targets: Option>, + replies_to: Option>, + #[serde(default)] + return_when: MailboxWaitReturnWhen, + timeout_ms: Option, + /// Compatibility only: mailbox waits are always native and event-driven. + native_event_wait: Option, +} + +impl ToolExecutor for Handler { + fn tool_name(&self) -> ToolName { + ToolName::plain("wait_agent") + } + + fn spec(&self) -> ToolSpec { + create_mailbox_wait_agent_tool() + } + + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(handle_wait(invocation)) + } +} + +impl CoreToolRuntime for Handler { + fn matches_kind(&self, payload: &ToolPayload) -> bool { + matches!(payload, ToolPayload::Function { .. }) + } + + fn waits_for_runtime_cancellation(&self) -> bool { + true + } +} + +async fn handle_wait(invocation: ToolInvocation) -> Result, FunctionCallError> { + let ToolInvocation { session, turn, payload, call_id, cancellation_token, .. } = invocation; + let args: WaitArgs = parse_arguments(&function_arguments(payload)?)?; + let _compatibility_only = args.native_event_wait; + let mailbox = mailbox_for(&session)?; + session.services.agent_control.register_session_root(session.thread_id, turn.parent_thread_id); + // Both subscriptions precede authoritative rescans. Quiet notifications may rescan, but + // never shorten the fixed deadline or return control to the provider. + let mut mailbox_rx = mailbox.subscribe(); + let (mut input_rx, _) = session.input_queue.subscribe_activity(/*turn_state*/ None).await; + let targetless = args.targets.is_none() && args.replies_to.is_none(); + let targets = args.targets.unwrap_or_default(); + let replies_to = args.replies_to.unwrap_or_default(); + if targets.len() + replies_to.len() > 100 { + return Err(mailbox_error("at most 100 work/reply selectors may be joined")); + } + let mut selectors = Vec::new(); + let mut agents = Vec::new(); + for target in &targets { + if target.starts_with("work_") { + selectors.push(Selector::Work(target.clone())); + } else { + let agent = resolve_agent_target(&session, &turn, target).await?; + reject_reverse_completion_join(&session, &turn, agent)?; + agents.push(agent); + selectors.extend(mailbox.issued_work_refs(session.thread_id, Some(agent), 100) + .await.map_err(mailbox_error)?.into_iter().map(|work| Selector::Work(work.id))); + } + } + selectors.extend(replies_to.iter().cloned().map(Selector::Reply)); + let mut seen = HashSet::new(); + selectors.retain(|selector| seen.insert(selector_key(selector))); + let return_when = match args.return_when { + MailboxWaitReturnWhen::Any => ReturnWhen::Any, + MailboxWaitReturnWhen::All => ReturnWhen::All, + }; + let deadline = args.timeout_ms.map(|timeout| { + Instant::now().checked_add(Duration::from_millis(timeout)) + .ok_or_else(|| mailbox_error("timeout_ms exceeds the monotonic clock range")) + }).transpose()?; + let mut report = MailboxWaitReport { + requested_targets: targets, + requested_replies_to: replies_to, + return_when: args.return_when, + completed: Vec::new(), + pending: Vec::new(), + completion_reason: None, + condition_satisfied: false, + quiet_count: 0, + wake_sender: None, + wake_ref: None, + }; + let mut selected = Vec::new(); + let mut attention_to_observe = None; + let mut started = false; + let mut last_report = None; + let mut registration = None; + loop { + drop(registration.take()); + if targetless { + selectors = mailbox.issued_work_refs(session.thread_id, None, 100) + .await.map_err(mailbox_error)?.into_iter() + .map(|work| Selector::Work(work.id)).collect(); + } + selected = if selectors.is_empty() { + Vec::new() + } else { + mailbox.snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: selectors.clone(), + return_when, + limit: 100, + }).await.map_err(mailbox_error)?.items + }; + for item in &selected { + if let SnapshotItem::Work(work) = item { + reject_reverse_completion_join(&session, &turn, work.work_ref.agent)?; + if !agents.contains(&work.work_ref.agent) { + agents.push(work.work_ref.agent); + } + } + } + update_report(&session, &mut report, &selected); + report.quiet_count = mailbox.quiet_count(session.thread_id).await.map_err(mailbox_error)?; + let ready = report.completed.len(); + let early_attention = selected.iter().any(needs_attention); + let satisfied = !selected.is_empty() && match return_when { + ReturnWhen::Any => ready > 0, + ReturnWhen::All => ready == selectors.len(), + }; + let all_observed = !selected.is_empty() && selected.iter().all(already_observed); + if !started { + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await; + started = true; + last_report = Some(report.clone()); + } else if last_report.as_ref() != Some(&report) { + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await; + last_report = Some(report.clone()); + } + if mailbox.has_runtime_fault(&agents) { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } + if !targetless && selected.len() != selectors.len() { + report.completion_reason = Some(MailboxWakeCause::NoMatchingWork); + break; + } + if !targetless && selectors.is_empty() { + report.completion_reason = Some(MailboxWakeCause::NoMatchingWork); + break; + } + if all_observed { + report.condition_satisfied = true; + report.completion_reason = Some(MailboxWakeCause::AlreadySatisfied); + break; + } + if early_attention { + report.completion_reason = Some(MailboxWakeCause::NeedsAttention); + break; + } + if satisfied { + report.condition_satisfied = true; + report.completion_reason = Some(if all_observed { + MailboxWakeCause::AlreadySatisfied + } else if selected.iter().any(|item| matches!(item, SnapshotItem::Request(request) if request.reply.is_some())) { + MailboxWakeCause::Reply + } else { + MailboxWakeCause::Result + }); + break; + } + // An addressed request/alert interrupts a join but never satisfies it. Its body is + // admitted before its one-shot notification is observed; unanswered obligation remains. + for attention in mailbox.attention(session.thread_id, 100).await.map_err(mailbox_error)? { + let Some(message_id) = attention.id.strip_prefix("attention:amsg_").map(|suffix| format!("amsg_{suffix}")) else { continue; }; + if let Some(envelope) = mailbox.get_message(session.thread_id, &message_id).await.map_err(mailbox_error)? { + deliver_envelope(&session, &turn, &envelope).await?; + report.wake_sender = Some(envelope.communication.author.to_string()); + report.wake_ref = Some(envelope.id); + report.completion_reason = Some(MailboxWakeCause::Request); + attention_to_observe = Some(attention.id); + break; + } + } + if report.completion_reason.is_some() { + break; + } + if let Some(activity) = session.input_queue.pending_wait_input_activity(&session.active_turn).await { + match activity { + InputQueueActivity::Steer => { + report.completion_reason = Some(MailboxWakeCause::Human); + break; + } + InputQueueActivity::RuntimeSystemEvent | InputQueueActivity::Mailbox => { + report.completion_reason = Some(MailboxWakeCause::Tool); + break; + } + InputQueueActivity::TerminalCompletion => {} + } + } + let peers = selected.iter().filter_map(|item| match item { + SnapshotItem::Work(work) if work.outcome.is_none() && !work.observed => Some(work.work_ref.agent), + SnapshotItem::Request(request) if request.reply.is_none() && !request.observed => Some(request.request.recipient), + SnapshotItem::Work(_) | SnapshotItem::Request(_) => None, + }).collect(); + match mailbox.begin_wait(call_id.clone(), session.thread_id, peers, return_when, deadline.is_some()) { + Ok(guard) => registration = Some(guard), + Err(codex_agent_mailbox::MailboxError::ClosedPeerCycle) => { + report.completion_reason = Some(MailboxWakeCause::Cycle); + break; + } + Err(error) => return Err(mailbox_error(error)), + } + let deadline_wait = async { + match deadline { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending::<()>().await, + } + }; + tokio::select! { + biased; + _ = cancellation_token.cancelled() => { + report.completion_reason = Some(MailboxWakeCause::Cancel); + break; + } + _ = deadline_wait => { + report.completion_reason = Some(MailboxWakeCause::Deadline); + break; + } + changed = mailbox_rx.changed() => { + if changed.is_err() { + report.completion_reason = Some(MailboxWakeCause::SubscriptionLoss); + break; + } + } + changed = input_rx.changed() => { + if changed.is_err() { + report.completion_reason = Some(MailboxWakeCause::SubscriptionLoss); + break; + } + } + } + } + drop(registration); + admit_selected(&session, &turn, &mailbox, &selected).await?; + if let Some(id) = attention_to_observe { + mailbox.acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) + .await.map_err(mailbox_error)?; + } + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::Completed).await; + Ok(boxed_tool_output(FunctionToolOutput::from_text( + tool_output_json_text(&report, "wait_agent"), Some(true), + ))) +} + +fn selector_key(selector: &Selector) -> String { + match selector { + Selector::Work(id) => format!("work:{id}"), + Selector::Reply(id) => format!("reply:{id}"), + } +} + +fn reject_reverse_completion_join(session: &Session, turn: &TurnContext, agent: ThreadId) -> Result<(), FunctionCallError> { + let current = turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root); + let target = session.services.agent_control.get_agent_metadata(agent).and_then(|metadata| metadata.agent_path); + if agent == session.thread_id || target.as_ref().is_some_and(|target| target == ¤t + || current.as_str().strip_prefix(target.as_str()).is_some_and(|suffix| suffix.starts_with('/'))) { + return Err(mailbox_error("self/ancestor completion joins are not allowed; explicit parent-reply waits remain valid")); + } + Ok(()) +} + +fn needs_attention(item: &SnapshotItem) -> bool { + match item { + SnapshotItem::Work(work) => !work.observed && (work.dispatch_state == DispatchState::RecoveryNeeded + || work.outcome.as_ref().is_some_and(|result| result.outcome != Outcome::Completed) + || work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention)), + SnapshotItem::Request(request) => !request.observed + && matches!(request.disposition, RequestDisposition::Superseded | RequestDisposition::Withdrawn), + } +} + +fn already_observed(item: &SnapshotItem) -> bool { + match item { + SnapshotItem::Work(work) => work.observed, + SnapshotItem::Request(request) => request.observed, + } +} + +fn update_report(session: &Session, report: &mut MailboxWaitReport, selected: &[SnapshotItem]) { + report.completed.clear(); + report.pending.clear(); + for item in selected { + let (metadata, ready) = match item { + SnapshotItem::Work(work) => { + let status = if work.dispatch_state == DispatchState::RecoveryNeeded { + "recovery_needed".to_string() + } else { + work.outcome.as_ref().map_or_else(|| "pending".to_string(), |outcome| match outcome.outcome { + Outcome::Completed => "completed", + Outcome::Failed => "failed", + Outcome::Cancelled => "cancelled", + Outcome::NeedsAttention => "needs_attention", + }.to_string()) + }; + (MailboxWaitSelectionItem { + work_ref: Some(work.work_ref.id.clone()), + message_id: work.outcome.as_ref().map(|outcome| outcome.id.clone()), + agent: session.services.agent_control.get_agent_metadata(work.work_ref.agent) + .and_then(|metadata| metadata.agent_path).map(|path| path.to_string()), + status, + }, work.outcome.is_some() || work.observed) + } + SnapshotItem::Request(request) => (MailboxWaitSelectionItem { + work_ref: None, + message_id: Some(request.request_id.clone()), + agent: Some(request.request.communication.recipient.to_string()), + status: match request.disposition { + RequestDisposition::Open => "pending".to_string(), + RequestDisposition::Replied => "replied".to_string(), + RequestDisposition::Superseded => format!("superseded:{}", request.replacement.as_deref().unwrap_or("unknown")), + RequestDisposition::Withdrawn => "withdrawn".to_string(), + }, + }, request.reply.is_some() || request.observed), + }; + if ready { report.completed.push(metadata); } else { report.pending.push(metadata); } + } +} + +async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox, selected: &[SnapshotItem]) -> Result<(), FunctionCallError> { + let mut observations = Vec::new(); + for item in selected { + match item { + SnapshotItem::Work(work) if !work.observed => { + if let Some(outcome) = &work.outcome { + let communication = outcome.response.clone().unwrap_or_else(|| { + let mut communication = InterAgentCommunication::new( + work.input.recipient.clone(), work.input.author.clone(), Vec::new(), + serde_json::json!({"work_ref":work.work_ref.id,"outcome":outcome.outcome,"continuation":outcome.continuation}).to_string(), + /*trigger_turn*/ false, + ); + communication.id = Some(ResponseItemId::from_server(outcome.id.clone())); + communication.origin = Some(AgentCommunicationOrigin::Result); + communication + }); + session.record_inter_agent_communication(turn, communication).await.map_err(mailbox_error)?; + mailbox.mark_outcome_canonical(session.thread_id, &outcome.id).await.map_err(mailbox_error)?; + observations.push(ObservationTarget::Work(work.work_ref.clone())); + } + } + SnapshotItem::Request(request) if !request.observed => { + if let Some(reply) = &request.reply { + deliver_envelope(session, turn, reply).await?; + observations.push(ObservationTarget::Reply(reply.id.clone())); + } + } + SnapshotItem::Work(_) | SnapshotItem::Request(_) => {} + } + } + if !observations.is_empty() { + mailbox.acknowledge_observed(session.thread_id, &observations).await.map_err(mailbox_error)?; + } + Ok(()) +} + +async fn emit_report(session: &Session, turn: &TurnContext, call_id: &str, agents: &[ThreadId], report: &MailboxWaitReport, status: CollabAgentToolCallStatus) { + let item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: call_id.to_string(), tool: CollabAgentTool::Wait, status, + sender_thread_id: session.thread_id, receiver_thread_ids: agents.to_vec(), receiver_agents: Vec::new(), + prompt: None, model: None, reasoning_effort: None, requested_model: None, requested_reasoning_effort: None, + agents_states: Default::default(), wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: Some(report.clone()), + }); + if status == CollabAgentToolCallStatus::InProgress { session.emit_turn_item_started(turn, &item).await; } + else { session.emit_turn_item_completed(turn, item).await; } +} diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index de551fdfeb07..8a39fc1cb935 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -136,9 +136,11 @@ pub(crate) struct AssignTaskArgs { struct FollowupTaskResult { task_name: String, recipient_task_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + work_ref: Option, effective_identity_scope: &'static str, - effective_model: String, - effective_model_provider_id: String, + effective_model: Option, + effective_model_provider_id: Option, effective_reasoning_effort: Option, effective_service_tier: Option, } @@ -265,6 +267,39 @@ async fn handle_message_submission_inner( let receiver_agent_path = receiver_agent.agent_path.clone().ok_or_else(|| { FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string()) })?; + if mode == MessageDeliveryMode::TriggerTurn + && let Some(work) = session.services.agent_control + .mailbox_work_for_key(session.thread_id, &call_id).await + .map_err(|err| collab_agent_error(receiver_thread_id, err))? + && let Some(mailbox) = session.services.agent_control.mailbox() + { + if work.agent != receiver_thread_id { + return Err(FunctionCallError::RespondToModel( + "native invocation was already accepted for a different agent".to_string(), + )); + } + mailbox.accept_work(codex_agent_mailbox::NativeWork { + native_key: call_id, + issuer: session.thread_id, + agent: work.agent, + submission_id: work.submission_id.clone(), + input: communication_from_tool_message( + turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root), + receiver_agent_path.clone(), message, + ), + }).await.map_err(super::mailbox_message::mailbox_error)?; + let snapshot = session.services.agent_control.get_agent_config_snapshot(receiver_thread_id).await; + return Ok(FunctionToolOutput::from_text(tool_output_json_text(&FollowupTaskResult { + task_name: receiver_agent_path.to_string(), + recipient_task_name: receiver_agent_path.to_string(), + work_ref: Some(work.id), + effective_identity_scope: "recipient", + effective_model: snapshot.as_ref().map(|config| config.model.clone()), + effective_model_provider_id: snapshot.as_ref().map(|config| config.model_provider_id.clone()), + effective_reasoning_effort: snapshot.as_ref().and_then(|config| config.reasoning_effort.clone()), + effective_service_tier: snapshot.as_ref().and_then(|config| config.service_tier.clone()), + }, "followup_task"), Some(true))); + } let delivery = match mode { MessageDeliveryMode::QueueOnly => { session @@ -321,7 +356,8 @@ async fn handle_message_submission_inner( MessageDeliveryMode::QueueOnly => AgentCommunicationKind::Message, MessageDeliveryMode::TriggerTurn => AgentCommunicationKind::Followup, }; - let context = AgentCommunicationContext::new(kind, session.thread_id); + let context = AgentCommunicationContext::new(kind, session.thread_id) + .with_native_key(call_id.clone()); let result = delivery .send(mode.apply(communication), context, interrupt) .await @@ -331,7 +367,7 @@ async fn handle_message_submission_inner( &session, &turn, SubAgentActivityItem { - id: call_id, + id: call_id.clone(), agent_thread_id: receiver_thread_id, agent_path: receiver_agent_path.clone(), model: None, @@ -371,9 +407,16 @@ async fn handle_message_submission_inner( &FollowupTaskResult { task_name: receiver_agent_path.to_string(), recipient_task_name: receiver_agent_path.to_string(), + work_ref: session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, &call_id) + .await + .map_err(|err| collab_agent_error(receiver_thread_id, err))? + .map(|work| work.id), effective_identity_scope: "recipient", - effective_model: receiver_config.model, - effective_model_provider_id: receiver_config.model_provider_id, + effective_model: Some(receiver_config.model), + effective_model_provider_id: Some(receiver_config.model_provider_id), effective_reasoning_effort: receiver_config.reasoning_effort, effective_service_tier: receiver_config.service_tier, }, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index a9a39ac51ac7..5e14ee3932c7 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -118,7 +118,8 @@ async fn handle_spawn_agent( .get_agent_path() .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, new_agent_path.clone(), message); - let context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); + let context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id) + .with_native_key(call_id.clone()); let spawned_agent = Box::pin( session .services @@ -140,6 +141,13 @@ async fn handle_spawn_agent( .await .map_err(collab_spawn_error)?; let new_thread_id = spawned_agent.thread_id; + let work_ref = session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, &call_id) + .await + .map_err(collab_spawn_error)? + .map(|work| work.id); let agent_snapshot = session .services .agent_control @@ -184,6 +192,7 @@ async fn handle_spawn_agent( if hide_agent_metadata { Ok(SpawnAgentResult { agent_id: None, + work_ref, task_name, nickname: None, requested_model: args.model.clone(), @@ -199,6 +208,7 @@ async fn handle_spawn_agent( } else { Ok(SpawnAgentResult { agent_id: Some(new_thread_id.to_string()), + work_ref, task_name, nickname, requested_model: args.model.clone(), @@ -346,6 +356,8 @@ fn validate_spawn_agent_expected_reasoning_effort( pub(crate) struct SpawnAgentResult { #[serde(skip_serializing_if = "Option::is_none")] agent_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + work_ref: Option, task_name: String, #[serde(skip_serializing_if = "Option::is_none")] nickname: Option, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs index 7b92a7739123..8978c2228ad5 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs @@ -269,6 +269,7 @@ impl Handler { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), ) .await; @@ -1049,6 +1050,7 @@ async fn emit_wait_completion( wake_notifications: (!notifications.is_empty()).then_some(notifications), completion_reason: Some(completion_reason), wake_cause, + mailbox_wait_report: None, }), ) .await; diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 10eae078311f..bee4f76843e5 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -47,6 +47,9 @@ use crate::tools::handlers::multi_agents_spec::WaitAgentTimeoutOptions; use crate::tools::handlers::multi_agents_v2::FollowupTaskHandler as FollowupTaskHandlerV2; use crate::tools::handlers::multi_agents_v2::InterruptAgentHandler; use crate::tools::handlers::multi_agents_v2::ListAgentsHandler as ListAgentsHandlerV2; +use crate::tools::handlers::multi_agents_v2::MailboxReadHandler; +use crate::tools::handlers::multi_agents_v2::MailboxSendMessageHandler; +use crate::tools::handlers::multi_agents_v2::MailboxWaitAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::SendMessageHandler as SendMessageHandlerV2; use crate::tools::handlers::multi_agents_v2::SpawnAgentHandler as SpawnAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::WaitAgentHandler as WaitAgentHandlerV2; @@ -824,6 +827,7 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu agent_type_description(turn_context, context.default_agent_type_description); let hide_spawn_agent_metadata = turn_context.config.multi_agent_v2.hide_spawn_agent_metadata; + let mailbox_enabled = turn_context.features.enabled(Feature::AgentMailbox); planned_tools.add_arc(override_tool_exposure( multi_agent_v2_handler( SpawnAgentHandlerV2::new(SpawnAgentToolOptions { @@ -842,20 +846,40 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu ), exposure, )); - planned_tools.add_arc(override_tool_exposure( - multi_agent_v2_handler(SendMessageHandlerV2, tool_namespace), - exposure, - )); + if mailbox_enabled { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler(MailboxSendMessageHandler, tool_namespace), + exposure, + )); + } else { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler(SendMessageHandlerV2, tool_namespace), + exposure, + )); + } planned_tools.add_arc(override_tool_exposure( multi_agent_v2_handler(FollowupTaskHandlerV2, tool_namespace), exposure, )); if turn_context.config.multi_agent_v2.wait_agent_enabled { + if mailbox_enabled { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler(MailboxWaitAgentHandlerV2, tool_namespace), + exposure, + )); + } else { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler( + WaitAgentHandlerV2::new(context.wait_agent_timeouts), + tool_namespace, + ), + exposure, + )); + } + } + if mailbox_enabled { planned_tools.add_arc(override_tool_exposure( - multi_agent_v2_handler( - WaitAgentHandlerV2::new(context.wait_agent_timeouts), - tool_namespace, - ), + multi_agent_v2_handler(MailboxReadHandler, tool_namespace), exposure, )); } diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 140c2c8f82f7..e8e61e8c6352 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -180,6 +180,178 @@ impl ToolPlanProbe { } } +fn namespace_function<'a>( + plan: &'a ToolPlanProbe, + namespace_name: &str, + tool_name: &str, +) -> &'a ResponsesApiTool { + let ToolSpec::Namespace(namespace) = plan.visible_spec(namespace_name) else { + panic!("expected {namespace_name} namespace"); + }; + namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == tool_name => Some(tool), + ResponsesApiNamespaceTool::Function(_) => None, + }) + .unwrap_or_else(|| panic!("expected {tool_name} in {namespace_name} namespace")) +} + +fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { + let send_message = namespace_function(plan, namespace_name, "send_message"); + let send_properties = send_message + .parameters + .properties + .as_ref() + .expect("send_message object parameters"); + assert_eq!( + send_properties["message"].encrypted, + Some(true), + "encrypted message marker must survive model exposure" + ); + assert_eq!( + send_properties["kind"].enum_values.as_ref(), + Some(&vec![ + json!("note"), + json!("progress"), + json!("request"), + json!("alert") + ]) + ); + assert!(!send_properties.contains_key("interrupt")); + for property in ["work_ref", "reply_to", "supersedes", "activate_if_idle", "wake_wait"] { + assert!(send_properties.contains_key(property), "missing `{property}`"); + } + let send_output = send_message + .output_schema + .as_ref() + .expect("send_message output schema"); + assert_eq!( + send_output["required"], + json!(["message_id", "durable_state", "activation_disposition"]) + ); + assert_eq!( + send_output["properties"]["activation_disposition"]["enum"], + json!(["not_requested", "already_active", "activated", "denied"]) + ); + assert!(send_output["properties"]["durable_state"]["description"] + .as_str() + .expect("durable receipt description") + .contains("not establish that the recipient read or understood it")); + + let wait_agent = namespace_function(plan, namespace_name, "wait_agent"); + let wait_properties = wait_agent + .parameters + .properties + .as_ref() + .expect("wait_agent object parameters"); + assert!(wait_properties.contains_key("targets")); + assert!(wait_properties.contains_key("replies_to")); + assert!(wait_properties.contains_key("timeout_ms")); + assert!(wait_properties.contains_key("native_event_wait")); + assert!(wait_agent.parameters.required.is_none()); + assert_eq!( + wait_properties["return_when"].enum_values.as_ref(), + Some(&vec![json!("any"), json!("all")]) + ); + assert_eq!( + wait_properties["timeout_ms"].schema_type, + Some(codex_tools::JsonSchemaType::Single( + codex_tools::JsonSchemaPrimitiveType::Integer + )) + ); + assert!(wait_properties["timeout_ms"] + .description + .as_deref() + .expect("timeout description") + .contains("unbounded event-only wait")); + let wait_output = wait_agent + .output_schema + .as_ref() + .expect("wait_agent output schema"); + assert!(wait_output["required"] + .as_array() + .expect("wait result required fields") + .contains(&json!("return_when"))); + assert_eq!(wait_output["properties"]["return_when"]["enum"], json!(["any", "all"])); + assert!(wait_output["properties"].get("condition_satisfied").is_some()); + assert!(wait_output["properties"].get("quiet_count").is_some()); + assert_eq!(wait_output["properties"]["completed"]["items"]["required"], json!(["status"])); + assert_eq!(wait_output["properties"]["pending"]["items"]["required"], json!(["status"])); + for result_list in ["completed", "pending"] { + for field in ["work_ref", "message_id", "agent", "status"] { + assert!(wait_output["properties"][result_list]["items"]["properties"] + .get(field) + .is_some()); + } + } + assert_eq!( + wait_output["properties"]["completion_reason"]["enum"], + json!([ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ]) + ); + + let mailbox_read = namespace_function(plan, namespace_name, "mailbox_read"); + let read_properties = mailbox_read + .parameters + .properties + .as_ref() + .expect("mailbox_read object parameters"); + assert!(read_properties.contains_key("cursor")); + assert!(read_properties.contains_key("limit")); + assert_eq!( + read_properties["limit"].schema_type, + Some(codex_tools::JsonSchemaType::Single( + codex_tools::JsonSchemaPrimitiveType::Integer + )) + ); + assert!(read_properties["limit"] + .description + .as_deref() + .expect("mailbox page limit description") + .contains("capped at 100")); + assert_eq!( + mailbox_read + .output_schema + .as_ref() + .expect("mailbox_read output schema")["required"], + json!(["messages", "next_cursor", "quiet_count"]) + ); + let read_item = &mailbox_read + .output_schema + .as_ref() + .expect("mailbox_read output schema")["properties"]["messages"]["items"]; + assert_eq!( + read_item["required"], + json!(["message_id", "sender", "kind", "canonical_delivered"]) + ); + assert!(read_item["properties"].get("message").is_none()); + + for tool_name in ["spawn_agent", "followup_task"] { + let tool = namespace_function(plan, namespace_name, tool_name); + let output = tool.output_schema.as_ref().expect("agent result schema"); + assert_eq!(output["properties"]["work_ref"]["type"], json!("string")); + assert!(!output["required"] + .as_array() + .expect("required result fields") + .contains(&json!("work_ref"))); + } +} + async fn probe_with( configure_turn: impl FnOnce(&mut TurnContext), inputs: ToolPlanInputs, @@ -1580,6 +1752,96 @@ async fn multi_agent_v2_message_schemas_are_encrypted() { } } +#[tokio::test] +async fn mailbox_schemas_survive_ordinary_namespaced_and_code_mode_exposure() { + let ordinary = probe(|turn| { + set_features(turn, &[Feature::MultiAgentV2, Feature::AgentMailbox]); + update_config(turn, |config| { + config.multi_agent_v2.tool_namespace = Some("agents".to_string()); + }); + }) + .await; + ordinary.assert_visible_contains(&["agents"]); + for tool_name in ["send_message", "wait_agent", "mailbox_read"] { + assert!( + ordinary.namespace_function_names("agents").iter().any(|name| name == tool_name), + "expected {tool_name} in the ordinary namespaced mailbox surface" + ); + } + assert_mailbox_namespace_schema(&ordinary, "agents"); + + let code_mode = probe(|turn| { + set_features( + turn, + &[ + Feature::CodeMode, + Feature::CodeModeOnly, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ], + ); + update_config(turn, |config| { + config.multi_agent_v2.non_code_mode_only = true; + config.multi_agent_v2.tool_namespace = Some("agents".to_string()); + }); + }) + .await; + code_mode.assert_visible_contains(&["exec", "agents"]); + assert_mailbox_namespace_schema(&code_mode, "agents"); +} + +#[tokio::test] +async fn mailbox_feature_disabled_keeps_legacy_v2_message_and_wait_schemas() { + let plan = probe(|turn| { + set_feature(turn, Feature::MultiAgentV2, /*enabled*/ true); + }) + .await; + let ToolSpec::Namespace(namespace) = plan.visible_spec(MULTI_AGENT_V2_NAMESPACE) else { + panic!("expected {MULTI_AGENT_V2_NAMESPACE} namespace"); + }; + assert!(!plan + .namespace_function_names(MULTI_AGENT_V2_NAMESPACE) + .iter() + .any(|name| name == "mailbox_read")); + let send_message = namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == "send_message" => Some(tool), + ResponsesApiNamespaceTool::Function(_) => None, + }) + .expect("legacy send_message schema"); + let send_properties = send_message + .parameters + .properties + .as_ref() + .expect("send_message object parameters"); + assert!(send_properties.contains_key("interrupt")); + assert!(!send_properties.contains_key("kind")); + + let wait_agent = namespace + .tools + .iter() + .find_map(|tool| match tool { + ResponsesApiNamespaceTool::Function(tool) if tool.name == "wait_agent" => Some(tool), + ResponsesApiNamespaceTool::Function(_) => None, + }) + .expect("legacy wait_agent schema"); + let wait_properties = wait_agent + .parameters + .properties + .as_ref() + .expect("wait_agent object parameters"); + assert!(wait_properties.contains_key("native_event_wait")); + assert!(!wait_properties.contains_key("replies_to")); + let wait_output = wait_agent + .output_schema + .as_ref() + .expect("legacy wait output schema"); + assert!(wait_output["properties"].get("requested_ids").is_some()); + assert!(wait_output["properties"].get("condition_satisfied").is_none()); +} + #[tokio::test] async fn multi_agent_v2_can_disable_wait_agent() { let plan = probe(|turn| { diff --git a/codex-rs/ext/goal/src/api.rs b/codex-rs/ext/goal/src/api.rs index 9a9694a00583..cff27733de3c 100644 --- a/codex-rs/ext/goal/src/api.rs +++ b/codex-rs/ext/goal/src/api.rs @@ -275,10 +275,13 @@ impl GoalService { fill_empty_thread_preview_if_possible(state_db, thread_id, &goal).await; } if let Some(runtime) = runtime.as_ref() - && !matches!(goal.status, codex_state::ThreadGoalStatus::Active) - && let Err(err) = runtime.finalize_pending_goal_notification().await + && let Err(err) = runtime + .finalize_pending_goal_notification(Some((goal.goal_id.clone(), goal.status))) + .await { - tracing::warn!("failed to forward deferred goal completion: {err}"); + return Err(GoalServiceError::Internal(format!( + "goal updated but deferred completion settlement failed: {err}" + ))); } Ok(GoalSetOutcome { goal: protocol_goal_from_state(goal.clone()), @@ -319,9 +322,11 @@ impl GoalService { })?; let cleared = cleared_goal.is_some(); if let Some(runtime) = runtime.as_ref() - && let Err(err) = runtime.finalize_pending_goal_notification().await + && let Err(err) = runtime.finalize_pending_goal_notification(None).await { - tracing::warn!("failed to forward deferred goal completion: {err}"); + return Err(GoalServiceError::Internal(format!( + "goal cleared but deferred completion settlement failed: {err}" + ))); } drop(goal_state_permit); drop(runtime); diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index d7e802df6896..3b660c4fbf96 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -5,6 +5,9 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use codex_core::GoalNotificationBinding; +use codex_core::GoalNotificationClassification; +use codex_core::GoalNotificationPhase; +use codex_core::GoalNotificationSnapshot; use codex_core::GoalNotificationStore; use codex_core::GoalNotificationTurnToken; use codex_core::ThreadManager; @@ -241,17 +244,82 @@ impl GoalRuntimeHandle { /// Completes a deferred child handback when an external goal mutation /// makes the child terminal while no new turn will run. The pending - /// completion is claimed before delivery and restored if the parent - /// transport rejects it, allowing a later mutation to retry. - pub(crate) async fn finalize_pending_goal_notification(&self) -> Result<(), String> { + /// completion is claimed before settlement and restored after a failed + /// durable mailbox write or native queue delivery, allowing a later mutation + /// to retry. + pub(crate) async fn finalize_pending_goal_notification( + &self, + external_goal: Option<(String, codex_state::ThreadGoalStatus)>, + ) -> Result<(), String> { let Some((token, result)) = self.inner.notification_store.take_pending_completion() else { if self.inner.notification_store.has_forwarded_completion() { return Ok(()); } + if let Some((goal_id, codex_state::ThreadGoalStatus::Active)) = + external_goal.as_ref() + { + let snapshot = self.inner.notification_store.snapshot(); + if snapshot + .as_ref() + .map_or(true, |snapshot| snapshot.binding.goal_id == *goal_id) + { + return Ok(()); + } + } self.invalidate_goal_notification(); return Ok(()); }; + let Some(mut goal_snapshot) = self.inner.notification_store.snapshot().filter(|snapshot| { + snapshot.binding == token.binding + && snapshot.last_completed_source_turn.as_deref() + == Some(token.source_turn_id.as_str()) + }) else { + self.inner + .notification_store + .restore_pending_completion(&token.binding); + return Err("deferred goal completion no longer matches its captured binding".into()); + }; + let external_goal_matches_binding = external_goal + .as_ref() + .is_some_and(|(goal_id, _)| goal_id == &token.binding.goal_id); + if external_goal_matches_binding + && matches!( + external_goal.as_ref().map(|(_, status)| status), + Some(codex_state::ThreadGoalStatus::Active) + ) + { + self.inner + .notification_store + .restore_pending_completion(&token.binding); + return Ok(()); + } + if let Some((_, status)) = external_goal.filter(|_| external_goal_matches_binding) { + goal_snapshot.status = match status { + codex_state::ThreadGoalStatus::Active => ThreadGoalStatus::Active, + codex_state::ThreadGoalStatus::Paused => ThreadGoalStatus::Paused, + codex_state::ThreadGoalStatus::Blocked => ThreadGoalStatus::Blocked, + codex_state::ThreadGoalStatus::UsageLimited => ThreadGoalStatus::UsageLimited, + codex_state::ThreadGoalStatus::BudgetLimited => ThreadGoalStatus::BudgetLimited, + codex_state::ThreadGoalStatus::Complete => ThreadGoalStatus::Complete, + }; + } + goal_snapshot.phase = if external_goal_matches_binding + && goal_snapshot.status == ThreadGoalStatus::Complete + && result.is_some() + { + GoalNotificationPhase::ResultReady + } else { + // Pausing, blocking, limiting, clearing, or replacing the goal ends + // this continuation without claiming that the goal itself completed. + GoalNotificationPhase::ActionRequired + }; + goal_snapshot.classification = match goal_snapshot.phase { + GoalNotificationPhase::ResultReady => GoalNotificationClassification::Result, + _ => GoalNotificationClassification::ActionRequired, + }; + goal_snapshot.revision = goal_snapshot.revision.saturating_add(1); + let Some(parent_thread_id) = self.inner.parent_thread_id else { self.invalidate_goal_notification(); return Ok(()); @@ -267,16 +335,18 @@ impl GoalRuntimeHandle { return Err("thread manager unavailable for deferred goal completion".to_string()); }; - let result = thread_manager + let delivery = thread_manager .notify_v2_child_completion( self.inner.thread_id, parent_thread_id, child_agent_path, + &token.source_turn_id, codex_protocol::protocol::AgentStatus::Completed(result), + Some(goal_snapshot), ) .await; - match result { - Ok(()) => { + match delivery { + Ok(true) => { let _ = self .inner .notification_store @@ -289,6 +359,12 @@ impl GoalRuntimeHandle { .clear_preserving_forwarded_completion(); Ok(()) } + Ok(false) => { + self.inner + .notification_store + .restore_pending_completion(&token.binding); + Ok(()) + } Err(err) => { self.inner .notification_store diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index 74628b2e8843..352bd902eabe 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -288,8 +288,14 @@ impl GoalToolExecutor { "cannot update goal because this thread has no goal".to_string(), ) })?; - if let Err(err) = self.runtime.finalize_pending_goal_notification().await { - tracing::warn!("failed to forward deferred goal completion: {err}"); + if let Err(err) = self + .runtime + .finalize_pending_goal_notification(Some((goal.goal_id.clone(), goal.status))) + .await + { + return Err(FunctionCallError::RespondToModel(format!( + "goal updated but deferred completion settlement failed: {err}" + ))); } self.metrics .record_terminal_if_status_changed(previous_status, &goal); diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 4f3ba9712d35..260bd36b4ccb 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -153,6 +153,10 @@ pub enum Feature { Collab, /// Enable task-path-based multi-agent routing. MultiAgentV2, + /// Use durable, selection-aware mailboxes for same-tree V2 collaboration. + AgentMailbox, + /// Recipient opt-in to idle activation by accepted same-tree mailbox messages. + AgentMailboxIdleActivation, /// Removed compatibility flag retained as a no-op. MultiAgentMode, /// Removed compatibility flag for the deleted agent-job tools. @@ -1076,6 +1080,18 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: false, }, + FeatureSpec { + id: Feature::AgentMailbox, + key: "agent_mailbox", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, + FeatureSpec { + id: Feature::AgentMailboxIdleActivation, + key: "agent_mailbox_idle_activation", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::MultiAgentMode, key: "multi_agent_mode", diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index ba3740e35e38..50ef387d2e63 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -345,6 +345,80 @@ pub struct CollabAgentToolCallItem { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub wake_cause: Option, + /// Selection-aware mailbox wait state/result. Missing values identify legacy + /// history, where the selected join and request identities were not recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mailbox_wait_report: Option, +} + +/// Whether a wait requires any one selected condition or every selected condition. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MailboxWaitReturnWhen { + #[default] + Any, + All, +} + +/// Actual outcome that ended a selection-aware mailbox wait. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MailboxWakeCause { + Result, + Reply, + Request, + Human, + Tool, + Deadline, + Cancel, + SubscriptionLoss, + RuntimeFault, + NoMatchingWork, + AlreadySatisfied, + Cycle, + NeedsAttention, +} + +/// One selected wait condition and its current or terminal state. +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +pub struct MailboxWaitSelectionItem { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub work_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub message_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub agent: Option, + pub status: String, +} + +/// Shared mailbox wait report carried by tool results and collaboration history. +/// +/// `completion_reason` is absent while the wait is in progress and populated with +/// the actual wake/result cause when it returns. It is always present in a terminal +/// wait tool result. +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +pub struct MailboxWaitReport { + pub requested_targets: Vec, + pub requested_replies_to: Vec, + #[serde(default)] + pub return_when: MailboxWaitReturnWhen, + pub completed: Vec, + pub pending: Vec, + #[serde(default)] + #[ts(optional = false)] + pub completion_reason: Option, + pub condition_satisfied: bool, + pub quiet_count: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub wake_sender: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub wake_ref: Option, } #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] @@ -874,6 +948,25 @@ mod tests { use pretty_assertions::assert_eq; use serde_json::json; + #[test] + fn mailbox_wait_report_preserves_in_progress_cause_and_defaults_any_join() { + let mut value = json!({ + "requested_targets": ["work_42"], + "requested_replies_to": [], + "completed": [], + "pending": [{"work_ref": "work_42", "status": "running"}], + "condition_satisfied": false, + "quiet_count": 2 + }); + let report: MailboxWaitReport = + serde_json::from_value(value.clone()).expect("decode mailbox wait report"); + assert_eq!(report.return_when, MailboxWaitReturnWhen::Any); + assert_eq!(report.completion_reason, None); + value["return_when"] = json!("any"); + value["completion_reason"] = json!(null); + assert_eq!(serde_json::to_value(report).expect("encode active wait"), value); + } + #[test] fn sleep_extension_item_preserves_type_and_kind() { let item = TurnItem::Extension(ExtensionItem::Sleep(SleepItem { diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index 39f1abbcc066..a605da9cd0f7 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -319,6 +319,7 @@ impl CollabAgentToolCallItem { receiver_thread_ids: self.receiver_thread_ids.clone(), receiver_agents: self.receiver_agents.clone(), call_id: self.id.clone(), + mailbox_wait_report: self.mailbox_wait_report.clone(), })), CollabAgentTool::CloseAgent => receiver_thread_id.map(|receiver_thread_id| { EventMsg::CollabCloseBegin(CollabCloseBeginEvent { @@ -399,6 +400,7 @@ impl CollabAgentToolCallItem { }) .collect(), statuses: self.agents_states.clone(), + mailbox_wait_report: self.mailbox_wait_report.clone(), })), CollabAgentTool::CloseAgent => receiver_thread_id.map(|receiver_thread_id| { let (receiver_agent_nickname, receiver_agent_role) = diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 082b41b77e5e..059c7c809c62 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -34,6 +34,7 @@ use crate::dynamic_tools::DynamicToolCallOutputContentItem; use crate::dynamic_tools::DynamicToolCallRequest; use crate::dynamic_tools::DynamicToolResponse; use crate::dynamic_tools::DynamicToolSpec; +use crate::items::MailboxWaitReport; use crate::items::TurnItem; use crate::mcp::CallToolResult; use crate::mcp::RequestId; @@ -4637,6 +4638,10 @@ pub struct CollabWaitingBeginEvent { pub receiver_agents: Vec, /// ID of the waiting call. pub call_id: String, + /// Selection-aware wait state when emitted by a mailbox-enabled wait. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mailbox_wait_report: Option, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)] @@ -4652,6 +4657,10 @@ pub struct CollabWaitingEndEvent { pub agent_statuses: Vec, /// Last known status of the receiver agents reported to the sender agent. pub statuses: HashMap, + /// Selection-aware wait result when emitted by a mailbox-enabled wait. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mailbox_wait_report: Option, } #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Default, JsonSchema, TS)] @@ -5930,6 +5939,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; let completed = ItemCompletedEvent { @@ -5952,6 +5962,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; @@ -6019,6 +6030,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; let historic_legacy_started = historic_started @@ -6079,6 +6091,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; let current_omitted = ItemStartedEvent { @@ -6101,6 +6114,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }), }; diff --git a/codex-rs/tui/src/app/tests.rs b/codex-rs/tui/src/app/tests.rs index b3a29245e716..a8549dfabae3 100644 --- a/codex-rs/tui/src/app/tests.rs +++ b/codex-rs/tui/src/app/tests.rs @@ -1375,6 +1375,7 @@ async fn collab_receiver_notification_caches_thread_without_app_server_read() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), )); @@ -1427,6 +1428,7 @@ async fn collab_receiver_notification_does_not_cache_not_found_thread() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), )); @@ -1474,6 +1476,7 @@ async fn receiver_and_activity_ingress_share_navigation_and_metadata_cap() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }); @@ -7447,6 +7450,7 @@ async fn replace_chat_widget_reseeds_collab_agent_metadata_for_replay() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }, ), diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index ab26a4202520..3381cfd5f7e3 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -612,6 +612,7 @@ pub(crate) struct ChatWidget { running_commands: HashMap, collab_agent_metadata: HashMap, pending_collab_spawn_requests: HashMap, + active_mailbox_waits: HashMap, suppressed_exec_calls: HashSet, skills_all: Vec, skills_initial_state: Option>, @@ -1262,6 +1263,11 @@ impl ChatWidget { fn flush_active_cell(&mut self) { if let Some(active) = self.transcript.active_cell.take() { self.transcript.needs_final_message_separator = true; + let active = active + .as_any() + .downcast_ref::() + .map(|cell| Box::new(cell.snapshot_cell()) as Box) + .unwrap_or(active); self.app_event_tx.send(AppEvent::InsertHistoryCell(active)); self.request_pending_usage_output_insertion(); } diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index 4f7bc5d3891a..b4b42403e54e 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -154,6 +154,7 @@ impl ChatWidget { running_commands: HashMap::new(), collab_agent_metadata: HashMap::new(), pending_collab_spawn_requests: HashMap::new(), + active_mailbox_waits: HashMap::new(), suppressed_exec_calls: HashSet::new(), last_unified_wait: None, unified_exec_wait_streak: None, diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 8a2980612082..2d0a59e8e620 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -394,6 +394,7 @@ impl ChatWidget { wake_notifications, completion_reason, wake_cause, + mailbox_wait_report, } => self.on_collab_agent_tool_call(ThreadItem::CollabAgentToolCall { id, tool, @@ -411,6 +412,7 @@ impl ChatWidget { wake_notifications, completion_reason, wake_cause, + mailbox_wait_report, }), item @ ThreadItem::SubAgentActivity { .. } => self.on_sub_agent_activity(item), ThreadItem::EnteredReviewMode { review, .. } if !from_replay => { diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index 9fea1c309307..69ab77865e49 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -214,6 +214,7 @@ impl ChatWidget { wake_notifications, completion_reason, wake_cause, + mailbox_wait_report, } => self.on_collab_agent_tool_call(ThreadItem::CollabAgentToolCall { id, tool, @@ -231,6 +232,7 @@ impl ChatWidget { wake_notifications, completion_reason, wake_cause, + mailbox_wait_report, }), item @ ThreadItem::SubAgentActivity { .. } => self.on_sub_agent_activity(item), ThreadItem::DynamicToolCall { .. } => {} diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index daaa836dcb99..965797041c06 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -514,6 +514,7 @@ async fn collab_spawn_end_shows_requested_model_and_effort() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -547,6 +548,7 @@ async fn collab_spawn_end_shows_requested_model_and_effort() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1028,6 +1030,7 @@ async fn live_app_server_collab_wait_items_render_history() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1074,6 +1077,7 @@ async fn live_app_server_collab_wait_items_render_history() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1087,6 +1091,181 @@ async fn live_app_server_collab_wait_items_render_history() { assert_chatwidget_snapshot!("app_server_collab_wait_items_render_history", combined); } +#[tokio::test] +async fn live_mailbox_wait_report_shows_join_pending_state_and_actual_wake() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + drain_insert_history(&mut rx); + let sender_thread_id = ThreadId::new(); + let receiver_thread_id = ThreadId::new(); + let mailbox_wait_item = |status, mailbox_wait_report| { + AppServerThreadItem::CollabAgentToolCall { + id: "wait-mailbox-1".to_string(), + tool: AppServerCollabAgentTool::Wait, + status, + sender_thread_id: sender_thread_id.to_string(), + receiver_thread_ids: vec![receiver_thread_id.to_string()], + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + effective_model: None, + effective_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report, + } + }; + let active_report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["Ada".to_string(), "work_42".to_string()], + requested_replies_to: vec!["msg_request".to_string()], + return_when: codex_protocol::items::MailboxWaitReturnWhen::All, + completed: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: None, + agent: Some("Ada".to_string()), + status: "completed".to_string(), + }], + pending: vec![ + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_42".to_string()), + message_id: None, + agent: None, + status: "running".to_string(), + }, + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: Some("msg_request".to_string()), + agent: Some("Robie".to_string()), + status: "awaiting_reply".to_string(), + }, + ], + completion_reason: None, + condition_satisfied: false, + quiet_count: 3, + wake_sender: None, + wake_ref: None, + }; + + chat.handle_server_notification( + ServerNotification::ItemStarted(ItemStartedNotification { + thread_id: "thread-mailbox-wait".to_string(), + turn_id: "turn-mailbox-wait".to_string(), + started_at_ms: 0, + item: mailbox_wait_item( + AppServerCollabAgentToolCallStatus::InProgress, + Some(active_report.clone()), + ), + }), + /*replay_kind*/ None, + ); + let initial_lines = lines_to_single_string( + &chat + .active_cell_transcript_lines(/*width*/ 80) + .expect("mailbox wait active row"), + ); + assert!(initial_lines.contains("Waiting for all of 3 selected conditions")); + assert!(initial_lines.contains("Join condition: all")); + assert!(initial_lines.contains("Pending work_42: running")); + assert!(initial_lines.contains("Pending Robie · msg_request: awaiting_reply")); + assert!(initial_lines.contains("Quiet count: 3")); + assert!(!initial_lines.contains("Completion reason:")); + + let progressed_report = codex_protocol::items::MailboxWaitReport { + requested_targets: active_report.requested_targets.clone(), + requested_replies_to: active_report.requested_replies_to.clone(), + return_when: active_report.return_when, + completed: vec![ + active_report.completed[0].clone(), + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_42".to_string()), + message_id: None, + agent: None, + status: "completed".to_string(), + }, + ], + pending: vec![codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: Some("msg_request".to_string()), + agent: Some("Robie".to_string()), + status: "awaiting_reply".to_string(), + }], + completion_reason: None, + condition_satisfied: false, + quiet_count: 4, + wake_sender: None, + wake_ref: None, + }; + chat.handle_server_notification( + ServerNotification::ItemStarted(ItemStartedNotification { + thread_id: "thread-mailbox-wait".to_string(), + turn_id: "turn-mailbox-wait".to_string(), + started_at_ms: 1, + item: mailbox_wait_item( + AppServerCollabAgentToolCallStatus::InProgress, + Some(progressed_report.clone()), + ), + }), + /*replay_kind*/ None, + ); + let updated_lines = lines_to_single_string( + &chat + .active_cell_transcript_lines(/*width*/ 80) + .expect("mailbox wait active row after selection progress"), + ); + assert!(updated_lines.contains("Completed work_42: completed")); + assert!(updated_lines.contains("Pending Robie · msg_request: awaiting_reply")); + assert!(updated_lines.contains("Quiet count: 4")); + assert!(!updated_lines.contains("Pending work_42:")); + assert!(drain_insert_history(&mut rx).is_empty()); + + let terminal_report = codex_protocol::items::MailboxWaitReport { + requested_targets: active_report.requested_targets.clone(), + requested_replies_to: active_report.requested_replies_to.clone(), + return_when: active_report.return_when, + completed: vec![ + progressed_report.completed[0].clone(), + progressed_report.completed[1].clone(), + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: Some("msg_reply".to_string()), + agent: Some("Robie".to_string()), + status: "replied".to_string(), + }, + ], + pending: Vec::new(), + completion_reason: Some(codex_protocol::items::MailboxWakeCause::Reply), + condition_satisfied: true, + quiet_count: 4, + wake_sender: Some("Robie".to_string()), + wake_ref: Some("msg_reply".to_string()), + }; + chat.handle_server_notification( + ServerNotification::ItemCompleted(ItemCompletedNotification { + thread_id: "thread-mailbox-wait".to_string(), + turn_id: "turn-mailbox-wait".to_string(), + completed_at_ms: 1, + item: mailbox_wait_item( + AppServerCollabAgentToolCallStatus::Completed, + Some(terminal_report), + ), + }), + /*replay_kind*/ None, + ); + let terminal_lines = drain_insert_history(&mut rx) + .into_iter() + .map(|lines| lines_to_single_string(&lines)) + .collect::>() + .join("\n"); + assert!(terminal_lines.contains("Mailbox wait returned: reply")); + assert!(terminal_lines.contains("Condition satisfied: yes")); + assert!(terminal_lines.contains("Wake sender: Robie")); + assert!(terminal_lines.contains("Wake reference: msg_reply")); + assert!(terminal_lines.contains("Quiet count: 4")); +} + #[tokio::test] async fn live_app_server_collab_spawn_completed_renders_requested_model_and_effort() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1118,6 +1297,7 @@ async fn live_app_server_collab_spawn_completed_renders_requested_model_and_effo wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1152,6 +1332,7 @@ async fn live_app_server_collab_spawn_completed_renders_requested_model_and_effo wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1206,6 +1387,7 @@ async fn live_app_server_spawn_completion_does_not_fill_missing_effective_identi wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, @@ -1239,6 +1421,7 @@ async fn live_app_server_spawn_completion_does_not_fill_missing_effective_identi wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, }), /*replay_kind*/ None, diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index c5c1f7e09dc1..e4f0ea99bc80 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -75,6 +75,77 @@ async fn resumed_initial_messages_render_history() { ); } +#[tokio::test] +async fn replayed_mailbox_wait_report_preserves_active_join_selection() { + let (mut chat, mut rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await; + let sender_thread_id = ThreadId::new(); + let report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["worker-A".to_string(), "work_17".to_string()], + requested_replies_to: vec!["msg_request".to_string()], + return_when: codex_protocol::items::MailboxWaitReturnWhen::Any, + completed: Vec::new(), + pending: vec![ + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: None, + agent: Some("worker-A".to_string()), + status: "running".to_string(), + }, + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_17".to_string()), + message_id: None, + agent: None, + status: "pending".to_string(), + }, + codex_protocol::items::MailboxWaitSelectionItem { + work_ref: None, + message_id: Some("msg_request".to_string()), + agent: Some("reviewer".to_string()), + status: "awaiting_reply".to_string(), + }, + ], + completion_reason: None, + condition_satisfied: false, + quiet_count: 2, + wake_sender: None, + wake_ref: None, + }; + chat.replay_thread_item( + AppServerThreadItem::CollabAgentToolCall { + id: "wait-replay".to_string(), + tool: AppServerCollabAgentTool::Wait, + status: AppServerCollabAgentToolCallStatus::InProgress, + sender_thread_id: sender_thread_id.to_string(), + receiver_thread_ids: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + effective_model: None, + effective_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(report), + }, + "turn-replayed-mailbox-wait".to_string(), + ReplayKind::ResumeInitialMessages, + ); + let lines = drain_insert_history(&mut rx) + .into_iter() + .map(|lines| lines_to_single_string(&lines)) + .collect::>() + .join("\n"); + assert!(lines.contains("Waiting for any of 3 selected conditions")); + assert!(lines.contains("Targets: worker-A, work_17")); + assert!(lines.contains("Replies to: msg_request")); + assert!(lines.contains("Pending work_17: pending")); + assert!(lines.contains("Quiet count: 2")); + assert!(!lines.contains("Completion reason:")); +} + #[tokio::test] async fn replayed_collab_spawn_terminal_uses_only_explicit_effective_identity() { let (mut chat, mut rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await; @@ -118,6 +189,7 @@ async fn replayed_collab_spawn_terminal_uses_only_explicit_effective_identity() wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, "turn-replay".to_string(), ReplayKind::ResumeInitialMessages, @@ -141,6 +213,7 @@ async fn replayed_collab_spawn_terminal_uses_only_explicit_effective_identity() wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, "turn-replay".to_string(), ReplayKind::ResumeInitialMessages, @@ -253,6 +326,7 @@ async fn replayed_failed_collab_spawn_without_receiver_keeps_requested_identity( wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, "turn-replay".to_string(), ReplayKind::ResumeInitialMessages, diff --git a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs index 537c91c52ff7..d61ba3107114 100644 --- a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs @@ -157,11 +157,75 @@ impl ChatWidget { pub(super) fn on_collab_agent_tool_call(&mut self, item: ThreadItem) { let ThreadItem::CollabAgentToolCall { - id, tool, status, .. + id, + tool, + status, + mailbox_wait_report, + .. } = &item else { return; }; + + if matches!(tool, CollabAgentTool::Wait) + && matches!(status, CollabAgentToolCallStatus::InProgress) + && let Some(report) = mailbox_wait_report.as_ref() + { + if let Some(progress) = self.active_mailbox_waits.get(id) { + progress.update(report.clone(), /*in_progress*/ true); + let wait_cell_is_active = self + .transcript + .active_cell + .as_ref() + .and_then(|cell| { + cell.as_any() + .downcast_ref::() + }) + .is_some_and(|cell| cell.call_id() == id.as_str()); + if wait_cell_is_active { + self.bump_active_cell_revision(); + } else { + self.flush_active_cell(); + self.transcript.active_cell = Some(Box::new(progress.cell(id.to_string()))); + self.bump_active_cell_revision(); + } + } else { + self.flush_answer_stream_with_separator(); + self.flush_active_cell(); + let (cell, progress) = + multi_agents::mailbox_wait::progress_cell(id.to_string(), report.clone()); + self.active_mailbox_waits.insert(id.clone(), progress); + self.transcript.active_cell = Some(Box::new(cell)); + self.bump_active_cell_revision(); + } + self.request_redraw(); + return; + } + if matches!(tool, CollabAgentTool::Wait) { + if let Some(progress) = self.active_mailbox_waits.remove(id) { + if let Some(report) = mailbox_wait_report.as_ref() { + progress.update(report.clone(), /*in_progress*/ false); + } else { + progress.finish(); + } + let wait_cell_is_active = self + .transcript + .active_cell + .as_ref() + .and_then(|cell| { + cell.as_any() + .downcast_ref::() + }) + .is_some_and(|cell| cell.call_id() == id.as_str()); + if wait_cell_is_active { + self.bump_active_cell_revision(); + self.flush_active_cell(); + self.request_redraw(); + return; + } + } + } + if matches!(tool, CollabAgentTool::SpawnAgent) && let Some(spawn_request) = multi_agents::spawn_request_summary(&item) { diff --git a/codex-rs/tui/src/multi_agents.rs b/codex-rs/tui/src/multi_agents.rs index d14ad0e2224b..807f9310c237 100644 --- a/codex-rs/tui/src/multi_agents.rs +++ b/codex-rs/tui/src/multi_agents.rs @@ -36,6 +36,8 @@ use ratatui::text::Line; use ratatui::text::Span; use std::collections::HashSet; +pub(crate) mod mailbox_wait; + const COLLAB_PROMPT_PREVIEW_GRAPHEMES: usize = 160; const COLLAB_AGENT_ERROR_PREVIEW_GRAPHEMES: usize = 160; const COLLAB_AGENT_RESPONSE_PREVIEW_GRAPHEMES: usize = 240; @@ -431,6 +433,7 @@ pub(crate) fn tool_call_history_cell( wake_notifications, completion_reason, wake_cause, + mailbox_wait_report, .. } = item else { @@ -506,7 +509,12 @@ pub(crate) fn tool_call_history_cell( }), CollabAgentTool::Wait => { if matches!(status, CollabAgentToolCallStatus::InProgress) { - Some(waiting_begin(receiver_thread_ids, &mut agent_metadata)) + Some(match mailbox_wait_report { + Some(report) => mailbox_wait::begin(report), + None => waiting_begin(receiver_thread_ids, &mut agent_metadata), + }) + } else if let Some(report) = mailbox_wait_report { + Some(mailbox_wait::end(report)) } else { Some(waiting_end( receiver_thread_ids, @@ -1640,6 +1648,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, bob_id), @@ -1668,6 +1677,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, bob_id), @@ -1693,6 +1703,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, bob_id), @@ -1727,6 +1738,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, bob_id), @@ -1755,6 +1767,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, bob_id), @@ -2071,6 +2084,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, ThreadId::new()), @@ -2120,6 +2134,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, /*cached_spawn_request*/ None, |thread_id| metadata_for(thread_id, robie_id, ThreadId::new()), @@ -2161,6 +2176,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } } diff --git a/codex-rs/tui/src/multi_agents/mailbox_wait.rs b/codex-rs/tui/src/multi_agents/mailbox_wait.rs new file mode 100644 index 000000000000..bbb9c173799f --- /dev/null +++ b/codex-rs/tui/src/multi_agents/mailbox_wait.rs @@ -0,0 +1,258 @@ +//! Rendering and live updates for mailbox wait history rows. + +use super::collab_event; +use super::sanitize_notification_preview; +use super::title_text; +use crate::history_cell::HistoryCell; +use crate::history_cell::PlainHistoryCell; +use codex_protocol::items::MailboxWaitReport; +use codex_protocol::items::MailboxWaitReturnWhen; +use codex_protocol::items::MailboxWaitSelectionItem; +use codex_protocol::items::MailboxWakeCause; +use ratatui::text::Line; +use std::sync::Arc; +use std::sync::Mutex; + +pub(crate) fn begin(report: &MailboxWaitReport) -> PlainHistoryCell { + let selection_count = report.requested_targets.len() + report.requested_replies_to.len(); + let title = if selection_count == 0 { + format!( + "Waiting for mailbox attention ({})", + return_when_label(report.return_when) + ) + } else { + format!( + "Waiting for {} of {selection_count} selected conditions", + return_when_label(report.return_when) + ) + }; + collab_event(title_text(title), mailbox_wait_report_lines(report, /*in_progress*/ true)) +} + +pub(crate) fn end(report: &MailboxWaitReport) -> PlainHistoryCell { + let title = match report.completion_reason { + Some(reason) => format!("Mailbox wait returned: {}", mailbox_wake_label(reason)), + None => "Mailbox wait returned (cause unavailable)".to_string(), + }; + collab_event(title_text(title), mailbox_wait_report_lines(report, /*in_progress*/ false)) +} + +#[derive(Clone, Debug)] +pub(crate) struct MailboxWaitProgressHandle(Arc>); + +impl MailboxWaitProgressHandle { + pub(crate) fn update(&self, report: MailboxWaitReport, in_progress: bool) { + match self.0.lock() { + Ok(mut current) => { + current.report = report; + current.in_progress = in_progress; + } + Err(poisoned) => { + let mut current = poisoned.into_inner(); + current.report = report; + current.in_progress = in_progress; + } + } + } + + pub(crate) fn cell(&self, call_id: impl Into) -> MailboxWaitProgressCell { + MailboxWaitProgressCell { + call_id: call_id.into(), + progress: Arc::clone(&self.0), + } + } + + pub(crate) fn finish(&self) { + match self.0.lock() { + Ok(mut current) => current.in_progress = false, + Err(poisoned) => poisoned.into_inner().in_progress = false, + } + } +} + +#[derive(Debug)] +struct MailboxWaitProgress { + report: MailboxWaitReport, + in_progress: bool, +} + +#[derive(Debug)] +pub(crate) struct MailboxWaitProgressCell { + call_id: String, + progress: Arc>, +} + +impl MailboxWaitProgressCell { + pub(crate) fn call_id(&self) -> &str { + &self.call_id + } + + pub(crate) fn snapshot_cell(&self) -> PlainHistoryCell { + let progress = snapshot(&self.progress); + if progress.in_progress { + begin(&progress.report) + } else { + end(&progress.report) + } + } +} + +pub(crate) fn progress_cell( + call_id: impl Into, + report: MailboxWaitReport, +) -> (MailboxWaitProgressCell, MailboxWaitProgressHandle) { + let progress = Arc::new(Mutex::new(MailboxWaitProgress { + report, + in_progress: true, + })); + ( + MailboxWaitProgressCell { + call_id: call_id.into(), + progress: Arc::clone(&progress), + }, + MailboxWaitProgressHandle(progress), + ) +} + +impl HistoryCell for MailboxWaitProgressCell { + fn display_lines(&self, width: u16) -> Vec> { + let progress = snapshot(&self.progress); + if progress.in_progress { + begin(&progress.report).display_lines(width) + } else { + end(&progress.report).display_lines(width) + } + } + + fn raw_lines(&self) -> Vec> { + let progress = snapshot(&self.progress); + if progress.in_progress { + begin(&progress.report).raw_lines() + } else { + end(&progress.report).raw_lines() + } + } +} + +fn snapshot(progress: &Mutex) -> MailboxWaitProgress { + match progress.lock() { + Ok(progress) => MailboxWaitProgress { + report: progress.report.clone(), + in_progress: progress.in_progress, + }, + Err(poisoned) => { + let progress = poisoned.into_inner(); + MailboxWaitProgress { + report: progress.report.clone(), + in_progress: progress.in_progress, + } + } + } +} + +fn mailbox_wait_report_lines(report: &MailboxWaitReport, in_progress: bool) -> Vec> { + let mut lines = vec![Line::from(format!( + "Join condition: {}", + return_when_label(report.return_when) + ))]; + if !report.requested_targets.is_empty() { + lines.push(Line::from(format!( + "Targets: {}", + report + .requested_targets + .iter() + .map(|target| sanitize_notification_preview(target)) + .collect::>() + .join(", ") + ))); + } + if !report.requested_replies_to.is_empty() { + lines.push(Line::from(format!( + "Replies to: {}", + report + .requested_replies_to + .iter() + .map(|message_id| sanitize_notification_preview(message_id)) + .collect::>() + .join(", ") + ))); + } + for selection in &report.completed { + lines.push(mailbox_selection_line("Completed", selection)); + } + for selection in &report.pending { + lines.push(mailbox_selection_line("Pending", selection)); + } + if !in_progress { + lines.push(Line::from(format!( + "Condition satisfied: {}", + if report.condition_satisfied { "yes" } else { "no" } + ))); + if let Some(reason) = report.completion_reason { + lines.push(Line::from(format!( + "Completion reason: {}", + mailbox_wake_label(reason) + ))); + } + if let Some(sender) = report.wake_sender.as_deref() { + lines.push(Line::from(format!( + "Wake sender: {}", + sanitize_notification_preview(sender) + ))); + } + if let Some(wake_ref) = report.wake_ref.as_deref() { + lines.push(Line::from(format!( + "Wake reference: {}", + sanitize_notification_preview(wake_ref) + ))); + } + } + lines.push(Line::from(format!("Quiet count: {}", report.quiet_count))); + lines +} + +fn mailbox_selection_line(phase: &str, selection: &MailboxWaitSelectionItem) -> Line<'static> { + let mut identity = Vec::new(); + if let Some(agent) = selection.agent.as_deref() { + identity.push(sanitize_notification_preview(agent)); + } + if let Some(work_ref) = selection.work_ref.as_deref() { + identity.push(sanitize_notification_preview(work_ref)); + } + if let Some(message_id) = selection.message_id.as_deref() { + identity.push(sanitize_notification_preview(message_id)); + } + if identity.is_empty() { + identity.push("selected item".to_string()); + } + Line::from(format!( + "{phase} {}: {}", + identity.join(" · "), + sanitize_notification_preview(&selection.status) + )) +} + +fn return_when_label(return_when: MailboxWaitReturnWhen) -> &'static str { + match return_when { + MailboxWaitReturnWhen::Any => "any", + MailboxWaitReturnWhen::All => "all", + } +} + +fn mailbox_wake_label(reason: MailboxWakeCause) -> &'static str { + match reason { + MailboxWakeCause::Result => "result", + MailboxWakeCause::Reply => "reply", + MailboxWakeCause::Request => "request", + MailboxWakeCause::Human => "human", + MailboxWakeCause::Tool => "tool", + MailboxWakeCause::Deadline => "deadline", + MailboxWakeCause::Cancel => "cancel", + MailboxWakeCause::SubscriptionLoss => "subscription loss", + MailboxWakeCause::RuntimeFault => "runtime fault", + MailboxWakeCause::NoMatchingWork => "no matching work", + MailboxWakeCause::AlreadySatisfied => "already satisfied", + MailboxWakeCause::Cycle => "cycle", + MailboxWakeCause::NeedsAttention => "needs attention", + } +} diff --git a/docs/carry-divergence-ledger.md b/docs/carry-divergence-ledger.md index e1fac980e87a..6b70641796c1 100644 --- a/docs/carry-divergence-ledger.md +++ b/docs/carry-divergence-ledger.md @@ -43,6 +43,33 @@ decisions. ### Exact-target wait ignores queue-only mailbox activity +The optional `agent_mailbox` feature extends this carry for same-tree V2 only. +Its library does not depend on core and owns `mailbox_1.sqlite` independently of +native state migrations. Spawn/followup receipts expose immutable accepted work +references; quiet notes/progress do not return a sleeping wait to the provider. +Request attention is one-shot and separate from an unresolved request: only an +exact durable final reply or the author's withdrawal/replacement resolves it. +Selected work and request replies form one any/all join, while attention, +human input, and native tool input interrupt without satisfying that join. + +The maintained adapter has six authoritative seams: native work acceptance, +local delivery eligibility, actual execution-input admission, producer terminal +outcome with typed goal disposition, fallible canonical append/flush and stable-ID +reconciliation, and existing collaboration live/replay projection. Canonical +custody is not execution admission; compact checkpoints preserve selected unread +bodies. Digests and UI rows do not consume omitted bodies. Body delivery is +bounded; longer final previews identify their canonical source turn explicitly. + +Both `agent_mailbox` and recipient `agent_mailbox_idle_activation` are off by +default. Disabled mailbox handling preserves pending records and cannot drain, +acknowledge, delete, or downgrade shared native state. A compatible explicit +resume may reconcile original IDs or report recovery needed; it never replaces +accepted work, mass-resumes old agents, or claims comprehension/exactly-once +business effects. Cross-root mail, subscriptions, shadow membership, watchdogs, +new scheduling, and provider/auth changes remain outside this carry. Future +upstream syncs retain the same semantic suite and retire equivalent mechanisms +and dependencies together, rather than restoring donor framework code. + - Downstream V2 `wait_agent` keeps queue-only inter-agent mail durable and visible in later notification summaries, but an exact-target wait wakes only for terminal target state, actionable (`trigger_turn`) mail, user steer, diff --git a/docs/divergences/index.yaml b/docs/divergences/index.yaml index d25561e5dcec..53d6b572af06 100644 --- a/docs/divergences/index.yaml +++ b/docs/divergences/index.yaml @@ -3587,34 +3587,44 @@ }, { "id": "exact-target-wait-mailbox-filter", - "title": "Exact-target V2 wait filters queue-only mailbox activity", + "title": "Exact-target waits and opt-in durable V2 mailboxes", "status": "live", "category": "agent-runtime", "surface_type": "wait-agent", - "surface": ["multi-agent V2 wait_agent", "session input queue"], + "surface": ["multi-agent V2 collaboration", "canonical history", "live and replay wait reports"], "files": [ + "codex-rs/agent-mailbox", + "codex-rs/core/src/agent/control/mailbox.rs", + "codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs", + "codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs", "codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs", "codex-rs/core/src/session/input_queue.rs" ], - "behavior": "Exact-target waits ignore queue-only inter-agent mailbox activity while retaining FIFO durable messages and later notification summaries; targetless waits remain broadly mailbox-wakeable.", + "behavior": "The native default keeps exact-target queue-only filtering. Opt-in agent_mailbox supplies a core-independent library and separately versioned mailbox_1.sqlite for same-tree V2 accepted work, quiet messages, exact request replies, bounded reads, per-item observation, and any/all work/reply joins. Native acceptance and execution admission remain authoritative; canonical append and successful flush precede observation. Optional recipient idle activation never implies comprehension or business acceptance.", "upstream_equivalent": false, "introduced_in": {"carry_commit": null, "upstream_commit": null}, "guardrail_lane": "codex.core-multi-agent-orchestration-targeted", "tests": [ + "native_retry_dispatch_reopen_and_capture_backpressure_are_safe", + "admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped", + "pending_deferred_and_terminal_continuations_bind_to_exact_generation", + "quiet_reply_and_author_dispositions_settle_only_the_exact_request", + "canonical_outcomes_are_observed_per_item_without_cursor_loss", + "only_closed_any_and_all_peer_cycles_are_provable", "multi_agent_v2_wait_agent_accepts_target_and_timeout_arguments", "multi_agent_v2_wait_agent_does_not_return_completed_content", "multi_agent_v2_wait_agent_wakes_on_trigger_turn_mailbox_notification", "input_queue_tracks_pending_trigger_turn_mail" ], "owner": "downstream", - "notes": "Regression repair for exact-target waits; preserve queue-only delivery and notification visibility until upstream adopts the typed actionable-input distinction.", + "notes": "Preserve six native seams during sync: accepted-work capture inside spawn/followup acceptance; local delivery eligibility; exact execution-input admission; captured producer terminal outcome and typed goal-continuation disposition; fallible canonical append/flush with stable-ID reconciliation; existing collaboration live/replay mapping. Retain compatible source/library/history/UI tests as one semantic guardrail, not private layout snapshots. Feature disable preserves pending custody without acknowledgement, deletion, stock draining, or shared-state rewind. Explicit resume reconciles exact retained work or reports recovery needed; it never mass-resumes historical agents or claims exactly-once business effects. Cross-root mail, subscriptions, shadow hubs, watchdogs, new schedulers, and provider/auth changes are not included. Retire each downstream mechanism only when upstream owns equivalent semantics and the same consumer suite passes; remove obsolete adapters with their dependencies rather than restoring copied agent/session implementations.", "upstreamability_tier": "upstream-pr", - "boundary_type": "runtime-regression-repair", + "boundary_type": "narrow-native-adapter", "hotspot_files": [ "codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs", "codex-rs/core/src/session/input_queue.rs" ], - "extraction_target": "upstream exact-target wait mailbox filtering" + "extraction_target": "upstream-equivalent accepted work and mailbox settlement semantics" }, { "id": "rmcp-streamable-http-cancellation", diff --git a/docs/downstream-regression-matrix.md b/docs/downstream-regression-matrix.md index bfab3b57e24b..071d6b97feb5 100644 --- a/docs/downstream-regression-matrix.md +++ b/docs/downstream-regression-matrix.md @@ -50,6 +50,18 @@ server, and natural first use must verify the actual request path. ## Core default path +The opt-in same-tree V2 `agent_mailbox` path is covered by the standalone +`codex-agent-mailbox` behavioral suite in +`codex.core-multi-agent-orchestration-targeted`, plus real core admission/history, +wrapped tool-schema, and collaboration live/replay tests. The native path remains +the default. Qualify accepted-work retries and caller cancellation, exact +incremental provider admission, mixed issuer/outcome and request joins, fixed +deadlines across quiet mail, typed goal continuation, canonical flush failure and +compaction, activation denial, backpressure, and conservative closed-peer cycles. +Library tests alone do not prove the native adapter, UI, final packages, landing, +or an installed session. Linux preview packages remain exact-SHA artifacts; +install/restart and runtime adoption are separate operator actions. + Exact-target V2 `wait_agent` ignores queue-only (`trigger_turn=false`) mailbox activity while preserving durable FIFO delivery and notification summaries; targetless waits retain broad mailbox wake behavior. The focused guardrail is diff --git a/docs/downstream-tool-surface-matrix.md b/docs/downstream-tool-surface-matrix.md index 4cd468024c33..d9a7f3cb06f9 100644 --- a/docs/downstream-tool-surface-matrix.md +++ b/docs/downstream-tool-surface-matrix.md @@ -9,6 +9,19 @@ divergences. Last reviewed: `2026-08-16`. +Opt-in mailbox increment (`agent_mailbox`, V2 only): `send_message` advertises +the bundled encrypted message contract with note/progress/request/alert intent, +exact work/reply/replacement references, legacy one-shot attention, and separate +recipient-opted-in idle activation. Its stable message receipt proves durable +acceptance, not acknowledgement or comprehension. Spawn/followup add an optional +immutable `work_ref`. `wait_agent` joins optional agent snapshots/exact work refs +and request IDs with any/all semantics; omitted timeout is event-only, selected +failures return pending refs, and attention does not satisfy the join. +`mailbox_read` is bounded and paginated: native canonical AgentMessage carries +the body, and the JSON result contains metadata rather than duplicate ciphertext. +The same structured report drives the tool result and live/replay display. +Disabling the feature restores native schemas without consuming pending mail. + Review baseline: - live `openai/codex` `main`: `49db349ffdd888f5e3c91abf9b7519d8631e6e9a` diff --git a/justfile b/justfile index 4d5e82d816d1..619a774372ed 100644 --- a/justfile +++ b/justfile @@ -377,6 +377,7 @@ tui-brokered-tool-replay-targeted: # and generation-safe V2 residency eviction. core-multi-agent-orchestration-targeted: + cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_spawn_cancellation_after_initial_delivery_returns_published_result context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --test all -- suite::spawn_agent_description::spawn_agent_description_lists_visible_models_and_reasoning_efforts suite::agent_execution::v2_evicted_completed_agent_keeps_final_status suite::agent_execution::v2_cold_mailbox_allows_eviction_and_replays_on_followup suite::multi_agent_mode::changing_configured_mode_hint_to_empty_appends_explicit_reset suite::pending_input::queue_only_agent_mail_wakes_sleeping_root_and_persists_message --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-exec --test all suite::completion_backfill::ignores_unrelated_turn_completion_before_backfilling_primary_turn -- --exact --test-threads=1 From f1202886071a28e874565613e15ed9666a92080e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:50:19 +1100 Subject: [PATCH 03/54] Repair mailbox custody and retry semantics with native regressions --- .../workflows/mailbox-source-preparation.yml | 5 +- codex-rs/agent-mailbox/src/activation.rs | 2 + codex-rs/agent-mailbox/src/mailbox_tests.rs | 175 ++++- .../agent-mailbox/src/message_delivery.rs | 16 +- codex-rs/agent-mailbox/src/messages.rs | 31 +- codex-rs/agent-mailbox/src/outcome.rs | 52 +- .../agent-mailbox/src/outcome_messages.rs | 54 +- codex-rs/agent-mailbox/src/schema.rs | 22 +- codex-rs/agent-mailbox/src/store.rs | 4 +- codex-rs/agent-mailbox/src/types.rs | 1 + codex-rs/agent-mailbox/src/wait_graph.rs | 21 +- codex-rs/agent-mailbox/src/work.rs | 16 +- codex-rs/core/src/agent/control/mailbox.rs | 29 +- codex-rs/core/src/session/mod.rs | 14 +- codex-rs/core/src/session/tests.rs | 476 +++++++++++ codex-rs/core/src/session/turn.rs | 72 +- .../src/tools/handlers/multi_agents_spec.rs | 17 +- .../src/tools/handlers/multi_agents_v2.rs | 1 + .../handlers/multi_agents_v2/followup_task.rs | 25 + .../multi_agents_v2/mailbox_message.rs | 39 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 117 ++- .../handlers/multi_agents_v2/message_tool.rs | 12 +- codex-rs/core/src/tools/parallel.rs | 171 +++- codex-rs/core/src/tools/spec_plan.rs | 14 +- codex-rs/core/tests/suite/agent_mailbox.rs | 739 ++++++++++++++++++ codex-rs/core/tests/suite/mod.rs | 1 + codex-rs/tui/src/multi_agents/mailbox_wait.rs | 7 +- 27 files changed, 1989 insertions(+), 144 deletions(-) create mode 100644 codex-rs/core/tests/suite/agent_mailbox.rs diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 389096620876..799c8789c270 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "1" + PREPARATION_ATTEMPT: "2" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -60,6 +60,9 @@ jobs: - name: Compile focused core mailbox and schema consumers working-directory: codex-rs run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 + - name: Run real native mailbox adapter contracts + working-directory: codex-rs + run: cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 - name: Compile focused UI mailbox consumers working-directory: codex-rs env: diff --git a/codex-rs/agent-mailbox/src/activation.rs b/codex-rs/agent-mailbox/src/activation.rs index fc37d3aba8fa..bf38d73cb509 100644 --- a/codex-rs/agent-mailbox/src/activation.rs +++ b/codex-rs/agent-mailbox/src/activation.rs @@ -108,6 +108,7 @@ fn activation_disposition_string(disposition: ActivationDisposition) -> &'static ActivationDisposition::AlreadyActive => "already_active", ActivationDisposition::Activated => "activated", ActivationDisposition::Denied => "denied", + ActivationDisposition::Unknown => "unknown", } } @@ -117,6 +118,7 @@ fn parse_activation_disposition(value: &str) -> MailboxResult Ok(ActivationDisposition::AlreadyActive), "activated" => Ok(ActivationDisposition::Activated), "denied" => Ok(ActivationDisposition::Denied), + "unknown" => Ok(ActivationDisposition::Unknown), _ => Err(MailboxError::InvalidInput("stored activation disposition".into())), } } diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 340770e8a351..15cc835e637a 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -6,8 +6,7 @@ use crate::{ Selector, Snapshot, SnapshotItem, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, }; use codex_protocol::{ - protocol::{AgentPath, InterAgentCommunication}, - ThreadId, + protocol::InterAgentCommunication, AgentPath, ThreadId, }; use codex_state::SqliteConfig; use codex_utils_absolute_path::test_support::PathExt; @@ -180,13 +179,63 @@ async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { assert!(fixture.mailbox.work_for_native_key(issuer, "too-large").await.unwrap().is_none()); } +#[tokio::test] +async fn native_accept_retry_after_pre_dispatch_handle_drop_reuses_original_identity() { + let TestMailbox { _home, config, mailbox } = test_mailbox().await; + let (issuer, agent) = (thread_id(22), thread_id(23)); + let original = native_work(issuer, agent, "pre-dispatch-drop", "preserve this accepted input"); + let accepted = mailbox.accept_work(original.clone()).await.unwrap(); + let accepted_input_id = mailbox + .accepted_input(&accepted) + .await + .unwrap() + .id + .unwrap() + .to_string(); + assert!(accepted_input_id.starts_with("amsg_")); + + // The durable accept has happened, but the caller drops the handle before dispatch begins. + drop(mailbox); + let retried_mailbox = Mailbox::open(&config).await.unwrap(); + let retried = retried_mailbox.accept_work(original).await.unwrap(); + assert_eq!(retried, accepted); + assert_eq!( + retried_mailbox.work_for_native_key(issuer, "pre-dispatch-drop").await.unwrap(), + Some(accepted.clone()) + ); + assert_eq!( + retried_mailbox + .accepted_input(&retried) + .await + .unwrap() + .id + .unwrap() + .to_string(), + accepted_input_id + ); + + assert_eq!( + retried_mailbox.begin_dispatch(&retried).await.unwrap(), + DispatchDecision::Dispatch + ); + assert_eq!( + retried_mailbox.begin_dispatch(&retried).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + retried_mailbox.mark_dispatched(&retried).await.unwrap(); + assert_eq!( + retried_mailbox.begin_dispatch(&retried).await.unwrap(), + DispatchDecision::AlreadyDispatched + ); +} + #[tokio::test] async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped() { let fixture = test_mailbox().await; let (issuer, other_issuer, agent) = (thread_id(3), thread_id(4), thread_id(5)); let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "a", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "b", "B").await; - let (c, id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; + let (c, _id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; assert_eq!( admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]).await.work_refs, vec![a.clone()] @@ -490,9 +539,123 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( let pending = fixture.mailbox.pending(recipient, 20).await.unwrap(); assert_eq!(pending.len(), 8); for kind in kinds { assert_eq!(pending.iter().filter(|m| m.kind == kind).count(), 2); } - assert_eq!(pending.iter().filter(|m| m.legacy_attention).count(), 4); - assert_eq!(fixture.mailbox.quiet_count(recipient).await.unwrap(), 4); - assert_eq!(fixture.mailbox.attention(recipient, 20).await.unwrap().len(), 4); + assert_eq!(pending.iter().filter(|m| m.legacy_attention).count(), 1); + assert_eq!(fixture.mailbox.quiet_count(recipient).await.unwrap(), 3); + assert_eq!(fixture.mailbox.attention(recipient, 20).await.unwrap().len(), 5); +} + +#[tokio::test] +async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_request() { + let fixture = test_mailbox().await; + let (sender, recipient) = (thread_id(24), thread_id(25)); + let request = send( + &fixture.mailbox, + draft("attention-request", sender, recipient, MessageKind::Request, "please decide", false), + ) + .await; + let alert = send( + &fixture.mailbox, + draft("attention-alert", sender, recipient, MessageKind::Alert, "needs attention", false), + ) + .await; + let note = send( + &fixture.mailbox, + draft("quiet-note", sender, recipient, MessageKind::Note, "quiet note", false), + ) + .await; + let progress = send( + &fixture.mailbox, + draft("quiet-progress", sender, recipient, MessageKind::Progress, "quiet progress", false), + ) + .await; + + let legacy_draft = draft( + "legacy-attention-once", sender, recipient, MessageKind::Note, "legacy wake", true, + ); + let legacy = send(&fixture.mailbox, legacy_draft.clone()).await; + assert_eq!(send(&fixture.mailbox, legacy_draft).await.id, legacy.id); + + let uncanonical = fixture.mailbox.pending_page(recipient, None, 20).await.unwrap(); + let uncanonical_ids = uncanonical.items.iter().map(|item| item.id.as_str()).collect::>(); + for message in [&request, &alert, ¬e, &progress, &legacy] { + assert!(uncanonical_ids.contains(&message.id.as_str())); + fixture.mailbox.mark_canonical(recipient, &message.id).await.unwrap(); + } + let canonical = fixture.mailbox.read_page(recipient, None, 20).await.unwrap(); + assert!(canonical.items.iter().any(|message| message.id == request.id)); + assert!(canonical.items.iter().any(|message| message.id == alert.id)); + + let mut attention_ids = fixture + .mailbox + .attention(recipient, 20) + .await + .unwrap() + .into_iter() + .map(|attention| attention.id) + .collect::>(); + attention_ids.sort(); + let mut expected_attention_ids = vec![ + format!("attention:{}", request.id), + format!("attention:{}", alert.id), + format!("attention:{}", legacy.id), + ]; + expected_attention_ids.sort(); + assert_eq!(attention_ids, expected_attention_ids); + + let observations = attention_ids + .iter() + .cloned() + .map(ObservationTarget::Attention) + .collect::>(); + fixture + .mailbox + .acknowledge_observed(recipient, &observations) + .await + .unwrap(); + assert!(fixture.mailbox.attention(recipient, 20).await.unwrap().is_empty()); + + let old_request = take_snapshot( + &fixture.mailbox, + sender, + Some(recipient), + vec![Selector::Reply(request.id.clone())], + ReturnWhen::Any, + ) + .await; + assert_eq!(old_request.state, SnapshotState::Pending); + assert_eq!(request_item(&old_request, &request.id).disposition, RequestDisposition::Open); + + let new_request = send( + &fixture.mailbox, + draft( + "new-attention-request", + sender, + recipient, + MessageKind::Request, + "new request", + false, + ), + ) + .await; + fixture.mailbox.mark_canonical(recipient, &new_request.id).await.unwrap(); + let new_attention = fixture.mailbox.attention(recipient, 20).await.unwrap(); + let new_attention_ids = new_attention + .iter() + .map(|attention| attention.id.clone()) + .collect::>(); + assert_eq!( + new_attention_ids, + vec![format!("attention:{}", new_request.id)] + ); + fixture + .mailbox + .acknowledge_observed( + recipient, + &[ObservationTarget::Attention(format!("attention:{}", new_request.id))], + ) + .await + .unwrap(); + assert!(fixture.mailbox.attention(recipient, 20).await.unwrap().is_empty()); } #[tokio::test] diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index b856639f1957..8de8699aff21 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -81,7 +81,11 @@ impl Mailbox { pub async fn withdraw_message(&self, sender: ThreadId, id: &str) -> MailboxResult<()> { let mut transaction = self.pool.begin().await?; let row = sqlx::query( - "SELECT sender, kind, canonical, input_admitted, legacy_attention, superseded_by, withdrawn + "SELECT sender, kind, canonical, input_admitted, legacy_attention, + CASE WHEN kind IN ('\"note\"', '\"progress\"') + AND (kind = '\"progress\"' OR legacy_attention = 0) + THEN 1 ELSE 0 END AS quiet, + superseded_by, withdrawn FROM mailbox_messages WHERE id = ?", ) .bind(id) @@ -113,7 +117,7 @@ impl Mailbox { } let canonical = row.try_get::("canonical")? != 0; let input_admitted = row.try_get::("input_admitted")? != 0; - let quiet = row.try_get::("legacy_attention")? == 0; + let quiet = row.try_get::("quiet")? != 0; let updated = sqlx::query( "UPDATE mailbox_messages SET withdrawn = 1 WHERE id = ? AND sender = ? AND withdrawn = 0", @@ -310,7 +314,11 @@ impl Mailbox { pub async fn mark_canonical(&self, recipient: ThreadId, id: &str) -> MailboxResult<()> { let mut transaction = self.pool.begin().await?; let current = sqlx::query( - "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn FROM mailbox_messages + "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn, + CASE WHEN kind IN ('\"note\"', '\"progress\"') + AND (kind = '\"progress\"' OR legacy_attention = 0) + THEN 1 ELSE 0 END AS quiet + FROM mailbox_messages WHERE id = ? AND recipient = ?", ) .bind(id) @@ -320,7 +328,7 @@ impl Mailbox { .ok_or(MailboxError::WrongRecipient)?; let was_canonical = current.try_get::("canonical")? != 0; let input_admitted = current.try_get::("input_admitted")? != 0; - let quiet = current.try_get::("legacy_attention")? == 0; + let quiet = current.try_get::("quiet")? != 0; let is_current = current.try_get::, _>("superseded_by")?.is_none() && current.try_get::("withdrawn")? == 0; let result = sqlx::query( diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index a0d7f95c4006..bb6af62e8238 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -1,7 +1,8 @@ use crate::error::{MailboxError, MailboxResult}; +use crate::message_delivery::envelope_from_row; +use crate::message_rules::{validate_reply_tx, validate_replacement_tx}; use crate::store::Mailbox; use crate::types::{Envelope, MessageDraft, MessageKind, WorkRef}; -use crate::message_rules::{validate_reply_tx, validate_replacement_tx}; use crate::work::exact_work; use codex_protocol::{ResponseItemId, ThreadId}; use serde::Serialize; @@ -46,6 +47,7 @@ impl Mailbox { transaction.commit().await?; envelope }; + self.invalidate_waits(envelope.recipient); self.notify_changed(); Ok(envelope) } @@ -58,6 +60,7 @@ pub(crate) async fn insert_message_tx( explicit_id: Option, ) -> MailboxResult { validate_key(&draft.invocation_key)?; + draft.legacy_attention &= draft.kind == MessageKind::Note; if draft.kind == MessageKind::Progress && draft.reply_to.is_some() { return Err(MailboxError::InvalidInput( "progress messages cannot be replies".into(), @@ -73,8 +76,8 @@ pub(crate) async fn insert_message_tx( fingerprint.communication.id = None; let request_json = serde_json::to_string(&DraftRequest { invocation_key: &fingerprint.invocation_key, - sender: &fingerprint.sender, - recipient: &fingerprint.recipient, + sender: fingerprint.sender, + recipient: fingerprint.recipient, communication: &fingerprint.communication, kind: fingerprint.kind, activate_if_idle: fingerprint.activate_if_idle, @@ -168,7 +171,7 @@ pub(crate) async fn insert_message_tx( ) .await?; add_invocation_tx(transaction, &draft.invocation_key, &request_json, &existing_id).await?; - if draft.legacy_attention { + if has_attention(&draft) { insert_attention_tx(transaction, &existing_id, draft.recipient.to_string().as_str()).await?; } let row = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") @@ -240,7 +243,10 @@ pub(crate) async fn insert_message_tx( add_invocation_tx(transaction, &draft.invocation_key, &request_json, &id).await?; if let Some(old_id) = draft.supersedes.as_deref() { let old = sqlx::query( - "SELECT recipient, canonical, input_admitted, legacy_attention + "SELECT recipient, canonical, input_admitted, legacy_attention, + CASE WHEN kind IN ('\"note\"', '\"progress\"') + AND (kind = '\"progress\"' OR legacy_attention = 0) + THEN 1 ELSE 0 END AS quiet FROM mailbox_messages WHERE id = ? AND superseded_by IS NULL AND withdrawn = 0", ) @@ -261,7 +267,7 @@ pub(crate) async fn insert_message_tx( || old.try_get::("input_admitted")? == 0; if unread { let recipient: String = old.try_get("recipient")?; - let quiet = old.try_get::("legacy_attention")? == 0; + let quiet = old.try_get::("quiet")? != 0; decrement_pending_count(transaction, &recipient, quiet).await?; } } @@ -273,10 +279,10 @@ pub(crate) async fn insert_message_tx( increment_pending_count( transaction, &draft.recipient.to_string(), - !draft.legacy_attention, + is_quiet_kind(draft.kind, draft.legacy_attention), ) .await?; - if draft.legacy_attention { + if has_attention(&draft) { insert_attention_tx(transaction, &id, draft.recipient.to_string().as_str()).await?; } let row = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") @@ -405,6 +411,15 @@ fn validate_key(key: &str) -> MailboxResult<()> { Ok(()) } +pub(crate) fn is_quiet_kind(kind: MessageKind, legacy_attention: bool) -> bool { + matches!(kind, MessageKind::Note | MessageKind::Progress) + && !(kind == MessageKind::Note && legacy_attention) +} + +fn has_attention(draft: &MessageDraft) -> bool { + draft.legacy_attention || matches!(draft.kind, MessageKind::Request | MessageKind::Alert) +} + fn validate_id(id: &str) -> MailboxResult<()> { if id.starts_with("amsg_") && id.len() > 5 && id.len() <= 256 { Ok(()) diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index b6248194f9eb..0596bfd9a41c 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -1,6 +1,7 @@ use crate::error::{MailboxError, MailboxResult}; use crate::outcome_messages::{ - insert_response_group, response_groups, validate_auto_response_size, ResponseGroup, + ensure_terminal_responses, insert_response_group, response_groups, + validate_auto_response_size, }; use crate::outcome_store::{ outcome_receipt_for, persist_continuation, save_outcome_tx, OutcomeData, @@ -182,6 +183,11 @@ impl Mailbox { .await?; } transaction.commit().await?; + for group in &groups { + if group.response.is_some() { + self.invalidate_waits(group.issuer); + } + } self.notify_changed(); return Ok(TurnOutcomeReceipt { outcomes: Vec::new(), @@ -191,6 +197,15 @@ impl Mailbox { }); } + ensure_terminal_responses( + &mut groups, + &refs, + outcome, + &continuation, + source_turn_id, + )?; + validate_auto_response_size(&groups)?; + let binding_json = match &continuation { Continuation::Terminal(binding) => Some(serde_json::to_string(binding)?), _ => None, @@ -288,6 +303,11 @@ impl Mailbox { } } transaction.commit().await?; + for group in &groups { + if group.response.is_some() { + self.invalidate_waits(group.issuer); + } + } if !already_recorded || binding_json.is_some() { self.notify_changed(); } @@ -297,13 +317,12 @@ impl Mailbox { .iter() .find(|group| group.issuer == work_ref.issuer) .and_then(|group| group.output_id.clone()); - let response_id = groups + let has_response = groups .iter() .find(|group| group.issuer == work_ref.issuer) - .and_then(|group| group.response.as_ref().map(|_| group.output_id.clone())) - .flatten(); + .is_some_and(|group| group.response.is_some()); receipts.push( - outcome_receipt_for(&self.pool, work_ref, group_id, response_id).await?, + outcome_receipt_for(&self.pool, work_ref, group_id, has_response).await?, ); } Ok(TurnOutcomeReceipt { @@ -333,8 +352,18 @@ impl Mailbox { } let mut transaction = self.pool.begin().await?; let mut groups = response_groups(&mut transaction, &[work_ref.clone()], response.as_ref()).await?; + let group = groups.pop().ok_or(MailboxError::ReferenceMismatch)?; + let mut groups = vec![group]; + ensure_terminal_responses( + &mut groups, + std::slice::from_ref(work_ref), + outcome, + &continuation, + &work_ref.id, + )?; let mut group = groups.pop().ok_or(MailboxError::ReferenceMismatch)?; validate_auto_response_size(std::slice::from_ref(&group))?; + let has_response = group.response.is_some(); let data = OutcomeData::new(outcome, continuation.clone(), group.response.clone()); let payload = serde_json::to_string(&data)?; let row = exact_work_tx(&mut transaction, work_ref).await?; @@ -344,7 +373,7 @@ impl Mailbox { } let id: String = row.try_get("outcome_id")?; transaction.commit().await?; - return outcome_receipt_for(&self.pool, work_ref.clone(), Some(id), response.is_some()).await; + return outcome_receipt_for(&self.pool, work_ref.clone(), Some(id), has_response).await; } if let Continuation::Terminal(binding) = &continuation { let binding_json = serde_json::to_string(binding)?; @@ -366,14 +395,14 @@ impl Mailbox { } group.output_id = insert_response_group( &mut transaction, - &group, + &mut group, format!("work-outcome:{}", work_ref.id), outcome, ) .await?; - let output_id = group.output_id.as_deref().unwrap_or_else(|| new_mailbox_id()); - ensure_mailbox_id(output_id)?; - save_outcome_tx(&mut transaction, work_ref, &payload, output_id, outcome).await?; + let output_id = group.output_id.clone().unwrap_or_else(new_mailbox_id); + ensure_mailbox_id(&output_id)?; + save_outcome_tx(&mut transaction, work_ref, &payload, &output_id, outcome).await?; if let Continuation::Terminal(binding) = &continuation { sqlx::query( "UPDATE mailbox_continuations SET state = 'terminal' @@ -386,8 +415,9 @@ impl Mailbox { .await?; } transaction.commit().await?; + self.invalidate_waits(group.issuer); self.notify_changed(); - outcome_receipt_for(&self.pool, work_ref.clone(), Some(output_id.to_string()), response.is_some()).await + outcome_receipt_for(&self.pool, work_ref.clone(), Some(output_id), has_response).await } } diff --git a/codex-rs/agent-mailbox/src/outcome_messages.rs b/codex-rs/agent-mailbox/src/outcome_messages.rs index 4653ad62ef32..8ff34febf6a0 100644 --- a/codex-rs/agent-mailbox/src/outcome_messages.rs +++ b/codex-rs/agent-mailbox/src/outcome_messages.rs @@ -1,9 +1,10 @@ use crate::error::{MailboxError, MailboxResult}; use crate::messages::insert_message_tx; -use crate::types::{MessageDraft, MessageKind, Outcome, WorkRef}; +use crate::types::{Continuation, MessageDraft, MessageKind, Outcome, WorkRef}; use crate::work::exact_work_tx; -use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::{AgentCommunicationOrigin, InterAgentCommunication}; use codex_protocol::{AgentPath, ThreadId}; +use serde::Serialize; use sqlx::{Row, Sqlite, Transaction}; const MAX_AUTO_RESPONSE_BYTES: usize = 16_384; @@ -78,6 +79,55 @@ pub(crate) fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxRe Ok(()) } +#[derive(Serialize)] +struct NativeOutcomeRecord<'a> { + work_refs: Vec<&'a str>, + outcome: Outcome, + continuation: &'a Continuation, + source_ref: &'a str, +} + +/// Ensure every terminal work outcome has a canonical, typed native envelope +/// even when the caller supplied no prose response. Nonterminal goal linkage +/// is deliberately excluded by the caller. +pub(crate) fn ensure_terminal_responses( + groups: &mut [ResponseGroup], + refs: &[WorkRef], + outcome: Outcome, + continuation: &Continuation, + source_ref: &str, +) -> MailboxResult<()> { + for group in groups.iter_mut().filter(|group| group.response.is_none()) { + let work_refs = refs + .iter() + .filter(|work_ref| work_ref.issuer == group.issuer) + .map(|work_ref| work_ref.id.as_str()) + .collect::>(); + if work_refs.is_empty() { + return Err(MailboxError::ReferenceMismatch); + } + let content = serde_json::to_string(&NativeOutcomeRecord { + work_refs, + outcome, + continuation, + source_ref, + })?; + if content.len() > MAX_RESPONSE_BYTES { + return Err(MailboxError::Backpressure("typed outcome envelope byte limit")); + } + let mut communication = InterAgentCommunication::new( + group.author.clone(), + group.recipient.clone(), + Vec::new(), + content, + false, + ); + communication.origin = Some(AgentCommunicationOrigin::Result); + group.response = Some(communication); + } + Ok(()) +} + pub(crate) async fn insert_response_group( transaction: &mut Transaction<'_, Sqlite>, group: &mut ResponseGroup, diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 99a7f2a6a5ed..2e20b7125849 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -132,14 +132,16 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; - sqlx::query( - "INSERT OR IGNORE INTO mailbox_input_admissions( - recipient, source_turn_id, input_message_id, work_id - ) SELECT recipient, source_turn_id, input_message_id, work_id - FROM mailbox_admissions", - ) - .execute(&mut *transaction) - .await?; + if version.is_some_and(|version| version < SCHEMA_VERSION) { + sqlx::query( + "INSERT OR IGNORE INTO mailbox_input_admissions( + recipient, source_turn_id, input_message_id, work_id + ) SELECT recipient, source_turn_id, input_message_id, work_id + FROM mailbox_admissions", + ) + .execute(&mut *transaction) + .await?; + } sqlx::query( "CREATE INDEX IF NOT EXISTS mailbox_input_admissions_turn ON mailbox_input_admissions(recipient, source_turn_id, input_message_id)", @@ -265,7 +267,9 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { WHERE message.recipient = mailbox_pending_counts.recipient AND (message.canonical = 0 OR message.input_admitted = 0) AND message.superseded_by IS NULL - AND message.withdrawn = 0 AND message.legacy_attention = 0 + AND message.withdrawn = 0 + AND message.kind IN ('\"note\"', '\"progress\"') + AND (message.kind = '\"progress\"' OR message.legacy_attention = 0) )", ) .execute(&mut *transaction) diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index fe9f0ac59426..6aba8f803472 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -1,6 +1,6 @@ use crate::error::MailboxResult; -use crate::schema; use std::collections::HashMap; +use std::sync::atomic::AtomicU64; use std::sync::{Arc, Mutex}; use sqlx::SqlitePool; use tokio::sync::watch; @@ -10,6 +10,7 @@ pub struct Mailbox { pub(crate) pool: SqlitePool, change_tx: watch::Sender, pub(crate) waits: Arc>>, + pub(crate) wait_generation: Arc, pub(crate) runtime_faults: Arc>, } @@ -24,6 +25,7 @@ impl Mailbox { pool, change_tx, waits: Arc::new(Mutex::new(HashMap::new())), + wait_generation: Arc::new(AtomicU64::new(1)), runtime_faults: Arc::new(Mutex::new(crate::runtime_fault::RuntimeFaults::default())), }) } diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 72377ce17f42..27ce8a5640be 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -35,6 +35,7 @@ pub enum ActivationDisposition { AlreadyActive, Activated, Denied, + Unknown, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs index 3eb183441808..1ef1188e9399 100644 --- a/codex-rs/agent-mailbox/src/wait_graph.rs +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -4,6 +4,7 @@ use crate::types::ReturnWhen; use codex_protocol::ThreadId; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, Weak}; +use std::sync::atomic::Ordering; const MAX_ACTIVE_WAITS: usize = 4_096; const MAX_PEERS_PER_WAIT: usize = 100; @@ -15,12 +16,14 @@ pub(crate) struct ActiveWait { peers: Vec, mode: ReturnWhen, has_escape: bool, + generation: u64, } /// RAII handle for one transient wait dependency. Dropping it removes only the /// exact registration; restarting the process never re-arms persisted waits. pub struct WaitRegistration { wait_id: String, + generation: u64, waits: Weak>>, } @@ -34,12 +37,25 @@ impl Drop for WaitRegistration { fn drop(&mut self) { if let Some(waits) = self.waits.upgrade() { let mut waits = lock(&waits); - waits.remove(&self.wait_id); + if waits + .get(&self.wait_id) + .is_some_and(|wait| wait.generation == self.generation) + { + waits.remove(&self.wait_id); + } } } } impl Mailbox { + /// Invalidate active dependency edges for an owner when eligible input is + /// about to become available. Stale RAII handles cannot remove replacements. + pub fn invalidate_waits(&self, owner: ThreadId) { + let owner = owner.to_string(); + let mut waits = lock(&self.waits); + waits.retain(|_, wait| wait.owner != owner); + } + /// Register only currently active peer dependencies. The caller sets /// `has_escape` for a finite deadline or any already-eligible input. /// A new registration is rejected only when the graph proves it closed. @@ -71,6 +87,7 @@ impl Mailbox { if waits.len() >= MAX_ACTIVE_WAITS { return Err(MailboxError::Backpressure("active wait registration count")); } + let generation = self.wait_generation.fetch_add(1, Ordering::Relaxed); waits.insert( wait_id.clone(), ActiveWait { @@ -78,6 +95,7 @@ impl Mailbox { peers, mode, has_escape, + generation, }, ); if is_closed_owner(&waits, &owner.to_string()) { @@ -86,6 +104,7 @@ impl Mailbox { } Ok(WaitRegistration { wait_id, + generation, waits: Arc::downgrade(&self.waits), }) } diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index f3ce61130d78..1fc423af1c73 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -236,7 +236,14 @@ impl Mailbox { match state { DispatchState::Dispatching => { sqlx::query( - "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' WHERE id = ?", + "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' + WHERE id = ? AND dispatch_state = 'dispatching' + AND outcome_json IS NULL + AND NOT EXISTS ( + SELECT 1 FROM mailbox_continuations AS continuation + WHERE continuation.work_id = mailbox_work.id + AND continuation.state IN ('pending_goal_binding', 'deferred') + )", ) .bind(&work_ref.id) .execute(&mut *transaction) @@ -360,7 +367,10 @@ impl Mailbox { } let message = sqlx::query( - "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn + "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn, + CASE WHEN kind IN ('\"note\"', '\"progress\"') + AND (kind = '\"progress\"' OR legacy_attention = 0) + THEN 1 ELSE 0 END AS quiet FROM mailbox_messages WHERE id = ? AND recipient = ? LIMIT 1", ) .bind(input_id) @@ -374,7 +384,7 @@ impl Mailbox { let canonical = message.try_get::("canonical")? != 0; let active = message.try_get::, _>("superseded_by")?.is_none() && message.try_get::("withdrawn")? == 0; - let quiet = message.try_get::("legacy_attention")? == 0; + let quiet = message.try_get::("quiet")? != 0; sqlx::query( "UPDATE mailbox_messages SET input_admitted = 1 WHERE id = ? AND recipient = ? AND input_admitted = 0", diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 6f35471815a4..e238bbe7b4ca 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -238,6 +238,29 @@ impl AgentControl { .map(|envelope| envelope.communication).collect()) } + pub(crate) async fn mailbox_reconcile_pending_work(&self, work: &WorkRef) -> CodexResult { + let Some(mailbox) = self.mailbox() else { + return Ok(false); + }; + let state = self.upgrade()?; + let can_proceed = match state.get_thread(work.agent).await { + Ok(thread) => { + let active = { thread.session.active_turn.lock().await.is_some() }; + active || thread.session.input_queue.has_pending_mailbox_items().await + || thread.session.input_queue.has_pending_terminal_finalizers() + || thread.session.has_outstanding_durable_sleep() + } + Err(_) => false, + }; + if can_proceed { + return Ok(false); + } + // This only changes recovery disposition, never replays input or invents a result. + // The store rechecks terminal/continuation custody atomically against this native read. + mailbox.mark_recovery_needed(work).await.map_err(storage_error)?; + Ok(true) + } + pub(crate) async fn mailbox_record_admission( &self, recipient: ThreadId, @@ -301,7 +324,9 @@ impl AgentControl { revision: snapshot.revision, source_turn_id: source_turn_id.to_string(), }; - match snapshot.phase { + if matches!(outcome, Outcome::Failed | Outcome::Cancelled) { + Continuation::Terminal(binding) + } else { match snapshot.phase { GoalNotificationPhase::Running | GoalNotificationPhase::ContinuationPending => { Continuation::PendingGoalBinding(binding) } @@ -311,7 +336,7 @@ impl AgentControl { Continuation::Deferred(binding) } GoalNotificationPhase::ResultReady => Continuation::Terminal(binding), - } + } } } None => Continuation::None, }; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index faa8d7e5f325..e41b264aea56 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2016,6 +2016,7 @@ impl Session { } let mailbox_terminal_before_publish = self.services.agent_control.mailbox_enabled() && turn_context.multi_agent_version == MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox) && matches!( &legacy_source, EventMsg::TurnComplete(_) | EventMsg::TurnAborted(_) @@ -2097,7 +2098,9 @@ impl Session { return None; }; - let mailbox_enabled = self.services.agent_control.mailbox_enabled(); + let mailbox_enabled = self.services.agent_control.mailbox_enabled() + && turn_context.multi_agent_version == MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox); let terminal_error = turn_context.terminal_error.lock().await.take(); let status = match terminal_error.as_ref() { Some(error) => { @@ -3420,7 +3423,9 @@ impl Session { }, RolloutItem::ResponseItem(response_item), ]; - let mailbox_enabled = self.services.agent_control.mailbox_enabled(); + let mailbox_enabled = self.services.agent_control.mailbox_enabled() + && turn_context.multi_agent_version == MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox); let custody_gate = if mailbox_enabled { Some(self.mailbox_custody_gate.lock().await) } else { @@ -3599,7 +3604,10 @@ impl Session { }; // Compaction starts a new history window, so its WorldState baseline must be full. let mut world_state_item = None; - if self.services.agent_control.mailbox_enabled() { + if self.services.agent_control.mailbox_enabled() + && self.features.enabled(Feature::AgentMailbox) + && self.multi_agent_version() == Some(MultiAgentVersion::V2) + { let _custody_gate = self.mailbox_custody_gate.lock().await; self.live_thread().ok_or_else(|| { std::io::Error::other( diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index b745e09738c5..c49fe8fba5c3 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2297,6 +2297,482 @@ async fn record_inter_agent_communication_preserves_item_id_in_rollout_and_resum ); } +struct FaultInjectingThreadStore { + inner: Arc, + fail_append_once: std::sync::atomic::AtomicBool, + fail_flush_once: std::sync::atomic::AtomicBool, +} + +impl FaultInjectingThreadStore { + fn new(inner: Arc) -> Self { + Self { + inner, + fail_append_once: std::sync::atomic::AtomicBool::new(false), + fail_flush_once: std::sync::atomic::AtomicBool::new(false), + } + } +} + +impl codex_thread_store::ThreadStore for FaultInjectingThreadStore { + fn as_any(&self) -> &dyn std::any::Any { + self.inner.as_any() + } + + fn create_thread( + &self, + params: codex_thread_store::CreateThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::create_thread(self.inner.as_ref(), params) + } + + fn resume_thread( + &self, + params: codex_thread_store::ResumeThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::resume_thread(self.inner.as_ref(), params) + } + + fn append_items( + &self, + params: codex_thread_store::AppendThreadItemsParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + if self + .fail_append_once + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Box::pin(async { + Err(codex_thread_store::ThreadStoreError::Internal { + message: "scripted mailbox append failure".to_string(), + }) + }); + } + codex_thread_store::ThreadStore::append_items(self.inner.as_ref(), params) + } + + fn persist_thread( + &self, + thread_id: ThreadId, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::persist_thread(self.inner.as_ref(), thread_id) + } + + fn flush_thread( + &self, + thread_id: ThreadId, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + if self + .fail_flush_once + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Box::pin(async { + Err(codex_thread_store::ThreadStoreError::Internal { + message: "scripted mailbox flush failure".to_string(), + }) + }); + } + codex_thread_store::ThreadStore::flush_thread(self.inner.as_ref(), thread_id) + } + + fn shutdown_thread( + &self, + thread_id: ThreadId, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::shutdown_thread(self.inner.as_ref(), thread_id) + } + + fn discard_thread( + &self, + thread_id: ThreadId, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::discard_thread(self.inner.as_ref(), thread_id) + } + + fn load_history( + &self, + params: codex_thread_store::LoadThreadHistoryParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::StoredThreadHistory> { + codex_thread_store::ThreadStore::load_history(self.inner.as_ref(), params) + } + + fn read_thread( + &self, + params: codex_thread_store::ReadThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::StoredThread> { + codex_thread_store::ThreadStore::read_thread(self.inner.as_ref(), params) + } + + fn read_thread_by_rollout_path( + &self, + params: codex_thread_store::ReadThreadByRolloutPathParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::StoredThread> { + codex_thread_store::ThreadStore::read_thread_by_rollout_path(self.inner.as_ref(), params) + } + + fn list_threads( + &self, + params: codex_thread_store::ListThreadsParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::ThreadPage> { + codex_thread_store::ThreadStore::list_threads(self.inner.as_ref(), params) + } + + fn update_thread_metadata( + &self, + params: codex_thread_store::UpdateThreadMetadataParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::StoredThread> { + codex_thread_store::ThreadStore::update_thread_metadata(self.inner.as_ref(), params) + } + + fn archive_thread( + &self, + params: codex_thread_store::ArchiveThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::archive_thread(self.inner.as_ref(), params) + } + + fn unarchive_thread( + &self, + params: codex_thread_store::ArchiveThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, codex_thread_store::StoredThread> { + codex_thread_store::ThreadStore::unarchive_thread(self.inner.as_ref(), params) + } + + fn delete_thread( + &self, + params: codex_thread_store::DeleteThreadParams, + ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + codex_thread_store::ThreadStore::delete_thread(self.inner.as_ref(), params) + } +} + +async fn accept_mailbox_note( + mailbox: &codex_agent_mailbox::Mailbox, + sender: ThreadId, + recipient: ThreadId, + invocation_key: &str, + body: &str, +) -> InterAgentCommunication { + mailbox + .accept_message(codex_agent_mailbox::MessageDraft { + invocation_key: invocation_key.to_string(), + sender, + recipient, + communication: InterAgentCommunication::new( + AgentPath::root(), + AgentPath::root(), + Vec::new(), + body.to_string(), + /*trigger_turn*/ false, + ), + kind: codex_agent_mailbox::MessageKind::Note, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention: false, + }) + .await + .expect("mailbox note should be accepted") + .communication +} + +#[tokio::test] +async fn canonical_append_and_flush_failures_do_not_acknowledge_mailbox_messages() { + let codex_home = tempfile::tempdir().expect("create temp directory"); + let (mut session, turn_context, _rx) = make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + |config| { + let _ = config.features.enable(Feature::MultiAgentV2); + let _ = config.features.enable(Feature::AgentMailbox); + }, + ) + .await; + session + .services + .agent_control + .mailbox_initialize(&turn_context.config) + .await + .expect("mailbox should initialize"); + let mailbox = session + .services + .agent_control + .mailbox() + .expect("mailbox should be enabled"); + let fault_store = Arc::new(FaultInjectingThreadStore::new(Arc::clone( + &session.services.thread_store, + ))); + session.services.thread_store = fault_store.clone(); + attach_thread_persistence(Arc::get_mut(&mut session).expect("unique session")).await; + + let append_communication = accept_mailbox_note( + mailbox.as_ref(), + ThreadId::new(), + session.thread_id, + "append-failure", + "append body", + ) + .await; + let append_id = append_communication + .id + .as_ref() + .expect("accepted input should have a stable ID") + .clone(); + fault_store + .fail_append_once + .store(true, std::sync::atomic::Ordering::SeqCst); + assert!( + session + .record_inter_agent_communication(&turn_context, append_communication) + .await + .is_err() + ); + assert!( + !mailbox + .get_message(session.thread_id, append_id.as_str()) + .await + .expect("message lookup should work") + .expect("accepted message should remain stored") + .canonical + ); + + let flush_communication = accept_mailbox_note( + mailbox.as_ref(), + ThreadId::new(), + session.thread_id, + "flush-failure", + "flush body", + ) + .await; + let flush_id = flush_communication + .id + .as_ref() + .expect("accepted input should have a stable ID") + .clone(); + fault_store + .fail_flush_once + .store(true, std::sync::atomic::Ordering::SeqCst); + assert!( + session + .record_inter_agent_communication(&turn_context, flush_communication) + .await + .is_err() + ); + assert!( + !mailbox + .get_message(session.thread_id, flush_id.as_str()) + .await + .expect("message lookup should work") + .expect("accepted message should remain stored") + .canonical + ); +} + +#[tokio::test] +async fn disabled_turn_does_not_settle_shared_mailbox_state() { + let codex_home = tempfile::tempdir().expect("create temp directory"); + let (enabled_session, enabled_turn_context, _rx) = + make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + |config| { + let _ = config.features.enable(Feature::MultiAgentV2); + let _ = config.features.enable(Feature::AgentMailbox); + }, + ) + .await; + enabled_session + .services + .agent_control + .mailbox_initialize(&enabled_turn_context.config) + .await + .expect("mailbox should initialize"); + let shared_control = enabled_session.services.agent_control.clone(); + let mailbox = shared_control + .mailbox() + .expect("enabled session should initialize mailbox"); + + let (mut disabled_session, disabled_turn_context, _rx) = + make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + |config| { + let _ = config.features.enable(Feature::MultiAgentV2); + let _ = config.features.disable(Feature::AgentMailbox); + }, + ) + .await; + disabled_session.services.agent_control = shared_control; + let communication = accept_mailbox_note( + mailbox.as_ref(), + ThreadId::new(), + disabled_session.thread_id, + "disabled-turn", + "must remain unacknowledged", + ) + .await; + let id = communication + .id + .as_ref() + .expect("accepted input should have a stable ID") + .clone(); + attach_thread_persistence(Arc::get_mut(&mut disabled_session).expect("unique session")).await; + + disabled_session + .record_inter_agent_communication(&disabled_turn_context, communication) + .await + .expect("native V2 history recording should continue when mailbox is disabled"); + + assert!( + !mailbox + .get_message(disabled_session.thread_id, id.as_str()) + .await + .expect("message lookup should work") + .expect("accepted message should remain stored") + .canonical + ); +} + +#[tokio::test] +async fn stable_mailbox_message_retry_and_resumed_history_do_not_duplicate_body() { + let codex_home = tempfile::tempdir().expect("create temp directory"); + let configure = |config: &mut crate::config::Config| { + let _ = config.features.enable(Feature::MultiAgentV2); + let _ = config.features.enable(Feature::AgentMailbox); + }; + let (mut session, turn_context, _rx) = make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + configure, + ) + .await; + session + .services + .agent_control + .mailbox_initialize(&turn_context.config) + .await + .expect("mailbox should initialize"); + let mailbox = session + .services + .agent_control + .mailbox() + .expect("mailbox should be enabled"); + let communication = accept_mailbox_note( + mailbox.as_ref(), + ThreadId::new(), + session.thread_id, + "stable-id-retry", + "one canonical body", + ) + .await; + let id = communication + .id + .as_ref() + .expect("accepted input should have a stable ID") + .clone(); + let rollout_path = + attach_thread_persistence(Arc::get_mut(&mut session).expect("unique session")).await; + session + .record_inter_agent_communication(&turn_context, communication.clone()) + .await + .expect("first canonical append should succeed"); + session + .replace_compacted_history( + Vec::new(), + None, + None, + crate::compact::CompactedHistoryMetadata { + message: "test checkpoint".to_string(), + window_number: 1, + window_ids: crate::state::AutoCompactWindowIds::new_initial(), + }, + ) + .await + .expect("checkpoint should preserve canonical but unadmitted mailbox body"); + let compacted_history = session.clone_history().await; + assert_eq!( + compacted_history + .raw_items() + .iter() + .filter(|item| item.id() == Some(&id)) + .count(), + 1 + ); + session + .record_inter_agent_communication(&turn_context, communication.clone()) + .await + .expect("stable-ID retry should reconcile without a duplicate append"); + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read canonical rollout") + else { + panic!("expected resumed rollout history"); + }; + let initial_history = InitialHistory::Resumed(resumed); + let restored_ids = Session::canonical_agent_message_ids(&initial_history); + assert!(restored_ids.contains(&id)); + + let (mut resumed_session, resumed_turn_context, _rx) = + make_session_and_context_with_auth_config_home_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + codex_home.path(), + configure, + ) + .await; + resumed_session + .services + .agent_control + .mailbox_initialize(&resumed_turn_context.config) + .await + .expect("resumed mailbox should initialize"); + let shared_control = session.services.agent_control.clone(); + let shared_store = Arc::clone(&session.services.thread_store); + let shared_live_thread = session + .services + .live_thread + .as_ref() + .expect("source rollout should remain open") + .clone(); + let resumed_session = Arc::get_mut(&mut resumed_session).expect("unique resumed session"); + resumed_session.services.agent_control = shared_control; + resumed_session.services.thread_store = shared_store; + resumed_session.services.live_thread = Some(shared_live_thread); + resumed_session + .mailbox_canonical_response_item_ids + .lock() + .await + .extend(restored_ids); + resumed_session + .record_initial_history(initial_history) + .await; + resumed_session + .record_inter_agent_communication(&resumed_turn_context, communication) + .await + .expect("resumed stable-ID retry should reconcile without a duplicate append"); + + let matching_items = resumed_session + .clone_history() + .await + .raw_items() + .iter() + .filter(|item| item.id() == Some(&id)) + .count(); + assert_eq!(matching_items, 1); + assert!( + mailbox + .get_message(session.thread_id, id.as_str()) + .await + .expect("message lookup should work") + .expect("accepted message should remain stored") + .canonical + ); +} + #[tokio::test] async fn prepares_image_failures_before_history_insertion() { let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx( diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 3ea30876c85e..edffcea72aa4 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -106,6 +106,7 @@ use codex_protocol::protocol::AgentReasoningSectionBreakEvent; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::PlanDeltaEvent; use codex_protocol::protocol::RawResponseCompletedEvent; use codex_protocol::protocol::ReasoningContentDeltaEvent; @@ -303,11 +304,15 @@ pub(crate) async fn run_turn( .record_step_world_state_if_changed(&world_state, step_context.as_ref()) .await; - for communication in sess.services.agent_control - .mailbox_automatic_inputs(sess.thread_id, &turn_context.sub_id) - .await? - { - sess.record_inter_agent_communication(&turn_context, communication).await?; + let mailbox_enabled_for_turn = turn_context.multi_agent_version == MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox); + if mailbox_enabled_for_turn { + for communication in sess.services.agent_control + .mailbox_automatic_inputs(sess.thread_id, &turn_context.sub_id) + .await? + { + sess.record_inter_agent_communication(&turn_context, communication).await?; + } } // Construct the input that we will send to the model. @@ -319,28 +324,6 @@ pub(crate) async fn run_turn( .instrument(trace_span!("run_turn.prepare_sampling_request_input")) .await; - let mailbox_message_ids = sampling_request_input - .iter() - .filter_map(|item| match item { - ResponseItem::AgentMessage { id: Some(id), .. } if !id.is_empty() => { - Some(id.clone()) - } - _ => None, - }) - .collect::>() - .into_iter() - .collect::>(); - if !mailbox_message_ids.is_empty() { - sess.services - .agent_control - .mailbox_record_admission( - sess.thread_id, - &turn_context.sub_id, - &mailbox_message_ids, - ) - .await?; - } - let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( sess.installation_id.clone(), window_id, @@ -2256,7 +2239,40 @@ async fn try_run_sampling_request( record_turn_ttft_metric(&turn_context, &event).await; match event { - ResponseEvent::Created => {} + ResponseEvent::Created => { + // ResponseEvent::Created is the provider's accepted-response boundary. In + // particular, WebSocket stream construction only queues a send task; marking + // admission earlier would let compaction drop mail when setup/send is rejected. + if turn_context.multi_agent_version == MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox) + { + let mailbox_message_ids = prompt + .input + .iter() + .filter_map(|item| match item { + ResponseItem::AgentMessage { id: Some(id), .. } if !id.is_empty() => { + Some(id.clone()) + } + _ => None, + }) + .collect::>() + .into_iter() + .collect::>(); + if !mailbox_message_ids.is_empty() + && let Err(error) = sess + .services + .agent_control + .mailbox_record_admission( + sess.thread_id, + &turn_context.sub_id, + &mailbox_message_ids, + ) + .await + { + break Err(error); + } + } + } ResponseEvent::OutputItemDone(mut item) => { assign_missing_streamed_response_item_id(&mut item, active_item.as_ref()); if let Some((_, mut consumer)) = active_tool_argument_diff_consumer.take() diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 25040cc1b916..34b4d9cae06e 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -702,6 +702,11 @@ fn followup_task_output_schema() -> Value { "type": "string", "description": "Immutable accepted work reference assigned to the follow-up." }, + "dispatch_state": { + "type": "string", + "enum": ["accepted", "dispatching", "dispatched", "recovery_needed"], + "description": "Native dispatch custody for the original work. Uncertain dispatch is not resent." + }, "recipient_task_name": { "type": "string", "description": "Explicit canonical task name of the recipient; the effective_* fields below describe this recipient, not the sending agent." @@ -770,12 +775,16 @@ fn mailbox_send_message_output_schema() -> Value { }, "activation_disposition": { "type": "string", - "enum": ["not_requested", "already_active", "activated", "denied"], - "description": "Disposition of the separately requested idle-target activation. Durable message acceptance is unaffected by a denied activation." + "enum": ["not_requested", "already_active", "activated", "denied", "unknown"], + "description": "Disposition of separately requested idle activation. Message acceptance is durable even when activation is denied or unknown; do not resend the message." }, "activation_reason": { "type": ["string", "null"], - "description": "Safe reason when activation was denied, such as native pause, budget, cancellation, or root protection." + "description": "Safe reason when activation was denied or its result could not be established." + }, + "recipient_task_name": { + "type": "string", + "description": "Canonical native task name addressed by the accepted message." } }, "required": ["message_id", "durable_state", "activation_disposition"], @@ -1131,7 +1140,7 @@ fn mailbox_wait_output_schema() -> Value { }, "quiet_count": { "type": "integer", - "description": "Count of messages safely quieted for this wait, if applicable." + "description": "Global retained quiet-message backlog for this recipient, not a per-wait filtered count. This metadata does not consume hidden message bodies." }, "wake_sender": { "type": ["string", "null"], diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs index 5646269b5c5c..e304bbe45e1c 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2.rs @@ -27,6 +27,7 @@ use serde::Serialize; use serde_json::Value as JsonValue; pub(crate) use followup_task::Handler as FollowupTaskHandler; +pub(crate) use followup_task::MailboxHandler as MailboxFollowupTaskHandler; pub(crate) use interrupt_agent::Handler as InterruptAgentHandler; pub(crate) use list_agents::Handler as ListAgentsHandler; pub(crate) use mailbox_message::ReadHandler as MailboxReadHandler; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs index b9625894cce5..f3c5918be56c 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs @@ -6,6 +6,7 @@ use crate::tools::handlers::multi_agents_spec::create_followup_task_tool; use codex_tools::ToolSpec; pub(crate) struct Handler; +pub(crate) struct MailboxHandler; impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { @@ -40,6 +41,30 @@ impl Handler { } } +impl ToolExecutor for MailboxHandler { + fn tool_name(&self) -> ToolName { ToolName::plain("followup_task") } + fn spec(&self) -> ToolSpec { create_followup_task_tool() } + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + if invocation.cancellation_token.is_cancelled() { + return Err(FunctionCallError::RespondToModel("follow-up was cancelled before native acceptance".to_string())); + } + Handler.handle_call(invocation).await + }) + } +} + +impl CoreToolRuntime for MailboxHandler { + fn matches_kind(&self, payload: &ToolPayload) -> bool { + matches!(payload, ToolPayload::Function { .. }) + } + fn waits_for_runtime_cancellation(&self) -> bool { + // A caller cancellation can suppress tool publication, but cannot drop a native + // acceptance transaction between its durable commit and dispatch settlement. + true + } +} + impl CoreToolRuntime for Handler { fn matches_kind(&self, payload: &ToolPayload) -> bool { matches!(payload, ToolPayload::Function { .. }) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs index 4f9ccbc8cd91..6c5a33aada1a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs @@ -155,27 +155,30 @@ impl ToolExecutor for SendHandler { activate_if_idle: args.activate_if_idle, }).await.map_err(mailbox_error)?; let (activation_disposition, activation_reason) = if args.activate_if_idle { - match mailbox.begin_activation(session.thread_id, &envelope.id).await.map_err(mailbox_error)? { - ActivationDecision::Activate => { + match mailbox.begin_activation(session.thread_id, &envelope.id).await { + Ok(ActivationDecision::Activate) => { let result = session.services.agent_control .mailbox_activate_if_idle(recipient, &envelope).await; let (disposition, reason) = match result { Ok(result) => result, - Err(_) => (ActivationDisposition::Denied, Some("native_activation_not_accepted".to_string())), + Err(_) => (ActivationDisposition::Unknown, Some("activation_outcome_unknown_original_message_retained".to_string())), }; - mailbox.record_activation(session.thread_id, &envelope.id, disposition) - .await.map_err(mailbox_error)?; - (disposition, reason) + if mailbox.record_activation(session.thread_id, &envelope.id, disposition).await.is_err() { + (ActivationDisposition::Unknown, Some("activation_receipt_not_persisted_original_message_retained".to_string())) + } else { + (disposition, reason) + } } - ActivationDecision::AlreadyDecided(disposition) => (disposition, None), - ActivationDecision::RecoveryNeeded => ( - ActivationDisposition::Denied, + Ok(ActivationDecision::AlreadyDecided(disposition)) => (disposition, None), + Ok(ActivationDecision::RecoveryNeeded) | Err(_) => ( + ActivationDisposition::Unknown, Some("activation_outcome_unknown_original_message_retained".to_string()), ), } } else { - mailbox.record_activation(session.thread_id, &envelope.id, ActivationDisposition::NotRequested) - .await.map_err(mailbox_error)?; + // The envelope is already committed. Optional activation bookkeeping must + // never hide acceptance or invite the caller to create a duplicate message. + let _ = mailbox.record_activation(session.thread_id, &envelope.id, ActivationDisposition::NotRequested).await; (ActivationDisposition::NotRequested, None) }; Ok(boxed_tool_output(FunctionToolOutput::from_text( @@ -214,7 +217,13 @@ impl ToolExecutor for ReadHandler { let cursor = args.cursor.as_deref().map(str::parse::).transpose() .map_err(|_| mailbox_error("cursor must be a returned mailbox cursor"))?; let mailbox = mailbox_for(&session)?; - let page = mailbox.pending_page(session.thread_id, cursor, args.limit.unwrap_or(20)) + let requested_limit = args.limit.unwrap_or(20); + if !(1..=100).contains(&requested_limit) { + return Err(mailbox_error("limit must be between 1 and 100")); + } + // An encrypted body cannot be safely truncated. Every accepted body is capped + // at 8KiB; admit at most two per call and retain the rest behind the page cursor. + let page = mailbox.pending_page(session.thread_id, cursor, requested_limit.min(2)) .await.map_err(mailbox_error)?; let mut messages = Vec::with_capacity(page.items.len()); for envelope in &page.items { @@ -228,6 +237,12 @@ impl ToolExecutor for ReadHandler { if envelope.reply_to.is_some() { observations.push(ObservationTarget::Reply(envelope.id.clone())); } + if envelope.communication.origin == Some(codex_protocol::protocol::AgentCommunicationOrigin::Result) + && let Some(work) = envelope.work_ref.as_ref() + && work.issuer == session.thread_id + { + observations.push(ObservationTarget::Work(work.clone())); + } // Reading a body can observe its own attention/reply only; it never observes // unrelated work outcomes or unresolved request obligations. if !observations.is_empty() { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 336c2a86d52e..6c8caa106233 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -121,6 +121,12 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, wake_sender: None, wake_ref: None, }; + report.pending = selectors.iter().map(|selector| MailboxWaitSelectionItem { + work_ref: match selector { Selector::Work(id) => Some(id.clone()), Selector::Reply(_) => None }, + message_id: match selector { Selector::Reply(id) => Some(id.clone()), Selector::Work(_) => None }, + agent: None, + status: "pending".to_string(), + }).collect(); let mut selected = Vec::new(); let mut attention_to_observe = None; let mut started = false; @@ -129,20 +135,30 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, loop { drop(registration.take()); if targetless { - selectors = mailbox.issued_work_refs(session.thread_id, None, 100) - .await.map_err(mailbox_error)?.into_iter() - .map(|work| Selector::Work(work.id)).collect(); + match mailbox.issued_work_refs(session.thread_id, None, 100).await { + Ok(work) => selectors = work.into_iter().map(|work| Selector::Work(work.id)).collect(), + Err(_) => { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } + } } selected = if selectors.is_empty() { Vec::new() } else { - mailbox.snapshot(&Selection { + match mailbox.snapshot(&Selection { caller: session.thread_id, agent: None, selectors: selectors.clone(), return_when, limit: 100, - }).await.map_err(mailbox_error)?.items + }).await { + Ok(snapshot) => snapshot.items, + Err(_) => { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } + } }; for item in &selected { if let SnapshotItem::Work(work) = item { @@ -152,8 +168,28 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } } } + let mut recovery_changed = false; + for item in &selected { + if let SnapshotItem::Work(work) = item + && work.dispatch_state == DispatchState::Dispatched + && work.outcome.is_none() + && work.continuations.is_empty() + { + recovery_changed |= session.services.agent_control + .mailbox_reconcile_pending_work(&work.work_ref).await.map_err(mailbox_error)?; + } + } + if recovery_changed { + continue; + } update_report(&session, &mut report, &selected); - report.quiet_count = mailbox.quiet_count(session.thread_id).await.map_err(mailbox_error)?; + match mailbox.quiet_count(session.thread_id).await { + Ok(count) => report.quiet_count = count, + Err(_) => { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } + } let ready = report.completed.len(); let early_attention = selected.iter().any(needs_attention); let satisfied = !selected.is_empty() && match return_when { @@ -181,15 +217,15 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, report.completion_reason = Some(MailboxWakeCause::NoMatchingWork); break; } + if early_attention { + report.completion_reason = Some(MailboxWakeCause::NeedsAttention); + break; + } if all_observed { report.condition_satisfied = true; report.completion_reason = Some(MailboxWakeCause::AlreadySatisfied); break; } - if early_attention { - report.completion_reason = Some(MailboxWakeCause::NeedsAttention); - break; - } if satisfied { report.condition_satisfied = true; report.completion_reason = Some(if all_observed { @@ -203,10 +239,20 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } // An addressed request/alert interrupts a join but never satisfies it. Its body is // admitted before its one-shot notification is observed; unanswered obligation remains. - for attention in mailbox.attention(session.thread_id, 100).await.map_err(mailbox_error)? { + let attention = match mailbox.attention(session.thread_id, 100).await { + Ok(attention) => attention, + Err(_) => { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } + }; + for attention in attention { let Some(message_id) = attention.id.strip_prefix("attention:amsg_").map(|suffix| format!("amsg_{suffix}")) else { continue; }; if let Some(envelope) = mailbox.get_message(session.thread_id, &message_id).await.map_err(mailbox_error)? { - deliver_envelope(&session, &turn, &envelope).await?; + if deliver_envelope(&session, &turn, &envelope).await.is_err() { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } report.wake_sender = Some(envelope.communication.author.to_string()); report.wake_ref = Some(envelope.id); report.completion_reason = Some(MailboxWakeCause::Request); @@ -234,14 +280,18 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, SnapshotItem::Work(work) if work.outcome.is_none() && !work.observed => Some(work.work_ref.agent), SnapshotItem::Request(request) if request.reply.is_none() && !request.observed => Some(request.request.recipient), SnapshotItem::Work(_) | SnapshotItem::Request(_) => None, - }).collect(); - match mailbox.begin_wait(call_id.clone(), session.thread_id, peers, return_when, deadline.is_some()) { + }).collect::>().into_iter().collect(); + let wait_id = format!("{}:{call_id}", session.thread_id); + match mailbox.begin_wait(wait_id, session.thread_id, peers, return_when, deadline.is_some()) { Ok(guard) => registration = Some(guard), Err(codex_agent_mailbox::MailboxError::ClosedPeerCycle) => { report.completion_reason = Some(MailboxWakeCause::Cycle); break; } - Err(error) => return Err(mailbox_error(error)), + Err(_) => { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } } let deadline_wait = async { match deadline { @@ -274,10 +324,21 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } } drop(registration); - admit_selected(&session, &turn, &mailbox, &selected).await?; + // Every accepted body is at most 8KiB. Bound each wait result to two bodies, + // including an interrupting attention message. Metadata is non-consuming; + // omitted bodies retain their per-item observation state and retrieval IDs. + let body_limit = if attention_to_observe.is_some() { 1 } else { 2 }; + if admit_selected(&session, &turn, &mailbox, &selected, body_limit).await.is_err() { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + report.condition_satisfied = false; + } if let Some(id) = attention_to_observe { - mailbox.acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) - .await.map_err(mailbox_error)?; + if mailbox.acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) + .await.is_err() + { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + report.condition_satisfied = false; + } } emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::Completed).await; Ok(boxed_tool_output(FunctionToolOutput::from_text( @@ -304,11 +365,10 @@ fn reject_reverse_completion_join(session: &Session, turn: &TurnContext, agent: fn needs_attention(item: &SnapshotItem) -> bool { match item { - SnapshotItem::Work(work) => !work.observed && (work.dispatch_state == DispatchState::RecoveryNeeded + SnapshotItem::Work(work) => work.dispatch_state == DispatchState::RecoveryNeeded || work.outcome.as_ref().is_some_and(|result| result.outcome != Outcome::Completed) - || work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention)), - SnapshotItem::Request(request) => !request.observed - && matches!(request.disposition, RequestDisposition::Superseded | RequestDisposition::Withdrawn), + || work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention), + SnapshotItem::Request(request) => matches!(request.disposition, RequestDisposition::Superseded | RequestDisposition::Withdrawn), } } @@ -327,6 +387,8 @@ fn update_report(session: &Session, report: &mut MailboxWaitReport, selected: &[ SnapshotItem::Work(work) => { let status = if work.dispatch_state == DispatchState::RecoveryNeeded { "recovery_needed".to_string() + } else if work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention) { + "needs_attention".to_string() } else { work.outcome.as_ref().map_or_else(|| "pending".to_string(), |outcome| match outcome.outcome { Outcome::Completed => "completed", @@ -359,12 +421,16 @@ fn update_report(session: &Session, report: &mut MailboxWaitReport, selected: &[ } } -async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox, selected: &[SnapshotItem]) -> Result<(), FunctionCallError> { +async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox, selected: &[SnapshotItem], body_limit: usize) -> Result<(), FunctionCallError> { let mut observations = Vec::new(); + let mut delivered = HashSet::new(); for item in selected { match item { SnapshotItem::Work(work) if !work.observed => { if let Some(outcome) = &work.outcome { + if !delivered.contains(&outcome.id) && delivered.len() >= body_limit { + continue; + } let communication = outcome.response.clone().unwrap_or_else(|| { let mut communication = InterAgentCommunication::new( work.input.recipient.clone(), work.input.author.clone(), Vec::new(), @@ -377,12 +443,17 @@ async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox }); session.record_inter_agent_communication(turn, communication).await.map_err(mailbox_error)?; mailbox.mark_outcome_canonical(session.thread_id, &outcome.id).await.map_err(mailbox_error)?; + delivered.insert(outcome.id.clone()); observations.push(ObservationTarget::Work(work.work_ref.clone())); } } SnapshotItem::Request(request) if !request.observed => { if let Some(reply) = &request.reply { + if !delivered.contains(&reply.id) && delivered.len() >= body_limit { + continue; + } deliver_envelope(session, turn, reply).await?; + delivered.insert(reply.id.clone()); observations.push(ObservationTarget::Reply(reply.id.clone())); } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 8a39fc1cb935..96de3cbc42fc 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -138,6 +138,8 @@ struct FollowupTaskResult { recipient_task_name: String, #[serde(skip_serializing_if = "Option::is_none")] work_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dispatch_state: Option, effective_identity_scope: &'static str, effective_model: Option, effective_model_provider_id: Option, @@ -279,26 +281,30 @@ async fn handle_message_submission_inner( )); } mailbox.accept_work(codex_agent_mailbox::NativeWork { - native_key: call_id, + native_key: call_id.clone(), issuer: session.thread_id, agent: work.agent, submission_id: work.submission_id.clone(), input: communication_from_tool_message( turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root), - receiver_agent_path.clone(), message, + receiver_agent_path.clone(), message.clone(), ), }).await.map_err(super::mailbox_message::mailbox_error)?; + let dispatch_state = mailbox.dispatch_state(&work).await.map_err(super::mailbox_message::mailbox_error)?; + if dispatch_state != codex_agent_mailbox::DispatchState::Accepted { let snapshot = session.services.agent_control.get_agent_config_snapshot(receiver_thread_id).await; return Ok(FunctionToolOutput::from_text(tool_output_json_text(&FollowupTaskResult { task_name: receiver_agent_path.to_string(), recipient_task_name: receiver_agent_path.to_string(), work_ref: Some(work.id), + dispatch_state: Some(dispatch_state), effective_identity_scope: "recipient", effective_model: snapshot.as_ref().map(|config| config.model.clone()), effective_model_provider_id: snapshot.as_ref().map(|config| config.model_provider_id.clone()), effective_reasoning_effort: snapshot.as_ref().and_then(|config| config.reasoning_effort.clone()), effective_service_tier: snapshot.as_ref().and_then(|config| config.service_tier.clone()), }, "followup_task"), Some(true))); + } } let delivery = match mode { MessageDeliveryMode::QueueOnly => { @@ -414,6 +420,8 @@ async fn handle_message_submission_inner( .await .map_err(|err| collab_agent_error(receiver_thread_id, err))? .map(|work| work.id), + dispatch_state: session.services.agent_control.mailbox_enabled() + .then_some(codex_agent_mailbox::DispatchState::Dispatched), effective_identity_scope: "recipient", effective_model: Some(receiver_config.model), effective_model_provider_id: Some(receiver_config.model_provider_id), diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index d1d651a0d600..29c4ba28935b 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -453,6 +453,9 @@ mod tests { use codex_protocol::AgentPath; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; + use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::Op; + use codex_protocol::protocol::TurnCompleteEvent; use codex_tools::ToolName; use pretty_assertions::assert_eq; use tokio::sync::Notify; @@ -1182,7 +1185,7 @@ mod tests { } #[tokio::test] - async fn configured_v2_spawn_cancellation_after_initial_delivery_returns_published_result() + async fn configured_v2_spawn_fast_completion_cancellation_retry_reuses_native_work() -> anyhow::Result<()> { let (mut session, mut turn_context) = crate::session::tests::make_session_and_context().await; @@ -1191,6 +1194,10 @@ mod tests { .features .enable(Feature::MultiAgentV2) .expect("test config should enable MultiAgentV2"); + config + .features + .enable(Feature::AgentMailbox) + .expect("test config should enable AgentMailbox"); config.multi_agent_v2.hide_spawn_agent_metadata = false; config.multi_agent_v2.tool_namespace = Some("profile_agents".to_string()); turn_context.multi_agent_version = config.multi_agent_version_from_features(); @@ -1201,10 +1208,11 @@ mod tests { built_in_model_providers(/*openai_base_url*/ None)["openai"].clone(), ); let root = manager - .start_thread(StartThreadOptions::new(config)) + .start_thread(StartThreadOptions::new(config.clone())) .await .expect("root thread should start"); session.services.agent_control = manager.agent_control(); + session.services.agent_control.mailbox_initialize(&config).await?; session.thread_id = root.thread_id; let (initial_delivery_finished, resume_spawn) = session .services @@ -1231,23 +1239,21 @@ mod tests { )); let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); let runtime = ToolCallRuntime::new(router, Arc::clone(&session), step_context, tracker); + let spawn_call = || ToolCall { + tool_name: ToolName::namespaced("profile_agents", "spawn_agent"), + call_id: "call-1".to_string(), + payload: ToolPayload::Function { + arguments: serde_json::json!({ + "message": "delivery owns this child", + "task_name": "delivery_owned_worker", + "fork_turns": "none" + }) + .to_string(), + }, + }; let cancellation_token = CancellationToken::new(); let mut response_task = tokio::spawn( - runtime.handle_tool_call( - ToolCall { - tool_name: ToolName::namespaced("profile_agents", "spawn_agent"), - call_id: "call-1".to_string(), - payload: ToolPayload::Function { - arguments: serde_json::json!({ - "message": "delivery owns this child", - "task_name": "delivery_owned_worker", - "fork_turns": "none" - }) - .to_string(), - }, - }, - cancellation_token.clone(), - ), + runtime.handle_tool_call(spawn_call(), cancellation_token.clone()), ); let child_thread_id = @@ -1267,10 +1273,67 @@ mod tests { created_threads.try_recv().is_err(), "initial configured-V2 delivery must not notify parent-visible creation" ); + let accepted_work = session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, "call-1") + .await? + .expect("the actual configured-V2 handler must durably accept native spawn work"); + let mailbox = session + .services + .agent_control + .mailbox() + .expect("AgentMailbox should be initialized for the actual runtime"); + let accepted_input = mailbox + .accepted_input(&accepted_work) + .await + .expect("accepted work should retain its original native input"); + let accepted_input_id = accepted_input + .id + .as_ref() + .expect("accepted native input should carry an amsg_ ID") + .to_string(); + assert!(accepted_input_id.starts_with("amsg_")); + let accepted_work_id = accepted_work.id.clone(); + + // Exercise the fast-completion edge while the real handler is paused before publication. + let child = manager + .get_thread(child_thread_id) + .await + .expect("provisional configured-V2 child should remain manager-owned"); + let terminal_turn = child.session.new_default_turn().await; + child + .session + .send_event( + terminal_turn.as_ref(), + EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: terminal_turn.sub_id.clone(), + started_at: None, + last_agent_message: Some("finished before publication".to_string()), + compaction_events_in_turn: 0, + final_model: None, + model_snapshot: None, + provider_usage: None, + error: None, + completed_at: None, + duration_ms: None, + time_to_first_token_ms: None, + }), + ) + .await; let initial_delivery_count = manager .captured_ops() .iter() - .filter(|(thread_id, _)| *thread_id == child_thread_id) + .filter(|(thread_id, op)| { + *thread_id == child_thread_id + && matches!( + op, + Op::InterAgentCommunication { communication } + if communication.id == accepted_input.id + && communication.encrypted_content == accepted_input.encrypted_content + && communication.trigger_turn + ) + }) .count(); assert_eq!( initial_delivery_count, 1, @@ -1315,6 +1378,16 @@ mod tests { !message.contains("aborted by user"), "delivery ownership must preserve the configured-V2 result instead of a false abort: {message}" ); + let first_result: serde_json::Value = serde_json::from_str(&message)?; + let first_agent_id = first_result["agent_id"] + .as_str() + .expect("configured-V2 spawn result should expose the published child ID"); + assert_eq!(first_agent_id, child_thread_id.to_string()); + assert_eq!( + first_result["work_ref"].as_str(), + Some(accepted_work_id.as_str()), + "the tool result should publish the exact accepted native work reference" + ); assert!( session .services @@ -1330,6 +1403,68 @@ mod tests { .expect("published configured-V2 child should notify once"), child_thread_id ); + assert!( + created_threads.try_recv().is_err(), + "the original publication should be the only child-created notification" + ); + + // A same-call retry models a tool result lost after acceptance/publication. It must + // resolve the retained work, not provision another child or send the input again. + let retry_response = tokio::time::timeout( + Duration::from_secs(5), + runtime.handle_tool_call(spawn_call(), CancellationToken::new()), + ) + .await + .expect("retrying a lost tool publication should resolve the accepted work")?; + let ResponseInputItem::FunctionCallOutput { output, .. } = retry_response else { + anyhow::bail!("configured-V2 retry should return a function output"); + }; + assert_eq!(output.success, Some(true)); + let FunctionCallOutputBody::Text(retry_message) = output.body else { + anyhow::bail!("configured-V2 retry output should be text"); + }; + let retry_result: serde_json::Value = serde_json::from_str(&retry_message)?; + assert_eq!(retry_result["agent_id"].as_str(), Some(first_agent_id)); + assert_eq!( + retry_result["work_ref"].as_str(), + Some(accepted_work_id.as_str()), + "same-call retry must return the original durable work reference" + ); + assert_eq!( + session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, "call-1") + .await?, + Some(accepted_work.clone()), + "same-call retry must resolve the original accepted child" + ); + assert_eq!( + mailbox.accepted_input(&accepted_work).await?, + accepted_input, + "same-call retry must preserve the original accepted native input" + ); + let redelivery_count = manager + .captured_ops() + .iter() + .filter(|(thread_id, op)| { + *thread_id == child_thread_id + && matches!( + op, + Op::InterAgentCommunication { communication } + if communication.id.as_ref().is_some_and(|id| id.as_str() == accepted_input_id) + && communication.trigger_turn + ) + }) + .count(); + assert_eq!( + redelivery_count, 1, + "same-call retry after caller cancellation must not resend accepted native input" + ); + assert!( + created_threads.try_recv().is_err(), + "same-call retry must not create a replacement child" + ); Ok(()) } diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index bee4f76843e5..cdf5689b7184 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -45,6 +45,7 @@ use crate::tools::handlers::multi_agents_common::MIN_WAIT_TIMEOUT_MS; use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions; use crate::tools::handlers::multi_agents_spec::WaitAgentTimeoutOptions; use crate::tools::handlers::multi_agents_v2::FollowupTaskHandler as FollowupTaskHandlerV2; +use crate::tools::handlers::multi_agents_v2::MailboxFollowupTaskHandler as MailboxFollowupTaskHandlerV2; use crate::tools::handlers::multi_agents_v2::InterruptAgentHandler; use crate::tools::handlers::multi_agents_v2::ListAgentsHandler as ListAgentsHandlerV2; use crate::tools::handlers::multi_agents_v2::MailboxReadHandler; @@ -857,10 +858,15 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu exposure, )); } - planned_tools.add_arc(override_tool_exposure( - multi_agent_v2_handler(FollowupTaskHandlerV2, tool_namespace), - exposure, - )); + if turn_context.config.features.enabled(Feature::AgentMailbox) { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler(MailboxFollowupTaskHandlerV2, tool_namespace), exposure, + )); + } else { + planned_tools.add_arc(override_tool_exposure( + multi_agent_v2_handler(FollowupTaskHandlerV2, tool_namespace), exposure, + )); + } if turn_context.config.multi_agent_v2.wait_agent_enabled { if mailbox_enabled { planned_tools.add_arc(override_tool_exposure( diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs new file mode 100644 index 000000000000..150c90f2c698 --- /dev/null +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -0,0 +1,739 @@ +use anyhow::Result; +use codex_features::Feature; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call_with_namespace; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once_match; +use core_test_support::responses::sse; +use core_test_support::responses::sse_response; +use core_test_support::responses::start_mock_server; +use core_test_support::test_codex::test_codex; +use serde_json::json; +use serde_json::Value; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use std::time::Instant; +use wiremock::Match; +use wiremock::Mock; +use wiremock::Request; +use wiremock::Respond; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path_regex; + +const NAMESPACE: &str = "agents"; +const SCHEMA_DISABLED_PROMPT: &str = "inspect mailbox tools without the mailbox feature"; +const SCHEMA_ENABLED_PROMPT: &str = "inspect mailbox tools with the mailbox feature"; +const FLOW_PROMPT: &str = "run the mailbox integration flow"; +const CHILD_TASK: &str = "finish quickly so the parent can join by immutable work reference"; +const REQUEST_BODY: &str = "opaque mailbox request body"; +const QUIET_REPLY_BODY: &str = "quiet final reply"; + +fn find_named_tool<'a>(value: &'a Value, name: &str) -> Option<&'a Value> { + match value { + Value::Object(object) => { + if object.get("name").and_then(Value::as_str) == Some(name) + && object.get("parameters").is_some() + { + return Some(value); + } + object.values().find_map(|child| find_named_tool(child, name)) + } + Value::Array(items) => items.iter().find_map(|child| find_named_tool(child, name)), + _ => None, + } +} + +fn function_output_text<'a>(request: &'a Value, call_id: &str) -> Option<&'a str> { + request + .get("input")? + .as_array()? + .iter() + .find(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + && item.get("call_id").and_then(Value::as_str) == Some(call_id) + })? + .get("output")? + .as_str() +} + +fn has_function_output(request: &Value, call_id: &str) -> bool { + function_output_text(request, call_id).is_some() +} + +fn function_call_response( + response_id: &str, + call_id: &str, + name: &str, + args: Value, +) -> ResponseTemplate { + let args = serde_json::to_string(&args).expect("tool arguments should serialize"); + sse_response(sse(vec![ + ev_response_created(response_id), + ev_function_call_with_namespace(call_id, NAMESPACE, name, &args), + ev_completed(response_id), + ])) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Result<()> { + let server = start_mock_server().await; + let disabled_response = mount_sse_once_match( + &server, + |request: &Request| String::from_utf8_lossy(&request.body).contains(SCHEMA_DISABLED_PROMPT), + sse(vec![ + ev_response_created("mailbox-schema-disabled"), + ev_assistant_message("mailbox-schema-disabled-message", "checked"), + ev_completed("mailbox-schema-disabled"), + ]), + ) + .await; + let enabled_response = mount_sse_once_match( + &server, + |request: &Request| String::from_utf8_lossy(&request.body).contains(SCHEMA_ENABLED_PROMPT), + sse(vec![ + ev_response_created("mailbox-schema-enabled"), + ev_assistant_message("mailbox-schema-enabled-message", "checked"), + ev_completed("mailbox-schema-enabled"), + ]), + ) + .await; + + let mut disabled_builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2] { + config + .features + .enable(feature) + .expect("test config should allow feature update"); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + }); + let disabled = disabled_builder.build_with_auto_env(&server).await?; + disabled.submit_turn(SCHEMA_DISABLED_PROMPT).await?; + + let mut enabled_builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + config + .features + .enable(feature) + .expect("test config should allow feature update"); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + }); + let enabled = enabled_builder.build_with_auto_env(&server).await?; + enabled.submit_turn(SCHEMA_ENABLED_PROMPT).await?; + + let disabled_schema = disabled_response.single_request().body_json(); + let disabled_wait = find_named_tool(&disabled_schema["tools"], "wait_agent") + .expect("legacy V2 wait tool should be present when mailbox is disabled"); + assert!(find_named_tool(&disabled_schema["tools"], "mailbox_read").is_none()); + assert!(disabled_wait["parameters"]["properties"].get("replies_to").is_none()); + + let enabled_schema = enabled_response.single_request().body_json(); + let wait = find_named_tool(&enabled_schema["tools"], "wait_agent") + .expect("mailbox V2 wait tool should be present"); + let wait_properties = &wait["parameters"]["properties"]; + assert!(wait_properties.get("targets").is_some()); + assert!(wait_properties.get("replies_to").is_some()); + assert!(wait_properties.get("timeout_ms").is_some()); + assert_eq!(wait_properties["return_when"]["enum"], json!(["any", "all"])); + + let send = find_named_tool(&enabled_schema["tools"], "send_message") + .expect("mailbox send tool should be present"); + let send_properties = &send["parameters"]["properties"]; + for property in ["kind", "reply_to", "supersedes", "work_ref"] { + assert!(send_properties.get(property).is_some(), "missing {property}"); + } + assert!(send_properties.get("interrupt").is_none()); + + let read = find_named_tool(&enabled_schema["tools"], "mailbox_read") + .expect("canonical mailbox read tool should be present"); + let read_properties = &read["parameters"]["properties"]; + assert!(read_properties.get("cursor").is_some()); + assert!(read_properties.get("limit").is_some()); + assert!(read["description"] + .as_str() + .is_some_and(|description| description.contains("canonical conversation history"))); + + Ok(()) +} + +#[derive(Clone)] +struct RootThreadMatcher { + thread_id: String, +} + +impl Match for RootThreadMatcher { + fn matches(&self, request: &Request) -> bool { + serde_json::from_slice::(&request.body) + .ok() + .and_then(|body| body.pointer("/client_metadata/thread_id").cloned()) + .and_then(|thread_id| thread_id.as_str().map(str::to_owned)) + .is_some_and(|thread_id| thread_id == self.thread_id) + } +} + +#[derive(Clone)] +struct ChildThreadMatcher { + root_thread_id: String, + task: String, +} + +impl Match for ChildThreadMatcher { + fn matches(&self, request: &Request) -> bool { + serde_json::from_slice::(&request.body).ok().is_some_and(|body| { + body.pointer("/client_metadata/thread_id") + .and_then(Value::as_str) + .is_some_and(|thread_id| thread_id != self.root_thread_id.as_str()) + && body.to_string().contains(self.task.as_str()) + }) + } +} + +#[derive(Clone)] +struct RootMailboxFlowResponder { + root_thread_id: String, + requests: Arc>>, +} + +impl Respond for RootMailboxFlowResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body) + .expect("provider request should contain JSON"); + self.requests + .lock() + .expect("request capture lock should not be poisoned") + .push(body.clone()); + + if has_function_output(&body, "wait-call") { + return sse_response(sse(vec![ + ev_response_created("mailbox-flow-final"), + ev_assistant_message("mailbox-flow-final-message", "finished"), + ev_completed("mailbox-flow-final"), + ])); + } + + if has_function_output(&body, "reply-call") { + let spawn_output: Value = serde_json::from_str( + function_output_text(&body, "spawn-call") + .expect("spawn output should remain in provider history"), + ) + .expect("spawn output should be JSON"); + let send_output: Value = serde_json::from_str( + function_output_text(&body, "request-call") + .expect("request receipt should remain in provider history"), + ) + .expect("request receipt should be JSON"); + return function_call_response( + "mailbox-wait-response", + "wait-call", + "wait_agent", + json!({ + "targets": [spawn_output["work_ref"]], + "replies_to": [send_output["message_id"]], + "return_when": "all", + }), + ); + } + + if has_function_output(&body, "read-call") { + let send_output: Value = serde_json::from_str( + function_output_text(&body, "request-call") + .expect("request receipt should remain in provider history"), + ) + .expect("request receipt should be JSON"); + return function_call_response( + "mailbox-reply-response", + "reply-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "note", + "reply_to": send_output["message_id"], + "message": QUIET_REPLY_BODY, + }), + ); + } + + if has_function_output(&body, "request-call") { + return function_call_response( + "mailbox-read-response", + "read-call", + "mailbox_read", + json!({}), + ); + } + + if has_function_output(&body, "spawn-call") { + return function_call_response( + "mailbox-request-response", + "request-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "request", + "message": REQUEST_BODY, + }), + ); + } + + function_call_response( + "mailbox-spawn-response", + "spawn-call", + "spawn_agent", + json!({ + "task_name": "quick-worker", + "message": CHILD_TASK, + }), + ) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() -> Result<()> { + let server = start_mock_server().await; + let mut builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + config + .features + .enable(feature) + .expect("test config should allow feature update"); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + config.multi_agent_v2.max_concurrent_threads_per_session = 2; + }); + let test = builder.build_with_auto_env(&server).await?; + let root_thread_id = test.session_configured.thread_id.to_string(); + let captured = Arc::new(Mutex::new(Vec::new())); + + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(RootThreadMatcher { + thread_id: root_thread_id.clone(), + }) + .respond_with(RootMailboxFlowResponder { + root_thread_id: root_thread_id.clone(), + requests: Arc::clone(&captured), + }) + .up_to_n_times(6) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(ChildThreadMatcher { + root_thread_id: root_thread_id.clone(), + task: CHILD_TASK.to_string(), + }) + .respond_with(sse_response(sse(vec![ + ev_response_created("quick-worker-response"), + ev_assistant_message("quick-worker-message", "done"), + ev_completed("quick-worker-response"), + ]))) + .up_to_n_times(1) + .mount(&server) + .await; + + test.submit_turn(FLOW_PROMPT).await?; + + let requests = captured + .lock() + .expect("request capture lock should not be poisoned") + .clone(); + assert_eq!(requests.len(), 6, "root should make one provider call per tool stage"); + + let spawn: Value = serde_json::from_str( + function_output_text(&requests[5], "spawn-call").expect("spawn output should be retained"), + )?; + let work_ref = spawn["work_ref"] + .as_str() + .expect("spawn should return an immutable mailbox work reference"); + assert!(work_ref.starts_with("work_"), "unexpected work reference: {work_ref}"); + + let request_receipt: Value = serde_json::from_str( + function_output_text(&requests[5], "request-call") + .expect("request receipt should be retained"), + )?; + let request_id = request_receipt["message_id"] + .as_str() + .expect("request receipt should contain a native message ID"); + assert!(request_id.starts_with("amsg_"), "unexpected native ID: {request_id}"); + + let read_output: Value = serde_json::from_str( + function_output_text(&requests[3], "read-call") + .expect("mailbox_read should return a receipt"), + )?; + assert_eq!(read_output["messages"][0]["message_id"], request_id); + assert_eq!(read_output["messages"][0]["canonical_delivered"], true); + assert!(read_output["messages"][0].get("message").is_none()); + assert!(read_output["messages"][0].get("body").is_none()); + assert!(!read_output.to_string().contains(REQUEST_BODY)); + + let history = requests[3]["input"] + .as_array() + .expect("provider input should be an array") + .iter() + .find(|item| { + item.get("type").and_then(Value::as_str) == Some("agent_message") + && item.get("id").and_then(Value::as_str) == Some(request_id) + }) + .expect("mailbox_read should append the request to canonical provider history"); + assert_eq!(history["content"][1]["type"], "encrypted_content"); + assert_eq!(history["content"][1]["encrypted_content"], REQUEST_BODY); + + let reply_receipt: Value = serde_json::from_str( + function_output_text(&requests[5], "reply-call") + .expect("final reply receipt should be retained"), + )?; + assert!(reply_receipt["message_id"].as_str().is_some_and(|id| id.starts_with("amsg_"))); + let wait_report: Value = serde_json::from_str( + function_output_text(&requests[5], "wait-call") + .expect("wait should return a selector report"), + )?; + assert_eq!(wait_report["requested_targets"], json!([work_ref])); + assert_eq!(wait_report["requested_replies_to"], json!([request_id])); + assert_eq!(wait_report["return_when"], "all"); + assert_eq!(wait_report["condition_satisfied"], true); + assert!(wait_report["completed"].as_array().is_some_and(|completed| { + completed.iter().any(|item| item["work_ref"] == work_ref && item["status"] == "completed") + && completed + .iter() + .any(|item| item["message_id"] == request_id && item["status"] == "replied") + })); + assert_eq!(wait_report["pending"], json!([])); + + Ok(()) +} + +const QUIET_CHILD_TASK: &str = "send quiet mailbox updates then an addressed request"; +const UNRELATED_CHILD_TASK: &str = "complete without being selected by the parent wait"; +const QUIET_NOTE_BODY: &str = "non-attention note while the parent waits"; +const QUIET_PROGRESS_BODY: &str = "non-attention progress while the parent waits"; +const ADDRESSED_REQUEST_BODY: &str = "please inspect this request"; + +#[derive(Clone)] +struct WaitProbeResponder { + root_thread_id: String, + requests: Arc>>, + first_wait_started: Arc>>, + event_only_arguments: Arc>>, +} + +impl Respond for WaitProbeResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body) + .expect("provider request should contain JSON"); + let received_at = Instant::now(); + self.requests + .lock() + .expect("request capture lock should not be poisoned") + .push((received_at, body.clone())); + + if has_function_output(&body, "wait-2-call") { + return sse_response(sse(vec![ + ev_response_created("wait-probe-final"), + ev_assistant_message("wait-probe-final-message", "finished"), + ev_completed("wait-probe-final"), + ])); + } + + if has_function_output(&body, "wait-1-call") { + let spawn_output: Value = serde_json::from_str( + function_output_text(&body, "spawn-a-call") + .expect("watched worker output should remain in provider history"), + ) + .expect("spawn output should be JSON"); + let arguments = json!({ + "targets": [spawn_output["work_ref"]], + "return_when": "all", + }); + *self + .event_only_arguments + .lock() + .expect("event-only argument lock should not be poisoned") = + Some(arguments.clone()); + return function_call_response( + "wait-probe-event-only-response", + "wait-2-call", + "wait_agent", + arguments, + ); + } + + if has_function_output(&body, "spawn-b-call") { + let spawn_output: Value = serde_json::from_str( + function_output_text(&body, "spawn-a-call") + .expect("watched worker output should remain in provider history"), + ) + .expect("spawn output should be JSON"); + *self + .first_wait_started + .lock() + .expect("wait start lock should not be poisoned") = Some(Instant::now()); + return function_call_response( + "wait-probe-deadline-response", + "wait-1-call", + "wait_agent", + json!({ + "targets": [spawn_output["work_ref"]], + "return_when": "all", + "timeout_ms": 1_000, + }), + ); + } + + if has_function_output(&body, "spawn-a-call") { + return function_call_response( + "wait-probe-unrelated-spawn-response", + "spawn-b-call", + "spawn_agent", + json!({ + "task_name": "unrelated-worker", + "message": UNRELATED_CHILD_TASK, + }), + ); + } + + function_call_response( + "wait-probe-watched-spawn-response", + "spawn-a-call", + "spawn_agent", + json!({ + "task_name": "watched-worker", + "message": QUIET_CHILD_TASK, + }), + ) + } +} + +#[derive(Clone)] +struct QuietThenRequestResponder { + root_thread_id: String, +} + +impl Respond for QuietThenRequestResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body) + .expect("child provider request should contain JSON"); + + let delayed = |response: ResponseTemplate| response.set_delay(Duration::from_millis(200)); + if has_function_output(&body, "addressed-request-call") { + return sse_response(sse(vec![ + ev_response_created("quiet-child-final"), + ev_assistant_message("quiet-child-final-message", "done"), + ev_completed("quiet-child-final"), + ])).set_delay(Duration::from_secs(3)); + } + + if has_function_output(&body, "quiet-progress-3-call") { + return function_call_response( + "quiet-child-request-response", + "addressed-request-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "request", + "message": ADDRESSED_REQUEST_BODY, + }), + ) + .set_delay(Duration::from_millis(400)); + } + + if has_function_output(&body, "quiet-progress-2-call") { + return delayed(function_call_response( + "quiet-child-progress-3-response", + "quiet-progress-3-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "progress", + "message": QUIET_PROGRESS_BODY, + }), + )); + } + + if has_function_output(&body, "quiet-progress-1-call") { + return delayed(function_call_response( + "quiet-child-progress-2-response", + "quiet-progress-2-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "progress", + "message": QUIET_PROGRESS_BODY, + }), + )); + } + + if has_function_output(&body, "quiet-note-call") { + return delayed(function_call_response( + "quiet-child-progress-1-response", + "quiet-progress-1-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "progress", + "message": QUIET_PROGRESS_BODY, + }), + )); + } + + delayed(function_call_response( + "quiet-child-note-response", + "quiet-note-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "note", + "message": QUIET_NOTE_BODY, + }), + )) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() -> Result<()> { + let server = start_mock_server().await; + let mut builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + config + .features + .enable(feature) + .expect("test config should allow feature update"); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + config.multi_agent_v2.max_concurrent_threads_per_session = 3; + }); + let test = builder.build_with_auto_env(&server).await?; + let root_thread_id = test.session_configured.thread_id.to_string(); + let captured = Arc::new(Mutex::new(Vec::new())); + let first_wait_started = Arc::new(Mutex::new(None)); + let event_only_arguments = Arc::new(Mutex::new(None)); + + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(RootThreadMatcher { + thread_id: root_thread_id.clone(), + }) + .respond_with(WaitProbeResponder { + root_thread_id: root_thread_id.clone(), + requests: Arc::clone(&captured), + first_wait_started: Arc::clone(&first_wait_started), + event_only_arguments: Arc::clone(&event_only_arguments), + }) + .up_to_n_times(5) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(ChildThreadMatcher { + root_thread_id: root_thread_id.clone(), + task: QUIET_CHILD_TASK.to_string(), + }) + .respond_with(QuietThenRequestResponder { + root_thread_id: root_thread_id.clone(), + }) + .up_to_n_times(6) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(ChildThreadMatcher { + root_thread_id: root_thread_id.clone(), + task: UNRELATED_CHILD_TASK.to_string(), + }) + .respond_with(sse_response(sse(vec![ + ev_response_created("unrelated-worker-response"), + ev_assistant_message("unrelated-worker-message", "done"), + ev_completed("unrelated-worker-response"), + ]))) + .up_to_n_times(1) + .mount(&server) + .await; + + test.submit_turn("verify quiet events, the deadline, and an addressed wake") + .await?; + + let requests = captured + .lock() + .expect("request capture lock should not be poisoned") + .clone(); + assert_eq!(requests.len(), 5, "quiet events must not add root provider calls"); + let wait_started = first_wait_started + .lock() + .expect("wait start lock should not be poisoned") + .expect("first targeted wait should have started"); + let deadline_wait_elapsed = requests[3].0.duration_since(wait_started); + assert!( + (Duration::from_millis(850)..Duration::from_millis(1_500)) + .contains(&deadline_wait_elapsed), + "quiet messages should not refresh the one-second deadline: {deadline_wait_elapsed:?}" + ); + + let watched_spawn: Value = serde_json::from_str( + function_output_text(&requests[4].1, "spawn-a-call") + .expect("watched spawn result should remain in history"), + )?; + let work_ref = watched_spawn["work_ref"] + .as_str() + .expect("watched spawn should return an immutable work reference"); + let unrelated_spawn: Value = serde_json::from_str( + function_output_text(&requests[4].1, "spawn-b-call") + .expect("unrelated spawn result should remain in history"), + )?; + let unrelated_work_ref = unrelated_spawn["work_ref"] + .as_str() + .expect("unrelated spawn should return an immutable work reference"); + assert_ne!(work_ref, unrelated_work_ref); + + let deadline_report: Value = serde_json::from_str( + function_output_text(&requests[3].1, "wait-1-call") + .expect("first wait should return its deadline report"), + )?; + assert_eq!(deadline_report["requested_targets"], json!([work_ref])); + assert_eq!(deadline_report["completion_reason"], "deadline"); + assert_eq!(deadline_report["condition_satisfied"], false); + assert!(deadline_report["quiet_count"].as_u64().unwrap_or_default() >= 4); + assert_eq!(deadline_report["completed"], json!([])); + assert_eq!(deadline_report["pending"][0]["work_ref"], work_ref); + assert_eq!(deadline_report["pending"][0]["status"], "pending"); + assert!(deadline_report.to_string().contains(work_ref)); + assert!(!deadline_report.to_string().contains(unrelated_work_ref)); + + let event_only_args = event_only_arguments + .lock() + .expect("event-only argument lock should not be poisoned") + .clone() + .expect("second wait call should follow the explicit deadline"); + assert!(event_only_args.get("timeout_ms").is_none()); + + let request_report: Value = serde_json::from_str( + function_output_text(&requests[4].1, "wait-2-call") + .expect("event-only wait should return the addressed request report"), + )?; + assert_eq!(request_report["requested_targets"], json!([work_ref])); + assert_eq!(request_report["completion_reason"], "request"); + assert_eq!(request_report["condition_satisfied"], false); + assert_eq!(request_report["completed"], json!([])); + assert_eq!(request_report["pending"][0]["work_ref"], work_ref); + assert_eq!(request_report["pending"][0]["status"], "pending"); + let wake_ref = request_report["wake_ref"] + .as_str() + .expect("addressed request should identify the exact wake message"); + assert!(wake_ref.starts_with("amsg_")); + + let canonical_request = requests[4].1["input"] + .as_array() + .expect("provider input should be an array") + .iter() + .find(|item| { + item.get("type").and_then(Value::as_str) == Some("agent_message") + && item.get("id").and_then(Value::as_str) == Some(wake_ref) + }) + .expect("wait should admit the addressed request into canonical history"); + assert_eq!(canonical_request["content"][1]["type"], "encrypted_content"); + assert_eq!(canonical_request["content"][1]["encrypted_content"], ADDRESSED_REQUEST_BODY); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index ea28172dc9df..76a1dd93fd6f 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -37,6 +37,7 @@ pub static CODEX_ALIASES_TEMP_DIR: Option = { mod abort_tasks; mod additional_context; mod agent_execution; +mod agent_mailbox; mod agent_websocket; mod agents_md; mod apply_patch_cli; diff --git a/codex-rs/tui/src/multi_agents/mailbox_wait.rs b/codex-rs/tui/src/multi_agents/mailbox_wait.rs index bbb9c173799f..0316cefd2c8a 100644 --- a/codex-rs/tui/src/multi_agents/mailbox_wait.rs +++ b/codex-rs/tui/src/multi_agents/mailbox_wait.rs @@ -14,12 +14,15 @@ use std::sync::Arc; use std::sync::Mutex; pub(crate) fn begin(report: &MailboxWaitReport) -> PlainHistoryCell { - let selection_count = report.requested_targets.len() + report.requested_replies_to.len(); - let title = if selection_count == 0 { + let selection_count = report.completed.len() + report.pending.len(); + let targetless = report.requested_targets.is_empty() && report.requested_replies_to.is_empty(); + let title = if targetless && selection_count == 0 { format!( "Waiting for mailbox attention ({})", return_when_label(report.return_when) ) + } else if targetless { + format!("Waiting for mailbox attention or {} of {selection_count} work outcomes", return_when_label(report.return_when)) } else { format!( "Waiting for {} of {selection_count} selected conditions", From 1b700653094c9d23e7dcee27b580163379ec3dc9 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:53:00 +1100 Subject: [PATCH 04/54] Quote the focused native integration test command in hosted workflow --- .github/workflows/mailbox-source-preparation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 799c8789c270..384c89a56163 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -62,7 +62,8 @@ jobs: run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 - name: Run real native mailbox adapter contracts working-directory: codex-rs - run: cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 + run: | + cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 - name: Compile focused UI mailbox consumers working-directory: codex-rs env: From 8589c836e2ebe0f6de7b72e84a0d36279ca1b39d Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:10:42 +1100 Subject: [PATCH 05/54] Bind wait observations to canonical result bodies and native input admission --- .../workflows/mailbox-source-preparation.yml | 7 +- codex-rs/agent-mailbox/src/error.rs | 2 + codex-rs/agent-mailbox/src/lib.rs | 1 + codex-rs/agent-mailbox/src/mailbox_tests.rs | 102 ++++++-- .../agent-mailbox/src/message_delivery.rs | 11 + codex-rs/agent-mailbox/src/messages.rs | 1 + codex-rs/agent-mailbox/src/outcome.rs | 2 + codex-rs/agent-mailbox/src/runtime_fault.rs | 1 + codex-rs/agent-mailbox/src/schema.rs | 7 + codex-rs/agent-mailbox/src/snapshot.rs | 13 +- codex-rs/agent-mailbox/src/store.rs | 5 + codex-rs/agent-mailbox/src/wait_graph.rs | 62 +++++ codex-rs/agent-mailbox/src/work.rs | 15 ++ codex-rs/core/src/session/input_queue.rs | 42 ++++ codex-rs/core/src/session/session.rs | 6 + .../multi_agents_v2/mailbox_message.rs | 15 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 35 +-- .../multi_agents_v2/mailbox_wait_tests.rs | 92 +++++++ codex-rs/core/tests/suite/agent_mailbox.rs | 238 ++++++++++++++++++ 19 files changed, 618 insertions(+), 39 deletions(-) create mode 100644 codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 384c89a56163..dd72c7e555bf 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "2" + PREPARATION_ATTEMPT: "3" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -51,6 +51,8 @@ jobs: - name: Refresh the Bazel dependency lock run: just bazel-lock-update - name: Run the owned library contracts + id: library + continue-on-error: true working-directory: codex-rs run: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 - name: Generate the affected native schemas @@ -69,6 +71,9 @@ jobs: env: INSTA_UPDATE: always run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 + - name: Retain the library contract result + if: always() + run: test '${{ steps.library.outcome }}' = success - name: Capture the exact generated source delta if: always() shell: bash diff --git a/codex-rs/agent-mailbox/src/error.rs b/codex-rs/agent-mailbox/src/error.rs index 1d47d5376aad..3fa9ee8c51cf 100644 --- a/codex-rs/agent-mailbox/src/error.rs +++ b/codex-rs/agent-mailbox/src/error.rs @@ -32,4 +32,6 @@ pub enum MailboxError { InvalidCursor, #[error("wait depends on a provably closed peer cycle")] ClosedPeerCycle, + #[error("eligible input changed while the wait condition was being selected")] + StaleWaitSnapshot, } diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index 31b9e2742e5e..95243d394ea4 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -24,6 +24,7 @@ pub use error::{MailboxError, MailboxResult}; pub use store::Mailbox; pub use snapshot::is_provably_closed_peer_cycle; pub use wait_graph::WaitRegistration; +pub use wait_graph::InputArrival; pub use types::{ ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, CanonicalConfirmation, Continuation, ContinuationReceipt, ContinuationState, Dependency, diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 15cc835e637a..31f1f3cb9a8a 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -451,11 +451,16 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom let mut update_draft = draft("p2", author, agent, MessageKind::Progress, "latest", false); update_draft.work_ref = Some(work_a.clone()); let update = send(&fixture.mailbox, update_draft).await; - assert_eq!(update.id, first.id); - assert_eq!( - fixture.mailbox.get_message(agent, &first.id).await.unwrap().unwrap().communication.content, - "latest" - ); + assert_ne!(update.id, first.id); + assert_eq!(update.supersedes.as_deref(), Some(first.id.as_str())); + let original = fixture.mailbox.get_message(agent, &first.id).await.unwrap().unwrap(); + assert_eq!(original.communication.content, "first"); + let latest = fixture.mailbox.get_message(agent, &update.id).await.unwrap().unwrap(); + assert_eq!(latest.communication.content, "latest"); + assert_eq!(latest.supersedes.as_deref(), Some(first.id.as_str())); + let pending = fixture.mailbox.pending(agent, 20).await.unwrap(); + assert!(pending.iter().any(|envelope| envelope.id == update.id)); + assert!(!pending.iter().any(|envelope| envelope.id == first.id)); let mut other_author = draft("p3", author, agent, MessageKind::Progress, "other path", false); other_author.work_ref = Some(work_a.clone()); @@ -491,7 +496,7 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { let outcome = finish( &fixture.mailbox, agent, turn, Outcome::Completed, None, Continuation::None, ).await; - fixture.mailbox.mark_outcome_canonical(agent, &outcome.outcomes[0].id).await.unwrap(); + fixture.mailbox.mark_outcome_canonical(issuer, &outcome.outcomes[0].id).await.unwrap(); assert_eq!(&outcome.outcomes[0].work_ref, work); } fixture.mailbox.acknowledge_observed( @@ -503,7 +508,7 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { ReturnWhen::All, ); let partial = fixture.mailbox.snapshot(&selection).await.unwrap(); - assert_eq!(partial.state, SnapshotState::Pending); + assert_eq!(partial.state, SnapshotState::AlreadySatisfied); assert!(!work_item(&partial, &a.id).observed); assert!(work_item(&partial, &b.id).observed); assert!( @@ -662,19 +667,23 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re async fn activation_claim_is_single_use_across_a_lost_result() { let fixture = test_mailbox().await; let (sender, recipient) = (thread_id(20), thread_id(21)); - let message = send( - &fixture.mailbox, - draft("activation", sender, recipient, MessageKind::Alert, "activate", false), - ).await; - assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + let mut message_draft = draft( + "activation", sender, recipient, MessageKind::Alert, "activate", false, + ); + message_draft.activate_if_idle = true; + let message = send(&fixture.mailbox, message_draft).await; + assert_eq!( + fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), ActivationDecision::Activate); - assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + assert_eq!( + fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), ActivationDecision::RecoveryNeeded); for _ in 0..2 { fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Activated) .await.unwrap(); } - assert_eq!(fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), + assert_eq!( + fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), ActivationDecision::AlreadyDecided(ActivationDisposition::Activated)); assert!(matches!( fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Denied).await, @@ -699,10 +708,15 @@ fn only_closed_any_and_all_peer_cycles_are_provable() { dependency("b", &["a"], DependencyMode::All), ]; assert!(!is_provably_closed_peer_cycle("a", &open)); - let open_all = [ + let closed_all = [ dependency("x", &["y", "outside"], DependencyMode::All), dependency("y", &["x"], DependencyMode::Any), ]; + assert!(is_provably_closed_peer_cycle("x", &closed_all)); + let open_all = [ + dependency("x", &["y"], DependencyMode::All), + dependency("y", &["x", "outside"], DependencyMode::Any), + ]; assert!(!is_provably_closed_peer_cycle("x", &open_all)); } @@ -732,3 +746,61 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { "open-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, ).is_ok()); } + +#[tokio::test] +async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until_rescan() { + let fixture = test_mailbox().await; + let (a, b) = (thread_id(28), thread_id(29)); + let before = fixture.mailbox.wait_input_generation(); + let old = fixture.mailbox.begin_wait_at( + "a".into(), a, vec![b], ReturnWhen::Any, false, before, + ).unwrap(); + let arrival = fixture.mailbox.begin_input_arrival(); + assert!(matches!(fixture.mailbox.begin_wait_at( + "stale".into(), b, vec![a], ReturnWhen::All, false, before, + ), Err(MailboxError::StaleWaitSnapshot))); + let during = fixture.mailbox.wait_input_generation(); + let provisional_a = fixture.mailbox.begin_wait_at( + "a".into(), a, vec![b], ReturnWhen::Any, false, during, + ).unwrap(); + // A stale RAII handle must not remove a replacement registration. + drop(old); + let provisional_b = fixture.mailbox.begin_wait_at( + "b".into(), b, vec![a], ReturnWhen::All, false, during, + ).unwrap(); + drop(arrival); + assert!(matches!(fixture.mailbox.begin_wait_at( + "needs-rescan".into(), b, vec![a], ReturnWhen::All, false, during, + ), Err(MailboxError::StaleWaitSnapshot))); + drop(provisional_a); + drop(provisional_b); + let current = fixture.mailbox.wait_input_generation(); + let _waiting_a = fixture.mailbox.begin_wait_at( + "fresh-a".into(), a, vec![b], ReturnWhen::Any, false, current, + ).unwrap(); + assert!(matches!(fixture.mailbox.begin_wait_at( + "fresh-b".into(), b, vec![a], ReturnWhen::All, false, current, + ), Err(MailboxError::ClosedPeerCycle))); +} + +#[tokio::test] +async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attention() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(30), thread_id(31)); + let (covered, input) = accept_work(&fixture.mailbox, issuer, agent, "covered", "A").await; + let (unrelated, _) = accept_work(&fixture.mailbox, issuer, agent, "unrelated", "B").await; + admit(&fixture.mailbox, agent, "covered-turn", &[input]).await; + let terminal = finish(&fixture.mailbox, agent, "covered-turn", Outcome::NeedsAttention, None, Continuation::None).await; + let outcome = &terminal.outcomes[0]; + let envelope = fixture.mailbox.get_message(issuer, &outcome.id).await.unwrap().unwrap(); + assert_eq!(envelope.communication.id.as_ref().unwrap().as_str(), outcome.id); + assert!(!fixture.mailbox.attention(issuer, 100).await.unwrap().is_empty()); + fixture.mailbox.mark_canonical(issuer, &outcome.id).await.unwrap(); + let exact = fixture.mailbox.work_for_outcome(issuer, &outcome.id).await.unwrap(); + assert_eq!(exact, vec![covered.clone()]); + fixture.mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(covered.clone())]).await.unwrap(); + assert!(fixture.mailbox.attention(issuer, 100).await.unwrap().is_empty()); + let snapshot = take_snapshot(&fixture.mailbox, issuer, Some(agent), vec![Selector::Work(covered.id.clone()), Selector::Work(unrelated.id.clone())], ReturnWhen::All).await; + assert!(work_item(&snapshot, &covered.id).observed); + assert!(!work_item(&snapshot, &unrelated.id).observed); +} diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index 8de8699aff21..2c0aa90b8e9d 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -79,6 +79,7 @@ impl Mailbox { /// Withdraw an unresolved request authored by this exact native thread. /// Withdrawal is durable and idempotent; replies cannot target it afterward. pub async fn withdraw_message(&self, sender: ThreadId, id: &str) -> MailboxResult<()> { + let _input_arrival = self.begin_input_arrival(); let mut transaction = self.pool.begin().await?; let row = sqlx::query( "SELECT sender, kind, canonical, input_admitted, legacy_attention, @@ -354,6 +355,16 @@ impl Mailbox { .bind(recipient.to_string()) .execute(&mut *transaction) .await?; + sqlx::query( + "UPDATE mailbox_attention SET canonical = 1 WHERE recipient = ? AND id IN ( + SELECT 'attention:' || id FROM mailbox_work WHERE outcome_id = ? AND issuer = ? + )", + ) + .bind(recipient.to_string()) + .bind(id) + .bind(recipient.to_string()) + .execute(&mut *transaction) + .await?; transaction.commit().await?; self.notify_changed(); Ok(()) diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index bb6af62e8238..fe86411af328 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -31,6 +31,7 @@ impl Mailbox { /// Durably accept a message intent. Retries with the same invocation key /// return its stable ID; reusing a key for a different request is rejected. pub async fn accept_message(&self, draft: MessageDraft) -> MailboxResult { + let _input_arrival = self.begin_input_arrival(); let id = match draft.communication.id.as_ref() { Some(id) if id.as_str().starts_with("amsg_") => id.to_string(), Some(_) => { diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index 0596bfd9a41c..721f459481c0 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -32,6 +32,7 @@ impl Mailbox { response: Option, continuation: Continuation, ) -> MailboxResult { + let _input_arrival = self.begin_input_arrival(); validate_key(source_turn_id)?; let mut transaction = self.pool.begin().await?; let admitted_rows = sqlx::query( @@ -342,6 +343,7 @@ impl Mailbox { response: Option, continuation: Continuation, ) -> MailboxResult { + let _input_arrival = self.begin_input_arrival(); if is_nonterminal(&continuation) { return Err(MailboxError::InvalidInput( "pending continuation requires admitted turn linkage".into(), diff --git a/codex-rs/agent-mailbox/src/runtime_fault.rs b/codex-rs/agent-mailbox/src/runtime_fault.rs index 330c62d5e3a4..82fa71bde4a0 100644 --- a/codex-rs/agent-mailbox/src/runtime_fault.rs +++ b/codex-rs/agent-mailbox/src/runtime_fault.rs @@ -15,6 +15,7 @@ impl Mailbox { /// Signal an in-memory observation that terminal custody could not settle. /// This does not create or fabricate a durable work outcome. pub fn notify_runtime_fault(&self, agent: ThreadId) { + let _input_arrival = self.begin_input_arrival(); let mut faults = lock(&self.runtime_faults); let agent = agent.to_string(); let changed = if faults.agents.contains(&agent) { diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 2e20b7125849..90320ad01c41 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -108,6 +108,13 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; + for index in [ + "CREATE INDEX IF NOT EXISTS mailbox_work_unobserved ON mailbox_work(issuer, outcome_observed, seq)", + "CREATE INDEX IF NOT EXISTS mailbox_work_agent_unobserved ON mailbox_work(issuer, agent, outcome_observed, seq)", + "CREATE INDEX IF NOT EXISTS mailbox_work_outcome ON mailbox_work(issuer, outcome_id, seq)", + ] { + sqlx::query(index).execute(&mut *transaction).await?; + } sqlx::query( "CREATE TABLE IF NOT EXISTS mailbox_admissions ( recipient TEXT NOT NULL, diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs index e1038e56f7b7..35e7ceba4dff 100644 --- a/codex-rs/agent-mailbox/src/snapshot.rs +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -177,7 +177,7 @@ impl Mailbox { return Err(MailboxError::WrongRecipient); } let row = sqlx::query( - "SELECT outcome_canonical, outcome_json FROM mailbox_work + "SELECT outcome_canonical, outcome_json, outcome_id FROM mailbox_work WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ?", ) .bind(&work_ref.id) @@ -203,6 +203,17 @@ impl Mailbox { .bind(&work_ref.submission_id) .execute(&mut *transaction) .await?; + // This notification describes the same exact admitted result body, + // not a separate request or an inference of business acceptance. + sqlx::query( + "UPDATE mailbox_attention SET observed = 1 + WHERE recipient = ? AND canonical = 1 AND id IN (?, ?)", + ) + .bind(recipient.to_string()) + .bind(format!("attention:{}", work_ref.id)) + .bind(format!("attention:{}", row.try_get::("outcome_id")?)) + .execute(&mut *transaction) + .await?; } ObservationTarget::Reply(id) => { let result = sqlx::query( diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index 6aba8f803472..7f8d0b8564df 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -1,6 +1,7 @@ use crate::error::MailboxResult; use std::collections::HashMap; use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; use sqlx::SqlitePool; use tokio::sync::watch; @@ -11,6 +12,8 @@ pub struct Mailbox { change_tx: watch::Sender, pub(crate) waits: Arc>>, pub(crate) wait_generation: Arc, + pub(crate) wait_input_generation: Arc, + pub(crate) pending_input_arrivals: Arc, pub(crate) runtime_faults: Arc>, } @@ -26,6 +29,8 @@ impl Mailbox { change_tx, waits: Arc::new(Mutex::new(HashMap::new())), wait_generation: Arc::new(AtomicU64::new(1)), + wait_input_generation: Arc::new(AtomicU64::new(0)), + pending_input_arrivals: Arc::new(AtomicUsize::new(0)), runtime_faults: Arc::new(Mutex::new(crate::runtime_fault::RuntimeFaults::default())), }) } diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs index 1ef1188e9399..7d5d9246aa50 100644 --- a/codex-rs/agent-mailbox/src/wait_graph.rs +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -27,6 +27,23 @@ pub struct WaitRegistration { waits: Weak>>, } +/// Short-lived custody of an input publication, not a scheduler or durable queue. +/// While an input transaction can make an old dependency stale, closed-cycle +/// claims are conservatively suspended. The adapter holds no mutex across await. +pub struct InputArrival { + mailbox: Mailbox, +} + +impl Drop for InputArrival { + fn drop(&mut self) { + let waits = lock(&self.mailbox.waits); + self.mailbox.wait_input_generation.fetch_add(1, Ordering::AcqRel); + self.mailbox.pending_input_arrivals.fetch_sub(1, Ordering::AcqRel); + drop(waits); + self.mailbox.notify_changed(); + } +} + impl WaitRegistration { pub fn wait_id(&self) -> &str { &self.wait_id @@ -48,6 +65,22 @@ impl Drop for WaitRegistration { } impl Mailbox { + /// Bracket an existing native enqueue or mailbox transaction before it can + /// make input eligible. A scalar in-flight count avoids a shadow agent registry. + pub fn begin_input_arrival(&self) -> InputArrival { + let mut waits = lock(&self.waits); + waits.clear(); + self.wait_input_generation.fetch_add(1, Ordering::AcqRel); + self.pending_input_arrivals.fetch_add(1, Ordering::AcqRel); + drop(waits); + InputArrival { mailbox: self.clone() } + } + + /// Capture before the authoritative selection/input rescan. + pub fn wait_input_generation(&self) -> u64 { + self.wait_input_generation.load(Ordering::Acquire) + } + /// Invalidate active dependency edges for an owner when eligible input is /// about to become available. Stale RAII handles cannot remove replacements. pub fn invalidate_waits(&self, owner: ThreadId) { @@ -66,6 +99,31 @@ impl Mailbox { peers: Vec, mode: ReturnWhen, has_escape: bool, + ) -> MailboxResult { + self.begin_wait_inner(wait_id, owner, peers, mode, has_escape, None) + } + + /// Register only if no eligible-input publication raced the caller's rescan. + pub fn begin_wait_at( + &self, + wait_id: String, + owner: ThreadId, + peers: Vec, + mode: ReturnWhen, + has_escape: bool, + input_generation: u64, + ) -> MailboxResult { + self.begin_wait_inner(wait_id, owner, peers, mode, has_escape, Some(input_generation)) + } + + fn begin_wait_inner( + &self, + wait_id: String, + owner: ThreadId, + peers: Vec, + mode: ReturnWhen, + has_escape: bool, + input_generation: Option, ) -> MailboxResult { if wait_id.is_empty() || wait_id.len() > MAX_WAIT_ID_BYTES { return Err(MailboxError::InvalidInput("invalid wait registration ID".into())); @@ -81,6 +139,10 @@ impl Mailbox { return Err(MailboxError::InvalidInput("duplicate wait dependency".into())); } let mut waits = lock(&self.waits); + if input_generation.is_some_and(|generation| generation != self.wait_input_generation.load(Ordering::Acquire)) { + return Err(MailboxError::StaleWaitSnapshot); + } + let has_escape = has_escape || self.pending_input_arrivals.load(Ordering::Acquire) != 0; if waits.contains_key(&wait_id) { return Err(MailboxError::IdempotencyConflict); } diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index 1fc423af1c73..80fe746355c3 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -15,6 +15,21 @@ const MAX_KEY_BYTES: usize = 256; const MAX_TURN_INPUTS: usize = 100; impl Mailbox { + /// Resolve only the work covered by this exact canonical result body. + pub async fn work_for_outcome(&self, issuer: ThreadId, outcome_id: &str) -> MailboxResult> { + let rows = sqlx::query( + "SELECT * FROM mailbox_work WHERE issuer = ? AND outcome_id = ? ORDER BY seq LIMIT 101", + ) + .bind(issuer.to_string()) + .bind(outcome_id) + .fetch_all(&self.pool) + .await?; + if rows.len() > 100 { + return Err(MailboxError::Backpressure("outcome work reference count")); + } + rows.iter().map(work_ref_from_row).collect() + } + /// Capture accepted native work and its input before the adapter activates it. /// Retries of the exact issuer-scoped native key return the same immutable ref. pub async fn accept_work(&self, mut work: NativeWork) -> MailboxResult { diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index bfd15ed1ad10..daf55d9e445e 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -6,10 +6,13 @@ use crate::state::TurnState; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::user_input::UserInput; +use codex_agent_mailbox::InputArrival; +use codex_agent_mailbox::Mailbox; use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex as StdMutex; +use std::sync::OnceLock; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -60,6 +63,7 @@ pub(crate) struct MailboxEntry { /// Session-scoped pending input storage and active-turn mailbox delivery coordination. pub(crate) struct InputQueue { activity_tx: watch::Sender, + durable_mailbox: OnceLock>, mailbox: Mutex, pending_mailbox_entries: Mutex>, next_mailbox_sequence: AtomicU64, @@ -80,6 +84,7 @@ impl InputQueue { let (activity_tx, _) = watch::channel(InputQueueActivity::Mailbox); Self { activity_tx, + durable_mailbox: OnceLock::new(), mailbox: Mutex::new(MailboxQueue::default()), pending_mailbox_entries: Mutex::new(VecDeque::new()), next_mailbox_sequence: AtomicU64::new(0), @@ -93,6 +98,15 @@ impl InputQueue { } } + /// Only enabled V2 sessions bind this adapter. Native queues remain authoritative. + pub(crate) fn bind_durable_mailbox(&self, mailbox: Arc) { + let _ = self.durable_mailbox.set(mailbox); + } + + fn reserve_mailbox_input(&self) -> Option { + self.durable_mailbox.get().map(|mailbox| mailbox.begin_input_arrival()) + } + pub(crate) async fn begin_residency_activity(&self) -> OwnedMutexGuard<()> { let guard = Arc::clone(&self.residency_transition).lock_owned().await; self.residency_activity_generation @@ -224,6 +238,7 @@ impl InputQueue { if communications.is_empty() { return; } + let _input_arrival = self.reserve_mailbox_input(); let communication_count = communications.len(); let mut mailbox = self.mailbox.lock().await; let enqueued_at_ms = current_time_ms(); @@ -249,6 +264,7 @@ impl InputQueue { if entries.is_empty() { return; } + let _input_arrival = self.reserve_mailbox_input(); let communication_count = entries.len(); let mut mailbox = self.mailbox.lock().await; mailbox.entries.extend(entries); @@ -301,6 +317,7 @@ impl InputQueue { if entries.is_empty() { return; } + let _input_arrival = self.reserve_mailbox_input(); let mut mailbox = self.mailbox.lock().await; for entry in entries.into_iter().rev() { mailbox.entries.push_front(entry); @@ -435,6 +452,7 @@ impl InputQueue { &self, mut completion: TerminalCompletionNotification, ) { + let _input_arrival = self.reserve_mailbox_input(); let mut pending = self.terminal_completions.lock().await; if pending .iter() @@ -686,6 +704,7 @@ impl InputQueue { turn_state: &Mutex, input: Vec, ) { + let _input_arrival = if input.is_empty() { None } else { self.reserve_mailbox_input() }; let activity = if input .iter() .any(|input| matches!(input, TurnInput::UserInput { .. })) @@ -712,6 +731,7 @@ impl InputQueue { turn_state: &Mutex, input: Vec, ) { + let _input_arrival = if input.is_empty() { None } else { self.reserve_mailbox_input() }; turn_state.lock().await.pending_input.items.extend(input); } @@ -889,6 +909,28 @@ mod tests { } } + #[tokio::test] + async fn durable_mailbox_input_arrival_invalidates_stale_cycle_snapshots() { + use codex_agent_mailbox::MailboxError; + use codex_agent_mailbox::ReturnWhen; + use codex_protocol::ThreadId; + use codex_utils_absolute_path::test_support::PathExt; + + let home = tempfile::tempdir().unwrap(); + let mailbox = Arc::new(Mailbox::open(&codex_state::SqliteConfig::new_for_testing(home.path().abs())).await.unwrap()); + let queue = InputQueue::new(); + queue.bind_durable_mailbox(Arc::clone(&mailbox)); + let (a, b) = (ThreadId::new(), ThreadId::new()); + let generation = mailbox.wait_input_generation(); + let _old = mailbox.begin_wait_at("old-a".into(), a, vec![b], ReturnWhen::Any, false, generation).unwrap(); + queue.enqueue_mailbox_communication(make_mail(AgentPath::root(), AgentPath::root(), "native input", true)).await; + assert!(queue.has_pending_mailbox_items().await); + assert!(matches!(mailbox.begin_wait_at("stale-b".into(), b, vec![a], ReturnWhen::All, false, generation), Err(MailboxError::StaleWaitSnapshot))); + // Native input invalidated the older edge before publication, so a peer cannot + // incorrectly prove that both owners remain blocked on one another. + let _peer = mailbox.begin_wait("peer-b".into(), b, vec![a], ReturnWhen::All, false).unwrap(); + } + #[tokio::test] async fn terminal_completion_notifies_subscriber_and_drains_once() { let input_queue = InputQueue::new(); diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 7ea1b57b3ba3..6f2f09fc7b20 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1248,6 +1248,12 @@ impl Session { fork_persistence, next_internal_sub_id: AtomicU64::new(0), }); + if config.multi_agent_version_from_features() == MultiAgentVersion::V2 + && config.features.enabled(Feature::AgentMailbox) + && let Some(mailbox) = sess.services.agent_control.mailbox() + { + sess.input_queue.bind_durable_mailbox(mailbox); + } if let Some(network_policy_decider_session) = network_policy_decider_session { let mut guard = network_policy_decider_session.write().await; *guard = Arc::downgrade(&sess); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs index 6c5a33aada1a..c068b714bf12 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs @@ -237,17 +237,18 @@ impl ToolExecutor for ReadHandler { if envelope.reply_to.is_some() { observations.push(ObservationTarget::Reply(envelope.id.clone())); } - if envelope.communication.origin == Some(codex_protocol::protocol::AgentCommunicationOrigin::Result) - && let Some(work) = envelope.work_ref.as_ref() - && work.issuer == session.thread_id - { - observations.push(ObservationTarget::Work(work.clone())); + if envelope.communication.origin == Some(codex_protocol::protocol::AgentCommunicationOrigin::Result) { + // A native terminal result can cover several admitted inputs. Resolve + // the bounded exact body relation rather than infer from agent history. + observations.extend(mailbox.work_for_outcome(session.thread_id, &envelope.id) + .await.map_err(mailbox_error)?.into_iter().map(ObservationTarget::Work)); } // Reading a body can observe its own attention/reply only; it never observes // unrelated work outcomes or unresolved request obligations. if !observations.is_empty() { - mailbox.acknowledge_observed(session.thread_id, &observations) - .await.map_err(mailbox_error)?; + for batch in observations.chunks(100) { + mailbox.acknowledge_observed(session.thread_id, batch).await.map_err(mailbox_error)?; + } } } let quiet_count = mailbox.quiet_count(session.thread_id).await.map_err(mailbox_error)?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 6c8caa106233..228cb978fb8e 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -18,13 +18,11 @@ use codex_agent_mailbox::ReturnWhen; use codex_agent_mailbox::Selection; use codex_agent_mailbox::Selector; use codex_agent_mailbox::SnapshotItem; -use codex_protocol::ResponseItemId; use codex_protocol::ThreadId; use codex_protocol::items::MailboxWaitReport; use codex_protocol::items::MailboxWaitReturnWhen; use codex_protocol::items::MailboxWaitSelectionItem; use codex_protocol::items::MailboxWakeCause; -use codex_protocol::protocol::AgentCommunicationOrigin; use codex_tools::ToolSpec; use std::collections::HashSet; use std::time::Duration; @@ -32,6 +30,10 @@ use tokio::time::Instant; pub(crate) struct Handler; +#[cfg(test)] +#[path = "mailbox_wait_tests.rs"] +mod tests; + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct WaitArgs { @@ -134,6 +136,7 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, let mut registration = None; loop { drop(registration.take()); + let input_generation = mailbox.wait_input_generation(); if targetless { match mailbox.issued_work_refs(session.thread_id, None, 100).await { Ok(work) => selectors = work.into_iter().map(|work| Selector::Work(work.id)).collect(), @@ -276,14 +279,23 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, InputQueueActivity::TerminalCompletion => {} } } + if cancellation_token.is_cancelled() { + report.completion_reason = Some(MailboxWakeCause::Cancel); + break; + } + if deadline.is_some_and(|at| Instant::now() >= at) { + report.completion_reason = Some(MailboxWakeCause::Deadline); + break; + } let peers = selected.iter().filter_map(|item| match item { SnapshotItem::Work(work) if work.outcome.is_none() && !work.observed => Some(work.work_ref.agent), SnapshotItem::Request(request) if request.reply.is_none() && !request.observed => Some(request.request.recipient), SnapshotItem::Work(_) | SnapshotItem::Request(_) => None, }).collect::>().into_iter().collect(); let wait_id = format!("{}:{call_id}", session.thread_id); - match mailbox.begin_wait(wait_id, session.thread_id, peers, return_when, deadline.is_some()) { + match mailbox.begin_wait_at(wait_id, session.thread_id, peers, return_when, deadline.is_some(), input_generation) { Ok(guard) => registration = Some(guard), + Err(codex_agent_mailbox::MailboxError::StaleWaitSnapshot) => continue, Err(codex_agent_mailbox::MailboxError::ClosedPeerCycle) => { report.completion_reason = Some(MailboxWakeCause::Cycle); break; @@ -431,18 +443,11 @@ async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox if !delivered.contains(&outcome.id) && delivered.len() >= body_limit { continue; } - let communication = outcome.response.clone().unwrap_or_else(|| { - let mut communication = InterAgentCommunication::new( - work.input.recipient.clone(), work.input.author.clone(), Vec::new(), - serde_json::json!({"work_ref":work.work_ref.id,"outcome":outcome.outcome,"continuation":outcome.continuation}).to_string(), - /*trigger_turn*/ false, - ); - communication.id = Some(ResponseItemId::from_server(outcome.id.clone())); - communication.origin = Some(AgentCommunicationOrigin::Result); - communication - }); - session.record_inter_agent_communication(turn, communication).await.map_err(mailbox_error)?; - mailbox.mark_outcome_canonical(session.thread_id, &outcome.id).await.map_err(mailbox_error)?; + let envelope = mailbox.get_message(session.thread_id, &outcome.id).await + .map_err(mailbox_error)?.ok_or_else(|| mailbox_error("terminal result body requires compatible recovery"))?; + // Session owns append + successful flush + exact canonical acknowledgement + // under one custody gate, including synthetic typed terminal envelopes. + deliver_envelope(session, turn, &envelope).await?; delivered.insert(outcome.id.clone()); observations.push(ObservationTarget::Work(work.work_ref.clone())); } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs new file mode 100644 index 000000000000..a0b7086cb7b9 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -0,0 +1,92 @@ +use super::*; +use crate::session::step_context::StepContext; +use crate::tools::context::ToolCallSource; +use crate::turn_diff_tracker::TurnDiffTracker; +use codex_agent_mailbox::Continuation; +use codex_agent_mailbox::MessageDraft; +use codex_agent_mailbox::MessageKind; +use codex_agent_mailbox::NativeWork; +use codex_agent_mailbox::WorkRef; +use codex_features::Feature; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::ResponseInputItem; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; + +async fn fixture() -> (Arc, Arc, Arc, ThreadId) { + let (session, mut turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + turn.multi_agent_version = config.multi_agent_version_from_features(); + turn.config = Arc::new(config); + session.services.agent_control.mailbox_initialize(&turn.config).await.unwrap(); + let mailbox = session.services.agent_control.mailbox().unwrap(); + (Arc::new(session), Arc::new(turn), mailbox, ThreadId::new()) +} + +async fn work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId, key: &str, outcome: Option) -> WorkRef { + let work = mailbox.accept_work(NativeWork { + native_key: key.to_string(), issuer, agent, submission_id: key.to_string(), + input: InterAgentCommunication::new(AgentPath::root(), AgentPath::from_string("/root/worker".to_string()).unwrap(), Vec::new(), key.to_string(), true), + }).await.unwrap(); + if let Some(outcome) = outcome { + let result = mailbox.record_outcome(&work, outcome, None, Continuation::None).await.unwrap(); + // This fixture represents a previously surfaced result. Writer custody has + // separate Session tests; this exercises replay through the actual wait facade. + mailbox.mark_canonical(issuer, &result.id).await.unwrap(); + mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(work.clone())]).await.unwrap(); + } + work +} + +async fn wait(session: Arc, turn: Arc, targets: Vec, mode: &str) -> MailboxWaitReport { + let payload = ToolPayload::Function { arguments: serde_json::json!({ "targets": targets, "return_when": mode, "timeout_ms": 0 }).to_string() }; + let output = handle_wait(ToolInvocation { + session, step_context: StepContext::for_test(Arc::clone(&turn)), turn, + cancellation_token: CancellationToken::new(), + tracker: Arc::new(Mutex::new(TurnDiffTracker::default())), + call_id: format!("replay-{mode}"), tool_name: ToolName::plain("wait_agent"), + source: ToolCallSource::Direct, payload: payload.clone(), + }).await.unwrap(); + let ResponseInputItem::FunctionCallOutput { output, .. } = output.to_response_item("replay", &payload) else { panic!("expected a function result") }; + let FunctionCallOutputBody::Text(body) = output.body else { panic!("expected report text") }; + serde_json::from_str(&body).unwrap() +} + +#[tokio::test] +async fn observed_unsuccessful_work_remains_non_success_in_exact_and_mixed_replays() { + for outcome in [Outcome::Failed, Outcome::Cancelled, Outcome::NeedsAttention] { + let (session, turn, mailbox, agent) = fixture().await; + let failed = work(&mailbox, session.thread_id, agent, "unsuccessful", Some(outcome)).await; + let pending = work(&mailbox, session.thread_id, agent, "pending", None).await; + for mode in ["any", "all"] { + for targets in [vec![failed.id.clone()], vec![failed.id.clone(), pending.id.clone()]] { + let report = wait(Arc::clone(&session), Arc::clone(&turn), targets.clone(), mode).await; + assert_eq!(report.completion_reason, Some(MailboxWakeCause::NeedsAttention)); + assert!(!report.condition_satisfied); + assert_eq!(report.pending.len(), targets.len() - 1); + } + } + } +} + +#[tokio::test] +async fn unrelated_attention_does_not_erase_a_successful_selected_join() { + let (session, turn, mailbox, agent) = fixture().await; + let successful = work(&mailbox, session.thread_id, agent, "successful", Some(Outcome::Completed)).await; + let attention = mailbox.accept_message(MessageDraft { + invocation_key: "unrelated-request".to_string(), sender: agent, recipient: session.thread_id, + communication: InterAgentCommunication::new(AgentPath::from_string("/root/worker".to_string()).unwrap(), AgentPath::root(), Vec::new(), "independent request".to_string(), false), + kind: MessageKind::Request, activate_if_idle: false, work_ref: None, reply_to: None, supersedes: None, legacy_attention: false, + }).await.unwrap(); + for mode in ["any", "all"] { + let report = wait(Arc::clone(&session), Arc::clone(&turn), vec![successful.id.clone()], mode).await; + assert_eq!(report.completion_reason, Some(MailboxWakeCause::AlreadySatisfied)); + assert!(report.condition_satisfied); + let pending_attention = mailbox.attention(session.thread_id, 100).await.unwrap(); + assert_eq!(pending_attention.len(), 1); + assert_eq!(pending_attention[0].id, format!("attention:{}", attention.id)); + } +} diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 150c90f2c698..1b69bbfca397 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -1,5 +1,9 @@ use anyhow::Result; +use codex_agent_mailbox::Mailbox; use codex_features::Feature; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::ThreadId; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call_with_namespace; @@ -9,6 +13,7 @@ use core_test_support::responses::sse; use core_test_support::responses::sse_response; use core_test_support::responses::start_mock_server; use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; use serde_json::json; use serde_json::Value; use std::sync::Arc; @@ -59,6 +64,17 @@ fn function_output_text<'a>(request: &'a Value, call_id: &str) -> Option<&'a str .as_str() } +fn agent_message_input<'a>(request: &'a Value, message_id: &str) -> Option<&'a Value> { + request + .get("input")? + .as_array()? + .iter() + .find(|item| { + item.get("type").and_then(Value::as_str) == Some("agent_message") + && item.get("id").and_then(Value::as_str) == Some(message_id) + }) +} + fn has_function_output(request: &Value, call_id: &str) -> bool { function_output_text(request, call_id).is_some() } @@ -411,6 +427,11 @@ const UNRELATED_CHILD_TASK: &str = "complete without being selected by the paren const QUIET_NOTE_BODY: &str = "non-attention note while the parent waits"; const QUIET_PROGRESS_BODY: &str = "non-attention progress while the parent waits"; const ADDRESSED_REQUEST_BODY: &str = "please inspect this request"; +const CUSTODY_PROMPT: &str = "read and preserve an unadmitted mailbox request"; +const RECOVERY_PROMPT: &str = "recover the still-pending mailbox request after compaction"; +const COMPACTION_PROMPT: &str = "summarize the current mailbox conversation"; +const COMPACTION_SUMMARY: &str = "the mailbox request remains pending for recovery"; +const CUSTODY_BODY: &str = "canonical request body that must survive compaction"; #[derive(Clone)] struct WaitProbeResponder { @@ -737,3 +758,220 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - Ok(()) } + +#[derive(Clone)] +struct MailboxCustodyResponder { + root_thread_id: String, + requests: Arc>>, +} + +impl Respond for MailboxCustodyResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body) + .expect("provider request should contain JSON"); + self.requests + .lock() + .expect("request capture lock should not be poisoned") + .push(body.clone()); + + if body.to_string().contains(COMPACTION_PROMPT) { + return sse_response(sse(vec![ + ev_response_created("mailbox-custody-compaction"), + ev_assistant_message("mailbox-custody-summary", COMPACTION_SUMMARY), + ev_completed("mailbox-custody-compaction"), + ])); + } + + if has_function_output(&body, "recovery-read-call") { + return sse_response(sse(vec![ + ev_response_created("mailbox-custody-recovered"), + ev_assistant_message("mailbox-custody-recovered-message", "recovered"), + ev_completed("mailbox-custody-recovered"), + ])); + } + + if has_function_output(&body, "initial-read-call") { + // A successful HTTP stream with no response.created event models setup ending + // before provider acceptance. Admission must remain unsettled for this input. + return sse_response(sse(Vec::new())); + } + + if has_function_output(&body, "send-request-call") { + return function_call_response( + "mailbox-custody-initial-read-response", + "initial-read-call", + "mailbox_read", + json!({}), + ); + } + + if body.to_string().contains(RECOVERY_PROMPT) { + return function_call_response( + "mailbox-custody-recovery-read-response", + "recovery-read-call", + "mailbox_read", + json!({}), + ); + } + + function_call_response( + "mailbox-custody-send-response", + "send-request-call", + "send_message", + json!({ + "target": self.root_thread_id, + "kind": "request", + "message": CUSTODY_BODY, + }), + ) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_compaction() +-> Result<()> { + let server = start_mock_server().await; + let mut builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + config + .features + .enable(feature) + .expect("test config should allow feature update"); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + config.model_provider.supports_websockets = false; + config.model_provider.stream_max_retries = Some(0); + config.model_provider.request_max_retries = Some(0); + config.compact_prompt = Some(COMPACTION_PROMPT.to_string()); + }); + let test = builder.build_with_auto_env(&server).await?; + let root_thread_id: ThreadId = test.session_configured.thread_id.clone(); + let root_thread_id_text = root_thread_id.to_string(); + let captured = Arc::new(Mutex::new(Vec::new())); + + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(RootThreadMatcher { + thread_id: root_thread_id_text.clone(), + }) + .respond_with(MailboxCustodyResponder { + root_thread_id: root_thread_id_text, + requests: Arc::clone(&captured), + }) + .up_to_n_times(8) + .mount(&server) + .await; + + test.submit_turn(CUSTODY_PROMPT).await?; + + let requests = captured + .lock() + .expect("request capture lock should not be poisoned") + .clone(); + assert_eq!(requests.len(), 3, "the pre-created failure should not retry"); + let send_receipt: Value = serde_json::from_str( + function_output_text(&requests[1], "send-request-call") + .expect("accepted request receipt should remain in provider history"), + )?; + let message_id = send_receipt["message_id"] + .as_str() + .expect("send receipt should include a native message ID"); + assert!(message_id.starts_with("amsg_"), "unexpected native ID: {message_id}"); + + let failed_read_receipt: Value = serde_json::from_str( + function_output_text(&requests[2], "initial-read-call") + .expect("mailbox_read receipt should be in the failed provider request"), + )?; + assert_eq!(failed_read_receipt["messages"][0]["message_id"], message_id); + let failed_request_item = agent_message_input(&requests[2], message_id) + .expect("the failed-before-created request must contain the canonical body"); + assert_eq!(failed_request_item["content"][1]["type"], "encrypted_content"); + assert_eq!( + failed_request_item["content"][1]["encrypted_content"], + CUSTODY_BODY + ); + + let mailbox = Mailbox::open(test.config.sqlite_config()).await?; + let message_id_owned = message_id.to_string(); + assert_eq!( + mailbox + .unadmitted_input_ids(root_thread_id.clone(), std::slice::from_ref(&message_id_owned)) + .await?, + vec![message_id_owned.clone()], + "a stream that ended before response.created must not admit the body" + ); + let envelope = mailbox + .get_message(root_thread_id.clone(), message_id) + .await? + .expect("accepted canonical body should remain retrievable"); + assert!(envelope.canonical); + assert_eq!( + envelope.communication.encrypted_content.as_deref(), + Some(CUSTODY_BODY) + ); + assert!(mailbox + .pending(root_thread_id.clone(), 10) + .await? + .iter() + .any(|pending| pending.id.as_str() == message_id)); + + test.codex.submit(Op::Compact).await?; + let compact_terminal = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + assert!(matches!(compact_terminal, EventMsg::TurnComplete(_))); + + let requests = captured + .lock() + .expect("request capture lock should not be poisoned") + .clone(); + assert_eq!(requests.len(), 4, "manual compaction should use one provider call"); + assert!(requests[3].to_string().contains(COMPACTION_PROMPT)); + let compaction_item = agent_message_input(&requests[3], message_id) + .expect("compaction should receive the canonical message as source history"); + assert_eq!(compaction_item["content"][1]["encrypted_content"], CUSTODY_BODY); + assert_eq!( + mailbox + .unadmitted_input_ids(root_thread_id.clone(), std::slice::from_ref(&message_id_owned)) + .await?, + vec![message_id_owned.clone()], + "compaction is not execution admission for a pending mailbox obligation" + ); + assert!(mailbox + .pending(root_thread_id.clone(), 10) + .await? + .iter() + .any(|pending| pending.id.as_str() == message_id)); + let retained = mailbox + .get_message(root_thread_id.clone(), message_id) + .await? + .expect("compaction must not delete the canonical body from mailbox custody"); + assert_eq!( + retained.communication.encrypted_content.as_deref(), + Some(CUSTODY_BODY) + ); + + test.submit_turn(RECOVERY_PROMPT).await?; + + let requests = captured + .lock() + .expect("request capture lock should not be poisoned") + .clone(); + assert_eq!(requests.len(), 6, "recovery should read the request then execute once"); + let recovery_read_receipt: Value = serde_json::from_str( + function_output_text(&requests[5], "recovery-read-call") + .expect("recovery mailbox_read receipt should remain in provider history"), + )?; + assert_eq!(recovery_read_receipt["messages"][0]["message_id"], message_id); + let recovered_item = agent_message_input(&requests[5], message_id) + .expect("recovery read must restore the exact canonical body to execution input"); + assert_eq!(recovered_item["content"][1]["type"], "encrypted_content"); + assert_eq!(recovered_item["content"][1]["encrypted_content"], CUSTODY_BODY); + assert!(mailbox + .unadmitted_input_ids(root_thread_id, &[message_id_owned]) + .await? + .is_empty()); + + Ok(()) +} From 9bd8df000cda1d772289637f2ca33f218b14e8b6 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:30:59 +1100 Subject: [PATCH 06/54] Fix native adapter compilation and immutable progress replacement --- .../workflows/mailbox-source-preparation.yml | 2 +- codex-rs/agent-mailbox/src/activation.rs | 19 +- codex-rs/agent-mailbox/src/lib.rs | 25 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 1034 +++++++++++++---- .../agent-mailbox/src/message_delivery.rs | 59 +- codex-rs/agent-mailbox/src/message_rules.rs | 33 +- codex-rs/agent-mailbox/src/messages.rs | 53 +- codex-rs/agent-mailbox/src/outcome.rs | 56 +- .../agent-mailbox/src/outcome_messages.rs | 13 +- codex-rs/agent-mailbox/src/outcome_store.rs | 2 +- codex-rs/agent-mailbox/src/runtime_fault.rs | 10 +- codex-rs/agent-mailbox/src/schema.rs | 8 + codex-rs/agent-mailbox/src/snapshot.rs | 36 +- codex-rs/agent-mailbox/src/store.rs | 1 + codex-rs/agent-mailbox/src/types.rs | 2 +- codex-rs/agent-mailbox/src/wait_graph.rs | 49 +- codex-rs/agent-mailbox/src/work.rs | 12 +- codex-rs/core/src/agent/control/lifecycle.rs | 19 +- codex-rs/core/src/agent/control/mailbox.rs | 127 +- codex-rs/core/src/agent/control/spawn.rs | 4 +- codex-rs/core/src/hook_runtime.rs | 2 +- codex-rs/core/src/session/handlers.rs | 4 +- .../tools/handlers/multi_agents_spec_tests.rs | 127 +- .../handlers/multi_agents_v2/followup_task.rs | 12 +- .../multi_agents_v2/mailbox_message.rs | 199 +++- .../handlers/multi_agents_v2/mailbox_wait.rs | 11 +- .../multi_agents_v2/mailbox_wait_tests.rs | 161 ++- .../handlers/multi_agents_v2/message_tool.rs | 87 +- codex-rs/core/src/tools/parallel.rs | 2 +- codex-rs/core/src/tools/spec_plan.rs | 4 +- codex-rs/core/src/tools/spec_plan_tests.rs | 117 +- codex-rs/core/tests/suite/agent_mailbox.rs | 238 ++-- codex-rs/ext/goal/src/runtime.rs | 4 +- .../tui/src/chatwidget/tests/app_server.rs | 7 +- docs/carry-divergence-ledger.md | 6 + docs/downstream-regression-matrix.md | 7 + justfile | 6 +- 37 files changed, 1838 insertions(+), 720 deletions(-) diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index dd72c7e555bf..88773d326ac4 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "3" + PREPARATION_ATTEMPT: "4" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: diff --git a/codex-rs/agent-mailbox/src/activation.rs b/codex-rs/agent-mailbox/src/activation.rs index bf38d73cb509..2a1145230bbd 100644 --- a/codex-rs/agent-mailbox/src/activation.rs +++ b/codex-rs/agent-mailbox/src/activation.rs @@ -13,14 +13,13 @@ impl Mailbox { message_id: &str, ) -> MailboxResult { let mut transaction = self.pool.begin().await?; - let message = sqlx::query( - "SELECT id FROM mailbox_messages WHERE id = ? AND sender = ? LIMIT 1", - ) - .bind(message_id) - .bind(sender.to_string()) - .fetch_optional(&mut *transaction) - .await? - .ok_or(MailboxError::ReferenceMismatch)?; + let message = + sqlx::query("SELECT id FROM mailbox_messages WHERE id = ? AND sender = ? LIMIT 1") + .bind(message_id) + .bind(sender.to_string()) + .fetch_optional(&mut *transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; let exact_id: String = message.try_get("id")?; let claimed = sqlx::query( "UPDATE mailbox_activations SET state = 'activating' @@ -119,6 +118,8 @@ fn parse_activation_disposition(value: &str) -> MailboxResult Ok(ActivationDisposition::Activated), "denied" => Ok(ActivationDisposition::Denied), "unknown" => Ok(ActivationDisposition::Unknown), - _ => Err(MailboxError::InvalidInput("stored activation disposition".into())), + _ => Err(MailboxError::InvalidInput( + "stored activation disposition".into(), + )), } } diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index 95243d394ea4..f47f79db7c27 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -4,11 +4,11 @@ //! trusted native-tree identity and perform canonical history writes; this crate //! persists the accepted work, delivery intent, and exact observation state. -mod error; mod activation; -mod messages; +mod error; mod message_delivery; mod message_rules; +mod messages; mod outcome; mod outcome_messages; mod outcome_store; @@ -16,24 +16,23 @@ mod runtime_fault; mod schema; mod snapshot; mod store; +mod types; mod wait_graph; mod work; -mod types; pub use error::{MailboxError, MailboxResult}; -pub use store::Mailbox; pub use snapshot::is_provably_closed_peer_cycle; -pub use wait_graph::WaitRegistration; -pub use wait_graph::InputArrival; +pub use store::Mailbox; pub use types::{ - ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, - CanonicalConfirmation, Continuation, ContinuationReceipt, ContinuationState, Dependency, - DependencyMode, DispatchDecision, - DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, - NativeWork, ObservationTarget, Outcome, OutcomeReceipt, - PendingPage, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, - SnapshotRequest, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, + ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, CanonicalConfirmation, + Continuation, ContinuationReceipt, ContinuationState, Dependency, DependencyMode, + DispatchDecision, DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, NativeWork, + ObservationTarget, Outcome, OutcomeReceipt, PendingPage, RequestDisposition, RequestSnapshot, + ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotRequest, SnapshotState, + TurnOutcomeReceipt, WorkRef, WorkSnapshot, }; +pub use wait_graph::InputArrival; +pub use wait_graph::WaitRegistration; #[cfg(test)] #[path = "mailbox_tests.rs"] diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 31f1f3cb9a8a..1eb9074919fa 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1,13 +1,11 @@ use crate::{ - is_provably_closed_peer_cycle, ActivationDecision, ActivationDisposition, AdmissionReceipt, - Continuation, ContinuationState, Dependency, DependencyMode, DispatchDecision, DispatchState, - Envelope, GoalBinding, Mailbox, MailboxError, MessageDraft, MessageKind, NativeWork, - ObservationTarget, Outcome, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, - Selector, Snapshot, SnapshotItem, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, -}; -use codex_protocol::{ - protocol::InterAgentCommunication, AgentPath, ThreadId, + ActivationDecision, ActivationDisposition, AdmissionReceipt, Continuation, ContinuationState, + Dependency, DependencyMode, DispatchDecision, DispatchState, Envelope, GoalBinding, Mailbox, + MailboxError, MessageDraft, MessageKind, NativeWork, ObservationTarget, Outcome, + RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, + SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, is_provably_closed_peer_cycle, }; +use codex_protocol::{AgentPath, ThreadId, protocol::InterAgentCommunication}; use codex_state::SqliteConfig; use codex_utils_absolute_path::test_support::PathExt; use pretty_assertions::assert_eq; @@ -23,7 +21,11 @@ async fn test_mailbox() -> TestMailbox { let home = tempfile::tempdir().expect("temporary mailbox home"); let config = SqliteConfig::new_for_testing(home.path().abs()); let mailbox = Mailbox::open(&config).await.expect("open mailbox"); - TestMailbox { _home: home, config, mailbox } + TestMailbox { + _home: home, + config, + mailbox, + } } fn thread_id(index: u64) -> ThreadId { @@ -33,25 +35,43 @@ fn thread_id(index: u64) -> ThreadId { fn communication(content: &str) -> InterAgentCommunication { InterAgentCommunication::new( - AgentPath::root(), AgentPath::root(), Vec::new(), content.to_string(), false, + AgentPath::root(), + AgentPath::root(), + Vec::new(), + content.to_string(), + false, ) } fn native_work(issuer: ThreadId, agent: ThreadId, key: &str, content: &str) -> NativeWork { NativeWork { - native_key: key.to_string(), issuer, agent, - submission_id: format!("submission-{key}"), input: communication(content), + native_key: key.to_string(), + issuer, + agent, + submission_id: format!("submission-{key}"), + input: communication(content), } } fn draft( - key: &str, sender: ThreadId, recipient: ThreadId, kind: MessageKind, content: &str, + key: &str, + sender: ThreadId, + recipient: ThreadId, + kind: MessageKind, + content: &str, legacy_attention: bool, ) -> MessageDraft { MessageDraft { - invocation_key: key.to_string(), sender, recipient, - communication: communication(content), kind, activate_if_idle: false, - work_ref: None, reply_to: None, supersedes: None, legacy_attention, + invocation_key: key.to_string(), + sender, + recipient, + communication: communication(content), + kind, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention, } } @@ -61,7 +81,13 @@ fn selection( selectors: Vec, when: ReturnWhen, ) -> Selection { - Selection { caller, agent, selectors, return_when: when, limit: 100 } + Selection { + caller, + agent, + selectors, + return_when: when, + limit: 100, + } } async fn take_snapshot( @@ -71,19 +97,32 @@ async fn take_snapshot( selectors: Vec, when: ReturnWhen, ) -> Snapshot { - mailbox.snapshot(&selection(caller, agent, selectors, when)).await.unwrap() + mailbox + .snapshot(&selection(caller, agent, selectors, when)) + .await + .unwrap() } fn work_item<'a>(snapshot: &'a Snapshot, id: &str) -> &'a WorkSnapshot { - snapshot.items.iter().find_map(|item| match item { - SnapshotItem::Work(work) if work.work_ref.id == id => Some(work), _ => None, - }).expect("selected work is in snapshot") + snapshot + .items + .iter() + .find_map(|item| match item { + SnapshotItem::Work(work) if work.work_ref.id == id => Some(work), + _ => None, + }) + .expect("selected work is in snapshot") } fn request_item<'a>(snapshot: &'a Snapshot, id: &str) -> &'a RequestSnapshot { - snapshot.items.iter().find_map(|item| match item { - SnapshotItem::Request(request) if request.request_id == id => Some(request), _ => None, - }).expect("selected request is in snapshot") + snapshot + .items + .iter() + .find_map(|item| match item { + SnapshotItem::Request(request) if request.request_id == id => Some(request), + _ => None, + }) + .expect("selected request is in snapshot") } async fn accept_work( @@ -101,24 +140,38 @@ async fn accept_work( .accepted_input(&work) .await .expect("read accepted input") - .id.expect("accepted input has provider ID").to_string(); + .id + .expect("accepted input has provider ID") + .to_string(); (work, input_id) } async fn admit(mailbox: &Mailbox, agent: ThreadId, turn: &str, ids: &[String]) -> AdmissionReceipt { - mailbox.record_admission(agent, turn, ids).await.expect("record admission") + mailbox + .record_admission(agent, turn, ids) + .await + .expect("record admission") } async fn finish( - mailbox: &Mailbox, agent: ThreadId, turn: &str, outcome: Outcome, - response: Option, continuation: Continuation, + mailbox: &Mailbox, + agent: ThreadId, + turn: &str, + outcome: Outcome, + response: Option, + continuation: Continuation, ) -> TurnOutcomeReceipt { - mailbox.record_turn_outcome(agent, turn, outcome, response, continuation).await + mailbox + .record_turn_outcome(agent, turn, outcome, response, continuation) + .await .expect("record turn outcome") } async fn send(mailbox: &Mailbox, message: MessageDraft) -> Envelope { - mailbox.accept_message(message).await.expect("accept message") + mailbox + .accept_message(message) + .await + .expect("accept message") } fn goal_binding( @@ -128,8 +181,12 @@ fn goal_binding( source_turn: &str, ) -> GoalBinding { GoalBinding { - goal_id: "goal-1".to_string(), parent_thread_id: parent, child_thread_id: child, - control_generation: generation, revision: 1, source_turn_id: source_turn.to_string(), + goal_id: "goal-1".to_string(), + parent_thread_id: parent, + child_thread_id: child, + control_generation: generation, + revision: 1, + source_turn_id: source_turn.to_string(), } } @@ -150,7 +207,10 @@ async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { let captured = fixture.mailbox.accepted_input(&work).await.unwrap(); assert!(captured.id.as_ref().unwrap().as_str().starts_with("amsg_")); assert_eq!(fixture.mailbox.accept_work(original).await.unwrap(), work); - assert_eq!(fixture.mailbox.accepted_input(&work).await.unwrap(), captured); + assert_eq!( + fixture.mailbox.accepted_input(&work).await.unwrap(), + captured + ); let mut conflict = native_work(issuer, agent, "native-1", "changed input"); conflict.input.trigger_turn = true; @@ -158,17 +218,29 @@ async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { fixture.mailbox.accept_work(conflict).await, Err(MailboxError::IdempotencyConflict) )); - assert_eq!(fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + assert_eq!( + fixture.mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::Dispatch + ); assert_eq!( fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::RecoveryNeeded ); let reopened = Mailbox::open(&fixture.config).await.unwrap(); - assert_eq!(reopened.dispatch_state(&work).await.unwrap(), DispatchState::Dispatching); - assert_eq!(reopened.begin_dispatch(&work).await.unwrap(), DispatchDecision::RecoveryNeeded); + assert_eq!( + reopened.dispatch_state(&work).await.unwrap(), + DispatchState::Dispatching + ); + assert_eq!( + reopened.begin_dispatch(&work).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); reopened.mark_dispatched(&work).await.unwrap(); - assert_eq!(reopened.begin_dispatch(&work).await.unwrap(), DispatchDecision::AlreadyDispatched); + assert_eq!( + reopened.begin_dispatch(&work).await.unwrap(), + DispatchDecision::AlreadyDispatched + ); let mut too_large = native_work(issuer, agent, "too-large", "x"); too_large.input.content = "x".repeat(1_000_001); @@ -176,14 +248,30 @@ async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { fixture.mailbox.accept_work(too_large).await, Err(MailboxError::Backpressure(_)) )); - assert!(fixture.mailbox.work_for_native_key(issuer, "too-large").await.unwrap().is_none()); + assert!( + fixture + .mailbox + .work_for_native_key(issuer, "too-large") + .await + .unwrap() + .is_none() + ); } #[tokio::test] async fn native_accept_retry_after_pre_dispatch_handle_drop_reuses_original_identity() { - let TestMailbox { _home, config, mailbox } = test_mailbox().await; + let TestMailbox { + _home, + config, + mailbox, + } = test_mailbox().await; let (issuer, agent) = (thread_id(22), thread_id(23)); - let original = native_work(issuer, agent, "pre-dispatch-drop", "preserve this accepted input"); + let original = native_work( + issuer, + agent, + "pre-dispatch-drop", + "preserve this accepted input", + ); let accepted = mailbox.accept_work(original.clone()).await.unwrap(); let accepted_input_id = mailbox .accepted_input(&accepted) @@ -200,7 +288,10 @@ async fn native_accept_retry_after_pre_dispatch_handle_drop_reuses_original_iden let retried = retried_mailbox.accept_work(original).await.unwrap(); assert_eq!(retried, accepted); assert_eq!( - retried_mailbox.work_for_native_key(issuer, "pre-dispatch-drop").await.unwrap(), + retried_mailbox + .work_for_native_key(issuer, "pre-dispatch-drop") + .await + .unwrap(), Some(accepted.clone()) ); assert_eq!( @@ -237,37 +328,72 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "b", "B").await; let (c, _id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; assert_eq!( - admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]).await.work_refs, + admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]) + .await + .work_refs, vec![a.clone()] ); - assert!(admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]).await.already_recorded); + assert!( + admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]) + .await + .already_recorded + ); assert_eq!( - admit(&fixture.mailbox, agent, "turn", &[id_b.clone()]).await.work_refs, + admit(&fixture.mailbox, agent, "turn", &[id_b.clone()]) + .await + .work_refs, vec![b.clone()] ); - assert!(matches!(fixture.mailbox.record_admission(agent, "other-turn", &[id_a.clone()]).await, - Err(MailboxError::IdempotencyConflict))); + assert!(matches!( + fixture + .mailbox + .record_admission(agent, "other-turn", &[id_a.clone()]) + .await, + Err(MailboxError::IdempotencyConflict) + )); // New work after admission remains outside the outcome; unrelated issuer history is hidden. let (late, _) = accept_work(&fixture.mailbox, issuer, agent, "late", "late").await; - let (foreign, _) = accept_work( - &fixture.mailbox, other_issuer, agent, "foreign", "foreign", - ).await; - fixture.mailbox.record_outcome( - &foreign, Outcome::Completed, None, Continuation::None, - ).await.unwrap(); + let (foreign, _) = + accept_work(&fixture.mailbox, other_issuer, agent, "foreign", "foreign").await; + fixture + .mailbox + .record_outcome(&foreign, Outcome::Completed, None, Continuation::None) + .await + .unwrap(); let outcomes = finish( - &fixture.mailbox, agent, "turn", Outcome::Completed, None, Continuation::None, - ).await; + &fixture.mailbox, + agent, + "turn", + Outcome::Completed, + None, + Continuation::None, + ) + .await; assert_eq!(outcomes.outcomes.len(), 2); assert!(outcomes.outcomes.iter().any(|item| item.work_ref == a)); assert!(outcomes.outcomes.iter().any(|item| item.work_ref == b)); - assert!(outcomes.outcomes.iter().all(|item| item.work_ref != c && item.work_ref != late)); + assert!( + outcomes + .outcomes + .iter() + .all(|item| item.work_ref != c && item.work_ref != late) + ); - let snapshot = take_snapshot(&fixture.mailbox, issuer, Some(agent), vec![ - Selector::Work(a.id.clone()), Selector::Work(b.id.clone()), Selector::Work(c.id.clone()), - Selector::Work(late.id.clone()), Selector::Work(foreign.id.clone()), - ], ReturnWhen::All).await; + let snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![ + Selector::Work(a.id.clone()), + Selector::Work(b.id.clone()), + Selector::Work(c.id.clone()), + Selector::Work(late.id.clone()), + Selector::Work(foreign.id.clone()), + ], + ReturnWhen::All, + ) + .await; assert_eq!(snapshot.state, SnapshotState::Pending); assert_eq!(snapshot.items.len(), 4); assert!(work_item(&snapshot, &a.id).outcome.is_some()); @@ -278,9 +404,16 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped item, SnapshotItem::Work(work) if work.work_ref.id == foreign.id ))); - let too_many = (0..101).map(|i| format!("amsg-limit-{i}")).collect::>(); - assert!(matches!(fixture.mailbox.record_admission(agent, "over-limit", &too_many).await, - Err(MailboxError::Backpressure(_)))); + let too_many = (0..101) + .map(|i| format!("amsg-limit-{i}")) + .collect::>(); + assert!(matches!( + fixture + .mailbox + .record_admission(agent, "over-limit", &too_many) + .await, + Err(MailboxError::Backpressure(_)) + )); } #[tokio::test] @@ -290,16 +423,36 @@ async fn mixed_issuer_turn_creates_distinct_handbacks() { let (a, id_a) = accept_work(&fixture.mailbox, issuer_a, agent, "multi-a", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer_b, agent, "multi-b", "B").await; admit(&fixture.mailbox, agent, "shared-turn", &[id_a, id_b]).await; - let receipt = finish(&fixture.mailbox, agent, "shared-turn", Outcome::Completed, - Some(communication("one result")), Continuation::None).await; + let receipt = finish( + &fixture.mailbox, + agent, + "shared-turn", + Outcome::Completed, + Some(communication("one result")), + Continuation::None, + ) + .await; assert_eq!(receipt.outcomes.len(), 2); - let id_a = receipt.outcomes.iter().find(|r| r.work_ref == a) - .and_then(|r| r.response_message_id.clone()).unwrap(); - let id_b = receipt.outcomes.iter().find(|r| r.work_ref == b) - .and_then(|r| r.response_message_id.clone()).unwrap(); + let id_a = receipt + .outcomes + .iter() + .find(|r| r.work_ref == a) + .and_then(|r| r.response_message_id.clone()) + .unwrap(); + let id_b = receipt + .outcomes + .iter() + .find(|r| r.work_ref == b) + .and_then(|r| r.response_message_id.clone()) + .unwrap(); assert_ne!(id_a, id_b); for (issuer, id) in [(issuer_a, id_a), (issuer_b, id_b)] { - let message = fixture.mailbox.get_message(issuer, &id).await.unwrap().unwrap(); + let message = fixture + .mailbox + .get_message(issuer, &id) + .await + .unwrap() + .unwrap(); assert_eq!((message.sender, message.recipient), (agent, issuer)); } } @@ -316,51 +469,98 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() ]; for (work, id, turn, continuation, state) in [ ( - &a, id_a, "turn-p", Continuation::PendingGoalBinding(bindings[0].clone()), + &a, + id_a, + "turn-p", + Continuation::PendingGoalBinding(bindings[0].clone()), ContinuationState::PendingGoalBinding, ), ( - &b, id_b, "turn-d", Continuation::Deferred(bindings[1].clone()), + &b, + id_b, + "turn-d", + Continuation::Deferred(bindings[1].clone()), ContinuationState::Deferred, ), ] { admit(&fixture.mailbox, agent, turn, &[id]).await; let receipt = finish( - &fixture.mailbox, agent, turn, Outcome::Completed, None, continuation, - ).await; + &fixture.mailbox, + agent, + turn, + Outcome::Completed, + None, + continuation, + ) + .await; assert!(receipt.outcomes.is_empty()); assert_eq!(receipt.continuation_work, vec![(*work).clone()]); let snapshot = take_snapshot( - &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(work.id.clone())], + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(work.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; let item = work_item(&snapshot, &work.id); assert!(item.outcome.is_none()); assert_eq!(item.continuations[0].state, state); } let mut stale = bindings[0].clone(); stale.control_generation += 1; - assert!(finish(&fixture.mailbox, agent, "stale-generation", Outcome::Completed, None, - Continuation::Terminal(stale)).await.no_matching_work); + assert!( + finish( + &fixture.mailbox, + agent, + "stale-generation", + Outcome::Completed, + None, + Continuation::Terminal(stale) + ) + .await + .no_matching_work + ); let still_deferred = take_snapshot( - &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(a.id.clone())], + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(a.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; assert!(work_item(&still_deferred, &a.id).outcome.is_none()); for (work, binding, turn) in [ (&a, bindings[0].clone(), "terminal-p"), (&b, bindings[1].clone(), "terminal-d"), ] { - let result = finish(&fixture.mailbox, agent, turn, Outcome::Completed, None, - Continuation::Terminal(binding.clone())).await; + let result = finish( + &fixture.mailbox, + agent, + turn, + Outcome::Completed, + None, + Continuation::Terminal(binding.clone()), + ) + .await; assert_eq!(result.outcomes.len(), 1); let snapshot = take_snapshot( - &fixture.mailbox, issuer, Some(agent), vec![Selector::Work(work.id.clone())], + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(work.id.clone())], ReturnWhen::Any, - ).await; - assert_eq!(work_item(&snapshot, &work.id).outcome.as_ref().unwrap().continuation, - Continuation::Terminal(binding)); + ) + .await; + assert_eq!( + work_item(&snapshot, &work.id) + .outcome + .as_ref() + .unwrap() + .continuation, + Continuation::Terminal(binding) + ); } } @@ -370,40 +570,96 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let (author, recipient, stranger) = (thread_id(11), thread_id(12), thread_id(13)); let request = send( &fixture.mailbox, - draft("request", author, recipient, MessageKind::Request, "decide", false), - ).await; + draft( + "request", + author, + recipient, + MessageKind::Request, + "decide", + false, + ), + ) + .await; let (_, input_id) = accept_work( - &fixture.mailbox, author, recipient, "unrelated", "other task", - ).await; + &fixture.mailbox, + author, + recipient, + "unrelated", + "other task", + ) + .await; admit(&fixture.mailbox, recipient, "unrelated-turn", &[input_id]).await; finish( - &fixture.mailbox, recipient, "unrelated-turn", Outcome::Completed, None, + &fixture.mailbox, + recipient, + "unrelated-turn", + Outcome::Completed, + None, Continuation::None, - ).await; + ) + .await; let open = take_snapshot( - &fixture.mailbox, author, Some(recipient), vec![Selector::Reply(request.id.clone())], + &fixture.mailbox, + author, + Some(recipient), + vec![Selector::Reply(request.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; assert_eq!(open.state, SnapshotState::Pending); - assert_eq!(request_item(&open, &request.id).disposition, RequestDisposition::Open); - fixture.mailbox.mark_canonical(recipient, &request.id).await.unwrap(); + assert_eq!( + request_item(&open, &request.id).disposition, + RequestDisposition::Open + ); + fixture + .mailbox + .mark_canonical(recipient, &request.id) + .await + .unwrap(); let mut reply_draft = draft("reply", recipient, author, MessageKind::Note, "done", false); reply_draft.reply_to = Some(request.id.clone()); let reply = send(&fixture.mailbox, reply_draft).await; let replied = take_snapshot( - &fixture.mailbox, author, Some(recipient), vec![Selector::Reply(request.id.clone())], + &fixture.mailbox, + author, + Some(recipient), + vec![Selector::Reply(request.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; assert_eq!(replied.state, SnapshotState::AlreadySatisfied); - assert_eq!(request_item(&replied, &request.id).disposition, RequestDisposition::Replied); - assert_eq!(request_item(&replied, &request.id).reply.as_ref().unwrap().id, reply.id); + assert_eq!( + request_item(&replied, &request.id).disposition, + RequestDisposition::Replied + ); + assert_eq!( + request_item(&replied, &request.id) + .reply + .as_ref() + .unwrap() + .id, + reply.id + ); let open_request = send( &fixture.mailbox, - draft("replace-me", author, recipient, MessageKind::Request, "old", false), - ).await; + draft( + "replace-me", + author, + recipient, + MessageKind::Request, + "old", + false, + ), + ) + .await; let mut unauthorized = draft( - "stranger-replace", stranger, recipient, MessageKind::Request, "no", false, + "stranger-replace", + stranger, + recipient, + MessageKind::Request, + "no", + false, ); unauthorized.supersedes = Some(open_request.id.clone()); assert!(matches!( @@ -411,14 +667,23 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { Err(MailboxError::WrongRecipient) )); let mut replacement_draft = draft( - "replacement", author, recipient, MessageKind::Request, "new", false, + "replacement", + author, + recipient, + MessageKind::Request, + "new", + false, ); replacement_draft.supersedes = Some(open_request.id.clone()); let replacement = send(&fixture.mailbox, replacement_draft).await; let superseded = take_snapshot( - &fixture.mailbox, author, None, vec![Selector::Reply(open_request.id.clone())], + &fixture.mailbox, + author, + None, + vec![Selector::Reply(open_request.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; assert_eq!(superseded.state, SnapshotState::AlreadySatisfied); let old = request_item(&superseded, &open_request.id); assert_eq!(old.disposition, RequestDisposition::Superseded); @@ -426,17 +691,41 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let withdrawal = send( &fixture.mailbox, - draft("withdraw-me", author, recipient, MessageKind::Request, "cancel", false), - ).await; - assert!(matches!(fixture.mailbox.withdraw_message(stranger, &withdrawal.id).await, - Err(MailboxError::WrongRecipient))); - fixture.mailbox.withdraw_message(author, &withdrawal.id).await.unwrap(); + draft( + "withdraw-me", + author, + recipient, + MessageKind::Request, + "cancel", + false, + ), + ) + .await; + assert!(matches!( + fixture + .mailbox + .withdraw_message(stranger, &withdrawal.id) + .await, + Err(MailboxError::WrongRecipient) + )); + fixture + .mailbox + .withdraw_message(author, &withdrawal.id) + .await + .unwrap(); let withdrawn = take_snapshot( - &fixture.mailbox, author, None, vec![Selector::Reply(withdrawal.id.clone())], + &fixture.mailbox, + author, + None, + vec![Selector::Reply(withdrawal.id.clone())], ReturnWhen::Any, - ).await; + ) + .await; assert_eq!(withdrawn.state, SnapshotState::AlreadySatisfied); - assert_eq!(request_item(&withdrawn, &withdrawal.id).disposition, RequestDisposition::Withdrawn); + assert_eq!( + request_item(&withdrawn, &withdrawal.id).disposition, + RequestDisposition::Withdrawn + ); } #[tokio::test] @@ -453,20 +742,44 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom let update = send(&fixture.mailbox, update_draft).await; assert_ne!(update.id, first.id); assert_eq!(update.supersedes.as_deref(), Some(first.id.as_str())); - let original = fixture.mailbox.get_message(agent, &first.id).await.unwrap().unwrap(); + let original = fixture + .mailbox + .get_message(agent, &first.id) + .await + .unwrap() + .unwrap(); assert_eq!(original.communication.content, "first"); - let latest = fixture.mailbox.get_message(agent, &update.id).await.unwrap().unwrap(); + let latest = fixture + .mailbox + .get_message(agent, &update.id) + .await + .unwrap() + .unwrap(); assert_eq!(latest.communication.content, "latest"); assert_eq!(latest.supersedes.as_deref(), Some(first.id.as_str())); let pending = fixture.mailbox.pending(agent, 20).await.unwrap(); assert!(pending.iter().any(|envelope| envelope.id == update.id)); assert!(!pending.iter().any(|envelope| envelope.id == first.id)); - let mut other_author = draft("p3", author, agent, MessageKind::Progress, "other path", false); + let mut other_author = draft( + "p3", + author, + agent, + MessageKind::Progress, + "other path", + false, + ); other_author.work_ref = Some(work_a.clone()); other_author.communication.author = AgentPath::from_string("/root/child".to_string()).unwrap(); assert_ne!(send(&fixture.mailbox, other_author).await.id, first.id); - let mut other_path = draft("p-path", author, agent, MessageKind::Progress, "other path", false); + let mut other_path = draft( + "p-path", + author, + agent, + MessageKind::Progress, + "other path", + false, + ); other_path.work_ref = Some(work_a.clone()); other_path.communication.recipient = AgentPath::from_string("/root/other".to_string()).unwrap(); assert_ne!(send(&fixture.mailbox, other_path).await.id, first.id); @@ -474,10 +787,19 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom other_work.work_ref = Some(work_b); assert_ne!(send(&fixture.mailbox, other_work).await.id, first.id); - fixture.mailbox.record_outcome( - &work_a, Outcome::Completed, None, Continuation::None, - ).await.unwrap(); - let mut late = draft("p-late", author, agent, MessageKind::Progress, "late", false); + fixture + .mailbox + .record_outcome(&work_a, Outcome::Completed, None, Continuation::None) + .await + .unwrap(); + let mut late = draft( + "p-late", + author, + agent, + MessageKind::Progress, + "late", + false, + ); late.work_ref = Some(work_a); assert!(matches!( fixture.mailbox.accept_message(late).await, @@ -494,16 +816,29 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { for (work, id, turn) in [(&a, id_a, "turn-a"), (&b, id_b, "turn-b")] { admit(&fixture.mailbox, agent, turn, &[id]).await; let outcome = finish( - &fixture.mailbox, agent, turn, Outcome::Completed, None, Continuation::None, - ).await; - fixture.mailbox.mark_outcome_canonical(issuer, &outcome.outcomes[0].id).await.unwrap(); + &fixture.mailbox, + agent, + turn, + Outcome::Completed, + None, + Continuation::None, + ) + .await; + fixture + .mailbox + .mark_outcome_canonical(issuer, &outcome.outcomes[0].id) + .await + .unwrap(); assert_eq!(&outcome.outcomes[0].work_ref, work); } - fixture.mailbox.acknowledge_observed( - issuer, &[ObservationTarget::Work(b.clone())], - ).await.unwrap(); + fixture + .mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(b.clone())]) + .await + .unwrap(); let selection = selection( - issuer, Some(agent), + issuer, + Some(agent), vec![Selector::Work(a.id.clone()), Selector::Work(b.id.clone())], ReturnWhen::All, ); @@ -515,8 +850,11 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { work_item(&partial, &a.id).outcome.is_some() && work_item(&partial, &b.id).outcome.is_some() ); - fixture.mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(a)]) - .await.unwrap(); + fixture + .mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(a)]) + .await + .unwrap(); assert_eq!( fixture.mailbox.snapshot(&selection).await.unwrap().state, SnapshotState::AlreadySatisfied @@ -528,25 +866,54 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( let fixture = test_mailbox().await; let (sender, recipient) = (thread_id(18), thread_id(19)); let kinds = [ - MessageKind::Note, MessageKind::Progress, MessageKind::Request, MessageKind::Alert, + MessageKind::Note, + MessageKind::Progress, + MessageKind::Request, + MessageKind::Alert, ]; assert_eq!(MessageKind::default(), MessageKind::Note); for kind in kinds { send( &fixture.mailbox, - draft(&format!("quiet-{kind:?}"), sender, recipient, kind, "quiet", false), - ).await; + draft( + &format!("quiet-{kind:?}"), + sender, + recipient, + kind, + "quiet", + false, + ), + ) + .await; send( &fixture.mailbox, - draft(&format!("legacy-{kind:?}"), sender, recipient, kind, "legacy", true), - ).await; + draft( + &format!("legacy-{kind:?}"), + sender, + recipient, + kind, + "legacy", + true, + ), + ) + .await; } let pending = fixture.mailbox.pending(recipient, 20).await.unwrap(); assert_eq!(pending.len(), 8); - for kind in kinds { assert_eq!(pending.iter().filter(|m| m.kind == kind).count(), 2); } + for kind in kinds { + assert_eq!(pending.iter().filter(|m| m.kind == kind).count(), 2); + } assert_eq!(pending.iter().filter(|m| m.legacy_attention).count(), 1); assert_eq!(fixture.mailbox.quiet_count(recipient).await.unwrap(), 3); - assert_eq!(fixture.mailbox.attention(recipient, 20).await.unwrap().len(), 5); + assert_eq!( + fixture + .mailbox + .attention(recipient, 20) + .await + .unwrap() + .len(), + 5 + ); } #[tokio::test] @@ -555,39 +922,93 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re let (sender, recipient) = (thread_id(24), thread_id(25)); let request = send( &fixture.mailbox, - draft("attention-request", sender, recipient, MessageKind::Request, "please decide", false), + draft( + "attention-request", + sender, + recipient, + MessageKind::Request, + "please decide", + false, + ), ) .await; let alert = send( &fixture.mailbox, - draft("attention-alert", sender, recipient, MessageKind::Alert, "needs attention", false), + draft( + "attention-alert", + sender, + recipient, + MessageKind::Alert, + "needs attention", + false, + ), ) .await; let note = send( &fixture.mailbox, - draft("quiet-note", sender, recipient, MessageKind::Note, "quiet note", false), + draft( + "quiet-note", + sender, + recipient, + MessageKind::Note, + "quiet note", + false, + ), ) .await; let progress = send( &fixture.mailbox, - draft("quiet-progress", sender, recipient, MessageKind::Progress, "quiet progress", false), + draft( + "quiet-progress", + sender, + recipient, + MessageKind::Progress, + "quiet progress", + false, + ), ) .await; let legacy_draft = draft( - "legacy-attention-once", sender, recipient, MessageKind::Note, "legacy wake", true, + "legacy-attention-once", + sender, + recipient, + MessageKind::Note, + "legacy wake", + true, ); let legacy = send(&fixture.mailbox, legacy_draft.clone()).await; assert_eq!(send(&fixture.mailbox, legacy_draft).await.id, legacy.id); - let uncanonical = fixture.mailbox.pending_page(recipient, None, 20).await.unwrap(); - let uncanonical_ids = uncanonical.items.iter().map(|item| item.id.as_str()).collect::>(); + let uncanonical = fixture + .mailbox + .pending_page(recipient, None, 20) + .await + .unwrap(); + let uncanonical_ids = uncanonical + .items + .iter() + .map(|item| item.id.as_str()) + .collect::>(); for message in [&request, &alert, ¬e, &progress, &legacy] { assert!(uncanonical_ids.contains(&message.id.as_str())); - fixture.mailbox.mark_canonical(recipient, &message.id).await.unwrap(); + fixture + .mailbox + .mark_canonical(recipient, &message.id) + .await + .unwrap(); } - let canonical = fixture.mailbox.read_page(recipient, None, 20).await.unwrap(); - assert!(canonical.items.iter().any(|message| message.id == request.id)); + let canonical = fixture + .mailbox + .read_page(recipient, None, 20) + .await + .unwrap(); + assert!( + canonical + .items + .iter() + .any(|message| message.id == request.id) + ); assert!(canonical.items.iter().any(|message| message.id == alert.id)); let mut attention_ids = fixture @@ -617,7 +1038,14 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re .acknowledge_observed(recipient, &observations) .await .unwrap(); - assert!(fixture.mailbox.attention(recipient, 20).await.unwrap().is_empty()); + assert!( + fixture + .mailbox + .attention(recipient, 20) + .await + .unwrap() + .is_empty() + ); let old_request = take_snapshot( &fixture.mailbox, @@ -628,7 +1056,10 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re ) .await; assert_eq!(old_request.state, SnapshotState::Pending); - assert_eq!(request_item(&old_request, &request.id).disposition, RequestDisposition::Open); + assert_eq!( + request_item(&old_request, &request.id).disposition, + RequestDisposition::Open + ); let new_request = send( &fixture.mailbox, @@ -642,7 +1073,11 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re ), ) .await; - fixture.mailbox.mark_canonical(recipient, &new_request.id).await.unwrap(); + fixture + .mailbox + .mark_canonical(recipient, &new_request.id) + .await + .unwrap(); let new_attention = fixture.mailbox.attention(recipient, 20).await.unwrap(); let new_attention_ids = new_attention .iter() @@ -656,11 +1091,21 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re .mailbox .acknowledge_observed( recipient, - &[ObservationTarget::Attention(format!("attention:{}", new_request.id))], + &[ObservationTarget::Attention(format!( + "attention:{}", + new_request.id + ))], ) .await .unwrap(); - assert!(fixture.mailbox.attention(recipient, 20).await.unwrap().is_empty()); + assert!( + fixture + .mailbox + .attention(recipient, 20) + .await + .unwrap() + .is_empty() + ); } #[tokio::test] @@ -668,25 +1113,51 @@ async fn activation_claim_is_single_use_across_a_lost_result() { let fixture = test_mailbox().await; let (sender, recipient) = (thread_id(20), thread_id(21)); let mut message_draft = draft( - "activation", sender, recipient, MessageKind::Alert, "activate", false, + "activation", + sender, + recipient, + MessageKind::Alert, + "activate", + false, ); message_draft.activate_if_idle = true; let message = send(&fixture.mailbox, message_draft).await; assert_eq!( - fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), - ActivationDecision::Activate); + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::Activate + ); assert_eq!( - fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), - ActivationDecision::RecoveryNeeded); + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::RecoveryNeeded + ); for _ in 0..2 { - fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Activated) - .await.unwrap(); + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Activated) + .await + .unwrap(); } assert_eq!( - fixture.mailbox.begin_activation(sender, &message.id).await.unwrap(), - ActivationDecision::AlreadyDecided(ActivationDisposition::Activated)); + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::AlreadyDecided(ActivationDisposition::Activated) + ); assert!(matches!( - fixture.mailbox.record_activation(sender, &message.id, ActivationDisposition::Denied).await, + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Denied) + .await, Err(MailboxError::IdempotencyConflict) )); } @@ -724,27 +1195,71 @@ fn only_closed_any_and_all_peer_cycles_are_provable() { async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { let fixture = test_mailbox().await; let (owner_a, owner_b) = (thread_id(22), thread_id(23)); - let wait_a = fixture.mailbox.begin_wait( - "wait-a".to_string(), owner_a, vec![owner_b], ReturnWhen::Any, false, - ).unwrap(); + let wait_a = fixture + .mailbox + .begin_wait( + "wait-a".to_string(), + owner_a, + vec![owner_b], + ReturnWhen::Any, + false, + ) + .unwrap(); assert_eq!(wait_a.wait_id(), "wait-a"); - assert!(matches!(fixture.mailbox.begin_wait( - "closed-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, - ), Err(MailboxError::ClosedPeerCycle))); - assert!(matches!(fixture.mailbox.begin_wait( - "still-closed-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, - ), Err(MailboxError::ClosedPeerCycle))); - let escape = fixture.mailbox.begin_wait( - "escape-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, true, - ).unwrap(); + assert!(matches!( + fixture.mailbox.begin_wait( + "closed-b".to_string(), + owner_b, + vec![owner_a], + ReturnWhen::All, + false, + ), + Err(MailboxError::ClosedPeerCycle) + )); + assert!(matches!( + fixture.mailbox.begin_wait( + "still-closed-b".to_string(), + owner_b, + vec![owner_a], + ReturnWhen::All, + false, + ), + Err(MailboxError::ClosedPeerCycle) + )); + let escape = fixture + .mailbox + .begin_wait( + "escape-b".to_string(), + owner_b, + vec![owner_a], + ReturnWhen::All, + true, + ) + .unwrap(); drop(escape); - assert!(matches!(fixture.mailbox.begin_wait( - "closed-again-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, - ), Err(MailboxError::ClosedPeerCycle))); + assert!(matches!( + fixture.mailbox.begin_wait( + "closed-again-b".to_string(), + owner_b, + vec![owner_a], + ReturnWhen::All, + false, + ), + Err(MailboxError::ClosedPeerCycle) + )); drop(wait_a); - assert!(fixture.mailbox.begin_wait( - "open-b".to_string(), owner_b, vec![owner_a], ReturnWhen::All, false, - ).is_ok()); + assert!( + fixture + .mailbox + .begin_wait( + "open-b".to_string(), + owner_b, + vec![owner_a], + ReturnWhen::All, + false, + ) + .is_ok() + ); } #[tokio::test] @@ -752,35 +1267,82 @@ async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until let fixture = test_mailbox().await; let (a, b) = (thread_id(28), thread_id(29)); let before = fixture.mailbox.wait_input_generation(); - let old = fixture.mailbox.begin_wait_at( - "a".into(), a, vec![b], ReturnWhen::Any, false, before, - ).unwrap(); + let old = fixture + .mailbox + .begin_wait_at("a".into(), a, vec![b], ReturnWhen::Any, false, before) + .unwrap(); let arrival = fixture.mailbox.begin_input_arrival(); - assert!(matches!(fixture.mailbox.begin_wait_at( - "stale".into(), b, vec![a], ReturnWhen::All, false, before, - ), Err(MailboxError::StaleWaitSnapshot))); + assert!(matches!( + fixture + .mailbox + .begin_wait_at("stale".into(), b, vec![a], ReturnWhen::All, false, before,), + Err(MailboxError::StaleWaitSnapshot) + )); let during = fixture.mailbox.wait_input_generation(); - let provisional_a = fixture.mailbox.begin_wait_at( - "a".into(), a, vec![b], ReturnWhen::Any, false, during, - ).unwrap(); + let provisional_a = fixture + .mailbox + .begin_wait_at("a".into(), a, vec![b], ReturnWhen::Any, false, during) + .unwrap(); // A stale RAII handle must not remove a replacement registration. drop(old); - let provisional_b = fixture.mailbox.begin_wait_at( - "b".into(), b, vec![a], ReturnWhen::All, false, during, - ).unwrap(); + let provisional_b = fixture + .mailbox + .begin_wait_at("b".into(), b, vec![a], ReturnWhen::All, false, during) + .unwrap(); drop(arrival); - assert!(matches!(fixture.mailbox.begin_wait_at( - "needs-rescan".into(), b, vec![a], ReturnWhen::All, false, during, - ), Err(MailboxError::StaleWaitSnapshot))); + assert!(matches!( + fixture.mailbox.begin_wait_at( + "needs-rescan".into(), + b, + vec![a], + ReturnWhen::All, + false, + during, + ), + Err(MailboxError::StaleWaitSnapshot) + )); drop(provisional_a); drop(provisional_b); let current = fixture.mailbox.wait_input_generation(); - let _waiting_a = fixture.mailbox.begin_wait_at( - "fresh-a".into(), a, vec![b], ReturnWhen::Any, false, current, - ).unwrap(); - assert!(matches!(fixture.mailbox.begin_wait_at( - "fresh-b".into(), b, vec![a], ReturnWhen::All, false, current, - ), Err(MailboxError::ClosedPeerCycle))); + let _waiting_a = fixture + .mailbox + .begin_wait_at( + "fresh-a".into(), + a, + vec![b], + ReturnWhen::Any, + false, + current, + ) + .unwrap(); + assert!(matches!( + fixture.mailbox.begin_wait_at( + "fresh-b".into(), + b, + vec![a], + ReturnWhen::All, + false, + current, + ), + Err(MailboxError::ClosedPeerCycle) + )); +} + +#[tokio::test] +async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recovery() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(32), thread_id(33)); + let (work, input) = accept_work(&fixture.mailbox, issuer, agent, "recovery", "original").await; + assert_eq!(fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + fixture.mailbox.mark_dispatched(&work).await.unwrap(); + fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); + assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::RecoveryNeeded); + admit(&fixture.mailbox, agent, "explicit-recovery-turn", &[input]).await; + assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched); + let terminal = finish(&fixture.mailbox, agent, "explicit-recovery-turn", Outcome::Completed, None, Continuation::None).await; + assert_eq!(terminal.outcomes[0].work_ref, work); + fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); + assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched); } #[tokio::test] @@ -790,17 +1352,69 @@ async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attentio let (covered, input) = accept_work(&fixture.mailbox, issuer, agent, "covered", "A").await; let (unrelated, _) = accept_work(&fixture.mailbox, issuer, agent, "unrelated", "B").await; admit(&fixture.mailbox, agent, "covered-turn", &[input]).await; - let terminal = finish(&fixture.mailbox, agent, "covered-turn", Outcome::NeedsAttention, None, Continuation::None).await; + let terminal = finish( + &fixture.mailbox, + agent, + "covered-turn", + Outcome::NeedsAttention, + None, + Continuation::None, + ) + .await; let outcome = &terminal.outcomes[0]; - let envelope = fixture.mailbox.get_message(issuer, &outcome.id).await.unwrap().unwrap(); - assert_eq!(envelope.communication.id.as_ref().unwrap().as_str(), outcome.id); - assert!(!fixture.mailbox.attention(issuer, 100).await.unwrap().is_empty()); - fixture.mailbox.mark_canonical(issuer, &outcome.id).await.unwrap(); - let exact = fixture.mailbox.work_for_outcome(issuer, &outcome.id).await.unwrap(); + let envelope = fixture + .mailbox + .get_message(issuer, &outcome.id) + .await + .unwrap() + .unwrap(); + assert_eq!( + envelope.communication.id.as_ref().unwrap().as_str(), + outcome.id + ); + assert!( + !fixture + .mailbox + .attention(issuer, 100) + .await + .unwrap() + .is_empty() + ); + fixture + .mailbox + .mark_canonical(issuer, &outcome.id) + .await + .unwrap(); + let exact = fixture + .mailbox + .work_for_outcome(issuer, &outcome.id) + .await + .unwrap(); assert_eq!(exact, vec![covered.clone()]); - fixture.mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(covered.clone())]).await.unwrap(); - assert!(fixture.mailbox.attention(issuer, 100).await.unwrap().is_empty()); - let snapshot = take_snapshot(&fixture.mailbox, issuer, Some(agent), vec![Selector::Work(covered.id.clone()), Selector::Work(unrelated.id.clone())], ReturnWhen::All).await; + fixture + .mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(covered.clone())]) + .await + .unwrap(); + assert!( + fixture + .mailbox + .attention(issuer, 100) + .await + .unwrap() + .is_empty() + ); + let snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![ + Selector::Work(covered.id.clone()), + Selector::Work(unrelated.id.clone()), + ], + ReturnWhen::All, + ) + .await; assert!(work_item(&snapshot, &covered.id).observed); assert!(!work_item(&snapshot, &unrelated.id).observed); } diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index 2c0aa90b8e9d..89fe4dd0b08c 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -3,8 +3,8 @@ use crate::messages::decrement_pending_count; use crate::store::Mailbox; use crate::types::{CanonicalConfirmation, Envelope, PendingPage}; use codex_protocol::ThreadId; -use std::future::Future; use sqlx::Row; +use std::future::Future; const MAX_PAGE_SIZE: usize = 100; @@ -66,13 +66,12 @@ impl Mailbox { recipient: ThreadId, id: &str, ) -> MailboxResult> { - let row = sqlx::query( - "SELECT * FROM mailbox_messages WHERE id = ? AND recipient = ? LIMIT 1", - ) - .bind(id) - .bind(recipient.to_string()) - .fetch_optional(&self.pool) - .await?; + let row = + sqlx::query("SELECT * FROM mailbox_messages WHERE id = ? AND recipient = ? LIMIT 1") + .bind(id) + .bind(recipient.to_string()) + .fetch_optional(&self.pool) + .await?; row.as_ref().map(envelope_from_row).transpose() } @@ -97,7 +96,9 @@ impl Mailbox { return Err(MailboxError::WrongRecipient); } if row.try_get::("kind")? != "\"request\"" { - return Err(MailboxError::InvalidInput("only requests may be withdrawn".into())); + return Err(MailboxError::InvalidInput( + "only requests may be withdrawn".into(), + )); } if row.try_get::("withdrawn")? != 0 { transaction.commit().await?; @@ -107,12 +108,11 @@ impl Mailbox { if superseded_by.is_some() { return Err(MailboxError::ReplacementNotAllowed); } - let reply_exists: Option = sqlx::query_scalar( - "SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1", - ) - .bind(id) - .fetch_optional(&mut *transaction) - .await?; + let reply_exists: Option = + sqlx::query_scalar("SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1") + .bind(id) + .fetch_optional(&mut *transaction) + .await?; if reply_exists.is_some() { return Err(MailboxError::RequestAlreadyResolved); } @@ -128,12 +128,11 @@ impl Mailbox { .execute(&mut *transaction) .await?; if updated.rows_affected() == 1 && (!canonical || !input_admitted) { - let recipient: String = sqlx::query_scalar( - "SELECT recipient FROM mailbox_messages WHERE id = ?", - ) - .bind(id) - .fetch_one(&mut *transaction) - .await?; + let recipient: String = + sqlx::query_scalar("SELECT recipient FROM mailbox_messages WHERE id = ?") + .bind(id) + .fetch_one(&mut *transaction) + .await?; decrement_pending_count(&mut transaction, &recipient, quiet).await?; } transaction.commit().await?; @@ -176,7 +175,9 @@ impl Mailbox { .fetch_all(&self.pool) .await?; if rows.len() > limit { - return Err(MailboxError::Backpressure("unadmitted canonical body limit")); + return Err(MailboxError::Backpressure( + "unadmitted canonical body limit", + )); } rows.iter() .map(|row| row.try_get("id").map_err(Into::into)) @@ -330,7 +331,9 @@ impl Mailbox { let was_canonical = current.try_get::("canonical")? != 0; let input_admitted = current.try_get::("input_admitted")? != 0; let quiet = current.try_get::("quiet")? != 0; - let is_current = current.try_get::, _>("superseded_by")?.is_none() + let is_current = current + .try_get::, _>("superseded_by")? + .is_none() && current.try_get::("withdrawn")? == 0; let result = sqlx::query( "UPDATE mailbox_messages SET canonical = 1 @@ -372,11 +375,7 @@ impl Mailbox { /// Mark a no-response or typed outcome after the adapter has appended its /// exact `OutcomeReceipt.id` and flushed canonical history. - pub async fn mark_outcome_canonical( - &self, - recipient: ThreadId, - id: &str, - ) -> MailboxResult<()> { + pub async fn mark_outcome_canonical(&self, recipient: ThreadId, id: &str) -> MailboxResult<()> { let mut transaction = self.pool.begin().await?; let result = sqlx::query( "UPDATE mailbox_work SET outcome_canonical = 1 @@ -464,7 +463,9 @@ pub(crate) fn envelope_from_row(row: &sqlx::sqlite::SqliteRow) -> MailboxResult< communication: serde_json::from_str(&row.try_get::("communication_json")?)?, kind: serde_json::from_str(&row.try_get::("kind")?)?, activate_if_idle: row.try_get::("activate_if_idle")? != 0, - work_ref: work_ref.map(|value| serde_json::from_str(&value)).transpose()?, + work_ref: work_ref + .map(|value| serde_json::from_str(&value)) + .transpose()?, reply_to: row.try_get("reply_to")?, supersedes: row.try_get("supersedes")?, legacy_attention: row.try_get::("legacy_attention")? != 0, diff --git a/codex-rs/agent-mailbox/src/message_rules.rs b/codex-rs/agent-mailbox/src/message_rules.rs index e276ec7a5d9b..d5caacf45562 100644 --- a/codex-rs/agent-mailbox/src/message_rules.rs +++ b/codex-rs/agent-mailbox/src/message_rules.rs @@ -16,7 +16,9 @@ pub(crate) async fn validate_reply_tx( .await? .ok_or(MailboxError::ReferenceMismatch)?; if request.try_get::("kind")? != "\"request\"" { - return Err(MailboxError::InvalidInput("reply_to must name a request".into())); + return Err(MailboxError::InvalidInput( + "reply_to must name a request".into(), + )); } if request.try_get::("recipient")? != draft.sender.to_string() || request.try_get::("sender")? != draft.recipient.to_string() @@ -26,17 +28,18 @@ pub(crate) async fn validate_reply_tx( if request.try_get::("canonical")? == 0 { return Err(MailboxError::NotCanonical); } - if request.try_get::, _>("superseded_by")?.is_some() + if request + .try_get::, _>("superseded_by")? + .is_some() || request.try_get::("withdrawn")? != 0 { return Err(MailboxError::RequestAlreadyResolved); } - let resolved: Option = sqlx::query_scalar( - "SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1", - ) - .bind(request_id) - .fetch_optional(&mut **transaction) - .await?; + let resolved: Option = + sqlx::query_scalar("SELECT id FROM mailbox_messages WHERE reply_to = ? LIMIT 1") + .bind(request_id) + .fetch_optional(&mut **transaction) + .await?; if resolved.is_some() { return Err(MailboxError::RequestAlreadyResolved); } @@ -57,7 +60,11 @@ pub(crate) async fn validate_replacement_tx( .await? .ok_or(MailboxError::ReferenceMismatch)?; let old_work: Option = old.try_get("work_ref_json")?; - let new_work = draft.work_ref.as_ref().map(serde_json::to_string).transpose()?; + let new_work = draft + .work_ref + .as_ref() + .map(serde_json::to_string) + .transpose()?; if old.try_get::("sender")? != draft.sender.to_string() || old.try_get::("recipient")? != draft.recipient.to_string() || old_work != new_work @@ -68,16 +75,12 @@ pub(crate) async fn validate_replacement_tx( let old_kind: String = old.try_get("kind")?; let valid_pair = match old_kind.as_str() { "\"progress\"" => { - draft.kind == MessageKind::Progress - && old.try_get::("canonical")? == 0 + draft.kind == MessageKind::Progress && old.try_get::("canonical")? == 0 } "\"request\"" => matches!(draft.kind, MessageKind::Request | MessageKind::Note), _ => false, }; - if !valid_pair - || already_replaced.is_some() - || old.try_get::("withdrawn")? != 0 - { + if !valid_pair || already_replaced.is_some() || old.try_get::("withdrawn")? != 0 { return Err(MailboxError::ReplacementNotAllowed); } if old_kind == "\"request\"" { diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index fe86411af328..77a6e11a5cd1 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -142,7 +142,7 @@ pub(crate) async fn insert_message_tx( "SELECT * FROM mailbox_messages WHERE sender = ? AND recipient = ? AND work_ref_json = ? AND author_path = ? AND recipient_path = ? - AND kind = 'progress' AND canonical = 0 AND input_admitted = 0 + AND kind = '\"progress\"' AND canonical = 0 AND input_admitted = 0 AND superseded_by IS NULL AND legacy_attention = ? AND NOT EXISTS ( @@ -160,26 +160,9 @@ pub(crate) async fn insert_message_tx( .fetch_optional(&mut **transaction) .await? { - let existing_id: String = existing.try_get("id")?; - draft.communication.id = Some(ResponseItemId::from_server(existing_id.clone())); - let coalesced_communication = serde_json::to_string(&draft.communication)?; - update_coalesced_tx( - transaction, - &existing_id, - &draft, - &request_json, - &coalesced_communication, - ) - .await?; - add_invocation_tx(transaction, &draft.invocation_key, &request_json, &existing_id).await?; - if has_attention(&draft) { - insert_attention_tx(transaction, &existing_id, draft.recipient.to_string().as_str()).await?; - } - let row = sqlx::query("SELECT * FROM mailbox_messages WHERE id = ?") - .bind(existing_id) - .fetch_one(&mut **transaction) - .await?; - return envelope_from_row(&row); + // Replacement gets its own stable ID. Never rewrite the body or + // idempotency binding behind the previously accepted identifier. + draft.supersedes = Some(existing.try_get("id")?); } } } @@ -293,34 +276,6 @@ pub(crate) async fn insert_message_tx( envelope_from_row(&row) } -async fn update_coalesced_tx( - transaction: &mut Transaction<'_, Sqlite>, - id: &str, - draft: &MessageDraft, - request_json: &str, - communication_json: &str, -) -> MailboxResult<()> { - sqlx::query( - "UPDATE mailbox_messages SET invocation_key = ?, request_json = ?, - communication_json = ?, author_path = ?, recipient_path = ?, kind = ?, - reply_to = ?, supersedes = ?, legacy_attention = ? - WHERE id = ? AND canonical = 0", - ) - .bind(&draft.invocation_key) - .bind(request_json) - .bind(communication_json) - .bind(serde_json::to_string(&draft.communication.author)?) - .bind(serde_json::to_string(&draft.communication.recipient)?) - .bind(serde_json::to_string(&draft.kind)?) - .bind(&draft.reply_to) - .bind(&draft.supersedes) - .bind(i64::from(draft.legacy_attention)) - .bind(id) - .execute(&mut **transaction) - .await?; - Ok(()) -} - async fn add_invocation_tx( transaction: &mut Transaction<'_, Sqlite>, invocation_key: &str, diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index 721f459481c0..d42be36287e0 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -1,21 +1,18 @@ use crate::error::{MailboxError, MailboxResult}; use crate::outcome_messages::{ - ensure_terminal_responses, insert_response_group, response_groups, - validate_auto_response_size, + ensure_terminal_responses, insert_response_group, response_groups, validate_auto_response_size, }; use crate::outcome_store::{ - outcome_receipt_for, persist_continuation, save_outcome_tx, OutcomeData, - MAX_CONTINUATIONS_PER_WORK, + MAX_CONTINUATIONS_PER_WORK, OutcomeData, outcome_receipt_for, persist_continuation, + save_outcome_tx, }; use crate::store::Mailbox; use crate::types::{ Continuation, GoalBinding, Outcome, OutcomeReceipt, TurnOutcomeReceipt, WorkRef, }; -use crate::work::{ - ensure_mailbox_id, exact_work_tx, new_mailbox_id, work_ref_from_row, -}; -use codex_protocol::protocol::InterAgentCommunication; +use crate::work::{ensure_mailbox_id, exact_work_tx, new_mailbox_id, work_ref_from_row}; use codex_protocol::ThreadId; +use codex_protocol::protocol::InterAgentCommunication; use sqlx::Row; const MAX_TURN_INPUTS: usize = 100; @@ -54,7 +51,10 @@ impl Mailbox { .map(work_ref_from_row) .collect::>>()?; - if matches!(&continuation, Continuation::None | Continuation::Terminal(_)) { + if matches!( + &continuation, + Continuation::None | Continuation::Terminal(_) + ) { let binding_json = match &continuation { Continuation::Terminal(binding) => Some(serde_json::to_string(binding)?), _ => None, @@ -74,7 +74,9 @@ impl Mailbox { return Err(MailboxError::Backpressure("work continuation count")); } if unresolved.is_empty() - || binding_json.as_ref().is_some_and(|binding| unresolved.contains(binding)) + || binding_json + .as_ref() + .is_some_and(|binding| unresolved.contains(binding)) { eligible.push(work_ref); } @@ -149,7 +151,10 @@ impl Mailbox { if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { return Err(MailboxError::Backpressure("work continuation count")); } - if unresolved.iter().any(|old_binding| old_binding != &binding_json) { + if unresolved + .iter() + .any(|old_binding| old_binding != &binding_json) + { return Err(MailboxError::ReferenceMismatch); } } @@ -157,7 +162,10 @@ impl Mailbox { let _response_id = insert_response_group( &mut transaction, group, - format!("continuation:{}:{}:{}", recipient, source_turn_id, group.issuer), + format!( + "continuation:{}:{}:{}", + recipient, source_turn_id, group.issuer + ), outcome, ) .await?; @@ -198,13 +206,7 @@ impl Mailbox { }); } - ensure_terminal_responses( - &mut groups, - &refs, - outcome, - &continuation, - source_turn_id, - )?; + ensure_terminal_responses(&mut groups, &refs, outcome, &continuation, source_turn_id)?; validate_auto_response_size(&groups)?; let binding_json = match &continuation { @@ -254,7 +256,10 @@ impl Mailbox { group.output_id = insert_response_group( &mut transaction, group, - format!("turn-outcome:{}:{}:{}", recipient, source_turn_id, group.issuer), + format!( + "turn-outcome:{}:{}:{}", + recipient, source_turn_id, group.issuer + ), outcome, ) .await?; @@ -322,9 +327,7 @@ impl Mailbox { .iter() .find(|group| group.issuer == work_ref.issuer) .is_some_and(|group| group.response.is_some()); - receipts.push( - outcome_receipt_for(&self.pool, work_ref, group_id, has_response).await?, - ); + receipts.push(outcome_receipt_for(&self.pool, work_ref, group_id, has_response).await?); } Ok(TurnOutcomeReceipt { outcomes: receipts, @@ -353,7 +356,8 @@ impl Mailbox { validate_binding_for_work(binding, work_ref)?; } let mut transaction = self.pool.begin().await?; - let mut groups = response_groups(&mut transaction, &[work_ref.clone()], response.as_ref()).await?; + let mut groups = + response_groups(&mut transaction, &[work_ref.clone()], response.as_ref()).await?; let group = groups.pop().ok_or(MailboxError::ReferenceMismatch)?; let mut groups = vec![group]; ensure_terminal_responses( @@ -464,7 +468,9 @@ fn validate_binding_for_work(binding: &GoalBinding, work_ref: &WorkRef) -> Mailb fn validate_key(key: &str) -> MailboxResult<()> { if key.is_empty() || key.len() > MAX_KEY_BYTES { - Err(MailboxError::InvalidInput("invalid native turn or binding key".into())) + Err(MailboxError::InvalidInput( + "invalid native turn or binding key".into(), + )) } else { Ok(()) } diff --git a/codex-rs/agent-mailbox/src/outcome_messages.rs b/codex-rs/agent-mailbox/src/outcome_messages.rs index 8ff34febf6a0..4ad6e04fd851 100644 --- a/codex-rs/agent-mailbox/src/outcome_messages.rs +++ b/codex-rs/agent-mailbox/src/outcome_messages.rs @@ -65,8 +65,8 @@ pub(crate) fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxRe let mut total = 0_usize; for group in groups { if let Some(response) = group.response.as_ref() { - let bytes = response.content.len() - + response.encrypted_content.as_ref().map_or(0, String::len); + let bytes = + response.content.len() + response.encrypted_content.as_ref().map_or(0, String::len); if bytes > MAX_RESPONSE_BYTES { return Err(MailboxError::Backpressure("response message byte limit")); } @@ -74,7 +74,9 @@ pub(crate) fn validate_auto_response_size(groups: &[ResponseGroup]) -> MailboxRe } } if total > MAX_AUTO_RESPONSE_BYTES { - return Err(MailboxError::Backpressure("automatic response aggregate byte limit")); + return Err(MailboxError::Backpressure( + "automatic response aggregate byte limit", + )); } Ok(()) } @@ -113,7 +115,9 @@ pub(crate) fn ensure_terminal_responses( source_ref, })?; if content.len() > MAX_RESPONSE_BYTES { - return Err(MailboxError::Backpressure("typed outcome envelope byte limit")); + return Err(MailboxError::Backpressure( + "typed outcome envelope byte limit", + )); } let mut communication = InterAgentCommunication::new( group.author.clone(), @@ -160,4 +164,3 @@ pub(crate) async fn insert_response_group( group.output_id = Some(envelope.id.clone()); Ok(group.output_id.clone()) } - diff --git a/codex-rs/agent-mailbox/src/outcome_store.rs b/codex-rs/agent-mailbox/src/outcome_store.rs index 4e396234270a..7530a70005c7 100644 --- a/codex-rs/agent-mailbox/src/outcome_store.rs +++ b/codex-rs/agent-mailbox/src/outcome_store.rs @@ -109,7 +109,7 @@ pub(crate) async fn save_outcome_tx( outcome: Outcome, ) -> MailboxResult<()> { let result = sqlx::query( - "UPDATE mailbox_work SET outcome_json = ?, outcome_id = ? + "UPDATE mailbox_work SET outcome_json = ?, outcome_id = ?, dispatch_state = 'dispatched' WHERE id = ? AND issuer = ? AND agent = ? AND submission_id = ? AND outcome_json IS NULL", ) diff --git a/codex-rs/agent-mailbox/src/runtime_fault.rs b/codex-rs/agent-mailbox/src/runtime_fault.rs index 82fa71bde4a0..7d2256d408de 100644 --- a/codex-rs/agent-mailbox/src/runtime_fault.rs +++ b/codex-rs/agent-mailbox/src/runtime_fault.rs @@ -43,7 +43,10 @@ impl Mailbox { return true; } let faults = lock(&self.runtime_faults); - faults.overflowed || agents.iter().any(|agent| faults.agents.contains(&agent.to_string())) + faults.overflowed + || agents + .iter() + .any(|agent| faults.agents.contains(&agent.to_string())) } /// Clear one exact agent's fault only after explicit adapter recovery. @@ -57,6 +60,7 @@ impl Mailbox { } fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { - mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) } - diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 90320ad01c41..3762b52525c9 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -297,6 +297,14 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS mailbox_messages_active_unread + ON mailbox_messages(recipient, seq) + WHERE (canonical = 0 OR input_admitted = 0) + AND superseded_by IS NULL AND withdrawn = 0", + ) + .execute(&mut *transaction) + .await?; sqlx::query( "CREATE INDEX IF NOT EXISTS mailbox_messages_unadmitted ON mailbox_messages(recipient, canonical, input_admitted, seq)", diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs index 35e7ceba4dff..a819af81c0b6 100644 --- a/codex-rs/agent-mailbox/src/snapshot.rs +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -1,12 +1,11 @@ use crate::error::{MailboxError, MailboxResult}; use crate::message_delivery::envelope_from_row; +use crate::outcome_store::{continuation_receipts_for, stored_outcome_receipt_for}; use crate::store::Mailbox; use crate::types::{ - Attention, Dependency, DependencyMode, ObservationTarget, ReturnWhen, Selection, - RequestDisposition, RequestSnapshot, Selector, Snapshot, SnapshotItem, SnapshotState, - WorkSnapshot, + Attention, Dependency, DependencyMode, ObservationTarget, RequestDisposition, RequestSnapshot, + ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotState, WorkSnapshot, }; -use crate::outcome_store::{continuation_receipts_for, stored_outcome_receipt_for}; use crate::work::{parse_dispatch_state, work_ref_from_row}; use codex_protocol::ThreadId; use sqlx::Row; @@ -211,7 +210,10 @@ impl Mailbox { ) .bind(recipient.to_string()) .bind(format!("attention:{}", work_ref.id)) - .bind(format!("attention:{}", row.try_get::("outcome_id")?)) + .bind(format!( + "attention:{}", + row.try_get::("outcome_id")? + )) .execute(&mut *transaction) .await?; } @@ -233,7 +235,8 @@ impl Mailbox { .await?; } ObservationTarget::Attention(id) => { - acknowledge_attention(&mut transaction, id, recipient.to_string().as_str()).await?; + acknowledge_attention(&mut transaction, id, recipient.to_string().as_str()) + .await?; } } } @@ -273,8 +276,10 @@ impl Mailbox { .map(|row| { Ok(Attention { id: row.try_get("id")?, - recipient: ThreadId::from_string(row.try_get::("recipient")?.as_str()) - .map_err(|_| MailboxError::InvalidInput("stored attention thread ID".into()))?, + recipient: ThreadId::from_string( + row.try_get::("recipient")?.as_str(), + ) + .map_err(|_| MailboxError::InvalidInput("stored attention thread ID".into()))?, canonical: row.try_get::("canonical")? != 0, observed: row.try_get::("observed")? != 0, }) @@ -306,14 +311,13 @@ async fn acknowledge_attention( .execute(&mut **transaction) .await?; if result.rows_affected() == 0 { - let row = sqlx::query( - "SELECT canonical FROM mailbox_attention WHERE id = ? AND recipient = ?", - ) - .bind(id) - .bind(recipient) - .fetch_optional(&mut **transaction) - .await? - .ok_or(MailboxError::ReferenceMismatch)?; + let row = + sqlx::query("SELECT canonical FROM mailbox_attention WHERE id = ? AND recipient = ?") + .bind(id) + .bind(recipient) + .fetch_optional(&mut **transaction) + .await? + .ok_or(MailboxError::ReferenceMismatch)?; if row.try_get::("canonical")? == 0 { return Err(MailboxError::NotCanonical); } diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index 7f8d0b8564df..665cb7613c88 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -47,6 +47,7 @@ impl Mailbox { }); } + #[cfg(test)] pub(crate) fn pool(&self) -> &SqlitePool { &self.pool } diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 27ce8a5640be..26e27e802633 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -1,5 +1,5 @@ -use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::ThreadId; +use codex_protocol::protocol::InterAgentCommunication; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs index 7d5d9246aa50..f07ebe93c621 100644 --- a/codex-rs/agent-mailbox/src/wait_graph.rs +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -3,8 +3,8 @@ use crate::store::Mailbox; use crate::types::ReturnWhen; use codex_protocol::ThreadId; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex, Weak}; use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex, Weak}; const MAX_ACTIVE_WAITS: usize = 4_096; const MAX_PEERS_PER_WAIT: usize = 100; @@ -37,8 +37,12 @@ pub struct InputArrival { impl Drop for InputArrival { fn drop(&mut self) { let waits = lock(&self.mailbox.waits); - self.mailbox.wait_input_generation.fetch_add(1, Ordering::AcqRel); - self.mailbox.pending_input_arrivals.fetch_sub(1, Ordering::AcqRel); + self.mailbox + .wait_input_generation + .fetch_add(1, Ordering::AcqRel); + self.mailbox + .pending_input_arrivals + .fetch_sub(1, Ordering::AcqRel); drop(waits); self.mailbox.notify_changed(); } @@ -73,7 +77,9 @@ impl Mailbox { self.wait_input_generation.fetch_add(1, Ordering::AcqRel); self.pending_input_arrivals.fetch_add(1, Ordering::AcqRel); drop(waits); - InputArrival { mailbox: self.clone() } + InputArrival { + mailbox: self.clone(), + } } /// Capture before the authoritative selection/input rescan. @@ -113,7 +119,14 @@ impl Mailbox { has_escape: bool, input_generation: u64, ) -> MailboxResult { - self.begin_wait_inner(wait_id, owner, peers, mode, has_escape, Some(input_generation)) + self.begin_wait_inner( + wait_id, + owner, + peers, + mode, + has_escape, + Some(input_generation), + ) } fn begin_wait_inner( @@ -126,7 +139,9 @@ impl Mailbox { input_generation: Option, ) -> MailboxResult { if wait_id.is_empty() || wait_id.len() > MAX_WAIT_ID_BYTES { - return Err(MailboxError::InvalidInput("invalid wait registration ID".into())); + return Err(MailboxError::InvalidInput( + "invalid wait registration ID".into(), + )); } if peers.len() > MAX_PEERS_PER_WAIT { return Err(MailboxError::Backpressure("wait dependency count")); @@ -136,10 +151,14 @@ impl Mailbox { .map(|peer| peer.to_string()) .collect::>(); if peers.iter().collect::>().len() != peers.len() { - return Err(MailboxError::InvalidInput("duplicate wait dependency".into())); + return Err(MailboxError::InvalidInput( + "duplicate wait dependency".into(), + )); } let mut waits = lock(&self.waits); - if input_generation.is_some_and(|generation| generation != self.wait_input_generation.load(Ordering::Acquire)) { + if input_generation.is_some_and(|generation| { + generation != self.wait_input_generation.load(Ordering::Acquire) + }) { return Err(MailboxError::StaleWaitSnapshot); } let has_escape = has_escape || self.pending_input_arrivals.load(Ordering::Acquire) != 0; @@ -196,10 +215,16 @@ fn is_closed_owner(waits: &HashMap, owner: &str) -> bool { owner_waits.iter().all(|wait| match wait.mode { // Any completes when one alternative is ready, so every peer // must itself be blocked for this waiter to remain trapped. - ReturnWhen::Any => wait.peers.iter().all(|peer| previous.contains(peer.as_str())), + ReturnWhen::Any => wait + .peers + .iter() + .all(|peer| previous.contains(peer.as_str())), // All remains blocked while at least one required peer is // blocked; other completed peers do not break the dependency. - ReturnWhen::All => wait.peers.iter().any(|peer| previous.contains(peer.as_str())), + ReturnWhen::All => wait + .peers + .iter() + .any(|peer| previous.contains(peer.as_str())), }) }); if closed.len() == previous_len { @@ -210,5 +235,7 @@ fn is_closed_owner(waits: &HashMap, owner: &str) -> bool { } fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { - mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) } diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index 80fe746355c3..8ee124665db9 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -249,10 +249,10 @@ impl Mailbox { let row = exact_work_tx(&mut transaction, work_ref).await?; let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; match state { - DispatchState::Dispatching => { + DispatchState::Dispatching | DispatchState::Dispatched => { sqlx::query( "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' - WHERE id = ? AND dispatch_state = 'dispatching' + WHERE id = ? AND dispatch_state IN ('dispatching', 'dispatched') AND outcome_json IS NULL AND NOT EXISTS ( SELECT 1 FROM mailbox_continuations AS continuation @@ -265,7 +265,7 @@ impl Mailbox { .await?; } DispatchState::RecoveryNeeded => {} - DispatchState::Accepted | DispatchState::Dispatched => { + DispatchState::Accepted => { return Err(MailboxError::ReferenceMismatch); } } @@ -376,6 +376,12 @@ impl Mailbox { .bind(input_id) .execute(&mut *transaction) .await?; + // The provider established this exact native input. This is stronger + // evidence than a stale dispatch-recovery marker, without replaying it. + sqlx::query("UPDATE mailbox_work SET dispatch_state = 'dispatched' WHERE id = ?") + .bind(&work_ref.id) + .execute(&mut *transaction) + .await?; work_refs.push(work_ref); added = true; continue; diff --git a/codex-rs/core/src/agent/control/lifecycle.rs b/codex-rs/core/src/agent/control/lifecycle.rs index 0d97e2891859..a1477facd82a 100644 --- a/codex-rs/core/src/agent/control/lifecycle.rs +++ b/codex-rs/core/src/agent/control/lifecycle.rs @@ -238,9 +238,10 @@ impl PreparedV2AgentDelivery { &mut submission_id, ) .await?; - let (work, dispatch) = accepted.map_or((None, DispatchDecision::Dispatch), |(work, decision)| { - (Some(work), decision) - }); + let (work, dispatch) = accepted + .map_or((None, DispatchDecision::Dispatch), |(work, decision)| { + (Some(work), decision) + }); if dispatch != DispatchDecision::Dispatch { return Ok(RecordedSubmission { id: submission_id, @@ -326,7 +327,11 @@ impl PreparedV2AgentDelivery { DispatchDecision::RecoveryNeeded => { return Err(CodexErr::UnsupportedOperation(format!( "accepted work {} requires native recovery; its original input is retained and was not resent", - submission.work.as_ref().map(|work| work.id.as_str()).unwrap_or("unknown") + submission + .work + .as_ref() + .map(|work| work.id.as_str()) + .unwrap_or("unknown") ))); } DispatchDecision::Dispatch => {} @@ -348,7 +353,11 @@ impl PreparedV2AgentDelivery { DispatchDecision::RecoveryNeeded => { return Err(CodexErr::UnsupportedOperation(format!( "accepted work {} requires native recovery; its original input is retained and was not resent", - submission.work.as_ref().map(|work| work.id.as_str()).unwrap_or("unknown") + submission + .work + .as_ref() + .map(|work| work.id.as_str()) + .unwrap_or("unknown") ))); } DispatchDecision::Dispatch => {} diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index e238bbe7b4ca..53822b511fdc 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -10,11 +10,11 @@ use crate::agent::goal_notifications::GoalNotificationSnapshot; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::config::Config; -use codex_agent_mailbox::Continuation; use codex_agent_mailbox::ActivationDisposition; +use codex_agent_mailbox::Continuation; use codex_agent_mailbox::DispatchDecision; -use codex_agent_mailbox::GoalBinding; use codex_agent_mailbox::Envelope; +use codex_agent_mailbox::GoalBinding; use codex_agent_mailbox::Mailbox; use codex_agent_mailbox::NativeWork; use codex_agent_mailbox::Outcome; @@ -24,12 +24,12 @@ use codex_protocol::ResponseItemId; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; +use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::ThreadGoalStatus; -use codex_protocol::models::ResponseItem; -use std::sync::Arc; use std::collections::HashSet; +use std::sync::Arc; fn storage_error(error: impl std::fmt::Display) -> CodexErr { // The store does not log payloads. Keep its typed diagnostic in the error chain presented @@ -94,8 +94,10 @@ impl AgentControl { return Ok(None); } if let Some(native_key) = context.native_key.as_deref() - && let Some(existing) = mailbox.work_for_native_key(context.sender_thread_id, native_key) - .await.map_err(storage_error)? + && let Some(existing) = mailbox + .work_for_native_key(context.sender_thread_id, native_key) + .await + .map_err(storage_error)? { *submission_id = existing.submission_id; } @@ -132,7 +134,11 @@ impl AgentControl { ) -> CodexResult<(ActivationDisposition, Option)> { let denied = |reason: &str| (ActivationDisposition::Denied, Some(reason.to_string())); let metadata = self.ensure_agent_known(recipient)?; - if metadata.agent_path.as_ref().is_none_or(|path| path.is_root()) { + if metadata + .agent_path + .as_ref() + .is_none_or(|path| path.is_root()) + { return Ok(denied("root_activation_is_not_allowed")); } let state = self.upgrade()?; @@ -151,7 +157,10 @@ impl AgentControl { if thread.session.active_turn.lock().await.is_some() { return Ok((ActivationDisposition::AlreadyActive, None)); } - if thread.session.services.thread_extension_data + if thread + .session + .services + .thread_extension_data .get::() .and_then(|store| store.snapshot()) .is_some_and(|goal| goal.status != ThreadGoalStatus::Active) @@ -179,8 +188,16 @@ impl AgentControl { return Ok(()); }; for id in message_ids { - if mailbox.get_message(recipient, id.as_str()).await.map_err(storage_error)?.is_some() { - mailbox.mark_canonical(recipient, id.as_str()).await.map_err(storage_error)?; + if mailbox + .get_message(recipient, id.as_str()) + .await + .map_err(storage_error)? + .is_some() + { + mailbox + .mark_canonical(recipient, id.as_str()) + .await + .map_err(storage_error)?; } } Ok(()) @@ -196,10 +213,21 @@ impl AgentControl { }; let mut cursor = None; loop { - let page = mailbox.pending_page(recipient, cursor, 100).await.map_err(storage_error)?; + let page = mailbox + .pending_page(recipient, cursor, 100) + .await + .map_err(storage_error)?; for envelope in &page.items { - if envelope.communication.id.as_ref().is_some_and(|id| canonical_ids.contains(id)) { - mailbox.mark_canonical(recipient, &envelope.id).await.map_err(storage_error)?; + if envelope + .communication + .id + .as_ref() + .is_some_and(|id| canonical_ids.contains(id)) + { + mailbox + .mark_canonical(recipient, &envelope.id) + .await + .map_err(storage_error)?; } } match page.next_cursor { @@ -218,8 +246,12 @@ impl AgentControl { let Some(mailbox) = self.mailbox() else { return Ok(Vec::new()); }; - let unread = mailbox.unadmitted_canonical_ids(recipient, 4096) - .await.map_err(storage_error)?.into_iter().collect::>(); + let unread = mailbox + .unadmitted_canonical_ids(recipient, 4096) + .await + .map_err(storage_error)? + .into_iter() + .collect::>(); Ok(current_active_history.iter().filter(|item| { matches!(item, ResponseItem::AgentMessage { id: Some(id), .. } if unread.contains(id.as_str())) }).cloned().collect()) @@ -233,9 +265,13 @@ impl AgentControl { let Some(mailbox) = self.mailbox() else { return Ok(Vec::new()); }; - Ok(mailbox.automatic_inputs(recipient, source_turn_id, 2) - .await.map_err(storage_error)?.into_iter() - .map(|envelope| envelope.communication).collect()) + Ok(mailbox + .automatic_inputs(recipient, source_turn_id, 2) + .await + .map_err(storage_error)? + .into_iter() + .map(|envelope| envelope.communication) + .collect()) } pub(crate) async fn mailbox_reconcile_pending_work(&self, work: &WorkRef) -> CodexResult { @@ -246,7 +282,8 @@ impl AgentControl { let can_proceed = match state.get_thread(work.agent).await { Ok(thread) => { let active = { thread.session.active_turn.lock().await.is_some() }; - active || thread.session.input_queue.has_pending_mailbox_items().await + active + || thread.session.input_queue.has_pending_mailbox_items().await || thread.session.input_queue.has_pending_terminal_finalizers() || thread.session.has_outstanding_durable_sleep() } @@ -257,7 +294,10 @@ impl AgentControl { } // This only changes recovery disposition, never replays input or invents a result. // The store rechecks terminal/continuation custody atomically against this native read. - mailbox.mark_recovery_needed(work).await.map_err(storage_error)?; + mailbox + .mark_recovery_needed(work) + .await + .map_err(storage_error)?; Ok(true) } @@ -270,13 +310,20 @@ impl AgentControl { let Some(mailbox) = self.mailbox() else { return Ok(()); }; - let ids = message_ids.iter().map(ToString::to_string).collect::>(); + let ids = message_ids + .iter() + .map(ToString::to_string) + .collect::>(); for candidates in ids.chunks(100) { - let admitted = mailbox.unadmitted_input_ids(recipient, candidates) - .await.map_err(storage_error)?; + let admitted = mailbox + .unadmitted_input_ids(recipient, candidates) + .await + .map_err(storage_error)?; if !admitted.is_empty() { - mailbox.record_admission(recipient, source_turn_id, &admitted) - .await.map_err(storage_error)?; + mailbox + .record_admission(recipient, source_turn_id, &admitted) + .await + .map_err(storage_error)?; } } Ok(()) @@ -326,23 +373,27 @@ impl AgentControl { }; if matches!(outcome, Outcome::Failed | Outcome::Cancelled) { Continuation::Terminal(binding) - } else { match snapshot.phase { - GoalNotificationPhase::Running | GoalNotificationPhase::ContinuationPending => { - Continuation::PendingGoalBinding(binding) - } - GoalNotificationPhase::DeferredWithOwner => Continuation::Deferred(binding), - GoalNotificationPhase::ActionRequired => { - outcome = Outcome::NeedsAttention; - Continuation::Deferred(binding) + } else { + match snapshot.phase { + GoalNotificationPhase::Running + | GoalNotificationPhase::ContinuationPending => { + Continuation::PendingGoalBinding(binding) + } + GoalNotificationPhase::DeferredWithOwner => Continuation::Deferred(binding), + GoalNotificationPhase::ActionRequired => { + outcome = Outcome::NeedsAttention; + Continuation::Deferred(binding) + } + GoalNotificationPhase::ResultReady => Continuation::Terminal(binding), } - GoalNotificationPhase::ResultReady => Continuation::Terminal(binding), - } } + } } None => Continuation::None, }; - if let Err(error) = mailbox.record_turn_outcome( - recipient, source_turn_id, outcome, completion, continuation, - ).await { + if let Err(error) = mailbox + .record_turn_outcome(recipient, source_turn_id, outcome, completion, continuation) + .await + { mailbox.notify_runtime_fault(recipient); return Err(storage_error(error)); } diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index cbfa5658401c..d8079c8c5273 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -420,7 +420,9 @@ impl AgentControl { input: communication, }) .await - .map_err(|err| CodexErr::Fatal(format!("agent mailbox acceptance failed: {err}")))?; + .map_err(|err| { + CodexErr::Fatal(format!("agent mailbox acceptance failed: {err}")) + })?; let metadata = self.get_agent_metadata(work.agent).ok_or_else(|| { CodexErr::UnsupportedOperation(format!( "accepted work {} retains original child {}; native recovery is required and no replacement was spawned", diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 42782e230e0b..dc7941574766 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -21,6 +21,7 @@ use codex_hooks::UserPromptSubmitOutcome; use codex_hooks::UserPromptSubmitRequest; use codex_otel::HOOK_RUN_DURATION_METRIC; use codex_otel::HOOK_RUN_METRIC; +use codex_protocol::error::Result as CodexResult; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::models::ResponseItem; @@ -35,7 +36,6 @@ use codex_protocol::protocol::HookSource; use codex_protocol::protocol::HookStartedEvent; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; -use codex_protocol::error::Result as CodexResult; use codex_thread_store::ReadThreadParams; use serde_json::Value; use tracing::instrument; diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index a35f2b76b3df..75dafa215aff 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -577,7 +577,7 @@ pub async fn thread_rollback(sess: &Arc, sub_id: String, num_turns: u32 sess.deliver_event_raw(Event { id: turn_context.sub_id.clone(), msg: rollback_msg, - }) + }, None) .await; } @@ -688,7 +688,7 @@ pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { sess.services .rollout_thread_trace .record_protocol_event(&event.msg); - sess.deliver_event_raw(event).await; + sess.deliver_event_raw(event, None).await; sess.services .rollout_thread_trace .record_ended(codex_rollout_trace::RolloutStatus::Completed); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index 18f26afa34f2..66e5983d58aa 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -362,20 +362,18 @@ fn spawn_agent_tool_hides_model_controls_without_override_exposure() { let output_schema = output_schema.expect("spawn_agent output schema"); assert_eq!( output_schema["required"], - json!([ - "task_name", - "effective_model", - "effective_reasoning_effort" - ]) + json!(["task_name", "effective_model", "effective_reasoning_effort"]) ); let output_properties = output_schema["properties"] .as_object() .expect("spawn result properties"); assert_eq!(output_properties["work_ref"]["type"], json!("string")); - assert!(output_properties["work_ref"]["description"] - .as_str() - .expect("work_ref description") - .contains("Immutable accepted work reference")); + assert!( + output_properties["work_ref"]["description"] + .as_str() + .expect("work_ref description") + .contains("Immutable accepted work reference") + ); assert!(!output_properties.contains_key("agent_id")); assert!(!output_properties.contains_key("nickname")); } @@ -426,17 +424,27 @@ fn mailbox_send_message_tool_exposes_encrypted_mailbox_contract() { json!("alert") ]) ); - assert!(properties["kind"].description.as_deref().unwrap().contains("Defaults to `note`")); - assert!(properties["reply_to"] - .description - .as_deref() - .unwrap() - .contains("Progress messages cannot set `reply_to`")); - assert!(properties["wake_wait"] - .description - .as_deref() - .unwrap() - .contains("Ignored when `kind` is supplied")); + assert!( + properties["kind"] + .description + .as_deref() + .unwrap() + .contains("Defaults to `note`") + ); + assert!( + properties["reply_to"] + .description + .as_deref() + .unwrap() + .contains("Progress messages cannot set `reply_to`") + ); + assert!( + properties["wake_wait"] + .description + .as_deref() + .unwrap() + .contains("Ignored when `kind` is supplied") + ); assert!(!properties.contains_key("items")); assert_eq!( properties @@ -508,11 +516,13 @@ fn mailbox_read_tool_exposes_bounded_cursor_pagination() { properties["limit"].schema_type, Some(JsonSchemaType::Single(JsonSchemaPrimitiveType::Integer)) ); - assert!(properties["limit"] - .description - .as_deref() - .unwrap() - .contains("capped at 100")); + assert!( + properties["limit"] + .description + .as_deref() + .unwrap() + .contains("capped at 100") + ); assert_eq!(parameters.required, None); let output_schema = output_schema.expect("mailbox_read output schema"); assert_eq!( @@ -523,9 +533,11 @@ fn mailbox_read_tool_exposes_bounded_cursor_pagination() { output_schema["properties"]["messages"]["items"]["required"], json!(["message_id", "sender", "kind", "canonical_delivered"]) ); - assert!(output_schema["properties"]["messages"]["items"]["properties"] - .get("message") - .is_none()); + assert!( + output_schema["properties"]["messages"]["items"]["properties"] + .get("message") + .is_none() + ); } #[test] @@ -654,14 +666,28 @@ fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result properties .get("timeout_ms") .and_then(|schema| schema.description.as_deref()), - Some("Optional timeout in milliseconds. Omit for an unbounded event-only wait; no short timeout is applied implicitly.") + Some( + "Optional timeout in milliseconds. Omit for an unbounded event-only wait; no short timeout is applied implicitly." + ) ); assert_eq!( properties["return_when"].enum_values.as_ref(), Some(&vec![json!("any"), json!("all")]) ); - assert!(properties["return_when"].description.as_deref().unwrap().contains("Defaults to `any`")); - assert!(properties["native_event_wait"].description.as_deref().unwrap().contains("has no effect")); + assert!( + properties["return_when"] + .description + .as_deref() + .unwrap() + .contains("Defaults to `any`") + ); + assert!( + properties["native_event_wait"] + .description + .as_deref() + .unwrap() + .contains("has no effect") + ); assert!(description.contains("unbounded event-only wait")); assert!(description.contains("does not satisfy a selected join")); let output_schema = output_schema.expect("wait output schema"); @@ -726,14 +752,18 @@ fn wait_agent_tool_v2_omits_runtime_fields_without_capability_provider() { assert!(!description.contains("return_when")); let output_schema = output_schema.expect("wait output schema"); - assert!(!output_schema["properties"] - .as_object() - .expect("properties should be object") - .contains_key("pending_ids")); - assert!(!output_schema["properties"] - .as_object() - .expect("properties should be object") - .contains_key("completion_reason")); + assert!( + !output_schema["properties"] + .as_object() + .expect("properties should be object") + .contains_key("pending_ids") + ); + assert!( + !output_schema["properties"] + .as_object() + .expect("properties should be object") + .contains_key("completion_reason") + ); assert_eq!( output_schema["required"], json!(["message", "requested_ids", "timed_out"]) @@ -757,7 +787,11 @@ fn mailbox_wait_agent_tool_has_no_native_capability_prerequisite() { assert!(properties.contains_key("return_when")); assert!(properties.contains_key("native_event_wait")); let output_schema = output_schema.expect("mailbox wait output schema"); - assert!(output_schema["properties"].get("condition_satisfied").is_some()); + assert!( + output_schema["properties"] + .get("condition_satisfied") + .is_some() + ); assert!(output_schema["properties"].get("quiet_count").is_some()); assert_eq!( output_schema["properties"]["completed"]["items"]["type"], @@ -878,7 +912,8 @@ fn inspect_agent_tree_tool_exposes_scope_and_compact_tree_fields() { #[test] fn mailbox_send_message_tool_declares_durable_receipt_without_claiming_comprehension() { - let ToolSpec::Function(ResponsesApiTool { output_schema, .. }) = create_mailbox_send_message_tool() + let ToolSpec::Function(ResponsesApiTool { output_schema, .. }) = + create_mailbox_send_message_tool() else { panic!("send_message should be a function tool"); }; @@ -896,8 +931,10 @@ fn mailbox_send_message_tool_declares_durable_receipt_without_claiming_comprehen output_schema["properties"]["activation_disposition"]["enum"], json!(["not_requested", "already_active", "activated", "denied"]) ); - assert!(output_schema["properties"]["durable_state"]["description"] - .as_str() - .expect("durable state description") - .contains("does not establish that the recipient read or understood it")); + assert!( + output_schema["properties"]["durable_state"]["description"] + .as_str() + .expect("durable state description") + .contains("does not establish that the recipient read or understood it") + ); } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs index f3c5918be56c..652bb8811b6f 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs @@ -42,12 +42,18 @@ impl Handler { } impl ToolExecutor for MailboxHandler { - fn tool_name(&self) -> ToolName { ToolName::plain("followup_task") } - fn spec(&self) -> ToolSpec { create_followup_task_tool() } + fn tool_name(&self) -> ToolName { + ToolName::plain("followup_task") + } + fn spec(&self) -> ToolSpec { + create_followup_task_tool() + } fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { Box::pin(async move { if invocation.cancellation_token.is_cancelled() { - return Err(FunctionCallError::RespondToModel("follow-up was cancelled before native acceptance".to_string())); + return Err(FunctionCallError::RespondToModel( + "follow-up was cancelled before native acceptance".to_string(), + )); } Handler.handle_call(invocation).await }) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs index c068b714bf12..19a19c9b72cd 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs @@ -90,7 +90,9 @@ pub(super) async fn deliver_envelope( envelope: &Envelope, ) -> Result { if envelope.recipient != session.thread_id { - return Err(mailbox_error("message is not addressed to this native thread")); + return Err(mailbox_error( + "message is not addressed to this native thread", + )); } session .record_inter_agent_communication(turn, envelope.communication.clone()) @@ -118,53 +120,100 @@ impl ToolExecutor for SendHandler { fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { Box::pin(async move { - let ToolInvocation { session, turn, payload, call_id, .. } = invocation; + let ToolInvocation { + session, + turn, + payload, + call_id, + .. + } = invocation; let args: SendArgs = parse_arguments(&function_arguments(payload)?)?; if args.message.trim().is_empty() { return Err(mailbox_error("message cannot be empty")); } let mailbox = mailbox_for(&session)?; let recipient = resolve_agent_target(&session, &turn, &args.target).await?; - let metadata = session.services.agent_control.ensure_agent_known(recipient) + let metadata = session + .services + .agent_control + .ensure_agent_known(recipient) .map_err(|err| collab_agent_error(recipient, err))?; - let recipient_path = metadata.agent_path.ok_or_else(|| mailbox_error("recipient has no native agent path"))?; - let sender_path = turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root); + let recipient_path = metadata + .agent_path + .ok_or_else(|| mailbox_error("recipient has no native agent path"))?; + let sender_path = turn + .session_source + .get_agent_path() + .unwrap_or_else(AgentPath::root); let work_ref = match args.work_ref.as_deref() { - Some(id) => Some(mailbox.work_for_message_scope(session.thread_id, recipient, id) - .await.map_err(mailbox_error)? - .ok_or_else(|| mailbox_error("work_ref does not belong to this exact author/recipient scope"))?), + Some(id) => Some( + mailbox + .work_for_message_scope(session.thread_id, recipient, id) + .await + .map_err(mailbox_error)? + .ok_or_else(|| { + mailbox_error( + "work_ref does not belong to this exact author/recipient scope", + ) + })?, + ), None => None, }; let legacy_attention = args.kind.is_none() && args.wake_wait; - let envelope = mailbox.accept_message(MessageDraft { - invocation_key: call_id, - sender: session.thread_id, - recipient, - communication: InterAgentCommunication::new_encrypted( - sender_path, - recipient_path.clone(), - Vec::new(), - args.message, - /*trigger_turn*/ false, - ), - kind: args.kind.unwrap_or_default(), - work_ref, - reply_to: args.reply_to, - supersedes: args.supersedes, - legacy_attention, - activate_if_idle: args.activate_if_idle, - }).await.map_err(mailbox_error)?; + let envelope = mailbox + .accept_message(MessageDraft { + invocation_key: call_id, + sender: session.thread_id, + recipient, + communication: InterAgentCommunication::new_encrypted( + sender_path, + recipient_path.clone(), + Vec::new(), + args.message, + /*trigger_turn*/ false, + ), + kind: args.kind.unwrap_or_default(), + work_ref, + reply_to: args.reply_to, + supersedes: args.supersedes, + legacy_attention, + activate_if_idle: args.activate_if_idle, + }) + .await + .map_err(mailbox_error)?; let (activation_disposition, activation_reason) = if args.activate_if_idle { - match mailbox.begin_activation(session.thread_id, &envelope.id).await { + match mailbox + .begin_activation(session.thread_id, &envelope.id) + .await + { Ok(ActivationDecision::Activate) => { - let result = session.services.agent_control - .mailbox_activate_if_idle(recipient, &envelope).await; + let result = session + .services + .agent_control + .mailbox_activate_if_idle(recipient, &envelope) + .await; let (disposition, reason) = match result { Ok(result) => result, - Err(_) => (ActivationDisposition::Unknown, Some("activation_outcome_unknown_original_message_retained".to_string())), + Err(_) => ( + ActivationDisposition::Unknown, + Some( + "activation_outcome_unknown_original_message_retained" + .to_string(), + ), + ), }; - if mailbox.record_activation(session.thread_id, &envelope.id, disposition).await.is_err() { - (ActivationDisposition::Unknown, Some("activation_receipt_not_persisted_original_message_retained".to_string())) + if mailbox + .record_activation(session.thread_id, &envelope.id, disposition) + .await + .is_err() + { + ( + ActivationDisposition::Unknown, + Some( + "activation_receipt_not_persisted_original_message_retained" + .to_string(), + ), + ) } else { (disposition, reason) } @@ -178,17 +227,26 @@ impl ToolExecutor for SendHandler { } else { // The envelope is already committed. Optional activation bookkeeping must // never hide acceptance or invite the caller to create a duplicate message. - let _ = mailbox.record_activation(session.thread_id, &envelope.id, ActivationDisposition::NotRequested).await; + let _ = mailbox + .record_activation( + session.thread_id, + &envelope.id, + ActivationDisposition::NotRequested, + ) + .await; (ActivationDisposition::NotRequested, None) }; Ok(boxed_tool_output(FunctionToolOutput::from_text( - tool_output_json_text(&SendReceipt { - message_id: envelope.id, - durable_state: "accepted", - activation_disposition, - activation_reason, - recipient_task_name: recipient_path.to_string(), - }, "send_message"), + tool_output_json_text( + &SendReceipt { + message_id: envelope.id, + durable_state: "accepted", + activation_disposition, + activation_reason, + recipient_task_name: recipient_path.to_string(), + }, + "send_message", + ), Some(true), ))) }) @@ -212,9 +270,18 @@ impl ToolExecutor for ReadHandler { fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { Box::pin(async move { - let ToolInvocation { session, turn, payload, .. } = invocation; + let ToolInvocation { + session, + turn, + payload, + .. + } = invocation; let args: ReadArgs = parse_arguments(&function_arguments(payload)?)?; - let cursor = args.cursor.as_deref().map(str::parse::).transpose() + let cursor = args + .cursor + .as_deref() + .map(str::parse::) + .transpose() .map_err(|_| mailbox_error("cursor must be a returned mailbox cursor"))?; let mailbox = mailbox_for(&session)?; let requested_limit = args.limit.unwrap_or(20); @@ -223,8 +290,10 @@ impl ToolExecutor for ReadHandler { } // An encrypted body cannot be safely truncated. Every accepted body is capped // at 8KiB; admit at most two per call and retain the rest behind the page cursor. - let page = mailbox.pending_page(session.thread_id, cursor, requested_limit.min(2)) - .await.map_err(mailbox_error)?; + let page = mailbox + .pending_page(session.thread_id, cursor, requested_limit.min(2)) + .await + .map_err(mailbox_error)?; let mut messages = Vec::with_capacity(page.items.len()); for envelope in &page.items { messages.push(deliver_envelope(&session, &turn, envelope).await?); @@ -232,32 +301,52 @@ impl ToolExecutor for ReadHandler { if matches!(envelope.kind, MessageKind::Request | MessageKind::Alert) || envelope.legacy_attention { - observations.push(ObservationTarget::Attention(format!("attention:{}", envelope.id))); + observations.push(ObservationTarget::Attention(format!( + "attention:{}", + envelope.id + ))); } if envelope.reply_to.is_some() { observations.push(ObservationTarget::Reply(envelope.id.clone())); } - if envelope.communication.origin == Some(codex_protocol::protocol::AgentCommunicationOrigin::Result) { + if envelope.communication.origin + == Some(codex_protocol::protocol::AgentCommunicationOrigin::Result) + { // A native terminal result can cover several admitted inputs. Resolve // the bounded exact body relation rather than infer from agent history. - observations.extend(mailbox.work_for_outcome(session.thread_id, &envelope.id) - .await.map_err(mailbox_error)?.into_iter().map(ObservationTarget::Work)); + observations.extend( + mailbox + .work_for_outcome(session.thread_id, &envelope.id) + .await + .map_err(mailbox_error)? + .into_iter() + .map(ObservationTarget::Work), + ); } // Reading a body can observe its own attention/reply only; it never observes // unrelated work outcomes or unresolved request obligations. if !observations.is_empty() { for batch in observations.chunks(100) { - mailbox.acknowledge_observed(session.thread_id, batch).await.map_err(mailbox_error)?; + mailbox + .acknowledge_observed(session.thread_id, batch) + .await + .map_err(mailbox_error)?; } } } - let quiet_count = mailbox.quiet_count(session.thread_id).await.map_err(mailbox_error)?; + let quiet_count = mailbox + .quiet_count(session.thread_id) + .await + .map_err(mailbox_error)?; Ok(boxed_tool_output(FunctionToolOutput::from_text( - tool_output_json_text(&ReadReceipt { - messages, - next_cursor: page.next_cursor.map(|cursor| cursor.to_string()), - quiet_count, - }, "mailbox_read"), + tool_output_json_text( + &ReadReceipt { + messages, + next_cursor: page.next_cursor.map(|cursor| cursor.to_string()), + quiet_count, + }, + "mailbox_read", + ), Some(true), ))) }) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 228cb978fb8e..84aaf662b753 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -172,16 +172,23 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } } let mut recovery_changed = false; + let mut recovery_fault = false; for item in &selected { if let SnapshotItem::Work(work) = item && work.dispatch_state == DispatchState::Dispatched && work.outcome.is_none() && work.continuations.is_empty() { - recovery_changed |= session.services.agent_control - .mailbox_reconcile_pending_work(&work.work_ref).await.map_err(mailbox_error)?; + match session.services.agent_control.mailbox_reconcile_pending_work(&work.work_ref).await { + Ok(changed) => recovery_changed |= changed, + Err(_) => { recovery_fault = true; break; } + } } } + if recovery_fault { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + break; + } if recovery_changed { continue; } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index a0b7086cb7b9..053dab602be3 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -21,37 +21,86 @@ async fn fixture() -> (Arc, Arc, Arc, ThreadId) { config.features.enable(Feature::AgentMailbox).unwrap(); turn.multi_agent_version = config.multi_agent_version_from_features(); turn.config = Arc::new(config); - session.services.agent_control.mailbox_initialize(&turn.config).await.unwrap(); + session + .services + .agent_control + .mailbox_initialize(&turn.config) + .await + .unwrap(); let mailbox = session.services.agent_control.mailbox().unwrap(); (Arc::new(session), Arc::new(turn), mailbox, ThreadId::new()) } -async fn work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId, key: &str, outcome: Option) -> WorkRef { - let work = mailbox.accept_work(NativeWork { - native_key: key.to_string(), issuer, agent, submission_id: key.to_string(), - input: InterAgentCommunication::new(AgentPath::root(), AgentPath::from_string("/root/worker".to_string()).unwrap(), Vec::new(), key.to_string(), true), - }).await.unwrap(); +async fn work( + mailbox: &Mailbox, + issuer: ThreadId, + agent: ThreadId, + key: &str, + outcome: Option, +) -> WorkRef { + let work = mailbox + .accept_work(NativeWork { + native_key: key.to_string(), + issuer, + agent, + submission_id: key.to_string(), + input: InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + key.to_string(), + true, + ), + }) + .await + .unwrap(); if let Some(outcome) = outcome { - let result = mailbox.record_outcome(&work, outcome, None, Continuation::None).await.unwrap(); + let result = mailbox + .record_outcome(&work, outcome, None, Continuation::None) + .await + .unwrap(); // This fixture represents a previously surfaced result. Writer custody has // separate Session tests; this exercises replay through the actual wait facade. mailbox.mark_canonical(issuer, &result.id).await.unwrap(); - mailbox.acknowledge_observed(issuer, &[ObservationTarget::Work(work.clone())]).await.unwrap(); + mailbox + .acknowledge_observed(issuer, &[ObservationTarget::Work(work.clone())]) + .await + .unwrap(); } work } -async fn wait(session: Arc, turn: Arc, targets: Vec, mode: &str) -> MailboxWaitReport { - let payload = ToolPayload::Function { arguments: serde_json::json!({ "targets": targets, "return_when": mode, "timeout_ms": 0 }).to_string() }; +async fn wait( + session: Arc, + turn: Arc, + targets: Vec, + mode: &str, +) -> MailboxWaitReport { + let payload = ToolPayload::Function { + arguments: serde_json::json!({ "targets": targets, "return_when": mode, "timeout_ms": 0 }) + .to_string(), + }; let output = handle_wait(ToolInvocation { - session, step_context: StepContext::for_test(Arc::clone(&turn)), turn, + session, + step_context: StepContext::for_test(Arc::clone(&turn)), + turn, cancellation_token: CancellationToken::new(), tracker: Arc::new(Mutex::new(TurnDiffTracker::default())), - call_id: format!("replay-{mode}"), tool_name: ToolName::plain("wait_agent"), - source: ToolCallSource::Direct, payload: payload.clone(), - }).await.unwrap(); - let ResponseInputItem::FunctionCallOutput { output, .. } = output.to_response_item("replay", &payload) else { panic!("expected a function result") }; - let FunctionCallOutputBody::Text(body) = output.body else { panic!("expected report text") }; + call_id: format!("replay-{mode}"), + tool_name: ToolName::plain("wait_agent"), + source: ToolCallSource::Direct, + payload: payload.clone(), + }) + .await + .unwrap(); + let ResponseInputItem::FunctionCallOutput { output, .. } = + output.to_response_item("replay", &payload) + else { + panic!("expected a function result") + }; + let FunctionCallOutputBody::Text(body) = output.body else { + panic!("expected report text") + }; serde_json::from_str(&body).unwrap() } @@ -59,12 +108,31 @@ async fn wait(session: Arc, turn: Arc, targets: Vec AgentCommunicationKind::Message, MessageDeliveryMode::TriggerTurn => AgentCommunicationKind::Followup, }; - let context = AgentCommunicationContext::new(kind, session.thread_id) - .with_native_key(call_id.clone()); + let context = + AgentCommunicationContext::new(kind, session.thread_id).with_native_key(call_id.clone()); let result = delivery .send(mode.apply(communication), context, interrupt) .await @@ -420,7 +448,10 @@ async fn handle_message_submission_inner( .await .map_err(|err| collab_agent_error(receiver_thread_id, err))? .map(|work| work.id), - dispatch_state: session.services.agent_control.mailbox_enabled() + dispatch_state: session + .services + .agent_control + .mailbox_enabled() .then_some(codex_agent_mailbox::DispatchState::Dispatched), effective_identity_scope: "recipient", effective_model: Some(receiver_config.model), diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 29c4ba28935b..422ad24b4a1b 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1185,7 +1185,7 @@ mod tests { } #[tokio::test] - async fn configured_v2_spawn_fast_completion_cancellation_retry_reuses_native_work() + async fn configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work() -> anyhow::Result<()> { let (mut session, mut turn_context) = crate::session::tests::make_session_and_context().await; diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index cdf5689b7184..34f62229d955 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -50,7 +50,7 @@ use crate::tools::handlers::multi_agents_v2::InterruptAgentHandler; use crate::tools::handlers::multi_agents_v2::ListAgentsHandler as ListAgentsHandlerV2; use crate::tools::handlers::multi_agents_v2::MailboxReadHandler; use crate::tools::handlers::multi_agents_v2::MailboxSendMessageHandler; -use crate::tools::handlers::multi_agents_v2::MailboxWaitAgentHandlerV2; +use crate::tools::handlers::multi_agents_v2::MailboxWaitAgentHandler as MailboxWaitAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::SendMessageHandler as SendMessageHandlerV2; use crate::tools::handlers::multi_agents_v2::SpawnAgentHandler as SpawnAgentHandlerV2; use crate::tools::handlers::multi_agents_v2::WaitAgentHandler as WaitAgentHandlerV2; @@ -828,7 +828,7 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu agent_type_description(turn_context, context.default_agent_type_description); let hide_spawn_agent_metadata = turn_context.config.multi_agent_v2.hide_spawn_agent_metadata; - let mailbox_enabled = turn_context.features.enabled(Feature::AgentMailbox); + let mailbox_enabled = turn_context.config.features.enabled(Feature::AgentMailbox); planned_tools.add_arc(override_tool_exposure( multi_agent_v2_handler( SpawnAgentHandlerV2::new(SpawnAgentToolOptions { diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index e8e61e8c6352..bcca77636683 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -220,8 +220,17 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { ]) ); assert!(!send_properties.contains_key("interrupt")); - for property in ["work_ref", "reply_to", "supersedes", "activate_if_idle", "wake_wait"] { - assert!(send_properties.contains_key(property), "missing `{property}`"); + for property in [ + "work_ref", + "reply_to", + "supersedes", + "activate_if_idle", + "wake_wait", + ] { + assert!( + send_properties.contains_key(property), + "missing `{property}`" + ); } let send_output = send_message .output_schema @@ -235,10 +244,12 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { send_output["properties"]["activation_disposition"]["enum"], json!(["not_requested", "already_active", "activated", "denied"]) ); - assert!(send_output["properties"]["durable_state"]["description"] - .as_str() - .expect("durable receipt description") - .contains("not establish that the recipient read or understood it")); + assert!( + send_output["properties"]["durable_state"]["description"] + .as_str() + .expect("durable receipt description") + .contains("not establish that the recipient read or understood it") + ); let wait_agent = namespace_function(plan, namespace_name, "wait_agent"); let wait_properties = wait_agent @@ -261,29 +272,48 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { codex_tools::JsonSchemaPrimitiveType::Integer )) ); - assert!(wait_properties["timeout_ms"] - .description - .as_deref() - .expect("timeout description") - .contains("unbounded event-only wait")); + assert!( + wait_properties["timeout_ms"] + .description + .as_deref() + .expect("timeout description") + .contains("unbounded event-only wait") + ); let wait_output = wait_agent .output_schema .as_ref() .expect("wait_agent output schema"); - assert!(wait_output["required"] - .as_array() - .expect("wait result required fields") - .contains(&json!("return_when"))); - assert_eq!(wait_output["properties"]["return_when"]["enum"], json!(["any", "all"])); - assert!(wait_output["properties"].get("condition_satisfied").is_some()); + assert!( + wait_output["required"] + .as_array() + .expect("wait result required fields") + .contains(&json!("return_when")) + ); + assert_eq!( + wait_output["properties"]["return_when"]["enum"], + json!(["any", "all"]) + ); + assert!( + wait_output["properties"] + .get("condition_satisfied") + .is_some() + ); assert!(wait_output["properties"].get("quiet_count").is_some()); - assert_eq!(wait_output["properties"]["completed"]["items"]["required"], json!(["status"])); - assert_eq!(wait_output["properties"]["pending"]["items"]["required"], json!(["status"])); + assert_eq!( + wait_output["properties"]["completed"]["items"]["required"], + json!(["status"]) + ); + assert_eq!( + wait_output["properties"]["pending"]["items"]["required"], + json!(["status"]) + ); for result_list in ["completed", "pending"] { for field in ["work_ref", "message_id", "agent", "status"] { - assert!(wait_output["properties"][result_list]["items"]["properties"] - .get(field) - .is_some()); + assert!( + wait_output["properties"][result_list]["items"]["properties"] + .get(field) + .is_some() + ); } } assert_eq!( @@ -319,11 +349,13 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { codex_tools::JsonSchemaPrimitiveType::Integer )) ); - assert!(read_properties["limit"] - .description - .as_deref() - .expect("mailbox page limit description") - .contains("capped at 100")); + assert!( + read_properties["limit"] + .description + .as_deref() + .expect("mailbox page limit description") + .contains("capped at 100") + ); assert_eq!( mailbox_read .output_schema @@ -345,10 +377,12 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { let tool = namespace_function(plan, namespace_name, tool_name); let output = tool.output_schema.as_ref().expect("agent result schema"); assert_eq!(output["properties"]["work_ref"]["type"], json!("string")); - assert!(!output["required"] - .as_array() - .expect("required result fields") - .contains(&json!("work_ref"))); + assert!( + !output["required"] + .as_array() + .expect("required result fields") + .contains(&json!("work_ref")) + ); } } @@ -1764,7 +1798,10 @@ async fn mailbox_schemas_survive_ordinary_namespaced_and_code_mode_exposure() { ordinary.assert_visible_contains(&["agents"]); for tool_name in ["send_message", "wait_agent", "mailbox_read"] { assert!( - ordinary.namespace_function_names("agents").iter().any(|name| name == tool_name), + ordinary + .namespace_function_names("agents") + .iter() + .any(|name| name == tool_name), "expected {tool_name} in the ordinary namespaced mailbox surface" ); } @@ -1799,10 +1836,12 @@ async fn mailbox_feature_disabled_keeps_legacy_v2_message_and_wait_schemas() { let ToolSpec::Namespace(namespace) = plan.visible_spec(MULTI_AGENT_V2_NAMESPACE) else { panic!("expected {MULTI_AGENT_V2_NAMESPACE} namespace"); }; - assert!(!plan - .namespace_function_names(MULTI_AGENT_V2_NAMESPACE) - .iter() - .any(|name| name == "mailbox_read")); + assert!( + !plan + .namespace_function_names(MULTI_AGENT_V2_NAMESPACE) + .iter() + .any(|name| name == "mailbox_read") + ); let send_message = namespace .tools .iter() @@ -1839,7 +1878,11 @@ async fn mailbox_feature_disabled_keeps_legacy_v2_message_and_wait_schemas() { .as_ref() .expect("legacy wait output schema"); assert!(wait_output["properties"].get("requested_ids").is_some()); - assert!(wait_output["properties"].get("condition_satisfied").is_none()); + assert!( + wait_output["properties"] + .get("condition_satisfied") + .is_none() + ); } #[tokio::test] diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 1b69bbfca397..478723ff32e6 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -1,9 +1,9 @@ use anyhow::Result; use codex_agent_mailbox::Mailbox; use codex_features::Feature; +use codex_protocol::ThreadId; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; -use codex_protocol::ThreadId; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call_with_namespace; @@ -14,8 +14,8 @@ use core_test_support::responses::sse_response; use core_test_support::responses::start_mock_server; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; -use serde_json::json; use serde_json::Value; +use serde_json::json; use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; @@ -44,7 +44,9 @@ fn find_named_tool<'a>(value: &'a Value, name: &str) -> Option<&'a Value> { { return Some(value); } - object.values().find_map(|child| find_named_tool(child, name)) + object + .values() + .find_map(|child| find_named_tool(child, name)) } Value::Array(items) => items.iter().find_map(|child| find_named_tool(child, name)), _ => None, @@ -65,14 +67,10 @@ fn function_output_text<'a>(request: &'a Value, call_id: &str) -> Option<&'a str } fn agent_message_input<'a>(request: &'a Value, message_id: &str) -> Option<&'a Value> { - request - .get("input")? - .as_array()? - .iter() - .find(|item| { - item.get("type").and_then(Value::as_str) == Some("agent_message") - && item.get("id").and_then(Value::as_str) == Some(message_id) - }) + request.get("input")?.as_array()?.iter().find(|item| { + item.get("type").and_then(Value::as_str) == Some("agent_message") + && item.get("id").and_then(Value::as_str) == Some(message_id) + }) } fn has_function_output(request: &Value, call_id: &str) -> bool { @@ -130,7 +128,11 @@ async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Resu disabled.submit_turn(SCHEMA_DISABLED_PROMPT).await?; let mut enabled_builder = test_codex().with_model("koffing").with_config(|config| { - for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + for feature in [ + Feature::Collab, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ] { config .features .enable(feature) @@ -145,7 +147,11 @@ async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Resu let disabled_wait = find_named_tool(&disabled_schema["tools"], "wait_agent") .expect("legacy V2 wait tool should be present when mailbox is disabled"); assert!(find_named_tool(&disabled_schema["tools"], "mailbox_read").is_none()); - assert!(disabled_wait["parameters"]["properties"].get("replies_to").is_none()); + assert!( + disabled_wait["parameters"]["properties"] + .get("replies_to") + .is_none() + ); let enabled_schema = enabled_response.single_request().body_json(); let wait = find_named_tool(&enabled_schema["tools"], "wait_agent") @@ -154,13 +160,19 @@ async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Resu assert!(wait_properties.get("targets").is_some()); assert!(wait_properties.get("replies_to").is_some()); assert!(wait_properties.get("timeout_ms").is_some()); - assert_eq!(wait_properties["return_when"]["enum"], json!(["any", "all"])); + assert_eq!( + wait_properties["return_when"]["enum"], + json!(["any", "all"]) + ); let send = find_named_tool(&enabled_schema["tools"], "send_message") .expect("mailbox send tool should be present"); let send_properties = &send["parameters"]["properties"]; for property in ["kind", "reply_to", "supersedes", "work_ref"] { - assert!(send_properties.get(property).is_some(), "missing {property}"); + assert!( + send_properties.get(property).is_some(), + "missing {property}" + ); } assert!(send_properties.get("interrupt").is_none()); @@ -169,9 +181,11 @@ async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Resu let read_properties = &read["parameters"]["properties"]; assert!(read_properties.get("cursor").is_some()); assert!(read_properties.get("limit").is_some()); - assert!(read["description"] - .as_str() - .is_some_and(|description| description.contains("canonical conversation history"))); + assert!( + read["description"] + .as_str() + .is_some_and(|description| description.contains("canonical conversation history")) + ); Ok(()) } @@ -199,12 +213,14 @@ struct ChildThreadMatcher { impl Match for ChildThreadMatcher { fn matches(&self, request: &Request) -> bool { - serde_json::from_slice::(&request.body).ok().is_some_and(|body| { - body.pointer("/client_metadata/thread_id") - .and_then(Value::as_str) - .is_some_and(|thread_id| thread_id != self.root_thread_id.as_str()) - && body.to_string().contains(self.task.as_str()) - }) + serde_json::from_slice::(&request.body) + .ok() + .is_some_and(|body| { + body.pointer("/client_metadata/thread_id") + .and_then(Value::as_str) + .is_some_and(|thread_id| thread_id != self.root_thread_id.as_str()) + && body.to_string().contains(self.task.as_str()) + }) } } @@ -216,8 +232,8 @@ struct RootMailboxFlowResponder { impl Respond for RootMailboxFlowResponder { fn respond(&self, request: &Request) -> ResponseTemplate { - let body: Value = serde_json::from_slice(&request.body) - .expect("provider request should contain JSON"); + let body: Value = + serde_json::from_slice(&request.body).expect("provider request should contain JSON"); self.requests .lock() .expect("request capture lock should not be poisoned") @@ -311,7 +327,11 @@ impl Respond for RootMailboxFlowResponder { async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() -> Result<()> { let server = start_mock_server().await; let mut builder = test_codex().with_model("koffing").with_config(|config| { - for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + for feature in [ + Feature::Collab, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ] { config .features .enable(feature) @@ -357,7 +377,11 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() .lock() .expect("request capture lock should not be poisoned") .clone(); - assert_eq!(requests.len(), 6, "root should make one provider call per tool stage"); + assert_eq!( + requests.len(), + 6, + "root should make one provider call per tool stage" + ); let spawn: Value = serde_json::from_str( function_output_text(&requests[5], "spawn-call").expect("spawn output should be retained"), @@ -365,7 +389,10 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() let work_ref = spawn["work_ref"] .as_str() .expect("spawn should return an immutable mailbox work reference"); - assert!(work_ref.starts_with("work_"), "unexpected work reference: {work_ref}"); + assert!( + work_ref.starts_with("work_"), + "unexpected work reference: {work_ref}" + ); let request_receipt: Value = serde_json::from_str( function_output_text(&requests[5], "request-call") @@ -374,7 +401,10 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() let request_id = request_receipt["message_id"] .as_str() .expect("request receipt should contain a native message ID"); - assert!(request_id.starts_with("amsg_"), "unexpected native ID: {request_id}"); + assert!( + request_id.starts_with("amsg_"), + "unexpected native ID: {request_id}" + ); let read_output: Value = serde_json::from_str( function_output_text(&requests[3], "read-call") @@ -402,7 +432,11 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() function_output_text(&requests[5], "reply-call") .expect("final reply receipt should be retained"), )?; - assert!(reply_receipt["message_id"].as_str().is_some_and(|id| id.starts_with("amsg_"))); + assert!( + reply_receipt["message_id"] + .as_str() + .is_some_and(|id| id.starts_with("amsg_")) + ); let wait_report: Value = serde_json::from_str( function_output_text(&requests[5], "wait-call") .expect("wait should return a selector report"), @@ -411,12 +445,18 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() assert_eq!(wait_report["requested_replies_to"], json!([request_id])); assert_eq!(wait_report["return_when"], "all"); assert_eq!(wait_report["condition_satisfied"], true); - assert!(wait_report["completed"].as_array().is_some_and(|completed| { - completed.iter().any(|item| item["work_ref"] == work_ref && item["status"] == "completed") - && completed - .iter() - .any(|item| item["message_id"] == request_id && item["status"] == "replied") - })); + assert!( + wait_report["completed"] + .as_array() + .is_some_and(|completed| { + completed + .iter() + .any(|item| item["work_ref"] == work_ref && item["status"] == "completed") + && completed + .iter() + .any(|item| item["message_id"] == request_id && item["status"] == "replied") + }) + ); assert_eq!(wait_report["pending"], json!([])); Ok(()) @@ -443,8 +483,8 @@ struct WaitProbeResponder { impl Respond for WaitProbeResponder { fn respond(&self, request: &Request) -> ResponseTemplate { - let body: Value = serde_json::from_slice(&request.body) - .expect("provider request should contain JSON"); + let body: Value = + serde_json::from_slice(&request.body).expect("provider request should contain JSON"); let received_at = Instant::now(); self.requests .lock() @@ -544,7 +584,8 @@ impl Respond for QuietThenRequestResponder { ev_response_created("quiet-child-final"), ev_assistant_message("quiet-child-final-message", "done"), ev_completed("quiet-child-final"), - ])).set_delay(Duration::from_secs(3)); + ])) + .set_delay(Duration::from_secs(3)); } if has_function_output(&body, "quiet-progress-3-call") { @@ -617,7 +658,11 @@ impl Respond for QuietThenRequestResponder { async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() -> Result<()> { let server = start_mock_server().await; let mut builder = test_codex().with_model("koffing").with_config(|config| { - for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + for feature in [ + Feature::Collab, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ] { config .features .enable(feature) @@ -680,15 +725,18 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - .lock() .expect("request capture lock should not be poisoned") .clone(); - assert_eq!(requests.len(), 5, "quiet events must not add root provider calls"); + assert_eq!( + requests.len(), + 5, + "quiet events must not add root provider calls" + ); let wait_started = first_wait_started .lock() .expect("wait start lock should not be poisoned") .expect("first targeted wait should have started"); let deadline_wait_elapsed = requests[3].0.duration_since(wait_started); assert!( - (Duration::from_millis(850)..Duration::from_millis(1_500)) - .contains(&deadline_wait_elapsed), + (Duration::from_millis(850)..Duration::from_millis(1_500)).contains(&deadline_wait_elapsed), "quiet messages should not refresh the one-second deadline: {deadline_wait_elapsed:?}" ); @@ -754,7 +802,10 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - }) .expect("wait should admit the addressed request into canonical history"); assert_eq!(canonical_request["content"][1]["type"], "encrypted_content"); - assert_eq!(canonical_request["content"][1]["encrypted_content"], ADDRESSED_REQUEST_BODY); + assert_eq!( + canonical_request["content"][1]["encrypted_content"], + ADDRESSED_REQUEST_BODY + ); Ok(()) } @@ -767,8 +818,8 @@ struct MailboxCustodyResponder { impl Respond for MailboxCustodyResponder { fn respond(&self, request: &Request) -> ResponseTemplate { - let body: Value = serde_json::from_slice(&request.body) - .expect("provider request should contain JSON"); + let body: Value = + serde_json::from_slice(&request.body).expect("provider request should contain JSON"); self.requests .lock() .expect("request capture lock should not be poisoned") @@ -832,7 +883,11 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com -> Result<()> { let server = start_mock_server().await; let mut builder = test_codex().with_model("koffing").with_config(|config| { - for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + for feature in [ + Feature::Collab, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ] { config .features .enable(feature) @@ -868,7 +923,11 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com .lock() .expect("request capture lock should not be poisoned") .clone(); - assert_eq!(requests.len(), 3, "the pre-created failure should not retry"); + assert_eq!( + requests.len(), + 3, + "the pre-created failure should not retry" + ); let send_receipt: Value = serde_json::from_str( function_output_text(&requests[1], "send-request-call") .expect("accepted request receipt should remain in provider history"), @@ -876,7 +935,10 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com let message_id = send_receipt["message_id"] .as_str() .expect("send receipt should include a native message ID"); - assert!(message_id.starts_with("amsg_"), "unexpected native ID: {message_id}"); + assert!( + message_id.starts_with("amsg_"), + "unexpected native ID: {message_id}" + ); let failed_read_receipt: Value = serde_json::from_str( function_output_text(&requests[2], "initial-read-call") @@ -885,7 +947,10 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com assert_eq!(failed_read_receipt["messages"][0]["message_id"], message_id); let failed_request_item = agent_message_input(&requests[2], message_id) .expect("the failed-before-created request must contain the canonical body"); - assert_eq!(failed_request_item["content"][1]["type"], "encrypted_content"); + assert_eq!( + failed_request_item["content"][1]["type"], + "encrypted_content" + ); assert_eq!( failed_request_item["content"][1]["encrypted_content"], CUSTODY_BODY @@ -895,7 +960,10 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com let message_id_owned = message_id.to_string(); assert_eq!( mailbox - .unadmitted_input_ids(root_thread_id.clone(), std::slice::from_ref(&message_id_owned)) + .unadmitted_input_ids( + root_thread_id.clone(), + std::slice::from_ref(&message_id_owned) + ) .await?, vec![message_id_owned.clone()], "a stream that ended before response.created must not admit the body" @@ -909,11 +977,13 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com envelope.communication.encrypted_content.as_deref(), Some(CUSTODY_BODY) ); - assert!(mailbox - .pending(root_thread_id.clone(), 10) - .await? - .iter() - .any(|pending| pending.id.as_str() == message_id)); + assert!( + mailbox + .pending(root_thread_id.clone(), 10) + .await? + .iter() + .any(|pending| pending.id.as_str() == message_id) + ); test.codex.submit(Op::Compact).await?; let compact_terminal = wait_for_event(&test.codex, |event| { @@ -926,23 +996,35 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com .lock() .expect("request capture lock should not be poisoned") .clone(); - assert_eq!(requests.len(), 4, "manual compaction should use one provider call"); + assert_eq!( + requests.len(), + 4, + "manual compaction should use one provider call" + ); assert!(requests[3].to_string().contains(COMPACTION_PROMPT)); let compaction_item = agent_message_input(&requests[3], message_id) .expect("compaction should receive the canonical message as source history"); - assert_eq!(compaction_item["content"][1]["encrypted_content"], CUSTODY_BODY); + assert_eq!( + compaction_item["content"][1]["encrypted_content"], + CUSTODY_BODY + ); assert_eq!( mailbox - .unadmitted_input_ids(root_thread_id.clone(), std::slice::from_ref(&message_id_owned)) + .unadmitted_input_ids( + root_thread_id.clone(), + std::slice::from_ref(&message_id_owned) + ) .await?, vec![message_id_owned.clone()], "compaction is not execution admission for a pending mailbox obligation" ); - assert!(mailbox - .pending(root_thread_id.clone(), 10) - .await? - .iter() - .any(|pending| pending.id.as_str() == message_id)); + assert!( + mailbox + .pending(root_thread_id.clone(), 10) + .await? + .iter() + .any(|pending| pending.id.as_str() == message_id) + ); let retained = mailbox .get_message(root_thread_id.clone(), message_id) .await? @@ -958,20 +1040,32 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com .lock() .expect("request capture lock should not be poisoned") .clone(); - assert_eq!(requests.len(), 6, "recovery should read the request then execute once"); + assert_eq!( + requests.len(), + 6, + "recovery should read the request then execute once" + ); let recovery_read_receipt: Value = serde_json::from_str( function_output_text(&requests[5], "recovery-read-call") .expect("recovery mailbox_read receipt should remain in provider history"), )?; - assert_eq!(recovery_read_receipt["messages"][0]["message_id"], message_id); + assert_eq!( + recovery_read_receipt["messages"][0]["message_id"], + message_id + ); let recovered_item = agent_message_input(&requests[5], message_id) .expect("recovery read must restore the exact canonical body to execution input"); assert_eq!(recovered_item["content"][1]["type"], "encrypted_content"); - assert_eq!(recovered_item["content"][1]["encrypted_content"], CUSTODY_BODY); - assert!(mailbox - .unadmitted_input_ids(root_thread_id, &[message_id_owned]) - .await? - .is_empty()); + assert_eq!( + recovered_item["content"][1]["encrypted_content"], + CUSTODY_BODY + ); + assert!( + mailbox + .unadmitted_input_ids(root_thread_id, &[message_id_owned]) + .await? + .is_empty() + ); Ok(()) } diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 3b660c4fbf96..97eebbf700b7 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -255,9 +255,7 @@ impl GoalRuntimeHandle { if self.inner.notification_store.has_forwarded_completion() { return Ok(()); } - if let Some((goal_id, codex_state::ThreadGoalStatus::Active)) = - external_goal.as_ref() - { + if let Some((goal_id, codex_state::ThreadGoalStatus::Active)) = external_goal.as_ref() { let snapshot = self.inner.notification_store.snapshot(); if snapshot .as_ref() diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 965797041c06..2890aea163f2 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -1097,8 +1097,8 @@ async fn live_mailbox_wait_report_shows_join_pending_state_and_actual_wake() { drain_insert_history(&mut rx); let sender_thread_id = ThreadId::new(); let receiver_thread_id = ThreadId::new(); - let mailbox_wait_item = |status, mailbox_wait_report| { - AppServerThreadItem::CollabAgentToolCall { + let mailbox_wait_item = + |status, mailbox_wait_report| AppServerThreadItem::CollabAgentToolCall { id: "wait-mailbox-1".to_string(), tool: AppServerCollabAgentTool::Wait, status, @@ -1116,8 +1116,7 @@ async fn live_mailbox_wait_report_shows_join_pending_state_and_actual_wake() { completion_reason: None, wake_cause: None, mailbox_wait_report, - } - }; + }; let active_report = codex_protocol::items::MailboxWaitReport { requested_targets: vec!["Ada".to_string(), "work_42".to_string()], requested_replies_to: vec!["msg_request".to_string()], diff --git a/docs/carry-divergence-ledger.md b/docs/carry-divergence-ledger.md index 6b70641796c1..6f9415ef5afb 100644 --- a/docs/carry-divergence-ledger.md +++ b/docs/carry-divergence-ledger.md @@ -59,6 +59,12 @@ reconciliation, and existing collaboration live/replay projection. Canonical custody is not execution admission; compact checkpoints preserve selected unread bodies. Digests and UI rows do not consume omitted bodies. Body delivery is bounded; longer final previews identify their canonical source turn explicitly. +Execution admission uses the provider's existing response-created event, not a +prepared request or a lazy stream wrapper. Failed pre-establishment attempts +leave the original IDs pending through compaction. Reading a native result body +observes only the exact work it covers; an observed failure remains non-success +on subsequent exact or mixed any/all joins. Input publication invalidates stale +cycle snapshots before a closed-peer claim can be made. Both `agent_mailbox` and recipient `agent_mailbox_idle_activation` are off by default. Disabled mailbox handling preserves pending records and cannot drain, diff --git a/docs/downstream-regression-matrix.md b/docs/downstream-regression-matrix.md index 071d6b97feb5..b1af3b6837fa 100644 --- a/docs/downstream-regression-matrix.md +++ b/docs/downstream-regression-matrix.md @@ -61,6 +61,13 @@ compaction, activation denial, backpressure, and conservative closed-peer cycles Library tests alone do not prove the native adapter, UI, final packages, landing, or an installed session. Linux preview packages remain exact-SHA artifacts; install/restart and runtime adoption are separate operator actions. +The maintained core lane runs the owned library, the core `mailbox` unit slice, +and `suite::agent_mailbox`, including +`unadmitted_canonical_message_survives_precreated_failure_and_manual_compaction`. +Its native cancellation slice includes +`configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work`. +The notification visibility lane also runs protocol/TUI `mailbox` tests so the +same structured report is exercised in live and replay consumers. Exact-target V2 `wait_agent` ignores queue-only (`trigger_turn=false`) mailbox activity while preserving durable FIFO delivery and notification summaries; diff --git a/justfile b/justfile index 619a774372ed..7cd312fc70f9 100644 --- a/justfile +++ b/justfile @@ -209,6 +209,8 @@ core-subagent-notification-contract-targeted: # tui_app_server -> tui cutover. core-subagent-notification-visibility-targeted: + cargo test --locked -p codex-protocol mailbox --lib -- --test-threads=1 + cargo test --locked -p codex-tui mailbox --lib -- --test-threads=1 cargo test -p codex-protocol parse_subagent_notification_response_item_ --lib -- --test-threads=1 cargo test -p codex-tui raw_response_subagent_notification_renders_history -- --exact --test-threads=1 @@ -378,7 +380,9 @@ tui-brokered-tool-replay-targeted: # and generation-safe V2 residency eviction. core-multi-agent-orchestration-targeted: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 - RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_spawn_cancellation_after_initial_delivery_returns_published_result context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact + RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 + RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 + RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --test all -- suite::spawn_agent_description::spawn_agent_description_lists_visible_models_and_reasoning_efforts suite::agent_execution::v2_evicted_completed_agent_keeps_final_status suite::agent_execution::v2_cold_mailbox_allows_eviction_and_replays_on_followup suite::multi_agent_mode::changing_configured_mode_hint_to_empty_appends_explicit_reset suite::pending_input::queue_only_agent_mail_wakes_sleeping_root_and_persists_message --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-exec --test all suite::completion_backfill::ignores_unrelated_turn_completion_before_backfilling_primary_turn -- --exact --test-threads=1 cargo nextest run -p codex-protocol --lib --no-tests=fail -- protocol::tests::turn_complete_without_provider_usage_remains_compatible --exact From 403e750b17bea37dd5da1c2649e71f97b33e75e9 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:56:09 +1100 Subject: [PATCH 07/54] Recover interrupted mailbox waits and verify exact body custody --- .github/validation-named-tests.json | 8 + .../workflows/mailbox-source-preparation.yml | 11 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 31 ++ codex-rs/agent-mailbox/src/messages.rs | 80 ++- codex-rs/agent-mailbox/src/outcome_store.rs | 29 +- codex-rs/agent-mailbox/src/schema.rs | 1 + codex-rs/agent-mailbox/src/store.rs | 2 +- codex-rs/agent-mailbox/src/work.rs | 36 ++ .../src/protocol/thread_history.rs | 115 ++++ .../src/protocol/v2/item.rs | 2 +- codex-rs/core/src/agent/control/mailbox.rs | 71 ++- codex-rs/core/src/session/handlers.rs | 11 +- codex-rs/core/src/session/input_queue.rs | 64 ++- codex-rs/core/src/session/mod.rs | 311 +++++++++++ codex-rs/core/src/session/session.rs | 20 + codex-rs/core/src/session/tests.rs | 248 ++++++++- codex-rs/core/src/session/turn.rs | 10 +- codex-rs/core/src/tasks/mod.rs | 13 +- codex-rs/core/src/thread_manager.rs | 2 +- .../src/tools/handlers/multi_agents_spec.rs | 2 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 6 + .../multi_agents_v2/mailbox_wait_tests.rs | 513 +++++++++++++++++- .../handlers/multi_agents_v2/message_tool.rs | 36 +- .../tools/handlers/multi_agents_v2/spawn.rs | 11 +- codex-rs/core/src/tools/parallel.rs | 2 +- codex-rs/core/src/tools/spec_plan.rs | 8 +- codex-rs/protocol/src/items.rs | 1 + codex-rs/tui/src/multi_agents/mailbox_wait.rs | 1 + docs/carry-divergence-ledger.md | 7 +- docs/downstream-regression-matrix.md | 7 +- justfile | 1 + 31 files changed, 1555 insertions(+), 105 deletions(-) diff --git a/.github/validation-named-tests.json b/.github/validation-named-tests.json index 5910cdc11d71..137d1a2a2690 100644 --- a/.github/validation-named-tests.json +++ b/.github/validation-named-tests.json @@ -17,6 +17,14 @@ "inventory_argv": ["cargo", "test", "--locked", "-p", "codex-app-server", "--lib", "--", "--list"], "execution_argv": ["cargo", "test", "--locked", "-p", "codex-app-server", "--lib", "--", "--test-threads=1"] }, + { + "package": "codex-app-server-protocol", + "target_kind": "lib", + "target": "", + "profiles": ["rust_minimal", "rust_integration"], + "inventory_argv": ["cargo", "test", "--locked", "-p", "codex-app-server-protocol", "--lib", "--", "--list"], + "execution_argv": ["cargo", "test", "--locked", "-p", "codex-app-server-protocol", "--lib", "--", "--test-threads=1"] + }, { "package": "codex-config", "target_kind": "lib", diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 88773d326ac4..51e4073d0bb6 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "4" + PREPARATION_ATTEMPT: "5" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -71,6 +71,9 @@ jobs: env: INSTA_UPDATE: always run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 + - name: Verify interrupted mailbox wait history replay + working-directory: codex-rs + run: cargo test --locked -p codex-app-server-protocol --lib mailbox -- --test-threads=1 - name: Retain the library contract result if: always() run: test '${{ steps.library.outcome }}' = success @@ -80,6 +83,12 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/mailbox-source-preparation" + for generated_type in MailboxWaitReport MailboxWaitReturnWhen MailboxWaitSelectionItem MailboxWakeCause; do + generated_path="codex-rs/app-server-protocol/schema/typescript/${generated_type}.ts" + if [[ -f "$generated_path" ]]; then + git add --intent-to-add -- "$generated_path" + fi + done git diff --binary > "$RUNNER_TEMP/mailbox-source-preparation/source.patch" git rev-parse HEAD > "$RUNNER_TEMP/mailbox-source-preparation/source-sha.txt" git status --short > "$RUNNER_TEMP/mailbox-source-preparation/status.txt" diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 1eb9074919fa..bb115dfcfb90 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1345,6 +1345,37 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched); } +#[tokio::test] +async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_batches() { + let fixture = test_mailbox().await; + let (issuer, agent, other_issuer) = (thread_id(34), thread_id(35), thread_id(36)); + let accepted_request = native_work(issuer, agent, "resume-accepted", "retained accepted input"); + let accepted = fixture.mailbox.accept_work(accepted_request.clone()).await.unwrap(); + let retained_input = fixture.mailbox.accepted_input(&accepted).await.unwrap(); + let (dispatching, _) = accept_work(&fixture.mailbox, issuer, agent, "resume-dispatching", "uncertain dispatch").await; + let (dispatched, _) = accept_work(&fixture.mailbox, issuer, agent, "resume-dispatched", "confirmed dispatch").await; + let (foreign, _) = accept_work(&fixture.mailbox, other_issuer, agent, "other-issuer", "foreign accepted input").await; + assert_eq!(fixture.mailbox.begin_dispatch(&dispatching).await.unwrap(), DispatchDecision::Dispatch); + assert_eq!(fixture.mailbox.begin_dispatch(&dispatched).await.unwrap(), DispatchDecision::Dispatch); + fixture.mailbox.mark_dispatched(&dispatched).await.unwrap(); + + let reopened = Mailbox::open(&fixture.config).await.unwrap(); + // Opening the store alone is not authority to interrupt another native session. + assert_eq!(reopened.dispatch_state(&accepted).await.unwrap(), DispatchState::Accepted); + assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 1); + assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 1); + assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 0); + assert_eq!(reopened.dispatch_state(&accepted).await.unwrap(), DispatchState::RecoveryNeeded); + assert_eq!(reopened.dispatch_state(&dispatching).await.unwrap(), DispatchState::RecoveryNeeded); + assert_eq!(reopened.dispatch_state(&dispatched).await.unwrap(), DispatchState::Dispatched); + assert_eq!(reopened.dispatch_state(&foreign).await.unwrap(), DispatchState::Accepted); + assert_eq!(reopened.accept_work(accepted_request).await.unwrap(), accepted); + assert_eq!(reopened.accepted_input(&accepted).await.unwrap(), retained_input); + assert_eq!(reopened.begin_dispatch(&accepted).await.unwrap(), DispatchDecision::RecoveryNeeded); + assert_eq!(reopened.issued_work_refs(issuer, None, 100).await.unwrap().len(), 3); + assert!(matches!(reopened.interrupt_issuer_dispatches(issuer, 0).await, Err(MailboxError::InvalidInput(_)))); +} + #[tokio::test] async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attention() { let fixture = test_mailbox().await; diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index 77a6e11a5cd1..d1447a8acf95 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -1,6 +1,6 @@ use crate::error::{MailboxError, MailboxResult}; use crate::message_delivery::envelope_from_row; -use crate::message_rules::{validate_reply_tx, validate_replacement_tx}; +use crate::message_rules::{validate_replacement_tx, validate_reply_tx}; use crate::store::Mailbox; use crate::types::{Envelope, MessageDraft, MessageKind, WorkRef}; use crate::work::exact_work; @@ -52,7 +52,6 @@ impl Mailbox { self.notify_changed(); Ok(envelope) } - } pub(crate) async fn insert_message_tx( @@ -89,7 +88,11 @@ pub(crate) async fn insert_message_tx( })?; let communication_json = serde_json::to_string(&draft.communication)?; let body_bytes = draft.communication.content.len() - + draft.communication.encrypted_content.as_ref().map_or(0, String::len); + + draft + .communication + .encrypted_content + .as_ref() + .map_or(0, String::len); if body_bytes > MAX_MESSAGE_BYTES || request_json.len() > MAX_RECORD_BYTES || communication_json.len() > MAX_RECORD_BYTES @@ -136,7 +139,8 @@ pub(crate) async fn insert_message_tx( validate_replacement_tx(transaction, supersedes, &draft).await?; } - if draft.supersedes.is_none() && draft.kind == MessageKind::Progress && !draft.activate_if_idle { + if draft.supersedes.is_none() && draft.kind == MessageKind::Progress && !draft.activate_if_idle + { if let Some(work_ref) = draft.work_ref.as_ref() { if let Some(existing) = sqlx::query( "SELECT * FROM mailbox_messages @@ -167,24 +171,24 @@ pub(crate) async fn insert_message_tx( } } - let collision: Option = sqlx::query_scalar( - "SELECT id FROM mailbox_messages WHERE id = ? LIMIT 1", - ) - .bind(&id) - .fetch_optional(&mut **transaction) - .await?; + let collision: Option = + sqlx::query_scalar("SELECT id FROM mailbox_messages WHERE id = ? LIMIT 1") + .bind(&id) + .fetch_optional(&mut **transaction) + .await?; if collision.is_some() { return Err(MailboxError::IdempotencyConflict); } let pending_count = pending_count_tx(transaction, &draft.recipient.to_string()).await?; if pending_count >= crate::messages::MAX_PENDING_PER_RECIPIENT { - return Err(MailboxError::Backpressure("recipient pending-message count")); + return Err(MailboxError::Backpressure( + "recipient pending-message count", + )); } - let total_count: i64 = sqlx::query_scalar( - "SELECT item_count FROM mailbox_counts WHERE name = 'message'", - ) - .fetch_one(&mut **transaction) - .await?; + let total_count: i64 = + sqlx::query_scalar("SELECT item_count FROM mailbox_counts WHERE name = 'message'") + .fetch_one(&mut **transaction) + .await?; if total_count >= MAX_TOTAL_MESSAGES { return Err(MailboxError::Backpressure("retained message count")); } @@ -206,7 +210,13 @@ pub(crate) async fn insert_message_tx( .bind(serde_json::to_string(&draft.communication.recipient)?) .bind(&communication_json) .bind(kind_json) - .bind(draft.work_ref.as_ref().map(serde_json::to_string).transpose()?) + .bind( + draft + .work_ref + .as_ref() + .map(serde_json::to_string) + .transpose()?, + ) .bind(draft.work_ref.as_ref().map(|work_ref| work_ref.id.as_str())) .bind(&draft.reply_to) .bind(&draft.supersedes) @@ -220,8 +230,16 @@ pub(crate) async fn insert_message_tx( ) .bind(&id) .bind(draft.sender.to_string()) - .bind(if draft.activate_if_idle { "pending" } else { "decided" }) - .bind(if draft.activate_if_idle { None } else { Some("not_requested") }) + .bind(if draft.activate_if_idle { + "pending" + } else { + "decided" + }) + .bind(if draft.activate_if_idle { + None + } else { + Some("not_requested") + }) .execute(&mut **transaction) .await?; add_invocation_tx(transaction, &draft.invocation_key, &request_json, &id).await?; @@ -241,10 +259,10 @@ pub(crate) async fn insert_message_tx( "UPDATE mailbox_messages SET superseded_by = ? WHERE id = ? AND superseded_by IS NULL AND withdrawn = 0", ) - .bind(&id) - .bind(old_id) - .execute(&mut **transaction) - .await?; + .bind(&id) + .bind(old_id) + .execute(&mut **transaction) + .await?; if replaced.rows_affected() == 1 { if let Some(old) = old { let unread = old.try_get::("canonical")? == 0 @@ -314,13 +332,13 @@ async fn pending_count_tx( transaction: &mut Transaction<'_, Sqlite>, recipient: &str, ) -> MailboxResult { - Ok(sqlx::query_scalar( - "SELECT item_count FROM mailbox_pending_counts WHERE recipient = ?", + Ok( + sqlx::query_scalar("SELECT item_count FROM mailbox_pending_counts WHERE recipient = ?") + .bind(recipient) + .fetch_optional(&mut **transaction) + .await? + .unwrap_or(0), ) - .bind(recipient) - .fetch_optional(&mut **transaction) - .await? - .unwrap_or(0)) } async fn increment_pending_count( @@ -380,7 +398,9 @@ fn validate_id(id: &str) -> MailboxResult<()> { if id.starts_with("amsg_") && id.len() > 5 && id.len() <= 256 { Ok(()) } else { - Err(MailboxError::InvalidInput("expected stable amsg_* ID".into())) + Err(MailboxError::InvalidInput( + "expected stable amsg_* ID".into(), + )) } } diff --git a/codex-rs/agent-mailbox/src/outcome_store.rs b/codex-rs/agent-mailbox/src/outcome_store.rs index 7530a70005c7..a0ae56caa7ec 100644 --- a/codex-rs/agent-mailbox/src/outcome_store.rs +++ b/codex-rs/agent-mailbox/src/outcome_store.rs @@ -75,9 +75,7 @@ pub(crate) async fn persist_continuation( } return Ok(()); } - let count_rows = sqlx::query( - "SELECT 1 FROM mailbox_continuations WHERE work_id = ? LIMIT ?", - ) + let count_rows = sqlx::query("SELECT 1 FROM mailbox_continuations WHERE work_id = ? LIMIT ?") .bind(&work_ref.id) .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) .fetch_all(&mut **transaction) @@ -149,7 +147,11 @@ pub(crate) async fn outcome_receipt_for( .ok_or(MailboxError::ReferenceMismatch)?; let mut data: OutcomeData = serde_json::from_str(&stored)?; let response_message_id = if has_response { - output_id.or_else(|| row.try_get::, _>("outcome_id").ok().flatten()) + output_id.or_else(|| { + row.try_get::, _>("outcome_id") + .ok() + .flatten() + }) } else { None }; @@ -173,11 +175,10 @@ pub(crate) async fn stored_outcome_receipt_for( work_ref: WorkRef, ) -> MailboxResult { let row = exact_work(pool, &work_ref).await?; - let has_response = serde_json::from_str::( - &row.try_get::("outcome_json")?, - )? - .response - .is_some(); + let has_response = + serde_json::from_str::(&row.try_get::("outcome_json")?)? + .response + .is_some(); outcome_receipt_for(pool, work_ref, None, has_response).await } @@ -204,7 +205,11 @@ pub(crate) async fn continuation_receipts_for( "pending_goal_binding" => ContinuationState::PendingGoalBinding, "deferred" => ContinuationState::Deferred, "terminal" => ContinuationState::Terminal, - _ => return Err(MailboxError::InvalidInput("stored continuation state".into())), + _ => { + return Err(MailboxError::InvalidInput( + "stored continuation state".into(), + )); + } }; Ok(ContinuationReceipt { binding: serde_json::from_str(&row.try_get::("binding_json")?)?, @@ -232,6 +237,8 @@ fn parse_outcome(value: &str) -> MailboxResult { "failed" => Ok(Outcome::Failed), "cancelled" => Ok(Outcome::Cancelled), "needs_attention" => Ok(Outcome::NeedsAttention), - _ => Err(MailboxError::InvalidInput("stored continuation outcome".into())), + _ => Err(MailboxError::InvalidInput( + "stored continuation outcome".into(), + )), } } diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 3762b52525c9..1bbaa4175eed 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -112,6 +112,7 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { "CREATE INDEX IF NOT EXISTS mailbox_work_unobserved ON mailbox_work(issuer, outcome_observed, seq)", "CREATE INDEX IF NOT EXISTS mailbox_work_agent_unobserved ON mailbox_work(issuer, agent, outcome_observed, seq)", "CREATE INDEX IF NOT EXISTS mailbox_work_outcome ON mailbox_work(issuer, outcome_id, seq)", + "CREATE INDEX IF NOT EXISTS mailbox_work_inflight ON mailbox_work(issuer, dispatch_state, seq) WHERE outcome_json IS NULL", ] { sqlx::query(index).execute(&mut *transaction).await?; } diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index 665cb7613c88..c1c847a7a4a4 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -1,9 +1,9 @@ use crate::error::MailboxResult; +use sqlx::SqlitePool; use std::collections::HashMap; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; -use sqlx::SqlitePool; use tokio::sync::watch; #[derive(Clone)] diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index 8ee124665db9..a46bc7fd0fc4 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -243,6 +243,42 @@ impl Mailbox { Ok(DispatchDecision::Dispatch) } + /// The native host calls this only while resuming this exact issuer, before it can + /// accept another invocation. A previous acceptance/dispatch claim cannot be retried + /// by guessing whether its side effect happened. Retain its input and identity, but + /// expose recovery-needed instead of leaving an ownerless join pending forever. + pub async fn interrupt_issuer_dispatches( + &self, + issuer: ThreadId, + limit: usize, + ) -> MailboxResult { + if limit == 0 || limit > 100 { + return Err(MailboxError::InvalidInput( + "dispatch recovery limit must be between 1 and 100".into(), + )); + } + let _arrival = self.begin_input_arrival(); + let result = sqlx::query( + "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' + WHERE id IN ( + SELECT id FROM mailbox_work + WHERE issuer = ? AND dispatch_state IN ('accepted', 'dispatching') + AND outcome_json IS NULL + AND NOT EXISTS ( + SELECT 1 FROM mailbox_continuations AS continuation + WHERE continuation.work_id = mailbox_work.id + AND continuation.state IN ('pending_goal_binding', 'deferred') + ) + LIMIT ? + )", + ) + .bind(issuer.to_string()) + .bind(limit as i64) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() as usize) + } + /// Record adapter-confirmed stale in-flight custody for this exact work. pub async fn mark_recovery_needed(&self, work_ref: &WorkRef) -> MailboxResult<()> { let mut transaction = self.pool.begin().await?; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 31a079719164..45a005c0d92e 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -5747,6 +5747,121 @@ mod tests { } } + #[test] + fn interrupted_mailbox_wait_replays_as_terminal_with_pending_refs() { + let sender = ThreadId::new(); + let turn_id = "turn-interrupted-mailbox-wait"; + let pending = codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_exact-42".to_string()), + message_id: None, + agent: Some("worker".to_string()), + status: "running".to_string(), + }; + let active_report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["work_exact-42".to_string()], + requested_replies_to: Vec::new(), + return_when: codex_protocol::items::MailboxWaitReturnWhen::All, + completed: Vec::new(), + pending: vec![pending.clone()], + completion_reason: None, + condition_satisfied: false, + quiet_count: 1, + wake_sender: None, + wake_ref: None, + }; + let interrupted_report = codex_protocol::items::MailboxWaitReport { + completion_reason: Some(codex_protocol::items::MailboxWakeCause::Interrupted), + ..active_report.clone() + }; + let turns = build_turns_from_rollout_items(&[ + RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: turn_id.to_string(), + trace_id: None, + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + })), + canonical_collab_lifecycle_item( + sender, + turn_id, + CoreCollabAgentToolCallItem { + id: "wait-interrupted".to_string(), + tool: CoreCollabAgentTool::Wait, + status: CoreCollabAgentToolCallStatus::InProgress, + sender_thread_id: sender, + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(active_report), + }, + /*started*/ true, + ), + canonical_collab_lifecycle_item( + sender, + turn_id, + CoreCollabAgentToolCallItem { + id: "wait-interrupted".to_string(), + tool: CoreCollabAgentTool::Wait, + status: CoreCollabAgentToolCallStatus::Failed, + sender_thread_id: sender, + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(interrupted_report.clone()), + }, + /*started*/ false, + ), + ]); + + let wait = turns[0] + .items + .iter() + .find(|item| { + matches!( + item, + ThreadItem::CollabAgentToolCall { + id, + tool: CollabAgentTool::Wait, + .. + } if id == "wait-interrupted" + ) + }) + .expect("replay should materialize the interrupted mailbox wait"); + let ThreadItem::CollabAgentToolCall { + status, + mailbox_wait_report, + .. + } = wait + else { + unreachable!("wait lookup must return the collaboration wait item"); + }; + assert_eq!(*status, CollabAgentToolCallStatus::Failed); + assert_eq!(mailbox_wait_report.as_ref(), Some(&interrupted_report)); + assert_eq!( + mailbox_wait_report + .as_ref() + .expect("interrupted wait report") + .pending, + vec![pending] + ); + } + fn replay_v1_spawn_terminal_compatibility( canonical_effective_model: Option<&str>, canonical_effective_reasoning_effort: Option< diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index 0b8efbe77572..aba9c7c080a5 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -28,11 +28,11 @@ use codex_protocol::items::AgentNotificationContent as CoreAgentNotificationCont use codex_protocol::items::AgentNotificationOrigin as CoreAgentNotificationOrigin; use codex_protocol::items::AgentNotificationSummary as CoreAgentNotificationSummary; use codex_protocol::items::AgentWakeCause as CoreAgentWakeCause; -use codex_protocol::items::MailboxWaitReport as CoreMailboxWaitReport; use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool; use codex_protocol::items::CollabAgentToolCallStatus as CoreCollabAgentToolCallStatus; use codex_protocol::items::CommandExecutionStatus as CoreCommandExecutionStatus; use codex_protocol::items::DynamicToolCallStatus as CoreDynamicToolCallStatus; +use codex_protocol::items::MailboxWaitReport as CoreMailboxWaitReport; use codex_protocol::items::McpToolCallStatus as CoreMcpToolCallStatus; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation; diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 53822b511fdc..69bcffea60a0 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -63,6 +63,24 @@ impl AgentControl { self.mailbox.get().cloned() } + pub(crate) async fn mailbox_interrupt_resumed_dispatches( + &self, + issuer: ThreadId, + ) -> CodexResult<()> { + let Some(mailbox) = self.mailbox() else { + return Ok(()); + }; + // Indexed bounded batches, scoped to the exact native session being resumed. + // Other loaded issuers and producer continuation custody are untouched. + while mailbox + .interrupt_issuer_dispatches(issuer, 100) + .await + .map_err(storage_error)? + != 0 + {} + Ok(()) + } + pub(crate) async fn mailbox_work_for_key( &self, issuer: ThreadId, @@ -93,8 +111,13 @@ impl AgentControl { ) { return Ok(None); } - if let Some(native_key) = context.native_key.as_deref() - && let Some(existing) = mailbox + // Only a mailbox-enabled invocation supplies this frozen native identity. A shared + // tree may contain a feature-disabled session; opening the store elsewhere does not + // opt its stock tools into mailbox capture. + let Some(native_key) = context.native_key.as_deref() else { + return Ok(None); + }; + if let Some(existing) = mailbox .work_for_native_key(context.sender_thread_id, native_key) .await .map_err(storage_error)? @@ -103,10 +126,7 @@ impl AgentControl { } let work = mailbox .accept_work(NativeWork { - native_key: context - .native_key - .clone() - .unwrap_or_else(|| submission_id.clone()), + native_key: native_key.to_string(), issuer: context.sender_thread_id, agent, submission_id: submission_id.clone(), @@ -400,3 +420,42 @@ impl AgentControl { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use codex_protocol::AgentPath; + + #[tokio::test] + async fn mailbox_shared_control_does_not_capture_stock_invocations() { + let (session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let control = &session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let recipient = ThreadId::new(); + let mut communication = InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + "native input".to_string(), + true, + ); + let original = communication.clone(); + let mut submission_id = "stock-submission".to_string(); + let context = AgentCommunicationContext::new(AgentCommunicationKind::Followup, session.thread_id); + assert!(control.mailbox_accept_native_submission(recipient, &context, &mut communication, &mut submission_id).await.unwrap().is_none()); + assert_eq!(communication, original); + assert_eq!(submission_id, "stock-submission"); + let mailbox = control.mailbox().unwrap(); + assert!(mailbox.issued_work_refs(session.thread_id, None, 100).await.unwrap().is_empty()); + + let enabled_context = context.with_native_key("enabled-call".to_string()); + let (accepted, decision) = control.mailbox_accept_native_submission(recipient, &enabled_context, &mut communication, &mut submission_id).await.unwrap().unwrap(); + assert_eq!(decision, DispatchDecision::Dispatch); + assert_eq!(accepted.issuer, session.thread_id); + assert_eq!(mailbox.accepted_input(&accepted).await.unwrap(), communication); + assert!(communication.id.unwrap().as_str().starts_with("amsg_")); + } +} diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 75dafa215aff..81a983c4b635 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -574,10 +574,13 @@ pub async fn thread_rollback(sess: &Arc, sub_id: String, num_turns: u32 .await; } - sess.deliver_event_raw(Event { - id: turn_context.sub_id.clone(), - msg: rollback_msg, - }, None) + sess.deliver_event_raw( + Event { + id: turn_context.sub_id.clone(), + msg: rollback_msg, + }, + None, + ) .await; } diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index daf55d9e445e..cfa9da11493f 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -3,11 +3,11 @@ use crate::context::TerminalCompletionNotification; use crate::state::ActiveTurn; use crate::state::MailboxDeliveryPhase; use crate::state::TurnState; +use codex_agent_mailbox::InputArrival; +use codex_agent_mailbox::Mailbox; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::user_input::UserInput; -use codex_agent_mailbox::InputArrival; -use codex_agent_mailbox::Mailbox; use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -104,7 +104,9 @@ impl InputQueue { } fn reserve_mailbox_input(&self) -> Option { - self.durable_mailbox.get().map(|mailbox| mailbox.begin_input_arrival()) + self.durable_mailbox + .get() + .map(|mailbox| mailbox.begin_input_arrival()) } pub(crate) async fn begin_residency_activity(&self) -> OwnedMutexGuard<()> { @@ -704,7 +706,11 @@ impl InputQueue { turn_state: &Mutex, input: Vec, ) { - let _input_arrival = if input.is_empty() { None } else { self.reserve_mailbox_input() }; + let _input_arrival = if input.is_empty() { + None + } else { + self.reserve_mailbox_input() + }; let activity = if input .iter() .any(|input| matches!(input, TurnInput::UserInput { .. })) @@ -731,7 +737,11 @@ impl InputQueue { turn_state: &Mutex, input: Vec, ) { - let _input_arrival = if input.is_empty() { None } else { self.reserve_mailbox_input() }; + let _input_arrival = if input.is_empty() { + None + } else { + self.reserve_mailbox_input() + }; turn_state.lock().await.pending_input.items.extend(input); } @@ -917,18 +927,52 @@ mod tests { use codex_utils_absolute_path::test_support::PathExt; let home = tempfile::tempdir().unwrap(); - let mailbox = Arc::new(Mailbox::open(&codex_state::SqliteConfig::new_for_testing(home.path().abs())).await.unwrap()); + let mailbox = Arc::new( + Mailbox::open(&codex_state::SqliteConfig::new_for_testing( + home.path().abs(), + )) + .await + .unwrap(), + ); let queue = InputQueue::new(); queue.bind_durable_mailbox(Arc::clone(&mailbox)); let (a, b) = (ThreadId::new(), ThreadId::new()); let generation = mailbox.wait_input_generation(); - let _old = mailbox.begin_wait_at("old-a".into(), a, vec![b], ReturnWhen::Any, false, generation).unwrap(); - queue.enqueue_mailbox_communication(make_mail(AgentPath::root(), AgentPath::root(), "native input", true)).await; + let _old = mailbox + .begin_wait_at( + "old-a".into(), + a, + vec![b], + ReturnWhen::Any, + false, + generation, + ) + .unwrap(); + queue + .enqueue_mailbox_communication(make_mail( + AgentPath::root(), + AgentPath::root(), + "native input", + true, + )) + .await; assert!(queue.has_pending_mailbox_items().await); - assert!(matches!(mailbox.begin_wait_at("stale-b".into(), b, vec![a], ReturnWhen::All, false, generation), Err(MailboxError::StaleWaitSnapshot))); + assert!(matches!( + mailbox.begin_wait_at( + "stale-b".into(), + b, + vec![a], + ReturnWhen::All, + false, + generation + ), + Err(MailboxError::StaleWaitSnapshot) + )); // Native input invalidated the older edge before publication, so a peer cannot // incorrectly prove that both owners remain blocked on one another. - let _peer = mailbox.begin_wait("peer-b".into(), b, vec![a], ReturnWhen::All, false).unwrap(); + let _peer = mailbox + .begin_wait("peer-b".into(), b, vec![a], ReturnWhen::All, false) + .unwrap(); } #[tokio::test] diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index e41b264aea56..02ad660f3c5d 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -103,12 +103,18 @@ use codex_protocol::config_types::Settings; use codex_protocol::config_types::WebSearchMode; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; +use codex_protocol::items::CollabAgentTool; +use codex_protocol::items::CollabAgentToolCallItem; +use codex_protocol::items::CollabAgentToolCallStatus; use codex_protocol::items::EnteredReviewModeItem; +use codex_protocol::items::MailboxWaitReport; +use codex_protocol::items::MailboxWakeCause; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; use codex_protocol::models::ActivePermissionProfile; use codex_protocol::models::AdditionalPermissionProfile; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::PermissionProfile; use codex_protocol::models::SandboxEnforcement; use codex_protocol::models::format_allow_prefixes; @@ -1439,6 +1445,209 @@ impl Session { state.clear_connector_selection(); } + async fn repair_interrupted_mailbox_waits( + &self, + conversation_history: InitialHistory, + ) -> CodexResult { + let mut resumed_history = match conversation_history { + InitialHistory::Resumed(resumed_history) => resumed_history, + other => return Ok(other), + }; + if resumed_history.conversation_id != self.thread_id { + return Err(CodexErr::Fatal( + "resumed mailbox-wait repair targeted a different thread".to_string(), + )); + } + + // Reconcile only lifecycle items already present in this exact resumed history. A + // mailbox wait has no durable timer or subscription to re-arm; its original provider + // call is closed with a tool result so it cannot be replayed as the same call. + let mut rollout_items = Arc::unwrap_or_clone(resumed_history.history); + let mut wait_states = HashMap::<(String, String), (CollabAgentToolCallItem, bool)>::new(); + let mut wait_order = Vec::new(); + let mut wait_call_ids = HashSet::::new(); + let mut wait_outputs = HashMap::::new(); + let mut saw_interrupted_terminal = false; + for rollout_item in &rollout_items { + match rollout_item { + RolloutItem::EventMsg(EventMsg::ItemStarted(event)) => { + let completed = false; + let TurnItem::CollabAgentToolCall(call) = &event.item else { + continue; + }; + if call.tool != CollabAgentTool::Wait + || call.mailbox_wait_report.is_none() + { + continue; + } + saw_interrupted_terminal |= call + .mailbox_wait_report + .as_ref() + .and_then(|report| report.completion_reason) + == Some(MailboxWakeCause::Interrupted); + let key = (event.turn_id.clone(), call.id.clone()); + if !wait_states.contains_key(&key) { + wait_order.push(key.clone()); + } + wait_states.insert(key, (call.clone(), completed)); + } + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => { + let completed = true; + let TurnItem::CollabAgentToolCall(call) = &event.item else { + continue; + }; + if call.tool != CollabAgentTool::Wait + || call.mailbox_wait_report.is_none() + { + continue; + } + let key = (event.turn_id.clone(), call.id.clone()); + if !wait_states.contains_key(&key) { + wait_order.push(key.clone()); + } + wait_states.insert(key, (call.clone(), completed)); + } + RolloutItem::ResponseItem(ResponseItem::FunctionCall { + call_id, + name, + .. + }) if name == "wait_agent" => { + wait_call_ids.insert(call_id.clone()); + } + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + call_id, output, .. + }) => { + if let Some(text) = output.body.to_text() { + wait_outputs.insert(call_id.clone(), text); + } + } + _ => {} + } + } + saw_interrupted_terminal |= wait_call_ids.iter().any(|call_id| { + wait_outputs + .get(call_id) + .and_then(|output| serde_json::from_str::(output).ok()) + .and_then(|report| report.completion_reason) + == Some(MailboxWakeCause::Interrupted) + }); + + let mut repair_items = Vec::new(); + let mut repair_events = Vec::new(); + for (turn_id, call_id) in wait_order { + let Some((mut call, already_completed)) = + wait_states.remove(&(turn_id.clone(), call_id.clone())) + else { + continue; + }; + + let report = match wait_outputs.get(&call_id) { + Some(output) => { + let report = serde_json::from_str::(output).map_err(|_| { + CodexErr::Fatal( + "cannot reconcile an unfinished mailbox wait with its stored tool result" + .to_string(), + ) + })?; + if report.completion_reason.is_none() { + return Err(CodexErr::Fatal( + "stored mailbox wait tool result has no terminal completion reason" + .to_string(), + )); + } + if !already_completed { + call.status = if report.completion_reason + == Some(MailboxWakeCause::Interrupted) + { + CollabAgentToolCallStatus::Failed + } else { + CollabAgentToolCallStatus::Completed + }; + } + report + } + None => { + let mut report = call + .mailbox_wait_report + .clone() + .expect("mailbox wait states always carry their report"); + if !already_completed { + report.completion_reason = Some(MailboxWakeCause::Interrupted); + report.condition_satisfied = false; + call.status = CollabAgentToolCallStatus::Failed; + } else if report.completion_reason.is_none() { + return Err(CodexErr::Fatal( + "completed mailbox wait has no terminal completion reason".to_string(), + )); + } + report + } + }; + if !already_completed { + call.mailbox_wait_report = Some(report.clone()); + let completed_event = EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: self.thread_id, + turn_id: turn_id.clone(), + item: TurnItem::CollabAgentToolCall(call), + completed_at_ms: now_unix_timestamp_ms(), + }); + repair_events.push((turn_id, completed_event.clone())); + repair_items.push(RolloutItem::EventMsg(completed_event)); + } + + if !wait_outputs.contains_key(&call_id) && wait_call_ids.contains(&call_id) { + let output_text = serde_json::to_string(&report).map_err(|error| { + CodexErr::Fatal(format!( + "failed to serialize recovered mailbox wait result: {error}" + )) + })?; + let mut output = FunctionCallOutputPayload::from_text(output_text); + output.success = Some( + report.completion_reason != Some(MailboxWakeCause::Interrupted), + ); + let mut output_item = ResponseItem::FunctionCallOutput { + id: Some(ResponseItemId::with_suffix( + "fco", + format!("mailbox-interrupted:{call_id}"), + )), + call_id, + output, + internal_chat_message_metadata_passthrough: None, + }; + output_item.set_turn_id_if_missing(&turn_id); + repair_items.push(RolloutItem::ResponseItem(output_item)); + } + } + + if !repair_items.is_empty() || saw_interrupted_terminal { + if self.live_thread().is_none() { + return Err(CodexErr::Fatal( + "cannot durably repair an interrupted mailbox wait without thread persistence" + .to_string(), + )); + } + if !repair_items.is_empty() { + self.try_persist_rollout_items(&repair_items).await?; + } + self.flush_rollout().await?; + for (turn_id, msg) in repair_events { + self.send_event_raw_with_persistence( + Event { id: turn_id, msg }, + /*persist*/ false, + ) + .await; + } + if !repair_items.is_empty() { + rollout_items.extend(repair_items); + } + } + + self.refresh_mailbox_interrupted_wait_receipts(&rollout_items) + .await; + resumed_history.history = Arc::new(rollout_items); + Ok(InitialHistory::Resumed(resumed_history)) + } + async fn record_initial_history(&self, conversation_history: InitialHistory) { let (is_subagent, is_paginated_subagent) = { let state = self.state.lock().await; @@ -3283,6 +3492,108 @@ impl Session { ids } + pub(super) fn mailbox_interrupted_wait_receipts( + initial_history: &InitialHistory, + ) -> HashMap { + let items: &[RolloutItem] = match initial_history { + InitialHistory::New | InitialHistory::Cleared => return HashMap::new(), + InitialHistory::Resumed(resumed) => resumed.history.as_slice(), + InitialHistory::Forked(items) => items, + }; + Self::mailbox_interrupted_wait_receipts_from_items(items) + } + + fn mailbox_interrupted_wait_receipts_from_items( + items: &[RolloutItem], + ) -> HashMap { + let mut wait_call_ids = HashSet::new(); + let mut output_receipts = HashMap::new(); + let mut receipts = HashMap::new(); + for item in items { + match item { + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => { + if let TurnItem::CollabAgentToolCall(call) = &event.item + && call.tool == CollabAgentTool::Wait + && let Some(report) = call.mailbox_wait_report.as_ref() + && report.completion_reason == Some(MailboxWakeCause::Interrupted) + { + receipts.insert(call.id.clone(), report.clone()); + } + } + RolloutItem::ResponseItem(response_item) => { + Self::record_interrupted_wait_response_item( + response_item, + &mut wait_call_ids, + &mut output_receipts, + ); + } + RolloutItem::Compacted(compacted) => { + for response_item in compacted.replacement_history.as_deref().unwrap_or_default() + { + Self::record_interrupted_wait_response_item( + response_item, + &mut wait_call_ids, + &mut output_receipts, + ); + } + } + _ => {} + } + } + for call_id in wait_call_ids { + if let Some(report) = output_receipts.remove(&call_id) { + receipts.insert(call_id, report); + } + } + receipts + } + + fn record_interrupted_wait_response_item( + item: &ResponseItem, + wait_call_ids: &mut HashSet, + output_receipts: &mut HashMap, + ) { + match item { + ResponseItem::FunctionCall { call_id, name, .. } if name == "wait_agent" => { + wait_call_ids.insert(call_id.clone()); + } + ResponseItem::FunctionCallOutput { call_id, output, .. } => { + let Some(report) = output + .body + .to_text() + .and_then(|text| serde_json::from_str::(&text).ok()) + .filter(|report| { + report.completion_reason == Some(MailboxWakeCause::Interrupted) + }) + else { + return; + }; + output_receipts.insert(call_id.clone(), report); + } + _ => {} + } + } + + pub(crate) async fn mailbox_interrupted_wait_receipt( + &self, + call_id: &str, + ) -> Option { + self.mailbox_interrupted_wait_receipts + .lock() + .await + .get(call_id) + .cloned() + } + + /// Rebuilds the in-memory same-call guard from canonical rollout history. + pub(crate) async fn refresh_mailbox_interrupted_wait_receipts( + &self, + items: &[RolloutItem], + ) { + *self.mailbox_interrupted_wait_receipts.lock().await = + Self::mailbox_interrupted_wait_receipts_from_items(items); + } + pub(crate) fn response_item_from_user_input(&self, input: Vec) -> ResponseItem { ResponseItem::from(ResponseInputItem::from_user_input( input, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 6f2f09fc7b20..a7bf3f390992 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -61,6 +61,9 @@ pub(crate) struct Session { pub(crate) services: SessionServices, /// Stable IDs of canonical native agent messages, retained even if model history is compacted. pub(super) mailbox_canonical_response_item_ids: Mutex>, + /// Terminal interrupted wait receipts, derived from canonical history and retained through + /// model-history compaction so re-emitting an old call ID cannot re-arm its deadline. + pub(super) mailbox_interrupted_wait_receipts: Mutex>, /// IDs whose append/flush had an indeterminate persistence result. They are never acked or /// retried in-place, because the canonical writer may have committed a partial append. pub(super) mailbox_unsettled_response_item_ids: Mutex>, @@ -589,6 +592,8 @@ impl Session { if let InitialHistory::Forked(items) = &mut initial_history { Self::assign_missing_rollout_response_item_ids(items); } + let mailbox_interrupted_wait_receipts = + Session::mailbox_interrupted_wait_receipts(&initial_history); let multi_agent_version = multi_agent_version.map(OnceLock::from).unwrap_or_default(); let initial_multi_agent_version = multi_agent_version.get().copied(); @@ -627,6 +632,14 @@ impl Session { .unwrap_or(usize::MAX), ); agent_control.mailbox_initialize(&config).await?; + if config.multi_agent_version_from_features() == MultiAgentVersion::V2 + && config.features.enabled(Feature::AgentMailbox) + && matches!(&initial_history, InitialHistory::Resumed(_)) + { + agent_control + .mailbox_interrupt_resumed_dispatches(thread_id) + .await?; + } let mailbox_canonical_response_item_ids = if agent_control.mailbox_enabled() { Session::canonical_agent_message_ids(&initial_history) } else { @@ -1242,6 +1255,9 @@ impl Session { mailbox_canonical_response_item_ids: Mutex::new( mailbox_canonical_response_item_ids, ), + mailbox_interrupted_wait_receipts: Mutex::new( + mailbox_interrupted_wait_receipts, + ), mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), mailbox_custody_gate: Mutex::new(()), git_enrichment_policy, @@ -1340,6 +1356,10 @@ impl Session { InitialHistory::Cleared => codex_hooks::SessionStartSource::Clear, }; + // A resumed mailbox wait has no surviving subscription/deadline. Repair its exact + // canonical call before rebuilding model history or exposing the session as ready. + let initial_history = + Box::pin(sess.repair_interrupted_mailbox_waits(initial_history)).await?; // record_initial_history can emit events. We record only after the SessionConfiguredEvent is emitted. Box::pin(sess.record_initial_history(initial_history)).await; if matches!(&sess.fork_persistence, ForkPersistence::Referenced { .. }) { diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index c49fe8fba5c3..8638dc8e24b4 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2073,6 +2073,240 @@ async fn record_initial_history_reconstructs_resumed_transcript() { assert_eq!(expected, history.raw_items()); } +#[tokio::test] +async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { + let (mut session, _turn_context, rx) = make_session_and_context_with_rx().await; + let fault_store = Arc::new(FaultInjectingThreadStore::new(Arc::clone( + &session.services.thread_store, + ))); + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .services + .thread_store = fault_store.clone(); + let rollout_path = attach_thread_persistence( + Arc::get_mut(&mut session).expect("session should be uniquely owned"), + ) + .await; + let turn_id = "turn-with-interrupted-mailbox-wait".to_string(); + let call_id = "wait-call-interrupted".to_string(); + let pending = codex_protocol::items::MailboxWaitSelectionItem { + work_ref: Some("work_exact-42".to_string()), + message_id: None, + agent: Some("worker".to_string()), + status: "running".to_string(), + }; + let report = codex_protocol::items::MailboxWaitReport { + requested_targets: vec!["work_exact-42".to_string()], + requested_replies_to: Vec::new(), + return_when: codex_protocol::items::MailboxWaitReturnWhen::All, + completed: Vec::new(), + pending: vec![pending.clone()], + completion_reason: None, + condition_satisfied: false, + quiet_count: 2, + wake_sender: None, + wake_ref: None, + }; + let started_item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: call_id.clone(), + tool: CollabAgentTool::Wait, + status: CollabAgentToolCallStatus::InProgress, + sender_thread_id: session.thread_id, + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(report), + }); + let initial_items = vec![ + RolloutItem::ResponseItem(ResponseItem::FunctionCall { + id: None, + name: "wait_agent".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: call_id.clone(), + internal_chat_message_metadata_passthrough: None, + }), + RolloutItem::EventMsg(EventMsg::ItemStarted(ItemStartedEvent { + thread_id: session.thread_id, + turn_id: turn_id.clone(), + item: started_item, + started_at_ms: 10, + })), + ]; + session + .try_persist_rollout_items(&initial_items) + .await + .expect("initial wait history should append"); + session.flush_rollout().await.expect("initial wait history should flush"); + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read initial rollout history") + else { + panic!("expected resumed history"); + }; + fault_store + .fail_flush_once + .store(true, std::sync::atomic::Ordering::SeqCst); + assert!( + session + .repair_interrupted_mailbox_waits(InitialHistory::Resumed(resumed)) + .await + .is_err(), + "repair must propagate a failed durability barrier" + ); + assert!( + rx.try_recv().is_err(), + "an interrupted completion must not be published before its flush" + ); + + let InitialHistory::Resumed(resumed_after_failed_flush) = + RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("reload history after failed durability barrier") + else { + panic!("expected resumed history after failed durability barrier"); + }; + session + .repair_interrupted_mailbox_waits(InitialHistory::Resumed(resumed_after_failed_flush)) + .await + .expect("retry should finish or reconcile a partially persisted repair"); + let InitialHistory::Resumed(persisted_repair) = + RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read repaired rollout history") + else { + panic!("expected resumed history after successful repair"); + }; + let repaired = InitialHistory::Resumed(persisted_repair); + let InitialHistory::Resumed(repaired_history) = &repaired else { + panic!("repair should preserve resumed history"); + }; + let repaired_event = repaired_history + .history + .iter() + .find_map(|item| match item { + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) + if event.item.id() == call_id => + { + Some(event) + } + _ => None, + }) + .expect("interrupted wait should have a terminal item event"); + let TurnItem::CollabAgentToolCall(repaired_call) = &repaired_event.item else { + panic!("expected a collaboration wait item"); + }; + assert_eq!(repaired_call.status, CollabAgentToolCallStatus::Failed); + let repaired_report = repaired_call + .mailbox_wait_report + .as_ref() + .expect("terminal wait should keep its structured report"); + assert_eq!( + repaired_report.completion_reason, + Some(codex_protocol::items::MailboxWakeCause::Interrupted) + ); + assert!(!repaired_report.condition_satisfied); + assert_eq!(repaired_report.pending, vec![pending.clone()]); + + let repaired_output_count = repaired_history + .history + .iter() + .filter(|item| { + matches!( + item, + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + call_id: output_call_id, + .. + }) if output_call_id == &call_id + ) + }) + .count(); + assert_eq!(repaired_output_count, 1, "same call must have one durable result"); + let output_text = repaired_history + .history + .iter() + .find_map(|item| match item { + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + call_id: output_call_id, + output, + .. + }) if output_call_id == &call_id => output.body.to_text(), + _ => None, + }) + .expect("same provider call should have a durable result"); + let output_report = serde_json::from_str::( + &output_text, + ) + .expect("durable provider result should preserve the structured wait report"); + assert_eq!( + output_report.completion_reason, + Some(codex_protocol::items::MailboxWakeCause::Interrupted) + ); + assert!(!output_report.condition_satisfied); + assert_eq!(output_report.pending, vec![pending.clone()]); + + let replayed = session + .repair_interrupted_mailbox_waits(repaired.clone()) + .await + .expect("replaying repaired history should be a no-op"); + session.record_initial_history(replayed.clone()).await; + let InitialHistory::Resumed(replayed_history) = replayed else { + panic!("replay should preserve resumed history"); + }; + let terminal_events = replayed_history + .history + .iter() + .filter(|item| { + matches!( + item, + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) + if event.item.id() == call_id + ) + }) + .count(); + let durable_outputs = replayed_history + .history + .iter() + .filter(|item| { + matches!( + item, + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + call_id: output_call_id, + .. + }) if output_call_id == &call_id + ) + }) + .count(); + assert_eq!(terminal_events, 1); + assert_eq!(durable_outputs, 1); + + let model_history = session.clone_history().await; + assert_eq!( + model_history + .raw_items() + .iter() + .filter(|item| { + matches!( + item, + ResponseItem::FunctionCallOutput { call_id: output_call_id, .. } + if output_call_id == &call_id + ) + }) + .count(), + 1, + "resume model history must not replay the same wait call" + ); +} + #[tokio::test] async fn record_conversation_items_stamps_missing_turn_id_and_preserves_existing_turn_id() { let (session, turn_context) = make_session_and_context().await; @@ -2502,7 +2736,10 @@ async fn canonical_append_and_flush_failures_do_not_acknowledge_mailbox_messages let fault_store = Arc::new(FaultInjectingThreadStore::new(Arc::clone( &session.services.thread_store, ))); - session.services.thread_store = fault_store.clone(); + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .services + .thread_store = fault_store.clone(); attach_thread_persistence(Arc::get_mut(&mut session).expect("unique session")).await; let append_communication = accept_mailbox_note( @@ -2604,7 +2841,10 @@ async fn disabled_turn_does_not_settle_shared_mailbox_state() { }, ) .await; - disabled_session.services.agent_control = shared_control; + Arc::get_mut(&mut disabled_session) + .expect("disabled session should be uniquely owned") + .services + .agent_control = shared_control; let communication = accept_mailbox_note( mailbox.as_ref(), ThreadId::new(), @@ -5094,7 +5334,7 @@ async fn open_thread_persistence(session: &mut Session) -> PathBuf { .expect("thread should have rollout path") } -async fn attach_thread_persistence(session: &mut Session) -> PathBuf { +pub(crate) async fn attach_thread_persistence(session: &mut Session) -> PathBuf { let rollout_path = open_thread_persistence(session).await; session.ensure_rollout_materialized().await; session @@ -6405,6 +6645,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(), services, mailbox_canonical_response_item_ids: Mutex::new(HashSet::new()), + mailbox_interrupted_wait_receipts: Mutex::new(HashMap::new()), mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), mailbox_custody_gate: Mutex::new(()), git_enrichment_policy: GitEnrichmentPolicy::Fresh, @@ -8645,6 +8886,7 @@ where guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(), services, mailbox_canonical_response_item_ids: Mutex::new(HashSet::new()), + mailbox_interrupted_wait_receipts: Mutex::new(HashMap::new()), mailbox_unsettled_response_item_ids: Mutex::new(HashSet::new()), mailbox_custody_gate: Mutex::new(()), git_enrichment_policy: GitEnrichmentPolicy::Fresh, diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index edffcea72aa4..b1ca3cd8e2b6 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -304,14 +304,18 @@ pub(crate) async fn run_turn( .record_step_world_state_if_changed(&world_state, step_context.as_ref()) .await; - let mailbox_enabled_for_turn = turn_context.multi_agent_version == MultiAgentVersion::V2 + let mailbox_enabled_for_turn = turn_context.multi_agent_version + == MultiAgentVersion::V2 && turn_context.config.features.enabled(Feature::AgentMailbox); if mailbox_enabled_for_turn { - for communication in sess.services.agent_control + for communication in sess + .services + .agent_control .mailbox_automatic_inputs(sess.thread_id, &turn_context.sub_id) .await? { - sess.record_inter_agent_communication(&turn_context, communication).await?; + sess.record_inter_agent_communication(&turn_context, communication) + .await?; } } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 00a6dbbe0e08..da5a8ed4c361 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -639,13 +639,22 @@ impl Session { ) .await; } else { - record_pending_input( + if let Err(error) = record_pending_input( self, &turn_context, pending_input_item, hook_outcome.additional_contexts, ) - .await; + .await + { + // Finalization is not execution admission. Accepted mailbox input + // retains its durable custody when canonical history cannot take it. + if let Some(mailbox) = self.services.agent_control.mailbox() { + mailbox.notify_runtime_fault(self.thread_id); + } + let error_event = error.to_error_event(/*message_prefix*/ None); + self.send_event(&turn_context, EventMsg::Error(error_event)).await; + } } } } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 13b34479a8bb..d4ce909bee02 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1420,7 +1420,7 @@ impl ThreadManager { } } - let Some(parent_agent_path) = parent_agent_path else { + let Some(_parent_agent_path) = parent_agent_path else { return Ok(true); }; let Some(communication) = communication else { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 34b4d9cae06e..52ac45cc3aac 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -1131,7 +1131,7 @@ fn mailbox_wait_output_schema() -> Value { }, "completion_reason": { "type": "string", - "enum": ["result", "reply", "request", "human", "tool", "deadline", "cancel", "subscription_loss", "runtime_fault", "no_matching_work", "already_satisfied", "cycle", "needs_attention"], + "enum": ["result", "reply", "request", "human", "tool", "deadline", "cancel", "interrupted", "subscription_loss", "runtime_fault", "no_matching_work", "already_satisfied", "cycle", "needs_attention"], "description": "Actual reason the wait returned. Attention and interruption do not themselves satisfy an outstanding join." }, "condition_satisfied": { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 84aaf662b753..7f6563ed5b48 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -73,6 +73,12 @@ impl CoreToolRuntime for Handler { async fn handle_wait(invocation: ToolInvocation) -> Result, FunctionCallError> { let ToolInvocation { session, turn, payload, call_id, cancellation_token, .. } = invocation; + if let Some(report) = session.mailbox_interrupted_wait_receipt(&call_id).await { + return Ok(boxed_tool_output(FunctionToolOutput::from_text( + tool_output_json_text(&report, "wait_agent"), + Some(false), + ))); + } let args: WaitArgs = parse_arguments(&function_arguments(payload)?)?; let _compatibility_only = args.native_event_wait; let mailbox = mailbox_for(&session)?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index 053dab602be3..2ca72a290143 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -9,7 +9,15 @@ use codex_agent_mailbox::NativeWork; use codex_agent_mailbox::WorkRef; use codex_features::Feature; use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ResponseItem; use codex_protocol::models::ResponseInputItem; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::InitialHistory; +use codex_protocol::protocol::ItemCompletedEvent; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::TurnItem; +use std::path::PathBuf; use std::sync::Arc; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; @@ -31,6 +39,36 @@ async fn fixture() -> (Arc, Arc, Arc, ThreadId) { (Arc::new(session), Arc::new(turn), mailbox, ThreadId::new()) } +async fn durable_fixture() -> ( + Arc, + Arc, + Arc, + ThreadId, + PathBuf, +) { + let (mut session, mut turn) = crate::session::tests::make_session_and_context().await; + let rollout_path = crate::session::tests::attach_thread_persistence(&mut session).await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + turn.multi_agent_version = config.multi_agent_version_from_features(); + turn.config = Arc::new(config); + session + .services + .agent_control + .mailbox_initialize(&turn.config) + .await + .unwrap(); + let mailbox = session.services.agent_control.mailbox().unwrap(); + ( + Arc::new(session), + Arc::new(turn), + mailbox, + ThreadId::new(), + rollout_path, + ) +} + async fn work( mailbox: &Mailbox, issuer: ThreadId, @@ -76,9 +114,24 @@ async fn wait( targets: Vec, mode: &str, ) -> MailboxWaitReport { + wait_with_call_id( + session, + turn, + serde_json::json!({ "targets": targets, "return_when": mode, "timeout_ms": 0 }), + format!("replay-{mode}"), + ) + .await + .0 +} + +async fn wait_with_call_id( + session: Arc, + turn: Arc, + arguments: serde_json::Value, + call_id: String, +) -> (MailboxWaitReport, Option) { let payload = ToolPayload::Function { - arguments: serde_json::json!({ "targets": targets, "return_when": mode, "timeout_ms": 0 }) - .to_string(), + arguments: arguments.to_string(), }; let output = handle_wait(ToolInvocation { session, @@ -86,7 +139,7 @@ async fn wait( turn, cancellation_token: CancellationToken::new(), tracker: Arc::new(Mutex::new(TurnDiffTracker::default())), - call_id: format!("replay-{mode}"), + call_id, tool_name: ToolName::plain("wait_agent"), source: ToolCallSource::Direct, payload: payload.clone(), @@ -98,10 +151,128 @@ async fn wait( else { panic!("expected a function result") }; + let success = output.success; let FunctionCallOutputBody::Text(body) = output.body else { panic!("expected report text") }; - serde_json::from_str(&body).unwrap() + (serde_json::from_str(&body).unwrap(), success) +} + +async fn record_result( + mailbox: &Mailbox, + work_ref: &WorkRef, + author: AgentPath, + body: &str, +) -> String { + mailbox + .record_outcome( + work_ref, + Outcome::Completed, + Some(InterAgentCommunication::new( + author, + AgentPath::root(), + Vec::new(), + body.to_string(), + false, + )), + Continuation::None, + ) + .await + .unwrap() + .id +} + +async fn assert_work_observed(mailbox: &Mailbox, recipient: ThreadId, work_ref: &WorkRef) { + let snapshot = mailbox + .snapshot(&Selection { + caller: recipient, + agent: None, + selectors: vec![Selector::Work(work_ref.id.clone())], + return_when: ReturnWhen::All, + limit: 100, + }) + .await + .unwrap(); + let [SnapshotItem::Work(work)] = snapshot.items.as_slice() else { + panic!("expected exact work snapshot") + }; + assert!(work.observed); +} + +async fn assert_persisted_message(rollout_path: &std::path::Path, id: &str, body: &str) { + let InitialHistory::Resumed(resumed) = + crate::rollout::recorder::RolloutRecorder::get_rollout_history(rollout_path) + .await + .unwrap() + else { + panic!("expected durable resumed rollout history") + }; + assert!(resumed.history.iter().any(|item| match item { + RolloutItem::ResponseItem(response) => { + response.id().is_some_and(|response_id| response_id.as_str() == id) + && serde_json::to_value(response) + .unwrap() + .to_string() + .contains(body) + } + _ => false, + })); +} + +async fn seed_interrupted_wait_receipt( + session: &Session, + call_id: &str, + report: MailboxWaitReport, +) { + // This exact call/output/terminal-event trio is the canonical history shape consumed by + // resume recovery; refresh the same derived receipt cache used by that path. + let terminal_item = TurnItem::CollabAgentToolCall( + codex_protocol::protocol::CollabAgentToolCallItem { + id: call_id.to_string(), + tool: codex_protocol::protocol::CollabAgentTool::Wait, + status: codex_protocol::protocol::CollabAgentToolCallStatus::Failed, + sender_thread_id: session.thread_id, + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: Default::default(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(report.clone()), + }, + ); + let items = [ + RolloutItem::ResponseItem(ResponseItem::FunctionCall { + id: None, + name: "wait_agent".to_string(), + namespace: None, + arguments: serde_json::json!({}).to_string(), + call_id: call_id.to_string(), + internal_chat_message_metadata_passthrough: None, + }), + RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { + id: None, + call_id: call_id.to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text(serde_json::to_string(&report).unwrap()), + success: Some(false), + }, + }), + RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: session.thread_id, + turn_id: "interrupted-wait-test".to_string(), + item: terminal_item, + completed_at_ms: 1, + })), + ]; + session + .refresh_mailbox_interrupted_wait_receipts(&items) + .await; } #[tokio::test] @@ -193,3 +364,337 @@ async fn unrelated_attention_does_not_erase_a_successful_selected_join() { ); } } + +#[tokio::test] +async fn wait_delivers_unobserved_result_before_acknowledging_observation() { + let (session, turn, mailbox, agent, rollout_path) = durable_fixture().await; + let work_ref = work(&mailbox, session.thread_id, agent, "deliver-result", None).await; + let body = "durable result body"; + let result_id = record_result( + &mailbox, + &work_ref, + AgentPath::from_string("/root/worker".to_string()).unwrap(), + body, + ) + .await; + let before = mailbox + .snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: vec![Selector::Work(work_ref.id.clone())], + return_when: ReturnWhen::All, + limit: 100, + }) + .await + .unwrap(); + let [SnapshotItem::Work(before)] = before.items.as_slice() else { + panic!("expected exact unobserved work snapshot") + }; + assert!(!before.observed); + assert!( + !mailbox + .get_message(session.thread_id, &result_id) + .await + .unwrap() + .unwrap() + .canonical + ); + + let (report, success) = wait_with_call_id( + Arc::clone(&session), + Arc::clone(&turn), + serde_json::json!({ + "targets": [work_ref.id.clone()], + "return_when": "all", + "timeout_ms": 0 + }), + "deliver-unobserved-result".to_string(), + ) + .await; + + assert_eq!(report.completion_reason, Some(MailboxWakeCause::Result)); + assert!(report.condition_satisfied); + assert_eq!(success, Some(true)); + assert!( + mailbox + .get_message(session.thread_id, &result_id) + .await + .unwrap() + .unwrap() + .canonical + ); + assert_work_observed(&mailbox, session.thread_id, &work_ref).await; + assert_persisted_message(&rollout_path, &result_id, body).await; +} + +#[tokio::test] +async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { + let (session, turn, mailbox, agent, rollout_path) = durable_fixture().await; + let worker_path = AgentPath::from_string("/root/worker".to_string()).unwrap(); + let request = mailbox + .accept_message(MessageDraft { + invocation_key: "quiet-reply-request".to_string(), + sender: session.thread_id, + recipient: agent, + communication: InterAgentCommunication::new( + AgentPath::root(), + worker_path.clone(), + Vec::new(), + "request requiring reply".to_string(), + false, + ), + kind: MessageKind::Request, + activate_if_idle: false, + work_ref: None, + reply_to: None, + supersedes: None, + legacy_attention: false, + }) + .await + .unwrap(); + let body = "quiet exact reply body"; + let reply = mailbox + .accept_message(MessageDraft { + invocation_key: "quiet-reply-answer".to_string(), + sender: agent, + recipient: session.thread_id, + communication: InterAgentCommunication::new( + worker_path, + AgentPath::root(), + Vec::new(), + body.to_string(), + false, + ), + kind: MessageKind::Note, + activate_if_idle: false, + work_ref: None, + reply_to: Some(request.id.clone()), + supersedes: None, + legacy_attention: false, + }) + .await + .unwrap(); + assert!(!reply.canonical); + assert_eq!(mailbox.quiet_count(session.thread_id).await.unwrap(), 1); + assert!(mailbox.attention(session.thread_id, 100).await.unwrap().is_empty()); + let before = mailbox + .snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: vec![Selector::Reply(request.id.clone())], + return_when: ReturnWhen::All, + limit: 100, + }) + .await + .unwrap(); + let [SnapshotItem::Request(before)] = before.items.as_slice() else { + panic!("expected exact unobserved request/reply snapshot") + }; + assert!(!before.observed); + + let (report, success) = wait_with_call_id( + Arc::clone(&session), + Arc::clone(&turn), + serde_json::json!({ + "replies_to": [request.id.clone()], + "return_when": "all", + "timeout_ms": 0 + }), + "deliver-quiet-reply".to_string(), + ) + .await; + + assert_eq!(report.completion_reason, Some(MailboxWakeCause::Reply)); + assert!(report.condition_satisfied); + assert_eq!(report.quiet_count, 1); + assert_eq!(success, Some(true)); + assert!(mailbox.attention(session.thread_id, 100).await.unwrap().is_empty()); + assert!( + mailbox + .get_message(session.thread_id, &reply.id) + .await + .unwrap() + .unwrap() + .canonical + ); + let snapshot = mailbox + .snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: vec![Selector::Reply(request.id)], + return_when: ReturnWhen::All, + limit: 100, + }) + .await + .unwrap(); + let [SnapshotItem::Request(request)] = snapshot.items.as_slice() else { + panic!("expected exact request/reply snapshot") + }; + assert!(request.observed); + assert_eq!( + request.reply.as_ref().map(|reply| reply.id.as_str()), + Some(reply.id.as_str()) + ); + assert_persisted_message(&rollout_path, &reply.id, body).await; +} + +#[tokio::test] +async fn wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delivers_them() { + let (session, turn, mailbox, agent, rollout_path) = durable_fixture().await; + let worker_path = AgentPath::from_string("/root/worker".to_string()).unwrap(); + let mut work_refs = Vec::new(); + let mut result_ids = Vec::new(); + let mut bodies = Vec::new(); + for index in 0..3 { + let work_ref = work( + &mailbox, + session.thread_id, + agent, + &format!("bounded-delivery-{index}"), + None, + ) + .await; + let body = format!("bounded body {index}"); + let result_id = record_result( + &mailbox, + &work_ref, + worker_path.clone(), + &body, + ) + .await; + work_refs.push(work_ref); + result_ids.push(result_id); + bodies.push(body); + } + + let (report, success) = wait_with_call_id( + Arc::clone(&session), + Arc::clone(&turn), + serde_json::json!({ + "targets": work_refs.iter().map(|work| work.id.clone()).collect::>(), + "return_when": "all", + "timeout_ms": 0 + }), + "deliver-bounded-results".to_string(), + ) + .await; + assert_eq!(report.completion_reason, Some(MailboxWakeCause::Result)); + assert!(report.condition_satisfied); + assert_eq!(report.completed.len(), 3); + assert_eq!(success, Some(true)); + + for index in 0..2 { + assert!( + mailbox + .get_message(session.thread_id, &result_ids[index]) + .await + .unwrap() + .unwrap() + .canonical + ); + assert_work_observed(&mailbox, session.thread_id, &work_refs[index]).await; + assert_persisted_message(&rollout_path, &result_ids[index], &bodies[index]).await; + } + let omitted = mailbox + .get_message(session.thread_id, &result_ids[2]) + .await + .unwrap() + .unwrap(); + assert!(!omitted.canonical); + assert_eq!(omitted.communication.content, bodies[2]); + let snapshot = mailbox + .snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: vec![Selector::Work(work_refs[2].id.clone())], + return_when: ReturnWhen::All, + limit: 100, + }) + .await + .unwrap(); + let [SnapshotItem::Work(work)] = snapshot.items.as_slice() else { + panic!("expected omitted exact work snapshot") + }; + assert!(!work.observed); + + let (later_report, later_success) = wait_with_call_id( + Arc::clone(&session), + Arc::clone(&turn), + serde_json::json!({ + "targets": [work_refs[2].id.clone()], + "return_when": "all", + "timeout_ms": 0 + }), + "deliver-omitted-result".to_string(), + ) + .await; + assert_eq!(later_report.completion_reason, Some(MailboxWakeCause::Result)); + assert!(later_report.condition_satisfied); + assert_eq!(later_success, Some(true)); + assert!( + mailbox + .get_message(session.thread_id, &result_ids[2]) + .await + .unwrap() + .unwrap() + .canonical + ); + assert_work_observed(&mailbox, session.thread_id, &work_refs[2]).await; + assert_persisted_message(&rollout_path, &result_ids[2], &bodies[2]).await; +} + +#[tokio::test] +async fn interrupted_wait_call_id_replays_failure_but_a_fresh_id_uses_the_wait_path() { + let (session, turn, mailbox, agent, _rollout_path) = durable_fixture().await; + let pending = work(&mailbox, session.thread_id, agent, "pending-after-interruption", None).await; + let call_id = "interrupted-wait-call"; + let interrupted = MailboxWaitReport { + requested_targets: vec![pending.id.clone()], + requested_replies_to: Vec::new(), + return_when: MailboxWaitReturnWhen::All, + completed: Vec::new(), + pending: vec![MailboxWaitSelectionItem { + work_ref: Some(pending.id.clone()), + message_id: None, + agent: Some("/root/worker".to_string()), + status: "running".to_string(), + }], + completion_reason: Some(MailboxWakeCause::Interrupted), + condition_satisfied: false, + quiet_count: 0, + wake_sender: None, + wake_ref: None, + }; + seed_interrupted_wait_receipt(&session, call_id, interrupted.clone()).await; + + let (replayed, replay_success) = wait_with_call_id( + Arc::clone(&session), + Arc::clone(&turn), + serde_json::json!({ + "targets": [pending.id.clone()], + "return_when": "all", + "timeout_ms": 0 + }), + call_id.to_string(), + ) + .await; + assert_eq!(replayed.completion_reason, Some(MailboxWakeCause::Interrupted)); + assert_eq!(replayed.pending, interrupted.pending); + assert!(!replayed.condition_satisfied); + assert_eq!(replay_success, Some(false)); + + let (fresh, fresh_success) = wait_with_call_id( + session, + turn, + serde_json::json!({ + "targets": [pending.id], + "return_when": "all", + "timeout_ms": 0 + }), + "fresh-wait-call".to_string(), + ) + .await; + assert_eq!(fresh.completion_reason, Some(MailboxWakeCause::Deadline)); + assert!(!fresh.condition_satisfied); + assert_eq!(fresh_success, Some(true)); +} diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 46ee13614c71..7881ff315adf 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -250,6 +250,7 @@ async fn handle_message_submission_inner( .. } = invocation; let _ = payload; + let mailbox_enabled = turn.config.features.enabled(codex_features::Feature::AgentMailbox); let receiver_thread_id = resolve_agent_target(&session, &turn, &target).await?; let receiver_agent = session .services @@ -269,7 +270,7 @@ async fn handle_message_submission_inner( let receiver_agent_path = receiver_agent.agent_path.clone().ok_or_else(|| { FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string()) })?; - if mode == MessageDeliveryMode::TriggerTurn + if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn && let Some(work) = session .services .agent_control @@ -390,8 +391,10 @@ async fn handle_message_submission_inner( MessageDeliveryMode::QueueOnly => AgentCommunicationKind::Message, MessageDeliveryMode::TriggerTurn => AgentCommunicationKind::Followup, }; - let context = - AgentCommunicationContext::new(kind, session.thread_id).with_native_key(call_id.clone()); + let mut context = AgentCommunicationContext::new(kind, session.thread_id); + if mailbox_enabled { + context = context.with_native_key(call_id.clone()); + } let result = delivery .send(mode.apply(communication), context, interrupt) .await @@ -411,6 +414,19 @@ async fn handle_message_submission_inner( ) .await; + let accepted_work = if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn { + session.services.agent_control.mailbox_work_for_key(session.thread_id, &call_id) + .await.map_err(|err| collab_agent_error(receiver_thread_id, err))? + } else { + None + }; + let dispatch_state = if let Some(work) = accepted_work.as_ref() + && let Some(mailbox) = session.services.agent_control.mailbox() + { + Some(mailbox.dispatch_state(work).await.map_err(super::mailbox_message::mailbox_error)?) + } else { + None + }; let output = match mode { MessageDeliveryMode::QueueOnly => { let receipt = SendMessageReceipt { @@ -441,18 +457,8 @@ async fn handle_message_submission_inner( &FollowupTaskResult { task_name: receiver_agent_path.to_string(), recipient_task_name: receiver_agent_path.to_string(), - work_ref: session - .services - .agent_control - .mailbox_work_for_key(session.thread_id, &call_id) - .await - .map_err(|err| collab_agent_error(receiver_thread_id, err))? - .map(|work| work.id), - dispatch_state: session - .services - .agent_control - .mailbox_enabled() - .then_some(codex_agent_mailbox::DispatchState::Dispatched), + work_ref: accepted_work.map(|work| work.id), + dispatch_state, effective_identity_scope: "recipient", effective_model: Some(receiver_config.model), effective_model_provider_id: Some(receiver_config.model_provider_id), diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index 5e14ee3932c7..93133a42957f 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -118,8 +118,11 @@ async fn handle_spawn_agent( .get_agent_path() .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, new_agent_path.clone(), message); - let context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id) - .with_native_key(call_id.clone()); + let mailbox_enabled = turn.config.features.enabled(codex_features::Feature::AgentMailbox); + let mut context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); + if mailbox_enabled { + context = context.with_native_key(call_id.clone()); + } let spawned_agent = Box::pin( session .services @@ -141,13 +144,13 @@ async fn handle_spawn_agent( .await .map_err(collab_spawn_error)?; let new_thread_id = spawned_agent.thread_id; - let work_ref = session + let work_ref = if mailbox_enabled { session .services .agent_control .mailbox_work_for_key(session.thread_id, &call_id) .await .map_err(collab_spawn_error)? - .map(|work| work.id); + .map(|work| work.id) } else { None }; let agent_snapshot = session .services .agent_control diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 422ad24b4a1b..af221daec593 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1253,7 +1253,7 @@ mod tests { }; let cancellation_token = CancellationToken::new(); let mut response_task = tokio::spawn( - runtime.handle_tool_call(spawn_call(), cancellation_token.clone()), + runtime.clone().handle_tool_call(spawn_call(), cancellation_token.clone()), ); let child_thread_id = diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 34f62229d955..69ccd5e951fd 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -45,9 +45,9 @@ use crate::tools::handlers::multi_agents_common::MIN_WAIT_TIMEOUT_MS; use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions; use crate::tools::handlers::multi_agents_spec::WaitAgentTimeoutOptions; use crate::tools::handlers::multi_agents_v2::FollowupTaskHandler as FollowupTaskHandlerV2; -use crate::tools::handlers::multi_agents_v2::MailboxFollowupTaskHandler as MailboxFollowupTaskHandlerV2; use crate::tools::handlers::multi_agents_v2::InterruptAgentHandler; use crate::tools::handlers::multi_agents_v2::ListAgentsHandler as ListAgentsHandlerV2; +use crate::tools::handlers::multi_agents_v2::MailboxFollowupTaskHandler as MailboxFollowupTaskHandlerV2; use crate::tools::handlers::multi_agents_v2::MailboxReadHandler; use crate::tools::handlers::multi_agents_v2::MailboxSendMessageHandler; use crate::tools::handlers::multi_agents_v2::MailboxWaitAgentHandler as MailboxWaitAgentHandlerV2; @@ -860,11 +860,13 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu } if turn_context.config.features.enabled(Feature::AgentMailbox) { planned_tools.add_arc(override_tool_exposure( - multi_agent_v2_handler(MailboxFollowupTaskHandlerV2, tool_namespace), exposure, + multi_agent_v2_handler(MailboxFollowupTaskHandlerV2, tool_namespace), + exposure, )); } else { planned_tools.add_arc(override_tool_exposure( - multi_agent_v2_handler(FollowupTaskHandlerV2, tool_namespace), exposure, + multi_agent_v2_handler(FollowupTaskHandlerV2, tool_namespace), + exposure, )); } if turn_context.config.multi_agent_v2.wait_agent_enabled { diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 50ef387d2e63..32727de353ff 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -372,6 +372,7 @@ pub enum MailboxWakeCause { Tool, Deadline, Cancel, + Interrupted, SubscriptionLoss, RuntimeFault, NoMatchingWork, diff --git a/codex-rs/tui/src/multi_agents/mailbox_wait.rs b/codex-rs/tui/src/multi_agents/mailbox_wait.rs index 0316cefd2c8a..7b18dd946ab7 100644 --- a/codex-rs/tui/src/multi_agents/mailbox_wait.rs +++ b/codex-rs/tui/src/multi_agents/mailbox_wait.rs @@ -251,6 +251,7 @@ fn mailbox_wake_label(reason: MailboxWakeCause) -> &'static str { MailboxWakeCause::Tool => "tool", MailboxWakeCause::Deadline => "deadline", MailboxWakeCause::Cancel => "cancel", + MailboxWakeCause::Interrupted => "interrupted", MailboxWakeCause::SubscriptionLoss => "subscription loss", MailboxWakeCause::RuntimeFault => "runtime fault", MailboxWakeCause::NoMatchingWork => "no matching work", diff --git a/docs/carry-divergence-ledger.md b/docs/carry-divergence-ledger.md index 6f9415ef5afb..759468918cdf 100644 --- a/docs/carry-divergence-ledger.md +++ b/docs/carry-divergence-ledger.md @@ -65,6 +65,11 @@ leave the original IDs pending through compaction. Reading a native result body observes only the exact work it covers; an observed failure remains non-success on subsequent exact or mixed any/all joins. Input publication invalidates stale cycle snapshots before a closed-peer claim can be made. +Ordinary resume durably closes an unfinished typed wait as `interrupted`, with +its pending references intact, rather than restarting its old deadline. Reusing +that interrupted call ID cannot start a new wait. Unconfirmed accepted/dispatching +work from that exact resumed issuer is marked recovery-needed in bounded indexed +batches, without resending its input or creating a replacement child. Both `agent_mailbox` and recipient `agent_mailbox_idle_activation` are off by default. Disabled mailbox handling preserves pending records and cannot drain, @@ -76,7 +81,7 @@ new scheduling, and provider/auth changes remain outside this carry. Future upstream syncs retain the same semantic suite and retire equivalent mechanisms and dependencies together, rather than restoring donor framework code. -- Downstream V2 `wait_agent` keeps queue-only inter-agent mail durable and +- Without the opt-in mailbox feature, downstream V2 `wait_agent` keeps queue-only inter-agent mail durable and visible in later notification summaries, but an exact-target wait wakes only for terminal target state, actionable (`trigger_turn`) mail, user steer, terminal completion input, cancellation, or subscription loss. Targetless diff --git a/docs/downstream-regression-matrix.md b/docs/downstream-regression-matrix.md index b1af3b6837fa..f338fd8ea14b 100644 --- a/docs/downstream-regression-matrix.md +++ b/docs/downstream-regression-matrix.md @@ -66,10 +66,11 @@ and `suite::agent_mailbox`, including `unadmitted_canonical_message_survives_precreated_failure_and_manual_compaction`. Its native cancellation slice includes `configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work`. -The notification visibility lane also runs protocol/TUI `mailbox` tests so the -same structured report is exercised in live and replay consumers. +The notification visibility lane also runs protocol/app-server-protocol/TUI +`mailbox` tests so the same structured report is exercised in live and replay +consumers, including interrupted-wait recovery with pending references retained. -Exact-target V2 `wait_agent` ignores queue-only (`trigger_turn=false`) mailbox +Without the opt-in mailbox feature, exact-target V2 `wait_agent` ignores queue-only (`trigger_turn=false`) mailbox activity while preserving durable FIFO delivery and notification summaries; targetless waits retain broad mailbox wake behavior. The focused guardrail is `codex.core-multi-agent-orchestration-targeted`, covering diff --git a/justfile b/justfile index 7cd312fc70f9..0f6910cd52f8 100644 --- a/justfile +++ b/justfile @@ -210,6 +210,7 @@ core-subagent-notification-contract-targeted: # tui_app_server -> tui cutover. core-subagent-notification-visibility-targeted: cargo test --locked -p codex-protocol mailbox --lib -- --test-threads=1 + cargo test --locked -p codex-app-server-protocol mailbox --lib -- --test-threads=1 cargo test --locked -p codex-tui mailbox --lib -- --test-threads=1 cargo test -p codex-protocol parse_subagent_notification_response_item_ --lib -- --test-threads=1 cargo test -p codex-tui raw_response_subagent_notification_renders_history -- --exact --test-threads=1 From aa56bcaa59f9b9f0b74e7ca715ce67b19fe4c9a8 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:12:52 +1100 Subject: [PATCH 08/54] Persist mailbox wait lifecycle before publication and align shared wire output --- .../workflows/mailbox-source-preparation.yml | 12 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 151 +++++++++++++++--- codex-rs/agent-mailbox/src/schema.rs | 78 +++------ codex-rs/agent-mailbox/src/work.rs | 98 +++++++----- codex-rs/app-server/README.md | 20 +++ codex-rs/core/src/agent/control/mailbox.rs | 44 +---- .../core/src/agent/control/mailbox_tests.rs | 36 +++++ codex-rs/core/src/session/mod.rs | 81 +++++++++- codex-rs/core/src/session/tests.rs | 79 ++++----- codex-rs/core/src/tasks/mod.rs | 3 +- .../src/tools/handlers/multi_agents_spec.rs | 26 +-- .../tools/handlers/multi_agents_spec_tests.rs | 37 +++-- .../handlers/multi_agents_v2/mailbox_wait.rs | 14 +- .../multi_agents_v2/mailbox_wait_tests.rs | 90 ++++++----- .../handlers/multi_agents_v2/message_tool.rs | 23 ++- .../tools/handlers/multi_agents_v2/spawn.rs | 26 +-- codex-rs/core/src/tools/parallel.rs | 10 +- codex-rs/core/tests/suite/agent_mailbox.rs | 36 +++-- codex-rs/protocol/src/items.rs | 96 +++++++++-- codex-rs/rollout/src/policy.rs | 14 +- codex-rs/rollout/src/policy_tests.rs | 82 ++++++++++ .../tui/src/chatwidget/tests/app_server.rs | 1 + .../src/chatwidget/tests/history_replay.rs | 65 +++++++- codex-rs/tui/src/multi_agents/mailbox_wait.rs | 21 ++- docs/divergences/index.yaml | 16 +- justfile | 1 + 26 files changed, 829 insertions(+), 331 deletions(-) create mode 100644 codex-rs/core/src/agent/control/mailbox_tests.rs create mode 100644 codex-rs/rollout/src/policy_tests.rs diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 51e4073d0bb6..08e56e09d726 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "5" + PREPARATION_ATTEMPT: "6" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -55,6 +55,9 @@ jobs: continue-on-error: true working-directory: codex-rs run: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 + - name: Verify canonical mailbox lifecycle retention + working-directory: codex-rs + run: cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 - name: Generate the affected native schemas run: | just write-config-schema @@ -69,7 +72,7 @@ jobs: - name: Compile focused UI mailbox consumers working-directory: codex-rs env: - INSTA_UPDATE: always + INSTA_UPDATE: new run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 - name: Verify interrupted mailbox wait history replay working-directory: codex-rs @@ -92,6 +95,11 @@ jobs: git diff --binary > "$RUNNER_TEMP/mailbox-source-preparation/source.patch" git rev-parse HEAD > "$RUNNER_TEMP/mailbox-source-preparation/source-sha.txt" git status --short > "$RUNNER_TEMP/mailbox-source-preparation/status.txt" + while IFS= read -r -d '' pending_snapshot; do + snapshot_target="$RUNNER_TEMP/mailbox-source-preparation/pending-snapshots/$pending_snapshot" + mkdir -p "$(dirname "$snapshot_target")" + cp -- "$pending_snapshot" "$snapshot_target" + done < <(git ls-files --others --exclude-standard -z -- 'codex-rs/tui/src/**/*.snap.new') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a if: always() with: diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index bb115dfcfb90..97c30c0d7158 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1333,16 +1333,36 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov let fixture = test_mailbox().await; let (issuer, agent) = (thread_id(32), thread_id(33)); let (work, input) = accept_work(&fixture.mailbox, issuer, agent, "recovery", "original").await; - assert_eq!(fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + assert_eq!( + fixture.mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::Dispatch + ); fixture.mailbox.mark_dispatched(&work).await.unwrap(); fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); - assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::RecoveryNeeded); + assert_eq!( + fixture.mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::RecoveryNeeded + ); admit(&fixture.mailbox, agent, "explicit-recovery-turn", &[input]).await; - assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched); - let terminal = finish(&fixture.mailbox, agent, "explicit-recovery-turn", Outcome::Completed, None, Continuation::None).await; + assert_eq!( + fixture.mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::Dispatched + ); + let terminal = finish( + &fixture.mailbox, + agent, + "explicit-recovery-turn", + Outcome::Completed, + None, + Continuation::None, + ) + .await; assert_eq!(terminal.outcomes[0].work_ref, work); fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); - assert_eq!(fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched); + assert_eq!( + fixture.mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::Dispatched + ); } #[tokio::test] @@ -1350,30 +1370,113 @@ async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_ba let fixture = test_mailbox().await; let (issuer, agent, other_issuer) = (thread_id(34), thread_id(35), thread_id(36)); let accepted_request = native_work(issuer, agent, "resume-accepted", "retained accepted input"); - let accepted = fixture.mailbox.accept_work(accepted_request.clone()).await.unwrap(); + let accepted = fixture + .mailbox + .accept_work(accepted_request.clone()) + .await + .unwrap(); let retained_input = fixture.mailbox.accepted_input(&accepted).await.unwrap(); - let (dispatching, _) = accept_work(&fixture.mailbox, issuer, agent, "resume-dispatching", "uncertain dispatch").await; - let (dispatched, _) = accept_work(&fixture.mailbox, issuer, agent, "resume-dispatched", "confirmed dispatch").await; - let (foreign, _) = accept_work(&fixture.mailbox, other_issuer, agent, "other-issuer", "foreign accepted input").await; - assert_eq!(fixture.mailbox.begin_dispatch(&dispatching).await.unwrap(), DispatchDecision::Dispatch); - assert_eq!(fixture.mailbox.begin_dispatch(&dispatched).await.unwrap(), DispatchDecision::Dispatch); + let (dispatching, _) = accept_work( + &fixture.mailbox, + issuer, + agent, + "resume-dispatching", + "uncertain dispatch", + ) + .await; + let (dispatched, _) = accept_work( + &fixture.mailbox, + issuer, + agent, + "resume-dispatched", + "confirmed dispatch", + ) + .await; + let (foreign, _) = accept_work( + &fixture.mailbox, + other_issuer, + agent, + "other-issuer", + "foreign accepted input", + ) + .await; + assert_eq!( + fixture.mailbox.begin_dispatch(&dispatching).await.unwrap(), + DispatchDecision::Dispatch + ); + assert_eq!( + fixture.mailbox.begin_dispatch(&dispatched).await.unwrap(), + DispatchDecision::Dispatch + ); fixture.mailbox.mark_dispatched(&dispatched).await.unwrap(); let reopened = Mailbox::open(&fixture.config).await.unwrap(); // Opening the store alone is not authority to interrupt another native session. - assert_eq!(reopened.dispatch_state(&accepted).await.unwrap(), DispatchState::Accepted); - assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 1); - assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 1); - assert_eq!(reopened.interrupt_issuer_dispatches(issuer, 1).await.unwrap(), 0); - assert_eq!(reopened.dispatch_state(&accepted).await.unwrap(), DispatchState::RecoveryNeeded); - assert_eq!(reopened.dispatch_state(&dispatching).await.unwrap(), DispatchState::RecoveryNeeded); - assert_eq!(reopened.dispatch_state(&dispatched).await.unwrap(), DispatchState::Dispatched); - assert_eq!(reopened.dispatch_state(&foreign).await.unwrap(), DispatchState::Accepted); - assert_eq!(reopened.accept_work(accepted_request).await.unwrap(), accepted); - assert_eq!(reopened.accepted_input(&accepted).await.unwrap(), retained_input); - assert_eq!(reopened.begin_dispatch(&accepted).await.unwrap(), DispatchDecision::RecoveryNeeded); - assert_eq!(reopened.issued_work_refs(issuer, None, 100).await.unwrap().len(), 3); - assert!(matches!(reopened.interrupt_issuer_dispatches(issuer, 0).await, Err(MailboxError::InvalidInput(_)))); + assert_eq!( + reopened.dispatch_state(&accepted).await.unwrap(), + DispatchState::Accepted + ); + assert_eq!( + reopened + .interrupt_issuer_dispatches(issuer, 1) + .await + .unwrap(), + 1 + ); + assert_eq!( + reopened + .interrupt_issuer_dispatches(issuer, 1) + .await + .unwrap(), + 1 + ); + assert_eq!( + reopened + .interrupt_issuer_dispatches(issuer, 1) + .await + .unwrap(), + 0 + ); + assert_eq!( + reopened.dispatch_state(&accepted).await.unwrap(), + DispatchState::RecoveryNeeded + ); + assert_eq!( + reopened.dispatch_state(&dispatching).await.unwrap(), + DispatchState::RecoveryNeeded + ); + assert_eq!( + reopened.dispatch_state(&dispatched).await.unwrap(), + DispatchState::Dispatched + ); + assert_eq!( + reopened.dispatch_state(&foreign).await.unwrap(), + DispatchState::Accepted + ); + assert_eq!( + reopened.accept_work(accepted_request).await.unwrap(), + accepted + ); + assert_eq!( + reopened.accepted_input(&accepted).await.unwrap(), + retained_input + ); + assert_eq!( + reopened.begin_dispatch(&accepted).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + assert_eq!( + reopened + .issued_work_refs(issuer, None, 100) + .await + .unwrap() + .len(), + 3 + ); + assert!(matches!( + reopened.interrupt_issuer_dispatches(issuer, 0).await, + Err(MailboxError::InvalidInput(_)) + )); } #[tokio::test] diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 1bbaa4175eed..5db9fd1aafe1 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -37,11 +37,10 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { .execute(&mut *transaction) .await?; - let version: Option = sqlx::query_scalar( - "SELECT version FROM mailbox_schema WHERE singleton = 1", - ) - .fetch_optional(&mut *transaction) - .await?; + let version: Option = + sqlx::query_scalar("SELECT version FROM mailbox_schema WHERE singleton = 1") + .fetch_optional(&mut *transaction) + .await?; if let Some(version) = version { if version > SCHEMA_VERSION { return Err(crate::MailboxError::NewerSchema { @@ -169,12 +168,7 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; - ensure_column( - &mut transaction, - "mailbox_continuations", - "outcome", - ) - .await?; + ensure_column(&mut transaction, "mailbox_continuations", "outcome").await?; sqlx::query( "CREATE INDEX IF NOT EXISTS mailbox_continuations_binding ON mailbox_continuations(binding_json, state, work_id)", @@ -218,48 +212,13 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { ) .execute(&mut *transaction) .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "withdrawn", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "activate_if_idle", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "input_admitted", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "author_path", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "recipient_path", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_messages", - "work_id", - ) - .await?; - ensure_column( - &mut transaction, - "mailbox_pending_counts", - "quiet_count", - ) - .await?; + ensure_column(&mut transaction, "mailbox_messages", "withdrawn").await?; + ensure_column(&mut transaction, "mailbox_messages", "activate_if_idle").await?; + ensure_column(&mut transaction, "mailbox_messages", "input_admitted").await?; + ensure_column(&mut transaction, "mailbox_messages", "author_path").await?; + ensure_column(&mut transaction, "mailbox_messages", "recipient_path").await?; + ensure_column(&mut transaction, "mailbox_messages", "work_id").await?; + ensure_column(&mut transaction, "mailbox_pending_counts", "quiet_count").await?; if version.is_some_and(|version| version < SCHEMA_VERSION) { sqlx::query( "UPDATE mailbox_pending_counts SET @@ -392,7 +351,11 @@ async fn ensure_column( "mailbox_continuations" => "PRAGMA table_info(mailbox_continuations)", "mailbox_messages" => "PRAGMA table_info(mailbox_messages)", "mailbox_pending_counts" => "PRAGMA table_info(mailbox_pending_counts)", - _ => return Err(crate::MailboxError::InvalidInput("unsupported mailbox schema table".into())), + _ => { + return Err(crate::MailboxError::InvalidInput( + "unsupported mailbox schema table".into(), + )); + } }; let rows = sqlx::query(pragma).fetch_all(&mut **transaction).await?; let exists = rows @@ -427,10 +390,13 @@ async fn ensure_column( ("mailbox_pending_counts", "quiet_count") => { "ALTER TABLE mailbox_pending_counts ADD COLUMN quiet_count INTEGER NOT NULL DEFAULT 0" } - _ => return Err(crate::MailboxError::InvalidInput("unsupported mailbox schema column".into())), + _ => { + return Err(crate::MailboxError::InvalidInput( + "unsupported mailbox schema column".into(), + )); + } }; sqlx::query(alter).execute(&mut **transaction).await?; } Ok(()) } - diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index a46bc7fd0fc4..a6884507be04 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -1,11 +1,9 @@ use crate::error::{MailboxError, MailboxResult}; use crate::messages::decrement_pending_count; use crate::store::Mailbox; -use crate::types::{ - AdmissionReceipt, DispatchDecision, DispatchState, NativeWork, WorkRef, -}; -use codex_protocol::protocol::InterAgentCommunication; +use crate::types::{AdmissionReceipt, DispatchDecision, DispatchState, NativeWork, WorkRef}; use codex_protocol::ThreadId; +use codex_protocol::protocol::InterAgentCommunication; use sqlx::{Row, Sqlite, Transaction}; const MAX_WORK_ITEMS: i64 = 50_000; @@ -16,7 +14,11 @@ const MAX_TURN_INPUTS: usize = 100; impl Mailbox { /// Resolve only the work covered by this exact canonical result body. - pub async fn work_for_outcome(&self, issuer: ThreadId, outcome_id: &str) -> MailboxResult> { + pub async fn work_for_outcome( + &self, + issuer: ThreadId, + outcome_id: &str, + ) -> MailboxResult> { let rows = sqlx::query( "SELECT * FROM mailbox_work WHERE issuer = ? AND outcome_id = ? ORDER BY seq LIMIT 101", ) @@ -38,10 +40,12 @@ impl Mailbox { let mut request = work.clone(); request.input.id = None; - let body_bytes = work.input.content.len() - + work.input.encrypted_content.as_ref().map_or(0, String::len); + let body_bytes = + work.input.content.len() + work.input.encrypted_content.as_ref().map_or(0, String::len); if body_bytes > MAX_NATIVE_BODY_BYTES { - return Err(MailboxError::Backpressure("accepted native body byte limit")); + return Err(MailboxError::Backpressure( + "accepted native body byte limit", + )); } let request_json = serde_json::to_string(&request)?; if request_json.len() > MAX_INPUT_BYTES { @@ -54,13 +58,12 @@ impl Mailbox { let input_json = serde_json::to_string(&work.input)?; let mut transaction = self.pool.begin().await?; - if let Some(row) = sqlx::query( - "SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ?", - ) - .bind(work.issuer.to_string()) - .bind(&work.native_key) - .fetch_optional(&mut *transaction) - .await? + if let Some(row) = + sqlx::query("SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ?") + .bind(work.issuer.to_string()) + .bind(&work.native_key) + .fetch_optional(&mut *transaction) + .await? { if row.try_get::("request_json")? != request_json { return Err(MailboxError::IdempotencyConflict); @@ -89,11 +92,10 @@ impl Mailbox { if duplicate_input.is_some() { return Err(MailboxError::IdempotencyConflict); } - let count: i64 = sqlx::query_scalar( - "SELECT item_count FROM mailbox_counts WHERE name = 'work'", - ) - .fetch_one(&mut *transaction) - .await?; + let count: i64 = + sqlx::query_scalar("SELECT item_count FROM mailbox_counts WHERE name = 'work'") + .fetch_one(&mut *transaction) + .await?; if count >= MAX_WORK_ITEMS { return Err(MailboxError::Backpressure("retained work-item count")); } @@ -121,9 +123,11 @@ impl Mailbox { .execute(&mut *transaction) .await?; if inserted.rows_affected() == 1 { - sqlx::query("UPDATE mailbox_counts SET item_count = item_count + 1 WHERE name = 'work'") - .execute(&mut *transaction) - .await?; + sqlx::query( + "UPDATE mailbox_counts SET item_count = item_count + 1 WHERE name = 'work'", + ) + .execute(&mut *transaction) + .await?; } transaction.commit().await?; self.notify_changed(); @@ -131,9 +135,14 @@ impl Mailbox { } /// Return the exact captured native input, including its assigned stable ID. - pub async fn accepted_input(&self, work_ref: &WorkRef) -> MailboxResult { + pub async fn accepted_input( + &self, + work_ref: &WorkRef, + ) -> MailboxResult { let row = exact_work(&self.pool, work_ref).await?; - Ok(serde_json::from_str(&row.try_get::("input_json")?)?) + Ok(serde_json::from_str( + &row.try_get::("input_json")?, + )?) } pub async fn work_for_native_key( @@ -141,13 +150,12 @@ impl Mailbox { issuer: ThreadId, native_key: &str, ) -> MailboxResult> { - let row = sqlx::query( - "SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ? LIMIT 1", - ) - .bind(issuer.to_string()) - .bind(native_key) - .fetch_optional(&self.pool) - .await?; + let row = + sqlx::query("SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ? LIMIT 1") + .bind(issuer.to_string()) + .bind(native_key) + .fetch_optional(&self.pool) + .await?; row.as_ref().map(work_ref_from_row).transpose() } @@ -156,13 +164,12 @@ impl Mailbox { agent: ThreadId, submission_id: &str, ) -> MailboxResult> { - let row = sqlx::query( - "SELECT * FROM mailbox_work WHERE agent = ? AND submission_id = ? LIMIT 1", - ) - .bind(agent.to_string()) - .bind(submission_id) - .fetch_optional(&self.pool) - .await?; + let row = + sqlx::query("SELECT * FROM mailbox_work WHERE agent = ? AND submission_id = ? LIMIT 1") + .bind(agent.to_string()) + .bind(submission_id) + .fetch_optional(&self.pool) + .await?; row.as_ref().map(work_ref_from_row).transpose() } @@ -353,7 +360,9 @@ impl Mailbox { let mut ids = input_message_ids.to_vec(); ids.sort_unstable(); if ids.windows(2).any(|pair| pair[0] == pair[1]) { - return Err(MailboxError::InvalidInput("duplicate input message ID".into())); + return Err(MailboxError::InvalidInput( + "duplicate input message ID".into(), + )); } let mut transaction = self.pool.begin().await?; @@ -439,7 +448,9 @@ impl Mailbox { return Err(MailboxError::IdempotencyConflict); } let canonical = message.try_get::("canonical")? != 0; - let active = message.try_get::, _>("superseded_by")?.is_none() + let active = message + .try_get::, _>("superseded_by")? + .is_none() && message.try_get::("withdrawn")? == 0; let quiet = message.try_get::("quiet")? != 0; sqlx::query( @@ -544,7 +555,8 @@ pub(crate) fn ensure_mailbox_id(id: &str) -> MailboxResult<()> { if id.starts_with("amsg_") && id.len() > 5 { Ok(()) } else { - Err(MailboxError::InvalidInput("expected stable amsg_* ID".into())) + Err(MailboxError::InvalidInput( + "expected stable amsg_* ID".into(), + )) } } - diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index de2d105515ff..f7605dbf209d 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -1471,6 +1471,26 @@ The app-server streams JSON-RPC notifications while a turn is running. Each turn - `contextCompaction` — `{id}` emitted when codex compacts the conversation history. This can happen automatically. - `compacted` - `{threadId, turnId}` when codex compacts the conversation history. This can happen automatically. **Deprecated:** Use `contextCompaction` instead. +For opt-in same-tree V2 mailbox waits, `collabAgentToolCall` also carries optional +`mailboxWaitReport`. It is the same structured result used by the `wait_agent` +tool: `requestedTargets`, `requestedRepliesTo`, `returnWhen`, `completed`, +`pending`, `completionReason`, `conditionSatisfied`, and `quietCount`, with nullable +`wakeSender` and `wakeRef`. Selection entries expose nullable `workRef`, +`messageId`, and `agent` plus their typed `status`. Repeated in-progress updates +retain the same item ID; clients should update the active row rather than append +duplicate bodies. `quietCount` is the recipient's retained quiet backlog, not a +count of satisfied conditions or an acknowledgement of hidden bodies. + +Only selected successful work or exact request replies satisfy the any/all join. +Attention, human/tool input, cancellation, deadline, and runtime faults can end a +wait with pending references intact. Resume closes an unfinished typed wait with +`status: "failed"` and `completionReason: "interrupted"`; it does not restart the +old deadline. Older history without `mailboxWaitReport` retains its native +projection, and current readers accept the earlier snake_case report fields. +Both `agent_mailbox` and the separate recipient `agent_mailbox_idle_activation` +permission are disabled by default; starting a binary alone does not opt a +session into mailbox tools or authorize idle activation. + All items emit shared lifecycle events: - `item/started` — emits the full `item` when a new unit of work begins so the UI can render it immediately; the `item.id` in this payload matches the `itemId` used by deltas. diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 69bcffea60a0..0eaa048927a0 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -168,6 +168,11 @@ impl AgentControl { return Ok(denied("recipient_is_not_live")); }; let config = thread.session.get_config().await; + if config.multi_agent_version_from_features() != MultiAgentVersion::V2 + || !config.features.enabled(Feature::AgentMailbox) + { + return Ok(denied("recipient_mailbox_not_enabled")); + } if !config.features.enabled(Feature::AgentMailboxIdleActivation) { return Ok(denied("recipient_idle_activation_not_enabled")); } @@ -422,40 +427,5 @@ impl AgentControl { } #[cfg(test)] -mod tests { - use super::*; - use codex_protocol::AgentPath; - - #[tokio::test] - async fn mailbox_shared_control_does_not_capture_stock_invocations() { - let (session, turn) = crate::session::tests::make_session_and_context().await; - let mut config = (*turn.config).clone(); - config.features.enable(Feature::MultiAgentV2).unwrap(); - config.features.enable(Feature::AgentMailbox).unwrap(); - let control = &session.services.agent_control; - control.mailbox_initialize(&config).await.unwrap(); - let recipient = ThreadId::new(); - let mut communication = InterAgentCommunication::new( - AgentPath::root(), - AgentPath::from_string("/root/worker".to_string()).unwrap(), - Vec::new(), - "native input".to_string(), - true, - ); - let original = communication.clone(); - let mut submission_id = "stock-submission".to_string(); - let context = AgentCommunicationContext::new(AgentCommunicationKind::Followup, session.thread_id); - assert!(control.mailbox_accept_native_submission(recipient, &context, &mut communication, &mut submission_id).await.unwrap().is_none()); - assert_eq!(communication, original); - assert_eq!(submission_id, "stock-submission"); - let mailbox = control.mailbox().unwrap(); - assert!(mailbox.issued_work_refs(session.thread_id, None, 100).await.unwrap().is_empty()); - - let enabled_context = context.with_native_key("enabled-call".to_string()); - let (accepted, decision) = control.mailbox_accept_native_submission(recipient, &enabled_context, &mut communication, &mut submission_id).await.unwrap().unwrap(); - assert_eq!(decision, DispatchDecision::Dispatch); - assert_eq!(accepted.issuer, session.thread_id); - assert_eq!(mailbox.accepted_input(&accepted).await.unwrap(), communication); - assert!(communication.id.unwrap().as_str().starts_with("amsg_")); - } -} +#[path = "mailbox_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs new file mode 100644 index 000000000000..e56d22a9a046 --- /dev/null +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -0,0 +1,36 @@ +use super::*; +use codex_protocol::AgentPath; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn mailbox_shared_control_does_not_capture_stock_invocations() { + let (session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let control = &session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let recipient = ThreadId::new(); + let mut communication = InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + "native input".to_string(), + true, + ); + let original = communication.clone(); + let mut submission_id = "stock-submission".to_string(); + let context = AgentCommunicationContext::new(AgentCommunicationKind::Followup, session.thread_id); + assert!(control.mailbox_accept_native_submission(recipient, &context, &mut communication, &mut submission_id).await.unwrap().is_none()); + assert_eq!(communication, original); + assert_eq!(submission_id, "stock-submission"); + let mailbox = control.mailbox().unwrap(); + assert!(mailbox.issued_work_refs(session.thread_id, None, 100).await.unwrap().is_empty()); + + let enabled_context = context.with_native_key("enabled-call".to_string()); + let (accepted, decision) = control.mailbox_accept_native_submission(recipient, &enabled_context, &mut communication, &mut submission_id).await.unwrap().unwrap(); + assert_eq!(decision, DispatchDecision::Dispatch); + assert_eq!(accepted.issuer, session.thread_id); + assert_eq!(mailbox.accepted_input(&accepted).await.unwrap(), communication); + assert!(communication.id.unwrap().as_str().starts_with("amsg_")); +} diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 02ad660f3c5d..fd5d3bbb4471 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1591,7 +1591,7 @@ impl Session { item: TurnItem::CollabAgentToolCall(call), completed_at_ms: now_unix_timestamp_ms(), }); - repair_events.push((turn_id, completed_event.clone())); + repair_events.push((turn_id.clone(), completed_event.clone())); repair_items.push(RolloutItem::EventMsg(completed_event)); } @@ -2210,6 +2210,21 @@ impl Session { /// Persist the event to rollout and send it to clients. pub(crate) async fn send_event(&self, turn_context: &TurnContext, msg: EventMsg) { + self.send_event_with_persistence(turn_context, msg, /*persist*/ true) + .await; + } + + async fn send_event_without_persistence(&self, turn_context: &TurnContext, msg: EventMsg) { + self.send_event_with_persistence(turn_context, msg, /*persist*/ false) + .await; + } + + async fn send_event_with_persistence( + &self, + turn_context: &TurnContext, + msg: EventMsg, + persist: bool, + ) { let legacy_source = msg.clone(); if let EventMsg::Error(error) = &legacy_source && error @@ -2259,7 +2274,7 @@ impl Session { self.send_event_raw_with_status_override(event, mailbox_status_override) .await; } else { - self.send_event_raw(event).await; + self.send_event_raw_with_persistence(event, persist).await; } if !mailbox_terminal_before_publish { let _ = self @@ -2280,8 +2295,68 @@ impl Session { id: turn_context.sub_id.clone(), msg: legacy, }; - self.send_event_raw(legacy_event).await; + self.send_event_raw_with_persistence(legacy_event, persist) + .await; + } + } + + /// Durably records a structured mailbox wait item before publishing it. The + /// ordinary collaboration item path is best-effort and its ItemStarted events + /// are intentionally filtered from canonical rollout history. + pub(crate) async fn emit_mailbox_wait_report( + &self, + turn_context: &TurnContext, + item: TurnItem, + ) -> std::io::Result<()> { + let (status, event_msg) = match &item { + TurnItem::CollabAgentToolCall(call) + if call.tool == CollabAgentTool::Wait + && call.mailbox_wait_report.is_some() + && matches!( + call.status, + CollabAgentToolCallStatus::InProgress + | CollabAgentToolCallStatus::Completed + ) => + { + let event_msg = match call.status { + CollabAgentToolCallStatus::InProgress => { + EventMsg::ItemStarted(ItemStartedEvent { + thread_id: self.thread_id, + turn_id: turn_context.sub_id.clone(), + item: item.clone(), + started_at_ms: now_unix_timestamp_ms(), + }) + } + CollabAgentToolCallStatus::Completed => { + EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: self.thread_id, + turn_id: turn_context.sub_id.clone(), + item: item.clone(), + completed_at_ms: now_unix_timestamp_ms(), + }) + } + CollabAgentToolCallStatus::Failed => unreachable!("status was checked above"), + }; + (call.status, event_msg) + } + _ => { + return Err(std::io::Error::other( + "durable mailbox wait reporting requires a typed wait item", + )); + } + }; + + self.live_thread_for_persistence("publish a mailbox wait report") + .map_err(std::io::Error::other)?; + self.try_persist_rollout_items(&[RolloutItem::EventMsg(event_msg.clone())]) + .await?; + self.flush_rollout().await?; + if status == CollabAgentToolCallStatus::Completed { + record_turn_ttfm_metric(turn_context, &item).await; } + self.send_event_without_persistence(turn_context, event_msg) + .await; + Ok(()) } /// Forwards terminal turn events from spawned MultiAgentV2 children to their direct parent. diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 8638dc8e24b4..e7f7f0b12ddc 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2075,7 +2075,7 @@ async fn record_initial_history_reconstructs_resumed_transcript() { #[tokio::test] async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { - let (mut session, _turn_context, rx) = make_session_and_context_with_rx().await; + let (mut session, turn_context, rx) = make_session_and_context_with_rx().await; let fault_store = Arc::new(FaultInjectingThreadStore::new(Arc::clone( &session.services.thread_store, ))); @@ -2087,7 +2087,6 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { Arc::get_mut(&mut session).expect("session should be uniquely owned"), ) .await; - let turn_id = "turn-with-interrupted-mailbox-wait".to_string(); let call_id = "wait-call-interrupted".to_string(); let pending = codex_protocol::items::MailboxWaitSelectionItem { work_ref: Some("work_exact-42".to_string()), @@ -2125,31 +2124,48 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { wake_cause: None, mailbox_wait_report: Some(report), }); - let initial_items = vec![ - RolloutItem::ResponseItem(ResponseItem::FunctionCall { + session + .try_persist_rollout_items(&[RolloutItem::ResponseItem(ResponseItem::FunctionCall { id: None, name: "wait_agent".to_string(), namespace: None, arguments: "{}".to_string(), call_id: call_id.clone(), internal_chat_message_metadata_passthrough: None, - }), - RolloutItem::EventMsg(EventMsg::ItemStarted(ItemStartedEvent { - thread_id: session.thread_id, - turn_id: turn_id.clone(), - item: started_item, - started_at_ms: 10, - })), - ]; + })]) + .await + .expect("wait call should append"); + session + .flush_rollout() + .await + .expect("initial wait history should flush"); + + fault_store + .fail_append_once + .store(true, std::sync::atomic::Ordering::SeqCst); + assert!( + session + .emit_mailbox_wait_report(&turn_context, started_item.clone()) + .await + .is_err(), + "a wait start must surface canonical append failure" + ); + assert!( + rx.try_recv().is_err(), + "a wait start must not publish before canonical persistence" + ); session - .try_persist_rollout_items(&initial_items) + .emit_mailbox_wait_report(&turn_context, started_item) .await - .expect("initial wait history should append"); - session.flush_rollout().await.expect("initial wait history should flush"); + .expect("a wait start should append, flush, then publish"); + assert!(matches!( + rx.try_recv().expect("durable wait start should be published").msg, + EventMsg::ItemStarted(event) if event.item.id() == call_id + )); let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) .await - .expect("read initial rollout history") + .expect("read durably started wait history") else { panic!("expected resumed history"); }; @@ -2194,9 +2210,7 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { .history .iter() .find_map(|item| match item { - RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) - if event.item.id() == call_id => - { + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) if event.item.id() == call_id => { Some(event) } _ => None, @@ -2230,7 +2244,10 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { ) }) .count(); - assert_eq!(repaired_output_count, 1, "same call must have one durable result"); + assert_eq!( + repaired_output_count, 1, + "same call must have one durable result" + ); let output_text = repaired_history .history .iter() @@ -2243,10 +2260,9 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { _ => None, }) .expect("same provider call should have a durable result"); - let output_report = serde_json::from_str::( - &output_text, - ) - .expect("durable provider result should preserve the structured wait report"); + let output_report = + serde_json::from_str::(&output_text) + .expect("durable provider result should preserve the structured wait report"); assert_eq!( output_report.completion_reason, Some(codex_protocol::items::MailboxWakeCause::Interrupted) @@ -2583,17 +2599,11 @@ impl codex_thread_store::ThreadStore for FaultInjectingThreadStore { codex_thread_store::ThreadStore::append_items(self.inner.as_ref(), params) } - fn persist_thread( - &self, - thread_id: ThreadId, - ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + fn persist_thread(&self, thread_id: ThreadId) -> codex_thread_store::ThreadStoreFuture<'_, ()> { codex_thread_store::ThreadStore::persist_thread(self.inner.as_ref(), thread_id) } - fn flush_thread( - &self, - thread_id: ThreadId, - ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + fn flush_thread(&self, thread_id: ThreadId) -> codex_thread_store::ThreadStoreFuture<'_, ()> { if self .fail_flush_once .swap(false, std::sync::atomic::Ordering::SeqCst) @@ -2614,10 +2624,7 @@ impl codex_thread_store::ThreadStore for FaultInjectingThreadStore { codex_thread_store::ThreadStore::shutdown_thread(self.inner.as_ref(), thread_id) } - fn discard_thread( - &self, - thread_id: ThreadId, - ) -> codex_thread_store::ThreadStoreFuture<'_, ()> { + fn discard_thread(&self, thread_id: ThreadId) -> codex_thread_store::ThreadStoreFuture<'_, ()> { codex_thread_store::ThreadStore::discard_thread(self.inner.as_ref(), thread_id) } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index da5a8ed4c361..72a35f30e95c 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -653,7 +653,8 @@ impl Session { mailbox.notify_runtime_fault(self.thread_id); } let error_event = error.to_error_event(/*message_prefix*/ None); - self.send_event(&turn_context, EventMsg::Error(error_event)).await; + self.send_event(&turn_context, EventMsg::Error(error_event)) + .await; } } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 52ac45cc3aac..cc9a24208775 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -1083,17 +1083,17 @@ fn mailbox_wait_output_schema() -> Value { json!({ "type": "object", "properties": { - "requested_targets": { + "requestedTargets": { "type": "array", "items": { "type": "string" }, "description": "Agent names or exact native work references requested by the wait call." }, - "requested_replies_to": { + "requestedRepliesTo": { "type": "array", "items": { "type": "string" }, "description": "Request message ids requested by the wait call." }, - "return_when": { + "returnWhen": { "type": "string", "enum": ["any", "all"], "description": "Whether any one selected condition or every selected condition must complete." @@ -1103,8 +1103,8 @@ fn mailbox_wait_output_schema() -> Value { "items": { "type": "object", "properties": { - "work_ref": { "type": ["string", "null"] }, - "message_id": { "type": ["string", "null"] }, + "workRef": { "type": ["string", "null"] }, + "messageId": { "type": ["string", "null"] }, "agent": { "type": ["string", "null"] }, "status": { "type": "string" } }, @@ -1119,8 +1119,8 @@ fn mailbox_wait_output_schema() -> Value { "items": { "type": "object", "properties": { - "work_ref": { "type": ["string", "null"] }, - "message_id": { "type": ["string", "null"] }, + "workRef": { "type": ["string", "null"] }, + "messageId": { "type": ["string", "null"] }, "agent": { "type": ["string", "null"] }, "status": { "type": "string" } }, @@ -1129,29 +1129,29 @@ fn mailbox_wait_output_schema() -> Value { }, "description": "Requested targets or replies still pending when the wait returned." }, - "completion_reason": { + "completionReason": { "type": "string", "enum": ["result", "reply", "request", "human", "tool", "deadline", "cancel", "interrupted", "subscription_loss", "runtime_fault", "no_matching_work", "already_satisfied", "cycle", "needs_attention"], "description": "Actual reason the wait returned. Attention and interruption do not themselves satisfy an outstanding join." }, - "condition_satisfied": { + "conditionSatisfied": { "type": "boolean", "description": "Whether the requested any/all join condition was satisfied." }, - "quiet_count": { + "quietCount": { "type": "integer", "description": "Global retained quiet-message backlog for this recipient, not a per-wait filtered count. This metadata does not consume hidden message bodies." }, - "wake_sender": { + "wakeSender": { "type": ["string", "null"], "description": "Safe sender identity metadata for a wake, when applicable." }, - "wake_ref": { + "wakeRef": { "type": ["string", "null"], "description": "Safe reference metadata for a wake, when applicable." } }, - "required": ["requested_targets", "requested_replies_to", "return_when", "completed", "pending", "completion_reason", "condition_satisfied", "quiet_count"], + "required": ["requestedTargets", "requestedRepliesTo", "returnWhen", "completed", "pending", "completionReason", "conditionSatisfied", "quietCount"], "additionalProperties": false }) } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index 66e5983d58aa..c28cb60ad2e6 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -657,6 +657,7 @@ fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result assert!(properties.contains_key("timeout_ms")); assert!(properties.contains_key("return_when")); assert!(properties.contains_key("native_event_wait")); + assert!(!properties.contains_key("returnWhen")); assert_eq!(parameters.required, None); assert_eq!( properties["timeout_ms"].schema_type, @@ -694,14 +695,14 @@ fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result assert_eq!( output_schema["required"], json!([ - "requested_targets", - "requested_replies_to", - "return_when", + "requestedTargets", + "requestedRepliesTo", + "returnWhen", "completed", "pending", - "completion_reason", - "condition_satisfied", - "quiet_count" + "completionReason", + "conditionSatisfied", + "quietCount" ]) ); for cause in [ @@ -720,7 +721,7 @@ fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result "needs_attention", ] { assert!( - output_schema["properties"]["completion_reason"]["enum"] + output_schema["properties"]["completionReason"]["enum"] .as_array() .expect("completion reason enum") .contains(&json!(cause)), @@ -789,14 +790,32 @@ fn mailbox_wait_agent_tool_has_no_native_capability_prerequisite() { let output_schema = output_schema.expect("mailbox wait output schema"); assert!( output_schema["properties"] - .get("condition_satisfied") + .get("conditionSatisfied") .is_some() ); - assert!(output_schema["properties"].get("quiet_count").is_some()); + assert!(output_schema["properties"].get("quietCount").is_some()); + assert!(output_schema["properties"].get("completionReason").is_some()); + assert!(output_schema["properties"].get("requestedTargets").is_some()); + assert!(output_schema["properties"].get("requestedRepliesTo").is_some()); + assert!(output_schema["properties"].get("returnWhen").is_some()); + assert!(output_schema["properties"].get("wakeSender").is_some()); + assert!(output_schema["properties"].get("wakeRef").is_some()); + assert!(output_schema["properties"].get("condition_satisfied").is_none()); + assert!(output_schema["properties"].get("quiet_count").is_none()); assert_eq!( output_schema["properties"]["completed"]["items"]["type"], json!("object") ); + assert!( + output_schema["properties"]["completed"]["items"]["properties"] + .get("workRef") + .is_some() + ); + assert!( + output_schema["properties"]["completed"]["items"]["properties"] + .get("messageId") + .is_some() + ); } #[test] diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 7f6563ed5b48..2d91802efa58 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -214,11 +214,11 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, }; let all_observed = !selected.is_empty() && selected.iter().all(already_observed); if !started { - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await; + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await?; started = true; last_report = Some(report.clone()); } else if last_report.as_ref() != Some(&report) { - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await; + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await?; last_report = Some(report.clone()); } if mailbox.has_runtime_fault(&agents) { @@ -365,7 +365,7 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, report.condition_satisfied = false; } } - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::Completed).await; + emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::Completed).await?; Ok(boxed_tool_output(FunctionToolOutput::from_text( tool_output_json_text(&report, "wait_agent"), Some(true), ))) @@ -484,7 +484,7 @@ async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox Ok(()) } -async fn emit_report(session: &Session, turn: &TurnContext, call_id: &str, agents: &[ThreadId], report: &MailboxWaitReport, status: CollabAgentToolCallStatus) { +async fn emit_report(session: &Session, turn: &TurnContext, call_id: &str, agents: &[ThreadId], report: &MailboxWaitReport, status: CollabAgentToolCallStatus) -> Result<(), FunctionCallError> { let item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { id: call_id.to_string(), tool: CollabAgentTool::Wait, status, sender_thread_id: session.thread_id, receiver_thread_ids: agents.to_vec(), receiver_agents: Vec::new(), @@ -492,6 +492,8 @@ async fn emit_report(session: &Session, turn: &TurnContext, call_id: &str, agent agents_states: Default::default(), wake_notifications: None, completion_reason: None, wake_cause: None, mailbox_wait_report: Some(report.clone()), }); - if status == CollabAgentToolCallStatus::InProgress { session.emit_turn_item_started(turn, &item).await; } - else { session.emit_turn_item_completed(turn, item).await; } + session + .emit_mailbox_wait_report(turn, item) + .await + .map_err(mailbox_error) } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index 2ca72a290143..eb4baa6d3c66 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -10,8 +10,8 @@ use codex_agent_mailbox::WorkRef; use codex_features::Feature; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; -use codex_protocol::models::ResponseItem; use codex_protocol::models::ResponseInputItem; +use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::ItemCompletedEvent; @@ -22,24 +22,7 @@ use std::sync::Arc; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; -async fn fixture() -> (Arc, Arc, Arc, ThreadId) { - let (session, mut turn) = crate::session::tests::make_session_and_context().await; - let mut config = (*turn.config).clone(); - config.features.enable(Feature::MultiAgentV2).unwrap(); - config.features.enable(Feature::AgentMailbox).unwrap(); - turn.multi_agent_version = config.multi_agent_version_from_features(); - turn.config = Arc::new(config); - session - .services - .agent_control - .mailbox_initialize(&turn.config) - .await - .unwrap(); - let mailbox = session.services.agent_control.mailbox().unwrap(); - (Arc::new(session), Arc::new(turn), mailbox, ThreadId::new()) -} - -async fn durable_fixture() -> ( +async fn fixture_with_path() -> ( Arc, Arc, Arc, @@ -69,6 +52,21 @@ async fn durable_fixture() -> ( ) } +async fn fixture() -> (Arc, Arc, Arc, ThreadId) { + let (session, turn, mailbox, agent, _rollout_path) = fixture_with_path().await; + (session, turn, mailbox, agent) +} + +async fn durable_fixture() -> ( + Arc, + Arc, + Arc, + ThreadId, + PathBuf, +) { + fixture_with_path().await +} + async fn work( mailbox: &Mailbox, issuer: ThreadId, @@ -209,7 +207,9 @@ async fn assert_persisted_message(rollout_path: &std::path::Path, id: &str, body }; assert!(resumed.history.iter().any(|item| match item { RolloutItem::ResponseItem(response) => { - response.id().is_some_and(|response_id| response_id.as_str() == id) + response + .id() + .is_some_and(|response_id| response_id.as_str() == id) && serde_json::to_value(response) .unwrap() .to_string() @@ -226,8 +226,8 @@ async fn seed_interrupted_wait_receipt( ) { // This exact call/output/terminal-event trio is the canonical history shape consumed by // resume recovery; refresh the same derived receipt cache used by that path. - let terminal_item = TurnItem::CollabAgentToolCall( - codex_protocol::protocol::CollabAgentToolCallItem { + let terminal_item = + TurnItem::CollabAgentToolCall(codex_protocol::protocol::CollabAgentToolCallItem { id: call_id.to_string(), tool: codex_protocol::protocol::CollabAgentTool::Wait, status: codex_protocol::protocol::CollabAgentToolCallStatus::Failed, @@ -244,8 +244,7 @@ async fn seed_interrupted_wait_receipt( completion_reason: None, wake_cause: None, mailbox_wait_report: Some(report.clone()), - }, - ); + }); let items = [ RolloutItem::ResponseItem(ResponseItem::FunctionCall { id: None, @@ -476,7 +475,13 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { .unwrap(); assert!(!reply.canonical); assert_eq!(mailbox.quiet_count(session.thread_id).await.unwrap(), 1); - assert!(mailbox.attention(session.thread_id, 100).await.unwrap().is_empty()); + assert!( + mailbox + .attention(session.thread_id, 100) + .await + .unwrap() + .is_empty() + ); let before = mailbox .snapshot(&Selection { caller: session.thread_id, @@ -508,7 +513,13 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { assert!(report.condition_satisfied); assert_eq!(report.quiet_count, 1); assert_eq!(success, Some(true)); - assert!(mailbox.attention(session.thread_id, 100).await.unwrap().is_empty()); + assert!( + mailbox + .attention(session.thread_id, 100) + .await + .unwrap() + .is_empty() + ); assert!( mailbox .get_message(session.thread_id, &reply.id) @@ -555,13 +566,7 @@ async fn wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delive ) .await; let body = format!("bounded body {index}"); - let result_id = record_result( - &mailbox, - &work_ref, - worker_path.clone(), - &body, - ) - .await; + let result_id = record_result(&mailbox, &work_ref, worker_path.clone(), &body).await; work_refs.push(work_ref); result_ids.push(result_id); bodies.push(body); @@ -628,7 +633,10 @@ async fn wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delive "deliver-omitted-result".to_string(), ) .await; - assert_eq!(later_report.completion_reason, Some(MailboxWakeCause::Result)); + assert_eq!( + later_report.completion_reason, + Some(MailboxWakeCause::Result) + ); assert!(later_report.condition_satisfied); assert_eq!(later_success, Some(true)); assert!( @@ -646,7 +654,14 @@ async fn wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delive #[tokio::test] async fn interrupted_wait_call_id_replays_failure_but_a_fresh_id_uses_the_wait_path() { let (session, turn, mailbox, agent, _rollout_path) = durable_fixture().await; - let pending = work(&mailbox, session.thread_id, agent, "pending-after-interruption", None).await; + let pending = work( + &mailbox, + session.thread_id, + agent, + "pending-after-interruption", + None, + ) + .await; let call_id = "interrupted-wait-call"; let interrupted = MailboxWaitReport { requested_targets: vec![pending.id.clone()], @@ -678,7 +693,10 @@ async fn interrupted_wait_call_id_replays_failure_but_a_fresh_id_uses_the_wait_p call_id.to_string(), ) .await; - assert_eq!(replayed.completion_reason, Some(MailboxWakeCause::Interrupted)); + assert_eq!( + replayed.completion_reason, + Some(MailboxWakeCause::Interrupted) + ); assert_eq!(replayed.pending, interrupted.pending); assert!(!replayed.condition_satisfied); assert_eq!(replay_success, Some(false)); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 7881ff315adf..6f37e616d40b 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -250,7 +250,10 @@ async fn handle_message_submission_inner( .. } = invocation; let _ = payload; - let mailbox_enabled = turn.config.features.enabled(codex_features::Feature::AgentMailbox); + let mailbox_enabled = turn + .config + .features + .enabled(codex_features::Feature::AgentMailbox); let receiver_thread_id = resolve_agent_target(&session, &turn, &target).await?; let receiver_agent = session .services @@ -270,7 +273,8 @@ async fn handle_message_submission_inner( let receiver_agent_path = receiver_agent.agent_path.clone().ok_or_else(|| { FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string()) })?; - if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn + if mailbox_enabled + && mode == MessageDeliveryMode::TriggerTurn && let Some(work) = session .services .agent_control @@ -415,15 +419,24 @@ async fn handle_message_submission_inner( .await; let accepted_work = if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn { - session.services.agent_control.mailbox_work_for_key(session.thread_id, &call_id) - .await.map_err(|err| collab_agent_error(receiver_thread_id, err))? + session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, &call_id) + .await + .map_err(|err| collab_agent_error(receiver_thread_id, err))? } else { None }; let dispatch_state = if let Some(work) = accepted_work.as_ref() && let Some(mailbox) = session.services.agent_control.mailbox() { - Some(mailbox.dispatch_state(work).await.map_err(super::mailbox_message::mailbox_error)?) + Some( + mailbox + .dispatch_state(work) + .await + .map_err(super::mailbox_message::mailbox_error)?, + ) } else { None }; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index 93133a42957f..8f714558cf20 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -118,8 +118,12 @@ async fn handle_spawn_agent( .get_agent_path() .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, new_agent_path.clone(), message); - let mailbox_enabled = turn.config.features.enabled(codex_features::Feature::AgentMailbox); - let mut context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); + let mailbox_enabled = turn + .config + .features + .enabled(codex_features::Feature::AgentMailbox); + let mut context = + AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); if mailbox_enabled { context = context.with_native_key(call_id.clone()); } @@ -144,13 +148,17 @@ async fn handle_spawn_agent( .await .map_err(collab_spawn_error)?; let new_thread_id = spawned_agent.thread_id; - let work_ref = if mailbox_enabled { session - .services - .agent_control - .mailbox_work_for_key(session.thread_id, &call_id) - .await - .map_err(collab_spawn_error)? - .map(|work| work.id) } else { None }; + let work_ref = if mailbox_enabled { + session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, &call_id) + .await + .map_err(collab_spawn_error)? + .map(|work| work.id) + } else { + None + }; let agent_snapshot = session .services .agent_control diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index af221daec593..dc8c3169bb17 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1212,7 +1212,11 @@ mod tests { .await .expect("root thread should start"); session.services.agent_control = manager.agent_control(); - session.services.agent_control.mailbox_initialize(&config).await?; + session + .services + .agent_control + .mailbox_initialize(&config) + .await?; session.thread_id = root.thread_id; let (initial_delivery_finished, resume_spawn) = session .services @@ -1253,7 +1257,9 @@ mod tests { }; let cancellation_token = CancellationToken::new(); let mut response_task = tokio::spawn( - runtime.clone().handle_tool_call(spawn_call(), cancellation_token.clone()), + runtime + .clone() + .handle_tool_call(spawn_call(), cancellation_token.clone()), ); let child_thread_id = diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 478723ff32e6..50fa762583fe 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -307,6 +307,7 @@ impl Respond for RootMailboxFlowResponder { "target": self.root_thread_id, "kind": "request", "message": REQUEST_BODY, + "activate_if_idle": true, }), ); } @@ -401,6 +402,9 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() let request_id = request_receipt["message_id"] .as_str() .expect("request receipt should contain a native message ID"); + assert_eq!(request_receipt["durable_state"], "accepted"); + assert_eq!(request_receipt["activation_disposition"], "denied"); + assert_eq!(request_receipt["activation_reason"], "root_activation_is_not_allowed"); assert!( request_id.starts_with("amsg_"), "unexpected native ID: {request_id}" @@ -441,20 +445,20 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() function_output_text(&requests[5], "wait-call") .expect("wait should return a selector report"), )?; - assert_eq!(wait_report["requested_targets"], json!([work_ref])); - assert_eq!(wait_report["requested_replies_to"], json!([request_id])); - assert_eq!(wait_report["return_when"], "all"); - assert_eq!(wait_report["condition_satisfied"], true); + assert_eq!(wait_report["requestedTargets"], json!([work_ref])); + assert_eq!(wait_report["requestedRepliesTo"], json!([request_id])); + assert_eq!(wait_report["returnWhen"], "all"); + assert_eq!(wait_report["conditionSatisfied"], true); assert!( wait_report["completed"] .as_array() .is_some_and(|completed| { completed .iter() - .any(|item| item["work_ref"] == work_ref && item["status"] == "completed") + .any(|item| item["workRef"] == work_ref && item["status"] == "completed") && completed .iter() - .any(|item| item["message_id"] == request_id && item["status"] == "replied") + .any(|item| item["messageId"] == request_id && item["status"] == "replied") }) ); assert_eq!(wait_report["pending"], json!([])); @@ -760,12 +764,12 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - function_output_text(&requests[3].1, "wait-1-call") .expect("first wait should return its deadline report"), )?; - assert_eq!(deadline_report["requested_targets"], json!([work_ref])); - assert_eq!(deadline_report["completion_reason"], "deadline"); - assert_eq!(deadline_report["condition_satisfied"], false); - assert!(deadline_report["quiet_count"].as_u64().unwrap_or_default() >= 4); + assert_eq!(deadline_report["requestedTargets"], json!([work_ref])); + assert_eq!(deadline_report["completionReason"], "deadline"); + assert_eq!(deadline_report["conditionSatisfied"], false); + assert!(deadline_report["quietCount"].as_u64().unwrap_or_default() >= 4); assert_eq!(deadline_report["completed"], json!([])); - assert_eq!(deadline_report["pending"][0]["work_ref"], work_ref); + assert_eq!(deadline_report["pending"][0]["workRef"], work_ref); assert_eq!(deadline_report["pending"][0]["status"], "pending"); assert!(deadline_report.to_string().contains(work_ref)); assert!(!deadline_report.to_string().contains(unrelated_work_ref)); @@ -781,13 +785,13 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - function_output_text(&requests[4].1, "wait-2-call") .expect("event-only wait should return the addressed request report"), )?; - assert_eq!(request_report["requested_targets"], json!([work_ref])); - assert_eq!(request_report["completion_reason"], "request"); - assert_eq!(request_report["condition_satisfied"], false); + assert_eq!(request_report["requestedTargets"], json!([work_ref])); + assert_eq!(request_report["completionReason"], "request"); + assert_eq!(request_report["conditionSatisfied"], false); assert_eq!(request_report["completed"], json!([])); - assert_eq!(request_report["pending"][0]["work_ref"], work_ref); + assert_eq!(request_report["pending"][0]["workRef"], work_ref); assert_eq!(request_report["pending"][0]["status"], "pending"); - let wake_ref = request_report["wake_ref"] + let wake_ref = request_report["wakeRef"] .as_str() .expect("addressed request should identify the exact wake message"); assert!(wake_ref.starts_with("amsg_")); diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 32727de353ff..32a82c6c1f3d 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -383,42 +383,48 @@ pub enum MailboxWakeCause { /// One selected wait condition and its current or terminal state. #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] pub struct MailboxWaitSelectionItem { - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] + #[serde(default, alias = "work_ref")] + #[ts(optional = false)] pub work_ref: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] + #[serde(default, alias = "message_id")] + #[ts(optional = false)] pub message_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] + #[serde(default)] + #[ts(optional = false)] pub agent: Option, pub status: String, } /// Shared mailbox wait report carried by tool results and collaboration history. /// -/// `completion_reason` is absent while the wait is in progress and populated with +/// `completion_reason` is null while the wait is in progress and populated with /// the actual wake/result cause when it returns. It is always present in a terminal /// wait tool result. #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] pub struct MailboxWaitReport { + #[serde(alias = "requested_targets")] pub requested_targets: Vec, + #[serde(alias = "requested_replies_to")] pub requested_replies_to: Vec, - #[serde(default)] + #[serde(default, alias = "return_when")] pub return_when: MailboxWaitReturnWhen, pub completed: Vec, pub pending: Vec, - #[serde(default)] + #[serde(default, alias = "completion_reason")] #[ts(optional = false)] pub completion_reason: Option, + #[serde(alias = "condition_satisfied")] pub condition_satisfied: bool, + #[serde(alias = "quiet_count")] pub quiet_count: usize, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] + #[serde(default, alias = "wake_sender")] + #[ts(optional = false)] pub wake_sender: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] + #[serde(default, alias = "wake_ref")] + #[ts(optional = false)] pub wake_ref: Option, } @@ -951,7 +957,7 @@ mod tests { #[test] fn mailbox_wait_report_preserves_in_progress_cause_and_defaults_any_join() { - let mut value = json!({ + let value = json!({ "requested_targets": ["work_42"], "requested_replies_to": [], "completed": [], @@ -963,9 +969,65 @@ mod tests { serde_json::from_value(value.clone()).expect("decode mailbox wait report"); assert_eq!(report.return_when, MailboxWaitReturnWhen::Any); assert_eq!(report.completion_reason, None); - value["return_when"] = json!("any"); - value["completion_reason"] = json!(null); - assert_eq!(serde_json::to_value(report).expect("encode active wait"), value); + assert_eq!( + serde_json::to_value(report).expect("encode active wait"), + json!({ + "requestedTargets": ["work_42"], + "requestedRepliesTo": [], + "returnWhen": "any", + "completed": [], + "pending": [{"workRef": "work_42", "messageId": null, "agent": null, "status": "running"}], + "completionReason": null, + "conditionSatisfied": false, + "quietCount": 2, + "wakeSender": null, + "wakeRef": null + }) + ); + } + + #[test] + fn mailbox_wait_report_decodes_legacy_snake_case_and_serializes_camel_case() { + let legacy = json!({ + "requested_targets": ["worker"], + "requested_replies_to": ["amsg_request"], + "return_when": "all", + "completed": [{ + "work_ref": "work_42", + "message_id": "amsg_reply", + "agent": "worker", + "status": "completed" + }], + "pending": [], + "completion_reason": "reply", + "condition_satisfied": true, + "quiet_count": 3, + "wake_sender": "worker", + "wake_ref": "amsg_reply" + }); + let report: MailboxWaitReport = + serde_json::from_value(legacy).expect("decode early mailbox report history"); + + assert_eq!( + serde_json::to_value(report).expect("encode current mailbox report"), + json!({ + "requestedTargets": ["worker"], + "requestedRepliesTo": ["amsg_request"], + "returnWhen": "all", + "completed": [{ + "workRef": "work_42", + "messageId": "amsg_reply", + "agent": "worker", + "status": "completed" + }], + "pending": [], + "completionReason": "reply", + "conditionSatisfied": true, + "quietCount": 3, + "wakeSender": "worker", + "wakeRef": "amsg_reply" + }) + ); } #[test] diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index 93a0c531aa0b..4a6949824db7 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -1,6 +1,8 @@ use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use codex_extension_items::ExtensionItem; +use codex_protocol::items::CollabAgentTool; +use codex_protocol::items::CollabAgentToolCallStatus; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::ThreadHistoryMode; @@ -169,7 +171,6 @@ pub fn should_persist_event_msg(ev: &EventMsg, history_mode: ThreadHistoryMode) | EventMsg::PlanUpdate(_) | EventMsg::ShutdownComplete | EventMsg::DeprecationNotice(_) - | EventMsg::ItemStarted(_) | EventMsg::HookStarted(_) | EventMsg::HookCompleted(_) | EventMsg::AgentMessageContentDelta(_) @@ -183,5 +184,16 @@ pub fn should_persist_event_msg(ev: &EventMsg, history_mode: ThreadHistoryMode) | EventMsg::CollabCloseBegin(_) | EventMsg::CollabResumeBegin(_) | EventMsg::Unknown => false, + EventMsg::ItemStarted(event) => matches!( + &event.item, + TurnItem::CollabAgentToolCall(call) + if call.tool == CollabAgentTool::Wait + && call.status == CollabAgentToolCallStatus::InProgress + && call.mailbox_wait_report.is_some() + ), } } + +#[cfg(test)] +#[path = "policy_tests.rs"] +mod tests; diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs new file mode 100644 index 000000000000..d38e81840396 --- /dev/null +++ b/codex-rs/rollout/src/policy_tests.rs @@ -0,0 +1,82 @@ +use std::collections::HashMap; + +use codex_protocol::ThreadId; +use codex_protocol::items::CollabAgentTool; +use codex_protocol::items::CollabAgentToolCallItem; +use codex_protocol::items::CollabAgentToolCallStatus; +use codex_protocol::items::MailboxWaitReport; +use codex_protocol::items::TurnItem; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ItemStartedEvent; +use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::protocol::RolloutItem; + +use crate::policy::is_persisted_rollout_item; + +fn started_call( + tool: CollabAgentTool, + status: CollabAgentToolCallStatus, + report: Option, +) -> RolloutItem { + RolloutItem::EventMsg(EventMsg::ItemStarted(ItemStartedEvent { + thread_id: ThreadId::new(), + turn_id: "turn".to_string(), + item: TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: "wait-call".to_string(), + tool, + status, + sender_thread_id: ThreadId::new(), + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: report, + }), + started_at_ms: 1, + })) +} + +fn started_wait(status: CollabAgentToolCallStatus, report: Option) -> RolloutItem { + started_call(CollabAgentTool::Wait, status, report) +} + +fn in_progress_report() -> MailboxWaitReport { + MailboxWaitReport { + requested_targets: Vec::new(), + requested_replies_to: Vec::new(), + return_when: Default::default(), + completed: Vec::new(), + pending: Vec::new(), + completion_reason: None, + condition_satisfied: false, + quiet_count: 0, + wake_sender: None, + wake_ref: None, + } +} + +#[test] +fn only_typed_in_progress_mailbox_wait_starts_are_persisted() { + let valid = started_wait(CollabAgentToolCallStatus::InProgress, Some(in_progress_report())); + let no_report = started_wait(CollabAgentToolCallStatus::InProgress, None); + let terminal = started_wait(CollabAgentToolCallStatus::Completed, Some(in_progress_report())); + let other_tool = started_call( + CollabAgentTool::SendInput, + CollabAgentToolCallStatus::InProgress, + Some(in_progress_report()), + ); + + for mode in [ThreadHistoryMode::Legacy, ThreadHistoryMode::Paginated] { + assert!(is_persisted_rollout_item(&valid, mode)); + assert!(!is_persisted_rollout_item(&no_report, mode)); + assert!(!is_persisted_rollout_item(&terminal, mode)); + assert!(!is_persisted_rollout_item(&other_tool, mode)); + } +} diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 2890aea163f2..f31162643225 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -1218,6 +1218,7 @@ async fn live_mailbox_wait_report_shows_join_pending_state_and_actual_wake() { assert!(updated_lines.contains("Pending Robie · msg_request: awaiting_reply")); assert!(updated_lines.contains("Quiet count: 4")); assert!(!updated_lines.contains("Pending work_42:")); + assert_chatwidget_snapshot!("live_mailbox_wait_report_active_progress", updated_lines); assert!(drain_insert_history(&mut rx).is_empty()); let terminal_report = codex_protocol::items::MailboxWaitReport { diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index e4f0ea99bc80..2d50d8f9212b 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -128,22 +128,71 @@ async fn replayed_mailbox_wait_report_preserves_active_join_selection() { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: Some(report), + mailbox_wait_report: Some(report.clone()), }, "turn-replayed-mailbox-wait".to_string(), ReplayKind::ResumeInitialMessages, ); - let lines = drain_insert_history(&mut rx) + let active_lines = lines_to_single_string( + &chat + .active_cell_transcript_lines(/*width*/ 80) + .expect("replayed mailbox wait active row"), + ); + assert!(active_lines.contains("Waiting for any of 3 selected conditions")); + assert!(active_lines.contains("Targets: worker-A, work_17")); + assert!(active_lines.contains("Replies to: msg_request")); + assert!(active_lines.contains("Pending work_17: pending")); + assert!(active_lines.contains("Quiet count: 2")); + assert!(!active_lines.contains("Completion reason:")); + + let interrupted_report = codex_protocol::items::MailboxWaitReport { + requested_targets: report.requested_targets.clone(), + requested_replies_to: report.requested_replies_to.clone(), + return_when: report.return_when, + completed: Vec::new(), + pending: report.pending.clone(), + completion_reason: Some(codex_protocol::items::MailboxWakeCause::Interrupted), + condition_satisfied: false, + quiet_count: 5, + wake_sender: None, + wake_ref: None, + }; + chat.replay_thread_item( + AppServerThreadItem::CollabAgentToolCall { + id: "wait-replay".to_string(), + tool: AppServerCollabAgentTool::Wait, + status: AppServerCollabAgentToolCallStatus::Failed, + sender_thread_id: sender_thread_id.to_string(), + receiver_thread_ids: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + effective_model: None, + effective_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(interrupted_report), + }, + "turn-replayed-mailbox-wait".to_string(), + ReplayKind::ResumeInitialMessages, + ); + let interrupted_lines = drain_insert_history(&mut rx) .into_iter() .map(|lines| lines_to_single_string(&lines)) .collect::>() .join("\n"); - assert!(lines.contains("Waiting for any of 3 selected conditions")); - assert!(lines.contains("Targets: worker-A, work_17")); - assert!(lines.contains("Replies to: msg_request")); - assert!(lines.contains("Pending work_17: pending")); - assert!(lines.contains("Quiet count: 2")); - assert!(!lines.contains("Completion reason:")); + assert!(interrupted_lines.contains("Mailbox wait returned: interrupted")); + assert!(interrupted_lines.contains("Pending work_17: pending")); + assert!(interrupted_lines.contains("Pending reviewer · msg_request: awaiting_reply")); + assert!(interrupted_lines.contains("Quiet count: 5")); + assert_chatwidget_snapshot!( + "replayed_mailbox_wait_report_interrupted_terminal", + interrupted_lines + ); } #[tokio::test] diff --git a/codex-rs/tui/src/multi_agents/mailbox_wait.rs b/codex-rs/tui/src/multi_agents/mailbox_wait.rs index 7b18dd946ab7..6b6215b9d0ee 100644 --- a/codex-rs/tui/src/multi_agents/mailbox_wait.rs +++ b/codex-rs/tui/src/multi_agents/mailbox_wait.rs @@ -22,14 +22,20 @@ pub(crate) fn begin(report: &MailboxWaitReport) -> PlainHistoryCell { return_when_label(report.return_when) ) } else if targetless { - format!("Waiting for mailbox attention or {} of {selection_count} work outcomes", return_when_label(report.return_when)) + format!( + "Waiting for mailbox attention or {} of {selection_count} work outcomes", + return_when_label(report.return_when) + ) } else { format!( "Waiting for {} of {selection_count} selected conditions", return_when_label(report.return_when) ) }; - collab_event(title_text(title), mailbox_wait_report_lines(report, /*in_progress*/ true)) + collab_event( + title_text(title), + mailbox_wait_report_lines(report, /*in_progress*/ true), + ) } pub(crate) fn end(report: &MailboxWaitReport) -> PlainHistoryCell { @@ -37,7 +43,10 @@ pub(crate) fn end(report: &MailboxWaitReport) -> PlainHistoryCell { Some(reason) => format!("Mailbox wait returned: {}", mailbox_wake_label(reason)), None => "Mailbox wait returned (cause unavailable)".to_string(), }; - collab_event(title_text(title), mailbox_wait_report_lines(report, /*in_progress*/ false)) + collab_event( + title_text(title), + mailbox_wait_report_lines(report, /*in_progress*/ false), + ) } #[derive(Clone, Debug)] @@ -189,7 +198,11 @@ fn mailbox_wait_report_lines(report: &MailboxWaitReport, in_progress: bool) -> V if !in_progress { lines.push(Line::from(format!( "Condition satisfied: {}", - if report.condition_satisfied { "yes" } else { "no" } + if report.condition_satisfied { + "yes" + } else { + "no" + } ))); if let Some(reason) = report.completion_reason { lines.push(Line::from(format!( diff --git a/docs/divergences/index.yaml b/docs/divergences/index.yaml index 53d6b572af06..e79344e513cb 100644 --- a/docs/divergences/index.yaml +++ b/docs/divergences/index.yaml @@ -3598,18 +3598,28 @@ "codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_message.rs", "codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs", "codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs", - "codex-rs/core/src/session/input_queue.rs" + "codex-rs/core/src/session/input_queue.rs", + "codex-rs/core/src/session/mod.rs", + "codex-rs/core/src/session/turn.rs", + "codex-rs/protocol/src/items.rs", + "codex-rs/app-server-protocol/src/protocol/v2/item.rs", + "codex-rs/tui/src/multi_agents/mailbox_wait.rs" ], "behavior": "The native default keeps exact-target queue-only filtering. Opt-in agent_mailbox supplies a core-independent library and separately versioned mailbox_1.sqlite for same-tree V2 accepted work, quiet messages, exact request replies, bounded reads, per-item observation, and any/all work/reply joins. Native acceptance and execution admission remain authoritative; canonical append and successful flush precede observation. Optional recipient idle activation never implies comprehension or business acceptance.", "upstream_equivalent": false, "introduced_in": {"carry_commit": null, "upstream_commit": null}, - "guardrail_lane": "codex.core-multi-agent-orchestration-targeted", + "guardrail_lane": "codex.core-multi-agent-orchestration-targeted; codex.core-subagent-notification-visibility-targeted", "tests": [ "native_retry_dispatch_reopen_and_capture_backpressure_are_safe", "admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped", "pending_deferred_and_terminal_continuations_bind_to_exact_generation", - "quiet_reply_and_author_dispositions_settle_only_the_exact_request", + "quiet_reply_and_author_dispositions_settle_only_exact_request", "canonical_outcomes_are_observed_per_item_without_cursor_loss", + "resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_batches", + "resumed_mailbox_wait_is_durably_interrupted_and_idempotent", + "interrupted_wait_call_id_replays_failure_but_a_fresh_id_uses_the_wait_path", + "wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delivers_them", + "unadmitted_canonical_message_survives_precreated_failure_and_manual_compaction", "only_closed_any_and_all_peer_cycles_are_provable", "multi_agent_v2_wait_agent_accepts_target_and_timeout_arguments", "multi_agent_v2_wait_agent_does_not_return_completed_content", diff --git a/justfile b/justfile index 0f6910cd52f8..2c2d5ea1fc47 100644 --- a/justfile +++ b/justfile @@ -381,6 +381,7 @@ tui-brokered-tool-replay-targeted: # and generation-safe V2 residency eviction. core-multi-agent-orchestration-targeted: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 + cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact From 09b428526b0ec41c93fa2b35e624702b623c4a99 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:41:22 +1100 Subject: [PATCH 09/54] fix(mailbox): retain exact goal custody across progress and recovery --- .../workflows/mailbox-source-preparation.yml | 104 ++++- codex-rs/agent-mailbox/src/lib.rs | 3 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 384 +++++++++++++++++- codex-rs/agent-mailbox/src/outcome.rs | 14 +- codex-rs/agent-mailbox/src/types.rs | 16 +- codex-rs/agent-mailbox/src/work.rs | 89 +++- .../schema/json/ServerNotification.json | 147 +++++++ .../codex_app_server_protocol.schemas.json | 147 +++++++ .../codex_app_server_protocol.v2.schemas.json | 147 +++++++ .../json/v2/ItemCompletedNotification.json | 147 +++++++ .../json/v2/ItemStartedNotification.json | 147 +++++++ .../schema/json/v2/ReviewStartResponse.json | 147 +++++++ .../schema/json/v2/ThreadForkResponse.json | 147 +++++++ .../schema/json/v2/ThreadListResponse.json | 147 +++++++ .../json/v2/ThreadMetadataUpdateResponse.json | 147 +++++++ .../schema/json/v2/ThreadReadResponse.json | 147 +++++++ .../schema/json/v2/ThreadResumeResponse.json | 147 +++++++ .../json/v2/ThreadRollbackResponse.json | 147 +++++++ .../schema/json/v2/ThreadStartResponse.json | 147 +++++++ .../json/v2/ThreadStartedNotification.json | 147 +++++++ .../json/v2/ThreadUnarchiveResponse.json | 147 +++++++ .../json/v2/TurnCompletedNotification.json | 147 +++++++ .../schema/json/v2/TurnStartResponse.json | 147 +++++++ .../json/v2/TurnStartedNotification.json | 147 +++++++ .../schema/typescript/MailboxWaitReport.ts | 15 + .../typescript/MailboxWaitReturnWhen.ts | 8 + .../typescript/MailboxWaitSelectionItem.ts | 8 + .../schema/typescript/MailboxWakeCause.ts | 8 + .../schema/typescript/index.ts | 4 + .../schema/typescript/v2/ThreadItem.ts | 7 +- codex-rs/core/config.schema.json | 12 + codex-rs/core/src/agent/control/mailbox.rs | 90 ++-- .../core/src/agent/control/mailbox_tests.rs | 175 +++++++- codex-rs/core/src/session/mod.rs | 109 ++--- codex-rs/core/src/thread_manager.rs | 34 +- .../src/tools/handlers/multi_agents_spec.rs | 3 +- .../tools/handlers/multi_agents_spec_tests.rs | 24 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 3 +- .../multi_agents_v2/mailbox_wait_tests.rs | 43 +- codex-rs/core/tests/suite/agent_mailbox.rs | 5 +- codex-rs/rollout/src/policy_tests.rs | 17 +- codex-rs/tui/src/chatwidget/tool_lifecycle.rs | 2 +- 42 files changed, 3665 insertions(+), 158 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/MailboxWaitReport.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/MailboxWaitReturnWhen.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/MailboxWaitSelectionItem.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/MailboxWakeCause.ts diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 08e56e09d726..39530b6ce83a 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,13 +15,16 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "6" + PREPARATION_ATTEMPT: "7" + SCCACHE_CACHE_SIZE: 2G steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: ref: ${{ github.sha }} fetch-depth: 0 persist-credentials: false + - name: Isolate workflow caches from source provenance + run: bash .github/scripts/isolate_workflow_paths_from_provenance.sh "$GITHUB_WORKSPACE" - uses: ./.github/actions/setup-bazel-ci with: target: x86_64-unknown-linux-gnu @@ -29,6 +32,46 @@ jobs: with: toolchain: 1.96.0 components: rustfmt, clippy + - name: Compute dependency cache identity + id: lockhash + shell: bash + run: | + set -euo pipefail + echo "hash=$(sha256sum codex-rs/Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum codex-rs/rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Restore cargo home cache + id: cache_cargo_home_restore + uses: actions/cache/restore@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/.rusty_v8/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: cargo-home-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} + - name: Install sccache + uses: taiki-e/install-action@7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c + with: + tool: sccache + - name: Configure the existing restore-only cache policy + id: sccache_backend + run: bash .github/scripts/configure_sccache_backend.sh restore-only + - name: Enable sccache wrapper + shell: bash + run: | + set -euo pipefail + echo "RUSTC_WRAPPER=$(command -v sccache)" >> "$GITHUB_ENV" + sccache --start-server || true + sccache --zero-stats || true + - name: Restore sccache cache + uses: actions/cache/restore@v6 + with: + path: ${{ github.workspace }}/.sccache/ + key: sccache-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }}-${{ github.run_id }} + restore-keys: | + sccache-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }}- + sccache-linux-rust-${{ steps.lockhash.outputs.hash }}- - uses: ./.github/actions/setup-rusty-v8 with: target: x86_64-unknown-linux-gnu @@ -56,30 +99,46 @@ jobs: working-directory: codex-rs run: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 - name: Verify canonical mailbox lifecycle retention + id: rollout + continue-on-error: true working-directory: codex-rs run: cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 - name: Generate the affected native schemas run: | just write-config-schema just write-app-server-schema - - name: Compile focused core mailbox and schema consumers - working-directory: codex-rs - run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 - - name: Run real native mailbox adapter contracts - working-directory: codex-rs - run: | - cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 - name: Compile focused UI mailbox consumers + id: ui + continue-on-error: true working-directory: codex-rs env: INSTA_UPDATE: new run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 - name: Verify interrupted mailbox wait history replay + id: protocol + continue-on-error: true working-directory: codex-rs run: cargo test --locked -p codex-app-server-protocol --lib mailbox -- --test-threads=1 - - name: Retain the library contract result + - name: Compile focused core mailbox and schema consumers + id: core + continue-on-error: true + working-directory: codex-rs + run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 + - name: Run real native mailbox adapter contracts + id: adapter + continue-on-error: true + working-directory: codex-rs + run: | + cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 + - name: Retain the focused contract results if: always() - run: test '${{ steps.library.outcome }}' = success + run: | + test '${{ steps.library.outcome }}' = success + test '${{ steps.rollout.outcome }}' = success + test '${{ steps.ui.outcome }}' = success + test '${{ steps.protocol.outcome }}' = success + test '${{ steps.core.outcome }}' = success + test '${{ steps.adapter.outcome }}' = success - name: Capture the exact generated source delta if: always() shell: bash @@ -106,3 +165,28 @@ jobs: name: mailbox-source-preparation-${{ github.sha }} path: ${{ runner.temp }}/mailbox-source-preparation/ retention-days: 3 + - name: Check post-lane cache-save disk headroom + if: ${{ always() && !cancelled() }} + id: disk_guard + shell: bash + run: | + set -euo pipefail + save_floor_bytes=$((8 * 1024 * 1024 * 1024)) + available_bytes="$(df --block-size=1 --output=avail "$GITHUB_WORKSPACE" | tail -n 1 | tr -d '[:space:]')" + if [[ "$available_bytes" -lt "$save_floor_bytes" ]]; then + echo "skip_cache_save=true" >> "$GITHUB_OUTPUT" + else + echo "skip_cache_save=false" >> "$GITHUB_OUTPUT" + fi + - name: Save cargo home cache after success or failure + if: ${{ always() && !cancelled() && steps.lockhash.outputs.hash != '' && steps.cache_cargo_home_restore.outputs.cache-hit != 'true' && steps.disk_guard.outputs.skip_cache_save == 'false' }} + continue-on-error: true + uses: actions/cache/save@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/.rusty_v8/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: cargo-home-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index f47f79db7c27..393fb572808e 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -27,7 +27,8 @@ pub use types::{ ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, CanonicalConfirmation, Continuation, ContinuationReceipt, ContinuationState, Dependency, DependencyMode, DispatchDecision, DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, NativeWork, - ObservationTarget, Outcome, OutcomeReceipt, PendingPage, RequestDisposition, RequestSnapshot, + ObservationTarget, Outcome, OutcomeReceipt, PendingPage, RecoveryEvidence, RequestDisposition, + RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotRequest, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, }; diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 97c30c0d7158..b10968d9d958 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -2,8 +2,9 @@ use crate::{ ActivationDecision, ActivationDisposition, AdmissionReceipt, Continuation, ContinuationState, Dependency, DependencyMode, DispatchDecision, DispatchState, Envelope, GoalBinding, Mailbox, MailboxError, MessageDraft, MessageKind, NativeWork, ObservationTarget, Outcome, - RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, - SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, is_provably_closed_peer_cycle, + RecoveryEvidence, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, + Snapshot, SnapshotItem, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, + is_provably_closed_peer_cycle, }; use codex_protocol::{AgentPath, ThreadId, protocol::InterAgentCommunication}; use codex_state::SqliteConfig; @@ -178,18 +179,57 @@ fn goal_binding( parent: ThreadId, child: ThreadId, generation: u64, - source_turn: &str, + incarnation: ThreadId, ) -> GoalBinding { GoalBinding { goal_id: "goal-1".to_string(), parent_thread_id: parent, child_thread_id: child, control_generation: generation, - revision: 1, - source_turn_id: source_turn.to_string(), + incarnation: Some(incarnation), } } +#[tokio::test] +async fn new_continuation_binding_requires_native_incarnation() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(40), thread_id(41)); + let (work, input) = + accept_work(&fixture.mailbox, issuer, agent, "missing-incarnation", "input").await; + assert_eq!( + fixture.mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&work).await.unwrap(); + admit(&fixture.mailbox, agent, "missing-incarnation-turn", &[input]).await; + + let mut legacy_binding = goal_binding(issuer, agent, 3, thread_id(42)); + legacy_binding.incarnation = None; + let result = fixture + .mailbox + .record_turn_outcome( + agent, + "missing-incarnation-turn", + Outcome::Completed, + None, + Continuation::PendingGoalBinding(legacy_binding), + ) + .await; + assert!(matches!(result, Err(MailboxError::InvalidInput(_)))); + + let snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(work.id.clone())], + ReturnWhen::Any, + ) + .await; + let item = work_item(&snapshot, &work.id); + assert!(item.outcome.is_none()); + assert!(item.continuations.is_empty()); +} + fn dependency(owner: &str, peers: &[&str], mode: DependencyMode) -> Dependency { Dependency { owner: owner.to_string(), @@ -464,8 +504,8 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "pending", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "deferred", "B").await; let bindings = [ - goal_binding(issuer, agent, 4, "turn-p"), - goal_binding(issuer, agent, 5, "turn-d"), + goal_binding(issuer, agent, 4, thread_id(44)), + goal_binding(issuer, agent, 5, thread_id(45)), ]; for (work, id, turn, continuation, state) in [ ( @@ -506,6 +546,7 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() let item = work_item(&snapshot, &work.id); assert!(item.outcome.is_none()); assert_eq!(item.continuations[0].state, state); + assert_eq!(item.continuations[0].source_turn_id, turn); } let mut stale = bindings[0].clone(); stale.control_generation += 1; @@ -531,6 +572,39 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() .await; assert!(work_item(&still_deferred, &a.id).outcome.is_none()); + let mut stale_incarnation = bindings[0].clone(); + stale_incarnation.incarnation = Some(thread_id(99)); + assert!( + finish( + &fixture.mailbox, + agent, + "stale-incarnation", + Outcome::Completed, + None, + Continuation::Terminal(stale_incarnation) + ) + .await + .no_matching_work + ); + let still_pending = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(a.id.clone())], + ReturnWhen::Any, + ) + .await; + assert!(work_item(&still_pending, &a.id).outcome.is_none()); + + let mut legacy_value = serde_json::to_value(&bindings[0]).unwrap(); + let legacy_fields = legacy_value.as_object_mut().unwrap(); + legacy_fields.remove("incarnation"); + legacy_fields.insert("revision".into(), serde_json::json!(8)); + legacy_fields.insert("source_turn_id".into(), serde_json::json!("old-turn")); + let legacy_binding: GoalBinding = serde_json::from_value(legacy_value).unwrap(); + assert_eq!(legacy_binding.incarnation, None); + assert_ne!(legacy_binding, bindings[0]); + for (work, binding, turn) in [ (&a, bindings[0].clone(), "terminal-p"), (&b, bindings[1].clone(), "terminal-d"), @@ -564,6 +638,290 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() } } +#[tokio::test] +async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(46), thread_id(47)); + let binding = goal_binding(issuer, agent, 6, thread_id(48)); + let (original, original_input) = + accept_work(&fixture.mailbox, issuer, agent, "continuation-origin", "origin").await; + assert_eq!( + fixture.mailbox.begin_dispatch(&original).await.unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&original).await.unwrap(); + admit( + &fixture.mailbox, + agent, + "source-origin", + &[original_input], + ) + .await; + finish( + &fixture.mailbox, + agent, + "source-origin", + Outcome::Completed, + None, + Continuation::PendingGoalBinding(binding.clone()), + ) + .await; + finish( + &fixture.mailbox, + agent, + "source-origin", + Outcome::Completed, + None, + Continuation::Deferred(binding.clone()), + ) + .await; + + let deferred_snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(original.id.clone())], + ReturnWhen::Any, + ) + .await; + let deferred = work_item(&deferred_snapshot, &original.id); + assert!(deferred.outcome.is_none()); + assert_eq!(deferred.continuations[0].binding, binding); + assert_eq!(deferred.continuations[0].state, ContinuationState::Deferred); + assert_eq!(deferred.continuations[0].source_turn_id, "source-origin"); + + let (later_work, later_input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "continuation-terminal", + "later continuation turn", + ) + .await; + assert_eq!( + fixture.mailbox.begin_dispatch(&later_work).await.unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&later_work).await.unwrap(); + admit( + &fixture.mailbox, + agent, + "source-terminal", + &[later_input], + ) + .await; + let terminal = finish( + &fixture.mailbox, + agent, + "source-terminal", + Outcome::Completed, + None, + Continuation::Terminal(binding.clone()), + ) + .await; + assert!( + terminal + .outcomes + .iter() + .any(|receipt| receipt.work_ref == original) + ); + + let terminal_snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(original.id.clone())], + ReturnWhen::Any, + ) + .await; + let settled = work_item(&terminal_snapshot, &original.id); + assert!(settled.outcome.is_some()); + assert_eq!(settled.continuations[0].binding, binding); + assert_eq!(settled.continuations[0].state, ContinuationState::Terminal); + assert_eq!(settled.continuations[0].source_turn_id, "source-origin"); +} + +#[tokio::test] +async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_terminal() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(50), thread_id(51)); + let binding = goal_binding(issuer, agent, 7, thread_id(52)); + let (original, original_input) = + accept_work(&fixture.mailbox, issuer, agent, "pending-continuation", "original").await; + assert_eq!( + fixture.mailbox.begin_dispatch(&original).await.unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&original).await.unwrap(); + admit( + &fixture.mailbox, + agent, + "source-pending", + &[original_input], + ) + .await; + let pending = finish( + &fixture.mailbox, + agent, + "source-pending", + Outcome::Completed, + None, + Continuation::PendingGoalBinding(binding.clone()), + ) + .await; + assert_eq!(pending.continuation_work, vec![original.clone()]); + + assert!( + !fixture + .mailbox + .mark_recovery_needed(&original, RecoveryEvidence::DeliveryUnavailable) + .await + .unwrap() + ); + let mut different_incarnation = binding.clone(); + different_incarnation.incarnation = Some(thread_id(53)); + assert!( + !fixture + .mailbox + .mark_recovery_needed( + &original, + RecoveryEvidence::ContinuationUnowned { + binding: different_incarnation.clone(), + }, + ) + .await + .unwrap() + ); + assert!( + fixture + .mailbox + .mark_recovery_needed( + &original, + RecoveryEvidence::ContinuationUnowned { + binding: binding.clone(), + }, + ) + .await + .unwrap() + ); + + let pending_snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(original.id.clone())], + ReturnWhen::Any, + ) + .await; + let retained = work_item(&pending_snapshot, &original.id); + assert_eq!(retained.dispatch_state, DispatchState::RecoveryNeeded); + assert!(retained.outcome.is_none()); + assert_eq!(retained.continuations.len(), 1); + assert_eq!(retained.continuations[0].binding, binding); + assert_eq!( + retained.continuations[0].state, + ContinuationState::PendingGoalBinding + ); + + let (wrong_terminal, wrong_input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "wrong-incarnation-terminal", + "wrong incarnation", + ) + .await; + assert_eq!( + fixture + .mailbox + .begin_dispatch(&wrong_terminal) + .await + .unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&wrong_terminal).await.unwrap(); + admit( + &fixture.mailbox, + agent, + "source-wrong-terminal", + &[wrong_input], + ) + .await; + finish( + &fixture.mailbox, + agent, + "source-wrong-terminal", + Outcome::Completed, + None, + Continuation::Terminal(different_incarnation), + ) + .await; + + let still_recovery_needed = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(original.id.clone())], + ReturnWhen::Any, + ) + .await; + assert_eq!( + work_item(&still_recovery_needed, &original.id).dispatch_state, + DispatchState::RecoveryNeeded + ); + assert!(work_item(&still_recovery_needed, &original.id).outcome.is_none()); + + let (exact_terminal, exact_input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "exact-incarnation-terminal", + "exact incarnation", + ) + .await; + assert_eq!( + fixture + .mailbox + .begin_dispatch(&exact_terminal) + .await + .unwrap(), + DispatchDecision::Dispatch + ); + fixture.mailbox.mark_dispatched(&exact_terminal).await.unwrap(); + admit( + &fixture.mailbox, + agent, + "source-exact-terminal", + &[exact_input], + ) + .await; + let terminal = finish( + &fixture.mailbox, + agent, + "source-exact-terminal", + Outcome::Completed, + None, + Continuation::Terminal(binding.clone()), + ) + .await; + assert!(terminal.outcomes.iter().any(|receipt| receipt.work_ref == original)); + + let settled = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(original.id.clone())], + ReturnWhen::Any, + ) + .await; + let settled_original = work_item(&settled, &original.id); + assert_eq!(settled_original.dispatch_state, DispatchState::Dispatched); + assert!(settled_original.outcome.is_some()); + assert_eq!(settled_original.continuations[0].binding, binding); + assert_eq!(settled_original.continuations[0].state, ContinuationState::Terminal); + assert_eq!(settled_original.continuations[0].source_turn_id, "source-pending"); +} + #[tokio::test] async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let fixture = test_mailbox().await; @@ -1338,7 +1696,11 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&work).await.unwrap(); - fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); + assert!(fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) + .await + .unwrap()); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::RecoveryNeeded @@ -1358,7 +1720,11 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov ) .await; assert_eq!(terminal.outcomes[0].work_ref, work); - fixture.mailbox.mark_recovery_needed(&work).await.unwrap(); + assert!(!fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) + .await + .unwrap()); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index d42be36287e0..fe37cd32a85d 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -455,15 +455,23 @@ fn validate_binding_for_agent(binding: &GoalBinding, agent: ThreadId) -> Mailbox if binding.child_thread_id != agent { return Err(MailboxError::WrongRecipient); } - validate_key(&binding.source_turn_id) + validate_native_binding(binding) +} + +fn validate_native_binding(binding: &GoalBinding) -> MailboxResult<()> { + if binding.incarnation.is_none() { + return Err(MailboxError::InvalidInput( + "native goal incarnation is required".into(), + )); + } + validate_key(&binding.goal_id) } fn validate_binding_for_work(binding: &GoalBinding, work_ref: &WorkRef) -> MailboxResult<()> { if binding.parent_thread_id != work_ref.issuer || binding.child_thread_id != work_ref.agent { return Err(MailboxError::WrongRecipient); } - validate_key(&binding.goal_id)?; - validate_key(&binding.source_turn_id) + validate_native_binding(binding) } fn validate_key(key: &str) -> MailboxResult<()> { diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 26e27e802633..159cd1d64db1 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -105,8 +105,10 @@ pub struct GoalBinding { pub parent_thread_id: ThreadId, pub child_thread_id: ThreadId, pub control_generation: u64, - pub revision: u64, - pub source_turn_id: String, + /// Process-local native goal-store identity. Older persisted bindings omit + /// this field and remain readable, but cannot match a new native producer. + #[serde(default)] + pub incarnation: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -118,6 +120,16 @@ pub enum DispatchState { RecoveryNeeded, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RecoveryEvidence { + /// The adapter proved there is no native delivery/continuation custody. + /// This is valid only when no pending or deferred goal binding remains. + DeliveryUnavailable, + /// The adapter proved this exact native goal binding has no owner. + /// The durable continuation itself is retained for later exact settlement. + ContinuationUnowned { binding: GoalBinding }, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DispatchDecision { Dispatch, diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index a6884507be04..e8469f7d6591 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -1,7 +1,11 @@ use crate::error::{MailboxError, MailboxResult}; use crate::messages::decrement_pending_count; +use crate::outcome_store::MAX_CONTINUATIONS_PER_WORK; use crate::store::Mailbox; -use crate::types::{AdmissionReceipt, DispatchDecision, DispatchState, NativeWork, WorkRef}; +use crate::types::{ + AdmissionReceipt, DispatchDecision, DispatchState, GoalBinding, NativeWork, RecoveryEvidence, + WorkRef, +}; use codex_protocol::ThreadId; use codex_protocol::protocol::InterAgentCommunication; use sqlx::{Row, Sqlite, Transaction}; @@ -287,12 +291,55 @@ impl Mailbox { } /// Record adapter-confirmed stale in-flight custody for this exact work. - pub async fn mark_recovery_needed(&self, work_ref: &WorkRef) -> MailboxResult<()> { + /// Returns whether this call changed the dispatch state. + pub async fn mark_recovery_needed( + &self, + work_ref: &WorkRef, + evidence: RecoveryEvidence, + ) -> MailboxResult { let mut transaction = self.pool.begin().await?; let row = exact_work_tx(&mut transaction, work_ref).await?; let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; - match state { - DispatchState::Dispatching | DispatchState::Dispatched => { + if state == DispatchState::Accepted { + return Err(MailboxError::ReferenceMismatch); + } + + let matching_binding_json = match &evidence { + RecoveryEvidence::DeliveryUnavailable => None, + RecoveryEvidence::ContinuationUnowned { binding } => { + if binding.parent_thread_id != work_ref.issuer + || binding.child_thread_id != work_ref.agent + { + return Err(MailboxError::WrongRecipient); + } + validate_key(&binding.goal_id, "goal id")?; + let rows = sqlx::query( + "SELECT binding_json FROM mailbox_continuations + WHERE work_id = ? AND state IN ('pending_goal_binding', 'deferred') + ORDER BY binding_json LIMIT ?", + ) + .bind(&work_ref.id) + .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .fetch_all(&mut *transaction) + .await?; + if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { + return Err(MailboxError::Backpressure("work continuation count")); + } + let mut matched = None; + for row in rows { + let stored_json: String = row.try_get("binding_json")?; + let stored: GoalBinding = serde_json::from_str(&stored_json)?; + if stored == *binding { + matched = Some(stored_json); + break; + } + } + matched + } + }; + + let update = match (&evidence, matching_binding_json) { + (RecoveryEvidence::DeliveryUnavailable, _) => { sqlx::query( "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' WHERE id = ? AND dispatch_state IN ('dispatching', 'dispatched') @@ -305,16 +352,36 @@ impl Mailbox { ) .bind(&work_ref.id) .execute(&mut *transaction) - .await?; + .await? } - DispatchState::RecoveryNeeded => {} - DispatchState::Accepted => { - return Err(MailboxError::ReferenceMismatch); + (RecoveryEvidence::ContinuationUnowned { .. }, Some(binding_json)) => { + sqlx::query( + "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' + WHERE id = ? AND dispatch_state IN ('dispatching', 'dispatched') + AND outcome_json IS NULL + AND EXISTS ( + SELECT 1 FROM mailbox_continuations AS continuation + WHERE continuation.work_id = mailbox_work.id + AND continuation.binding_json = ? + AND continuation.state IN ('pending_goal_binding', 'deferred') + )", + ) + .bind(&work_ref.id) + .bind(binding_json) + .execute(&mut *transaction) + .await? } - } + (RecoveryEvidence::ContinuationUnowned { .. }, None) => { + transaction.commit().await?; + return Ok(false); + } + }; + let changed = update.rows_affected() == 1; transaction.commit().await?; - self.notify_changed(); - Ok(()) + if changed { + self.notify_changed(); + } + Ok(changed) } pub async fn mark_dispatched(&self, work_ref: &WorkRef) -> MailboxResult<()> { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b014e45342ec..29897abc712f 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2985,6 +2985,142 @@ ], "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpServerOauthLoginCompletedNotification": { "properties": { "error": { @@ -6098,6 +6234,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 79e69933aa8d..1199a690e7cb 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -12234,6 +12234,142 @@ "title": "LogoutAccountResponse", "type": "object" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/definitions/v2/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/definitions/v2/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/definitions/v2/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/definitions/v2/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "ManagedHooksRequirements": { "properties": { "PermissionRequest": { @@ -19112,6 +19248,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/definitions/v2/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 1846d134db3a..44866489b58f 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -8641,6 +8641,142 @@ "title": "LogoutAccountResponse", "type": "object" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/definitions/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/definitions/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/definitions/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/definitions/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "ManagedHooksRequirements": { "properties": { "PermissionRequest": { @@ -16801,6 +16937,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/definitions/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index c63a0e08b55c..3a261a25d738 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -639,6 +639,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1451,6 +1587,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 21cce96a698a..02bf9bb3c9ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -639,6 +639,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1451,6 +1587,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 26e8826898b1..2d4ad7e507d6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -781,6 +781,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1600,6 +1736,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index af415ccc329c..fa3330bf9d0c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -882,6 +882,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -2143,6 +2279,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 6d668065917f..e9fa814baa7e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index d9aaa20f5fc6..962cdd41505e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index b67cb8f58800..24775bb0d716 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 10880a20d113..13d0117a703e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -882,6 +882,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -2143,6 +2279,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index a9941f4958f0..e159fce7bae5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 13cda1ffbd77..dbbc13422ff4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -882,6 +882,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -2143,6 +2279,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index d3a1b7d9d1ea..0539ac1bc154 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 9ef50ef080dc..ecaaaecf7f97 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -804,6 +804,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1946,6 +2082,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 372044da2b11..47c09935f732 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -781,6 +781,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1600,6 +1736,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 78c329ce5d49..f8d3ac6705e5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -781,6 +781,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1600,6 +1736,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index ad2cdd486eb1..c5b0f85c8a97 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -781,6 +781,142 @@ "LegacyAppPathString": { "type": "string" }, + "MailboxWaitReport": { + "description": "Shared mailbox wait report carried by tool results and collaboration history.\n\n`completion_reason` is null while the wait is in progress and populated with\nthe actual wake/result cause when it returns. It is always present in a terminal\nwait tool result.", + "properties": { + "completed": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "completionReason": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWakeCause" + }, + { + "type": "null" + } + ], + "default": null + }, + "conditionSatisfied": { + "type": "boolean" + }, + "pending": { + "items": { + "$ref": "#/$defs/MailboxWaitSelectionItem" + }, + "type": "array" + }, + "quietCount": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "requestedRepliesTo": { + "items": { + "type": "string" + }, + "type": "array" + }, + "requestedTargets": { + "items": { + "type": "string" + }, + "type": "array" + }, + "returnWhen": { + "$ref": "#/$defs/MailboxWaitReturnWhen", + "default": "any" + }, + "wakeRef": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "wakeSender": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "requestedTargets", + "requestedRepliesTo", + "completed", + "pending", + "conditionSatisfied", + "quietCount" + ], + "type": "object" + }, + "MailboxWaitReturnWhen": { + "description": "Whether a wait requires any one selected condition or every selected condition.", + "enum": [ + "any", + "all" + ], + "type": "string" + }, + "MailboxWaitSelectionItem": { + "description": "One selected wait condition and its current or terminal state.", + "properties": { + "agent": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "messageId": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "workRef": { + "default": null, + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "MailboxWakeCause": { + "description": "Actual outcome that ended a selection-aware mailbox wait.", + "enum": [ + "result", + "reply", + "request", + "human", + "tool", + "deadline", + "cancel", + "interrupted", + "subscription_loss", + "runtime_fault", + "no_matching_work", + "already_satisfied", + "cycle", + "needs_attention" + ], + "type": "string" + }, "McpToolCallAppContext": { "properties": { "actionName": { @@ -1600,6 +1736,17 @@ "description": "Unique identifier for this collab tool call.", "type": "string" }, + "mailboxWaitReport": { + "anyOf": [ + { + "$ref": "#/$defs/MailboxWaitReport" + }, + { + "type": "null" + } + ], + "description": "Selection-aware mailbox wait state/result, when reported by the producer." + }, "model": { "description": "Established model alias for a spawned-agent lifecycle item.\n\nOn spawn start, this is the caller-requested model. On a terminal spawn item, this is the observed effective model. An unknown terminal effective model is null.", "type": [ diff --git a/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReport.ts b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReport.ts new file mode 100644 index 000000000000..39a375be7be3 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReport.ts @@ -0,0 +1,15 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { MailboxWaitReturnWhen } from "./MailboxWaitReturnWhen"; +import type { MailboxWaitSelectionItem } from "./MailboxWaitSelectionItem"; +import type { MailboxWakeCause } from "./MailboxWakeCause"; + +/** + * Shared mailbox wait report carried by tool results and collaboration history. + * + * `completion_reason` is null while the wait is in progress and populated with + * the actual wake/result cause when it returns. It is always present in a terminal + * wait tool result. + */ +export type MailboxWaitReport = { requestedTargets: Array, requestedRepliesTo: Array, returnWhen: MailboxWaitReturnWhen, completed: Array, pending: Array, completionReason: MailboxWakeCause | null, conditionSatisfied: boolean, quietCount: number, wakeSender: string | null, wakeRef: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReturnWhen.ts b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReturnWhen.ts new file mode 100644 index 000000000000..85a310256cbe --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitReturnWhen.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Whether a wait requires any one selected condition or every selected condition. + */ +export type MailboxWaitReturnWhen = "any" | "all"; diff --git a/codex-rs/app-server-protocol/schema/typescript/MailboxWaitSelectionItem.ts b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitSelectionItem.ts new file mode 100644 index 000000000000..2b406623a15c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/MailboxWaitSelectionItem.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * One selected wait condition and its current or terminal state. + */ +export type MailboxWaitSelectionItem = { workRef: string | null, messageId: string | null, agent: string | null, status: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/MailboxWakeCause.ts b/codex-rs/app-server-protocol/schema/typescript/MailboxWakeCause.ts new file mode 100644 index 000000000000..d6e964d2a47c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/MailboxWakeCause.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Actual outcome that ended a selection-aware mailbox wait. + */ +export type MailboxWakeCause = "result" | "reply" | "request" | "human" | "tool" | "deadline" | "cancel" | "interrupted" | "subscription_loss" | "runtime_fault" | "no_matching_work" | "already_satisfied" | "cycle" | "needs_attention"; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index 0ba7c7adb86a..6b96fe984678 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -51,6 +51,10 @@ export type { LegacyAppPathString } from "./LegacyAppPathString"; export type { LocalShellAction } from "./LocalShellAction"; export type { LocalShellExecAction } from "./LocalShellExecAction"; export type { LocalShellStatus } from "./LocalShellStatus"; +export type { MailboxWaitReport } from "./MailboxWaitReport"; +export type { MailboxWaitReturnWhen } from "./MailboxWaitReturnWhen"; +export type { MailboxWaitSelectionItem } from "./MailboxWaitSelectionItem"; +export type { MailboxWakeCause } from "./MailboxWakeCause"; export type { McpServerInfo } from "./McpServerInfo"; export type { MessagePhase } from "./MessagePhase"; export type { ModeKind } from "./ModeKind"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index a6e37404af42..f4db43a00dbb 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -4,6 +4,7 @@ import type { CollabWaitingCompletionReason } from "../CollabWaitingCompletionReason"; import type { ImageGenerationItem } from "../ImageGenerationItem"; import type { LegacyAppPathString } from "../LegacyAppPathString"; +import type { MailboxWaitReport } from "../MailboxWaitReport"; import type { MessagePhase } from "../MessagePhase"; import type { ReasoningEffort } from "../ReasoningEffort"; import type { SleepItem } from "../SleepItem"; @@ -160,7 +161,11 @@ agentsStates: { [key in string]: CollabAgentState }, /** * Safe mailbox notifications observed by a native wait. */ -wakeNotifications: Array | null, completionReason: CollabWaitingCompletionReason | null, wakeCause?: AgentWakeCause, } | { "type": "subAgentActivity", id: string, kind: SubAgentActivityKind, agentThreadId: string, agentPath: string, +wakeNotifications: Array | null, completionReason: CollabWaitingCompletionReason | null, wakeCause?: AgentWakeCause, +/** + * Selection-aware mailbox wait state/result, when reported by the producer. + */ +mailboxWaitReport?: MailboxWaitReport, } | { "type": "subAgentActivity", id: string, kind: SubAgentActivityKind, agentThreadId: string, agentPath: string, /** * Effective model selected for the affected child, when known. */ diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 61903c1cb020..d33706902dfa 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -493,6 +493,12 @@ "default": null, "description": "Optional feature toggles scoped to this profile.", "properties": { + "agent_mailbox": { + "type": "boolean" + }, + "agent_mailbox_idle_activation": { + "type": "boolean" + }, "apply_patch_freeform": { "type": "boolean" }, @@ -6414,6 +6420,12 @@ "default": null, "description": "Centralized feature flags (new). Prefer this over individual toggles.", "properties": { + "agent_mailbox": { + "type": "boolean" + }, + "agent_mailbox_idle_activation": { + "type": "boolean" + }, "apply_patch_freeform": { "type": "boolean" }, diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 0eaa048927a0..f8f26f2febf5 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -12,13 +12,16 @@ use crate::agent_communication::AgentCommunicationKind; use crate::config::Config; use codex_agent_mailbox::ActivationDisposition; use codex_agent_mailbox::Continuation; +use codex_agent_mailbox::ContinuationState; use codex_agent_mailbox::DispatchDecision; use codex_agent_mailbox::Envelope; use codex_agent_mailbox::GoalBinding; use codex_agent_mailbox::Mailbox; use codex_agent_mailbox::NativeWork; use codex_agent_mailbox::Outcome; +use codex_agent_mailbox::RecoveryEvidence; use codex_agent_mailbox::WorkRef; +use codex_agent_mailbox::WorkSnapshot; use codex_features::Feature; use codex_protocol::ResponseItemId; use codex_protocol::ThreadId; @@ -37,6 +40,18 @@ fn storage_error(error: impl std::fmt::Display) -> CodexErr { CodexErr::Fatal(format!("agent mailbox storage failed: {error}")) } +fn goal_binding_matches_native( + binding: &GoalBinding, + incarnation: ThreadId, + snapshot: &GoalNotificationSnapshot, +) -> bool { + binding.incarnation == Some(incarnation) + && binding.goal_id == snapshot.binding.goal_id + && binding.parent_thread_id == snapshot.binding.parent_thread_id + && binding.child_thread_id == snapshot.binding.child_thread_id + && binding.control_generation == snapshot.binding.control_generation +} + impl AgentControl { pub(crate) async fn mailbox_initialize(&self, config: &Config) -> CodexResult<()> { if config.multi_agent_version_from_features() != MultiAgentVersion::V2 @@ -118,9 +133,9 @@ impl AgentControl { return Ok(None); }; if let Some(existing) = mailbox - .work_for_native_key(context.sender_thread_id, native_key) - .await - .map_err(storage_error)? + .work_for_native_key(context.sender_thread_id, native_key) + .await + .map_err(storage_error)? { *submission_id = existing.submission_id; } @@ -299,31 +314,54 @@ impl AgentControl { .collect()) } - pub(crate) async fn mailbox_reconcile_pending_work(&self, work: &WorkRef) -> CodexResult { + pub(crate) async fn mailbox_reconcile_pending_work(&self, work: &WorkSnapshot) -> CodexResult { let Some(mailbox) = self.mailbox() else { return Ok(false); }; let state = self.upgrade()?; - let can_proceed = match state.get_thread(work.agent).await { - Ok(thread) => { + let (can_proceed, goal_owner) = match state.get_thread(work.work_ref.agent).await { + Ok(thread) if thread.is_running() => { let active = { thread.session.active_turn.lock().await.is_some() }; - active + let can_proceed = active || thread.session.input_queue.has_pending_mailbox_items().await || thread.session.input_queue.has_pending_terminal_finalizers() - || thread.session.has_outstanding_durable_sleep() + || thread.session.has_outstanding_durable_sleep(); + let goal_owner = thread.session.services.thread_extension_data + .get::() + .and_then(|store| store.snapshot().map(|snapshot| (store.incarnation(), snapshot))); + (can_proceed, goal_owner) } - Err(_) => false, + Ok(_) | Err(_) => (false, None), }; - if can_proceed { + let unresolved = work.continuations.iter().find(|continuation| { + matches!(continuation.state, ContinuationState::PendingGoalBinding | ContinuationState::Deferred) + }); + let evidence = if let Some(continuation) = unresolved { + if goal_owner.as_ref().is_some_and(|(incarnation, snapshot)| { + goal_binding_matches_native(&continuation.binding, *incarnation, snapshot) + && ((snapshot.status == ThreadGoalStatus::Active + && matches!(snapshot.phase, GoalNotificationPhase::Running + | GoalNotificationPhase::ContinuationPending + | GoalNotificationPhase::DeferredWithOwner)) + || (snapshot.phase == GoalNotificationPhase::ResultReady && can_proceed)) + }) { + return Ok(false); + } + // A cold, restarted, replaced, or action-required producer cannot prove custody + // of this exact old logical continuation. Keep its binding and original refs; + // return recovery-needed instead of waiting for a producer that cannot advance it. + RecoveryEvidence::ContinuationUnowned { binding: continuation.binding.clone() } + } else if can_proceed { return Ok(false); - } + } else { + RecoveryEvidence::DeliveryUnavailable + }; // This only changes recovery disposition, never replays input or invents a result. // The store rechecks terminal/continuation custody atomically against this native read. mailbox - .mark_recovery_needed(work) + .mark_recovery_needed(&work.work_ref, evidence) .await - .map_err(storage_error)?; - Ok(true) + .map_err(storage_error) } pub(crate) async fn mailbox_record_admission( @@ -359,11 +397,11 @@ impl AgentControl { recipient: ThreadId, source_turn_id: &str, status: AgentStatus, - goal: Option, + goal: Option<(ThreadId, GoalNotificationSnapshot)>, mut completion: Option, - ) -> CodexResult<()> { + ) -> CodexResult { let Some(mailbox) = self.mailbox() else { - return Ok(()); + return Ok(false); }; if let Some(communication) = completion.as_mut() && communication.encrypted_content.is_none() @@ -387,14 +425,13 @@ impl AgentControl { } }; let continuation = match goal { - Some(snapshot) => { + Some((incarnation, snapshot)) => { let binding = GoalBinding { goal_id: snapshot.binding.goal_id, parent_thread_id: snapshot.binding.parent_thread_id, child_thread_id: snapshot.binding.child_thread_id, control_generation: snapshot.binding.control_generation, - revision: snapshot.revision, - source_turn_id: source_turn_id.to_string(), + incarnation: Some(incarnation), }; if matches!(outcome, Outcome::Failed | Outcome::Cancelled) { Continuation::Terminal(binding) @@ -415,14 +452,17 @@ impl AgentControl { } None => Continuation::None, }; - if let Err(error) = mailbox + let receipt = match mailbox .record_turn_outcome(recipient, source_turn_id, outcome, completion, continuation) .await { - mailbox.notify_runtime_fault(recipient); - return Err(storage_error(error)); - } - Ok(()) + Ok(receipt) => receipt, + Err(error) => { + mailbox.notify_runtime_fault(recipient); + return Err(storage_error(error)); + } + }; + Ok(!receipt.no_matching_work) } } diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index e56d22a9a046..8ce9d80913b8 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -1,4 +1,11 @@ use super::*; +use crate::agent::goal_notifications::GoalNotificationBinding; +use crate::agent::goal_notifications::GoalNotificationClassification; +use codex_agent_mailbox::DispatchState; +use codex_agent_mailbox::ReturnWhen; +use codex_agent_mailbox::Selection; +use codex_agent_mailbox::Selector; +use codex_agent_mailbox::SnapshotItem; use codex_protocol::AgentPath; use pretty_assertions::assert_eq; @@ -20,17 +27,175 @@ async fn mailbox_shared_control_does_not_capture_stock_invocations() { ); let original = communication.clone(); let mut submission_id = "stock-submission".to_string(); - let context = AgentCommunicationContext::new(AgentCommunicationKind::Followup, session.thread_id); - assert!(control.mailbox_accept_native_submission(recipient, &context, &mut communication, &mut submission_id).await.unwrap().is_none()); + let context = + AgentCommunicationContext::new(AgentCommunicationKind::Followup, session.thread_id); + assert!( + control + .mailbox_accept_native_submission( + recipient, + &context, + &mut communication, + &mut submission_id + ) + .await + .unwrap() + .is_none() + ); assert_eq!(communication, original); assert_eq!(submission_id, "stock-submission"); let mailbox = control.mailbox().unwrap(); - assert!(mailbox.issued_work_refs(session.thread_id, None, 100).await.unwrap().is_empty()); + assert!( + mailbox + .issued_work_refs(session.thread_id, None, 100) + .await + .unwrap() + .is_empty() + ); let enabled_context = context.with_native_key("enabled-call".to_string()); - let (accepted, decision) = control.mailbox_accept_native_submission(recipient, &enabled_context, &mut communication, &mut submission_id).await.unwrap().unwrap(); + let (accepted, decision) = control + .mailbox_accept_native_submission( + recipient, + &enabled_context, + &mut communication, + &mut submission_id, + ) + .await + .unwrap() + .unwrap(); assert_eq!(decision, DispatchDecision::Dispatch); assert_eq!(accepted.issuer, session.thread_id); - assert_eq!(mailbox.accepted_input(&accepted).await.unwrap(), communication); + assert_eq!( + mailbox.accepted_input(&accepted).await.unwrap(), + communication + ); assert!(communication.id.unwrap().as_str().starts_with("amsg_")); } + +fn goal_snapshot(issuer: ThreadId, agent: ThreadId) -> GoalNotificationSnapshot { + GoalNotificationSnapshot { + binding: GoalNotificationBinding { + parent_thread_id: issuer, + child_thread_id: agent, + goal_id: "native-goal".to_string(), + control_generation: 7, + }, + status: ThreadGoalStatus::Active, + phase: GoalNotificationPhase::ContinuationPending, + revision: 1, + last_completed_source_turn: Some("accepted-turn".to_string()), + result_reference: None, + classification: GoalNotificationClassification::Progress, + } +} + +async fn accepted_goal_work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId) -> WorkRef { + let work = mailbox.accept_work(NativeWork { + native_key: "goal-call".to_string(), + issuer, + agent, + submission_id: "goal-submission".to_string(), + input: InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + "complete the accepted work".to_string(), + true, + ), + }).await.unwrap(); + assert_eq!(mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + mailbox.mark_dispatched(&work).await.unwrap(); + let input = mailbox.accepted_input(&work).await.unwrap(); + mailbox.record_admission(agent, "accepted-turn", &[input.id.unwrap().to_string()]).await.unwrap(); + work +} + +async fn exact_work_snapshot(mailbox: &Mailbox, work: &WorkRef) -> WorkSnapshot { + let snapshot = mailbox.snapshot(&Selection { + caller: work.issuer, + agent: Some(work.agent), + selectors: vec![Selector::Work(work.id.clone())], + return_when: ReturnWhen::All, + limit: 1, + }).await.unwrap(); + match snapshot.items.into_iter().next().unwrap() { + SnapshotItem::Work(work) => work, + SnapshotItem::Request(_) => panic!("exact work selector returned a request"), + } +} + +#[tokio::test] +async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_and_turns() { + let (session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let control = &session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let mailbox = control.mailbox().unwrap(); + let agent = ThreadId::new(); + let incarnation = ThreadId::new(); + let work = accepted_goal_work(&mailbox, session.thread_id, agent).await; + let mut goal = goal_snapshot(session.thread_id, agent); + + assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + goal.phase = GoalNotificationPhase::DeferredWithOwner; + goal.revision += 1; + assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + let pending = exact_work_snapshot(&mailbox, &work).await; + assert!(pending.outcome.is_none()); + assert_eq!(pending.continuations.len(), 1); + assert_eq!(pending.continuations[0].source_turn_id, "accepted-turn"); + assert_eq!(pending.continuations[0].state, ContinuationState::Deferred); + + goal.phase = GoalNotificationPhase::ResultReady; + goal.status = ThreadGoalStatus::Complete; + goal.revision += 1; + goal.last_completed_source_turn = Some("continuation-turn".to_string()); + goal.classification = GoalNotificationClassification::Result; + // A restarted/replaced native store is not evidence for the old producer, + // even when its goal ID, generation and thread IDs happen to be identical. + assert!(!control.mailbox_record_terminal(agent, "continuation-turn", AgentStatus::Completed(None), Some((ThreadId::new(), goal.clone())), None).await.unwrap()); + assert!(exact_work_snapshot(&mailbox, &work).await.outcome.is_none()); + assert!(control.mailbox_record_terminal(agent, "continuation-turn", AgentStatus::Completed(None), Some((incarnation, goal)), None).await.unwrap()); + let completed = exact_work_snapshot(&mailbox, &work).await; + assert_eq!(completed.outcome.unwrap().outcome, Outcome::Completed); + assert_eq!(completed.continuations[0].state, ContinuationState::Terminal); + // An unrelated native result must retain the existing native fallback path. + assert!(!control.mailbox_record_terminal(ThreadId::new(), "uncaptured-turn", AgentStatus::Completed(None), None, None).await.unwrap()); +} + +#[tokio::test] +async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody() { + let (session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let control = &session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let mailbox = control.mailbox().unwrap(); + let agent = ThreadId::new(); + let incarnation = ThreadId::new(); + let work = accepted_goal_work(&mailbox, session.thread_id, agent).await; + let mut goal = goal_snapshot(session.thread_id, agent); + goal.phase = GoalNotificationPhase::DeferredWithOwner; + assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + let pending = exact_work_snapshot(&mailbox, &work).await; + + // This exact producer has no loaded native thread or continuation owner. + assert!(control.mailbox_reconcile_pending_work(&pending).await.unwrap()); + let recovered = exact_work_snapshot(&mailbox, &work).await; + assert_eq!(recovered.work_ref, work); + assert_eq!(recovered.dispatch_state, DispatchState::RecoveryNeeded); + assert!(recovered.outcome.is_none()); + assert_eq!(recovered.input, pending.input); + assert_eq!(recovered.continuations, pending.continuations); + assert!(!control.mailbox_reconcile_pending_work(&recovered).await.unwrap()); + + goal.phase = GoalNotificationPhase::ResultReady; + goal.status = ThreadGoalStatus::Complete; + goal.revision += 1; + goal.classification = GoalNotificationClassification::Result; + assert!(control.mailbox_record_terminal(agent, "recovered-turn", AgentStatus::Completed(None), Some((incarnation, goal)), None).await.unwrap()); + assert_eq!(exact_work_snapshot(&mailbox, &work).await.outcome.unwrap().outcome, Outcome::Completed); +} diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index fd5d3bbb4471..ecc377803cfb 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1475,9 +1475,7 @@ impl Session { let TurnItem::CollabAgentToolCall(call) = &event.item else { continue; }; - if call.tool != CollabAgentTool::Wait - || call.mailbox_wait_report.is_none() - { + if call.tool != CollabAgentTool::Wait || call.mailbox_wait_report.is_none() { continue; } saw_interrupted_terminal |= call @@ -1496,9 +1494,7 @@ impl Session { let TurnItem::CollabAgentToolCall(call) = &event.item else { continue; }; - if call.tool != CollabAgentTool::Wait - || call.mailbox_wait_report.is_none() - { + if call.tool != CollabAgentTool::Wait || call.mailbox_wait_report.is_none() { continue; } let key = (event.turn_id.clone(), call.id.clone()); @@ -1507,15 +1503,15 @@ impl Session { } wait_states.insert(key, (call.clone(), completed)); } - RolloutItem::ResponseItem(ResponseItem::FunctionCall { - call_id, - name, - .. - }) if name == "wait_agent" => { + RolloutItem::ResponseItem(ResponseItem::FunctionCall { call_id, name, .. }) + if name == "wait_agent" => + { wait_call_ids.insert(call_id.clone()); } RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { - call_id, output, .. + call_id, + output, + .. }) => { if let Some(text) = output.body.to_text() { wait_outputs.insert(call_id.clone(), text); @@ -1556,13 +1552,12 @@ impl Session { )); } if !already_completed { - call.status = if report.completion_reason - == Some(MailboxWakeCause::Interrupted) - { - CollabAgentToolCallStatus::Failed - } else { - CollabAgentToolCallStatus::Completed - }; + call.status = + if report.completion_reason == Some(MailboxWakeCause::Interrupted) { + CollabAgentToolCallStatus::Failed + } else { + CollabAgentToolCallStatus::Completed + }; } report } @@ -1602,9 +1597,8 @@ impl Session { )) })?; let mut output = FunctionCallOutputPayload::from_text(output_text); - output.success = Some( - report.completion_reason != Some(MailboxWakeCause::Interrupted), - ); + output.success = + Some(report.completion_reason != Some(MailboxWakeCause::Interrupted)); let mut output_item = ResponseItem::FunctionCallOutput { id: Some(ResponseItemId::with_suffix( "fco", @@ -2416,41 +2410,49 @@ impl Session { let _ = store.publish_turn_complete(&token, event.last_agent_message.clone()); } + let mut mailbox_captured = false; if mailbox_enabled { let completion = Self::child_completion_communication(child_agent_path, &status); - let goal_snapshot = self + let goal = self .services .thread_extension_data .get::() - .and_then(|store| store.snapshot()); - if let Err(error) = self + .and_then(|store| { + store + .snapshot() + .map(|snapshot| (store.incarnation(), snapshot)) + }); + match self .services .agent_control .mailbox_record_terminal( self.thread_id, &turn_context.sub_id, status.clone(), - goal_snapshot, + goal, completion, ) .await { - warn!( - thread_id = %self.thread_id, - turn_id = %turn_context.sub_id, - error = %error, - "failed to durably record mailbox turn outcome" - ); - return Some(AgentStatus::Errored(format!( - "Mailbox turn outcome could not be durably recorded: {error}" - ))); + Ok(captured) => mailbox_captured = captured, + Err(error) => { + warn!( + thread_id = %self.thread_id, + turn_id = %turn_context.sub_id, + error = %error, + "failed to durably record mailbox turn outcome" + ); + return Some(AgentStatus::Errored(format!( + "Mailbox turn outcome could not be durably recorded: {error}" + ))); + } } } // An interrupted turn is terminal for mailbox accounting but not for native agent // residency, so record it above and leave the existing native completion behavior alone. if !is_final(&status) { - return mailbox_enabled.then_some(status); + return mailbox_captured.then_some(status); } // Only a normal, error-free TurnComplete may be deferred. Aborts and @@ -2469,15 +2471,15 @@ impl Session { }) == Some(false) { - if mailbox_enabled { + if mailbox_captured { return Some(status); } return None; } - // The mailbox adapter owns result delivery to every exact issuer. If its persistence - // fails, report the error above and do not publish a success-shaped native fallback. - if mailbox_enabled { + // Only a captured mailbox work item owns result delivery. A shared initialized + // mailbox is not enough to suppress the existing native completion path. + if mailbox_captured { return Some(status); } @@ -2485,10 +2487,10 @@ impl Session { turn_context, *parent_thread_id, child_agent_path, - status, + status.clone(), ) .await; - None + mailbox_enabled.then_some(status) } /// Sends the standard completion envelope from a spawned MultiAgentV2 child to its parent. @@ -3603,7 +3605,8 @@ impl Session { ); } RolloutItem::Compacted(compacted) => { - for response_item in compacted.replacement_history.as_deref().unwrap_or_default() + for response_item in + compacted.replacement_history.as_deref().unwrap_or_default() { Self::record_interrupted_wait_response_item( response_item, @@ -3632,7 +3635,9 @@ impl Session { ResponseItem::FunctionCall { call_id, name, .. } if name == "wait_agent" => { wait_call_ids.insert(call_id.clone()); } - ResponseItem::FunctionCallOutput { call_id, output, .. } => { + ResponseItem::FunctionCallOutput { + call_id, output, .. + } => { let Some(report) = output .body .to_text() @@ -3661,10 +3666,7 @@ impl Session { } /// Rebuilds the in-memory same-call guard from canonical rollout history. - pub(crate) async fn refresh_mailbox_interrupted_wait_receipts( - &self, - items: &[RolloutItem], - ) { + pub(crate) async fn refresh_mailbox_interrupted_wait_receipts(&self, items: &[RolloutItem]) { *self.mailbox_interrupted_wait_receipts.lock().await = Self::mailbox_interrupted_wait_receipts_from_items(items); } @@ -3844,10 +3846,7 @@ impl Session { // must not duplicate the body in in-memory history or emit it a second time. self.services .agent_control - .mailbox_mark_canonical( - self.thread_id, - std::slice::from_ref(&response_item_id), - ) + .mailbox_mark_canonical(self.thread_id, std::slice::from_ref(&response_item_id)) .await?; drop(canonical_ids); drop(custody_gate); @@ -4011,7 +4010,11 @@ impl Session { ) .into()); } - let canonical_ids = self.mailbox_canonical_response_item_ids.lock().await.clone(); + let canonical_ids = self + .mailbox_canonical_response_item_ids + .lock() + .await + .clone(); self.services .agent_control .mailbox_reconcile_canonical_ids(self.thread_id, &canonical_ids) diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index d4ce909bee02..bd97f0f7a1a5 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1,6 +1,7 @@ use crate::CodexAppsToolsCache; use crate::SkillsService; use crate::agent::AgentControl; +use crate::agent::goal_notifications::GoalNotificationStore; use crate::agent::goal_notifications::GoalNotificationSnapshot; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; @@ -1405,18 +1406,43 @@ impl ThreadManager { }; if let Ok(child_thread) = self.get_thread(child_thread_id).await { let child_agent_control = child_thread.session.services.agent_control.clone(); - if child_agent_control.mailbox_enabled() { - child_agent_control + let child_config = child_thread.config().await; + let mailbox_configured = child_thread.multi_agent_version() + == Some(MultiAgentVersion::V2) + && child_config.multi_agent_version_from_features() == MultiAgentVersion::V2 + && child_config.features.enabled(Feature::AgentMailbox) + && child_agent_control.mailbox_enabled(); + if mailbox_configured { + let goal = match goal { + Some(snapshot) => { + let Some(store) = child_thread + .session + .services + .thread_extension_data + .get::() + else { + return Err( + "mailbox goal handback lacks the loaded child's native goal incarnation" + .to_string(), + ); + }; + Some((store.incarnation(), snapshot)) + } + None => None, + }; + let captured = child_agent_control .mailbox_record_terminal( child_thread_id, source_turn_id, status, goal, - communication, + communication.clone(), ) .await .map_err(|err| err.to_string())?; - return Ok(true); + if captured { + return Ok(true); + } } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index cc9a24208775..2f82524ef48d 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -318,8 +318,7 @@ pub fn create_mailbox_read_tool() -> ToolSpec { ( "limit".to_string(), JsonSchema::integer(Some( - "Maximum messages to return. Defaults to 20 and is capped at 100." - .to_string(), + "Maximum messages to return. Defaults to 20 and is capped at 100.".to_string(), )), ), ]); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index c28cb60ad2e6..bf4b1efbf829 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -794,13 +794,29 @@ fn mailbox_wait_agent_tool_has_no_native_capability_prerequisite() { .is_some() ); assert!(output_schema["properties"].get("quietCount").is_some()); - assert!(output_schema["properties"].get("completionReason").is_some()); - assert!(output_schema["properties"].get("requestedTargets").is_some()); - assert!(output_schema["properties"].get("requestedRepliesTo").is_some()); + assert!( + output_schema["properties"] + .get("completionReason") + .is_some() + ); + assert!( + output_schema["properties"] + .get("requestedTargets") + .is_some() + ); + assert!( + output_schema["properties"] + .get("requestedRepliesTo") + .is_some() + ); assert!(output_schema["properties"].get("returnWhen").is_some()); assert!(output_schema["properties"].get("wakeSender").is_some()); assert!(output_schema["properties"].get("wakeRef").is_some()); - assert!(output_schema["properties"].get("condition_satisfied").is_none()); + assert!( + output_schema["properties"] + .get("condition_satisfied") + .is_none() + ); assert!(output_schema["properties"].get("quiet_count").is_none()); assert_eq!( output_schema["properties"]["completed"]["items"]["type"], diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 2d91802efa58..3681b24e14d6 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -183,9 +183,8 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, if let SnapshotItem::Work(work) = item && work.dispatch_state == DispatchState::Dispatched && work.outcome.is_none() - && work.continuations.is_empty() { - match session.services.agent_control.mailbox_reconcile_pending_work(&work.work_ref).await { + match session.services.agent_control.mailbox_reconcile_pending_work(work).await { Ok(changed) => recovery_changed |= changed, Err(_) => { recovery_fault = true; break; } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index eb4baa6d3c66..63e9fb6f3169 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -8,6 +8,10 @@ use codex_agent_mailbox::MessageKind; use codex_agent_mailbox::NativeWork; use codex_agent_mailbox::WorkRef; use codex_features::Feature; +use codex_protocol::items::CollabAgentTool; +use codex_protocol::items::CollabAgentToolCallItem; +use codex_protocol::items::CollabAgentToolCallStatus; +use codex_protocol::items::TurnItem; use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; @@ -16,7 +20,6 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::RolloutItem; -use codex_protocol::protocol::TurnItem; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::Mutex; @@ -226,25 +229,24 @@ async fn seed_interrupted_wait_receipt( ) { // This exact call/output/terminal-event trio is the canonical history shape consumed by // resume recovery; refresh the same derived receipt cache used by that path. - let terminal_item = - TurnItem::CollabAgentToolCall(codex_protocol::protocol::CollabAgentToolCallItem { - id: call_id.to_string(), - tool: codex_protocol::protocol::CollabAgentTool::Wait, - status: codex_protocol::protocol::CollabAgentToolCallStatus::Failed, - sender_thread_id: session.thread_id, - receiver_thread_ids: Vec::new(), - receiver_agents: Vec::new(), - prompt: None, - model: None, - reasoning_effort: None, - requested_model: None, - requested_reasoning_effort: None, - agents_states: Default::default(), - wake_notifications: None, - completion_reason: None, - wake_cause: None, - mailbox_wait_report: Some(report.clone()), - }); + let terminal_item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: call_id.to_string(), + tool: CollabAgentTool::Wait, + status: CollabAgentToolCallStatus::Failed, + sender_thread_id: session.thread_id, + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: Default::default(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: Some(report.clone()), + }); let items = [ RolloutItem::ResponseItem(ResponseItem::FunctionCall { id: None, @@ -257,6 +259,7 @@ async fn seed_interrupted_wait_receipt( RolloutItem::ResponseItem(ResponseItem::FunctionCallOutput { id: None, call_id: call_id.to_string(), + internal_chat_message_metadata_passthrough: None, output: FunctionCallOutputPayload { body: FunctionCallOutputBody::Text(serde_json::to_string(&report).unwrap()), success: Some(false), diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 50fa762583fe..c34874e39f29 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -404,7 +404,10 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() .expect("request receipt should contain a native message ID"); assert_eq!(request_receipt["durable_state"], "accepted"); assert_eq!(request_receipt["activation_disposition"], "denied"); - assert_eq!(request_receipt["activation_reason"], "root_activation_is_not_allowed"); + assert_eq!( + request_receipt["activation_reason"], + "root_activation_is_not_allowed" + ); assert!( request_id.starts_with("amsg_"), "unexpected native ID: {request_id}" diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs index d38e81840396..3cc3b4c552bb 100644 --- a/codex-rs/rollout/src/policy_tests.rs +++ b/codex-rs/rollout/src/policy_tests.rs @@ -8,8 +8,8 @@ use codex_protocol::items::MailboxWaitReport; use codex_protocol::items::TurnItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ItemStartedEvent; -use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::ThreadHistoryMode; use crate::policy::is_persisted_rollout_item; @@ -43,7 +43,10 @@ fn started_call( })) } -fn started_wait(status: CollabAgentToolCallStatus, report: Option) -> RolloutItem { +fn started_wait( + status: CollabAgentToolCallStatus, + report: Option, +) -> RolloutItem { started_call(CollabAgentTool::Wait, status, report) } @@ -64,9 +67,15 @@ fn in_progress_report() -> MailboxWaitReport { #[test] fn only_typed_in_progress_mailbox_wait_starts_are_persisted() { - let valid = started_wait(CollabAgentToolCallStatus::InProgress, Some(in_progress_report())); + let valid = started_wait( + CollabAgentToolCallStatus::InProgress, + Some(in_progress_report()), + ); let no_report = started_wait(CollabAgentToolCallStatus::InProgress, None); - let terminal = started_wait(CollabAgentToolCallStatus::Completed, Some(in_progress_report())); + let terminal = started_wait( + CollabAgentToolCallStatus::Completed, + Some(in_progress_report()), + ); let other_tool = started_call( CollabAgentTool::SendInput, CollabAgentToolCallStatus::InProgress, diff --git a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs index d61ba3107114..b5aab9ea6367 100644 --- a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs @@ -171,7 +171,7 @@ impl ChatWidget { && matches!(status, CollabAgentToolCallStatus::InProgress) && let Some(report) = mailbox_wait_report.as_ref() { - if let Some(progress) = self.active_mailbox_waits.get(id) { + if let Some(progress) = self.active_mailbox_waits.get(id).cloned() { progress.update(report.clone(), /*in_progress*/ true); let wait_cell_is_active = self .transcript From ed3f4399cea6947b920b59d9b6c6af630efaa0c5 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:11:19 +1100 Subject: [PATCH 10/54] fix(mailbox): bind deferred handback to captured native token --- .../workflows/mailbox-source-preparation.yml | 2 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 2 +- .../agent-mailbox/src/outcome_messages.rs | 2 +- codex-rs/agent-mailbox/src/store.rs | 6 + .../core/src/agent/control/mailbox_tests.rs | 53 +++++++- codex-rs/core/src/session/input_queue.rs | 30 +++++ codex-rs/core/src/session/mod.rs | 118 ++++++++++++++---- codex-rs/core/src/session/tests.rs | 63 ++++++++++ codex-rs/core/src/tasks/mod.rs | 9 ++ codex-rs/core/src/thread_manager.rs | 116 +++++++++++++---- codex-rs/core/src/thread_manager_tests.rs | 71 +++++++++++ .../multi_agents_v2/mailbox_wait_tests.rs | 10 +- codex-rs/core/tests/suite/agent_mailbox.rs | 104 +++++++++++++++ codex-rs/ext/goal/src/runtime.rs | 2 +- docs/carry-divergence-ledger.md | 32 +++++ docs/divergences/index.yaml | 6 + docs/downstream-regression-matrix.md | 5 + 17 files changed, 574 insertions(+), 57 deletions(-) diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 39530b6ce83a..6e320c1b1f77 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "7" + PREPARATION_ATTEMPT: "8" SCCACHE_CACHE_SIZE: 2G steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index b10968d9d958..7bb2640fea1d 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -40,7 +40,7 @@ fn communication(content: &str) -> InterAgentCommunication { AgentPath::root(), Vec::new(), content.to_string(), - false, + /*trigger_turn*/ false, ) } diff --git a/codex-rs/agent-mailbox/src/outcome_messages.rs b/codex-rs/agent-mailbox/src/outcome_messages.rs index 4ad6e04fd851..e38f1ea350fc 100644 --- a/codex-rs/agent-mailbox/src/outcome_messages.rs +++ b/codex-rs/agent-mailbox/src/outcome_messages.rs @@ -124,7 +124,7 @@ pub(crate) fn ensure_terminal_responses( group.recipient.clone(), Vec::new(), content, - false, + /*trigger_turn*/ false, ); communication.origin = Some(AgentCommunicationOrigin::Result); group.response = Some(communication); diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index c1c847a7a4a4..e7e234965b76 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -41,6 +41,12 @@ impl Mailbox { self.change_tx.subscribe() } + /// Advisory wake after native producer custody changes. Consumers must rescan + /// native eligibility and durable records; this is not a result or an ack. + pub fn notify_native_custody_changed(&self) { + self.notify_changed(); + } + pub(crate) fn notify_changed(&self) { self.change_tx.send_modify(|generation| { *generation = generation.wrapping_add(1); diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 8ce9d80913b8..057469c5be47 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -23,7 +23,7 @@ async fn mailbox_shared_control_does_not_capture_stock_invocations() { AgentPath::from_string("/root/worker".to_string()).unwrap(), Vec::new(), "native input".to_string(), - true, + /*trigger_turn*/ true, ); let original = communication.clone(); let mut submission_id = "stock-submission".to_string(); @@ -100,7 +100,7 @@ async fn accepted_goal_work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId AgentPath::from_string("/root/worker".to_string()).unwrap(), Vec::new(), "complete the accepted work".to_string(), - true, + /*trigger_turn*/ true, ), }).await.unwrap(); assert_eq!(mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); @@ -167,10 +167,15 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a #[tokio::test] async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody() { - let (session, turn) = crate::session::tests::make_session_and_context().await; + let (mut session, turn) = crate::session::tests::make_session_and_context().await; let mut config = (*turn.config).clone(); config.features.enable(Feature::MultiAgentV2).unwrap(); config.features.enable(Feature::AgentMailbox).unwrap(); + let manager = crate::ThreadManager::with_models_provider_for_tests( + codex_login::CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + ); + session.services.agent_control = manager.agent_control(); let control = &session.services.agent_control; control.mailbox_initialize(&config).await.unwrap(); let mailbox = control.mailbox().unwrap(); @@ -199,3 +204,45 @@ async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_c assert!(control.mailbox_record_terminal(agent, "recovered-turn", AgentStatus::Completed(None), Some((incarnation, goal)), None).await.unwrap()); assert_eq!(exact_work_snapshot(&mailbox, &work).await.outcome.unwrap().outcome, Outcome::Completed); } + +#[tokio::test] +async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { + let (_session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let manager = crate::ThreadManager::with_models_provider_for_tests( + codex_login::CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + ); + let parent = manager.start_thread(crate::thread_manager::StartThreadOptions::new(config.clone())).await.unwrap(); + let control = &parent.thread.session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let mailbox = control.mailbox().unwrap(); + let agent = ThreadId::new(); + let incarnation = ThreadId::new(); + let work = accepted_goal_work(&mailbox, parent.thread_id, agent).await; + let mut goal = goal_snapshot(parent.thread_id, agent); + assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + assert!(manager.get_thread(agent).await.is_err()); + assert!(!parent.thread.session.input_queue.has_pending_mailbox_items().await); + + goal.phase = GoalNotificationPhase::ResultReady; + goal.status = ThreadGoalStatus::Complete; + goal.revision += 1; + goal.classification = GoalNotificationClassification::Result; + assert!(manager.notify_v2_child_completion( + agent, + parent.thread_id, + AgentPath::from_string("/root/worker".to_string()).unwrap(), + "accepted-turn", + AgentStatus::Completed(Some("durable goal result".to_string())), + Some((incarnation, goal)), + ).await.unwrap()); + let settled = exact_work_snapshot(&mailbox, &work).await; + assert_eq!(settled.work_ref, work); + assert_eq!(settled.outcome.unwrap().outcome, Outcome::Completed); + assert_eq!(settled.continuations[0].state, ContinuationState::Terminal); + assert!(manager.get_thread(agent).await.is_err()); + assert!(!parent.thread.session.input_queue.has_pending_mailbox_items().await); +} diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index cfa9da11493f..6513e1a8e1c3 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -132,6 +132,12 @@ impl InputQueue { pub(crate) fn finish_terminal_finalizer(&self) { self.pending_terminal_finalizers .fetch_sub(1, Ordering::AcqRel); + if let Some(mailbox) = self.durable_mailbox.get() { + // A turn can fail before provider admission, so it has no mailbox + // outcome to publish. Wake only after native finalizer custody ends; + // selected waits then reconcile the still-unadmitted exact work. + mailbox.notify_native_custody_changed(); + } } pub(crate) fn has_pending_terminal_finalizers(&self) -> bool { @@ -919,6 +925,30 @@ mod tests { } } + #[tokio::test] + async fn durable_mailbox_finalizer_release_wakes_rescan_without_fabricating_input() { + use codex_utils_absolute_path::test_support::PathExt; + + let home = tempfile::tempdir().unwrap(); + let mailbox = Arc::new( + Mailbox::open(&codex_state::SqliteConfig::new_for_testing(home.path().abs())) + .await + .unwrap(), + ); + let queue = InputQueue::new(); + queue.bind_durable_mailbox(Arc::clone(&mailbox)); + queue.register_terminal_finalizer(); + let mut changes = mailbox.subscribe(); + let input_generation = mailbox.wait_input_generation(); + queue.finish_terminal_finalizer(); + + assert!(changes.has_changed().unwrap()); + changes.borrow_and_update(); + assert!(!queue.has_pending_terminal_finalizers()); + assert!(!queue.has_pending_mailbox_items().await); + assert_eq!(mailbox.wait_input_generation(), input_generation); + } + #[tokio::test] async fn durable_mailbox_input_arrival_invalidates_stale_cycle_snapshots() { use codex_agent_mailbox::MailboxError; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index ecc377803cfb..7181d14d1b04 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2396,32 +2396,50 @@ impl Session { } }; - if let EventMsg::TurnComplete(event) = msg - && matches!(status, AgentStatus::Completed(_)) - && let Some(token) = turn_context - .extension_data - .get::( - ) - && let Some(store) = self - .services - .thread_extension_data - .get::() - { - let _ = store.publish_turn_complete(&token, event.last_agent_message.clone()); - } - let mut mailbox_captured = false; if mailbox_enabled { let completion = Self::child_completion_communication(child_agent_path, &status); - let goal = self - .services - .thread_extension_data - .get::() - .and_then(|store| { - store - .snapshot() - .map(|snapshot| (store.incarnation(), snapshot)) - }); + let goal = match turn_context + .extension_data + .get::() + { + Some(token) => { + let Some(store) = self + .services + .thread_extension_data + .get::() + else { + self.notify_mailbox_terminal_correlation_fault(); + return Some(AgentStatus::Errored( + "Mailbox turn outcome could not be correlated: native goal store is unavailable" + .to_string(), + )); + }; + let publishes_completion = matches!(msg, EventMsg::TurnComplete(_)) + && matches!(status, AgentStatus::Completed(_)); + let result = match msg { + EventMsg::TurnComplete(event) if publishes_completion => { + event.last_agent_message.clone() + } + _ => None, + }; + match mailbox_goal_for_terminal_turn( + store.as_ref(), + &token, + publishes_completion, + result, + ) { + Ok(goal) => Some(goal), + Err(error) => { + self.notify_mailbox_terminal_correlation_fault(); + return Some(AgentStatus::Errored(format!( + "Mailbox turn outcome could not be correlated: {error}" + ))); + } + } + } + None => None, + }; match self .services .agent_control @@ -2447,6 +2465,19 @@ impl Session { ))); } } + } else if let EventMsg::TurnComplete(event) = msg + && matches!(status, AgentStatus::Completed(_)) + && let Some(token) = turn_context + .extension_data + .get::( + ) + && let Some(store) = self + .services + .thread_extension_data + .get::() + { + // Preserve native goal notifications when mailbox custody is disabled. + let _ = store.publish_turn_complete(&token, event.last_agent_message.clone()); } // An interrupted turn is terminal for mailbox accounting but not for native agent @@ -2493,6 +2524,12 @@ impl Session { mailbox_enabled.then_some(status) } + fn notify_mailbox_terminal_correlation_fault(&self) { + if let Some(mailbox) = self.services.agent_control.mailbox() { + mailbox.notify_runtime_fault(self.thread_id); + } + } + /// Sends the standard completion envelope from a spawned MultiAgentV2 child to its parent. async fn forward_child_completion_to_parent( &self, @@ -5076,6 +5113,41 @@ async fn build_hooks_for_config( }) } +fn mailbox_goal_for_terminal_turn( + store: &crate::agent::goal_notifications::GoalNotificationStore, + token: &crate::agent::goal_notifications::GoalNotificationTurnToken, + publishes_completion: bool, + result: Option, +) -> Result<( + ThreadId, + crate::agent::goal_notifications::GoalNotificationSnapshot, +), String> { + if store.incarnation() != token.incarnation { + return Err("native goal store incarnation changed".into()); + } + if publishes_completion { + if !store.publish_turn_complete(token, result) { + return Err("native goal completion publication rejected the turn token".into()); + } + } else if store + .terminal_turn_is_wake_eligible_for_token(token) + .is_none() + { + return Err("native goal store no longer recognizes the turn token".into()); + } + + let snapshot = store + .snapshot() + .filter(|snapshot| snapshot.binding == token.binding) + .ok_or_else(|| "native goal snapshot no longer matches the turn binding".to_string())?; + if publishes_completion + && snapshot.last_completed_source_turn.as_deref() != Some(token.source_turn_id.as_str()) + { + return Err("native goal snapshot does not record the captured source turn".into()); + } + Ok((token.incarnation, snapshot)) +} + #[cfg(test)] #[path = "elicitation_holders_tests.rs"] mod elicitation_holders_tests; diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index e7f7f0b12ddc..094738a605c0 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -12594,3 +12594,66 @@ async fn session_start_hooks_require_project_trust_without_config_toml() -> std: Ok(()) } + +#[test] +fn mailbox_terminal_goal_requires_successful_publication_for_exact_turn() { + use crate::agent::goal_notifications::GoalNotificationBinding; + use crate::agent::goal_notifications::GoalNotificationPhase; + use crate::agent::goal_notifications::GoalNotificationProjection; + use crate::agent::goal_notifications::GoalNotificationStore; + use crate::agent::goal_notifications::GoalNotificationTurnToken; + + let parent_thread_id = ThreadId::new(); + let child_thread_id = ThreadId::new(); + let binding = GoalNotificationBinding { + parent_thread_id, + child_thread_id, + goal_id: "goal-a".to_string(), + control_generation: 7, + }; + let store = GoalNotificationStore::default(); + let mut projection = GoalNotificationProjection::new( + binding.clone(), + codex_protocol::protocol::ThreadGoalStatus::Active, + ); + assert!(projection.opt_in(&binding)); + store.install_authoritative(projection); + store.set_source_turn("turn-a"); + let token = GoalNotificationTurnToken { + incarnation: store.incarnation(), + binding: binding.clone(), + source_turn_id: "turn-a".to_string(), + generation: binding.control_generation, + }; + + let (incarnation, snapshot) = mailbox_goal_for_terminal_turn( + &store, + &token, + /*publishes_completion*/ true, + Some("result".to_string()), + ) + .expect("matching native completion should be capturable"); + assert_eq!(incarnation, token.incarnation); + assert_eq!(snapshot.binding, token.binding); + assert_eq!(snapshot.last_completed_source_turn.as_deref(), Some("turn-a")); + assert_eq!(snapshot.phase, GoalNotificationPhase::ContinuationPending); + + let mut replacement = binding; + replacement.goal_id = "goal-b".to_string(); + replacement.control_generation += 1; + let mut replacement_projection = GoalNotificationProjection::new( + replacement.clone(), + codex_protocol::protocol::ThreadGoalStatus::Active, + ); + assert!(replacement_projection.opt_in(&replacement)); + store.install_authoritative(replacement_projection); + store.set_source_turn("turn-a"); + assert!(mailbox_goal_for_terminal_turn( + &store, + &token, + /*publishes_completion*/ true, + Some("stale result".to_string()), + ) + .is_err()); + assert_eq!(store.snapshot().unwrap().binding, replacement); +} diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 72a35f30e95c..9923386fc5ad 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -882,6 +882,15 @@ impl Session { if cleared_active_turn { self.maybe_start_turn_for_pending_work().await; } + if turn_context.multi_agent_version == codex_protocol::protocol::MultiAgentVersion::V2 + && turn_context.config.features.enabled(Feature::AgentMailbox) + && let Some(mailbox) = self.services.agent_control.mailbox() + { + // This follows active-turn release and the native pending-work decision. + // A pre-admission producer failure has no terminal mailbox result, but an + // event-only issuer wait must still rescan exact native custody. + mailbox.notify_native_custody_changed(); + } } async fn take_active_turn(&self) -> Option { diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index bd97f0f7a1a5..e24129c788cb 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1372,8 +1372,9 @@ impl ThreadManager { /// Settles a deferred V2 child completion after a later external state /// change makes it terminal while the child is idle. Mailbox-enabled /// sessions persist the outcome and delivery intent instead of queueing a - /// separate native completion. A false return means the legacy native - /// enqueue was rejected; mailbox persistence failures return an error. + /// separate native completion. A false return means the native enqueue was + /// rejected after no matching mailbox work was captured; persistence or + /// required same-tree custody failures return an error. pub async fn notify_v2_child_completion( &self, child_thread_id: ThreadId, @@ -1381,7 +1382,7 @@ impl ThreadManager { child_agent_path: AgentPath, source_turn_id: &str, status: AgentStatus, - goal: Option, + goal: Option<(ThreadId, GoalNotificationSnapshot)>, ) -> Result { let parent_agent_path = child_agent_path .as_str() @@ -1407,35 +1408,79 @@ impl ThreadManager { if let Ok(child_thread) = self.get_thread(child_thread_id).await { let child_agent_control = child_thread.session.services.agent_control.clone(); let child_config = child_thread.config().await; + let goal_store = child_thread + .session + .services + .thread_extension_data + .get::(); + if goal.is_some() && goal_store.is_none() { + return Err( + "deferred goal handback lacks the loaded child's native goal incarnation" + .to_string(), + ); + } + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + source_turn_id, + goal.as_ref(), + goal_store.as_deref(), + )?; let mailbox_configured = child_thread.multi_agent_version() == Some(MultiAgentVersion::V2) && child_config.multi_agent_version_from_features() == MultiAgentVersion::V2 && child_config.features.enabled(Feature::AgentMailbox) && child_agent_control.mailbox_enabled(); if mailbox_configured { - let goal = match goal { - Some(snapshot) => { - let Some(store) = child_thread - .session - .services - .thread_extension_data - .get::() - else { - return Err( - "mailbox goal handback lacks the loaded child's native goal incarnation" - .to_string(), - ); - }; - Some((store.incarnation(), snapshot)) - } - None => None, - }; let captured = child_agent_control .mailbox_record_terminal( child_thread_id, source_turn_id, - status, - goal, + status.clone(), + goal.clone(), + communication.clone(), + ) + .await + .map_err(|err| err.to_string())?; + if captured { + return Ok(true); + } + } + } else if let Some((incarnation, snapshot)) = goal.as_ref() { + // A deferred goal runtime can outlive the child's residency. The loaded + // immediate parent shares the tree's AgentControl and therefore its + // mailbox; the manager's freshly constructed control does not. + let parent_thread = self.get_thread(parent_thread_id).await.map_err(|err| { + format!( + "cannot establish same-tree mailbox custody for deferred child goal handback: {err}" + ) + })?; + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + source_turn_id, + goal.as_ref(), + /*goal_store*/ None, + )?; + let parent_agent_control = parent_thread.session.services.agent_control.clone(); + let parent_config = parent_thread.config().await; + let parent_mailbox_configured = parent_thread.multi_agent_version() + == Some(MultiAgentVersion::V2) + && parent_config.multi_agent_version_from_features() == MultiAgentVersion::V2 + && parent_config.features.enabled(Feature::AgentMailbox); + if parent_mailbox_configured && !parent_agent_control.mailbox_enabled() { + return Err( + "loaded parent has mailbox enabled but lacks initialized same-tree mailbox custody" + .to_string(), + ); + } + if parent_mailbox_configured { + let captured = parent_agent_control + .mailbox_record_terminal( + child_thread_id, + source_turn_id, + status.clone(), + Some((*incarnation, snapshot.clone())), communication.clone(), ) .await @@ -2598,6 +2643,33 @@ fn append_interrupted_boundary( } } +fn validate_deferred_goal_identity( + child_thread_id: ThreadId, + parent_thread_id: ThreadId, + source_turn_id: &str, + goal: Option<&(ThreadId, GoalNotificationSnapshot)>, + goal_store: Option<&GoalNotificationStore>, +) -> Result<(), String> { + let Some((incarnation, snapshot)) = goal else { + return Ok(()); + }; + if snapshot.binding.child_thread_id != child_thread_id + || snapshot.binding.parent_thread_id != parent_thread_id + { + return Err("deferred goal handback does not match its captured child/parent binding".into()); + } + if snapshot.last_completed_source_turn.as_deref() != Some(source_turn_id) { + return Err("deferred goal handback does not match its captured source turn".into()); + } + if goal_store.is_some_and(|store| store.incarnation() != *incarnation) { + return Err( + "deferred goal handback incarnation does not match the loaded child's native goal store" + .into(), + ); + } + Ok(()) +} + #[cfg(test)] #[path = "thread_manager_tests.rs"] mod tests; diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 1c2119f281cb..8524309e43a2 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -2158,3 +2158,74 @@ async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_ 1, ); } + +#[test] +fn mailbox_deferred_goal_handoff_rejects_replaced_native_incarnation() { + let parent_thread_id = ThreadId::new(); + let child_thread_id = ThreadId::new(); + let binding = crate::agent::goal_notifications::GoalNotificationBinding { + parent_thread_id, + child_thread_id, + goal_id: "goal-a".to_string(), + control_generation: 1, + }; + let mut projection = crate::agent::goal_notifications::GoalNotificationProjection::new( + binding, + codex_protocol::protocol::ThreadGoalStatus::Active, + ); + let binding = projection.snapshot().binding.clone(); + assert!(projection.observe( + &binding, + crate::agent::goal_notifications::GoalNotificationInput::TurnComplete { + source_turn: "turn-a".to_string(), + result: None, + }, + )); + let snapshot = projection.snapshot().clone(); + let store = GoalNotificationStore::default(); + let exact_goal = (store.incarnation(), snapshot.clone()); + + assert!(validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&exact_goal), + Some(&store), + ) + .is_ok()); + assert!(validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-b", + Some(&exact_goal), + Some(&store), + ) + .is_err()); + assert!(validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&(ThreadId::new(), snapshot.clone())), + Some(&store), + ) + .is_err()); + assert!(validate_deferred_goal_identity( + ThreadId::new(), + parent_thread_id, + "turn-a", + Some(&exact_goal), + Some(&store), + ) + .is_err()); + + // An unloaded child has no current native store to compare against. Its exact + // incarnation is carried from the original runtime token and binding checked. + assert!(validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&exact_goal), + None, + ) + .is_ok()); +} diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index 63e9fb6f3169..8af7d82afa29 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -88,7 +88,7 @@ async fn work( AgentPath::from_string("/root/worker".to_string()).unwrap(), Vec::new(), key.to_string(), - true, + /*trigger_turn*/ true, ), }) .await @@ -174,7 +174,7 @@ async fn record_result( AgentPath::root(), Vec::new(), body.to_string(), - false, + /*trigger_turn*/ false, )), Continuation::None, ) @@ -334,7 +334,7 @@ async fn unrelated_attention_does_not_erase_a_successful_selected_join() { AgentPath::root(), Vec::new(), "independent request".to_string(), - false, + /*trigger_turn*/ false, ), kind: MessageKind::Request, activate_if_idle: false, @@ -443,7 +443,7 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { worker_path.clone(), Vec::new(), "request requiring reply".to_string(), - false, + /*trigger_turn*/ false, ), kind: MessageKind::Request, activate_if_idle: false, @@ -465,7 +465,7 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { AgentPath::root(), Vec::new(), body.to_string(), - false, + /*trigger_turn*/ false, ), kind: MessageKind::Note, activate_if_idle: false, diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index c34874e39f29..41aa8f2d5d99 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -91,6 +91,110 @@ fn function_call_response( ])) } +#[derive(Clone)] +struct UnadmittedWorkResponder; + +impl Respond for UnadmittedWorkResponder { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body).unwrap(); + if has_function_output(&body, "unadmitted-wait") { + return sse_response(sse(vec![ + ev_response_created("unadmitted-final"), + ev_assistant_message("unadmitted-final-message", "recovery required"), + ev_completed("unadmitted-final"), + ])); + } + if has_function_output(&body, "unadmitted-spawn") { + let spawn: Value = serde_json::from_str(function_output_text(&body, "unadmitted-spawn").unwrap()).unwrap(); + return function_call_response("unadmitted-wait-response", "unadmitted-wait", "wait_agent", json!({ + "targets": [spawn["work_ref"]], + "return_when": "all", + })); + } + function_call_response("unadmitted-spawn-response", "unadmitted-spawn", "spawn_agent", json!({ + "task_name": "unadmitted-worker", + "message": "fail before provider acceptance of this work", + })) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mailbox_event_only_wait_wakes_when_producer_fails_before_input_admission() -> Result<()> { + use codex_agent_mailbox::DispatchState; + use codex_agent_mailbox::ReturnWhen; + use codex_agent_mailbox::Selection; + use codex_agent_mailbox::Selector; + use codex_agent_mailbox::SnapshotItem; + use codex_protocol::items::CollabAgentToolCallStatus; + use codex_protocol::items::TurnItem; + use codex_protocol::protocol::ThreadSettingsOverrides; + use codex_protocol::user_input::UserInput; + + let server = start_mock_server().await; + let mut builder = test_codex().with_model("koffing").with_config(|config| { + for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + config.features.enable(feature).unwrap(); + } + config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); + config.model_provider.supports_websockets = false; + config.model_provider.stream_max_retries = Some(0); + config.model_provider.request_max_retries = Some(0); + }); + let test = builder.build_with_auto_env(&server).await?; + let root_id = test.session_configured.thread_id; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(RootThreadMatcher { thread_id: root_id.to_string() }) + .respond_with(UnadmittedWorkResponder) + .up_to_n_times(3) + .mount(&server).await; + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .and(ChildThreadMatcher { root_thread_id: root_id.to_string(), task: "fail before provider acceptance of this work".to_string() }) + // No response-created event: this child never admits its accepted input. + .respond_with(sse_response(String::new()).set_delay(Duration::from_secs(2))) + .up_to_n_times(1) + .mount(&server).await; + test.codex.submit(Op::UserInput { + items: vec![UserInput::Text { text: "join the accepted work without a timeout".to_string(), text_elements: Vec::new() }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: ThreadSettingsOverrides::default(), + }).await?; + // Prove the selected wait was sleeping while native custody still existed, + // rather than only testing an already-cold initial snapshot. + wait_for_event(&test.codex, |event| matches!(event, + EventMsg::ItemStarted(event) if matches!(&event.item, + TurnItem::CollabAgentToolCall(call) if call.id == "unadmitted-wait" + && call.status == CollabAgentToolCallStatus::InProgress + && call.mailbox_wait_report.as_ref().is_some_and(|report| report.pending.iter().any(|item| item.status == "pending")) + ) + )).await; + let terminal = wait_for_event(&test.codex, |event| matches!(event, + EventMsg::ItemCompleted(event) if matches!(&event.item, + TurnItem::CollabAgentToolCall(call) if call.id == "unadmitted-wait" + ) + )).await; + let EventMsg::ItemCompleted(terminal) = terminal else { unreachable!() }; + let TurnItem::CollabAgentToolCall(call) = terminal.item else { unreachable!() }; + let report = call.mailbox_wait_report.unwrap(); + assert_eq!(report.completion_reason, Some(codex_protocol::items::MailboxWakeCause::NeedsAttention)); + assert!(!report.condition_satisfied); + assert_eq!(report.pending.len(), 1); + assert_eq!(report.pending[0].status, "recovery_needed"); + wait_for_event(&test.codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let mailbox = Mailbox::open(test.config.sqlite_config()).await?; + let snapshot = mailbox.snapshot(&Selection { caller: root_id, agent: None, selectors: vec![Selector::Work(report.pending[0].work_ref.clone().unwrap())], return_when: ReturnWhen::All, limit: 1 }).await?; + let SnapshotItem::Work(work) = &snapshot.items[0] else { unreachable!() }; + assert_eq!(work.dispatch_state, DispatchState::RecoveryNeeded); + assert!(work.outcome.is_none()); + let input_id = work.input.id.as_ref().unwrap().to_string(); + assert_eq!(mailbox.unadmitted_input_ids(work.work_ref.agent, &[input_id.clone()]).await?, vec![input_id]); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mailbox_provider_schema_is_opt_in_and_exposes_the_v2_contract() -> Result<()> { let server = start_mock_server().await; diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 97eebbf700b7..725a6f957796 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -340,7 +340,7 @@ impl GoalRuntimeHandle { child_agent_path, &token.source_turn_id, codex_protocol::protocol::AgentStatus::Completed(result), - Some(goal_snapshot), + Some((token.incarnation, goal_snapshot)), ) .await; match delivery { diff --git a/docs/carry-divergence-ledger.md b/docs/carry-divergence-ledger.md index 759468918cdf..f82005f20107 100644 --- a/docs/carry-divergence-ledger.md +++ b/docs/carry-divergence-ledger.md @@ -70,6 +70,20 @@ its pending references intact, rather than restarting its old deadline. Reusing that interrupted call ID cannot start a new wait. Unconfirmed accepted/dispatching work from that exact resumed issuer is marked recovery-needed in bounded indexed batches, without resending its input or creating a replacement child. +The in-progress typed wait itself is appended and flushed before the wait sleeps, +so interruption recovery does not depend on a UI-only start event. Goal +continuations retain their accepted source-turn provenance but correlate by the +logical goal, exact parent/child and control generation, and native goal-store +incarnation. Revision or later-turn progress does not break that correlation; +a cold or replaced producer yields recovery-needed without deleting the original +binding or inventing completion. Native result delivery is suppressed only when +the mailbox actually captured the accepted work. +Deferred handback carries the original native token incarnation end-to-end; it +cannot be relabelled with a replacement loaded child's identity. A cold producer +can settle through its loaded parent's same-tree store, otherwise handback +retains retry custody. Native finalizer/turn release emits an advisory rescan so +a producer failure before input admission cannot leave an event-only wait asleep +without an outcome or a producer able to advance it. Both `agent_mailbox` and recipient `agent_mailbox_idle_activation` are off by default. Disabled mailbox handling preserves pending records and cannot drain, @@ -81,6 +95,24 @@ new scheduling, and provider/auth changes remain outside this carry. Future upstream syncs retain the same semantic suite and retire equivalent mechanisms and dependencies together, rather than restoring donor framework code. +To explicitly select this experimental surface for a new session, enable both +`multi_agent_v2` and `agent_mailbox`, for example: + +```toml +[features] +multi_agent_v2 = true +agent_mailbox = true +agent_mailbox_idle_activation = false +``` + +The equivalent one-session CLI selection is +`codex --enable multi_agent_v2 --enable agent_mailbox`. This does not grant +idle activation: that requires the recipient's separate +`agent_mailbox_idle_activation` permission and still obeys native root, +approval, pause, budget, and cancellation controls. A default-off binary startup +is not evidence that the mailbox tool contract is active. Installing a preview, +restarting a session, and opting in remain explicit operator actions. + - Without the opt-in mailbox feature, downstream V2 `wait_agent` keeps queue-only inter-agent mail durable and visible in later notification summaries, but an exact-target wait wakes only for terminal target state, actionable (`trigger_turn`) mail, user steer, diff --git a/docs/divergences/index.yaml b/docs/divergences/index.yaml index e79344e513cb..119c4d47879e 100644 --- a/docs/divergences/index.yaml +++ b/docs/divergences/index.yaml @@ -3613,6 +3613,12 @@ "native_retry_dispatch_reopen_and_capture_backpressure_are_safe", "admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped", "pending_deferred_and_terminal_continuations_bind_to_exact_generation", + "mailbox_goal_terminal_matches_logical_binding_across_native_revisions_and_turns", + "mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody", + "mailbox_cold_deferred_handback_uses_loaded_parents_shared_store", + "mailbox_event_only_wait_wakes_when_producer_fails_before_input_admission", + "mailbox_terminal_goal_requires_successful_publication_for_exact_turn", + "mailbox_deferred_goal_handoff_rejects_replaced_native_incarnation", "quiet_reply_and_author_dispositions_settle_only_exact_request", "canonical_outcomes_are_observed_per_item_without_cursor_loss", "resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_batches", diff --git a/docs/downstream-regression-matrix.md b/docs/downstream-regression-matrix.md index f338fd8ea14b..2ba80c31ace5 100644 --- a/docs/downstream-regression-matrix.md +++ b/docs/downstream-regression-matrix.md @@ -69,6 +69,11 @@ Its native cancellation slice includes The notification visibility lane also runs protocol/app-server-protocol/TUI `mailbox` tests so the same structured report is exercised in live and replay consumers, including interrupted-wait recovery with pending references retained. +The native goal adapter also exercises +`mailbox_goal_terminal_matches_logical_binding_across_native_revisions_and_turns` +and `mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody`: +later logical progress may settle exact original work, but a replacement native +goal incarnation cannot, and an unowned deferred continuation must not hang. Without the opt-in mailbox feature, exact-target V2 `wait_agent` ignores queue-only (`trigger_turn=false`) mailbox activity while preserving durable FIFO delivery and notification summaries; From be6ca99f10e2d27e1471193b77ca51de24b10fba Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:22:22 +1100 Subject: [PATCH 11/54] test(mailbox): reconcile hosted fixtures and reviewed wait snapshots --- .../workflows/mailbox-source-preparation.yml | 2 +- codex-rs/agent-mailbox/src/lib.rs | 5 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 121 ++++-- codex-rs/core/src/agent/control/mailbox.rs | 34 +- .../core/src/agent/control/mailbox_tests.rs | 10 + codex-rs/core/src/session/tests.rs | 10 + .../tools/handlers/multi_agents_spec_tests.rs | 4 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 387 +++++++++++++----- .../multi_agents_v2/mailbox_wait_tests.rs | 3 + codex-rs/core/src/tools/spec_plan_tests.rs | 2 +- codex-rs/core/tests/suite/agent_mailbox.rs | 11 +- ...e_mailbox_wait_report_active_progress.snap | 12 + ...lbox_wait_report_interrupted_terminal.snap | 14 + 13 files changed, 460 insertions(+), 155 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_mailbox_wait_report_active_progress.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__replayed_mailbox_wait_report_interrupted_terminal.snap diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index 6e320c1b1f77..af6c2add62bc 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "8" + PREPARATION_ATTEMPT: "9" SCCACHE_CACHE_SIZE: 2G steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index 393fb572808e..8ab34f3de22a 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -28,9 +28,8 @@ pub use types::{ Continuation, ContinuationReceipt, ContinuationState, Dependency, DependencyMode, DispatchDecision, DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, NativeWork, ObservationTarget, Outcome, OutcomeReceipt, PendingPage, RecoveryEvidence, RequestDisposition, - RequestSnapshot, - ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotRequest, SnapshotState, - TurnOutcomeReceipt, WorkRef, WorkSnapshot, + RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotRequest, + SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, }; pub use wait_graph::InputArrival; pub use wait_graph::WaitRegistration; diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 7bb2640fea1d..ca8e9dd71a86 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -194,14 +194,26 @@ fn goal_binding( async fn new_continuation_binding_requires_native_incarnation() { let fixture = test_mailbox().await; let (issuer, agent) = (thread_id(40), thread_id(41)); - let (work, input) = - accept_work(&fixture.mailbox, issuer, agent, "missing-incarnation", "input").await; + let (work, input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "missing-incarnation", + "input", + ) + .await; assert_eq!( fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&work).await.unwrap(); - admit(&fixture.mailbox, agent, "missing-incarnation-turn", &[input]).await; + admit( + &fixture.mailbox, + agent, + "missing-incarnation-turn", + &[input], + ) + .await; let mut legacy_binding = goal_binding(issuer, agent, 3, thread_id(42)); legacy_binding.incarnation = None; @@ -643,20 +655,20 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { let fixture = test_mailbox().await; let (issuer, agent) = (thread_id(46), thread_id(47)); let binding = goal_binding(issuer, agent, 6, thread_id(48)); - let (original, original_input) = - accept_work(&fixture.mailbox, issuer, agent, "continuation-origin", "origin").await; + let (original, original_input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "continuation-origin", + "origin", + ) + .await; assert_eq!( fixture.mailbox.begin_dispatch(&original).await.unwrap(), DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&original).await.unwrap(); - admit( - &fixture.mailbox, - agent, - "source-origin", - &[original_input], - ) - .await; + admit(&fixture.mailbox, agent, "source-origin", &[original_input]).await; finish( &fixture.mailbox, agent, @@ -703,13 +715,7 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&later_work).await.unwrap(); - admit( - &fixture.mailbox, - agent, - "source-terminal", - &[later_input], - ) - .await; + admit(&fixture.mailbox, agent, "source-terminal", &[later_input]).await; let terminal = finish( &fixture.mailbox, agent, @@ -746,20 +752,20 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t let fixture = test_mailbox().await; let (issuer, agent) = (thread_id(50), thread_id(51)); let binding = goal_binding(issuer, agent, 7, thread_id(52)); - let (original, original_input) = - accept_work(&fixture.mailbox, issuer, agent, "pending-continuation", "original").await; + let (original, original_input) = accept_work( + &fixture.mailbox, + issuer, + agent, + "pending-continuation", + "original", + ) + .await; assert_eq!( fixture.mailbox.begin_dispatch(&original).await.unwrap(), DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&original).await.unwrap(); - admit( - &fixture.mailbox, - agent, - "source-pending", - &[original_input], - ) - .await; + admit(&fixture.mailbox, agent, "source-pending", &[original_input]).await; let pending = finish( &fixture.mailbox, agent, @@ -839,7 +845,11 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t .unwrap(), DispatchDecision::Dispatch ); - fixture.mailbox.mark_dispatched(&wrong_terminal).await.unwrap(); + fixture + .mailbox + .mark_dispatched(&wrong_terminal) + .await + .unwrap(); admit( &fixture.mailbox, agent, @@ -869,7 +879,11 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t work_item(&still_recovery_needed, &original.id).dispatch_state, DispatchState::RecoveryNeeded ); - assert!(work_item(&still_recovery_needed, &original.id).outcome.is_none()); + assert!( + work_item(&still_recovery_needed, &original.id) + .outcome + .is_none() + ); let (exact_terminal, exact_input) = accept_work( &fixture.mailbox, @@ -887,7 +901,11 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t .unwrap(), DispatchDecision::Dispatch ); - fixture.mailbox.mark_dispatched(&exact_terminal).await.unwrap(); + fixture + .mailbox + .mark_dispatched(&exact_terminal) + .await + .unwrap(); admit( &fixture.mailbox, agent, @@ -904,7 +922,12 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t Continuation::Terminal(binding.clone()), ) .await; - assert!(terminal.outcomes.iter().any(|receipt| receipt.work_ref == original)); + assert!( + terminal + .outcomes + .iter() + .any(|receipt| receipt.work_ref == original) + ); let settled = take_snapshot( &fixture.mailbox, @@ -918,8 +941,14 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t assert_eq!(settled_original.dispatch_state, DispatchState::Dispatched); assert!(settled_original.outcome.is_some()); assert_eq!(settled_original.continuations[0].binding, binding); - assert_eq!(settled_original.continuations[0].state, ContinuationState::Terminal); - assert_eq!(settled_original.continuations[0].source_turn_id, "source-pending"); + assert_eq!( + settled_original.continuations[0].state, + ContinuationState::Terminal + ); + assert_eq!( + settled_original.continuations[0].source_turn_id, + "source-pending" + ); } #[tokio::test] @@ -1696,11 +1725,13 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov DispatchDecision::Dispatch ); fixture.mailbox.mark_dispatched(&work).await.unwrap(); - assert!(fixture - .mailbox - .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) - .await - .unwrap()); + assert!( + fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) + .await + .unwrap() + ); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::RecoveryNeeded @@ -1720,11 +1751,13 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov ) .await; assert_eq!(terminal.outcomes[0].work_ref, work); - assert!(!fixture - .mailbox - .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) - .await - .unwrap()); + assert!( + !fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) + .await + .unwrap() + ); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index f8f26f2febf5..8db81fb0a62d 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -314,7 +314,10 @@ impl AgentControl { .collect()) } - pub(crate) async fn mailbox_reconcile_pending_work(&self, work: &WorkSnapshot) -> CodexResult { + pub(crate) async fn mailbox_reconcile_pending_work( + &self, + work: &WorkSnapshot, + ) -> CodexResult { let Some(mailbox) = self.mailbox() else { return Ok(false); }; @@ -326,23 +329,36 @@ impl AgentControl { || thread.session.input_queue.has_pending_mailbox_items().await || thread.session.input_queue.has_pending_terminal_finalizers() || thread.session.has_outstanding_durable_sleep(); - let goal_owner = thread.session.services.thread_extension_data + let goal_owner = thread + .session + .services + .thread_extension_data .get::() - .and_then(|store| store.snapshot().map(|snapshot| (store.incarnation(), snapshot))); + .and_then(|store| { + store + .snapshot() + .map(|snapshot| (store.incarnation(), snapshot)) + }); (can_proceed, goal_owner) } Ok(_) | Err(_) => (false, None), }; let unresolved = work.continuations.iter().find(|continuation| { - matches!(continuation.state, ContinuationState::PendingGoalBinding | ContinuationState::Deferred) + matches!( + continuation.state, + ContinuationState::PendingGoalBinding | ContinuationState::Deferred + ) }); let evidence = if let Some(continuation) = unresolved { if goal_owner.as_ref().is_some_and(|(incarnation, snapshot)| { goal_binding_matches_native(&continuation.binding, *incarnation, snapshot) && ((snapshot.status == ThreadGoalStatus::Active - && matches!(snapshot.phase, GoalNotificationPhase::Running - | GoalNotificationPhase::ContinuationPending - | GoalNotificationPhase::DeferredWithOwner)) + && matches!( + snapshot.phase, + GoalNotificationPhase::Running + | GoalNotificationPhase::ContinuationPending + | GoalNotificationPhase::DeferredWithOwner + )) || (snapshot.phase == GoalNotificationPhase::ResultReady && can_proceed)) }) { return Ok(false); @@ -350,7 +366,9 @@ impl AgentControl { // A cold, restarted, replaced, or action-required producer cannot prove custody // of this exact old logical continuation. Keep its binding and original refs; // return recovery-needed instead of waiting for a producer that cannot advance it. - RecoveryEvidence::ContinuationUnowned { binding: continuation.binding.clone() } + RecoveryEvidence::ContinuationUnowned { + binding: continuation.binding.clone(), + } } else if can_proceed { return Ok(false); } else { diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 057469c5be47..37197b62717f 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -7,12 +7,15 @@ use codex_agent_mailbox::Selection; use codex_agent_mailbox::Selector; use codex_agent_mailbox::SnapshotItem; use codex_protocol::AgentPath; +use codex_utils_absolute_path::test_support::PathExt; use pretty_assertions::assert_eq; #[tokio::test] async fn mailbox_shared_control_does_not_capture_stock_invocations() { let (session, turn) = crate::session::tests::make_session_and_context().await; let mut config = (*turn.config).clone(); + let sqlite_home = tempfile::tempdir().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.path().abs()); config.features.enable(Feature::MultiAgentV2).unwrap(); config.features.enable(Feature::AgentMailbox).unwrap(); let control = &session.services.agent_control; @@ -128,6 +131,8 @@ async fn exact_work_snapshot(mailbox: &Mailbox, work: &WorkRef) -> WorkSnapshot async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_and_turns() { let (session, turn) = crate::session::tests::make_session_and_context().await; let mut config = (*turn.config).clone(); + let sqlite_home = tempfile::tempdir().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.path().abs()); config.features.enable(Feature::MultiAgentV2).unwrap(); config.features.enable(Feature::AgentMailbox).unwrap(); let control = &session.services.agent_control; @@ -169,6 +174,8 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody() { let (mut session, turn) = crate::session::tests::make_session_and_context().await; let mut config = (*turn.config).clone(); + let sqlite_home = tempfile::tempdir().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.path().abs()); config.features.enable(Feature::MultiAgentV2).unwrap(); config.features.enable(Feature::AgentMailbox).unwrap(); let manager = crate::ThreadManager::with_models_provider_for_tests( @@ -209,6 +216,8 @@ async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_c async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { let (_session, turn) = crate::session::tests::make_session_and_context().await; let mut config = (*turn.config).clone(); + let sqlite_home = tempfile::tempdir().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.path().abs()); config.features.enable(Feature::MultiAgentV2).unwrap(); config.features.enable(Feature::AgentMailbox).unwrap(); let manager = crate::ThreadManager::with_models_provider_for_tests( @@ -216,6 +225,7 @@ async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { config.model_provider.clone(), ); let parent = manager.start_thread(crate::thread_manager::StartThreadOptions::new(config.clone())).await.unwrap(); + parent.thread.session.set_multi_agent_version_if_unset(MultiAgentVersion::V2); let control = &parent.thread.session.services.agent_control; control.mailbox_initialize(&config).await.unwrap(); let mailbox = control.mailbox().unwrap(); diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 094738a605c0..f6233d74d400 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2162,6 +2162,16 @@ async fn resumed_mailbox_wait_is_durably_interrupted_and_idempotent() { rx.try_recv().expect("durable wait start should be published").msg, EventMsg::ItemStarted(event) if event.item.id() == call_id )); + assert!(matches!( + rx.try_recv() + .expect("durable wait start should publish its legacy companion") + .msg, + EventMsg::CollabWaitingBegin(event) if event.call_id == call_id + )); + assert!( + rx.try_recv().is_err(), + "all start notifications should be consumed before testing terminal publication" + ); let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) .await diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index bf4b1efbf829..9450aa44bf9a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -690,7 +690,7 @@ fn mailbox_wait_agent_tool_exposes_unbounded_join_contract_and_structured_result .contains("has no effect") ); assert!(description.contains("unbounded event-only wait")); - assert!(description.contains("does not satisfy a selected join")); + assert!(description.contains("do not satisfy a selected join")); let output_schema = output_schema.expect("wait output schema"); assert_eq!( output_schema["required"], @@ -964,7 +964,7 @@ fn mailbox_send_message_tool_declares_durable_receipt_without_claiming_comprehen ); assert_eq!( output_schema["properties"]["activation_disposition"]["enum"], - json!(["not_requested", "already_active", "activated", "denied"]) + json!(["not_requested", "already_active", "activated", "denied", "unknown"]) ); assert!( output_schema["properties"]["durable_state"]["description"] diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 3681b24e14d6..a3f778521076 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -1,9 +1,9 @@ //! Selection-aware waits over immutable native work and explicit request replies. -use super::*; use super::mailbox_message::deliver_envelope; use super::mailbox_message::mailbox_error; use super::mailbox_message::mailbox_for; +use super::*; use crate::session::input_queue::InputQueueActivity; use crate::session::session::Session; use crate::session::turn_context::TurnContext; @@ -72,7 +72,14 @@ impl CoreToolRuntime for Handler { } async fn handle_wait(invocation: ToolInvocation) -> Result, FunctionCallError> { - let ToolInvocation { session, turn, payload, call_id, cancellation_token, .. } = invocation; + let ToolInvocation { + session, + turn, + payload, + call_id, + cancellation_token, + .. + } = invocation; if let Some(report) = session.mailbox_interrupted_wait_receipt(&call_id).await { return Ok(boxed_tool_output(FunctionToolOutput::from_text( tool_output_json_text(&report, "wait_agent"), @@ -82,16 +89,24 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, let args: WaitArgs = parse_arguments(&function_arguments(payload)?)?; let _compatibility_only = args.native_event_wait; let mailbox = mailbox_for(&session)?; - session.services.agent_control.register_session_root(session.thread_id, turn.parent_thread_id); + session + .services + .agent_control + .register_session_root(session.thread_id, turn.parent_thread_id); // Both subscriptions precede authoritative rescans. Quiet notifications may rescan, but // never shorten the fixed deadline or return control to the provider. let mut mailbox_rx = mailbox.subscribe(); - let (mut input_rx, _) = session.input_queue.subscribe_activity(/*turn_state*/ None).await; + let (mut input_rx, _) = session + .input_queue + .subscribe_activity(/*turn_state*/ None) + .await; let targetless = args.targets.is_none() && args.replies_to.is_none(); let targets = args.targets.unwrap_or_default(); let replies_to = args.replies_to.unwrap_or_default(); if targets.len() + replies_to.len() > 100 { - return Err(mailbox_error("at most 100 work/reply selectors may be joined")); + return Err(mailbox_error( + "at most 100 work/reply selectors may be joined", + )); } let mut selectors = Vec::new(); let mut agents = Vec::new(); @@ -102,8 +117,14 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, let agent = resolve_agent_target(&session, &turn, target).await?; reject_reverse_completion_join(&session, &turn, agent)?; agents.push(agent); - selectors.extend(mailbox.issued_work_refs(session.thread_id, Some(agent), 100) - .await.map_err(mailbox_error)?.into_iter().map(|work| Selector::Work(work.id))); + selectors.extend( + mailbox + .issued_work_refs(session.thread_id, Some(agent), 100) + .await + .map_err(mailbox_error)? + .into_iter() + .map(|work| Selector::Work(work.id)), + ); } } selectors.extend(replies_to.iter().cloned().map(Selector::Reply)); @@ -113,10 +134,14 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, MailboxWaitReturnWhen::Any => ReturnWhen::Any, MailboxWaitReturnWhen::All => ReturnWhen::All, }; - let deadline = args.timeout_ms.map(|timeout| { - Instant::now().checked_add(Duration::from_millis(timeout)) - .ok_or_else(|| mailbox_error("timeout_ms exceeds the monotonic clock range")) - }).transpose()?; + let deadline = args + .timeout_ms + .map(|timeout| { + Instant::now() + .checked_add(Duration::from_millis(timeout)) + .ok_or_else(|| mailbox_error("timeout_ms exceeds the monotonic clock range")) + }) + .transpose()?; let mut report = MailboxWaitReport { requested_targets: targets, requested_replies_to: replies_to, @@ -129,12 +154,21 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, wake_sender: None, wake_ref: None, }; - report.pending = selectors.iter().map(|selector| MailboxWaitSelectionItem { - work_ref: match selector { Selector::Work(id) => Some(id.clone()), Selector::Reply(_) => None }, - message_id: match selector { Selector::Reply(id) => Some(id.clone()), Selector::Work(_) => None }, - agent: None, - status: "pending".to_string(), - }).collect(); + report.pending = selectors + .iter() + .map(|selector| MailboxWaitSelectionItem { + work_ref: match selector { + Selector::Work(id) => Some(id.clone()), + Selector::Reply(_) => None, + }, + message_id: match selector { + Selector::Reply(id) => Some(id.clone()), + Selector::Work(_) => None, + }, + agent: None, + status: "pending".to_string(), + }) + .collect(); let mut selected = Vec::new(); let mut attention_to_observe = None; let mut started = false; @@ -145,7 +179,12 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, let input_generation = mailbox.wait_input_generation(); if targetless { match mailbox.issued_work_refs(session.thread_id, None, 100).await { - Ok(work) => selectors = work.into_iter().map(|work| Selector::Work(work.id)).collect(), + Ok(work) => { + selectors = work + .into_iter() + .map(|work| Selector::Work(work.id)) + .collect() + } Err(_) => { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); break; @@ -155,13 +194,16 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, selected = if selectors.is_empty() { Vec::new() } else { - match mailbox.snapshot(&Selection { - caller: session.thread_id, - agent: None, - selectors: selectors.clone(), - return_when, - limit: 100, - }).await { + match mailbox + .snapshot(&Selection { + caller: session.thread_id, + agent: None, + selectors: selectors.clone(), + return_when, + limit: 100, + }) + .await + { Ok(snapshot) => snapshot.items, Err(_) => { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); @@ -184,9 +226,17 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, && work.dispatch_state == DispatchState::Dispatched && work.outcome.is_none() { - match session.services.agent_control.mailbox_reconcile_pending_work(work).await { + match session + .services + .agent_control + .mailbox_reconcile_pending_work(work) + .await + { Ok(changed) => recovery_changed |= changed, - Err(_) => { recovery_fault = true; break; } + Err(_) => { + recovery_fault = true; + break; + } } } } @@ -207,17 +257,34 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } let ready = report.completed.len(); let early_attention = selected.iter().any(needs_attention); - let satisfied = !selected.is_empty() && match return_when { - ReturnWhen::Any => ready > 0, - ReturnWhen::All => ready == selectors.len(), - }; + let satisfied = !selected.is_empty() + && match return_when { + ReturnWhen::Any => ready > 0, + ReturnWhen::All => ready == selectors.len(), + }; let all_observed = !selected.is_empty() && selected.iter().all(already_observed); if !started { - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await?; + emit_report( + &session, + &turn, + &call_id, + &agents, + &report, + CollabAgentToolCallStatus::InProgress, + ) + .await?; started = true; last_report = Some(report.clone()); } else if last_report.as_ref() != Some(&report) { - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::InProgress).await?; + emit_report( + &session, + &turn, + &call_id, + &agents, + &report, + CollabAgentToolCallStatus::InProgress, + ) + .await?; last_report = Some(report.clone()); } if mailbox.has_runtime_fault(&agents) { @@ -245,7 +312,9 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, report.condition_satisfied = true; report.completion_reason = Some(if all_observed { MailboxWakeCause::AlreadySatisfied - } else if selected.iter().any(|item| matches!(item, SnapshotItem::Request(request) if request.reply.is_some())) { + } else if selected.iter().any( + |item| matches!(item, SnapshotItem::Request(request) if request.reply.is_some()), + ) { MailboxWakeCause::Reply } else { MailboxWakeCause::Result @@ -262,8 +331,18 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } }; for attention in attention { - let Some(message_id) = attention.id.strip_prefix("attention:amsg_").map(|suffix| format!("amsg_{suffix}")) else { continue; }; - if let Some(envelope) = mailbox.get_message(session.thread_id, &message_id).await.map_err(mailbox_error)? { + let Some(message_id) = attention + .id + .strip_prefix("attention:amsg_") + .map(|suffix| format!("amsg_{suffix}")) + else { + continue; + }; + if let Some(envelope) = mailbox + .get_message(session.thread_id, &message_id) + .await + .map_err(mailbox_error)? + { if deliver_envelope(&session, &turn, &envelope).await.is_err() { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); break; @@ -278,7 +357,11 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, if report.completion_reason.is_some() { break; } - if let Some(activity) = session.input_queue.pending_wait_input_activity(&session.active_turn).await { + if let Some(activity) = session + .input_queue + .pending_wait_input_activity(&session.active_turn) + .await + { match activity { InputQueueActivity::Steer => { report.completion_reason = Some(MailboxWakeCause::Human); @@ -299,13 +382,29 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, report.completion_reason = Some(MailboxWakeCause::Deadline); break; } - let peers = selected.iter().filter_map(|item| match item { - SnapshotItem::Work(work) if work.outcome.is_none() && !work.observed => Some(work.work_ref.agent), - SnapshotItem::Request(request) if request.reply.is_none() && !request.observed => Some(request.request.recipient), - SnapshotItem::Work(_) | SnapshotItem::Request(_) => None, - }).collect::>().into_iter().collect(); + let peers = selected + .iter() + .filter_map(|item| match item { + SnapshotItem::Work(work) if work.outcome.is_none() && !work.observed => { + Some(work.work_ref.agent) + } + SnapshotItem::Request(request) if request.reply.is_none() && !request.observed => { + Some(request.request.recipient) + } + SnapshotItem::Work(_) | SnapshotItem::Request(_) => None, + }) + .collect::>() + .into_iter() + .collect(); let wait_id = format!("{}:{call_id}", session.thread_id); - match mailbox.begin_wait_at(wait_id, session.thread_id, peers, return_when, deadline.is_some(), input_generation) { + match mailbox.begin_wait_at( + wait_id, + session.thread_id, + peers, + return_when, + deadline.is_some(), + input_generation, + ) { Ok(guard) => registration = Some(guard), Err(codex_agent_mailbox::MailboxError::StaleWaitSnapshot) => continue, Err(codex_agent_mailbox::MailboxError::ClosedPeerCycle) => { @@ -352,21 +451,35 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, // including an interrupting attention message. Metadata is non-consuming; // omitted bodies retain their per-item observation state and retrieval IDs. let body_limit = if attention_to_observe.is_some() { 1 } else { 2 }; - if admit_selected(&session, &turn, &mailbox, &selected, body_limit).await.is_err() { + if admit_selected(&session, &turn, &mailbox, &selected, body_limit) + .await + .is_err() + { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); report.condition_satisfied = false; } if let Some(id) = attention_to_observe { - if mailbox.acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) - .await.is_err() + if mailbox + .acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) + .await + .is_err() { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); report.condition_satisfied = false; } } - emit_report(&session, &turn, &call_id, &agents, &report, CollabAgentToolCallStatus::Completed).await?; + emit_report( + &session, + &turn, + &call_id, + &agents, + &report, + CollabAgentToolCallStatus::Completed, + ) + .await?; Ok(boxed_tool_output(FunctionToolOutput::from_text( - tool_output_json_text(&report, "wait_agent"), Some(true), + tool_output_json_text(&report, "wait_agent"), + Some(true), ))) } @@ -377,22 +490,53 @@ fn selector_key(selector: &Selector) -> String { } } -fn reject_reverse_completion_join(session: &Session, turn: &TurnContext, agent: ThreadId) -> Result<(), FunctionCallError> { - let current = turn.session_source.get_agent_path().unwrap_or_else(AgentPath::root); - let target = session.services.agent_control.get_agent_metadata(agent).and_then(|metadata| metadata.agent_path); - if agent == session.thread_id || target.as_ref().is_some_and(|target| target == ¤t - || current.as_str().strip_prefix(target.as_str()).is_some_and(|suffix| suffix.starts_with('/'))) { - return Err(mailbox_error("self/ancestor completion joins are not allowed; explicit parent-reply waits remain valid")); +fn reject_reverse_completion_join( + session: &Session, + turn: &TurnContext, + agent: ThreadId, +) -> Result<(), FunctionCallError> { + let current = turn + .session_source + .get_agent_path() + .unwrap_or_else(AgentPath::root); + let target = session + .services + .agent_control + .get_agent_metadata(agent) + .and_then(|metadata| metadata.agent_path); + if agent == session.thread_id + || target.as_ref().is_some_and(|target| { + target == ¤t + || current + .as_str() + .strip_prefix(target.as_str()) + .is_some_and(|suffix| suffix.starts_with('/')) + }) + { + return Err(mailbox_error( + "self/ancestor completion joins are not allowed; explicit parent-reply waits remain valid", + )); } Ok(()) } fn needs_attention(item: &SnapshotItem) -> bool { match item { - SnapshotItem::Work(work) => work.dispatch_state == DispatchState::RecoveryNeeded - || work.outcome.as_ref().is_some_and(|result| result.outcome != Outcome::Completed) - || work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention), - SnapshotItem::Request(request) => matches!(request.disposition, RequestDisposition::Superseded | RequestDisposition::Withdrawn), + SnapshotItem::Work(work) => { + work.dispatch_state == DispatchState::RecoveryNeeded + || work + .outcome + .as_ref() + .is_some_and(|result| result.outcome != Outcome::Completed) + || work + .continuations + .iter() + .any(|continuation| continuation.outcome == Outcome::NeedsAttention) + } + SnapshotItem::Request(request) => matches!( + request.disposition, + RequestDisposition::Superseded | RequestDisposition::Withdrawn + ), } } @@ -411,41 +555,74 @@ fn update_report(session: &Session, report: &mut MailboxWaitReport, selected: &[ SnapshotItem::Work(work) => { let status = if work.dispatch_state == DispatchState::RecoveryNeeded { "recovery_needed".to_string() - } else if work.continuations.iter().any(|continuation| continuation.outcome == Outcome::NeedsAttention) { + } else if work + .continuations + .iter() + .any(|continuation| continuation.outcome == Outcome::NeedsAttention) + { "needs_attention".to_string() } else { - work.outcome.as_ref().map_or_else(|| "pending".to_string(), |outcome| match outcome.outcome { - Outcome::Completed => "completed", - Outcome::Failed => "failed", - Outcome::Cancelled => "cancelled", - Outcome::NeedsAttention => "needs_attention", - }.to_string()) + work.outcome.as_ref().map_or_else( + || "pending".to_string(), + |outcome| { + match outcome.outcome { + Outcome::Completed => "completed", + Outcome::Failed => "failed", + Outcome::Cancelled => "cancelled", + Outcome::NeedsAttention => "needs_attention", + } + .to_string() + }, + ) }; - (MailboxWaitSelectionItem { - work_ref: Some(work.work_ref.id.clone()), - message_id: work.outcome.as_ref().map(|outcome| outcome.id.clone()), - agent: session.services.agent_control.get_agent_metadata(work.work_ref.agent) - .and_then(|metadata| metadata.agent_path).map(|path| path.to_string()), - status, - }, work.outcome.is_some() || work.observed) + ( + MailboxWaitSelectionItem { + work_ref: Some(work.work_ref.id.clone()), + message_id: work.outcome.as_ref().map(|outcome| outcome.id.clone()), + agent: session + .services + .agent_control + .get_agent_metadata(work.work_ref.agent) + .and_then(|metadata| metadata.agent_path) + .map(|path| path.to_string()), + status, + }, + work.outcome.is_some() || work.observed, + ) } - SnapshotItem::Request(request) => (MailboxWaitSelectionItem { - work_ref: None, - message_id: Some(request.request_id.clone()), - agent: Some(request.request.communication.recipient.to_string()), - status: match request.disposition { - RequestDisposition::Open => "pending".to_string(), - RequestDisposition::Replied => "replied".to_string(), - RequestDisposition::Superseded => format!("superseded:{}", request.replacement.as_deref().unwrap_or("unknown")), - RequestDisposition::Withdrawn => "withdrawn".to_string(), + SnapshotItem::Request(request) => ( + MailboxWaitSelectionItem { + work_ref: None, + message_id: Some(request.request_id.clone()), + agent: Some(request.request.communication.recipient.to_string()), + status: match request.disposition { + RequestDisposition::Open => "pending".to_string(), + RequestDisposition::Replied => "replied".to_string(), + RequestDisposition::Superseded => format!( + "superseded:{}", + request.replacement.as_deref().unwrap_or("unknown") + ), + RequestDisposition::Withdrawn => "withdrawn".to_string(), + }, }, - }, request.reply.is_some() || request.observed), + request.reply.is_some() || request.observed, + ), }; - if ready { report.completed.push(metadata); } else { report.pending.push(metadata); } + if ready { + report.completed.push(metadata); + } else { + report.pending.push(metadata); + } } } -async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox, selected: &[SnapshotItem], body_limit: usize) -> Result<(), FunctionCallError> { +async fn admit_selected( + session: &Session, + turn: &TurnContext, + mailbox: &Mailbox, + selected: &[SnapshotItem], + body_limit: usize, +) -> Result<(), FunctionCallError> { let mut observations = Vec::new(); let mut delivered = HashSet::new(); for item in selected { @@ -455,8 +632,13 @@ async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox if !delivered.contains(&outcome.id) && delivered.len() >= body_limit { continue; } - let envelope = mailbox.get_message(session.thread_id, &outcome.id).await - .map_err(mailbox_error)?.ok_or_else(|| mailbox_error("terminal result body requires compatible recovery"))?; + let envelope = mailbox + .get_message(session.thread_id, &outcome.id) + .await + .map_err(mailbox_error)? + .ok_or_else(|| { + mailbox_error("terminal result body requires compatible recovery") + })?; // Session owns append + successful flush + exact canonical acknowledgement // under one custody gate, including synthetic typed terminal envelopes. deliver_envelope(session, turn, &envelope).await?; @@ -478,17 +660,38 @@ async fn admit_selected(session: &Session, turn: &TurnContext, mailbox: &Mailbox } } if !observations.is_empty() { - mailbox.acknowledge_observed(session.thread_id, &observations).await.map_err(mailbox_error)?; + mailbox + .acknowledge_observed(session.thread_id, &observations) + .await + .map_err(mailbox_error)?; } Ok(()) } -async fn emit_report(session: &Session, turn: &TurnContext, call_id: &str, agents: &[ThreadId], report: &MailboxWaitReport, status: CollabAgentToolCallStatus) -> Result<(), FunctionCallError> { +async fn emit_report( + session: &Session, + turn: &TurnContext, + call_id: &str, + agents: &[ThreadId], + report: &MailboxWaitReport, + status: CollabAgentToolCallStatus, +) -> Result<(), FunctionCallError> { let item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { - id: call_id.to_string(), tool: CollabAgentTool::Wait, status, - sender_thread_id: session.thread_id, receiver_thread_ids: agents.to_vec(), receiver_agents: Vec::new(), - prompt: None, model: None, reasoning_effort: None, requested_model: None, requested_reasoning_effort: None, - agents_states: Default::default(), wake_notifications: None, completion_reason: None, wake_cause: None, + id: call_id.to_string(), + tool: CollabAgentTool::Wait, + status, + sender_thread_id: session.thread_id, + receiver_thread_ids: agents.to_vec(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: Default::default(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, mailbox_wait_report: Some(report.clone()), }); session diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index 8af7d82afa29..dbccaed09bd0 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -454,6 +454,9 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { }) .await .unwrap(); + // The replying agent must first have the request in canonical history; reply + // admission is intentionally rejected until that exact request is canonical. + mailbox.mark_canonical(agent, &request.id).await.unwrap(); let body = "quiet exact reply body"; let reply = mailbox .accept_message(MessageDraft { diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index bcca77636683..43045cd504f2 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -242,7 +242,7 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { ); assert_eq!( send_output["properties"]["activation_disposition"]["enum"], - json!(["not_requested", "already_active", "activated", "denied"]) + json!(["not_requested", "already_active", "activated", "denied", "unknown"]) ); assert!( send_output["properties"]["durable_state"]["description"] diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 41aa8f2d5d99..9c710b4d83ba 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -112,7 +112,7 @@ impl Respond for UnadmittedWorkResponder { })); } function_call_response("unadmitted-spawn-response", "unadmitted-spawn", "spawn_agent", json!({ - "task_name": "unadmitted-worker", + "task_name": "unadmitted_worker", "message": "fail before provider acceptance of this work", })) } @@ -421,7 +421,7 @@ impl Respond for RootMailboxFlowResponder { "spawn-call", "spawn_agent", json!({ - "task_name": "quick-worker", + "task_name": "quick_worker", "message": CHILD_TASK, }), ) @@ -661,7 +661,7 @@ impl Respond for WaitProbeResponder { "spawn-b-call", "spawn_agent", json!({ - "task_name": "unrelated-worker", + "task_name": "unrelated_worker", "message": UNRELATED_CHILD_TASK, }), ); @@ -672,7 +672,7 @@ impl Respond for WaitProbeResponder { "spawn-a-call", "spawn_agent", json!({ - "task_name": "watched-worker", + "task_name": "watched_worker", "message": QUIET_CHILD_TASK, }), ) @@ -1008,6 +1008,9 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com config.model_provider.supports_websockets = false; config.model_provider.stream_max_retries = Some(0); config.model_provider.request_max_retries = Some(0); + // This fixture supplies a local summarization prompt, not a remote + // compaction endpoint. Match the maintained local-compaction fixtures. + config.model_provider.name = "OpenAI (test)".to_string(); config.compact_prompt = Some(COMPACTION_PROMPT.to_string()); }); let test = builder.build_with_auto_env(&server).await?; diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_mailbox_wait_report_active_progress.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_mailbox_wait_report_active_progress.snap new file mode 100644 index 000000000000..67ea3e83f20c --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_mailbox_wait_report_active_progress.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/chatwidget/tests/app_server.rs +expression: updated_lines +--- +• Waiting for all of 3 selected conditions + â”” Join condition: all + Targets: Ada, work_42 + Replies to: msg_request + Completed Ada: completed + Completed work_42: completed + Pending Robie · msg_request: awaiting_reply + Quiet count: 4 diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__replayed_mailbox_wait_report_interrupted_terminal.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__replayed_mailbox_wait_report_interrupted_terminal.snap new file mode 100644 index 000000000000..9b6ef0aa45fb --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__replayed_mailbox_wait_report_interrupted_terminal.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/tests/history_replay.rs +expression: interrupted_lines +--- +• Mailbox wait returned: interrupted + â”” Join condition: any + Targets: worker-A, work_17 + Replies to: msg_request + Pending worker-A: running + Pending work_17: pending + Pending reviewer · msg_request: awaiting_reply + Condition satisfied: no + Completion reason: interrupted + Quiet count: 5 From 2b1fe6158b0b72cb9113493239784e76189e9aa8 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:41:06 +1100 Subject: [PATCH 12/54] fix(mailbox): reject incompatible native recipients before acceptance --- codex-rs/agent-mailbox/src/mailbox_tests.rs | 91 ++++++++ codex-rs/agent-mailbox/src/types.rs | 3 + codex-rs/agent-mailbox/src/work.rs | 18 +- codex-rs/core/src/agent/control/lifecycle.rs | 52 ++++- codex-rs/core/src/agent/control/mailbox.rs | 120 ++++++++++- .../core/src/agent/control/mailbox_tests.rs | 4 +- .../core/src/agent/control/residency_tests.rs | 198 ++++++++++++++++++ codex-rs/core/src/agent/control/spawn.rs | 38 +++- .../src/tools/handlers/multi_agents_spec.rs | 8 + .../tools/handlers/multi_agents_spec_tests.rs | 4 + .../handlers/multi_agents_v2/message_tool.rs | 57 ++++- .../tools/handlers/multi_agents_v2/spawn.rs | 30 ++- codex-rs/core/src/tools/parallel.rs | 54 +++++ codex-rs/core/src/tools/spec_plan_tests.rs | 4 + docs/carry-divergence-ledger.md | 7 + 15 files changed, 656 insertions(+), 32 deletions(-) diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index ca8e9dd71a86..108da654d18d 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1758,10 +1758,101 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov .await .unwrap() ); + assert!( + !fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .unwrap() + ); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::Dispatched ); + let snapshot = take_snapshot( + &fixture.mailbox, + issuer, + Some(agent), + vec![Selector::Work(work.id.clone())], + ReturnWhen::Any, + ) + .await; + assert_eq!( + work_item(&snapshot, &work.id).outcome, + Some(terminal.outcomes[0].clone()) + ); +} + +#[tokio::test] +async fn recipient_incompatibility_recovers_accepted_work_without_losing_or_resending_it() { + let fixture = test_mailbox().await; + let (issuer, agent) = (thread_id(37), thread_id(38)); + let (work, input_id) = accept_work( + &fixture.mailbox, + issuer, + agent, + "recipient-incompatible", + "captured input", + ) + .await; + let (unrelated, unrelated_input_id) = accept_work( + &fixture.mailbox, + issuer, + agent, + "recipient-still-compatible", + "unrelated captured input", + ) + .await; + + assert!(fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .unwrap()); + assert_eq!( + fixture.mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::RecoveryNeeded + ); + assert_eq!( + fixture.mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::RecoveryNeeded + ); + assert!(!fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .unwrap()); + + assert_eq!( + fixture + .mailbox + .work_for_native_key(issuer, "recipient-incompatible") + .await + .unwrap(), + Some(work.clone()) + ); + let retained_input = fixture.mailbox.accepted_input(&work).await.unwrap(); + assert_eq!(retained_input.id.as_deref(), Some(input_id.as_str())); + assert_eq!(retained_input.content, "captured input"); + + assert_eq!( + fixture.mailbox.dispatch_state(&unrelated).await.unwrap(), + DispatchState::Accepted + ); + assert_eq!( + fixture + .mailbox + .work_for_native_key(issuer, "recipient-still-compatible") + .await + .unwrap(), + Some(unrelated.clone()) + ); + let unrelated_input = fixture.mailbox.accepted_input(&unrelated).await.unwrap(); + assert_eq!( + unrelated_input.id.as_deref(), + Some(unrelated_input_id.as_str()) + ); + assert_eq!(unrelated_input.content, "unrelated captured input"); } #[tokio::test] diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 159cd1d64db1..1ce228a3cdc6 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -128,6 +128,9 @@ pub enum RecoveryEvidence { /// The adapter proved this exact native goal binding has no owner. /// The durable continuation itself is retained for later exact settlement. ContinuationUnowned { binding: GoalBinding }, + /// The adapter proved the accepted recipient no longer supports mailbox delivery. + /// Keep the accepted work and its captured input for observation; never resend it. + RecipientIncompatible, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index e8469f7d6591..8009921bcc07 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -290,7 +290,7 @@ impl Mailbox { Ok(result.rows_affected() as usize) } - /// Record adapter-confirmed stale in-flight custody for this exact work. + /// Record adapter-confirmed recovery evidence for this exact work. /// Returns whether this call changed the dispatch state. pub async fn mark_recovery_needed( &self, @@ -300,12 +300,14 @@ impl Mailbox { let mut transaction = self.pool.begin().await?; let row = exact_work_tx(&mut transaction, work_ref).await?; let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; - if state == DispatchState::Accepted { + if state == DispatchState::Accepted + && !matches!(&evidence, RecoveryEvidence::RecipientIncompatible) + { return Err(MailboxError::ReferenceMismatch); } let matching_binding_json = match &evidence { - RecoveryEvidence::DeliveryUnavailable => None, + RecoveryEvidence::DeliveryUnavailable | RecoveryEvidence::RecipientIncompatible => None, RecoveryEvidence::ContinuationUnowned { binding } => { if binding.parent_thread_id != work_ref.issuer || binding.child_thread_id != work_ref.agent @@ -354,6 +356,16 @@ impl Mailbox { .execute(&mut *transaction) .await? } + (RecoveryEvidence::RecipientIncompatible, _) => { + sqlx::query( + "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' + WHERE id = ? AND dispatch_state IN ('accepted', 'dispatching', 'dispatched') + AND outcome_json IS NULL", + ) + .bind(&work_ref.id) + .execute(&mut *transaction) + .await? + } (RecoveryEvidence::ContinuationUnowned { .. }, Some(binding_json)) => { sqlx::query( "UPDATE mailbox_work SET dispatch_state = 'recovery_needed' diff --git a/codex-rs/core/src/agent/control/lifecycle.rs b/codex-rs/core/src/agent/control/lifecycle.rs index a1477facd82a..8975f639c845 100644 --- a/codex-rs/core/src/agent/control/lifecycle.rs +++ b/codex-rs/core/src/agent/control/lifecycle.rs @@ -227,17 +227,27 @@ impl PreparedV2AgentDelivery { &self, communication: &mut InterAgentCommunication, context: &AgentCommunicationContext, + recipient_config: Option<&Config>, ) -> CodexResult { let mut submission_id = new_submission_id(); - let accepted = self - .control - .mailbox_accept_native_submission( + let accepted = if let Some(recipient_config) = recipient_config { + self.control + .mailbox_accept_native_submission( self.agent_id, context, communication, &mut submission_id, + recipient_config, ) - .await?; + .await? + } else if context.native_key.is_some() { + return Err(CodexErr::UnsupportedOperation(format!( + "cannot verify the resolved configuration for mailbox recipient {}; the original input was not submitted", + self.agent_id + ))); + } else { + None + }; let (work, dispatch) = accepted .map_or((None, DispatchDecision::Dispatch), |(work, decision)| { (Some(work), decision) @@ -316,12 +326,19 @@ impl PreparedV2AgentDelivery { if !thread.is_running() { return Err(CodexErr::InternalAgentDied); } - if interrupt { - self.state - .record_submitted_op(self.agent_id, &Op::Interrupt); - thread.session.interrupt_task().await; - } - let submission = self.record_submission(&mut communication, &context).await?; + let recipient_config = thread.session.get_config().await; + self.control + .mailbox_require_native_recipient_compatible( + self.agent_id, + &context, + &communication, + &recipient_config, + self.metadata.agent_role.as_deref(), + ) + .await?; + let submission = self + .record_submission(&mut communication, &context, Some(&recipient_config)) + .await?; match submission.dispatch { DispatchDecision::AlreadyDispatched => return Ok(submission.id), DispatchDecision::RecoveryNeeded => { @@ -336,6 +353,11 @@ impl PreparedV2AgentDelivery { } DispatchDecision::Dispatch => {} } + if interrupt { + self.state + .record_submitted_op(self.agent_id, &Op::Interrupt); + thread.session.interrupt_task().await; + } let send = crate::session::inter_agent_communication; send(&thread.session, submission.id.clone(), communication).await; if let Some(work) = &submission.work { @@ -347,7 +369,15 @@ impl PreparedV2AgentDelivery { if communication.trigger_turn { return Err(CodexErr::ThreadNotFound(self.agent_id)); } - let submission = self.record_submission(&mut communication, &context).await?; + if context.native_key.is_some() { + return Err(CodexErr::UnsupportedOperation(format!( + "cannot verify the resolved configuration for mailbox recipient {}; the original input was not submitted", + self.agent_id + ))); + } + let submission = self + .record_submission(&mut communication, &context, None) + .await?; match submission.dispatch { DispatchDecision::AlreadyDispatched => return Ok(submission.id), DispatchDecision::RecoveryNeeded => { diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 8db81fb0a62d..0864e2dc04d9 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -7,6 +7,7 @@ use super::AgentControl; use crate::agent::AgentStatus; use crate::agent::goal_notifications::GoalNotificationPhase; use crate::agent::goal_notifications::GoalNotificationSnapshot; +use crate::agent::role::apply_role_to_config; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::config::Config; @@ -53,10 +54,26 @@ fn goal_binding_matches_native( } impl AgentControl { + pub(crate) fn mailbox_config_compatible(config: &Config) -> bool { + config.multi_agent_version_from_features() == MultiAgentVersion::V2 + && config.features.enabled(Feature::AgentMailbox) + } + + pub(super) fn mailbox_config_incompatibility_error( + recipient: &str, + role: Option<&str>, + ) -> CodexErr { + let target = role.map_or_else( + || recipient.to_string(), + |role| format!("role `{role}` ({recipient})"), + ); + CodexErr::UnsupportedOperation(format!( + "tracked agent delivery to {target} requires MultiAgentV2 and agent_mailbox to be enabled" + )) + } + pub(crate) async fn mailbox_initialize(&self, config: &Config) -> CodexResult<()> { - if config.multi_agent_version_from_features() != MultiAgentVersion::V2 - || !config.features.enabled(Feature::AgentMailbox) - { + if !Self::mailbox_config_compatible(config) { return Ok(()); } self.mailbox @@ -74,6 +91,30 @@ impl AgentControl { self.mailbox.get().is_some() } + /// Resolve compatibility without reloading a cold recipient or restoring its queued input. + /// The retained role is the authoritative per-agent config selector while the runtime is cold. + pub(crate) async fn mailbox_recipient_config_compatible( + &self, + recipient: ThreadId, + fallback_config: &Config, + ) -> CodexResult { + let state = self.upgrade()?; + if let Ok(thread) = state.get_thread(recipient).await { + let config = thread.session.get_config().await; + return Ok(Self::mailbox_config_compatible(&config)); + } + + let metadata = self + .state + .agent_metadata_for_thread(recipient) + .ok_or(CodexErr::ThreadNotFound(recipient))?; + let mut resolved_config = fallback_config.clone(); + apply_role_to_config(&mut resolved_config, metadata.agent_role.as_deref()) + .await + .map_err(CodexErr::InvalidRequest)?; + Ok(Self::mailbox_config_compatible(&resolved_config)) + } + pub(crate) fn mailbox(&self) -> Option> { self.mailbox.get().cloned() } @@ -116,6 +157,7 @@ impl AgentControl { context: &AgentCommunicationContext, communication: &mut InterAgentCommunication, submission_id: &mut String, + recipient_config: &Config, ) -> CodexResult> { let Some(mailbox) = self.mailbox() else { return Ok(None); @@ -132,6 +174,14 @@ impl AgentControl { let Some(native_key) = context.native_key.as_deref() else { return Ok(None); }; + self.mailbox_require_native_recipient_compatible( + agent, + context, + communication, + recipient_config, + /*recipient_role*/ None, + ) + .await?; if let Some(existing) = mailbox .work_for_native_key(context.sender_thread_id, native_key) .await @@ -155,6 +205,57 @@ impl AgentControl { Ok(Some((work, dispatch))) } + pub(super) async fn mailbox_require_native_recipient_compatible( + &self, + recipient: ThreadId, + context: &AgentCommunicationContext, + communication: &InterAgentCommunication, + recipient_config: &Config, + recipient_role: Option<&str>, + ) -> CodexResult<()> { + if !matches!( + context.kind, + AgentCommunicationKind::Spawn | AgentCommunicationKind::Followup + ) || context.native_key.is_none() + || Self::mailbox_config_compatible(recipient_config) + { + return Ok(()); + } + + let incompatibility = Self::mailbox_config_incompatibility_error( + &format!("recipient {recipient}"), + recipient_role, + ); + let Some(mailbox) = self.mailbox() else { + return Err(incompatibility); + }; + if let Some(native_key) = context.native_key.as_deref() + && let Some(work) = mailbox + .work_for_native_key(context.sender_thread_id, native_key) + .await + .map_err(storage_error)? + { + let mut accepted = mailbox.accepted_input(&work).await.map_err(storage_error)?; + let mut requested = communication.clone(); + accepted.id = None; + requested.id = None; + if work.agent != recipient || accepted != requested { + return Err(CodexErr::InvalidRequest(format!( + "native mailbox key `{native_key}` was reused with a different recipient or input" + ))); + } + mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .map_err(storage_error)?; + return Err(CodexErr::UnsupportedOperation(format!( + "accepted work {} for {recipient} is incompatible with the current recipient configuration; its original input is retained and recovery is required", + work.id + ))); + } + Err(incompatibility) + } + pub(super) async fn mailbox_dispatched(&self, work: &WorkRef) -> CodexResult<()> { if let Some(mailbox) = self.mailbox() { mailbox.mark_dispatched(work).await.map_err(storage_error)?; @@ -162,6 +263,19 @@ impl AgentControl { Ok(()) } + pub(super) async fn mailbox_mark_recipient_incompatible( + &self, + work: &WorkRef, + ) -> CodexResult<()> { + if let Some(mailbox) = self.mailbox() { + mailbox + .mark_recovery_needed(work, RecoveryEvidence::RecipientIncompatible) + .await + .map_err(storage_error)?; + } + Ok(()) + } + pub(crate) async fn mailbox_activate_if_idle( &self, recipient: ThreadId, diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 37197b62717f..8b3a8fcf1e3e 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -38,7 +38,8 @@ async fn mailbox_shared_control_does_not_capture_stock_invocations() { recipient, &context, &mut communication, - &mut submission_id + &mut submission_id, + &config, ) .await .unwrap() @@ -62,6 +63,7 @@ async fn mailbox_shared_control_does_not_capture_stock_invocations() { &enabled_context, &mut communication, &mut submission_id, + &config, ) .await .unwrap() diff --git a/codex-rs/core/src/agent/control/residency_tests.rs b/codex-rs/core/src/agent/control/residency_tests.rs index 1aaaef1ea093..f758ac31e3f3 100644 --- a/codex-rs/core/src/agent/control/residency_tests.rs +++ b/codex-rs/core/src/agent/control/residency_tests.rs @@ -14,6 +14,7 @@ use crate::context::TerminalCompletionStatus; use crate::init_state_db; use crate::thread_manager::ThreadManagerState; use crate::thread_manager::V2ThreadUnloadResult; +use codex_agent_mailbox::DispatchState; use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::AgentPath; @@ -32,6 +33,7 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::user_input::UserInput; +use codex_utils_absolute_path::test_support::PathExt; use pretty_assertions::assert_eq; use std::sync::Arc; use std::task::Context; @@ -608,6 +610,202 @@ async fn spawned_v2_terminal_child_unloads_at_terminal_idle_deadline() { assert_eq!(control.v2_residency.resident_count(), 0); } +#[tokio::test] +async fn mailbox_rejects_incompatible_spawn_and_followup_before_capture() { + let mut config = test_config().await; + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + config.multi_agent_v2.max_concurrent_threads_per_session = 4; + let temp_home = tempfile::tempdir().expect("create temp home"); + config.codex_home = temp_home.path().to_path_buf().try_into().unwrap(); + config.cwd = temp_home.path().to_path_buf().try_into().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(temp_home.path().abs()); + let manager = ThreadManager::with_models_provider_and_home_for_tests( + CodexAuth::from_api_key("dummy"), + config.model_provider.clone(), + config.codex_home.to_path_buf(), + Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), + ); + let root = manager + .start_thread(StartThreadOptions::new(config.clone())) + .await + .expect("start mailbox-enabled V2 root"); + let control = manager.agent_control(); + let mut disabled_config = config.clone(); + disabled_config.features.disable(Feature::AgentMailbox).unwrap(); + + let disabled_spawn_source = Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(AgentPath::root().join("disabled_spawn").unwrap()), + agent_nickname: None, + agent_role: None, + })); + let before_rejected_spawn = manager.list_thread_ids().await; + let rejected_spawn = control + .spawn_agent_with_communication( + disabled_config.clone(), + test_communication("incompatible spawn", /*trigger_turn*/ true), + AgentCommunicationContext::new(AgentCommunicationKind::Spawn, root.thread_id) + .with_native_key("disabled-spawn-call".to_string()), + disabled_spawn_source, + SpawnAgentOptions::default(), + ) + .await; + let rejected_spawn_error = rejected_spawn + .expect_err("incompatible tracked spawn must fail before child creation"); + assert!(rejected_spawn_error.to_string().contains("disabled_spawn")); + assert!( + rejected_spawn_error + .to_string() + .contains("MultiAgentV2 and agent_mailbox") + ); + let after_rejected_spawn = manager.list_thread_ids().await; + assert_eq!(after_rejected_spawn.len(), before_rejected_spawn.len()); + assert!( + before_rejected_spawn + .iter() + .all(|thread_id| after_rejected_spawn.contains(thread_id)) + ); + assert!( + control + .mailbox_work_for_key(root.thread_id, "disabled-spawn-call") + .await + .unwrap() + .is_none() + ); + + let accepted_spawn_source = Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(AgentPath::root().join("mailbox_retry").unwrap()), + agent_nickname: None, + agent_role: None, + })); + let accepted_communication = + test_communication("accepted spawn input", /*trigger_turn*/ false); + let accepted_context = AgentCommunicationContext::new( + AgentCommunicationKind::Spawn, + root.thread_id, + ) + .with_native_key("accepted-spawn-call".to_string()); + let accepted_spawn = control + .spawn_agent_with_communication( + config.clone(), + accepted_communication.clone(), + accepted_context.clone(), + accepted_spawn_source.clone(), + SpawnAgentOptions::default(), + ) + .await + .expect("compatible tracked spawn should succeed"); + let accepted_work = control + .mailbox_work_for_key(root.thread_id, "accepted-spawn-call") + .await + .unwrap() + .expect("accepted spawn work"); + let accepted_thread = manager + .get_thread(accepted_spawn.thread_id) + .await + .expect("accepted child remains loaded"); + let queued_before_retry = accepted_thread + .session + .input_queue + .snapshot_mailbox_communications() + .await; + let retry = control + .spawn_agent_with_communication( + disabled_config.clone(), + accepted_communication, + accepted_context, + accepted_spawn_source, + SpawnAgentOptions::default(), + ) + .await + .expect("incompatible retry must preserve the original child identity"); + assert_eq!(retry.thread_id, accepted_spawn.thread_id); + assert_eq!(accepted_work.agent, accepted_spawn.thread_id); + assert_eq!( + control + .mailbox_work_for_key(root.thread_id, "accepted-spawn-call") + .await + .unwrap(), + Some(accepted_work.clone()) + ); + assert_eq!( + accepted_thread + .session + .input_queue + .snapshot_mailbox_communications() + .await, + queued_before_retry + ); + assert_eq!( + control + .mailbox() + .unwrap() + .dispatch_state(&accepted_work) + .await + .unwrap(), + DispatchState::RecoveryNeeded + ); + + let disabled_child_source = Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: root.thread_id, + depth: 1, + agent_path: Some(AgentPath::root().join("disabled_recipient").unwrap()), + agent_nickname: None, + agent_role: None, + })); + let disabled_child = control + .spawn_agent_with_communication( + disabled_config, + test_communication("stock child initialization", /*trigger_turn*/ false), + AgentCommunicationContext::new(AgentCommunicationKind::Message, root.thread_id), + disabled_child_source, + SpawnAgentOptions::default(), + ) + .await + .expect("stock V2 child spawn should remain available"); + let disabled_thread = manager + .get_thread(disabled_child.thread_id) + .await + .expect("disabled recipient is loaded"); + let queued_before = disabled_thread + .session + .input_queue + .snapshot_mailbox_communications() + .await; + let delivery = control + .prepare_v2_agent_delivery(disabled_child.thread_id) + .await + .expect("prepare loaded disabled recipient"); + let rejected_followup = delivery + .send( + test_communication("incompatible followup", /*trigger_turn*/ false), + AgentCommunicationContext::new(AgentCommunicationKind::Followup, root.thread_id) + .with_native_key("disabled-followup-call".to_string()), + /*interrupt*/ false, + ) + .await; + assert!(rejected_followup.is_err()); + assert_eq!( + disabled_thread + .session + .input_queue + .snapshot_mailbox_communications() + .await, + queued_before + ); + assert!( + control + .mailbox_work_for_key(root.thread_id, "disabled-followup-call") + .await + .unwrap() + .is_none() + ); +} + #[tokio::test(start_paused = true)] async fn terminal_idle_unload_rearms_after_accepted_send_input_invalidates_deadline() { let (_home, _config, manager, control, first, metadata) = terminal_idle_test_agent( diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index d8079c8c5273..dae14f2128f5 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -411,7 +411,7 @@ impl AgentControl { { // A repeated native invocation retains its original child, including when the // original tool publication was lost. Never provision a replacement for a retry. - mailbox + let work = mailbox .accept_work(codex_agent_mailbox::NativeWork { native_key: native_key.to_string(), issuer: context.sender_thread_id, @@ -423,6 +423,28 @@ impl AgentControl { .map_err(|err| { CodexErr::Fatal(format!("agent mailbox acceptance failed: {err}")) })?; + let recipient_compatible = if !Self::mailbox_config_compatible(&config) { + false + } else { + self.mailbox_recipient_config_compatible(work.agent, &config) + .await + .map_err(|error| { + CodexErr::UnsupportedOperation(format!( + "accepted work {} retains original child {}; recipient compatibility is unknown: {error}; its original input remains retained and no replacement child or new dispatch was attempted", + work.id, work.agent + )) + })? + }; + if !recipient_compatible { + self.mailbox_mark_recipient_incompatible(&work) + .await + .map_err(|error| { + CodexErr::Fatal(format!( + "accepted work {} retains original child {}; incompatible-recipient recovery could not be persisted: {error}; its original input remains retained and no replacement child or new dispatch was attempted", + work.id, work.agent + )) + })?; + } let metadata = self.get_agent_metadata(work.agent).ok_or_else(|| { CodexErr::UnsupportedOperation(format!( "accepted work {} retains original child {}; native recovery is required and no replacement was spawned", @@ -435,6 +457,20 @@ impl AgentControl { status: self.get_status(work.agent).await, }); } + if context.native_key.is_some() && !Self::mailbox_config_compatible(&config) { + let role = session_source + .as_ref() + .and_then(SessionSource::get_agent_role); + let recipient = session_source + .as_ref() + .and_then(SessionSource::get_agent_path) + .map(ToString::to_string) + .unwrap_or_else(|| format!("new child of {}", context.sender_thread_id)); + return Err(Self::mailbox_config_incompatibility_error( + &recipient, + role.as_deref(), + )); + } Box::pin(self.spawn_agent_internal( config, SpawnInitialInput::InterAgentCommunication(communication, context), diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 2f82524ef48d..2892a78cc7ca 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -627,6 +627,14 @@ fn spawn_agent_output_schema_v2(hide_agent_metadata: bool) -> Value { "description": "Immutable accepted work reference assigned to the spawned agent." }), ), + ( + "dispatch_state".to_string(), + json!({ + "type": "string", + "enum": ["accepted", "dispatching", "dispatched", "recovery_needed"], + "description": "Native dispatch custody for the original work. Uncertain or incompatible dispatch is retained, not resent." + }), + ), ( "requested_model".to_string(), json!({ diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index 9450aa44bf9a..0fcd780b1d03 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -368,6 +368,10 @@ fn spawn_agent_tool_hides_model_controls_without_override_exposure() { .as_object() .expect("spawn result properties"); assert_eq!(output_properties["work_ref"]["type"], json!("string")); + assert_eq!( + output_properties["dispatch_state"]["enum"], + json!(["accepted", "dispatching", "dispatched", "recovery_needed"]) + ); assert!( output_properties["work_ref"]["description"] .as_str() diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs index 6f37e616d40b..82d73ef905d2 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/message_tool.rs @@ -10,6 +10,7 @@ use crate::agent_communication::AgentCommunicationKind; use crate::tools::context::FunctionToolOutput; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::user_input::UserInput; use futures::future::BoxFuture; use serde::Serialize; @@ -250,10 +251,11 @@ async fn handle_message_submission_inner( .. } = invocation; let _ = payload; - let mailbox_enabled = turn - .config - .features - .enabled(codex_features::Feature::AgentMailbox); + let mailbox_enabled = turn.multi_agent_version == MultiAgentVersion::V2 + && turn + .config + .features + .enabled(codex_features::Feature::AgentMailbox); let receiver_thread_id = resolve_agent_target(&session, &turn, &target).await?; let receiver_agent = session .services @@ -273,14 +275,41 @@ async fn handle_message_submission_inner( let receiver_agent_path = receiver_agent.agent_path.clone().ok_or_else(|| { FunctionCallError::RespondToModel("target agent is missing an agent_path".to_string()) })?; - if mailbox_enabled - && mode == MessageDeliveryMode::TriggerTurn - && let Some(work) = session + let existing_work = if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn { + session .services .agent_control .mailbox_work_for_key(session.thread_id, &call_id) .await .map_err(|err| collab_agent_error(receiver_thread_id, err))? + } else { + None + }; + // Resolve compatibility without reloading a cold recipient: reloading may restore + // already queued native input, so a rejected new operation must not use it as a probe. + let recipient_compatible = if mailbox_enabled && mode == MessageDeliveryMode::TriggerTurn { + let resume_config = build_agent_resume_config(turn.as_ref())?; + Some( + session + .services + .agent_control + .mailbox_recipient_config_compatible(receiver_thread_id, &resume_config) + .await + .map_err(|err| { + if let Some(work) = existing_work.as_ref() { + FunctionCallError::RespondToModel(format!( + "accepted work {} for recipient {} is blocked pending native recovery: {err}; original custody is retained and no input was resent", + work.id, work.agent + )) + } else { + collab_agent_error(receiver_thread_id, err) + } + })?, + ) + } else { + None + }; + if let Some(work) = existing_work && let Some(mailbox) = session.services.agent_control.mailbox() { if work.agent != receiver_thread_id { @@ -304,6 +333,15 @@ async fn handle_message_submission_inner( }) .await .map_err(super::mailbox_message::mailbox_error)?; + if recipient_compatible == Some(false) { + mailbox + .mark_recovery_needed( + &work, + codex_agent_mailbox::RecoveryEvidence::RecipientIncompatible, + ) + .await + .map_err(super::mailbox_message::mailbox_error)?; + } let dispatch_state = mailbox .dispatch_state(&work) .await @@ -339,6 +377,11 @@ async fn handle_message_submission_inner( )); } } + if recipient_compatible == Some(false) { + return Err(FunctionCallError::RespondToModel(format!( + "follow-up task was not accepted: recipient {receiver_agent_path} requires MultiAgentV2 and agent_mailbox to be enabled", + ))); + } let delivery = match mode { MessageDeliveryMode::QueueOnly => { session diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index 8f714558cf20..afd2b5265446 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -10,6 +10,7 @@ use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions; use crate::tools::handlers::multi_agents_spec::create_spawn_agent_tool_v2; use crate::tools::handlers::multi_agents_v2::message_tool::message_content; use codex_protocol::AgentPath; +use codex_protocol::protocol::MultiAgentVersion; use codex_tools::ToolSpec; #[derive(Default)] @@ -118,10 +119,11 @@ async fn handle_spawn_agent( .get_agent_path() .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, new_agent_path.clone(), message); - let mailbox_enabled = turn - .config - .features - .enabled(codex_features::Feature::AgentMailbox); + let mailbox_enabled = turn.multi_agent_version == MultiAgentVersion::V2 + && turn + .config + .features + .enabled(codex_features::Feature::AgentMailbox); let mut context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); if mailbox_enabled { @@ -148,17 +150,29 @@ async fn handle_spawn_agent( .await .map_err(collab_spawn_error)?; let new_thread_id = spawned_agent.thread_id; - let work_ref = if mailbox_enabled { + let accepted_work = if mailbox_enabled { session .services .agent_control .mailbox_work_for_key(session.thread_id, &call_id) .await .map_err(collab_spawn_error)? - .map(|work| work.id) } else { None }; + let dispatch_state = if let Some(work) = accepted_work.as_ref() + && let Some(mailbox) = session.services.agent_control.mailbox() + { + Some( + mailbox + .dispatch_state(work) + .await + .map_err(super::mailbox_message::mailbox_error)?, + ) + } else { + None + }; + let work_ref = accepted_work.map(|work| work.id); let agent_snapshot = session .services .agent_control @@ -204,6 +218,7 @@ async fn handle_spawn_agent( Ok(SpawnAgentResult { agent_id: None, work_ref, + dispatch_state, task_name, nickname: None, requested_model: args.model.clone(), @@ -220,6 +235,7 @@ async fn handle_spawn_agent( Ok(SpawnAgentResult { agent_id: Some(new_thread_id.to_string()), work_ref, + dispatch_state, task_name, nickname, requested_model: args.model.clone(), @@ -369,6 +385,8 @@ pub(crate) struct SpawnAgentResult { agent_id: Option, #[serde(skip_serializing_if = "Option::is_none")] work_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dispatch_state: Option, task_name: String, #[serde(skip_serializing_if = "Option::is_none")] nickname: Option, diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index dc8c3169bb17..4e55f08178d4 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1198,6 +1198,20 @@ mod tests { .features .enable(Feature::AgentMailbox) .expect("test config should enable AgentMailbox"); + let role_home = tempfile::tempdir()?; + let disabled_role_path = role_home.path().join("mailbox_disabled.toml"); + tokio::fs::write( + &disabled_role_path, + "[features]\nmulti_agent_v2 = true\nagent_mailbox = false\n", + ) + .await?; + config.agent_roles.insert( + "mailbox_disabled".to_string(), + crate::config::AgentRoleConfig { + config_file: Some(disabled_role_path), + ..Default::default() + }, + ); config.multi_agent_v2.hide_spawn_agent_metadata = false; config.multi_agent_v2.tool_namespace = Some("profile_agents".to_string()); turn_context.multi_agent_version = config.multi_agent_version_from_features(); @@ -1243,6 +1257,45 @@ mod tests { )); let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); let runtime = ToolCallRuntime::new(router, Arc::clone(&session), step_context, tracker); + let thread_ids_before_rejection = manager.list_thread_ids().await; + let rejected_response = runtime + .handle_tool_call( + ToolCall { + tool_name: ToolName::namespaced("profile_agents", "spawn_agent"), + call_id: "disabled-role".to_string(), + payload: ToolPayload::Function { + arguments: serde_json::json!({ + "message": "must not be submitted", + "task_name": "disabled_worker", + "agent_type": "mailbox_disabled", + "fork_turns": "none" + }) + .to_string(), + }, + }, + CancellationToken::new(), + ) + .await?; + let ResponseInputItem::FunctionCallOutput { output, .. } = rejected_response else { + anyhow::bail!("incompatible role rejection should return a function output"); + }; + assert_eq!(output.success, Some(false)); + let FunctionCallOutputBody::Text(rejection) = output.body else { + anyhow::bail!("incompatible role rejection should be text"); + }; + assert!(rejection.contains("agent_mailbox"), "{rejection}"); + assert!(rejection.contains("mailbox_disabled"), "{rejection}"); + assert_eq!(manager.list_thread_ids().await, thread_ids_before_rejection); + assert!( + session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, "disabled-role") + .await? + .is_none(), + "a rejected resolved role must not accept tracked native work" + ); + assert!(created_threads.try_recv().is_err()); let spawn_call = || ToolCall { tool_name: ToolName::namespaced("profile_agents", "spawn_agent"), call_id: "call-1".to_string(), @@ -1394,6 +1447,7 @@ mod tests { Some(accepted_work_id.as_str()), "the tool result should publish the exact accepted native work reference" ); + assert_eq!(first_result["dispatch_state"], "dispatched"); assert!( session .services diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 43045cd504f2..91146a7f2232 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -377,6 +377,10 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { let tool = namespace_function(plan, namespace_name, tool_name); let output = tool.output_schema.as_ref().expect("agent result schema"); assert_eq!(output["properties"]["work_ref"]["type"], json!("string")); + assert_eq!( + output["properties"]["dispatch_state"]["enum"], + json!(["accepted", "dispatching", "dispatched", "recovery_needed"]) + ); assert!( !output["required"] .as_array() diff --git a/docs/carry-divergence-ledger.md b/docs/carry-divergence-ledger.md index f82005f20107..99c19b119566 100644 --- a/docs/carry-divergence-ledger.md +++ b/docs/carry-divergence-ledger.md @@ -84,6 +84,13 @@ can settle through its loaded parent's same-tree store, otherwise handback retains retry custody. Native finalizer/turn release emits an advisory rescan so a producer failure before input admission cannot leave an event-only wait asleep without an outcome or a producer able to advance it. +New tracked spawn/followup operations require both the caller and the resolved +recipient role to enable V2 and `agent_mailbox`. An explicitly incompatible role +or recipient is rejected before child creation, work acceptance, or submission; +it is never silently enabled or downgraded to native-only durability. If the +native invocation was already accepted before the recipient became incompatible, +its original work reference and input remain in custody with `recovery_needed` +dispatch state rather than being resent or replaced. Both `agent_mailbox` and recipient `agent_mailbox_idle_activation` are off by default. Disabled mailbox handling preserves pending records and cannot drain, From f8f78bd2132c4583266cd7e69a03bd1572776522 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:55:56 +1100 Subject: [PATCH 13/54] fix(mailbox): resolve accepted spawn retries before role reload --- .../tools/handlers/multi_agents_v2/spawn.rs | 97 +++++++++++-------- codex-rs/core/src/tools/parallel.rs | 17 ++++ 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index afd2b5265446..84454d198904 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -60,47 +60,65 @@ async fn handle_spawn_agent( let message = message_content(args.message)?; let session_source = turn.session_source.clone(); let child_depth = next_thread_spawn_depth(&session_source); + let mailbox_enabled = turn.multi_agent_version == MultiAgentVersion::V2 + && turn + .config + .features + .enabled(codex_features::Feature::AgentMailbox); + let accepted_retry = mailbox_enabled + && session + .services + .agent_control + .mailbox_work_for_key(session.thread_id, &call_id) + .await + .map_err(collab_spawn_error)? + .is_some(); let mut config = build_agent_spawn_config(&session.get_base_instructions().await, turn.as_ref())?; - if let Some(service_tier) = args.service_tier.as_ref() { - config.service_tier = Some(service_tier.clone()); - } - let is_full_history_fork = matches!(fork_mode, Some(SpawnAgentForkMode::FullHistory)); - apply_requested_spawn_agent_model_overrides( - &session, - turn.as_ref(), - &mut config, - args.model.as_deref(), - args.reasoning_effort.clone(), - role_name, - ) - .await?; - if !is_full_history_fork || role_name.is_some() { - apply_spawn_agent_role(&session, &mut config, role_name).await?; - if is_full_history_fork && config.developer_instructions.is_none() { - config - .developer_instructions - .clone_from(&turn.developer_instructions); + // An accepted invocation belongs to its retained child, even if the requested role + // is now missing or cannot be loaded. The native retry path validates its original + // input and recipient configuration without provisioning or submitting again. + if !accepted_retry { + if let Some(service_tier) = args.service_tier.as_ref() { + config.service_tier = Some(service_tier.clone()); + } + let is_full_history_fork = matches!(fork_mode, Some(SpawnAgentForkMode::FullHistory)); + apply_requested_spawn_agent_model_overrides( + &session, + turn.as_ref(), + &mut config, + args.model.as_deref(), + args.reasoning_effort.clone(), + role_name, + ) + .await?; + if !is_full_history_fork || role_name.is_some() { + apply_spawn_agent_role(&session, &mut config, role_name).await?; + if is_full_history_fork && config.developer_instructions.is_none() { + config + .developer_instructions + .clone_from(&turn.developer_instructions); + } } + apply_spawn_agent_service_tier( + &session, + &mut config, + turn.config.service_tier.as_deref(), + args.service_tier.as_deref(), + ) + .await?; + apply_spawn_agent_runtime_overrides(&mut config, turn.as_ref())?; + validate_spawn_agent_expected_model( + &config, + args.model.as_deref(), + args.expected_model.as_deref(), + )?; + validate_spawn_agent_expected_reasoning_effort( + &config, + args.reasoning_effort.as_ref(), + args.expected_reasoning_effort.as_ref(), + )?; } - apply_spawn_agent_service_tier( - &session, - &mut config, - turn.config.service_tier.as_deref(), - args.service_tier.as_deref(), - ) - .await?; - apply_spawn_agent_runtime_overrides(&mut config, turn.as_ref())?; - validate_spawn_agent_expected_model( - &config, - args.model.as_deref(), - args.expected_model.as_deref(), - )?; - validate_spawn_agent_expected_reasoning_effort( - &config, - args.reasoning_effort.as_ref(), - args.expected_reasoning_effort.as_ref(), - )?; let spawn_source = thread_spawn_source( session.thread_id, @@ -119,11 +137,6 @@ async fn handle_spawn_agent( .get_agent_path() .unwrap_or_else(AgentPath::root); let communication = communication_from_tool_message(author, new_agent_path.clone(), message); - let mailbox_enabled = turn.multi_agent_version == MultiAgentVersion::V2 - && turn - .config - .features - .enabled(codex_features::Feature::AgentMailbox); let mut context = AgentCommunicationContext::new(AgentCommunicationKind::Spawn, session.thread_id); if mailbox_enabled { diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 4e55f08178d4..87aa5b89b3d2 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1212,6 +1212,19 @@ mod tests { ..Default::default() }, ); + let enabled_role_path = role_home.path().join("mailbox_enabled.toml"); + tokio::fs::write( + &enabled_role_path, + "[features]\nmulti_agent_v2 = true\nagent_mailbox = true\n", + ) + .await?; + config.agent_roles.insert( + "mailbox_enabled".to_string(), + crate::config::AgentRoleConfig { + config_file: Some(enabled_role_path.clone()), + ..Default::default() + }, + ); config.multi_agent_v2.hide_spawn_agent_metadata = false; config.multi_agent_v2.tool_namespace = Some("profile_agents".to_string()); turn_context.multi_agent_version = config.multi_agent_version_from_features(); @@ -1303,6 +1316,7 @@ mod tests { arguments: serde_json::json!({ "message": "delivery owns this child", "task_name": "delivery_owned_worker", + "agent_type": "mailbox_enabled", "fork_turns": "none" }) .to_string(), @@ -1470,6 +1484,8 @@ mod tests { // A same-call retry models a tool result lost after acceptance/publication. It must // resolve the retained work, not provision another child or send the input again. + // Resolving the old request must also survive its role file becoming unavailable. + tokio::fs::remove_file(&enabled_role_path).await?; let retry_response = tokio::time::timeout( Duration::from_secs(5), runtime.handle_tool_call(spawn_call(), CancellationToken::new()), @@ -1485,6 +1501,7 @@ mod tests { }; let retry_result: serde_json::Value = serde_json::from_str(&retry_message)?; assert_eq!(retry_result["agent_id"].as_str(), Some(first_agent_id)); + assert_eq!(retry_result["dispatch_state"], first_result["dispatch_state"]); assert_eq!( retry_result["work_ref"].as_str(), Some(accepted_work_id.as_str()), From 97adde10b295df7a28e719af1c1631713ae5b579 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:55:56 +1100 Subject: [PATCH 14/54] ci: keep fork Bazel execution on standard hosted runners --- .../scripts/test_run_bazel_with_buildbuddy.py | 27 +++++++++++++++++++ .../validation-lanes/workflow-ci-sanity.sh | 1 + .github/workflows/bazel.yml | 11 ++++---- .github/workflows/rust-ci-full.yml | 5 ++-- .github/workflows/v8-canary.yml | 5 ++-- justfile | 1 + 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 1351df4e9f64..b6ff218d05c0 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -48,6 +48,33 @@ def test_keyless_invocation_drops_remote_ci_configuration(self) -> None: ["build", "--", "//codex-rs/cli:codex"], ) + def test_hosted_only_fork_callers_withhold_optional_remote_credentials(self) -> None: + workflow_root = Path(__file__).resolve().parents[1] / "workflows" + expected = ( + "BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' " + "&& secrets.BUILDBUDDY_API_KEY || '' }}" + ) + for workflow in ("bazel.yml", "rust-ci-full.yml", "v8-canary.yml"): + with self.subTest(workflow=workflow): + source = (workflow_root / workflow).read_text(encoding="utf-8") + assignments = [ + line.strip() + for line in source.splitlines() + if line.strip().startswith("BUILDBUDDY_API_KEY:") + ] + self.assertTrue(assignments) + self.assertTrue(all(line == expected for line in assignments)) + + def test_keyless_hosted_fallback_preserves_every_remote_ci_target(self) -> None: + for config in run_bazel_with_buildbuddy.REMOTE_EXECUTION_CONFIGS: + with self.subTest(config=config): + self.assertEqual( + run_bazel_with_buildbuddy.bazel_args_with_remote_config( + ["test", config, "--", "//codex-rs/cli:codex", "//..."], {} + ), + ["test", "--", "//codex-rs/cli:codex", "//..."], + ) + def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> None: args = ["run", "//codex-rs/cli:codex", "--", "--config=remote"] diff --git a/.github/scripts/validation-lanes/workflow-ci-sanity.sh b/.github/scripts/validation-lanes/workflow-ci-sanity.sh index bf53d5576a3c..4312f0e67970 100755 --- a/.github/scripts/validation-lanes/workflow-ci-sanity.sh +++ b/.github/scripts/validation-lanes/workflow-ci-sanity.sh @@ -23,6 +23,7 @@ python3 -m py_compile \ python3 -m unittest discover -s .github/scripts -p 'test_ci_planners.py' python3 -m unittest discover -s .github/scripts -p 'test_validate.py' python3 -m unittest discover -s .github/scripts -p 'test_run_named_rust_tests.py' +python3 -m unittest discover -s .github/scripts -p 'test_run_bazel_with_buildbuddy.py' python3 .codex/skills/babysit-gh-workflow-run/tests/test_gh_workflow_run_watch.py python3 .github/scripts/check_workflow_policy.py ruby -e 'require "yaml"; paths = Dir.glob([".github/workflows/*.{yml,yaml}", "codex-rs/.github/workflows/*.{yml,yaml}"]).sort; paths.each { |path| YAML.load_file(path) }; puts "yaml-ok #{paths.length}"' diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1a2ee0b9d59c..6baf9ca884c0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -228,7 +228,8 @@ jobs: - name: bazel test //... env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + # This fork uses hosted-local Bazel; preserve upstream's remote opt-in elsewhere. + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | bazel_targets=( @@ -326,7 +327,7 @@ jobs: env: BAZEL_TEST_SHARD: ${{ matrix.shard }} BAZEL_TEST_SHARD_COUNT: 4 - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | set -euo pipefail @@ -449,7 +450,7 @@ jobs: - name: bazel build --config=clippy lint targets env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | bazel_clippy_args=( @@ -554,7 +555,7 @@ jobs: - name: bazel build verify-release-build targets env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | # This job exists to compile Rust code behind @@ -604,7 +605,7 @@ jobs: - name: Verify Bazel builds bwrap if: runner.os == 'Linux' env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | ./.github/scripts/run-bazel-ci.sh \ diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index 6f287f0646d3..4e23c4cd9465 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -306,7 +306,8 @@ jobs: - name: Run argument comment lint on codex-rs via Bazel if: ${{ runner.os != 'Windows' }} env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + # This fork uses hosted-local Bazel; preserve upstream's remote opt-in elsewhere. + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | bazel_extra_args=() @@ -329,7 +330,7 @@ jobs: - name: Run argument comment lint on codex-rs via Bazel if: ${{ runner.os == 'Windows' }} env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | ./.github/scripts/run-argument-comment-lint-bazel.sh \ diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index fb3948522a3b..1d373b34b0e2 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -193,7 +193,8 @@ jobs: - name: Build Bazel V8 release pair env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + # This fork uses hosted-local Bazel; preserve upstream's remote opt-in elsewhere. + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} PLATFORM: ${{ matrix.platform }} SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} @@ -229,7 +230,7 @@ jobs: - name: Stage release pair env: BAZEL_CONFIG: ${{ matrix.bazel_config }} - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} PLATFORM: ${{ matrix.platform }} SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} diff --git a/justfile b/justfile index 2c2d5ea1fc47..57f3034e555b 100644 --- a/justfile +++ b/justfile @@ -761,6 +761,7 @@ downstream-docs-check: workflow-ci-sanity: cd "{{ justfile_directory() }}" && python3 -m py_compile .github/scripts/aggregate_validation_summary.py .github/scripts/check_markdown_links.py .github/scripts/resolve_rust_ci_mode.py .github/scripts/resolve_sedna_release_version.py .github/scripts/resolve_validation_plan.py .github/scripts/test_ci_planners.py scripts/downstream-divergence-audit.py cd "{{ justfile_directory() }}" && python3 -m unittest discover -s .github/scripts -p 'test_ci_planners.py' + cd "{{ justfile_directory() }}" && python3 -m unittest discover -s .github/scripts -p 'test_run_bazel_with_buildbuddy.py' cd "{{ justfile_directory() }}" && ruby -e 'require "yaml"; %w[.github/workflows/_sedna-linux-rust.yml .github/workflows/codeql.yml .github/workflows/docs-sanity.yml .github/workflows/rust-ci-full.yml .github/workflows/rust-ci.yml .github/workflows/sedna-heavy-tests.yml .github/workflows/sedna-release.yml .github/workflows/validation-lab.yml].each { |path| YAML.load_file(path) }; puts "yaml-ok"' [no-cd] From 3ca85f3c3c272fdac1b4c490ffd48e91f0c4e8b9 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:00:37 +1100 Subject: [PATCH 15/54] fix(ci): emit canonical single-plus preview versions --- .github/scripts/test_ci_planners.py | 57 ++++++++++++++++++++++++ .github/workflows/sedna-branch-build.yml | 7 ++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 35cfbf29729e..182766f58d5a 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -4872,6 +4872,63 @@ def test_sedna_branch_build_uses_safe_ref_env_and_cached_macos_x64_preview( self.assertIn('"notarized": False', macos_stage_script) self.assertNotIn("${{ needs.metadata.outputs.display_ref }}", macos_stage_script) + def test_sedna_branch_preview_version_preserves_one_metadata_separator(self) -> None: + metadata_step = workflow_step_by_name( + REPO_ROOT / ".github/workflows/sedna-branch-build.yml", + "metadata", + "Compute preview version", + ) + script = metadata_step.get("run") or "" + git_fixture = """ +git() { + case "$1" in + rev-parse) printf '%s\\n' deadbeef ;; + tag) printf '%s\\n' "${PREVIEW_TEST_TAG}" ;; + *) return 2 ;; + esac +} +""" + cases = ( + ("", "0.0.0-sedna.0+ci.42.gdeadbeef"), + ("v1.2.3-sedna.4", "1.2.3-sedna.4+ci.42.gdeadbeef"), + ( + "v0.146.0-alpha.8-sedna.10", + "0.146.0-alpha.8-sedna.10+ci.42.gdeadbeef", + ), + ( + "v0.146.0-alpha.8-sedna.10+upstream.8593", + "0.146.0-alpha.8-sedna.10+upstream.8593.ci.42.gdeadbeef", + ), + ) + commit = "deadbeef" + "0" * 32 + for tag, expected in cases: + with self.subTest(tag=tag), tempfile.TemporaryDirectory() as tmpdir: + output_path = Path(tmpdir) / "github-output" + proc = subprocess.run( + ["bash", "-c", git_fixture + script], + check=False, + capture_output=True, + text=True, + env={ + **os.environ, + "PREVIEW_TEST_TAG": tag, + "GITHUB_OUTPUT": str(output_path), + "GITHUB_RUN_NUMBER": "42", + "CHECKOUT_REF": commit, + "DISPLAY_REF": "feature/preview", + "HOST_ONLY": "false", + "PLATFORM": "linux-x86_64", + }, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + outputs = parse_github_output_file(output_path) + self.assertEqual(outputs["preview_version"], expected) + self.assertEqual(outputs["preview_version"].count("+"), 1) + self.assertEqual(outputs["checkout_ref"], commit) + self.assertEqual( + outputs["artifact_name"], "sedna-branch-feature-preview-deadbeef" + ) + def test_sedna_branch_build_host_only_command_has_valid_shell_syntax(self) -> None: workflow_path = REPO_ROOT / ".github/workflows/sedna-branch-build.yml" payload = load_workflow_payload(workflow_path) diff --git a/.github/workflows/sedna-branch-build.yml b/.github/workflows/sedna-branch-build.yml index 4d87bb002317..6552c2f7e47c 100644 --- a/.github/workflows/sedna-branch-build.yml +++ b/.github/workflows/sedna-branch-build.yml @@ -72,7 +72,12 @@ jobs: branch_name="${DISPLAY_REF}" branch_slug="$(printf '%s' "${branch_name}" | tr '/:@ ' '-' | tr -cd '[:alnum:]._-' | cut -c1-48)" - preview_version="${base_version}-ci.${GITHUB_RUN_NUMBER}+g${short_sha}" + preview_suffix="ci.${GITHUB_RUN_NUMBER}.g${short_sha}" + if [[ "${base_version}" == *+* ]]; then + preview_version="${base_version}.${preview_suffix}" + else + preview_version="${base_version}+${preview_suffix}" + fi { echo "checkout_ref=${checkout_ref}" From d64f96f3c1db446f8bdb09c151943bf20195b68e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:00:37 +1100 Subject: [PATCH 16/54] fix(mailbox): correct owned path conversion and apply hosted formatting --- .../workflows/mailbox-source-preparation.yml | 2 +- codex-rs/agent-mailbox/src/mailbox_tests.rs | 24 +- codex-rs/core/src/agent/control/lifecycle.rs | 12 +- .../core/src/agent/control/mailbox_tests.rs | 239 ++++++++++++++---- .../core/src/agent/control/residency_tests.rs | 17 +- codex-rs/core/src/agent/control/spawn.rs | 2 +- codex-rs/core/src/session/input_queue.rs | 8 +- codex-rs/core/src/session/mod.rs | 19 +- codex-rs/core/src/session/tests.rs | 21 +- codex-rs/core/src/thread_manager.rs | 6 +- codex-rs/core/src/thread_manager_tests.rs | 90 ++++--- .../tools/handlers/multi_agents_spec_tests.rs | 8 +- codex-rs/core/src/tools/spec_plan_tests.rs | 8 +- codex-rs/core/tests/suite/agent_mailbox.rs | 120 ++++++--- 14 files changed, 409 insertions(+), 167 deletions(-) diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml index af6c2add62bc..36129cb0914e 100644 --- a/.github/workflows/mailbox-source-preparation.yml +++ b/.github/workflows/mailbox-source-preparation.yml @@ -15,7 +15,7 @@ jobs: env: CARGO_INCREMENTAL: "0" RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "9" + PREPARATION_ATTEMPT: "10" SCCACHE_CACHE_SIZE: 2G steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 108da654d18d..1c79f4567306 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1804,11 +1804,13 @@ async fn recipient_incompatibility_recovers_accepted_work_without_losing_or_rese ) .await; - assert!(fixture - .mailbox - .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) - .await - .unwrap()); + assert!( + fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .unwrap() + ); assert_eq!( fixture.mailbox.dispatch_state(&work).await.unwrap(), DispatchState::RecoveryNeeded @@ -1817,11 +1819,13 @@ async fn recipient_incompatibility_recovers_accepted_work_without_losing_or_rese fixture.mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::RecoveryNeeded ); - assert!(!fixture - .mailbox - .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) - .await - .unwrap()); + assert!( + !fixture + .mailbox + .mark_recovery_needed(&work, RecoveryEvidence::RecipientIncompatible) + .await + .unwrap() + ); assert_eq!( fixture diff --git a/codex-rs/core/src/agent/control/lifecycle.rs b/codex-rs/core/src/agent/control/lifecycle.rs index 8975f639c845..5f7144de7bd2 100644 --- a/codex-rs/core/src/agent/control/lifecycle.rs +++ b/codex-rs/core/src/agent/control/lifecycle.rs @@ -233,12 +233,12 @@ impl PreparedV2AgentDelivery { let accepted = if let Some(recipient_config) = recipient_config { self.control .mailbox_accept_native_submission( - self.agent_id, - context, - communication, - &mut submission_id, - recipient_config, - ) + self.agent_id, + context, + communication, + &mut submission_id, + recipient_config, + ) .await? } else if context.native_key.is_some() { return Err(CodexErr::UnsupportedOperation(format!( diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 8b3a8fcf1e3e..6b51ee97d746 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -95,34 +95,46 @@ fn goal_snapshot(issuer: ThreadId, agent: ThreadId) -> GoalNotificationSnapshot } async fn accepted_goal_work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId) -> WorkRef { - let work = mailbox.accept_work(NativeWork { - native_key: "goal-call".to_string(), - issuer, - agent, - submission_id: "goal-submission".to_string(), - input: InterAgentCommunication::new( - AgentPath::root(), - AgentPath::from_string("/root/worker".to_string()).unwrap(), - Vec::new(), - "complete the accepted work".to_string(), - /*trigger_turn*/ true, - ), - }).await.unwrap(); - assert_eq!(mailbox.begin_dispatch(&work).await.unwrap(), DispatchDecision::Dispatch); + let work = mailbox + .accept_work(NativeWork { + native_key: "goal-call".to_string(), + issuer, + agent, + submission_id: "goal-submission".to_string(), + input: InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + "complete the accepted work".to_string(), + /*trigger_turn*/ true, + ), + }) + .await + .unwrap(); + assert_eq!( + mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::Dispatch + ); mailbox.mark_dispatched(&work).await.unwrap(); let input = mailbox.accepted_input(&work).await.unwrap(); - mailbox.record_admission(agent, "accepted-turn", &[input.id.unwrap().to_string()]).await.unwrap(); + mailbox + .record_admission(agent, "accepted-turn", &[input.id.unwrap().to_string()]) + .await + .unwrap(); work } async fn exact_work_snapshot(mailbox: &Mailbox, work: &WorkRef) -> WorkSnapshot { - let snapshot = mailbox.snapshot(&Selection { - caller: work.issuer, - agent: Some(work.agent), - selectors: vec![Selector::Work(work.id.clone())], - return_when: ReturnWhen::All, - limit: 1, - }).await.unwrap(); + let snapshot = mailbox + .snapshot(&Selection { + caller: work.issuer, + agent: Some(work.agent), + selectors: vec![Selector::Work(work.id.clone())], + return_when: ReturnWhen::All, + limit: 1, + }) + .await + .unwrap(); match snapshot.items.into_iter().next().unwrap() { SnapshotItem::Work(work) => work, SnapshotItem::Request(_) => panic!("exact work selector returned a request"), @@ -145,10 +157,32 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a let work = accepted_goal_work(&mailbox, session.thread_id, agent).await; let mut goal = goal_snapshot(session.thread_id, agent); - assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + assert!( + control + .mailbox_record_terminal( + agent, + "accepted-turn", + AgentStatus::Completed(None), + Some((incarnation, goal.clone())), + None + ) + .await + .unwrap() + ); goal.phase = GoalNotificationPhase::DeferredWithOwner; goal.revision += 1; - assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + assert!( + control + .mailbox_record_terminal( + agent, + "accepted-turn", + AgentStatus::Completed(None), + Some((incarnation, goal.clone())), + None + ) + .await + .unwrap() + ); let pending = exact_work_snapshot(&mailbox, &work).await; assert!(pending.outcome.is_none()); assert_eq!(pending.continuations.len(), 1); @@ -162,14 +196,50 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a goal.classification = GoalNotificationClassification::Result; // A restarted/replaced native store is not evidence for the old producer, // even when its goal ID, generation and thread IDs happen to be identical. - assert!(!control.mailbox_record_terminal(agent, "continuation-turn", AgentStatus::Completed(None), Some((ThreadId::new(), goal.clone())), None).await.unwrap()); + assert!( + !control + .mailbox_record_terminal( + agent, + "continuation-turn", + AgentStatus::Completed(None), + Some((ThreadId::new(), goal.clone())), + None + ) + .await + .unwrap() + ); assert!(exact_work_snapshot(&mailbox, &work).await.outcome.is_none()); - assert!(control.mailbox_record_terminal(agent, "continuation-turn", AgentStatus::Completed(None), Some((incarnation, goal)), None).await.unwrap()); + assert!( + control + .mailbox_record_terminal( + agent, + "continuation-turn", + AgentStatus::Completed(None), + Some((incarnation, goal)), + None + ) + .await + .unwrap() + ); let completed = exact_work_snapshot(&mailbox, &work).await; assert_eq!(completed.outcome.unwrap().outcome, Outcome::Completed); - assert_eq!(completed.continuations[0].state, ContinuationState::Terminal); + assert_eq!( + completed.continuations[0].state, + ContinuationState::Terminal + ); // An unrelated native result must retain the existing native fallback path. - assert!(!control.mailbox_record_terminal(ThreadId::new(), "uncaptured-turn", AgentStatus::Completed(None), None, None).await.unwrap()); + assert!( + !control + .mailbox_record_terminal( + ThreadId::new(), + "uncaptured-turn", + AgentStatus::Completed(None), + None, + None + ) + .await + .unwrap() + ); } #[tokio::test] @@ -193,25 +263,64 @@ async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_c let work = accepted_goal_work(&mailbox, session.thread_id, agent).await; let mut goal = goal_snapshot(session.thread_id, agent); goal.phase = GoalNotificationPhase::DeferredWithOwner; - assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + assert!( + control + .mailbox_record_terminal( + agent, + "accepted-turn", + AgentStatus::Completed(None), + Some((incarnation, goal.clone())), + None + ) + .await + .unwrap() + ); let pending = exact_work_snapshot(&mailbox, &work).await; // This exact producer has no loaded native thread or continuation owner. - assert!(control.mailbox_reconcile_pending_work(&pending).await.unwrap()); + assert!( + control + .mailbox_reconcile_pending_work(&pending) + .await + .unwrap() + ); let recovered = exact_work_snapshot(&mailbox, &work).await; assert_eq!(recovered.work_ref, work); assert_eq!(recovered.dispatch_state, DispatchState::RecoveryNeeded); assert!(recovered.outcome.is_none()); assert_eq!(recovered.input, pending.input); assert_eq!(recovered.continuations, pending.continuations); - assert!(!control.mailbox_reconcile_pending_work(&recovered).await.unwrap()); + assert!( + !control + .mailbox_reconcile_pending_work(&recovered) + .await + .unwrap() + ); goal.phase = GoalNotificationPhase::ResultReady; goal.status = ThreadGoalStatus::Complete; goal.revision += 1; goal.classification = GoalNotificationClassification::Result; - assert!(control.mailbox_record_terminal(agent, "recovered-turn", AgentStatus::Completed(None), Some((incarnation, goal)), None).await.unwrap()); - assert_eq!(exact_work_snapshot(&mailbox, &work).await.outcome.unwrap().outcome, Outcome::Completed); + assert!( + control + .mailbox_record_terminal( + agent, + "recovered-turn", + AgentStatus::Completed(None), + Some((incarnation, goal)), + None + ) + .await + .unwrap() + ); + assert_eq!( + exact_work_snapshot(&mailbox, &work) + .await + .outcome + .unwrap() + .outcome, + Outcome::Completed + ); } #[tokio::test] @@ -226,8 +335,16 @@ async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { codex_login::CodexAuth::from_api_key("dummy"), config.model_provider.clone(), ); - let parent = manager.start_thread(crate::thread_manager::StartThreadOptions::new(config.clone())).await.unwrap(); - parent.thread.session.set_multi_agent_version_if_unset(MultiAgentVersion::V2); + let parent = manager + .start_thread(crate::thread_manager::StartThreadOptions::new( + config.clone(), + )) + .await + .unwrap(); + parent + .thread + .session + .set_multi_agent_version_if_unset(MultiAgentVersion::V2); let control = &parent.thread.session.services.agent_control; control.mailbox_initialize(&config).await.unwrap(); let mailbox = control.mailbox().unwrap(); @@ -235,26 +352,56 @@ async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { let incarnation = ThreadId::new(); let work = accepted_goal_work(&mailbox, parent.thread_id, agent).await; let mut goal = goal_snapshot(parent.thread_id, agent); - assert!(control.mailbox_record_terminal(agent, "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), None).await.unwrap()); + assert!( + control + .mailbox_record_terminal( + agent, + "accepted-turn", + AgentStatus::Completed(None), + Some((incarnation, goal.clone())), + None + ) + .await + .unwrap() + ); assert!(manager.get_thread(agent).await.is_err()); - assert!(!parent.thread.session.input_queue.has_pending_mailbox_items().await); + assert!( + !parent + .thread + .session + .input_queue + .has_pending_mailbox_items() + .await + ); goal.phase = GoalNotificationPhase::ResultReady; goal.status = ThreadGoalStatus::Complete; goal.revision += 1; goal.classification = GoalNotificationClassification::Result; - assert!(manager.notify_v2_child_completion( - agent, - parent.thread_id, - AgentPath::from_string("/root/worker".to_string()).unwrap(), - "accepted-turn", - AgentStatus::Completed(Some("durable goal result".to_string())), - Some((incarnation, goal)), - ).await.unwrap()); + assert!( + manager + .notify_v2_child_completion( + agent, + parent.thread_id, + AgentPath::from_string("/root/worker".to_string()).unwrap(), + "accepted-turn", + AgentStatus::Completed(Some("durable goal result".to_string())), + Some((incarnation, goal)), + ) + .await + .unwrap() + ); let settled = exact_work_snapshot(&mailbox, &work).await; assert_eq!(settled.work_ref, work); assert_eq!(settled.outcome.unwrap().outcome, Outcome::Completed); assert_eq!(settled.continuations[0].state, ContinuationState::Terminal); assert!(manager.get_thread(agent).await.is_err()); - assert!(!parent.thread.session.input_queue.has_pending_mailbox_items().await); + assert!( + !parent + .thread + .session + .input_queue + .has_pending_mailbox_items() + .await + ); } diff --git a/codex-rs/core/src/agent/control/residency_tests.rs b/codex-rs/core/src/agent/control/residency_tests.rs index f758ac31e3f3..da8238a5634f 100644 --- a/codex-rs/core/src/agent/control/residency_tests.rs +++ b/codex-rs/core/src/agent/control/residency_tests.rs @@ -632,7 +632,10 @@ async fn mailbox_rejects_incompatible_spawn_and_followup_before_capture() { .expect("start mailbox-enabled V2 root"); let control = manager.agent_control(); let mut disabled_config = config.clone(); - disabled_config.features.disable(Feature::AgentMailbox).unwrap(); + disabled_config + .features + .disable(Feature::AgentMailbox) + .unwrap(); let disabled_spawn_source = Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { parent_thread_id: root.thread_id, @@ -652,8 +655,8 @@ async fn mailbox_rejects_incompatible_spawn_and_followup_before_capture() { SpawnAgentOptions::default(), ) .await; - let rejected_spawn_error = rejected_spawn - .expect_err("incompatible tracked spawn must fail before child creation"); + let rejected_spawn_error = + rejected_spawn.expect_err("incompatible tracked spawn must fail before child creation"); assert!(rejected_spawn_error.to_string().contains("disabled_spawn")); assert!( rejected_spawn_error @@ -684,11 +687,9 @@ async fn mailbox_rejects_incompatible_spawn_and_followup_before_capture() { })); let accepted_communication = test_communication("accepted spawn input", /*trigger_turn*/ false); - let accepted_context = AgentCommunicationContext::new( - AgentCommunicationKind::Spawn, - root.thread_id, - ) - .with_native_key("accepted-spawn-call".to_string()); + let accepted_context = + AgentCommunicationContext::new(AgentCommunicationKind::Spawn, root.thread_id) + .with_native_key("accepted-spawn-call".to_string()); let accepted_spawn = control .spawn_agent_with_communication( config.clone(), diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index dae14f2128f5..7d3bc079194d 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -464,7 +464,7 @@ impl AgentControl { let recipient = session_source .as_ref() .and_then(SessionSource::get_agent_path) - .map(ToString::to_string) + .map(|path| path.to_string()) .unwrap_or_else(|| format!("new child of {}", context.sender_thread_id)); return Err(Self::mailbox_config_incompatibility_error( &recipient, diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index 6513e1a8e1c3..1d9aee212b84 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -931,9 +931,11 @@ mod tests { let home = tempfile::tempdir().unwrap(); let mailbox = Arc::new( - Mailbox::open(&codex_state::SqliteConfig::new_for_testing(home.path().abs())) - .await - .unwrap(), + Mailbox::open(&codex_state::SqliteConfig::new_for_testing( + home.path().abs(), + )) + .await + .unwrap(), ); let queue = InputQueue::new(); queue.bind_durable_mailbox(Arc::clone(&mailbox)); diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 7181d14d1b04..1aa2ef5ce43c 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2404,10 +2404,10 @@ impl Session { .get::() { Some(token) => { - let Some(store) = self - .services - .thread_extension_data - .get::() + let Some(store) = + self.services + .thread_extension_data + .get::() else { self.notify_mailbox_terminal_correlation_fault(); return Some(AgentStatus::Errored( @@ -5118,10 +5118,13 @@ fn mailbox_goal_for_terminal_turn( token: &crate::agent::goal_notifications::GoalNotificationTurnToken, publishes_completion: bool, result: Option, -) -> Result<( - ThreadId, - crate::agent::goal_notifications::GoalNotificationSnapshot, -), String> { +) -> Result< + ( + ThreadId, + crate::agent::goal_notifications::GoalNotificationSnapshot, + ), + String, +> { if store.incarnation() != token.incarnation { return Err("native goal store incarnation changed".into()); } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index f6233d74d400..8089a115c2ac 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -12645,7 +12645,10 @@ fn mailbox_terminal_goal_requires_successful_publication_for_exact_turn() { .expect("matching native completion should be capturable"); assert_eq!(incarnation, token.incarnation); assert_eq!(snapshot.binding, token.binding); - assert_eq!(snapshot.last_completed_source_turn.as_deref(), Some("turn-a")); + assert_eq!( + snapshot.last_completed_source_turn.as_deref(), + Some("turn-a") + ); assert_eq!(snapshot.phase, GoalNotificationPhase::ContinuationPending); let mut replacement = binding; @@ -12658,12 +12661,14 @@ fn mailbox_terminal_goal_requires_successful_publication_for_exact_turn() { assert!(replacement_projection.opt_in(&replacement)); store.install_authoritative(replacement_projection); store.set_source_turn("turn-a"); - assert!(mailbox_goal_for_terminal_turn( - &store, - &token, - /*publishes_completion*/ true, - Some("stale result".to_string()), - ) - .is_err()); + assert!( + mailbox_goal_for_terminal_turn( + &store, + &token, + /*publishes_completion*/ true, + Some("stale result".to_string()), + ) + .is_err() + ); assert_eq!(store.snapshot().unwrap().binding, replacement); } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index e24129c788cb..78a2fdad77a2 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -1,8 +1,8 @@ use crate::CodexAppsToolsCache; use crate::SkillsService; use crate::agent::AgentControl; -use crate::agent::goal_notifications::GoalNotificationStore; use crate::agent::goal_notifications::GoalNotificationSnapshot; +use crate::agent::goal_notifications::GoalNotificationStore; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; @@ -2656,7 +2656,9 @@ fn validate_deferred_goal_identity( if snapshot.binding.child_thread_id != child_thread_id || snapshot.binding.parent_thread_id != parent_thread_id { - return Err("deferred goal handback does not match its captured child/parent binding".into()); + return Err( + "deferred goal handback does not match its captured child/parent binding".into(), + ); } if snapshot.last_completed_source_turn.as_deref() != Some(source_turn_id) { return Err("deferred goal handback does not match its captured source turn".into()); diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 8524309e43a2..3ab6945e311b 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -2185,47 +2185,57 @@ fn mailbox_deferred_goal_handoff_rejects_replaced_native_incarnation() { let store = GoalNotificationStore::default(); let exact_goal = (store.incarnation(), snapshot.clone()); - assert!(validate_deferred_goal_identity( - child_thread_id, - parent_thread_id, - "turn-a", - Some(&exact_goal), - Some(&store), - ) - .is_ok()); - assert!(validate_deferred_goal_identity( - child_thread_id, - parent_thread_id, - "turn-b", - Some(&exact_goal), - Some(&store), - ) - .is_err()); - assert!(validate_deferred_goal_identity( - child_thread_id, - parent_thread_id, - "turn-a", - Some(&(ThreadId::new(), snapshot.clone())), - Some(&store), - ) - .is_err()); - assert!(validate_deferred_goal_identity( - ThreadId::new(), - parent_thread_id, - "turn-a", - Some(&exact_goal), - Some(&store), - ) - .is_err()); + assert!( + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&exact_goal), + Some(&store), + ) + .is_ok() + ); + assert!( + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-b", + Some(&exact_goal), + Some(&store), + ) + .is_err() + ); + assert!( + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&(ThreadId::new(), snapshot.clone())), + Some(&store), + ) + .is_err() + ); + assert!( + validate_deferred_goal_identity( + ThreadId::new(), + parent_thread_id, + "turn-a", + Some(&exact_goal), + Some(&store), + ) + .is_err() + ); // An unloaded child has no current native store to compare against. Its exact // incarnation is carried from the original runtime token and binding checked. - assert!(validate_deferred_goal_identity( - child_thread_id, - parent_thread_id, - "turn-a", - Some(&exact_goal), - None, - ) - .is_ok()); + assert!( + validate_deferred_goal_identity( + child_thread_id, + parent_thread_id, + "turn-a", + Some(&exact_goal), + None, + ) + .is_ok() + ); } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index 0fcd780b1d03..c0f3f4b87303 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -968,7 +968,13 @@ fn mailbox_send_message_tool_declares_durable_receipt_without_claiming_comprehen ); assert_eq!( output_schema["properties"]["activation_disposition"]["enum"], - json!(["not_requested", "already_active", "activated", "denied", "unknown"]) + json!([ + "not_requested", + "already_active", + "activated", + "denied", + "unknown" + ]) ); assert!( output_schema["properties"]["durable_state"]["description"] diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 91146a7f2232..9dddc29af900 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -242,7 +242,13 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { ); assert_eq!( send_output["properties"]["activation_disposition"]["enum"], - json!(["not_requested", "already_active", "activated", "denied", "unknown"]) + json!([ + "not_requested", + "already_active", + "activated", + "denied", + "unknown" + ]) ); assert!( send_output["properties"]["durable_state"]["description"] diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 9c710b4d83ba..b5831142a090 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -105,16 +105,28 @@ impl Respond for UnadmittedWorkResponder { ])); } if has_function_output(&body, "unadmitted-spawn") { - let spawn: Value = serde_json::from_str(function_output_text(&body, "unadmitted-spawn").unwrap()).unwrap(); - return function_call_response("unadmitted-wait-response", "unadmitted-wait", "wait_agent", json!({ - "targets": [spawn["work_ref"]], - "return_when": "all", - })); + let spawn: Value = + serde_json::from_str(function_output_text(&body, "unadmitted-spawn").unwrap()) + .unwrap(); + return function_call_response( + "unadmitted-wait-response", + "unadmitted-wait", + "wait_agent", + json!({ + "targets": [spawn["work_ref"]], + "return_when": "all", + }), + ); } - function_call_response("unadmitted-spawn-response", "unadmitted-spawn", "spawn_agent", json!({ - "task_name": "unadmitted_worker", - "message": "fail before provider acceptance of this work", - })) + function_call_response( + "unadmitted-spawn-response", + "unadmitted-spawn", + "spawn_agent", + json!({ + "task_name": "unadmitted_worker", + "message": "fail before provider acceptance of this work", + }), + ) } } @@ -132,7 +144,11 @@ async fn mailbox_event_only_wait_wakes_when_producer_fails_before_input_admissio let server = start_mock_server().await; let mut builder = test_codex().with_model("koffing").with_config(|config| { - for feature in [Feature::Collab, Feature::MultiAgentV2, Feature::AgentMailbox] { + for feature in [ + Feature::Collab, + Feature::MultiAgentV2, + Feature::AgentMailbox, + ] { config.features.enable(feature).unwrap(); } config.multi_agent_v2.tool_namespace = Some(NAMESPACE.to_string()); @@ -144,24 +160,36 @@ async fn mailbox_event_only_wait_wakes_when_producer_fails_before_input_admissio let root_id = test.session_configured.thread_id; Mock::given(method("POST")) .and(path_regex(".*/responses$")) - .and(RootThreadMatcher { thread_id: root_id.to_string() }) + .and(RootThreadMatcher { + thread_id: root_id.to_string(), + }) .respond_with(UnadmittedWorkResponder) .up_to_n_times(3) - .mount(&server).await; + .mount(&server) + .await; Mock::given(method("POST")) .and(path_regex(".*/responses$")) - .and(ChildThreadMatcher { root_thread_id: root_id.to_string(), task: "fail before provider acceptance of this work".to_string() }) + .and(ChildThreadMatcher { + root_thread_id: root_id.to_string(), + task: "fail before provider acceptance of this work".to_string(), + }) // No response-created event: this child never admits its accepted input. .respond_with(sse_response(String::new()).set_delay(Duration::from_secs(2))) .up_to_n_times(1) - .mount(&server).await; - test.codex.submit(Op::UserInput { - items: vec![UserInput::Text { text: "join the accepted work without a timeout".to_string(), text_elements: Vec::new() }], - final_output_json_schema: None, - responsesapi_client_metadata: None, - additional_context: Default::default(), - thread_settings: ThreadSettingsOverrides::default(), - }).await?; + .mount(&server) + .await; + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "join the accepted work without a timeout".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: ThreadSettingsOverrides::default(), + }) + .await?; // Prove the selected wait was sleeping while native custody still existed, // rather than only testing an already-cold initial snapshot. wait_for_event(&test.codex, |event| matches!(event, @@ -171,27 +199,55 @@ async fn mailbox_event_only_wait_wakes_when_producer_fails_before_input_admissio && call.mailbox_wait_report.as_ref().is_some_and(|report| report.pending.iter().any(|item| item.status == "pending")) ) )).await; - let terminal = wait_for_event(&test.codex, |event| matches!(event, - EventMsg::ItemCompleted(event) if matches!(&event.item, - TurnItem::CollabAgentToolCall(call) if call.id == "unadmitted-wait" + let terminal = wait_for_event(&test.codex, |event| { + matches!(event, + EventMsg::ItemCompleted(event) if matches!(&event.item, + TurnItem::CollabAgentToolCall(call) if call.id == "unadmitted-wait" + ) ) - )).await; - let EventMsg::ItemCompleted(terminal) = terminal else { unreachable!() }; - let TurnItem::CollabAgentToolCall(call) = terminal.item else { unreachable!() }; + }) + .await; + let EventMsg::ItemCompleted(terminal) = terminal else { + unreachable!() + }; + let TurnItem::CollabAgentToolCall(call) = terminal.item else { + unreachable!() + }; let report = call.mailbox_wait_report.unwrap(); - assert_eq!(report.completion_reason, Some(codex_protocol::items::MailboxWakeCause::NeedsAttention)); + assert_eq!( + report.completion_reason, + Some(codex_protocol::items::MailboxWakeCause::NeedsAttention) + ); assert!(!report.condition_satisfied); assert_eq!(report.pending.len(), 1); assert_eq!(report.pending[0].status, "recovery_needed"); - wait_for_event(&test.codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; let mailbox = Mailbox::open(test.config.sqlite_config()).await?; - let snapshot = mailbox.snapshot(&Selection { caller: root_id, agent: None, selectors: vec![Selector::Work(report.pending[0].work_ref.clone().unwrap())], return_when: ReturnWhen::All, limit: 1 }).await?; - let SnapshotItem::Work(work) = &snapshot.items[0] else { unreachable!() }; + let snapshot = mailbox + .snapshot(&Selection { + caller: root_id, + agent: None, + selectors: vec![Selector::Work(report.pending[0].work_ref.clone().unwrap())], + return_when: ReturnWhen::All, + limit: 1, + }) + .await?; + let SnapshotItem::Work(work) = &snapshot.items[0] else { + unreachable!() + }; assert_eq!(work.dispatch_state, DispatchState::RecoveryNeeded); assert!(work.outcome.is_none()); let input_id = work.input.id.as_ref().unwrap().to_string(); - assert_eq!(mailbox.unadmitted_input_ids(work.work_ref.agent, &[input_id.clone()]).await?, vec![input_id]); + assert_eq!( + mailbox + .unadmitted_input_ids(work.work_ref.agent, &[input_id.clone()]) + .await?, + vec![input_id] + ); Ok(()) } From e12fbb60f296b8f7ac67c3e2f1473f162dee40d7 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:03:23 +1100 Subject: [PATCH 17/54] ci(mailbox): include owned crate in protected analysis coverage --- .github/scripts/test_ci_planners.py | 4 ++-- .github/workflows/codeql.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 182766f58d5a..3f18a04c8021 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1012,7 +1012,7 @@ def test_v2_residency_route_stays_on_multi_agent_orchestration_lane(self) -> Non ] ) self.assertEqual(recipe.count("cargo nextest run"), 4) - self.assertEqual(recipe.count("RUST_MIN_STACK="), 4) + self.assertEqual(recipe.count("RUST_MIN_STACK="), 6) self.assertEqual(recipe.count("--no-tests=fail"), 4) for test_name in ( "agent::control::residency::tests::" @@ -5598,7 +5598,7 @@ def test_stack_sensitive_targeted_recipes_set_rust_min_stack(self) -> None: multi_agent_recipe = "\n".join( recipes["core-multi-agent-orchestration-targeted"] ) - self.assertEqual(multi_agent_recipe.count("RUST_MIN_STACK="), 4) + self.assertEqual(multi_agent_recipe.count("RUST_MIN_STACK="), 6) blocking_waits_core_recipe = "\n".join( recipes["blocking-waits-core-targeted"] diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a61372987da8..a1006dbf6a15 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -297,7 +297,7 @@ jobs: ;; codex-rs-rest-a) scope_paths=( - codex-rs/.cargo codex-rs/analytics codex-rs/arg0 codex-rs/browser-computer-use + codex-rs/.cargo codex-rs/agent-mailbox codex-rs/analytics codex-rs/arg0 codex-rs/browser-computer-use codex-rs/cloud-tasks codex-rs/code-mode-host codex-rs/codex-client codex-rs/collaboration-mode-templates codex-rs/core-api codex-rs/exec-server-protocol codex-rs/features codex-rs/git-utils codex-rs/lmstudio codex-rs/message-history From 08988d69a9c13764250b99a94cdd275230ac7f6a Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:13:23 +1100 Subject: [PATCH 18/54] ci(mailbox): retire temporary source preparation workflow --- .../workflows/mailbox-source-preparation.yml | 192 ------------------ 1 file changed, 192 deletions(-) delete mode 100644 .github/workflows/mailbox-source-preparation.yml diff --git a/.github/workflows/mailbox-source-preparation.yml b/.github/workflows/mailbox-source-preparation.yml deleted file mode 100644 index 36129cb0914e..000000000000 --- a/.github/workflows/mailbox-source-preparation.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: Mailbox focused hosted source preparation - -on: - push: - branches: [feat/quiet-durable-mailbox] - paths: [.github/workflows/mailbox-source-preparation.yml] - -permissions: - contents: read - -jobs: - prepare-and-compile: - runs-on: ubuntu-24.04 - timeout-minutes: 90 - env: - CARGO_INCREMENTAL: "0" - RUST_MIN_STACK: "8388608" - PREPARATION_ATTEMPT: "10" - SCCACHE_CACHE_SIZE: 2G - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Isolate workflow caches from source provenance - run: bash .github/scripts/isolate_workflow_paths_from_provenance.sh "$GITHUB_WORKSPACE" - - uses: ./.github/actions/setup-bazel-ci - with: - target: x86_64-unknown-linux-gnu - - uses: dtolnay/rust-toolchain@ebb3d1676050bfd0971c36c1e215b5751473994d - with: - toolchain: 1.96.0 - components: rustfmt, clippy - - name: Compute dependency cache identity - id: lockhash - shell: bash - run: | - set -euo pipefail - echo "hash=$(sha256sum codex-rs/Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - echo "toolchain_hash=$(sha256sum codex-rs/rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - - name: Restore cargo home cache - id: cache_cargo_home_restore - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/.rusty_v8/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: cargo-home-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} - - name: Install sccache - uses: taiki-e/install-action@7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c - with: - tool: sccache - - name: Configure the existing restore-only cache policy - id: sccache_backend - run: bash .github/scripts/configure_sccache_backend.sh restore-only - - name: Enable sccache wrapper - shell: bash - run: | - set -euo pipefail - echo "RUSTC_WRAPPER=$(command -v sccache)" >> "$GITHUB_ENV" - sccache --start-server || true - sccache --zero-stats || true - - name: Restore sccache cache - uses: actions/cache/restore@v6 - with: - path: ${{ github.workspace }}/.sccache/ - key: sccache-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }}-${{ github.run_id }} - restore-keys: | - sccache-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }}- - sccache-linux-rust-${{ steps.lockhash.outputs.hash }}- - - uses: ./.github/actions/setup-rusty-v8 - with: - target: x86_64-unknown-linux-gnu - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 - with: - node-version: 22 - - name: Install standard Linux build dependencies - run: | - sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev - - name: Prepare only this branch's changed Rust files and lock metadata - shell: bash - run: | - set -euo pipefail - while IFS= read -r source; do - rustfmt --edition 2024 --config skip_children=true "$source" - done < <(git diff --name-only 2d91cb7437de6e146c8ec4eb654755dac2fb5ddd HEAD -- '*.rs') - cd codex-rs - cargo metadata --no-deps --format-version 1 > /dev/null - - name: Refresh the Bazel dependency lock - run: just bazel-lock-update - - name: Run the owned library contracts - id: library - continue-on-error: true - working-directory: codex-rs - run: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 - - name: Verify canonical mailbox lifecycle retention - id: rollout - continue-on-error: true - working-directory: codex-rs - run: cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 - - name: Generate the affected native schemas - run: | - just write-config-schema - just write-app-server-schema - - name: Compile focused UI mailbox consumers - id: ui - continue-on-error: true - working-directory: codex-rs - env: - INSTA_UPDATE: new - run: cargo test --locked -p codex-tui --lib mailbox -- --test-threads=1 - - name: Verify interrupted mailbox wait history replay - id: protocol - continue-on-error: true - working-directory: codex-rs - run: cargo test --locked -p codex-app-server-protocol --lib mailbox -- --test-threads=1 - - name: Compile focused core mailbox and schema consumers - id: core - continue-on-error: true - working-directory: codex-rs - run: cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 - - name: Run real native mailbox adapter contracts - id: adapter - continue-on-error: true - working-directory: codex-rs - run: | - cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 - - name: Retain the focused contract results - if: always() - run: | - test '${{ steps.library.outcome }}' = success - test '${{ steps.rollout.outcome }}' = success - test '${{ steps.ui.outcome }}' = success - test '${{ steps.protocol.outcome }}' = success - test '${{ steps.core.outcome }}' = success - test '${{ steps.adapter.outcome }}' = success - - name: Capture the exact generated source delta - if: always() - shell: bash - run: | - set -euo pipefail - mkdir -p "$RUNNER_TEMP/mailbox-source-preparation" - for generated_type in MailboxWaitReport MailboxWaitReturnWhen MailboxWaitSelectionItem MailboxWakeCause; do - generated_path="codex-rs/app-server-protocol/schema/typescript/${generated_type}.ts" - if [[ -f "$generated_path" ]]; then - git add --intent-to-add -- "$generated_path" - fi - done - git diff --binary > "$RUNNER_TEMP/mailbox-source-preparation/source.patch" - git rev-parse HEAD > "$RUNNER_TEMP/mailbox-source-preparation/source-sha.txt" - git status --short > "$RUNNER_TEMP/mailbox-source-preparation/status.txt" - while IFS= read -r -d '' pending_snapshot; do - snapshot_target="$RUNNER_TEMP/mailbox-source-preparation/pending-snapshots/$pending_snapshot" - mkdir -p "$(dirname "$snapshot_target")" - cp -- "$pending_snapshot" "$snapshot_target" - done < <(git ls-files --others --exclude-standard -z -- 'codex-rs/tui/src/**/*.snap.new') - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - if: always() - with: - name: mailbox-source-preparation-${{ github.sha }} - path: ${{ runner.temp }}/mailbox-source-preparation/ - retention-days: 3 - - name: Check post-lane cache-save disk headroom - if: ${{ always() && !cancelled() }} - id: disk_guard - shell: bash - run: | - set -euo pipefail - save_floor_bytes=$((8 * 1024 * 1024 * 1024)) - available_bytes="$(df --block-size=1 --output=avail "$GITHUB_WORKSPACE" | tail -n 1 | tr -d '[:space:]')" - if [[ "$available_bytes" -lt "$save_floor_bytes" ]]; then - echo "skip_cache_save=true" >> "$GITHUB_OUTPUT" - else - echo "skip_cache_save=false" >> "$GITHUB_OUTPUT" - fi - - name: Save cargo home cache after success or failure - if: ${{ always() && !cancelled() && steps.lockhash.outputs.hash != '' && steps.cache_cargo_home_restore.outputs.cache-hit != 'true' && steps.disk_guard.outputs.skip_cache_save == 'false' }} - continue-on-error: true - uses: actions/cache/save@v6 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/.rusty_v8/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: cargo-home-linux-rust-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} From c4794f7efdd3cdc43c76d6b3a6bcb46e5bb4d6d7 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:38:30 +1100 Subject: [PATCH 19/54] fix(agents): reconcile mailbox recovery and bounded wait contracts --- codex-rs/agent-mailbox/src/activation.rs | 6 +- codex-rs/agent-mailbox/src/lib.rs | 45 ++++-- codex-rs/agent-mailbox/src/mailbox_tests.rs | 45 ++++-- .../agent-mailbox/src/message_delivery.rs | 16 +- codex-rs/agent-mailbox/src/message_rules.rs | 10 +- codex-rs/agent-mailbox/src/messages.rs | 53 ++++--- codex-rs/agent-mailbox/src/outcome.rs | 54 ++++--- .../agent-mailbox/src/outcome_messages.rs | 23 ++- codex-rs/agent-mailbox/src/outcome_store.rs | 26 ++-- codex-rs/agent-mailbox/src/schema.rs | 17 ++- codex-rs/agent-mailbox/src/snapshot.rs | 32 ++-- codex-rs/agent-mailbox/src/store.rs | 3 +- codex-rs/agent-mailbox/src/types.rs | 12 +- codex-rs/agent-mailbox/src/wait_graph.rs | 19 ++- codex-rs/agent-mailbox/src/work.rs | 29 +++- .../src/protocol/thread_history.rs | 8 +- codex-rs/core/src/agent/control/mailbox.rs | 76 +++++++++- .../core/src/agent/control/mailbox_tests.rs | 137 ++++++++++++++++++ .../src/tools/handlers/multi_agents_spec.rs | 23 ++- .../handlers/multi_agents_v2/mailbox_wait.rs | 57 +++++++- .../multi_agents_v2/mailbox_wait_tests.rs | 38 +++++ codex-rs/core/src/tools/parallel.rs | 6 +- codex-rs/core/src/tools/spec_plan_tests.rs | 37 ++++- codex-rs/core/tests/suite/agent_mailbox.rs | 15 +- 24 files changed, 627 insertions(+), 160 deletions(-) diff --git a/codex-rs/agent-mailbox/src/activation.rs b/codex-rs/agent-mailbox/src/activation.rs index 2a1145230bbd..4d3fac89dd16 100644 --- a/codex-rs/agent-mailbox/src/activation.rs +++ b/codex-rs/agent-mailbox/src/activation.rs @@ -1,6 +1,8 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::store::Mailbox; -use crate::types::{ActivationDecision, ActivationDisposition}; +use crate::types::ActivationDecision; +use crate::types::ActivationDisposition; use codex_protocol::ThreadId; use sqlx::Row; diff --git a/codex-rs/agent-mailbox/src/lib.rs b/codex-rs/agent-mailbox/src/lib.rs index 8ab34f3de22a..3278f65b5de6 100644 --- a/codex-rs/agent-mailbox/src/lib.rs +++ b/codex-rs/agent-mailbox/src/lib.rs @@ -20,17 +20,44 @@ mod types; mod wait_graph; mod work; -pub use error::{MailboxError, MailboxResult}; +pub use error::MailboxError; +pub use error::MailboxResult; pub use snapshot::is_provably_closed_peer_cycle; pub use store::Mailbox; -pub use types::{ - ActivationDecision, ActivationDisposition, AdmissionReceipt, Attention, CanonicalConfirmation, - Continuation, ContinuationReceipt, ContinuationState, Dependency, DependencyMode, - DispatchDecision, DispatchState, Envelope, GoalBinding, MessageDraft, MessageKind, NativeWork, - ObservationTarget, Outcome, OutcomeReceipt, PendingPage, RecoveryEvidence, RequestDisposition, - RequestSnapshot, ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotRequest, - SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, -}; +pub use types::ActivationDecision; +pub use types::ActivationDisposition; +pub use types::AdmissionReceipt; +pub use types::Attention; +pub use types::CanonicalConfirmation; +pub use types::Continuation; +pub use types::ContinuationReceipt; +pub use types::ContinuationState; +pub use types::Dependency; +pub use types::DependencyMode; +pub use types::DispatchDecision; +pub use types::DispatchState; +pub use types::Envelope; +pub use types::GoalBinding; +pub use types::MessageDraft; +pub use types::MessageKind; +pub use types::NativeWork; +pub use types::ObservationTarget; +pub use types::Outcome; +pub use types::OutcomeReceipt; +pub use types::PendingPage; +pub use types::RecoveryEvidence; +pub use types::RequestDisposition; +pub use types::RequestSnapshot; +pub use types::ReturnWhen; +pub use types::Selection; +pub use types::Selector; +pub use types::Snapshot; +pub use types::SnapshotItem; +pub use types::SnapshotRequest; +pub use types::SnapshotState; +pub use types::TurnOutcomeReceipt; +pub use types::WorkRef; +pub use types::WorkSnapshot; pub use wait_graph::InputArrival; pub use wait_graph::WaitRegistration; diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 1c79f4567306..6229078ac4dc 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1,12 +1,37 @@ -use crate::{ - ActivationDecision, ActivationDisposition, AdmissionReceipt, Continuation, ContinuationState, - Dependency, DependencyMode, DispatchDecision, DispatchState, Envelope, GoalBinding, Mailbox, - MailboxError, MessageDraft, MessageKind, NativeWork, ObservationTarget, Outcome, - RecoveryEvidence, RequestDisposition, RequestSnapshot, ReturnWhen, Selection, Selector, - Snapshot, SnapshotItem, SnapshotState, TurnOutcomeReceipt, WorkRef, WorkSnapshot, - is_provably_closed_peer_cycle, -}; -use codex_protocol::{AgentPath, ThreadId, protocol::InterAgentCommunication}; +use crate::ActivationDecision; +use crate::ActivationDisposition; +use crate::AdmissionReceipt; +use crate::Continuation; +use crate::ContinuationState; +use crate::Dependency; +use crate::DependencyMode; +use crate::DispatchDecision; +use crate::DispatchState; +use crate::Envelope; +use crate::GoalBinding; +use crate::Mailbox; +use crate::MailboxError; +use crate::MessageDraft; +use crate::MessageKind; +use crate::NativeWork; +use crate::ObservationTarget; +use crate::Outcome; +use crate::RecoveryEvidence; +use crate::RequestDisposition; +use crate::RequestSnapshot; +use crate::ReturnWhen; +use crate::Selection; +use crate::Selector; +use crate::Snapshot; +use crate::SnapshotItem; +use crate::SnapshotState; +use crate::TurnOutcomeReceipt; +use crate::WorkRef; +use crate::WorkSnapshot; +use crate::is_provably_closed_peer_cycle; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::protocol::InterAgentCommunication; use codex_state::SqliteConfig; use codex_utils_absolute_path::test_support::PathExt; use pretty_assertions::assert_eq; @@ -1369,7 +1394,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re let uncanonical = fixture .mailbox - .pending_page(recipient, None, 20) + .pending_page(recipient, /*after_seq*/ None, 20) .await .unwrap(); let uncanonical_ids = uncanonical diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index 89fe4dd0b08c..3e0ffd3defe9 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -1,7 +1,10 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::messages::decrement_pending_count; use crate::store::Mailbox; -use crate::types::{CanonicalConfirmation, Envelope, PendingPage}; +use crate::types::CanonicalConfirmation; +use crate::types::Envelope; +use crate::types::PendingPage; use codex_protocol::ThreadId; use sqlx::Row; use std::future::Future; @@ -35,7 +38,10 @@ impl Mailbox { /// Convenience first page; use `pending_page` to continue pagination. pub async fn pending(&self, recipient: ThreadId, limit: usize) -> MailboxResult> { - Ok(self.pending_page(recipient, None, limit).await?.items) + Ok(self + .pending_page(recipient, /*after_seq*/ None, limit) + .await? + .items) } /// Read a bounded page including canonical records, without consuming them. @@ -416,7 +422,9 @@ impl Mailbox { F: FnMut(Envelope) -> Fut, Fut: Future>, { - let page = self.pending_page(recipient, None, limit).await?; + let page = self + .pending_page(recipient, /*after_seq*/ None, limit) + .await?; let mut reconciled = 0; for envelope in page.items { let id = envelope.id.clone(); diff --git a/codex-rs/agent-mailbox/src/message_rules.rs b/codex-rs/agent-mailbox/src/message_rules.rs index d5caacf45562..3b29e20b7d0e 100644 --- a/codex-rs/agent-mailbox/src/message_rules.rs +++ b/codex-rs/agent-mailbox/src/message_rules.rs @@ -1,6 +1,10 @@ -use crate::error::{MailboxError, MailboxResult}; -use crate::types::{MessageDraft, MessageKind}; -use sqlx::{Row, Sqlite, Transaction}; +use crate::error::MailboxError; +use crate::error::MailboxResult; +use crate::types::MessageDraft; +use crate::types::MessageKind; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::Transaction; pub(crate) async fn validate_reply_tx( transaction: &mut Transaction<'_, Sqlite>, diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index d1447a8acf95..1a5b9b6d4b41 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -1,12 +1,20 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::message_delivery::envelope_from_row; -use crate::message_rules::{validate_replacement_tx, validate_reply_tx}; +use crate::message_rules::validate_replacement_tx; +use crate::message_rules::validate_reply_tx; use crate::store::Mailbox; -use crate::types::{Envelope, MessageDraft, MessageKind, WorkRef}; +use crate::types::Envelope; +use crate::types::MessageDraft; +use crate::types::MessageKind; +use crate::types::WorkRef; use crate::work::exact_work; -use codex_protocol::{ResponseItemId, ThreadId}; +use codex_protocol::ResponseItemId; +use codex_protocol::ThreadId; use serde::Serialize; -use sqlx::{Row, Sqlite, Transaction}; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::Transaction; const MAX_MESSAGE_BYTES: usize = 8_192; const MAX_RECORD_BYTES: usize = 16_384; @@ -139,10 +147,11 @@ pub(crate) async fn insert_message_tx( validate_replacement_tx(transaction, supersedes, &draft).await?; } - if draft.supersedes.is_none() && draft.kind == MessageKind::Progress && !draft.activate_if_idle - { - if let Some(work_ref) = draft.work_ref.as_ref() { - if let Some(existing) = sqlx::query( + if draft.supersedes.is_none() + && draft.kind == MessageKind::Progress + && !draft.activate_if_idle + && let Some(work_ref) = draft.work_ref.as_ref() + && let Some(existing) = sqlx::query( "SELECT * FROM mailbox_messages WHERE sender = ? AND recipient = ? AND work_ref_json = ? AND author_path = ? AND recipient_path = ? @@ -163,12 +172,10 @@ pub(crate) async fn insert_message_tx( .bind(i64::from(draft.legacy_attention)) .fetch_optional(&mut **transaction) .await? - { - // Replacement gets its own stable ID. Never rewrite the body or - // idempotency binding behind the previously accepted identifier. - draft.supersedes = Some(existing.try_get("id")?); - } - } + { + // Replacement gets its own stable ID. Never rewrite the body or + // idempotency binding behind the previously accepted identifier. + draft.supersedes = Some(existing.try_get("id")?); } let collision: Option = @@ -263,15 +270,13 @@ pub(crate) async fn insert_message_tx( .bind(old_id) .execute(&mut **transaction) .await?; - if replaced.rows_affected() == 1 { - if let Some(old) = old { - let unread = old.try_get::("canonical")? == 0 - || old.try_get::("input_admitted")? == 0; - if unread { - let recipient: String = old.try_get("recipient")?; - let quiet = old.try_get::("quiet")? != 0; - decrement_pending_count(transaction, &recipient, quiet).await?; - } + if replaced.rows_affected() == 1 && let Some(old) = old { + let unread = old.try_get::("canonical")? == 0 + || old.try_get::("input_admitted")? == 0; + if unread { + let recipient: String = old.try_get("recipient")?; + let quiet = old.try_get::("quiet")? != 0; + decrement_pending_count(transaction, &recipient, quiet).await?; } } } diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index fe37cd32a85d..5fa97e9a19c4 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -1,16 +1,25 @@ -use crate::error::{MailboxError, MailboxResult}; -use crate::outcome_messages::{ - ensure_terminal_responses, insert_response_group, response_groups, validate_auto_response_size, -}; -use crate::outcome_store::{ - MAX_CONTINUATIONS_PER_WORK, OutcomeData, outcome_receipt_for, persist_continuation, - save_outcome_tx, -}; +use crate::error::MailboxError; +use crate::error::MailboxResult; +use crate::outcome_messages::ensure_terminal_responses; +use crate::outcome_messages::insert_response_group; +use crate::outcome_messages::response_groups; +use crate::outcome_messages::validate_auto_response_size; +use crate::outcome_store::MAX_CONTINUATIONS_PER_WORK; +use crate::outcome_store::OutcomeData; +use crate::outcome_store::outcome_receipt_for; +use crate::outcome_store::persist_continuation; +use crate::outcome_store::save_outcome_tx; use crate::store::Mailbox; -use crate::types::{ - Continuation, GoalBinding, Outcome, OutcomeReceipt, TurnOutcomeReceipt, WorkRef, -}; -use crate::work::{ensure_mailbox_id, exact_work_tx, new_mailbox_id, work_ref_from_row}; +use crate::types::Continuation; +use crate::types::GoalBinding; +use crate::types::Outcome; +use crate::types::OutcomeReceipt; +use crate::types::TurnOutcomeReceipt; +use crate::types::WorkRef; +use crate::work::ensure_mailbox_id; +use crate::work::exact_work_tx; +use crate::work::new_mailbox_id; +use crate::work::work_ref_from_row; use codex_protocol::ThreadId; use codex_protocol::protocol::InterAgentCommunication; use sqlx::Row; @@ -67,7 +76,7 @@ impl Mailbox { ORDER BY binding_json LIMIT ?", ) .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(&mut *transaction) .await?; if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { @@ -135,8 +144,11 @@ impl Mailbox { let mut groups = response_groups(&mut transaction, &refs, response.as_ref()).await?; validate_auto_response_size(&groups)?; if is_nonterminal(&continuation) { - let binding = continuation_binding(&continuation) - .expect("nonterminal continuation always carries a binding"); + let binding = continuation_binding(&continuation).ok_or_else(|| { + MailboxError::InvalidInput( + "nonterminal continuation is missing its binding".into(), + ) + })?; let binding_json = serde_json::to_string(binding)?; for work_ref in &refs { let unresolved = sqlx::query_scalar::<_, String>( @@ -145,7 +157,7 @@ impl Mailbox { ORDER BY binding_json LIMIT ?", ) .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(&mut *transaction) .await?; if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { @@ -356,8 +368,12 @@ impl Mailbox { validate_binding_for_work(binding, work_ref)?; } let mut transaction = self.pool.begin().await?; - let mut groups = - response_groups(&mut transaction, &[work_ref.clone()], response.as_ref()).await?; + let mut groups = response_groups( + &mut transaction, + std::slice::from_ref(work_ref), + response.as_ref(), + ) + .await?; let group = groups.pop().ok_or(MailboxError::ReferenceMismatch)?; let mut groups = vec![group]; ensure_terminal_responses( @@ -389,7 +405,7 @@ impl Mailbox { ORDER BY binding_json LIMIT ?", ) .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(&mut *transaction) .await?; if unresolved.len() > MAX_CONTINUATIONS_PER_WORK as usize { diff --git a/codex-rs/agent-mailbox/src/outcome_messages.rs b/codex-rs/agent-mailbox/src/outcome_messages.rs index e38f1ea350fc..0917bc30f42a 100644 --- a/codex-rs/agent-mailbox/src/outcome_messages.rs +++ b/codex-rs/agent-mailbox/src/outcome_messages.rs @@ -1,11 +1,20 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::messages::insert_message_tx; -use crate::types::{Continuation, MessageDraft, MessageKind, Outcome, WorkRef}; +use crate::types::Continuation; +use crate::types::MessageDraft; +use crate::types::MessageKind; +use crate::types::Outcome; +use crate::types::WorkRef; use crate::work::exact_work_tx; -use codex_protocol::protocol::{AgentCommunicationOrigin, InterAgentCommunication}; -use codex_protocol::{AgentPath, ThreadId}; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::protocol::AgentCommunicationOrigin; +use codex_protocol::protocol::InterAgentCommunication; use serde::Serialize; -use sqlx::{Row, Sqlite, Transaction}; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::Transaction; const MAX_AUTO_RESPONSE_BYTES: usize = 16_384; const MAX_RESPONSE_BYTES: usize = 8_192; @@ -158,9 +167,9 @@ pub(crate) async fn insert_response_group( supersedes: None, legacy_attention: outcome == Outcome::NeedsAttention, }, - None, + /*explicit_id*/ None, ) .await?; - group.output_id = Some(envelope.id.clone()); + group.output_id = Some(envelope.id); Ok(group.output_id.clone()) } diff --git a/codex-rs/agent-mailbox/src/outcome_store.rs b/codex-rs/agent-mailbox/src/outcome_store.rs index a0ae56caa7ec..2bc6c0e9337b 100644 --- a/codex-rs/agent-mailbox/src/outcome_store.rs +++ b/codex-rs/agent-mailbox/src/outcome_store.rs @@ -1,11 +1,19 @@ -use crate::error::{MailboxError, MailboxResult}; -use crate::types::{ - Continuation, ContinuationReceipt, ContinuationState, Outcome, OutcomeReceipt, WorkRef, -}; +use crate::error::MailboxError; +use crate::error::MailboxResult; +use crate::types::Continuation; +use crate::types::ContinuationReceipt; +use crate::types::ContinuationState; +use crate::types::Outcome; +use crate::types::OutcomeReceipt; +use crate::types::WorkRef; use crate::work::exact_work; use codex_protocol::protocol::InterAgentCommunication; -use serde::{Deserialize, Serialize}; -use sqlx::{Row, Sqlite, SqlitePool, Transaction}; +use serde::Deserialize; +use serde::Serialize; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::SqlitePool; +use sqlx::Transaction; pub(crate) const MAX_CONTINUATIONS_PER_WORK: i64 = 32; @@ -77,7 +85,7 @@ pub(crate) async fn persist_continuation( } let count_rows = sqlx::query("SELECT 1 FROM mailbox_continuations WHERE work_id = ? LIMIT ?") .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(&mut **transaction) .await?; if count_rows.len() >= MAX_CONTINUATIONS_PER_WORK as usize { @@ -179,7 +187,7 @@ pub(crate) async fn stored_outcome_receipt_for( serde_json::from_str::(&row.try_get::("outcome_json")?)? .response .is_some(); - outcome_receipt_for(pool, work_ref, None, has_response).await + outcome_receipt_for(pool, work_ref, /*output_id*/ None, has_response).await } pub(crate) async fn continuation_receipts_for( @@ -193,7 +201,7 @@ pub(crate) async fn continuation_receipts_for( ORDER BY source_turn_id, binding_json LIMIT ?", ) .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(pool) .await?; if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { diff --git a/codex-rs/agent-mailbox/src/schema.rs b/codex-rs/agent-mailbox/src/schema.rs index 5db9fd1aafe1..bbdc264a45b2 100644 --- a/codex-rs/agent-mailbox/src/schema.rs +++ b/codex-rs/agent-mailbox/src/schema.rs @@ -1,5 +1,6 @@ use crate::error::MailboxResult; -use sqlx::{Row, SqlitePool}; +use sqlx::Row; +use sqlx::SqlitePool; const SCHEMA_VERSION: i64 = 5; @@ -41,13 +42,13 @@ pub(crate) async fn initialize(pool: &SqlitePool) -> MailboxResult<()> { sqlx::query_scalar("SELECT version FROM mailbox_schema WHERE singleton = 1") .fetch_optional(&mut *transaction) .await?; - if let Some(version) = version { - if version > SCHEMA_VERSION { - return Err(crate::MailboxError::NewerSchema { - found: version, - supported: SCHEMA_VERSION, - }); - } + if let Some(version) = version + && version > SCHEMA_VERSION + { + return Err(crate::MailboxError::NewerSchema { + found: version, + supported: SCHEMA_VERSION, + }); } sqlx::query( diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs index a819af81c0b6..8625bf4c7402 100644 --- a/codex-rs/agent-mailbox/src/snapshot.rs +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -1,15 +1,28 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::message_delivery::envelope_from_row; -use crate::outcome_store::{continuation_receipts_for, stored_outcome_receipt_for}; +use crate::outcome_store::continuation_receipts_for; +use crate::outcome_store::stored_outcome_receipt_for; use crate::store::Mailbox; -use crate::types::{ - Attention, Dependency, DependencyMode, ObservationTarget, RequestDisposition, RequestSnapshot, - ReturnWhen, Selection, Selector, Snapshot, SnapshotItem, SnapshotState, WorkSnapshot, -}; -use crate::work::{parse_dispatch_state, work_ref_from_row}; +use crate::types::Attention; +use crate::types::Dependency; +use crate::types::DependencyMode; +use crate::types::ObservationTarget; +use crate::types::RequestDisposition; +use crate::types::RequestSnapshot; +use crate::types::ReturnWhen; +use crate::types::Selection; +use crate::types::Selector; +use crate::types::Snapshot; +use crate::types::SnapshotItem; +use crate::types::SnapshotState; +use crate::types::WorkSnapshot; +use crate::work::parse_dispatch_state; +use crate::work::work_ref_from_row; use codex_protocol::ThreadId; use sqlx::Row; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +use std::collections::HashSet; const MAX_SELECTIONS: usize = 100; const MAX_OBSERVATIONS: usize = 100; @@ -293,7 +306,8 @@ impl Mailbox { recipient: ThreadId, limit: usize, ) -> MailboxResult> { - self.pending_attention(recipient, None, limit).await + self.pending_attention(recipient, /*after_seq*/ None, limit) + .await } } diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index e7e234965b76..d671fde918eb 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -1,9 +1,10 @@ use crate::error::MailboxResult; use sqlx::SqlitePool; use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; -use std::sync::{Arc, Mutex}; use tokio::sync::watch; #[derive(Clone)] diff --git a/codex-rs/agent-mailbox/src/types.rs b/codex-rs/agent-mailbox/src/types.rs index 1ce228a3cdc6..726f494cded5 100644 --- a/codex-rs/agent-mailbox/src/types.rs +++ b/codex-rs/agent-mailbox/src/types.rs @@ -1,6 +1,7 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::InterAgentCommunication; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkRef { @@ -19,9 +20,10 @@ pub struct NativeWork { pub input: InterAgentCommunication, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MessageKind { + #[default] Note, Progress, Request, @@ -45,12 +47,6 @@ pub enum ActivationDecision { RecoveryNeeded, } -impl Default for MessageKind { - fn default() -> Self { - Self::Note - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct MessageDraft { pub invocation_key: String, diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs index f07ebe93c621..4d8b26ee1875 100644 --- a/codex-rs/agent-mailbox/src/wait_graph.rs +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -1,10 +1,14 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::store::Mailbox; use crate::types::ReturnWhen; use codex_protocol::ThreadId; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::Weak; use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex, Weak}; const MAX_ACTIVE_WAITS: usize = 4_096; const MAX_PEERS_PER_WAIT: usize = 100; @@ -106,7 +110,14 @@ impl Mailbox { mode: ReturnWhen, has_escape: bool, ) -> MailboxResult { - self.begin_wait_inner(wait_id, owner, peers, mode, has_escape, None) + self.begin_wait_inner( + wait_id, + owner, + peers, + mode, + has_escape, + /*input_generation*/ None, + ) } /// Register only if no eligible-input publication raced the caller's rescan. diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index 8009921bcc07..f727c4ac9c0a 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -1,14 +1,20 @@ -use crate::error::{MailboxError, MailboxResult}; +use crate::error::MailboxError; +use crate::error::MailboxResult; use crate::messages::decrement_pending_count; use crate::outcome_store::MAX_CONTINUATIONS_PER_WORK; use crate::store::Mailbox; -use crate::types::{ - AdmissionReceipt, DispatchDecision, DispatchState, GoalBinding, NativeWork, RecoveryEvidence, - WorkRef, -}; +use crate::types::AdmissionReceipt; +use crate::types::DispatchDecision; +use crate::types::DispatchState; +use crate::types::GoalBinding; +use crate::types::NativeWork; +use crate::types::RecoveryEvidence; +use crate::types::WorkRef; use codex_protocol::ThreadId; use codex_protocol::protocol::InterAgentCommunication; -use sqlx::{Row, Sqlite, Transaction}; +use sqlx::Row; +use sqlx::Sqlite; +use sqlx::Transaction; const MAX_WORK_ITEMS: i64 = 50_000; const MAX_INPUT_BYTES: usize = 1_000_000; @@ -58,7 +64,14 @@ impl Mailbox { if !work.input.id.as_ref().is_some_and(is_mailbox_id) { work.input.id = Some(codex_protocol::ResponseItemId::from_server(new_mailbox_id())); } - let input_message_id = work.input.id.as_ref().expect("assigned above").to_string(); + let input_message_id = work + .input + .id + .as_ref() + .map(ToString::to_string) + .ok_or_else(|| { + MailboxError::InvalidInput("accepted native input ID is missing".into()) + })?; let input_json = serde_json::to_string(&work.input)?; let mut transaction = self.pool.begin().await?; @@ -321,7 +334,7 @@ impl Mailbox { ORDER BY binding_json LIMIT ?", ) .bind(&work_ref.id) - .bind((MAX_CONTINUATIONS_PER_WORK + 1) as i64) + .bind(MAX_CONTINUATIONS_PER_WORK + 1) .fetch_all(&mut *transaction) .await?; if rows.len() > MAX_CONTINUATIONS_PER_WORK as usize { diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 45a005c0d92e..adc3fcf7fa22 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -4120,7 +4120,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: Some(active_wait_report.clone()), + mailbox_wait_report: None, }, /*started*/ true, ), @@ -4143,7 +4143,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: Some(completed_wait_report.clone()), + mailbox_wait_report: None, }, /*started*/ false, ), @@ -4166,7 +4166,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: None, + mailbox_wait_report: Some(active_wait_report.clone()), }, /*started*/ true, ), @@ -4191,7 +4191,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: None, + mailbox_wait_report: Some(completed_wait_report.clone()), }, /*started*/ false, ), diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 0864e2dc04d9..527268070485 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -15,6 +15,7 @@ use codex_agent_mailbox::ActivationDisposition; use codex_agent_mailbox::Continuation; use codex_agent_mailbox::ContinuationState; use codex_agent_mailbox::DispatchDecision; +use codex_agent_mailbox::DispatchState; use codex_agent_mailbox::Envelope; use codex_agent_mailbox::GoalBinding; use codex_agent_mailbox::Mailbox; @@ -53,6 +54,46 @@ fn goal_binding_matches_native( && binding.control_generation == snapshot.binding.control_generation } +async fn recover_exact_unadmitted_submission( + mailbox: &Mailbox, + recipient: ThreadId, + submission_id: &str, +) -> CodexResult { + let Some(work) = mailbox + .work_for_submission(recipient, submission_id) + .await + .map_err(storage_error)? + else { + return Ok(false); + }; + let input = mailbox.accepted_input(&work).await.map_err(storage_error)?; + let input_id = input + .id + .as_ref() + .ok_or_else(|| storage_error("accepted mailbox submission has no stable input ID"))? + .to_string(); + if mailbox + .unadmitted_input_ids(recipient, std::slice::from_ref(&input_id)) + .await + .map_err(storage_error)? + .is_empty() + { + return Ok(false); + } + + mailbox + .mark_recovery_needed(&work, RecoveryEvidence::DeliveryUnavailable) + .await + .map_err(storage_error)?; + let dispatch_state = mailbox.dispatch_state(&work).await.map_err(storage_error)?; + if dispatch_state != DispatchState::RecoveryNeeded { + return Err(storage_error( + "exact unadmitted mailbox submission could not enter recovery-needed state", + )); + } + Ok(true) +} + impl AgentControl { pub(crate) fn mailbox_config_compatible(config: &Config) -> bool { config.multi_agent_version_from_features() == MultiAgentVersion::V2 @@ -257,8 +298,27 @@ impl AgentControl { } pub(super) async fn mailbox_dispatched(&self, work: &WorkRef) -> CodexResult<()> { - if let Some(mailbox) = self.mailbox() { - mailbox.mark_dispatched(work).await.map_err(storage_error)?; + if let Some(mailbox) = self.mailbox() + && let Err(mark_error) = mailbox.mark_dispatched(work).await + { + // Native input is already queued. A producer may have durably captured its + // pre-admission failure while this fast sender was finishing; preserve that + // accepted recovery receipt instead of making spawn roll the child back. + match mailbox.dispatch_state(work).await { + Ok(DispatchState::Dispatched | DispatchState::RecoveryNeeded) => return Ok(()), + Ok(state) => { + return Err(storage_error(format!( + "accepted work {} could not be marked dispatched (state {state:?}): {mark_error}", + work.id + ))); + } + Err(state_error) => { + return Err(storage_error(format!( + "accepted work {} could not be marked dispatched and its retained state could not be verified: {mark_error}; {state_error}", + work.id + ))); + } + } } Ok(()) } @@ -594,7 +654,17 @@ impl AgentControl { return Err(storage_error(error)); } }; - Ok(!receipt.no_matching_work) + if receipt.no_matching_work { + match recover_exact_unadmitted_submission(&mailbox, recipient, source_turn_id).await { + Ok(captured) => Ok(captured), + Err(error) => { + mailbox.notify_runtime_fault(recipient); + Err(error) + } + } + } else { + Ok(true) + } } } diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 6b51ee97d746..6e32261e4fc6 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -124,6 +124,42 @@ async fn accepted_goal_work(mailbox: &Mailbox, issuer: ThreadId, agent: ThreadId work } +async fn accepted_unadmitted_work( + mailbox: &Mailbox, + issuer: ThreadId, + agent: ThreadId, + submission_id: &str, +) -> (WorkRef, String) { + let work = mailbox + .accept_work(NativeWork { + native_key: "pre-admission-failure".to_string(), + issuer, + agent, + submission_id: submission_id.to_string(), + input: InterAgentCommunication::new( + AgentPath::root(), + AgentPath::from_string("/root/worker".to_string()).unwrap(), + Vec::new(), + "accepted input not yet admitted to a provider request".to_string(), + /*trigger_turn*/ true, + ), + }) + .await + .unwrap(); + assert_eq!( + mailbox.begin_dispatch(&work).await.unwrap(), + DispatchDecision::Dispatch + ); + let input_id = mailbox + .accepted_input(&work) + .await + .unwrap() + .id + .expect("accepted mailbox input should carry a stable ID") + .to_string(); + (work, input_id) +} + async fn exact_work_snapshot(mailbox: &Mailbox, work: &WorkRef) -> WorkSnapshot { let snapshot = mailbox .snapshot(&Selection { @@ -242,6 +278,107 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a ); } +#[tokio::test] +async fn mailbox_terminal_recovers_exact_unadmitted_submission_idempotently() { + let (session, turn) = crate::session::tests::make_session_and_context().await; + let mut config = (*turn.config).clone(); + let sqlite_home = tempfile::tempdir().unwrap(); + config.sqlite = codex_state::SqliteConfig::new_for_testing(sqlite_home.path().abs()); + config.features.enable(Feature::MultiAgentV2).unwrap(); + config.features.enable(Feature::AgentMailbox).unwrap(); + let control = &session.services.agent_control; + control.mailbox_initialize(&config).await.unwrap(); + let mailbox = control.mailbox().unwrap(); + let agent = ThreadId::new(); + let submission_id = "producer-failed-before-created"; + let (work, input_id) = accepted_unadmitted_work( + &mailbox, + session.thread_id, + agent, + submission_id, + ) + .await; + assert_eq!( + mailbox + .unadmitted_input_ids(agent, std::slice::from_ref(&input_id)) + .await + .unwrap(), + vec![input_id.clone()] + ); + + assert!( + control + .mailbox_record_terminal( + agent, + submission_id, + AgentStatus::Errored("provider request setup failed".to_string()), + None, + None, + ) + .await + .unwrap() + ); + assert_eq!( + mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::RecoveryNeeded + ); + assert!(exact_work_snapshot(&mailbox, &work).await.outcome.is_none()); + assert_eq!( + mailbox + .unadmitted_input_ids(agent, std::slice::from_ref(&input_id)) + .await + .unwrap(), + vec![input_id.clone()], + "recovery capture must not consume or admit the accepted input" + ); + + // A fast sender finishing after producer failure must preserve the truthful receipt. + control.mailbox_dispatched(&work).await.unwrap(); + assert_eq!( + mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::RecoveryNeeded + ); + + assert!( + !control + .mailbox_record_terminal( + agent, + "unrelated-source-turn", + AgentStatus::Errored("unrelated failure".to_string()), + None, + None, + ) + .await + .unwrap(), + "an unrelated source turn must not capture the accepted submission" + ); + assert!( + control + .mailbox_record_terminal( + agent, + submission_id, + AgentStatus::Errored("provider request setup failed".to_string()), + None, + None, + ) + .await + .unwrap(), + "retrying the exact producer terminal must retain the same recovery custody" + ); + assert_eq!( + mailbox.dispatch_state(&work).await.unwrap(), + DispatchState::RecoveryNeeded + ); + assert!(exact_work_snapshot(&mailbox, &work).await.outcome.is_none()); + assert_eq!( + mailbox + .unadmitted_input_ids(agent, std::slice::from_ref(&input_id)) + .await + .unwrap(), + vec![input_id] + ); +} + #[tokio::test] async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_custody() { let (mut session, turn) = crate::session::tests::make_session_and_context().await; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 2892a78cc7ca..54a07b18bd4b 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -15,6 +15,10 @@ use crate::tools::tool_runtime_capabilities::ToolRuntimeCapabilities; use crate::tools::tool_runtime_capabilities::registered_tool_runtime_capabilities; pub const MULTI_AGENT_V1_NAMESPACE: &str = "multi_agent_v1"; +pub(crate) const MAX_MAILBOX_WAIT_SELECTORS: usize = 100; +pub(crate) const MAX_MAILBOX_WAIT_SELECTOR_BYTES: usize = 512; +pub(crate) const MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES: usize = 8_192; +pub(crate) const MAX_MAILBOX_WAIT_REPORT_BYTES: usize = 32_768; const MULTI_AGENT_V1_NAMESPACE_DESCRIPTION: &str = "Tools for spawning and managing sub-agents."; const SPAWN_AGENT_INHERITED_MODEL_GUIDANCE: &str = "Spawned agents inherit your current model by default. Omit `model` to use that preferred default; set `model` only when an explicit override is needed."; @@ -1092,12 +1096,14 @@ fn mailbox_wait_output_schema() -> Value { "properties": { "requestedTargets": { "type": "array", - "items": { "type": "string" }, + "items": { "type": "string", "maxLength": MAX_MAILBOX_WAIT_SELECTOR_BYTES }, + "maxItems": MAX_MAILBOX_WAIT_SELECTORS, "description": "Agent names or exact native work references requested by the wait call." }, "requestedRepliesTo": { "type": "array", - "items": { "type": "string" }, + "items": { "type": "string", "maxLength": MAX_MAILBOX_WAIT_SELECTOR_BYTES }, + "maxItems": MAX_MAILBOX_WAIT_SELECTORS, "description": "Request message ids requested by the wait call." }, "returnWhen": { @@ -1605,9 +1611,11 @@ fn mailbox_wait_agent_tool_parameters() -> JsonSchema { ( "targets".to_string(), JsonSchema::array( - JsonSchema::string(/*description*/ None), + JsonSchema::string(Some(format!( + "Agent name or exact work reference, at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes." + ))), Some( - "Optional names of agents in this tree or exact native work references (for example, `work_...`). Pass multiple targets to wait for whichever completes first unless `return_when` is `all`. Omit to wait only for matching mailbox attention/replies or until `timeout_ms`; without a timeout this wait is unbounded." + "Optional names of agents in this tree or exact native work references (for example, `work_...`). Pass multiple targets to wait for whichever completes first unless `return_when` is `all`. Omit both `targets` and `replies_to` to wait for caller-issued unsurfaced work or new attention. If `replies_to` is provided, omitted `targets` adds no work joins. An explicit empty selection returns `no_matching_work`. Without a timeout this wait is unbounded." .to_string(), ), ), @@ -1615,7 +1623,9 @@ fn mailbox_wait_agent_tool_parameters() -> JsonSchema { ( "replies_to".to_string(), JsonSchema::array( - JsonSchema::string(Some("Request message id.".to_string())), + JsonSchema::string(Some(format!( + "Request message id, at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes." + ))), Some( "Optional request message ids whose intentional final replies should satisfy this wait." .to_string(), @@ -1652,8 +1662,7 @@ fn mailbox_wait_agent_tool_parameters() -> JsonSchema { } fn mailbox_wait_agent_description() -> String { - "Use this for blocking coordination within the current agent tree while awaiting agent, native work, or request-reply completion. Omit targets and `replies_to` to wait for caller-issued unsurfaced work or new attention; such attention, human/tool interrupts, and ordinary request notifications do not satisfy a selected join. `return_when` defaults to `any`; `all` requires every selected join to complete. A selected failure, cancellation, or needs-attention state may return early and is reported as the actual cause. Omit `timeout_ms` for an unbounded event-only wait. Results report requested, completed, and pending joins; no matching work returns an empty result rather than polling." - .to_string() + format!("Use this for blocking coordination within the current agent tree while awaiting agent, native work, or request-reply completion. Omit targets and `replies_to` to wait for caller-issued unsurfaced work or new attention; such attention, human/tool interrupts, and ordinary request notifications do not satisfy a selected join. `return_when` defaults to `any`; `all` requires every selected join to complete. A selected failure, cancellation, or needs-attention state may return early and is reported as the actual cause. Omit `timeout_ms` for an unbounded event-only wait. Results report requested, completed, and pending joins; no matching work returns an empty result rather than polling. At most {MAX_MAILBOX_WAIT_SELECTORS} selectors and resolved joins are accepted, each selector is at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes, and their combined content is at most {MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES} bytes. Reports are limited to {MAX_MAILBOX_WAIT_REPORT_BYTES} serialized bytes; choose fewer exact references if a selection exceeds this bound.") } fn wait_agent_tool_parameters_v2( diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index a3f778521076..08e4a4ca37de 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -8,6 +8,10 @@ use crate::session::input_queue::InputQueueActivity; use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::tools::context::FunctionToolOutput; +use crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_REPORT_BYTES; +use crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_SELECTOR_BYTES; +use crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES; +use crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_SELECTORS; use crate::tools::handlers::multi_agents_spec::create_mailbox_wait_agent_tool; use codex_agent_mailbox::DispatchState; use codex_agent_mailbox::Mailbox; @@ -88,6 +92,10 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } let args: WaitArgs = parse_arguments(&function_arguments(payload)?)?; let _compatibility_only = args.native_event_wait; + let targetless = args.targets.is_none() && args.replies_to.is_none(); + let targets = args.targets.unwrap_or_default(); + let replies_to = args.replies_to.unwrap_or_default(); + validate_wait_selectors(&targets, &replies_to)?; let mailbox = mailbox_for(&session)?; session .services @@ -100,14 +108,6 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, .input_queue .subscribe_activity(/*turn_state*/ None) .await; - let targetless = args.targets.is_none() && args.replies_to.is_none(); - let targets = args.targets.unwrap_or_default(); - let replies_to = args.replies_to.unwrap_or_default(); - if targets.len() + replies_to.len() > 100 { - return Err(mailbox_error( - "at most 100 work/reply selectors may be joined", - )); - } let mut selectors = Vec::new(); let mut agents = Vec::new(); for target in &targets { @@ -130,6 +130,11 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, selectors.extend(replies_to.iter().cloned().map(Selector::Reply)); let mut seen = HashSet::new(); selectors.retain(|selector| seen.insert(selector_key(selector))); + if selectors.len() > MAX_MAILBOX_WAIT_SELECTORS { + return Err(mailbox_error(format!( + "selection expands beyond {MAX_MAILBOX_WAIT_SELECTORS} joins; select fewer exact work references" + ))); + } let return_when = match args.return_when { MailboxWaitReturnWhen::Any => ReturnWhen::Any, MailboxWaitReturnWhen::All => ReturnWhen::All, @@ -483,6 +488,41 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, ))) } +fn validate_wait_selectors( + targets: &[String], + replies_to: &[String], +) -> Result<(), FunctionCallError> { + if targets.len() + replies_to.len() > MAX_MAILBOX_WAIT_SELECTORS { + return Err(mailbox_error(format!( + "at most {MAX_MAILBOX_WAIT_SELECTORS} work/reply selectors may be joined" + ))); + } + let mut total_bytes = 0; + for selector in targets.iter().chain(replies_to) { + if selector.len() > MAX_MAILBOX_WAIT_SELECTOR_BYTES { + return Err(mailbox_error(format!( + "each work/reply selector must be at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes" + ))); + } + total_bytes += selector.len(); + } + if total_bytes > MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES { + return Err(mailbox_error(format!( + "combined work/reply selectors must be at most {MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES} UTF-8 bytes" + ))); + } + Ok(()) +} + +fn validate_wait_report(report: &MailboxWaitReport) -> Result<(), FunctionCallError> { + if serde_json::to_vec(report).map_err(mailbox_error)?.len() > MAX_MAILBOX_WAIT_REPORT_BYTES { + return Err(mailbox_error(format!( + "resolved wait report exceeds {MAX_MAILBOX_WAIT_REPORT_BYTES} bytes; select fewer exact work/reply references" + ))); + } + Ok(()) +} + fn selector_key(selector: &Selector) -> String { match selector { Selector::Work(id) => format!("work:{id}"), @@ -676,6 +716,7 @@ async fn emit_report( report: &MailboxWaitReport, status: CollabAgentToolCallStatus, ) -> Result<(), FunctionCallError> { + validate_wait_report(report)?; let item = TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { id: call_id.to_string(), tool: CollabAgentTool::Wait, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index dbccaed09bd0..6d75aa00b57f 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -25,6 +25,44 @@ use std::sync::Arc; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; +#[test] +fn mailbox_wait_selectors_bound_utf8_count_and_combined_content() { + let at_limit = "x".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES); + assert!( + validate_wait_selectors(&vec![at_limit.clone(); 8], &vec![at_limit.clone(); 8]).is_ok() + ); + assert!(validate_wait_selectors(&vec![at_limit.clone(); 8], &vec![at_limit; 9]).is_err()); + assert!( + validate_wait_selectors(&vec![String::new(); MAX_MAILBOX_WAIT_SELECTORS + 1], &[]).is_err() + ); + assert!( + validate_wait_selectors(&["x".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES + 1)], &[]).is_err() + ); + assert!( + validate_wait_selectors(&[], &["é".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES / 2 + 1)]).is_err() + ); +} + +#[test] +fn mailbox_wait_report_bounds_escaped_selector_echoes() { + let mut report = MailboxWaitReport { + requested_targets: Vec::new(), + requested_replies_to: Vec::new(), + return_when: MailboxWaitReturnWhen::Any, + completed: Vec::new(), + pending: Vec::new(), + completion_reason: None, + condition_satisfied: false, + quiet_count: 0, + wake_sender: None, + wake_ref: None, + }; + assert!(validate_wait_report(&report).is_ok()); + report.requested_targets = vec!["\0".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES); 16]; + assert!(validate_wait_selectors(&report.requested_targets, &[]).is_ok()); + assert!(validate_wait_report(&report).is_err()); +} + async fn fixture_with_path() -> ( Arc, Arc, diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 87aa5b89b3d2..dcb7534b552e 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -1272,6 +1272,7 @@ mod tests { let runtime = ToolCallRuntime::new(router, Arc::clone(&session), step_context, tracker); let thread_ids_before_rejection = manager.list_thread_ids().await; let rejected_response = runtime + .clone() .handle_tool_call( ToolCall { tool_name: ToolName::namespaced("profile_agents", "spawn_agent"), @@ -1501,7 +1502,10 @@ mod tests { }; let retry_result: serde_json::Value = serde_json::from_str(&retry_message)?; assert_eq!(retry_result["agent_id"].as_str(), Some(first_agent_id)); - assert_eq!(retry_result["dispatch_state"], first_result["dispatch_state"]); + assert_eq!( + retry_result["dispatch_state"], + first_result["dispatch_state"] + ); assert_eq!( retry_result["work_ref"].as_str(), Some(accepted_work_id.as_str()), diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 9dddc29af900..6e9f6cf0f8a8 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -268,6 +268,20 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { assert!(wait_properties.contains_key("timeout_ms")); assert!(wait_properties.contains_key("native_event_wait")); assert!(wait_agent.parameters.required.is_none()); + assert!( + wait_properties["targets"] + .description + .as_deref() + .expect("target selection description") + .contains("Omit both `targets` and `replies_to`") + ); + assert!( + wait_properties["targets"] + .description + .as_deref() + .expect("target selection description") + .contains("caller-issued unsurfaced work or new attention") + ); assert_eq!( wait_properties["return_when"].enum_values.as_ref(), Some(&vec![json!("any"), json!("all")]) @@ -289,22 +303,32 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { .output_schema .as_ref() .expect("wait_agent output schema"); + for selector_list in ["requestedTargets", "requestedRepliesTo"] { + assert_eq!( + wait_output["properties"][selector_list]["maxItems"], + json!(crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_SELECTORS) + ); + assert_eq!( + wait_output["properties"][selector_list]["items"]["maxLength"], + json!(crate::tools::handlers::multi_agents_spec::MAX_MAILBOX_WAIT_SELECTOR_BYTES) + ); + } assert!( wait_output["required"] .as_array() .expect("wait result required fields") - .contains(&json!("return_when")) + .contains(&json!("returnWhen")) ); assert_eq!( - wait_output["properties"]["return_when"]["enum"], + wait_output["properties"]["returnWhen"]["enum"], json!(["any", "all"]) ); assert!( wait_output["properties"] - .get("condition_satisfied") + .get("conditionSatisfied") .is_some() ); - assert!(wait_output["properties"].get("quiet_count").is_some()); + assert!(wait_output["properties"].get("quietCount").is_some()); assert_eq!( wait_output["properties"]["completed"]["items"]["required"], json!(["status"]) @@ -314,7 +338,7 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { json!(["status"]) ); for result_list in ["completed", "pending"] { - for field in ["work_ref", "message_id", "agent", "status"] { + for field in ["workRef", "messageId", "agent", "status"] { assert!( wait_output["properties"][result_list]["items"]["properties"] .get(field) @@ -323,7 +347,7 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { } } assert_eq!( - wait_output["properties"]["completion_reason"]["enum"], + wait_output["properties"]["completionReason"]["enum"], json!([ "result", "reply", @@ -332,6 +356,7 @@ fn assert_mailbox_namespace_schema(plan: &ToolPlanProbe, namespace_name: &str) { "tool", "deadline", "cancel", + "interrupted", "subscription_loss", "runtime_fault", "no_matching_work", diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index b5831142a090..71cf56fbbe69 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -1213,18 +1213,21 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com assert_eq!( requests.len(), 6, - "recovery should read the request then execute once" + "recovery should execute the pending input, then read it once" + ); + let automatically_delivered = agent_message_input(&requests[4], message_id) + .expect("the next ordinary provider request should receive the pending canonical input"); + assert_eq!( + automatically_delivered["content"][1]["encrypted_content"], + CUSTODY_BODY ); let recovery_read_receipt: Value = serde_json::from_str( function_output_text(&requests[5], "recovery-read-call") .expect("recovery mailbox_read receipt should remain in provider history"), )?; - assert_eq!( - recovery_read_receipt["messages"][0]["message_id"], - message_id - ); + assert_eq!(recovery_read_receipt["messages"], json!([])); let recovered_item = agent_message_input(&requests[5], message_id) - .expect("recovery read must restore the exact canonical body to execution input"); + .expect("the admitted canonical body should remain in provider history"); assert_eq!(recovered_item["content"][1]["type"], "encrypted_content"); assert_eq!( recovered_item["content"][1]["encrypted_content"], From 99ac7f50dea9bd7167082b341348fd70235b99c6 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:38:40 +1100 Subject: [PATCH 20/54] fix(build): retain pinned zlib downloads on Ubuntu snapshots --- MODULE.bazel | 1 + patches/BUILD.bazel | 1 + patches/rules_rs_zlib_snapshot_urls.patch | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 patches/rules_rs_zlib_snapshot_urls.patch diff --git a/MODULE.bazel b/MODULE.bazel index da0bd8c28469..5c6e3e591c80 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -97,6 +97,7 @@ single_version_override( patches = [ "//patches:rules_rs_build_script_deps_annotation.patch", "//patches:rules_rs_windows_gnullvm_exec_toolchain.patch", + "//patches:rules_rs_zlib_snapshot_urls.patch", ], version = "0.0.96", ) diff --git a/patches/BUILD.bazel b/patches/BUILD.bazel index 1909fed6cf65..4545a709ce43 100644 --- a/patches/BUILD.bazel +++ b/patches/BUILD.bazel @@ -10,6 +10,7 @@ exports_files([ "rules_cc_rusty_v8_custom_libcxx.patch", "rules_rs_build_script_deps_annotation.patch", "rules_rs_windows_gnullvm_exec_toolchain.patch", + "rules_rs_zlib_snapshot_urls.patch", "rusty_v8_prebuilt_out_dir.patch", "ring_windows_msvc_include_dirs.patch", "v8_bazel_rules.patch", diff --git a/patches/rules_rs_zlib_snapshot_urls.patch b/patches/rules_rs_zlib_snapshot_urls.patch new file mode 100644 index 000000000000..53274f7af26b --- /dev/null +++ b/patches/rules_rs_zlib_snapshot_urls.patch @@ -0,0 +1,23 @@ +# What: retain the pinned Linux compiler runtime libraries on immutable Ubuntu snapshots. +# Why: the moving Ubuntu mirrors remove superseded packages and return 404. +# Scope: URLs only; retain the existing package versions and SHA-256 verification. +# Source: hermeticbuild/rules_rs@81336041f1c092b94ba0d35384c4dfc03ce45a07. + +diff --git a/rs/private/rustc_repository.bzl b/rs/private/rustc_repository.bzl +--- a/rs/private/rustc_repository.bzl ++++ b/rs/private/rustc_repository.bzl +@@ -11,12 +11,12 @@ _LINUX_ZLIB = { + "aarch64": struct( + libdir = "usr/lib/aarch64-linux-gnu", + sha256 = "cbe3d39ec32d3cc27c021ae4af11e7c67bdf9d700d573207e0941d4038056278", +- url = "https://ports.ubuntu.com/ubuntu-ports/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_arm64.deb", ++ url = "https://snapshot.ubuntu.com/ubuntu/20260801T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_arm64.deb", + ), + "x86_64": struct( + libdir = "usr/lib/x86_64-linux-gnu", + sha256 = "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", +- url = "https://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb", ++ url = "https://snapshot.ubuntu.com/ubuntu/20260801T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb", + ), + } + From 60f8730e2be169f934f6666be14df6fa606ce187 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:10:06 +1100 Subject: [PATCH 21/54] fix(agents): retain mailbox wait terminals across legacy history --- .github/scripts/test_ci_planners.py | 19 +++ .github/workflows/bazel.yml | 4 + codex-rs/agent-mailbox/src/messages.rs | 24 +-- codex-rs/agent-mailbox/src/outcome.rs | 4 +- codex-rs/agent-mailbox/src/wait_graph.rs | 7 +- codex-rs/agent-mailbox/src/work.rs | 16 +- .../analytics/src/analytics_client_tests.rs | 1 + codex-rs/core/src/agent/control/lifecycle.rs | 2 +- codex-rs/core/src/agent/control/mailbox.rs | 8 +- .../core/src/agent/control/mailbox_tests.rs | 9 +- codex-rs/core/src/session/handlers.rs | 4 +- codex-rs/core/src/session/mod.rs | 6 +- .../src/tools/handlers/multi_agents_spec.rs | 4 +- .../handlers/multi_agents_v2/mailbox_wait.rs | 9 +- .../multi_agents_v2/mailbox_wait_tests.rs | 3 +- codex-rs/rollout/src/policy.rs | 12 ++ codex-rs/rollout/src/policy_tests.rs | 137 ++++++++++++++---- 17 files changed, 193 insertions(+), 76 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 3f18a04c8021..0d707250b9dd 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1598,6 +1598,25 @@ def test_bazel_macos_clippy_caps_hosted_runner_fanout(self) -> None: self.assertIn("--jobs=96", clippy_run) self.assertIn("--loading_phase_threads=8", clippy_run) + def test_bazel_windows_clippy_fork_uses_native_host(self) -> None: + payload = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") + clippy_steps = ((payload.get("jobs") or {}).get("clippy") or {}).get("steps") or [] + clippy_run = next( + step + for step in clippy_steps + if step.get("name") == "bazel build --config=clippy lint targets" + ).get("run") or "" + fork_no_key_condition = ( + '[[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" ' + '&& -z "${BUILDBUDDY_API_KEY:-}" ]]' + ) + self.assertIn('[[ "${RUNNER_OS}" == "Windows" ]]', clippy_run) + self.assertIn(fork_no_key_condition, clippy_run) + self.assertIn( + "bazel_clippy_args+=(--host_platform=//:local_windows)", clippy_run + ) + self.assertEqual(clippy_run.count("--host_platform=//:local_windows"), 1) + def test_bazel_windows_tests_serialize_host_global_policy_state(self) -> None: payload = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") windows_steps = ( diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6baf9ca884c0..651155b759bf 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -479,6 +479,10 @@ jobs: # the old local-fallback behavior there. bazel_clippy_args+=(--skip_incompatible_explicit_targets) fi + if [[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + # Match native Rust execution tools to the fork's gnullvm C++ host. + bazel_clippy_args+=(--host_platform=//:local_windows) + fi fi bazel_target_lines="$(./scripts/list-bazel-clippy-targets.sh "${bazel_target_list_args[@]}")" diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index 1a5b9b6d4b41..bc85500fb572 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -152,7 +152,7 @@ pub(crate) async fn insert_message_tx( && !draft.activate_if_idle && let Some(work_ref) = draft.work_ref.as_ref() && let Some(existing) = sqlx::query( - "SELECT * FROM mailbox_messages + "SELECT * FROM mailbox_messages WHERE sender = ? AND recipient = ? AND work_ref_json = ? AND author_path = ? AND recipient_path = ? AND kind = '\"progress\"' AND canonical = 0 AND input_admitted = 0 @@ -163,15 +163,15 @@ pub(crate) async fn insert_message_tx( WHERE newer.supersedes = mailbox_messages.id ) ORDER BY seq DESC LIMIT 1", - ) - .bind(draft.sender.to_string()) - .bind(draft.recipient.to_string()) - .bind(serde_json::to_string(work_ref)?) - .bind(serde_json::to_string(&draft.communication.author)?) - .bind(serde_json::to_string(&draft.communication.recipient)?) - .bind(i64::from(draft.legacy_attention)) - .fetch_optional(&mut **transaction) - .await? + ) + .bind(draft.sender.to_string()) + .bind(draft.recipient.to_string()) + .bind(serde_json::to_string(work_ref)?) + .bind(serde_json::to_string(&draft.communication.author)?) + .bind(serde_json::to_string(&draft.communication.recipient)?) + .bind(i64::from(draft.legacy_attention)) + .fetch_optional(&mut **transaction) + .await? { // Replacement gets its own stable ID. Never rewrite the body or // idempotency binding behind the previously accepted identifier. @@ -270,7 +270,9 @@ pub(crate) async fn insert_message_tx( .bind(old_id) .execute(&mut **transaction) .await?; - if replaced.rows_affected() == 1 && let Some(old) = old { + if replaced.rows_affected() == 1 + && let Some(old) = old + { let unread = old.try_get::("canonical")? == 0 || old.try_get::("input_admitted")? == 0; if unread { diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index 5fa97e9a19c4..2a839e9ba6cf 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -145,9 +145,7 @@ impl Mailbox { validate_auto_response_size(&groups)?; if is_nonterminal(&continuation) { let binding = continuation_binding(&continuation).ok_or_else(|| { - MailboxError::InvalidInput( - "nonterminal continuation is missing its binding".into(), - ) + MailboxError::InvalidInput("nonterminal continuation is missing its binding".into()) })?; let binding_json = serde_json::to_string(binding)?; for work_ref in &refs { diff --git a/codex-rs/agent-mailbox/src/wait_graph.rs b/codex-rs/agent-mailbox/src/wait_graph.rs index 4d8b26ee1875..43ed59a5260a 100644 --- a/codex-rs/agent-mailbox/src/wait_graph.rs +++ b/codex-rs/agent-mailbox/src/wait_graph.rs @@ -111,12 +111,7 @@ impl Mailbox { has_escape: bool, ) -> MailboxResult { self.begin_wait_inner( - wait_id, - owner, - peers, - mode, - has_escape, - /*input_generation*/ None, + wait_id, owner, peers, mode, has_escape, /*input_generation*/ None, ) } diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index f727c4ac9c0a..a1aa01c478f1 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -64,14 +64,14 @@ impl Mailbox { if !work.input.id.as_ref().is_some_and(is_mailbox_id) { work.input.id = Some(codex_protocol::ResponseItemId::from_server(new_mailbox_id())); } - let input_message_id = work - .input - .id - .as_ref() - .map(ToString::to_string) - .ok_or_else(|| { - MailboxError::InvalidInput("accepted native input ID is missing".into()) - })?; + let input_message_id = + work.input + .id + .as_ref() + .map(ToString::to_string) + .ok_or_else(|| { + MailboxError::InvalidInput("accepted native input ID is missing".into()) + })?; let input_json = serde_json::to_string(&work.input)?; let mut transaction = self.pool.begin().await?; diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index be716e005218..1a46f939abd6 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -4807,6 +4807,7 @@ async fn turn_event_counts_completed_tool_items() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, ThreadItem::SubAgentActivity { id: "sub-agent-activity-1".to_string(), diff --git a/codex-rs/core/src/agent/control/lifecycle.rs b/codex-rs/core/src/agent/control/lifecycle.rs index 5f7144de7bd2..74198528bbab 100644 --- a/codex-rs/core/src/agent/control/lifecycle.rs +++ b/codex-rs/core/src/agent/control/lifecycle.rs @@ -376,7 +376,7 @@ impl PreparedV2AgentDelivery { ))); } let submission = self - .record_submission(&mut communication, &context, None) + .record_submission(&mut communication, &context, /*recipient_config*/ None) .await?; match submission.dispatch { DispatchDecision::AlreadyDispatched => return Ok(submission.id), diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 527268070485..29c24618d1bd 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -170,7 +170,7 @@ impl AgentControl { // Indexed bounded batches, scoped to the exact native session being resumed. // Other loaded issuers and producer continuation custody are untouched. while mailbox - .interrupt_issuer_dispatches(issuer, 100) + .interrupt_issuer_dispatches(issuer, /*limit*/ 100) .await .map_err(storage_error)? != 0 @@ -428,7 +428,7 @@ impl AgentControl { let mut cursor = None; loop { let page = mailbox - .pending_page(recipient, cursor, 100) + .pending_page(recipient, cursor, /*limit*/ 100) .await .map_err(storage_error)?; for envelope in &page.items { @@ -461,7 +461,7 @@ impl AgentControl { return Ok(Vec::new()); }; let unread = mailbox - .unadmitted_canonical_ids(recipient, 4096) + .unadmitted_canonical_ids(recipient, /*limit*/ 4096) .await .map_err(storage_error)? .into_iter() @@ -480,7 +480,7 @@ impl AgentControl { return Ok(Vec::new()); }; Ok(mailbox - .automatic_inputs(recipient, source_turn_id, 2) + .automatic_inputs(recipient, source_turn_id, /*limit*/ 2) .await .map_err(storage_error)? .into_iter() diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index 6e32261e4fc6..e6c7e82dc97e 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -291,13 +291,8 @@ async fn mailbox_terminal_recovers_exact_unadmitted_submission_idempotently() { let mailbox = control.mailbox().unwrap(); let agent = ThreadId::new(); let submission_id = "producer-failed-before-created"; - let (work, input_id) = accepted_unadmitted_work( - &mailbox, - session.thread_id, - agent, - submission_id, - ) - .await; + let (work, input_id) = + accepted_unadmitted_work(&mailbox, session.thread_id, agent, submission_id).await; assert_eq!( mailbox .unadmitted_input_ids(agent, std::slice::from_ref(&input_id)) diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 81a983c4b635..81504c3cdb83 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -579,7 +579,7 @@ pub async fn thread_rollback(sess: &Arc, sub_id: String, num_turns: u32 id: turn_context.sub_id.clone(), msg: rollback_msg, }, - None, + /*status_override*/ None, ) .await; } @@ -691,7 +691,7 @@ pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { sess.services .rollout_thread_trace .record_protocol_event(&event.msg); - sess.deliver_event_raw(event, None).await; + sess.deliver_event_raw(event, /*status_override*/ None).await; sess.services .rollout_thread_trace .record_ended(codex_rollout_trace::RolloutStatus::Completed); diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 1aa2ef5ce43c..9a15312bc714 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -2683,8 +2683,10 @@ impl Session { } async fn send_event_raw_with_persistence(&self, event: Event, persist: bool) { - self.send_event_raw_with_persistence_and_status_override(event, persist, None) - .await; + self.send_event_raw_with_persistence_and_status_override( + event, persist, /*status_override*/ None, + ) + .await; } async fn send_event_raw_with_persistence_and_status_override( diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 54a07b18bd4b..d2d8f14888f2 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -1662,7 +1662,9 @@ fn mailbox_wait_agent_tool_parameters() -> JsonSchema { } fn mailbox_wait_agent_description() -> String { - format!("Use this for blocking coordination within the current agent tree while awaiting agent, native work, or request-reply completion. Omit targets and `replies_to` to wait for caller-issued unsurfaced work or new attention; such attention, human/tool interrupts, and ordinary request notifications do not satisfy a selected join. `return_when` defaults to `any`; `all` requires every selected join to complete. A selected failure, cancellation, or needs-attention state may return early and is reported as the actual cause. Omit `timeout_ms` for an unbounded event-only wait. Results report requested, completed, and pending joins; no matching work returns an empty result rather than polling. At most {MAX_MAILBOX_WAIT_SELECTORS} selectors and resolved joins are accepted, each selector is at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes, and their combined content is at most {MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES} bytes. Reports are limited to {MAX_MAILBOX_WAIT_REPORT_BYTES} serialized bytes; choose fewer exact references if a selection exceeds this bound.") + format!( + "Use this for blocking coordination within the current agent tree while awaiting agent, native work, or request-reply completion. Omit targets and `replies_to` to wait for caller-issued unsurfaced work or new attention; such attention, human/tool interrupts, and ordinary request notifications do not satisfy a selected join. `return_when` defaults to `any`; `all` requires every selected join to complete. A selected failure, cancellation, or needs-attention state may return early and is reported as the actual cause. Omit `timeout_ms` for an unbounded event-only wait. Results report requested, completed, and pending joins; no matching work returns an empty result rather than polling. At most {MAX_MAILBOX_WAIT_SELECTORS} selectors and resolved joins are accepted, each selector is at most {MAX_MAILBOX_WAIT_SELECTOR_BYTES} UTF-8 bytes, and their combined content is at most {MAX_MAILBOX_WAIT_SELECTOR_TOTAL_BYTES} bytes. Reports are limited to {MAX_MAILBOX_WAIT_REPORT_BYTES} serialized bytes; choose fewer exact references if a selection exceeds this bound." + ) } fn wait_agent_tool_parameters_v2( diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 08e4a4ca37de..6cb788eee3ea 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -119,7 +119,7 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, agents.push(agent); selectors.extend( mailbox - .issued_work_refs(session.thread_id, Some(agent), 100) + .issued_work_refs(session.thread_id, Some(agent), /*limit*/ 100) .await .map_err(mailbox_error)? .into_iter() @@ -183,7 +183,10 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, drop(registration.take()); let input_generation = mailbox.wait_input_generation(); if targetless { - match mailbox.issued_work_refs(session.thread_id, None, 100).await { + match mailbox + .issued_work_refs(session.thread_id, /*agent*/ None, /*limit*/ 100) + .await + { Ok(work) => { selectors = work .into_iter() @@ -328,7 +331,7 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, } // An addressed request/alert interrupts a join but never satisfies it. Its body is // admitted before its one-shot notification is observed; unanswered obligation remains. - let attention = match mailbox.attention(session.thread_id, 100).await { + let attention = match mailbox.attention(session.thread_id, /*limit*/ 100).await { Ok(attention) => attention, Err(_) => { report.completion_reason = Some(MailboxWakeCause::RuntimeFault); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index 6d75aa00b57f..e8a3c1694989 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -39,7 +39,8 @@ fn mailbox_wait_selectors_bound_utf8_count_and_combined_content() { validate_wait_selectors(&["x".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES + 1)], &[]).is_err() ); assert!( - validate_wait_selectors(&[], &["é".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES / 2 + 1)]).is_err() + validate_wait_selectors(&[], &["é".repeat(MAX_MAILBOX_WAIT_SELECTOR_BYTES / 2 + 1)]) + .is_err() ); } diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index 4a6949824db7..65a556735133 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -91,7 +91,19 @@ pub fn should_persist_event_msg(ev: &EventMsg, history_mode: ThreadHistoryMode) EventMsg::ItemCompleted(event) => { // Paginated rollouts store TurnItems. // Legacy rollouts keep only items with no raw ResponseItem or legacy equivalent. + // Typed mailbox terminal reports must survive legacy rollout filtering. matches!(history_mode, ThreadHistoryMode::Paginated) + || matches!( + &event.item, + TurnItem::CollabAgentToolCall(call) + if call.tool == CollabAgentTool::Wait + && matches!( + call.status, + CollabAgentToolCallStatus::Completed + | CollabAgentToolCallStatus::Failed + ) + && call.mailbox_wait_report.is_some() + ) || matches!( event.item, TurnItem::Plan(_) | TurnItem::Extension(ExtensionItem::Sleep(_)) diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs index 3cc3b4c552bb..1cd2772e4bec 100644 --- a/codex-rs/rollout/src/policy_tests.rs +++ b/codex-rs/rollout/src/policy_tests.rs @@ -5,8 +5,10 @@ use codex_protocol::items::CollabAgentTool; use codex_protocol::items::CollabAgentToolCallItem; use codex_protocol::items::CollabAgentToolCallStatus; use codex_protocol::items::MailboxWaitReport; +use codex_protocol::items::MailboxWakeCause; use codex_protocol::items::TurnItem; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::ItemStartedEvent; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::ThreadHistoryMode; @@ -21,28 +23,49 @@ fn started_call( RolloutItem::EventMsg(EventMsg::ItemStarted(ItemStartedEvent { thread_id: ThreadId::new(), turn_id: "turn".to_string(), - item: TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { - id: "wait-call".to_string(), - tool, - status, - sender_thread_id: ThreadId::new(), - receiver_thread_ids: Vec::new(), - receiver_agents: Vec::new(), - prompt: None, - model: None, - reasoning_effort: None, - requested_model: None, - requested_reasoning_effort: None, - agents_states: HashMap::new(), - wake_notifications: None, - completion_reason: None, - wake_cause: None, - mailbox_wait_report: report, - }), + item: collab_call_item(tool, status, report), started_at_ms: 1, })) } +fn completed_call( + tool: CollabAgentTool, + status: CollabAgentToolCallStatus, + report: Option, +) -> RolloutItem { + RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: ThreadId::new(), + turn_id: "turn".to_string(), + item: collab_call_item(tool, status, report), + completed_at_ms: 2, + })) +} + +fn collab_call_item( + tool: CollabAgentTool, + status: CollabAgentToolCallStatus, + report: Option, +) -> TurnItem { + TurnItem::CollabAgentToolCall(CollabAgentToolCallItem { + id: "wait-call".to_string(), + tool, + status, + sender_thread_id: ThreadId::new(), + receiver_thread_ids: Vec::new(), + receiver_agents: Vec::new(), + prompt: None, + model: None, + reasoning_effort: None, + requested_model: None, + requested_reasoning_effort: None, + agents_states: HashMap::new(), + wake_notifications: None, + completion_reason: None, + wake_cause: None, + mailbox_wait_report: report, + }) +} + fn started_wait( status: CollabAgentToolCallStatus, report: Option, @@ -50,6 +73,13 @@ fn started_wait( started_call(CollabAgentTool::Wait, status, report) } +fn completed_wait( + status: CollabAgentToolCallStatus, + report: Option, +) -> RolloutItem { + completed_call(CollabAgentTool::Wait, status, report) +} + fn in_progress_report() -> MailboxWaitReport { MailboxWaitReport { requested_targets: Vec::new(), @@ -65,27 +95,80 @@ fn in_progress_report() -> MailboxWaitReport { } } +fn terminal_report(cause: MailboxWakeCause, condition_satisfied: bool) -> MailboxWaitReport { + MailboxWaitReport { + completion_reason: Some(cause), + condition_satisfied, + ..in_progress_report() + } +} + #[test] -fn only_typed_in_progress_mailbox_wait_starts_are_persisted() { - let valid = started_wait( +fn typed_mailbox_wait_lifecycle_is_persisted_across_history_modes() { + let start = started_wait( CollabAgentToolCallStatus::InProgress, Some(in_progress_report()), ); - let no_report = started_wait(CollabAgentToolCallStatus::InProgress, None); - let terminal = started_wait( + let start_without_report = started_wait(CollabAgentToolCallStatus::InProgress, None); + let non_in_progress_start = started_wait( CollabAgentToolCallStatus::Completed, Some(in_progress_report()), ); - let other_tool = started_call( + let other_tool_start = started_call( CollabAgentTool::SendInput, CollabAgentToolCallStatus::InProgress, Some(in_progress_report()), ); + let completed = completed_wait( + CollabAgentToolCallStatus::Completed, + Some(terminal_report(MailboxWakeCause::Result, true)), + ); + let failed = completed_wait( + CollabAgentToolCallStatus::Failed, + Some(terminal_report(MailboxWakeCause::Interrupted, false)), + ); + let nonterminal_completion = completed_wait( + CollabAgentToolCallStatus::InProgress, + Some(in_progress_report()), + ); + let terminal_without_report = completed_wait(CollabAgentToolCallStatus::Failed, None); + let other_tool_terminal = completed_call( + CollabAgentTool::SendInput, + CollabAgentToolCallStatus::Failed, + Some(in_progress_report()), + ); for mode in [ThreadHistoryMode::Legacy, ThreadHistoryMode::Paginated] { - assert!(is_persisted_rollout_item(&valid, mode)); - assert!(!is_persisted_rollout_item(&no_report, mode)); - assert!(!is_persisted_rollout_item(&terminal, mode)); - assert!(!is_persisted_rollout_item(&other_tool, mode)); + assert!(is_persisted_rollout_item(&start, mode)); + assert!(!is_persisted_rollout_item(&start_without_report, mode)); + assert!(!is_persisted_rollout_item(&non_in_progress_start, mode)); + assert!(!is_persisted_rollout_item(&other_tool_start, mode)); + assert!(is_persisted_rollout_item(&completed, mode)); + assert!(is_persisted_rollout_item(&failed, mode)); } + + assert!(!is_persisted_rollout_item( + &other_tool_terminal, + ThreadHistoryMode::Legacy + )); + assert!(!is_persisted_rollout_item( + &nonterminal_completion, + ThreadHistoryMode::Legacy + )); + assert!(is_persisted_rollout_item( + &other_tool_terminal, + ThreadHistoryMode::Paginated + )); + assert!(is_persisted_rollout_item( + &nonterminal_completion, + ThreadHistoryMode::Paginated + )); + assert!(!is_persisted_rollout_item( + &terminal_without_report, + ThreadHistoryMode::Legacy + )); + assert!(is_persisted_rollout_item( + &terminal_without_report, + ThreadHistoryMode::Paginated + )); } From 38beece879b5bf39fab97cd4892923f2cd65e88e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:16:02 +1100 Subject: [PATCH 22/54] style(agents): apply hosted await formatting --- codex-rs/core/src/session/handlers.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 81504c3cdb83..0b2c70aa6099 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -691,7 +691,8 @@ pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { sess.services .rollout_thread_trace .record_protocol_event(&event.msg); - sess.deliver_event_raw(event, /*status_override*/ None).await; + sess.deliver_event_raw(event, /*status_override*/ None) + .await; sess.services .rollout_thread_trace .record_ended(codex_rollout_trace::RolloutStatus::Completed); From 088400bc88061ea02cf9ff8e8a3f08dc9b85045d Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:32:06 +1100 Subject: [PATCH 23/54] fix(mailbox): resolve protected lint and hosted Windows failures --- .github/scripts/test_ci_planners.py | 8 +++++ .github/workflows/bazel.yml | 4 +++ codex-rs/agent-mailbox/src/mailbox_tests.rs | 8 ++--- codex-rs/agent-mailbox/src/store.rs | 5 --- .../src/protocol/thread_history.rs | 2 +- codex-rs/core/src/agent/control/mailbox.rs | 2 +- codex-rs/core/src/session/mod.rs | 35 ++++++++++++------- .../handlers/multi_agents_v2/mailbox_wait.rs | 11 +++--- codex-rs/protocol/src/items.rs | 2 +- codex-rs/rollout/src/policy_tests.rs | 16 ++++++--- 10 files changed, 58 insertions(+), 35 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 0d707250b9dd..8ece13a8b412 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1627,6 +1627,14 @@ def test_bazel_windows_tests_serialize_host_global_policy_state(self) -> None: step for step in windows_steps if step.get("name") == "bazel test shard" ).get("run") or "" self.assertIn("--local_test_jobs=1", windows_test_run) + self.assertIn( + '[[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" ' + '&& -z "${BUILDBUDDY_API_KEY:-}" ]]', + windows_test_run, + ) + self.assertIn( + "bazel_test_args+=(--host_platform=//:local_windows)", windows_test_run + ) def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> None: bazel = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 651155b759bf..3ac36d27a51f 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -369,6 +369,10 @@ jobs: --build_metadata=COMMIT_SHA=${GITHUB_SHA} --build_metadata=TAG_windows_test_shard=${BAZEL_TEST_SHARD} ) + if [[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + # Use the same native gnullvm host platform as fork clippy. + bazel_test_args+=(--host_platform=//:local_windows) + fi ./.github/scripts/run-bazel-ci.sh \ --print-failed-action-summary \ diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 6229078ac4dc..a3db02a8e0f6 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -405,18 +405,18 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "b", "B").await; let (c, _id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; assert_eq!( - admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]) + admit(&fixture.mailbox, agent, "turn", std::slice::from_ref(&id_a)) .await .work_refs, vec![a.clone()] ); assert!( - admit(&fixture.mailbox, agent, "turn", &[id_a.clone()]) + admit(&fixture.mailbox, agent, "turn", std::slice::from_ref(&id_a)) .await .already_recorded ); assert_eq!( - admit(&fixture.mailbox, agent, "turn", &[id_b.clone()]) + admit(&fixture.mailbox, agent, "turn", std::slice::from_ref(&id_b)) .await .work_refs, vec![b.clone()] @@ -424,7 +424,7 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped assert!(matches!( fixture .mailbox - .record_admission(agent, "other-turn", &[id_a.clone()]) + .record_admission(agent, "other-turn", std::slice::from_ref(&id_a)) .await, Err(MailboxError::IdempotencyConflict) )); diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index d671fde918eb..400710c6723e 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -53,9 +53,4 @@ impl Mailbox { *generation = generation.wrapping_add(1); }); } - - #[cfg(test)] - pub(crate) fn pool(&self) -> &SqlitePool { - &self.pool - } } diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index adc3fcf7fa22..d9f6a5c061be 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -4166,7 +4166,7 @@ mod tests { wake_notifications: None, completion_reason: None, wake_cause: None, - mailbox_wait_report: Some(active_wait_report.clone()), + mailbox_wait_report: Some(active_wait_report), }, /*started*/ true, ), diff --git a/codex-rs/core/src/agent/control/mailbox.rs b/codex-rs/core/src/agent/control/mailbox.rs index 29c24618d1bd..63a983014714 100644 --- a/codex-rs/core/src/agent/control/mailbox.rs +++ b/codex-rs/core/src/agent/control/mailbox.rs @@ -346,7 +346,7 @@ impl AgentControl { if metadata .agent_path .as_ref() - .is_none_or(|path| path.is_root()) + .is_none_or(codex_protocol::AgentPath::is_root) { return Ok(denied("root_activation_is_not_allowed")); } diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 9a15312bc714..749215c88200 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1565,7 +1565,11 @@ impl Session { let mut report = call .mailbox_wait_report .clone() - .expect("mailbox wait states always carry their report"); + .ok_or_else(|| { + CodexErr::Fatal( + "stored mailbox wait has no structured report".to_string(), + ) + })?; if !already_completed { report.completion_reason = Some(MailboxWakeCause::Interrupted); report.condition_satisfied = false; @@ -2388,12 +2392,7 @@ impl Session { } status } - None => { - let Some(status) = agent_status_from_event(msg) else { - return None; - }; - status - } + None => agent_status_from_event(msg)?, }; let mut mailbox_captured = false; @@ -3843,7 +3842,7 @@ impl Session { let response_item_id = response_item .id() .cloned() - .expect("prepared agent message should have a stable ID"); + .ok_or_else(|| std::io::Error::other("prepared agent message has no stable ID"))?; let rollout_items = [ RolloutItem::InterAgentCommunicationMetadata { trigger_turn: communication.trigger_turn, @@ -3853,6 +3852,8 @@ impl Session { let mailbox_enabled = self.services.agent_control.mailbox_enabled() && turn_context.multi_agent_version == MultiAgentVersion::V2 && turn_context.config.features.enabled(Feature::AgentMailbox); + // Serialize append, successful flush, and per-item acknowledgement with compaction. + // The short-lived canonical-ID locks below never span persistence awaits. let custody_gate = if mailbox_enabled { Some(self.mailbox_custody_gate.lock().await) } else { @@ -3875,8 +3876,11 @@ impl Session { ) .into()); } - let mut canonical_ids = self.mailbox_canonical_response_item_ids.lock().await; - let already_canonical = canonical_ids.contains(&response_item_id); + let already_canonical = self + .mailbox_canonical_response_item_ids + .lock() + .await + .contains(&response_item_id); if already_canonical { // InitialHistory and this set are the canonical source across compaction. Flush // before acknowledging even when the exact message was already present. @@ -3887,7 +3891,6 @@ impl Session { .agent_control .mailbox_mark_canonical(self.thread_id, std::slice::from_ref(&response_item_id)) .await?; - drop(canonical_ids); drop(custody_gate); return Ok(response_item_id); } @@ -3917,12 +3920,14 @@ impl Session { turn_context.model_info.truncation_policy.into(), ); } - canonical_ids.insert(response_item_id.clone()); + self.mailbox_canonical_response_item_ids + .lock() + .await + .insert(response_item_id.clone()); self.services .agent_control .mailbox_mark_canonical(self.thread_id, std::slice::from_ref(&response_item_id)) .await?; - drop(canonical_ids); } else { self.persist_rollout_items(&rollout_items).await; let mut state = self.state.lock().await; @@ -4007,6 +4012,10 @@ impl Session { state.replace_history(items, reference_context_item); } + #[expect( + clippy::await_holding_invalid_type, + reason = "mailbox compaction must retain exclusive custody through canonical append and flush" + )] pub(crate) async fn replace_compacted_history( &self, items: Vec, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs index 6cb788eee3ea..ad7ad774a43b 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait.rs @@ -466,15 +466,14 @@ async fn handle_wait(invocation: ToolInvocation) -> Result, report.completion_reason = Some(MailboxWakeCause::RuntimeFault); report.condition_satisfied = false; } - if let Some(id) = attention_to_observe { - if mailbox + if let Some(id) = attention_to_observe + && mailbox .acknowledge_observed(session.thread_id, &[ObservationTarget::Attention(id)]) .await .is_err() - { - report.completion_reason = Some(MailboxWakeCause::RuntimeFault); - report.condition_satisfied = false; - } + { + report.completion_reason = Some(MailboxWakeCause::RuntimeFault); + report.condition_satisfied = false; } emit_report( &session, diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 32a82c6c1f3d..4d4e69798c2c 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -966,7 +966,7 @@ mod tests { "quiet_count": 2 }); let report: MailboxWaitReport = - serde_json::from_value(value.clone()).expect("decode mailbox wait report"); + serde_json::from_value(value).expect("decode mailbox wait report"); assert_eq!(report.return_when, MailboxWaitReturnWhen::Any); assert_eq!(report.completion_reason, None); assert_eq!( diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs index 1cd2772e4bec..65cbe37dd34a 100644 --- a/codex-rs/rollout/src/policy_tests.rs +++ b/codex-rs/rollout/src/policy_tests.rs @@ -109,7 +109,8 @@ fn typed_mailbox_wait_lifecycle_is_persisted_across_history_modes() { CollabAgentToolCallStatus::InProgress, Some(in_progress_report()), ); - let start_without_report = started_wait(CollabAgentToolCallStatus::InProgress, None); + let start_without_report = + started_wait(CollabAgentToolCallStatus::InProgress, /*report*/ None); let non_in_progress_start = started_wait( CollabAgentToolCallStatus::Completed, Some(in_progress_report()), @@ -121,17 +122,24 @@ fn typed_mailbox_wait_lifecycle_is_persisted_across_history_modes() { ); let completed = completed_wait( CollabAgentToolCallStatus::Completed, - Some(terminal_report(MailboxWakeCause::Result, true)), + Some(terminal_report( + MailboxWakeCause::Result, + /*condition_satisfied*/ true, + )), ); let failed = completed_wait( CollabAgentToolCallStatus::Failed, - Some(terminal_report(MailboxWakeCause::Interrupted, false)), + Some(terminal_report( + MailboxWakeCause::Interrupted, + /*condition_satisfied*/ false, + )), ); let nonterminal_completion = completed_wait( CollabAgentToolCallStatus::InProgress, Some(in_progress_report()), ); - let terminal_without_report = completed_wait(CollabAgentToolCallStatus::Failed, None); + let terminal_without_report = + completed_wait(CollabAgentToolCallStatus::Failed, /*report*/ None); let other_tool_terminal = completed_call( CollabAgentTool::SendInput, CollabAgentToolCallStatus::Failed, From ca9a4735805606277e95393aed34494356df19ab Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:36:55 +1100 Subject: [PATCH 24/54] test(mailbox): complete legacy execution fixtures --- codex-rs/core/src/session/mod.rs | 11 +++-------- .../exec/tests/event_processor_with_json_output.rs | 2 ++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 749215c88200..f4cca2bb15f0 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1562,14 +1562,9 @@ impl Session { report } None => { - let mut report = call - .mailbox_wait_report - .clone() - .ok_or_else(|| { - CodexErr::Fatal( - "stored mailbox wait has no structured report".to_string(), - ) - })?; + let mut report = call.mailbox_wait_report.clone().ok_or_else(|| { + CodexErr::Fatal("stored mailbox wait has no structured report".to_string()) + })?; if !already_completed { report.completion_reason = Some(MailboxWakeCause::Interrupted); report.condition_satisfied = false; diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index d1e45eae28f7..eea595c40ae3 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -972,6 +972,7 @@ fn collab_spawn_begin_and_end_emit_item_events() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, thread_id: "thread-parent".to_string(), turn_id: "turn-1".to_string(), @@ -1003,6 +1004,7 @@ fn collab_spawn_begin_and_end_emit_item_events() { wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, }, thread_id: "thread-parent".to_string(), turn_id: "turn-1".to_string(), From c7445be6be1b30c24f0277ebc3d1cfa9ca35c57f Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:48:17 +1100 Subject: [PATCH 25/54] fix(mailbox): clean goal continuation lint diagnostics --- codex-rs/ext/goal/src/runtime.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 725a6f957796..8cdfc862ba76 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -7,7 +7,6 @@ use std::sync::atomic::Ordering; use codex_core::GoalNotificationBinding; use codex_core::GoalNotificationClassification; use codex_core::GoalNotificationPhase; -use codex_core::GoalNotificationSnapshot; use codex_core::GoalNotificationStore; use codex_core::GoalNotificationTurnToken; use codex_core::ThreadManager; @@ -259,7 +258,7 @@ impl GoalRuntimeHandle { let snapshot = self.inner.notification_store.snapshot(); if snapshot .as_ref() - .map_or(true, |snapshot| snapshot.binding.goal_id == *goal_id) + .is_none_or(|snapshot| snapshot.binding.goal_id == *goal_id) { return Ok(()); } From 28558eef5003196076a50a15441a2afcdbe6db55 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:03:43 +1100 Subject: [PATCH 26/54] test(mailbox): satisfy workspace compatibility and test lints --- codex-rs/agent-mailbox/src/mailbox_tests.rs | 313 ++++++++++++------ .../app-server/tests/suite/v2/turn_start.rs | 2 + codex-rs/core/src/thread_manager_tests.rs | 2 +- codex-rs/core/tests/suite/agent_mailbox.rs | 32 +- codex-rs/tui/src/chatwidget/tool_lifecycle.rs | 44 +-- 5 files changed, 252 insertions(+), 141 deletions(-) diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index a3db02a8e0f6..bb0f969a7f72 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -218,7 +218,7 @@ fn goal_binding( #[tokio::test] async fn new_continuation_binding_requires_native_incarnation() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(40), thread_id(41)); + let (issuer, agent) = (thread_id(/*index*/ 40), thread_id(/*index*/ 41)); let (work, input) = accept_work( &fixture.mailbox, issuer, @@ -240,7 +240,12 @@ async fn new_continuation_binding_requires_native_incarnation() { ) .await; - let mut legacy_binding = goal_binding(issuer, agent, 3, thread_id(42)); + let mut legacy_binding = goal_binding( + issuer, + agent, + /*generation*/ 3, + thread_id(/*index*/ 42), + ); legacy_binding.incarnation = None; let result = fixture .mailbox @@ -248,7 +253,7 @@ async fn new_continuation_binding_requires_native_incarnation() { agent, "missing-incarnation-turn", Outcome::Completed, - None, + /*response*/ None, Continuation::PendingGoalBinding(legacy_binding), ) .await; @@ -278,7 +283,7 @@ fn dependency(owner: &str, peers: &[&str], mode: DependencyMode) -> Dependency { #[tokio::test] async fn native_retry_dispatch_reopen_and_capture_backpressure_are_safe() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(1), thread_id(2)); + let (issuer, agent) = (thread_id(/*index*/ 1), thread_id(/*index*/ 2)); let original = native_work(issuer, agent, "native-1", "do the work"); let work = fixture.mailbox.accept_work(original.clone()).await.unwrap(); let captured = fixture.mailbox.accepted_input(&work).await.unwrap(); @@ -342,7 +347,7 @@ async fn native_accept_retry_after_pre_dispatch_handle_drop_reuses_original_iden config, mailbox, } = test_mailbox().await; - let (issuer, agent) = (thread_id(22), thread_id(23)); + let (issuer, agent) = (thread_id(/*index*/ 22), thread_id(/*index*/ 23)); let original = native_work( issuer, agent, @@ -400,7 +405,11 @@ async fn native_accept_retry_after_pre_dispatch_handle_drop_reuses_original_iden #[tokio::test] async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped() { let fixture = test_mailbox().await; - let (issuer, other_issuer, agent) = (thread_id(3), thread_id(4), thread_id(5)); + let (issuer, other_issuer, agent) = ( + thread_id(/*index*/ 3), + thread_id(/*index*/ 4), + thread_id(/*index*/ 5), + ); let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "a", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "b", "B").await; let (c, _id_c) = accept_work(&fixture.mailbox, issuer, agent, "c", "C").await; @@ -435,7 +444,12 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped accept_work(&fixture.mailbox, other_issuer, agent, "foreign", "foreign").await; fixture .mailbox - .record_outcome(&foreign, Outcome::Completed, None, Continuation::None) + .record_outcome( + &foreign, + Outcome::Completed, + /*response*/ None, + Continuation::None, + ) .await .unwrap(); let outcomes = finish( @@ -443,7 +457,7 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped agent, "turn", Outcome::Completed, - None, + /*response*/ None, Continuation::None, ) .await; @@ -496,7 +510,11 @@ async fn admission_is_incremental_but_outcomes_and_snapshots_stay_exactly_scoped #[tokio::test] async fn mixed_issuer_turn_creates_distinct_handbacks() { let fixture = test_mailbox().await; - let (issuer_a, issuer_b, agent) = (thread_id(6), thread_id(7), thread_id(8)); + let (issuer_a, issuer_b, agent) = ( + thread_id(/*index*/ 6), + thread_id(/*index*/ 7), + thread_id(/*index*/ 8), + ); let (a, id_a) = accept_work(&fixture.mailbox, issuer_a, agent, "multi-a", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer_b, agent, "multi-b", "B").await; admit(&fixture.mailbox, agent, "shared-turn", &[id_a, id_b]).await; @@ -537,12 +555,22 @@ async fn mixed_issuer_turn_creates_distinct_handbacks() { #[tokio::test] async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(9), thread_id(10)); + let (issuer, agent) = (thread_id(/*index*/ 9), thread_id(/*index*/ 10)); let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "pending", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "deferred", "B").await; let bindings = [ - goal_binding(issuer, agent, 4, thread_id(44)), - goal_binding(issuer, agent, 5, thread_id(45)), + goal_binding( + issuer, + agent, + /*generation*/ 4, + thread_id(/*index*/ 44), + ), + goal_binding( + issuer, + agent, + /*generation*/ 5, + thread_id(/*index*/ 45), + ), ]; for (work, id, turn, continuation, state) in [ ( @@ -566,7 +594,7 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() agent, turn, Outcome::Completed, - None, + /*response*/ None, continuation, ) .await; @@ -593,7 +621,7 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() agent, "stale-generation", Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(stale) ) .await @@ -610,14 +638,14 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() assert!(work_item(&still_deferred, &a.id).outcome.is_none()); let mut stale_incarnation = bindings[0].clone(); - stale_incarnation.incarnation = Some(thread_id(99)); + stale_incarnation.incarnation = Some(thread_id(/*index*/ 99)); assert!( finish( &fixture.mailbox, agent, "stale-incarnation", Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(stale_incarnation) ) .await @@ -651,7 +679,7 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() agent, turn, Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(binding.clone()), ) .await; @@ -678,8 +706,13 @@ async fn pending_deferred_and_terminal_continuations_bind_to_exact_generation() #[tokio::test] async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(46), thread_id(47)); - let binding = goal_binding(issuer, agent, 6, thread_id(48)); + let (issuer, agent) = (thread_id(/*index*/ 46), thread_id(/*index*/ 47)); + let binding = goal_binding( + issuer, + agent, + /*generation*/ 6, + thread_id(/*index*/ 48), + ); let (original, original_input) = accept_work( &fixture.mailbox, issuer, @@ -699,7 +732,7 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { agent, "source-origin", Outcome::Completed, - None, + /*response*/ None, Continuation::PendingGoalBinding(binding.clone()), ) .await; @@ -708,7 +741,7 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { agent, "source-origin", Outcome::Completed, - None, + /*response*/ None, Continuation::Deferred(binding.clone()), ) .await; @@ -746,7 +779,7 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { agent, "source-terminal", Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(binding.clone()), ) .await; @@ -775,8 +808,13 @@ async fn continuation_binding_survives_deferred_phase_and_later_source_turn() { #[tokio::test] async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_terminal() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(50), thread_id(51)); - let binding = goal_binding(issuer, agent, 7, thread_id(52)); + let (issuer, agent) = (thread_id(/*index*/ 50), thread_id(/*index*/ 51)); + let binding = goal_binding( + issuer, + agent, + /*generation*/ 7, + thread_id(/*index*/ 52), + ); let (original, original_input) = accept_work( &fixture.mailbox, issuer, @@ -796,7 +834,7 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t agent, "source-pending", Outcome::Completed, - None, + /*response*/ None, Continuation::PendingGoalBinding(binding.clone()), ) .await; @@ -810,7 +848,7 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t .unwrap() ); let mut different_incarnation = binding.clone(); - different_incarnation.incarnation = Some(thread_id(53)); + different_incarnation.incarnation = Some(thread_id(/*index*/ 53)); assert!( !fixture .mailbox @@ -887,7 +925,7 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t agent, "source-wrong-terminal", Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(different_incarnation), ) .await; @@ -943,7 +981,7 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t agent, "source-exact-terminal", Outcome::Completed, - None, + /*response*/ None, Continuation::Terminal(binding.clone()), ) .await; @@ -979,7 +1017,11 @@ async fn unowned_continuation_recovery_retains_binding_until_exact_incarnation_t #[tokio::test] async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let fixture = test_mailbox().await; - let (author, recipient, stranger) = (thread_id(11), thread_id(12), thread_id(13)); + let (author, recipient, stranger) = ( + thread_id(/*index*/ 11), + thread_id(/*index*/ 12), + thread_id(/*index*/ 13), + ); let request = send( &fixture.mailbox, draft( @@ -988,7 +1030,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, MessageKind::Request, "decide", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1006,7 +1048,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, "unrelated-turn", Outcome::Completed, - None, + /*response*/ None, Continuation::None, ) .await; @@ -1028,7 +1070,14 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { .mark_canonical(recipient, &request.id) .await .unwrap(); - let mut reply_draft = draft("reply", recipient, author, MessageKind::Note, "done", false); + let mut reply_draft = draft( + "reply", + recipient, + author, + MessageKind::Note, + "done", + /*legacy_attention*/ false, + ); reply_draft.reply_to = Some(request.id.clone()); let reply = send(&fixture.mailbox, reply_draft).await; let replied = take_snapshot( @@ -1061,7 +1110,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, MessageKind::Request, "old", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1071,7 +1120,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, MessageKind::Request, "no", - false, + /*legacy_attention*/ false, ); unauthorized.supersedes = Some(open_request.id.clone()); assert!(matches!( @@ -1084,14 +1133,14 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, MessageKind::Request, "new", - false, + /*legacy_attention*/ false, ); replacement_draft.supersedes = Some(open_request.id.clone()); let replacement = send(&fixture.mailbox, replacement_draft).await; let superseded = take_snapshot( &fixture.mailbox, author, - None, + /*agent*/ None, vec![Selector::Reply(open_request.id.clone())], ReturnWhen::Any, ) @@ -1109,7 +1158,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { recipient, MessageKind::Request, "cancel", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1128,7 +1177,7 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { let withdrawn = take_snapshot( &fixture.mailbox, author, - None, + /*agent*/ None, vec![Selector::Reply(withdrawal.id.clone())], ReturnWhen::Any, ) @@ -1143,13 +1192,27 @@ async fn quiet_reply_and_author_dispositions_settle_only_the_exact_request() { #[tokio::test] async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcome() { let fixture = test_mailbox().await; - let (author, agent) = (thread_id(14), thread_id(15)); + let (author, agent) = (thread_id(/*index*/ 14), thread_id(/*index*/ 15)); let (work_a, _) = accept_work(&fixture.mailbox, author, agent, "work-a", "A").await; let (work_b, _) = accept_work(&fixture.mailbox, author, agent, "work-b", "B").await; - let mut first_draft = draft("p1", author, agent, MessageKind::Progress, "first", false); + let mut first_draft = draft( + "p1", + author, + agent, + MessageKind::Progress, + "first", + /*legacy_attention*/ false, + ); first_draft.work_ref = Some(work_a.clone()); let first = send(&fixture.mailbox, first_draft).await; - let mut update_draft = draft("p2", author, agent, MessageKind::Progress, "latest", false); + let mut update_draft = draft( + "p2", + author, + agent, + MessageKind::Progress, + "latest", + /*legacy_attention*/ false, + ); update_draft.work_ref = Some(work_a.clone()); let update = send(&fixture.mailbox, update_draft).await; assert_ne!(update.id, first.id); @@ -1169,7 +1232,7 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom .unwrap(); assert_eq!(latest.communication.content, "latest"); assert_eq!(latest.supersedes.as_deref(), Some(first.id.as_str())); - let pending = fixture.mailbox.pending(agent, 20).await.unwrap(); + let pending = fixture.mailbox.pending(agent, /*limit*/ 20).await.unwrap(); assert!(pending.iter().any(|envelope| envelope.id == update.id)); assert!(!pending.iter().any(|envelope| envelope.id == first.id)); @@ -1179,7 +1242,7 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom agent, MessageKind::Progress, "other path", - false, + /*legacy_attention*/ false, ); other_author.work_ref = Some(work_a.clone()); other_author.communication.author = AgentPath::from_string("/root/child".to_string()).unwrap(); @@ -1190,18 +1253,30 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom agent, MessageKind::Progress, "other path", - false, + /*legacy_attention*/ false, ); other_path.work_ref = Some(work_a.clone()); other_path.communication.recipient = AgentPath::from_string("/root/other".to_string()).unwrap(); assert_ne!(send(&fixture.mailbox, other_path).await.id, first.id); - let mut other_work = draft("p4", author, agent, MessageKind::Progress, "work B", false); + let mut other_work = draft( + "p4", + author, + agent, + MessageKind::Progress, + "work B", + /*legacy_attention*/ false, + ); other_work.work_ref = Some(work_b); assert_ne!(send(&fixture.mailbox, other_work).await.id, first.id); fixture .mailbox - .record_outcome(&work_a, Outcome::Completed, None, Continuation::None) + .record_outcome( + &work_a, + Outcome::Completed, + /*response*/ None, + Continuation::None, + ) .await .unwrap(); let mut late = draft( @@ -1210,7 +1285,7 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom agent, MessageKind::Progress, "late", - false, + /*legacy_attention*/ false, ); late.work_ref = Some(work_a); assert!(matches!( @@ -1222,7 +1297,7 @@ async fn progress_coalesces_exact_unadmitted_scope_and_closes_at_terminal_outcom #[tokio::test] async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(16), thread_id(17)); + let (issuer, agent) = (thread_id(/*index*/ 16), thread_id(/*index*/ 17)); let (a, id_a) = accept_work(&fixture.mailbox, issuer, agent, "observe-a", "A").await; let (b, id_b) = accept_work(&fixture.mailbox, issuer, agent, "observe-b", "B").await; for (work, id, turn) in [(&a, id_a, "turn-a"), (&b, id_b, "turn-b")] { @@ -1232,7 +1307,7 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { agent, turn, Outcome::Completed, - None, + /*response*/ None, Continuation::None, ) .await; @@ -1276,7 +1351,7 @@ async fn canonical_outcomes_are_observed_per_item_without_cursor_loss() { #[tokio::test] async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate() { let fixture = test_mailbox().await; - let (sender, recipient) = (thread_id(18), thread_id(19)); + let (sender, recipient) = (thread_id(/*index*/ 18), thread_id(/*index*/ 19)); let kinds = [ MessageKind::Note, MessageKind::Progress, @@ -1293,7 +1368,7 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( recipient, kind, "quiet", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1305,12 +1380,16 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( recipient, kind, "legacy", - true, + /*legacy_attention*/ true, ), ) .await; } - let pending = fixture.mailbox.pending(recipient, 20).await.unwrap(); + let pending = fixture + .mailbox + .pending(recipient, /*limit*/ 20) + .await + .unwrap(); assert_eq!(pending.len(), 8); for kind in kinds { assert_eq!(pending.iter().filter(|m| m.kind == kind).count(), 2); @@ -1320,7 +1399,7 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( assert_eq!( fixture .mailbox - .attention(recipient, 20) + .attention(recipient, /*limit*/ 20) .await .unwrap() .len(), @@ -1331,7 +1410,7 @@ async fn message_kinds_default_to_quiet_delivery_with_legacy_attention_separate( #[tokio::test] async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_request() { let fixture = test_mailbox().await; - let (sender, recipient) = (thread_id(24), thread_id(25)); + let (sender, recipient) = (thread_id(/*index*/ 24), thread_id(/*index*/ 25)); let request = send( &fixture.mailbox, draft( @@ -1340,7 +1419,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Request, "please decide", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1352,7 +1431,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Alert, "needs attention", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1364,7 +1443,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Note, "quiet note", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1376,7 +1455,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Progress, "quiet progress", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1387,14 +1466,14 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Note, "legacy wake", - true, + /*legacy_attention*/ true, ); let legacy = send(&fixture.mailbox, legacy_draft.clone()).await; assert_eq!(send(&fixture.mailbox, legacy_draft).await.id, legacy.id); let uncanonical = fixture .mailbox - .pending_page(recipient, /*after_seq*/ None, 20) + .pending_page(recipient, /*after_seq*/ None, /*limit*/ 20) .await .unwrap(); let uncanonical_ids = uncanonical @@ -1412,7 +1491,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re } let canonical = fixture .mailbox - .read_page(recipient, None, 20) + .read_page(recipient, /*after_seq*/ None, /*limit*/ 20) .await .unwrap(); assert!( @@ -1425,7 +1504,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re let mut attention_ids = fixture .mailbox - .attention(recipient, 20) + .attention(recipient, /*limit*/ 20) .await .unwrap() .into_iter() @@ -1453,7 +1532,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re assert!( fixture .mailbox - .attention(recipient, 20) + .attention(recipient, /*limit*/ 20) .await .unwrap() .is_empty() @@ -1481,7 +1560,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re recipient, MessageKind::Request, "new request", - false, + /*legacy_attention*/ false, ), ) .await; @@ -1490,7 +1569,11 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re .mark_canonical(recipient, &new_request.id) .await .unwrap(); - let new_attention = fixture.mailbox.attention(recipient, 20).await.unwrap(); + let new_attention = fixture + .mailbox + .attention(recipient, /*limit*/ 20) + .await + .unwrap(); let new_attention_ids = new_attention .iter() .map(|attention| attention.id.clone()) @@ -1513,7 +1596,7 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re assert!( fixture .mailbox - .attention(recipient, 20) + .attention(recipient, /*limit*/ 20) .await .unwrap() .is_empty() @@ -1523,14 +1606,14 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re #[tokio::test] async fn activation_claim_is_single_use_across_a_lost_result() { let fixture = test_mailbox().await; - let (sender, recipient) = (thread_id(20), thread_id(21)); + let (sender, recipient) = (thread_id(/*index*/ 20), thread_id(/*index*/ 21)); let mut message_draft = draft( "activation", sender, recipient, MessageKind::Alert, "activate", - false, + /*legacy_attention*/ false, ); message_draft.activate_if_idle = true; let message = send(&fixture.mailbox, message_draft).await; @@ -1606,7 +1689,7 @@ fn only_closed_any_and_all_peer_cycles_are_provable() { #[tokio::test] async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { let fixture = test_mailbox().await; - let (owner_a, owner_b) = (thread_id(22), thread_id(23)); + let (owner_a, owner_b) = (thread_id(/*index*/ 22), thread_id(/*index*/ 23)); let wait_a = fixture .mailbox .begin_wait( @@ -1614,7 +1697,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_a, vec![owner_b], ReturnWhen::Any, - false, + /*has_escape*/ false, ) .unwrap(); assert_eq!(wait_a.wait_id(), "wait-a"); @@ -1624,7 +1707,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_b, vec![owner_a], ReturnWhen::All, - false, + /*has_escape*/ false, ), Err(MailboxError::ClosedPeerCycle) )); @@ -1634,7 +1717,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_b, vec![owner_a], ReturnWhen::All, - false, + /*has_escape*/ false, ), Err(MailboxError::ClosedPeerCycle) )); @@ -1645,7 +1728,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_b, vec![owner_a], ReturnWhen::All, - true, + /*has_escape*/ true, ) .unwrap(); drop(escape); @@ -1655,7 +1738,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_b, vec![owner_a], ReturnWhen::All, - false, + /*has_escape*/ false, ), Err(MailboxError::ClosedPeerCycle) )); @@ -1668,7 +1751,7 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { owner_b, vec![owner_a], ReturnWhen::All, - false, + /*has_escape*/ false, ) .is_ok() ); @@ -1677,29 +1760,55 @@ async fn wait_cycle_rejects_only_the_new_registration_and_drop_is_exact() { #[tokio::test] async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until_rescan() { let fixture = test_mailbox().await; - let (a, b) = (thread_id(28), thread_id(29)); + let (a, b) = (thread_id(/*index*/ 28), thread_id(/*index*/ 29)); let before = fixture.mailbox.wait_input_generation(); let old = fixture .mailbox - .begin_wait_at("a".into(), a, vec![b], ReturnWhen::Any, false, before) + .begin_wait_at( + "a".into(), + a, + vec![b], + ReturnWhen::Any, + /*has_escape*/ false, + before, + ) .unwrap(); let arrival = fixture.mailbox.begin_input_arrival(); assert!(matches!( - fixture - .mailbox - .begin_wait_at("stale".into(), b, vec![a], ReturnWhen::All, false, before,), + fixture.mailbox.begin_wait_at( + "stale".into(), + b, + vec![a], + ReturnWhen::All, + /*has_escape*/ false, + before, + ), Err(MailboxError::StaleWaitSnapshot) )); let during = fixture.mailbox.wait_input_generation(); let provisional_a = fixture .mailbox - .begin_wait_at("a".into(), a, vec![b], ReturnWhen::Any, false, during) + .begin_wait_at( + "a".into(), + a, + vec![b], + ReturnWhen::Any, + /*has_escape*/ false, + during, + ) .unwrap(); // A stale RAII handle must not remove a replacement registration. drop(old); let provisional_b = fixture .mailbox - .begin_wait_at("b".into(), b, vec![a], ReturnWhen::All, false, during) + .begin_wait_at( + "b".into(), + b, + vec![a], + ReturnWhen::All, + /*has_escape*/ false, + during, + ) .unwrap(); drop(arrival); assert!(matches!( @@ -1708,7 +1817,7 @@ async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until b, vec![a], ReturnWhen::All, - false, + /*has_escape*/ false, during, ), Err(MailboxError::StaleWaitSnapshot) @@ -1723,7 +1832,7 @@ async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until a, vec![b], ReturnWhen::Any, - false, + /*has_escape*/ false, current, ) .unwrap(); @@ -1733,7 +1842,7 @@ async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until b, vec![a], ReturnWhen::All, - false, + /*has_escape*/ false, current, ), Err(MailboxError::ClosedPeerCycle) @@ -1743,7 +1852,7 @@ async fn input_publication_invalidates_stale_waits_and_defers_cycle_claims_until #[tokio::test] async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recovery() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(32), thread_id(33)); + let (issuer, agent) = (thread_id(/*index*/ 32), thread_id(/*index*/ 33)); let (work, input) = accept_work(&fixture.mailbox, issuer, agent, "recovery", "original").await; assert_eq!( fixture.mailbox.begin_dispatch(&work).await.unwrap(), @@ -1771,7 +1880,7 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov agent, "explicit-recovery-turn", Outcome::Completed, - None, + /*response*/ None, Continuation::None, ) .await; @@ -1811,7 +1920,7 @@ async fn confirmed_execution_and_terminal_capture_supersede_stale_dispatch_recov #[tokio::test] async fn recipient_incompatibility_recovers_accepted_work_without_losing_or_resending_it() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(37), thread_id(38)); + let (issuer, agent) = (thread_id(/*index*/ 37), thread_id(/*index*/ 38)); let (work, input_id) = accept_work( &fixture.mailbox, issuer, @@ -1887,7 +1996,11 @@ async fn recipient_incompatibility_recovers_accepted_work_without_losing_or_rese #[tokio::test] async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_batches() { let fixture = test_mailbox().await; - let (issuer, agent, other_issuer) = (thread_id(34), thread_id(35), thread_id(36)); + let (issuer, agent, other_issuer) = ( + thread_id(/*index*/ 34), + thread_id(/*index*/ 35), + thread_id(/*index*/ 36), + ); let accepted_request = native_work(issuer, agent, "resume-accepted", "retained accepted input"); let accepted = fixture .mailbox @@ -1937,21 +2050,21 @@ async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_ba ); assert_eq!( reopened - .interrupt_issuer_dispatches(issuer, 1) + .interrupt_issuer_dispatches(issuer, /*limit*/ 1) .await .unwrap(), 1 ); assert_eq!( reopened - .interrupt_issuer_dispatches(issuer, 1) + .interrupt_issuer_dispatches(issuer, /*limit*/ 1) .await .unwrap(), 1 ); assert_eq!( reopened - .interrupt_issuer_dispatches(issuer, 1) + .interrupt_issuer_dispatches(issuer, /*limit*/ 1) .await .unwrap(), 0 @@ -1986,14 +2099,16 @@ async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_ba ); assert_eq!( reopened - .issued_work_refs(issuer, None, 100) + .issued_work_refs(issuer, /*agent*/ None, /*limit*/ 100) .await .unwrap() .len(), 3 ); assert!(matches!( - reopened.interrupt_issuer_dispatches(issuer, 0).await, + reopened + .interrupt_issuer_dispatches(issuer, /*limit*/ 0) + .await, Err(MailboxError::InvalidInput(_)) )); } @@ -2001,7 +2116,7 @@ async fn resumed_issuer_interrupts_only_its_unconfirmed_dispatches_in_bounded_ba #[tokio::test] async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attention() { let fixture = test_mailbox().await; - let (issuer, agent) = (thread_id(30), thread_id(31)); + let (issuer, agent) = (thread_id(/*index*/ 30), thread_id(/*index*/ 31)); let (covered, input) = accept_work(&fixture.mailbox, issuer, agent, "covered", "A").await; let (unrelated, _) = accept_work(&fixture.mailbox, issuer, agent, "unrelated", "B").await; admit(&fixture.mailbox, agent, "covered-turn", &[input]).await; @@ -2010,7 +2125,7 @@ async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attentio agent, "covered-turn", Outcome::NeedsAttention, - None, + /*response*/ None, Continuation::None, ) .await; @@ -2028,7 +2143,7 @@ async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attentio assert!( !fixture .mailbox - .attention(issuer, 100) + .attention(issuer, /*limit*/ 100) .await .unwrap() .is_empty() @@ -2052,7 +2167,7 @@ async fn canonical_result_body_resolves_only_its_exact_covered_work_and_attentio assert!( fixture .mailbox - .attention(issuer, 100) + .attention(issuer, /*limit*/ 100) .await .unwrap() .is_empty() diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index bf231f1faebf..fdf9584eecad 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -3243,6 +3243,7 @@ async fn turn_start_emits_multi_agent_v1_spawn_requested_and_effective_identity_ wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); @@ -3612,6 +3613,7 @@ config_file = "./custom-role.toml" wake_notifications: None, completion_reason: None, wake_cause: None, + mailbox_wait_report: None, } ); diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 3ab6945e311b..429d4fb9a910 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -2210,7 +2210,7 @@ fn mailbox_deferred_goal_handoff_rejects_replaced_native_incarnation() { child_thread_id, parent_thread_id, "turn-a", - Some(&(ThreadId::new(), snapshot.clone())), + Some(&(ThreadId::new(), snapshot)), Some(&store), ) .is_err() diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 71cf56fbbe69..6bc99318bb53 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -96,7 +96,8 @@ struct UnadmittedWorkResponder; impl Respond for UnadmittedWorkResponder { fn respond(&self, request: &Request) -> ResponseTemplate { - let body: Value = serde_json::from_slice(&request.body).unwrap(); + let body: Value = + serde_json::from_slice(&request.body).expect("provider request should contain JSON"); if has_function_output(&body, "unadmitted-wait") { return sse_response(sse(vec![ ev_response_created("unadmitted-final"), @@ -105,9 +106,10 @@ impl Respond for UnadmittedWorkResponder { ])); } if has_function_output(&body, "unadmitted-spawn") { + let spawn_output = function_output_text(&body, "unadmitted-spawn") + .expect("spawn output should remain in provider history"); let spawn: Value = - serde_json::from_str(function_output_text(&body, "unadmitted-spawn").unwrap()) - .unwrap(); + serde_json::from_str(spawn_output).expect("spawn output should be JSON"); return function_call_response( "unadmitted-wait-response", "unadmitted-wait", @@ -244,7 +246,7 @@ async fn mailbox_event_only_wait_wakes_when_producer_fails_before_input_admissio let input_id = work.input.id.as_ref().unwrap().to_string(); assert_eq!( mailbox - .unadmitted_input_ids(work.work_ref.agent, &[input_id.clone()]) + .unadmitted_input_ids(work.work_ref.agent, std::slice::from_ref(&input_id)) .await?, vec![input_id] ); @@ -642,7 +644,6 @@ const CUSTODY_BODY: &str = "canonical request body that must survive compaction" #[derive(Clone)] struct WaitProbeResponder { - root_thread_id: String, requests: Arc>>, first_wait_started: Arc>>, event_only_arguments: Arc>>, @@ -850,7 +851,6 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - thread_id: root_thread_id.clone(), }) .respond_with(WaitProbeResponder { - root_thread_id: root_thread_id.clone(), requests: Arc::clone(&captured), first_wait_started: Arc::clone(&first_wait_started), event_only_arguments: Arc::clone(&event_only_arguments), @@ -1070,7 +1070,7 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com config.compact_prompt = Some(COMPACTION_PROMPT.to_string()); }); let test = builder.build_with_auto_env(&server).await?; - let root_thread_id: ThreadId = test.session_configured.thread_id.clone(); + let root_thread_id: ThreadId = test.session_configured.thread_id; let root_thread_id_text = root_thread_id.to_string(); let captured = Arc::new(Mutex::new(Vec::new())); @@ -1130,16 +1130,13 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com let message_id_owned = message_id.to_string(); assert_eq!( mailbox - .unadmitted_input_ids( - root_thread_id.clone(), - std::slice::from_ref(&message_id_owned) - ) + .unadmitted_input_ids(root_thread_id, std::slice::from_ref(&message_id_owned)) .await?, vec![message_id_owned.clone()], "a stream that ended before response.created must not admit the body" ); let envelope = mailbox - .get_message(root_thread_id.clone(), message_id) + .get_message(root_thread_id, message_id) .await? .expect("accepted canonical body should remain retrievable"); assert!(envelope.canonical); @@ -1149,7 +1146,7 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com ); assert!( mailbox - .pending(root_thread_id.clone(), 10) + .pending(root_thread_id, 10) .await? .iter() .any(|pending| pending.id.as_str() == message_id) @@ -1180,23 +1177,20 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com ); assert_eq!( mailbox - .unadmitted_input_ids( - root_thread_id.clone(), - std::slice::from_ref(&message_id_owned) - ) + .unadmitted_input_ids(root_thread_id, std::slice::from_ref(&message_id_owned)) .await?, vec![message_id_owned.clone()], "compaction is not execution admission for a pending mailbox obligation" ); assert!( mailbox - .pending(root_thread_id.clone(), 10) + .pending(root_thread_id, 10) .await? .iter() .any(|pending| pending.id.as_str() == message_id) ); let retained = mailbox - .get_message(root_thread_id.clone(), message_id) + .get_message(root_thread_id, message_id) .await? .expect("compaction must not delete the canonical body from mailbox custody"); assert_eq!( diff --git a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs index b5aab9ea6367..3404449df658 100644 --- a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs @@ -201,28 +201,28 @@ impl ChatWidget { self.request_redraw(); return; } - if matches!(tool, CollabAgentTool::Wait) { - if let Some(progress) = self.active_mailbox_waits.remove(id) { - if let Some(report) = mailbox_wait_report.as_ref() { - progress.update(report.clone(), /*in_progress*/ false); - } else { - progress.finish(); - } - let wait_cell_is_active = self - .transcript - .active_cell - .as_ref() - .and_then(|cell| { - cell.as_any() - .downcast_ref::() - }) - .is_some_and(|cell| cell.call_id() == id.as_str()); - if wait_cell_is_active { - self.bump_active_cell_revision(); - self.flush_active_cell(); - self.request_redraw(); - return; - } + if matches!(tool, CollabAgentTool::Wait) + && let Some(progress) = self.active_mailbox_waits.remove(id) + { + if let Some(report) = mailbox_wait_report.as_ref() { + progress.update(report.clone(), /*in_progress*/ false); + } else { + progress.finish(); + } + let wait_cell_is_active = self + .transcript + .active_cell + .as_ref() + .and_then(|cell| { + cell.as_any() + .downcast_ref::() + }) + .is_some_and(|cell| cell.call_id() == id.as_str()); + if wait_cell_is_active { + self.bump_active_cell_revision(); + self.flush_active_cell(); + self.request_redraw(); + return; } } From f94ba4c764c6e134a352d49b5006ce676c73652f Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:12:26 +1100 Subject: [PATCH 27/54] style(mailbox): label cleared-goal settlement argument --- codex-rs/ext/goal/src/api.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codex-rs/ext/goal/src/api.rs b/codex-rs/ext/goal/src/api.rs index cff27733de3c..d5aedba3d62f 100644 --- a/codex-rs/ext/goal/src/api.rs +++ b/codex-rs/ext/goal/src/api.rs @@ -322,7 +322,9 @@ impl GoalService { })?; let cleared = cleared_goal.is_some(); if let Some(runtime) = runtime.as_ref() - && let Err(err) = runtime.finalize_pending_goal_notification(None).await + && let Err(err) = runtime + .finalize_pending_goal_notification(/*external_goal*/ None) + .await { return Err(GoalServiceError::Internal(format!( "goal cleared but deferred completion settlement failed: {err}" From 78bbda50504d821edbe4697f0b5b22419f3a8f73 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:16:01 +1100 Subject: [PATCH 28/54] ci: keep SDK and post-main health on hosted execution --- .github/scripts/test_ci_planners.py | 8 ++++++++ .github/scripts/test_run_bazel_with_buildbuddy.py | 8 +++++++- .github/workflows/native-windows-bazel-health.yml | 6 +++++- .github/workflows/sdk.yml | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 8ece13a8b412..ddd9a0f63f5f 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1692,6 +1692,14 @@ def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> No native_test_run, ) self.assertIn("--local_test_jobs=1", native_test_run) + self.assertIn( + '[[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" ' + '&& -z "${BUILDBUDDY_API_KEY:-}" ]]', + native_test_run, + ) + self.assertIn( + "bazel_test_args+=(--host_platform=//:local_windows)", native_test_run + ) module_bazel = (REPO_ROOT / "MODULE.bazel").read_text(encoding="utf-8") self.assertIn( diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index b6ff218d05c0..0b256e12590b 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -54,7 +54,13 @@ def test_hosted_only_fork_callers_withhold_optional_remote_credentials(self) -> "BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' " "&& secrets.BUILDBUDDY_API_KEY || '' }}" ) - for workflow in ("bazel.yml", "rust-ci-full.yml", "v8-canary.yml"): + for workflow in ( + "bazel.yml", + "native-windows-bazel-health.yml", + "rust-ci-full.yml", + "sdk.yml", + "v8-canary.yml", + ): with self.subTest(workflow=workflow): source = (workflow_root / workflow).read_text(encoding="utf-8") assignments = [ diff --git a/.github/workflows/native-windows-bazel-health.yml b/.github/workflows/native-windows-bazel-health.yml index 650df857a42e..a57bb3556024 100644 --- a/.github/workflows/native-windows-bazel-health.yml +++ b/.github/workflows/native-windows-bazel-health.yml @@ -129,7 +129,7 @@ jobs: - name: bazel test //... id: native_bazel_test env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | bazel_targets=( @@ -154,6 +154,10 @@ jobs: --build_metadata=COMMIT_SHA=${GITHUB_SHA} --build_metadata=TAG_windows_native_main=true ) + if [[ "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + # Match the fork's hosted-local gnullvm test and clippy host. + bazel_test_args+=(--host_platform=//:local_windows) + fi ./.github/scripts/run-bazel-ci.sh \ --print-failed-action-summary \ diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index dddaf03f5539..6976023d6db8 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -96,7 +96,7 @@ jobs: - name: Build codex with Bazel env: - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + BUILDBUDDY_API_KEY: ${{ github.repository != 'sednalabs/codex' && secrets.BUILDBUDDY_API_KEY || '' }} shell: bash run: | set -euo pipefail From 01ec0b2e8978a387208e3a0006277b21b318f263 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:30:41 +1100 Subject: [PATCH 29/54] test(mailbox): move final replay fixture pending refs --- codex-rs/tui/src/chatwidget/tests/history_replay.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index 2d50d8f9212b..118e4b047805 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -150,7 +150,7 @@ async fn replayed_mailbox_wait_report_preserves_active_join_selection() { requested_replies_to: report.requested_replies_to.clone(), return_when: report.return_when, completed: Vec::new(), - pending: report.pending.clone(), + pending: report.pending, completion_reason: Some(codex_protocol::items::MailboxWakeCause::Interrupted), condition_satisfied: false, quiet_count: 5, From 8000e30da76a4c49ba11b4a3b2e4564b522e8173 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:42:17 +1100 Subject: [PATCH 30/54] test(mailbox): annotate native fixture arguments --- .../core/src/agent/control/mailbox_tests.rs | 32 +++++++++--------- codex-rs/core/src/session/input_queue.rs | 14 +++++--- codex-rs/core/src/session/tests.rs | 4 +-- codex-rs/core/src/thread_manager_tests.rs | 2 +- .../multi_agents_v2/mailbox_wait_tests.rs | 33 ++++++++++++++----- codex-rs/core/tests/suite/agent_mailbox.rs | 4 +-- 6 files changed, 56 insertions(+), 33 deletions(-) diff --git a/codex-rs/core/src/agent/control/mailbox_tests.rs b/codex-rs/core/src/agent/control/mailbox_tests.rs index e6c7e82dc97e..4ebd269d29d1 100644 --- a/codex-rs/core/src/agent/control/mailbox_tests.rs +++ b/codex-rs/core/src/agent/control/mailbox_tests.rs @@ -50,7 +50,7 @@ async fn mailbox_shared_control_does_not_capture_stock_invocations() { let mailbox = control.mailbox().unwrap(); assert!( mailbox - .issued_work_refs(session.thread_id, None, 100) + .issued_work_refs(session.thread_id, /*agent*/ None, /*limit*/ 100) .await .unwrap() .is_empty() @@ -200,7 +200,7 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), - None + /*completion*/ None ) .await .unwrap() @@ -214,7 +214,7 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), - None + /*completion*/ None ) .await .unwrap() @@ -239,7 +239,7 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a "continuation-turn", AgentStatus::Completed(None), Some((ThreadId::new(), goal.clone())), - None + /*completion*/ None ) .await .unwrap() @@ -252,7 +252,7 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a "continuation-turn", AgentStatus::Completed(None), Some((incarnation, goal)), - None + /*completion*/ None ) .await .unwrap() @@ -270,8 +270,8 @@ async fn mailbox_goal_terminal_matches_logical_binding_across_native_revisions_a ThreadId::new(), "uncaptured-turn", AgentStatus::Completed(None), - None, - None + /*goal*/ None, + /*completion*/ None ) .await .unwrap() @@ -307,8 +307,8 @@ async fn mailbox_terminal_recovers_exact_unadmitted_submission_idempotently() { agent, submission_id, AgentStatus::Errored("provider request setup failed".to_string()), - None, - None, + /*goal*/ None, + /*completion*/ None, ) .await .unwrap() @@ -340,8 +340,8 @@ async fn mailbox_terminal_recovers_exact_unadmitted_submission_idempotently() { agent, "unrelated-source-turn", AgentStatus::Errored("unrelated failure".to_string()), - None, - None, + /*goal*/ None, + /*completion*/ None, ) .await .unwrap(), @@ -353,8 +353,8 @@ async fn mailbox_terminal_recovers_exact_unadmitted_submission_idempotently() { agent, submission_id, AgentStatus::Errored("provider request setup failed".to_string()), - None, - None, + /*goal*/ None, + /*completion*/ None, ) .await .unwrap(), @@ -402,7 +402,7 @@ async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_c "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), - None + /*completion*/ None ) .await .unwrap() @@ -440,7 +440,7 @@ async fn mailbox_cold_goal_continuation_returns_recovery_needed_without_losing_c "recovered-turn", AgentStatus::Completed(None), Some((incarnation, goal)), - None + /*completion*/ None ) .await .unwrap() @@ -491,7 +491,7 @@ async fn mailbox_cold_deferred_handback_uses_loaded_parents_shared_store() { "accepted-turn", AgentStatus::Completed(None), Some((incarnation, goal.clone())), - None + /*completion*/ None ) .await .unwrap() diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index 1d9aee212b84..cae252bb13a1 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -976,7 +976,7 @@ mod tests { a, vec![b], ReturnWhen::Any, - false, + /*has_escape*/ false, generation, ) .unwrap(); @@ -985,7 +985,7 @@ mod tests { AgentPath::root(), AgentPath::root(), "native input", - true, + /*trigger_turn*/ true, )) .await; assert!(queue.has_pending_mailbox_items().await); @@ -995,7 +995,7 @@ mod tests { b, vec![a], ReturnWhen::All, - false, + /*has_escape*/ false, generation ), Err(MailboxError::StaleWaitSnapshot) @@ -1003,7 +1003,13 @@ mod tests { // Native input invalidated the older edge before publication, so a peer cannot // incorrectly prove that both owners remain blocked on one another. let _peer = mailbox - .begin_wait("peer-b".into(), b, vec![a], ReturnWhen::All, false) + .begin_wait( + "peer-b".into(), + b, + vec![a], + ReturnWhen::All, + /*has_escape*/ false, + ) .unwrap(); } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 8089a115c2ac..4a242071630c 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2939,8 +2939,8 @@ async fn stable_mailbox_message_retry_and_resumed_history_do_not_duplicate_body( session .replace_compacted_history( Vec::new(), - None, - None, + /*reference_context_item*/ None, + /*world_state_baseline*/ None, crate::compact::CompactedHistoryMetadata { message: "test checkpoint".to_string(), window_number: 1, diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 429d4fb9a910..f219f0c0b9df 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -2234,7 +2234,7 @@ fn mailbox_deferred_goal_handoff_rejects_replaced_native_incarnation() { parent_thread_id, "turn-a", Some(&exact_goal), - None, + /*goal_store*/ None, ) .is_ok() ); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs index e8a3c1694989..e24014266b86 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/mailbox_wait_tests.rs @@ -134,7 +134,7 @@ async fn work( .unwrap(); if let Some(outcome) = outcome { let result = mailbox - .record_outcome(&work, outcome, None, Continuation::None) + .record_outcome(&work, outcome, /*response*/ None, Continuation::None) .await .unwrap(); // This fixture represents a previously surfaced result. Writer custody has @@ -328,7 +328,14 @@ async fn observed_unsuccessful_work_remains_non_success_in_exact_and_mixed_repla Some(outcome), ) .await; - let pending = work(&mailbox, session.thread_id, agent, "pending", None).await; + let pending = work( + &mailbox, + session.thread_id, + agent, + "pending", + /*outcome*/ None, + ) + .await; for mode in ["any", "all"] { for targets in [ vec![failed.id.clone()], @@ -397,7 +404,10 @@ async fn unrelated_attention_does_not_erase_a_successful_selected_join() { Some(MailboxWakeCause::AlreadySatisfied) ); assert!(report.condition_satisfied); - let pending_attention = mailbox.attention(session.thread_id, 100).await.unwrap(); + let pending_attention = mailbox + .attention(session.thread_id, /*limit*/ 100) + .await + .unwrap(); assert_eq!(pending_attention.len(), 1); assert_eq!( pending_attention[0].id, @@ -409,7 +419,14 @@ async fn unrelated_attention_does_not_erase_a_successful_selected_join() { #[tokio::test] async fn wait_delivers_unobserved_result_before_acknowledging_observation() { let (session, turn, mailbox, agent, rollout_path) = durable_fixture().await; - let work_ref = work(&mailbox, session.thread_id, agent, "deliver-result", None).await; + let work_ref = work( + &mailbox, + session.thread_id, + agent, + "deliver-result", + /*outcome*/ None, + ) + .await; let body = "durable result body"; let result_id = record_result( &mailbox, @@ -522,7 +539,7 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { assert_eq!(mailbox.quiet_count(session.thread_id).await.unwrap(), 1); assert!( mailbox - .attention(session.thread_id, 100) + .attention(session.thread_id, /*limit*/ 100) .await .unwrap() .is_empty() @@ -560,7 +577,7 @@ async fn wait_delivers_exact_quiet_reply_and_persists_it_before_observation() { assert_eq!(success, Some(true)); assert!( mailbox - .attention(session.thread_id, 100) + .attention(session.thread_id, /*limit*/ 100) .await .unwrap() .is_empty() @@ -607,7 +624,7 @@ async fn wait_leaves_over_limit_bodies_retrievable_and_a_later_exact_wait_delive session.thread_id, agent, &format!("bounded-delivery-{index}"), - None, + /*outcome*/ None, ) .await; let body = format!("bounded body {index}"); @@ -704,7 +721,7 @@ async fn interrupted_wait_call_id_replays_failure_but_a_fresh_id_uses_the_wait_p session.thread_id, agent, "pending-after-interruption", - None, + /*outcome*/ None, ) .await; let call_id = "interrupted-wait-call"; diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 6bc99318bb53..1e117affdcf9 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -1146,7 +1146,7 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com ); assert!( mailbox - .pending(root_thread_id, 10) + .pending(root_thread_id, /*limit*/ 10) .await? .iter() .any(|pending| pending.id.as_str() == message_id) @@ -1184,7 +1184,7 @@ async fn unadmitted_canonical_message_survives_precreated_failure_and_manual_com ); assert!( mailbox - .pending(root_thread_id, 10) + .pending(root_thread_id, /*limit*/ 10) .await? .iter() .any(|pending| pending.id.as_str() == message_id) From a985d87b4fb1c2ea2e89a215096252f47fcdf143 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:01:31 +1100 Subject: [PATCH 31/54] fix(ci): use GNU host for fork Windows release checks --- .github/scripts/test_ci_planners.py | 21 +++++++++++++++++++++ .github/workflows/bazel.yml | 11 ++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index ddd9a0f63f5f..7b7a05ed3486 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1636,6 +1636,27 @@ def test_bazel_windows_tests_serialize_host_global_policy_state(self) -> None: "bazel_test_args+=(--host_platform=//:local_windows)", windows_test_run ) + def test_bazel_windows_release_build_fork_uses_gnullvm_host(self) -> None: + payload = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") + release_steps = ( + ((payload.get("jobs") or {}).get("verify-release-build") or {}).get("steps") + or [] + ) + release_run = next( + step + for step in release_steps + if step.get("name") == "bazel build verify-release-build targets" + ).get("run") or "" + fork_no_key_windows_condition = ( + '[[ "${RUNNER_OS}" == "Windows" ' + '&& "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" ' + '&& -z "${BUILDBUDDY_API_KEY:-}" ]]' + ) + self.assertIn(fork_no_key_windows_condition, release_run) + self.assertIn( + "bazel_build_args+=(--host_platform=//:local_windows)", release_run + ) + def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> None: bazel = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") self.assertNotIn("test-windows-native-main", bazel.get("jobs") or {}) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 3ac36d27a51f..790adc9bbdeb 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -574,11 +574,9 @@ jobs: # Rust debug assertions explicitly. bazel_wrapper_args=() if [[ "${RUNNER_OS}" == "Windows" ]]; then - # This is build-only signal, so use the same Linux-RBE - # cross-compile path as the fast Windows test and clippy jobs. - # Fork/community PRs without the BuildBuddy secret fall back - # inside `run-bazel-ci.sh` to the previous local Windows MSVC - # host-platform shape. + # Use the same Windows cross-compile wrapper path as the fast test + # and Clippy jobs. Without RBE, the wrapper keeps its generic MSVC + # fallback; this fork overrides it with gnullvm below. bazel_wrapper_args+=(--windows-cross-compile) fi @@ -590,6 +588,9 @@ jobs: --build_metadata=TAG_job=verify-release-build --build_metadata=TAG_rust_debug_assertions=off ) + if [[ "${RUNNER_OS}" == "Windows" && "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" && -z "${BUILDBUDDY_API_KEY:-}" ]]; then + bazel_build_args+=(--host_platform=//:local_windows) + fi if [[ "${RUNNER_OS}" == "macOS" ]]; then # Hosted macOS runners can exhaust the Bazel client thread/process # ceiling when inheriting the high remote-execution job count. From 69b2dfea47d8684f308955b6c9c5f5439cca52e7 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:36:59 +1100 Subject: [PATCH 32/54] fix(ci): budget cold hosted-local compiler jobs --- .github/scripts/test_ci_planners.py | 10 ++++++++++ .github/workflows/bazel.yml | 4 +++- .github/workflows/sdk.yml | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 7b7a05ed3486..33d14604a9f1 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1657,6 +1657,16 @@ def test_bazel_windows_release_build_fork_uses_gnullvm_host(self) -> None: "bazel_build_args+=(--host_platform=//:local_windows)", release_run ) + def test_fork_hosted_local_compile_jobs_keep_bounded_full_build_budget(self) -> None: + fork_budget = "${{ github.repository == 'sednalabs/codex' && 120 || 30 }}" + for workflow_name, job_name in (("bazel.yml", "clippy"), ("sdk.yml", "sdks")): + with self.subTest(workflow=workflow_name, job=job_name): + payload = load_workflow_payload( + REPO_ROOT / ".github/workflows" / workflow_name + ) + job = (payload.get("jobs") or {}).get(job_name) or {} + self.assertEqual(job.get("timeout-minutes"), fork_budget) + def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> None: bazel = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") self.assertNotIn("test-windows-native-main", bazel.get("jobs") or {}) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 790adc9bbdeb..e0cc43d1c8bd 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -417,7 +417,9 @@ jobs: clippy: needs: plan if: ${{ needs.plan.outputs.run_bazel == 'true' }} - timeout-minutes: 30 + # Cold hosted-local V8/toolchain builds need the full Bazel-test budget. + # Preserve upstream's remote-enabled budget outside this fork. + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 120 || 30 }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 6976023d6db8..5d27f60c4602 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -59,7 +59,8 @@ jobs: sdks: if: ${{ inputs.run_typescript }} runs-on: ubuntu-24.04 - timeout-minutes: 30 + # The fork builds Codex locally on the standard hosted runner before SDK tests. + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 120 || 30 }} environment: name: bazel deployment: false From a3a3d44a50c1073b391e34e6828d8fd9bb955948 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:08:34 +1100 Subject: [PATCH 33/54] test(pty): restore Ctrl-C handling in Windows fixture --- codex-rs/utils/pty/src/windows_tests.rs | 26 ++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/codex-rs/utils/pty/src/windows_tests.rs b/codex-rs/utils/pty/src/windows_tests.rs index f778aab7385f..883a42409624 100644 --- a/codex-rs/utils/pty/src/windows_tests.rs +++ b/codex-rs/utils/pty/src/windows_tests.rs @@ -302,7 +302,25 @@ async fn conpty_ctrl_c_interrupts_powershell_foreground_child() -> anyhow::Resul let Some(program) = find_powershell() else { return Ok(()); }; - let args = vec!["-NoLogo".to_string(), "-NoProfile".to_string()]; + // Bazel starts Windows test processes in a new process group with Ctrl-C + // disabled, and children inherit that setting. Restore it only inside this + // fixture's shell, not process-globally in the parallel test harness. + let restore_ctrl_c = r#" +Add-Type -Namespace CodexPtyTests -Name ConsoleControl -MemberDefinition ' +[System.Runtime.InteropServices.DllImport("kernel32.dll", SetLastError = true)] +public static extern bool SetConsoleCtrlHandler(System.IntPtr handler, bool add); +' +if (-not [CodexPtyTests.ConsoleControl]::SetConsoleCtrlHandler([IntPtr]::Zero, $false)) { + throw "SetConsoleCtrlHandler failed: $([System.Runtime.InteropServices.Marshal]::GetLastWin32Error())" +} +"#; + let args = vec![ + "-NoLogo".to_string(), + "-NoProfile".to_string(), + "-NoExit".to_string(), + "-Command".to_string(), + format!("{restore_ctrl_c}\n[Console]::WriteLine('{SHELL_READY_MARKER}')"), + ]; let env: HashMap = std::env::vars().collect(); let spawned = spawn_pty_process( &program, @@ -316,6 +334,12 @@ async fn conpty_ctrl_c_interrupts_powershell_foreground_child() -> anyhow::Resul .await?; let (session, mut output_rx, exit_rx) = combine_spawned_output(spawned); let writer = session.writer_sender(); + wait_for_output_contains( + &mut output_rx, + SHELL_READY_MARKER, + /*timeout_ms*/ 10_000, + ) + .await?; writer.send(b"ping.exe -4 -t localhost\n".to_vec()).await?; wait_for_output_contains(&mut output_rx, "127.0.0.1", /*timeout_ms*/ 10_000).await?; From 0a5adf98c96ec7991cc8fc2d30a99baf4e23eb75 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:21:26 +1100 Subject: [PATCH 34/54] test(exec-server): allow normal Windows handshake startup --- codex-rs/exec-server/src/client.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 1ff98fcc7ce0..ea47c12d1dd4 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -1852,7 +1852,8 @@ mod tests { cwd: None, }, client_name: "stdio-test-client".to_string(), - initialize_timeout: Duration::from_secs(1), + // This verifies the handshake, not cold PowerShell startup latency. + initialize_timeout: DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT, resume_session_id: None, }) .await From 91c55ba6fea5662345a99fe00819dc783dc39c54 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:21:27 +1100 Subject: [PATCH 35/54] fix(bazel): keep WebRTC build caches in action outputs --- .../webrtc-sys_hermetic_darwin_sysroot.patch | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/patches/webrtc-sys_hermetic_darwin_sysroot.patch b/patches/webrtc-sys_hermetic_darwin_sysroot.patch index 1420dea6d9b1..355ce8136742 100644 --- a/patches/webrtc-sys_hermetic_darwin_sysroot.patch +++ b/patches/webrtc-sys_hermetic_darwin_sysroot.patch @@ -6,7 +6,18 @@ index 3b4a24a..6aa06ff 100644 -use std::path::Path; +use std::fs; +use std::path::Path; -@@ -118,1 +119,9 @@ +@@ -63,1 +64,9 @@ +- let mut builder = cxx_build::bridges(rust_files); ++ if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_some() { ++ // Keep cxx-build's shared debug cache in this action's writable output, ++ // not scratch's dependency OUT_DIR, which is read-only under Bazel. ++ let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); ++ fs::create_dir_all(&out_dir).unwrap(); ++ env::set_var("CARGO_TARGET_DIR", out_dir.canonicalize().unwrap()); ++ } ++ ++ let mut builder = cxx_build::bridges(rust_files); +@@ -118,1 +127,9 @@ - println!("cargo:rustc-link-search=native={}", webrtc_lib.to_str().unwrap()); + let webrtc_link_lib = if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_some() { + let out_lib = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("lib"); @@ -17,10 +28,10 @@ index 3b4a24a..6aa06ff 100644 + webrtc_lib + }; + println!("cargo:rustc-link-search=native={}", webrtc_link_lib.to_str().unwrap()); -@@ -255,1 +264,1 @@ +@@ -255,1 +272,1 @@ - println!("cargo:rustc-link-lib=framework=Appkit"); + println!("cargo:rustc-link-lib=framework=AppKit"); -@@ -320,1 +329,18 @@ +@@ -320,1 +337,18 @@ - builder.warnings(false).compile("webrtcsys-cxx"); + builder.warnings(false).compile("webrtcsys-cxx"); + let cxxbridge_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("cxxbridge"); @@ -40,12 +51,12 @@ index 3b4a24a..6aa06ff 100644 + fs::remove_file(crate_link).unwrap(); + } + -@@ -375,1 +384,3 @@ +@@ -375,1 +409,3 @@ - println!("cargo:rustc-link-lib={}", clang_rt); + if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_none() { + println!("cargo:rustc-link-lib={}", clang_rt); + } -@@ -378,15 +389,32 @@ +@@ -378,15 +414,32 @@ - let sysroot = Command::new("xcrun").args(["--sdk", sdk, "--show-sdk-path"]).output().unwrap(); - - let sysroot = String::from_utf8_lossy(&sysroot.stdout); @@ -93,3 +104,23 @@ index 3b4a24a..6aa06ff 100644 + } + } + } +diff --git a/webrtc-sys/build/src/lib.rs b/webrtc-sys/build/src/lib.rs +--- a/webrtc-sys/build/src/lib.rs ++++ b/webrtc-sys/build/src/lib.rs +@@ -87,1 +87,10 @@ ++fn webrtc_scratch_dir() -> path::PathBuf { ++ // Dependency outputs are read-only in Bazel; use the calling action's output. ++ if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_some() { ++ path::PathBuf::from(env::var_os("OUT_DIR").unwrap()).join(SCRATH_PATH) ++ } else { ++ scratch::path(SCRATH_PATH) ++ } ++} ++ + /// Location of the downloaded webrtc binaries +@@ -92,1 +101,1 @@ +- let target_dir = scratch::path(SCRATH_PATH); ++ let target_dir = webrtc_scratch_dir(); +@@ -200,1 +209,1 @@ +- let dir = scratch::path(SCRATH_PATH); ++ let dir = webrtc_scratch_dir(); From 027cd1ee6604cbed5d53677bedd5bf88c4d6151e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:30:03 +1100 Subject: [PATCH 36/54] test(exec-server): import handshake timeout on Windows --- codex-rs/exec-server/src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index ea47c12d1dd4..4020f9cde612 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -1549,7 +1549,6 @@ mod tests { use super::LazyRemoteExecServerClient; use crate::EnvironmentObservedStatus; use crate::ProcessId; - #[cfg(not(windows))] use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT; use crate::client_api::ExecServerTransportParams; use crate::client_api::RemoteExecServerConnectArgs; From 59b1472fa447618bd39fb61ac22de4ca9c176203 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:39:57 +1100 Subject: [PATCH 37/54] fix(ci): budget cold hosted-local validation --- .github/scripts/test_ci_planners.py | 19 ++++++++++++++++--- .github/workflows/bazel.yml | 10 +++++++--- .github/workflows/sdk.yml | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 33d14604a9f1..6b16526bbca9 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1658,14 +1658,27 @@ def test_bazel_windows_release_build_fork_uses_gnullvm_host(self) -> None: ) def test_fork_hosted_local_compile_jobs_keep_bounded_full_build_budget(self) -> None: - fork_budget = "${{ github.repository == 'sednalabs/codex' && 120 || 30 }}" - for workflow_name, job_name in (("bazel.yml", "clippy"), ("sdk.yml", "sdks")): + fork_budget = "${{ github.repository == 'sednalabs/codex' && 180 || 30 }}" + for workflow_name, job_name, expected_budget in ( + ("bazel.yml", "clippy", fork_budget), + ("sdk.yml", "sdks", fork_budget), + ( + "bazel.yml", + "test", + "${{ github.repository == 'sednalabs/codex' && 180 || 120 }}", + ), + ( + "bazel.yml", + "verify-release-build", + "${{ github.repository == 'sednalabs/codex' && 180 || 60 }}", + ), + ): with self.subTest(workflow=workflow_name, job=job_name): payload = load_workflow_payload( REPO_ROOT / ".github/workflows" / workflow_name ) job = (payload.get("jobs") or {}).get(job_name) or {} - self.assertEqual(job.get("timeout-minutes"), fork_budget) + self.assertEqual(job.get("timeout-minutes"), expected_budget) def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> None: bazel = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index e0cc43d1c8bd..7d45a0490aec 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -170,7 +170,9 @@ jobs: # the default window while build actions execute remotely. needs: plan if: ${{ needs.plan.outputs.run_bazel == 'true' }} - timeout-minutes: 120 + # This fork executes cold builds on standard hosted runners only. + # Other repositories retain their existing remote-enabled budget. + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 120 }} strategy: fail-fast: false matrix: @@ -419,7 +421,7 @@ jobs: if: ${{ needs.plan.outputs.run_bazel == 'true' }} # Cold hosted-local V8/toolchain builds need the full Bazel-test budget. # Preserve upstream's remote-enabled budget outside this fork. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 120 || 30 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 30 }} strategy: fail-fast: false matrix: @@ -532,7 +534,9 @@ jobs: verify-release-build: needs: plan if: ${{ needs.plan.outputs.run_bazel == 'true' }} - timeout-minutes: 60 + # Hosted-local cold builds need the same bounded budget as test jobs. + # Other repositories retain their existing release-verifier limit. + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 60 }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 5d27f60c4602..bca9989a16de 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -60,7 +60,7 @@ jobs: if: ${{ inputs.run_typescript }} runs-on: ubuntu-24.04 # The fork builds Codex locally on the standard hosted runner before SDK tests. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 120 || 30 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 30 }} environment: name: bazel deployment: false From 07e313c9873a891e7b239e9691f680ad50b3dc38 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:21:01 +1100 Subject: [PATCH 38/54] fix(ci): allow cold shell startup and reclaim unused SDK space --- .github/actions/setup-bazel-ci/action.yml | 18 +++++++++++++++ .github/scripts/test_ci_planners.py | 27 +++++++++++++++++++++++ codex-rs/utils/pty/src/windows_tests.rs | 17 +++++++++----- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-bazel-ci/action.yml b/.github/actions/setup-bazel-ci/action.yml index 1948867a8357..87f05dc3ac72 100644 --- a/.github/actions/setup-bazel-ci/action.yml +++ b/.github/actions/setup-bazel-ci/action.yml @@ -15,6 +15,24 @@ runs: - id: setup_ci uses: ./.github/actions/setup-ci + - name: Reclaim unused hosted Linux Android SDK + if: ${{ github.repository == 'sednalabs/codex' && runner.os == 'Linux' && runner.environment == 'github-hosted' }} + shell: bash + run: | + set -euo pipefail + # Native Linux Rust targets do not use this preinstalled image SDK. + # Keep the workspace, tool cache, and Bazel/Cargo caches intact. + df -h / + if [[ -d /usr/local/lib/android ]]; then + if [[ "$(realpath /usr/local/lib/android)" != /usr/local/lib/android ]]; then + echo "::error::Refusing to reclaim a non-canonical runner SDK path" + exit 1 + fi + du -sh /usr/local/lib/android + sudo rm -rf -- /usr/local/lib/android + fi + df -h / + - name: Set up Bazel uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 # Without an explicit Bazelisk version, setup-bazel leaves PATH unchanged diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 6b16526bbca9..03d18f607627 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -2121,6 +2121,33 @@ def test_bazel_ci_docs_only_plan_is_fail_closed_and_preserves_required_signal(se ) self.assertEqual(proc.returncode, 0, proc.stderr) + def test_fork_hosted_linux_bazel_reclaims_only_unused_image_sdk(self) -> None: + payload = load_workflow_payload( + REPO_ROOT / ".github/actions/setup-bazel-ci/action.yml" + ) + steps = (payload.get("runs") or {}).get("steps") or [] + cleanup = next( + step + for step in steps + if step.get("name") == "Reclaim unused hosted Linux Android SDK" + ) + self.assertEqual( + cleanup.get("if"), + "${{ github.repository == 'sednalabs/codex' && runner.os == 'Linux' " + "&& runner.environment == 'github-hosted' }}", + ) + self.assertEqual(cleanup.get("shell"), "bash") + script = cleanup.get("run") or "" + self.assertIn("realpath /usr/local/lib/android", script) + removal_commands = [ + line.strip() + for line in script.splitlines() + if line.strip().startswith("sudo rm ") + ] + self.assertEqual(removal_commands, ["sudo rm -rf -- /usr/local/lib/android"]) + setup_bazel = next(step for step in steps if step.get("name") == "Set up Bazel") + self.assertLess(steps.index(cleanup), steps.index(setup_bazel)) + def test_bazel_cache_writes_are_limited_to_trusted_post_merge_pushes(self) -> None: setup_action = load_workflow_payload(REPO_ROOT / ".github/actions/setup-bazel-ci/action.yml") setup_bazel_step = next( diff --git a/codex-rs/utils/pty/src/windows_tests.rs b/codex-rs/utils/pty/src/windows_tests.rs index 883a42409624..4c7343016fe3 100644 --- a/codex-rs/utils/pty/src/windows_tests.rs +++ b/codex-rs/utils/pty/src/windows_tests.rs @@ -11,6 +11,9 @@ use std::time::Duration; const READY_MARKER: &str = "__CODEX_CHILD_READY__"; const SHELL_READY_MARKER: &str = "__CODEX_SHELL_READY__"; +// Cold PowerShell startup, including Add-Type initialization, is a fixture +// precondition rather than the foreground input or Ctrl-C behavior under test. +const SHELL_STARTUP_TIMEOUT_MS: u64 = 120_000; const VALUE_MARKER: &str = "__CODEX_CHILD_VALUE__"; struct WindowsShell { @@ -255,11 +258,13 @@ async fn conpty_delivers_input_to_foreground_children() -> anyhow::Result<()> { let (session, mut output_rx, exit_rx) = combine_spawned_output(spawned); let writer = session.writer_sender(); if let Some(marker) = shell.ready_marker { - wait_for_output_contains(&mut output_rx, marker, /*timeout_ms*/ 10_000) - .await - .map_err(|err| { - anyhow::anyhow!("{} shell did not become ready: {err}", shell.name) - })?; + wait_for_output_contains( + &mut output_rx, + marker, + /*timeout_ms*/ SHELL_STARTUP_TIMEOUT_MS, + ) + .await + .map_err(|err| anyhow::anyhow!("{} shell did not become ready: {err}", shell.name))?; } writer .send(format!("{}\n", shell.child_command).into_bytes()) @@ -337,7 +342,7 @@ if (-not [CodexPtyTests.ConsoleControl]::SetConsoleCtrlHandler([IntPtr]::Zero, $ wait_for_output_contains( &mut output_rx, SHELL_READY_MARKER, - /*timeout_ms*/ 10_000, + /*timeout_ms*/ SHELL_STARTUP_TIMEOUT_MS, ) .await?; writer.send(b"ping.exe -4 -t localhost\n".to_vec()).await?; From 027d3407b17ef6c7e3158727938277d5f79eb46e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:45:31 +1100 Subject: [PATCH 39/54] fix(ci): match native fork budgets to the hosted runner limit --- .github/scripts/test_ci_planners.py | 6 +++--- .github/workflows/bazel.yml | 6 +++--- .github/workflows/sdk.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 03d18f607627..73f158729d59 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1658,19 +1658,19 @@ def test_bazel_windows_release_build_fork_uses_gnullvm_host(self) -> None: ) def test_fork_hosted_local_compile_jobs_keep_bounded_full_build_budget(self) -> None: - fork_budget = "${{ github.repository == 'sednalabs/codex' && 180 || 30 }}" + fork_budget = "${{ github.repository == 'sednalabs/codex' && 360 || 30 }}" for workflow_name, job_name, expected_budget in ( ("bazel.yml", "clippy", fork_budget), ("sdk.yml", "sdks", fork_budget), ( "bazel.yml", "test", - "${{ github.repository == 'sednalabs/codex' && 180 || 120 }}", + "${{ github.repository == 'sednalabs/codex' && 360 || 120 }}", ), ( "bazel.yml", "verify-release-build", - "${{ github.repository == 'sednalabs/codex' && 180 || 60 }}", + "${{ github.repository == 'sednalabs/codex' && 360 || 60 }}", ), ): with self.subTest(workflow=workflow_name, job=job_name): diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 7d45a0490aec..f14a0bad701a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -172,7 +172,7 @@ jobs: if: ${{ needs.plan.outputs.run_bazel == 'true' }} # This fork executes cold builds on standard hosted runners only. # Other repositories retain their existing remote-enabled budget. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 120 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 360 || 120 }} strategy: fail-fast: false matrix: @@ -421,7 +421,7 @@ jobs: if: ${{ needs.plan.outputs.run_bazel == 'true' }} # Cold hosted-local V8/toolchain builds need the full Bazel-test budget. # Preserve upstream's remote-enabled budget outside this fork. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 30 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 360 || 30 }} strategy: fail-fast: false matrix: @@ -536,7 +536,7 @@ jobs: if: ${{ needs.plan.outputs.run_bazel == 'true' }} # Hosted-local cold builds need the same bounded budget as test jobs. # Other repositories retain their existing release-verifier limit. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 60 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 360 || 60 }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index bca9989a16de..184c22e37a88 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -60,7 +60,7 @@ jobs: if: ${{ inputs.run_typescript }} runs-on: ubuntu-24.04 # The fork builds Codex locally on the standard hosted runner before SDK tests. - timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 180 || 30 }} + timeout-minutes: ${{ github.repository == 'sednalabs/codex' && 360 || 30 }} environment: name: bazel deployment: false From f219529ee4ed176cb552e6b4eaf06a28fe231d94 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:48:02 +1100 Subject: [PATCH 40/54] fix(ci): prepare hosted Linux namespaces for sandbox tests --- .github/scripts/test_ci_planners.py | 34 +++++++++++++++++++++++++++++ .github/workflows/bazel.yml | 11 ++++++++++ 2 files changed, 45 insertions(+) diff --git a/.github/scripts/test_ci_planners.py b/.github/scripts/test_ci_planners.py index 73f158729d59..6cbb488c916f 100644 --- a/.github/scripts/test_ci_planners.py +++ b/.github/scripts/test_ci_planners.py @@ -1680,6 +1680,40 @@ def test_fork_hosted_local_compile_jobs_keep_bounded_full_build_budget(self) -> job = (payload.get("jobs") or {}).get(job_name) or {} self.assertEqual(job.get("timeout-minutes"), expected_budget) + def test_fork_hosted_linux_bazel_tests_prepare_user_namespaces(self) -> None: + payload = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") + steps = ((payload.get("jobs") or {}).get("test") or {}).get("steps") or [] + prepare_step = next( + step + for step in steps + if step.get("name") == "Enable user namespaces for hosted Linux sandbox tests" + ) + self.assertEqual( + prepare_step.get("if"), + "${{ github.repository == 'sednalabs/codex' && runner.os == 'Linux' " + "&& runner.environment == 'github-hosted' }}", + ) + self.assertEqual(prepare_step.get("shell"), "bash") + prepare_run = prepare_step.get("run") or "" + self.assertIn("sudo sysctl -w kernel.unprivileged_userns_clone=1", prepare_run) + self.assertIn( + "if sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns " + ">/dev/null 2>&1; then", + prepare_run, + ) + self.assertIn( + "sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0", + prepare_run, + ) + self.assertLess( + steps.index(prepare_step), + next( + index + for index, step in enumerate(steps) + if step.get("name") == "bazel test //..." + ), + ) + def test_bazel_windows_native_main_uses_gnullvm_proc_macro_toolchain(self) -> None: bazel = load_workflow_payload(REPO_ROOT / ".github/workflows/bazel.yml") self.assertNotIn("test-windows-native-main", bazel.get("jobs") or {}) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index f14a0bad701a..933ce37e7197 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -216,6 +216,17 @@ jobs: target: ${{ matrix.target }} cache-scope: bazel-${{ github.job }} + - name: Enable user namespaces for hosted Linux sandbox tests + if: ${{ github.repository == 'sednalabs/codex' && runner.os == 'Linux' && runner.environment == 'github-hosted' }} + shell: bash + run: | + set -euo pipefail + # Match the native Linux test prerequisites on this disposable runner. + sudo sysctl -w kernel.unprivileged_userns_clone=1 + if sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + - name: Check rusty_v8 MODULE.bazel checksums if: matrix.os == 'ubuntu-24.04' && matrix.target == 'x86_64-unknown-linux-gnu' shell: bash From d24c920d4b3d522efa37c3f1116cf0cfd71310b8 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:22:23 +1100 Subject: [PATCH 41/54] fix(ci): retain lean local settings in hosted fork fallback --- .github/scripts/run_bazel_with_buildbuddy.py | 12 ++ .../scripts/test_run_bazel_with_buildbuddy.py | 113 ++++++++++++++++++ bazel/rules/testing/wine/BUILD.bazel | 3 +- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run_bazel_with_buildbuddy.py b/.github/scripts/run_bazel_with_buildbuddy.py index afbf0b656ec8..675f1a971b29 100755 --- a/.github/scripts/run_bazel_with_buildbuddy.py +++ b/.github/scripts/run_bazel_with_buildbuddy.py @@ -141,6 +141,18 @@ def bazel_args_with_remote_config( config = remote_config(args, env) if config is None: configured_args = bazel_args_without_remote_execution(args) + if ( + env.get("GITHUB_ACTIONS") == "true" + and env.get("GITHUB_REPOSITORY") == "sednalabs/codex" + and env.get("RUNNER_ENVIRONMENT") == "github-hosted" + and uses_remote_execution(args) + ): + # Removing a remote platform must not also lose the shared CI + # settings: lean Rust artifacts and no duplicate on-runner disk CAS. + local_command_idx = next( + idx for idx, arg in enumerate(configured_args) if not arg.startswith("-") + ) + configured_args.insert(local_command_idx + 1, "--config=ci-bazel") else: # `remote_config()` returns a configuration only when this key is present. api_key = env["BUILDBUDDY_API_KEY"] diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 0b256e12590b..c3c2b5a0609b 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -81,6 +81,108 @@ def test_keyless_hosted_fallback_preserves_every_remote_ci_target(self) -> None: ["test", "--", "//codex-rs/cli:codex", "//..."], ) + def test_keyless_hosted_fork_retains_only_shared_local_ci_settings(self) -> None: + env = { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + } + for config in run_bazel_with_buildbuddy.REMOTE_EXECUTION_CONFIGS: + with self.subTest(config=config): + self.assertEqual( + run_bazel_with_buildbuddy.bazel_args_with_remote_config( + [ + "--output_user_root=/tmp/test-bazel-output", + "test", + config, + "--test_tag_filters=-argument-comment-lint", + "--", + "//codex-rs/cli:codex", + "//...", + ], + env, + ), + [ + "--output_user_root=/tmp/test-bazel-output", + "test", + "--config=ci-bazel", + "--test_tag_filters=-argument-comment-lint", + "--", + "//codex-rs/cli:codex", + "//...", + ], + ) + + def test_shared_local_ci_fallback_is_limited_to_this_hosted_fork(self) -> None: + for env in ( + {}, + {"GITHUB_REPOSITORY": "sednalabs/codex"}, + { + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + }, + { + "GITHUB_ACTIONS": "false", + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + }, + { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "openai/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + }, + { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "example/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + }, + {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": "sednalabs/codex"}, + { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "self-hosted", + }, + ): + with self.subTest(env=env): + self.assertEqual( + run_bazel_with_buildbuddy.bazel_args_with_remote_config( + ["build", "--config=ci-linux", "--", "//codex-rs/cli:codex"], + env, + ), + ["build", "--", "//codex-rs/cli:codex"], + ) + + def test_shared_ci_config_is_lean_without_remote_endpoints_or_disk_cas(self) -> None: + bazelrc = Path(__file__).resolve().parents[2] / ".bazelrc" + shared_ci = [ + line + for line in bazelrc.read_text(encoding="utf-8").splitlines() + if line.startswith(("common:ci ", "common:ci-bazel ")) + ] + self.assertIn("common:ci-bazel --config=ci", shared_ci) + self.assertIn("common:ci --disk_cache=", shared_ci) + self.assertIn( + "common:ci --@rules_rust//rust/settings:extra_rustc_flag=-Cdebuginfo=0", + shared_ci, + ) + self.assertIn( + "common:ci --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebuginfo=0", + shared_ci, + ) + for forbidden in ( + "--config=buildbuddy-", + "--config=remote", + "--remote_executor=", + "--remote_cache=", + "--bes_backend=", + "--experimental_remote_downloader=", + "--strategy=remote", + "--platforms=//:rbe", + "--extra_execution_platforms=", + ): + with self.subTest(forbidden=forbidden): + self.assertFalse(any(forbidden in line for line in shared_ci)) + def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> None: args = ["run", "//codex-rs/cli:codex", "--", "--config=remote"] @@ -88,6 +190,17 @@ def test_program_arguments_after_separator_do_not_select_or_lose_rbe(self) -> No run_bazel_with_buildbuddy.bazel_args_with_remote_config(args, {}), args, ) + self.assertEqual( + run_bazel_with_buildbuddy.bazel_args_with_remote_config( + ["run", "//codex-rs/cli:codex", "--", "--config=ci-linux"], + { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + }, + ), + ["run", "//codex-rs/cli:codex", "--", "--config=ci-linux"], + ) self.assertEqual( run_bazel_with_buildbuddy.remote_config( args, {"BUILDBUDDY_API_KEY": "fork-token"} diff --git a/bazel/rules/testing/wine/BUILD.bazel b/bazel/rules/testing/wine/BUILD.bazel index 4bd9acf6d7a6..85403241367b 100644 --- a/bazel/rules/testing/wine/BUILD.bazel +++ b/bazel/rules/testing/wine/BUILD.bazel @@ -42,7 +42,8 @@ rust_binary( wine_rust_test( name = "wine-test-support-unit-tests", - timeout = "short", + # Allow isolated Wine prefix startup and teardown to finish reporting. + timeout = "moderate", crate = ":wine_test_support", windows_binaries = { "wine-smoke": ":windows-smoke", From 9cfdec16f46c617b5369fc0a7bdf4d6d2e752eea Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:26:27 +1100 Subject: [PATCH 42/54] fix(ci): retain interrupted sharded test diagnostics --- .github/scripts/run-bazel-ci.sh | 15 ++++- .../scripts/test_run_bazel_with_buildbuddy.py | 62 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 9e80d65817c5..047addb41f26 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -141,7 +141,20 @@ print_bazel_test_log_tails() { if [[ -f "$test_log" ]]; then tail -n 200 "$test_log" else - echo "Missing test log: $test_log" + # A build failure can interrupt a sharded target before its final test.log + # exists. Retain the failed attempt evidence already written by Bazel. + local found_attempt=0 + local attempt_log + for attempt_log in "${testlogs_dir}/${rel_path}"/shard_*_of_*/test_attempts/attempt_*.log; do + if [[ -f "$attempt_log" ]]; then + found_attempt=1 + echo "Bazel sharded attempt log: ${attempt_log}" + tail -n 200 "$attempt_log" + fi + done + if [[ $found_attempt -eq 0 ]]; then + echo "Missing test log: $test_log" + fi fi echo "::endgroup::" done diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index c3c2b5a0609b..0238299e0e65 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -425,6 +425,68 @@ def test_bazel_command_adds_local_caches_before_separator(self) -> None: ], ) + def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + testlogs = root / "testlogs" + markers = [] + for shard in (2, 3, 5): + attempt = ( + testlogs + / "codex-rs/core/core-unit-tests" + / f"shard_{shard}_of_8/test_attempts/attempt_1.log" + ) + attempt.parent.mkdir(parents=True) + marker = f"failed-shard-{shard}-evidence" + attempt.write_text(marker + "\n", encoding="utf-8") + markers.append(marker) + + fake_bazel = root / "fake-bazel" + fake_bazel.write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "if 'info' in sys.argv:\n" + " print(os.environ['TEST_BAZEL_TESTLOGS_DIR'])\n" + " sys.exit(0)\n" + "for shard in (2, 3, 5):\n" + " path = os.environ['TEST_BAZEL_TESTLOGS_DIR'] + " + "f'/codex-rs/core/core-unit-tests/shard_{shard}_of_8/' " + "+ 'test_attempts/attempt_1.log'\n" + " print(f'FAIL: //codex-rs/core:core-unit-tests " + "(shard {shard} of 8) (Exit 101) (see {path})')\n" + "sys.exit(1)\n", + encoding="utf-8", + ) + fake_bazel.chmod(0o755) + env = os.environ.copy() + env.pop("BUILDBUDDY_API_KEY", None) + env.update( + CODEX_BAZEL_BIN=str(fake_bazel), + TEST_BAZEL_TESTLOGS_DIR=str(testlogs), + RUNNER_TEMP=str(root), + ) + result = subprocess.run( + [ + "bash", + str(repo_root / ".github/scripts/run-bazel-ci.sh"), + "--print-failed-test-logs", + "--", + "test", + "--", + "//codex-rs/core:core-unit-tests", + ], + cwd=repo_root, + env=env, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 1, result.stderr) + for marker in markers: + self.assertIn(marker, result.stdout) + self.assertNotIn("Missing test log:", result.stdout) + def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None: spaced_arg = ( r"--test_env=PATH=C:\Program Files\PowerShell\7;C:\Program Files\Git\bin" From d50fadb6c4395778c3fca5f02cc392191ad4776e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:14:22 +1100 Subject: [PATCH 43/54] fix(ci): run diagnostic fixture with native Windows shell --- .../scripts/test_run_bazel_with_buildbuddy.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 0238299e0e65..7b95b9ecece5 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -442,8 +442,8 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - attempt.write_text(marker + "\n", encoding="utf-8") markers.append(marker) - fake_bazel = root / "fake-bazel" - fake_bazel.write_text( + fake_script = root / "fake-bazel.py" + fake_script.write_text( f"#!{sys.executable}\n" "import os, sys\n" "if 'info' in sys.argv:\n" @@ -458,18 +458,29 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - "sys.exit(1)\n", encoding="utf-8", ) - fake_bazel.chmod(0o755) + bash = "bash" + if os.name == "nt": + # A bare bash can resolve to WSL rather than Git for Windows. + bash = str(Path(os.environ["ProgramFiles"]) / "Git/bin/bash.exe") + fake_bazel = root / "fake-bazel.cmd" + fake_bazel.write_text( + f'@echo off\n"{sys.executable}" "{fake_script}" %*\n', + encoding="utf-8", + ) + else: + fake_bazel = fake_script + fake_bazel.chmod(0o755) env = os.environ.copy() env.pop("BUILDBUDDY_API_KEY", None) env.update( CODEX_BAZEL_BIN=str(fake_bazel), - TEST_BAZEL_TESTLOGS_DIR=str(testlogs), - RUNNER_TEMP=str(root), + TEST_BAZEL_TESTLOGS_DIR=testlogs.as_posix(), + RUNNER_TEMP=root.as_posix(), ) result = subprocess.run( [ - "bash", - str(repo_root / ".github/scripts/run-bazel-ci.sh"), + bash, + (repo_root / ".github/scripts/run-bazel-ci.sh").as_posix(), "--print-failed-test-logs", "--", "test", From d2418d704922bb435e1ea5c9904b869e447b3dbe Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:28:12 +1100 Subject: [PATCH 44/54] test(ci): supply native Windows environment for log fixture --- .github/scripts/test_run_bazel_with_buildbuddy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 7b95b9ecece5..17f4eb4c6bc0 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -446,6 +446,7 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - fake_script.write_text( f"#!{sys.executable}\n" "import os, sys\n" + "sys.stdout.reconfigure(newline='\\n')\n" "if 'info' in sys.argv:\n" " print(os.environ['TEST_BAZEL_TESTLOGS_DIR'])\n" " sys.exit(0)\n" @@ -477,6 +478,9 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - TEST_BAZEL_TESTLOGS_DIR=testlogs.as_posix(), RUNNER_TEMP=root.as_posix(), ) + if os.name == "nt": + # This test runs before Prepare Bazel CI supplies the native PATH. + env["CODEX_BAZEL_WINDOWS_PATH"] = env["PATH"] result = subprocess.run( [ bash, @@ -495,7 +499,7 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - ) self.assertEqual(result.returncode, 1, result.stderr) for marker in markers: - self.assertIn(marker, result.stdout) + self.assertIn(marker, result.stdout, result.stdout + result.stderr) self.assertNotIn("Missing test log:", result.stdout) def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None: From a5d7070c36cff09803f24926e03cf8c8ea69927e Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:35:58 +1100 Subject: [PATCH 45/54] test(ci): bound fake Bazel arguments on Windows --- .github/scripts/test_run_bazel_with_buildbuddy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index 17f4eb4c6bc0..f76ead4edd33 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -479,8 +479,9 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - RUNNER_TEMP=root.as_posix(), ) if os.name == "nt": - # This test runs before Prepare Bazel CI supplies the native PATH. - env["CODEX_BAZEL_WINDOWS_PATH"] = env["PATH"] + # The fake only needs a valid PATH, not the runner's long tool list + # repeated in arguments to a command with cmd.exe's length limit. + env["CODEX_BAZEL_WINDOWS_PATH"] = str(Path(sys.executable).parent) result = subprocess.run( [ bash, From c7fdf23909df1ce80d1c63a6690a9e0df127669f Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:07:57 +1100 Subject: [PATCH 46/54] test(pty): await resumed PowerShell prompt after Ctrl-C --- codex-rs/utils/pty/src/windows_tests.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/codex-rs/utils/pty/src/windows_tests.rs b/codex-rs/utils/pty/src/windows_tests.rs index 4c7343016fe3..82b9be6e8f19 100644 --- a/codex-rs/utils/pty/src/windows_tests.rs +++ b/codex-rs/utils/pty/src/windows_tests.rs @@ -304,6 +304,7 @@ async fn conpty_delivers_input_to_foreground_children() -> anyhow::Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn conpty_ctrl_c_interrupts_powershell_foreground_child() -> anyhow::Result<()> { + const PROMPT_READY_MARKER: &str = "__CODEX_SHELL_PROMPT__"; let Some(program) = find_powershell() else { return Ok(()); }; @@ -324,7 +325,9 @@ if (-not [CodexPtyTests.ConsoleControl]::SetConsoleCtrlHandler([IntPtr]::Zero, $ "-NoProfile".to_string(), "-NoExit".to_string(), "-Command".to_string(), - format!("{restore_ctrl_c}\n[Console]::WriteLine('{SHELL_READY_MARKER}')"), + format!( + "{restore_ctrl_c}\nfunction global:prompt {{ '{PROMPT_READY_MARKER}' }}\n[Console]::WriteLine('{SHELL_READY_MARKER}')" + ), ]; let env: HashMap = std::env::vars().collect(); let spawned = spawn_pty_process( @@ -349,14 +352,23 @@ if (-not [CodexPtyTests.ConsoleControl]::SetConsoleCtrlHandler([IntPtr]::Zero, $ wait_for_output_contains(&mut output_rx, "127.0.0.1", /*timeout_ms*/ 10_000).await?; writer.send(vec![0x03]).await?; - tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; - writer.send(b"cmd.exe /D /C ver\n".to_vec()).await?; + let resume_deadline = tokio::time::Instant::now() + Duration::from_millis(10_000); + // Use the next prompt as the resumed-input boundary, not a fixed sleep. + // Prompt readiness and follow-up output share the existing ten-second window. let mut output = wait_for_output_contains( &mut output_rx, - "Microsoft Windows", + PROMPT_READY_MARKER, /*timeout_ms*/ 10_000, ) .await?; + writer.send(b"cmd.exe /D /C ver\n".to_vec()).await?; + let remaining_ms = resume_deadline + .saturating_duration_since(tokio::time::Instant::now()) + .as_millis() + .try_into()?; + output.extend_from_slice( + &wait_for_output_contains(&mut output_rx, "Microsoft Windows", remaining_ms).await?, + ); writer.send(b"exit 0\n".to_vec()).await?; let (remaining, exit_code) = From 3992c1d5b49b6954fc85d0d8b8deaadf2bf8e888 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:26:43 +1100 Subject: [PATCH 47/54] fix(ci): retain lean settings through the hosted shell entry point --- .github/scripts/run-bazel-ci.sh | 12 ++- .github/scripts/run_bazel_with_buildbuddy.py | 4 + .../scripts/test_run_bazel_with_buildbuddy.py | 102 ++++++++++++++---- 3 files changed, 96 insertions(+), 22 deletions(-) diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index 047addb41f26..4b085fb1f325 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -88,7 +88,7 @@ print_bazel_test_log_tails() { local testlogs_dir local -a bazel_info_args=(info) - if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + if [[ -n "${BUILDBUDDY_API_KEY:-}" || "$ci_config" == "ci-bazel" ]]; then # `bazel info` needs the same CI config as the failed test invocation so # platform-specific output roots match. On Windows, omitting `ci-windows` # would point at `local_windows-fastbuild` even when the test ran with the @@ -390,6 +390,12 @@ if [[ "${RUNNER_OS:-}" == "Windows" ]]; then post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}") fi +if [[ -z "${BUILDBUDDY_API_KEY:-}" && "${GITHUB_ACTIONS:-}" == "true" && "${GITHUB_REPOSITORY:-}" == "sednalabs/codex" && "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then + # This entry point omits remote configs when keyless, so the Python fallback + # cannot infer that it must retain the shared lean, hosted-local CI settings. + ci_config=ci-bazel +fi + bazel_console_log="$(mktemp)" trap 'rm -f "$bazel_console_log"' EXIT @@ -399,6 +405,10 @@ if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then bazel_run_args+=("--config=${ci_config}") else echo "BuildBuddy API key is not available; using local Bazel configuration." + if [[ "$ci_config" == "ci-bazel" ]]; then + echo "Retaining shared hosted-local CI settings: --config=ci-bazel" + bazel_run_args+=(--config=ci-bazel) + fi fi if (( ${#bazel_args[@]} > 1 )); then # Explicit caller flags must follow rc configs so workflow-level safety and diff --git a/.github/scripts/run_bazel_with_buildbuddy.py b/.github/scripts/run_bazel_with_buildbuddy.py index 675f1a971b29..d9929ca4c882 100755 --- a/.github/scripts/run_bazel_with_buildbuddy.py +++ b/.github/scripts/run_bazel_with_buildbuddy.py @@ -141,11 +141,15 @@ def bazel_args_with_remote_config( config = remote_config(args, env) if config is None: configured_args = bazel_args_without_remote_execution(args) + local_config_args = configured_args + if "--" in configured_args: + local_config_args = configured_args[: configured_args.index("--")] if ( env.get("GITHUB_ACTIONS") == "true" and env.get("GITHUB_REPOSITORY") == "sednalabs/codex" and env.get("RUNNER_ENVIRONMENT") == "github-hosted" and uses_remote_execution(args) + and "--config=ci-bazel" not in local_config_args ): # Removing a remote platform must not also lose the shared CI # settings: lean Rust artifacts and no duplicate on-runner disk CAS. diff --git a/.github/scripts/test_run_bazel_with_buildbuddy.py b/.github/scripts/test_run_bazel_with_buildbuddy.py index f76ead4edd33..ad8531df7449 100644 --- a/.github/scripts/test_run_bazel_with_buildbuddy.py +++ b/.github/scripts/test_run_bazel_with_buildbuddy.py @@ -152,6 +152,28 @@ def test_shared_local_ci_fallback_is_limited_to_this_hosted_fork(self) -> None: ["build", "--", "//codex-rs/cli:codex"], ) + def test_shared_local_ci_is_not_duplicated_or_selected_by_payload(self) -> None: + env = { + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "sednalabs/codex", + "RUNNER_ENVIRONMENT": "github-hosted", + } + for args, expected in ( + ( + ["test", "--config=ci-bazel", "--config=ci-linux", "--", "//..."], + ["test", "--config=ci-bazel", "--", "//..."], + ), + ( + ["run", "--config=ci-linux", "//...", "--", "--config=ci-bazel"], + ["run", "--config=ci-bazel", "//...", "--", "--config=ci-bazel"], + ), + ): + with self.subTest(args=args): + self.assertEqual( + run_bazel_with_buildbuddy.bazel_args_with_remote_config(args, env), + expected, + ) + def test_shared_ci_config_is_lean_without_remote_endpoints_or_disk_cas(self) -> None: bazelrc = Path(__file__).resolve().parents[2] / ".bazelrc" shared_ci = [ @@ -445,8 +467,12 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - fake_script = root / "fake-bazel.py" fake_script.write_text( f"#!{sys.executable}\n" - "import os, sys\n" + "import json, os, sys\n" "sys.stdout.reconfigure(newline='\\n')\n" + "invocation = 'info' if 'info' in sys.argv else 'test'\n" + "with open(os.environ['TEST_BAZEL_ARGS_FILE'] + '-' + invocation, " + "'w', encoding='utf-8') as args_file:\n" + " json.dump(sys.argv[1:], args_file)\n" "if 'info' in sys.argv:\n" " print(os.environ['TEST_BAZEL_TESTLOGS_DIR'])\n" " sys.exit(0)\n" @@ -476,32 +502,66 @@ def test_failed_sharded_attempt_logs_survive_interrupted_build_reporting(self) - env.update( CODEX_BAZEL_BIN=str(fake_bazel), TEST_BAZEL_TESTLOGS_DIR=testlogs.as_posix(), + TEST_BAZEL_ARGS_FILE=(root / "invocation").as_posix(), RUNNER_TEMP=root.as_posix(), ) if os.name == "nt": # The fake only needs a valid PATH, not the runner's long tool list # repeated in arguments to a command with cmd.exe's length limit. env["CODEX_BAZEL_WINDOWS_PATH"] = str(Path(sys.executable).parent) - result = subprocess.run( - [ - bash, - (repo_root / ".github/scripts/run-bazel-ci.sh").as_posix(), - "--print-failed-test-logs", - "--", - "test", - "--", - "//codex-rs/core:core-unit-tests", - ], - cwd=repo_root, - env=env, - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 1, result.stderr) - for marker in markers: - self.assertIn(marker, result.stdout, result.stdout + result.stderr) - self.assertNotIn("Missing test log:", result.stdout) + windows_options = ["--windows-cross-compile"] if os.name == "nt" else [] + host_options = ["--host_platform=//:local_windows"] if os.name == "nt" else [] + for actions, repository, runner, shared_ci in ( + ("true", "sednalabs/codex", "github-hosted", True), + ("true", "openai/codex", "github-hosted", False), + ("true", "sednalabs/codex", "self-hosted", False), + ("false", "sednalabs/codex", "github-hosted", False), + ("true", "sednalabs/codex", "", False), + ): + with self.subTest(actions=actions, repository=repository, runner=runner): + env.update( + GITHUB_ACTIONS=actions, + GITHUB_REPOSITORY=repository, + RUNNER_ENVIRONMENT=runner, + ) + result = subprocess.run( + [ + bash, + (repo_root / ".github/scripts/run-bazel-ci.sh").as_posix(), + "--print-failed-test-logs", + *windows_options, + "--", + "test", + *host_options, + "--", + "//codex-rs/core:core-unit-tests", + ], + cwd=repo_root, + env=env, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 1, result.stderr) + for marker in markers: + self.assertIn(marker, result.stdout, result.stdout + result.stderr) + self.assertNotIn("Missing test log:", result.stdout) + for invocation in ("test", "info"): + forwarded = json.loads( + (root / f"invocation-{invocation}").read_text(encoding="utf-8") + ) + self.assertEqual(forwarded.count("--config=ci-bazel"), int(shared_ci)) + self.assertFalse( + any( + arg in run_bazel_with_buildbuddy.REMOTE_EXECUTION_CONFIGS + or arg.startswith("--remote_header=") + for arg in forwarded + ) + ) + self.assertEqual( + json.loads((root / "invocation-test").read_text(encoding="utf-8"))[-2:], + ["--", "//codex-rs/core:core-unit-tests"], + ) def test_main_preserves_spaced_argument_and_child_exit_status(self) -> None: spaced_arg = ( From 6fbbd5c775b71643e7604045b6c123254a8af4e3 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:38:43 +1100 Subject: [PATCH 48/54] test(agents): align accepted-work receipt schema expectations --- .../src/tools/handlers/multi_agents_spec_tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index c0f3f4b87303..35b2cfec1544 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -164,6 +164,7 @@ fn spawn_agent_tool_v2_requires_task_name_and_lists_visible_models() { .collect::>(), [ "agent_id", + "dispatch_state", "effective_model", "effective_reasoning_effort", "nickname", @@ -171,6 +172,7 @@ fn spawn_agent_tool_v2_requires_task_name_and_lists_visible_models() { "requested_model_honored", "requested_reasoning_effort", "task_name", + "work_ref", ] .into_iter() .map(str::to_string) @@ -597,6 +599,11 @@ fn followup_task_tool_requires_message_and_describes_model_receipt() { "type": "string", "description": "Immutable accepted work reference assigned to the follow-up." }, + "dispatch_state": { + "type": "string", + "enum": ["accepted", "dispatching", "dispatched", "recovery_needed"], + "description": "Native dispatch custody for the original work. Uncertain dispatch is not resent." + }, "recipient_task_name": { "type": "string", "description": "Explicit canonical task name of the recipient; the effective_* fields below describe this recipient, not the sending agent." @@ -607,11 +614,11 @@ fn followup_task_tool_requires_message_and_describes_model_receipt() { "description": "Identity scope for the effective_* fields in this receipt." }, "effective_model": { - "type": "string", + "type": ["string", "null"], "description": "Effective model retained by the recipient agent for the follow-up turn." }, "effective_model_provider_id": { - "type": "string", + "type": ["string", "null"], "description": "Effective model provider retained by the recipient agent for the follow-up turn." }, "effective_reasoning_effort": { From 016a30d50e9c06c40b1824d247af4fce0d01e8f4 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:41:22 +1100 Subject: [PATCH 49/54] test(agents): retain receipt schemas in focused carry coverage --- justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/justfile b/justfile index 57f3034e555b..16a9273c1cf5 100644 --- a/justfile +++ b/justfile @@ -383,6 +383,7 @@ core-multi-agent-orchestration-targeted: cargo test --locked -p codex-agent-mailbox --lib -- --test-threads=1 cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 + RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents_spec::tests::spawn_agent_tool_v2_requires_task_name_and_lists_visible_models tools::handlers::multi_agents_spec::tests::followup_task_tool_requires_message_and_describes_model_receipt --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --test all -- suite::spawn_agent_description::spawn_agent_description_lists_visible_models_and_reasoning_efforts suite::agent_execution::v2_evicted_completed_agent_keeps_final_status suite::agent_execution::v2_cold_mailbox_allows_eviction_and_replays_on_followup suite::multi_agent_mode::changing_configured_mode_hint_to_empty_appends_explicit_reset suite::pending_input::queue_only_agent_mail_wakes_sleeping_root_and_persists_message --exact From 69c8c8b70a29ada74e6368152cf1790af78f376f Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:12:23 +1100 Subject: [PATCH 50/54] test(mailbox): retain structured join failure diagnostics --- codex-rs/core/tests/suite/agent_mailbox.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index 1e117affdcf9..f2bf49bba2df 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -613,7 +613,10 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() assert_eq!(wait_report["requestedTargets"], json!([work_ref])); assert_eq!(wait_report["requestedRepliesTo"], json!([request_id])); assert_eq!(wait_report["returnWhen"], "all"); - assert_eq!(wait_report["conditionSatisfied"], true); + assert_eq!( + wait_report["conditionSatisfied"], true, + "exact join report: {wait_report}" + ); assert!( wait_report["completed"] .as_array() @@ -901,10 +904,14 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - .lock() .expect("wait start lock should not be poisoned") .expect("first targeted wait should have started"); + let deadline_report: Value = serde_json::from_str( + function_output_text(&requests[3].1, "wait-1-call") + .expect("first wait should return its deadline report"), + )?; let deadline_wait_elapsed = requests[3].0.duration_since(wait_started); assert!( (Duration::from_millis(850)..Duration::from_millis(1_500)).contains(&deadline_wait_elapsed), - "quiet messages should not refresh the one-second deadline: {deadline_wait_elapsed:?}" + "quiet messages should not refresh the one-second deadline: {deadline_wait_elapsed:?}; report: {deadline_report}" ); let watched_spawn: Value = serde_json::from_str( @@ -923,10 +930,6 @@ async fn quiet_events_keep_deadline_fixed_then_request_wakes_event_only_wait() - .expect("unrelated spawn should return an immutable work reference"); assert_ne!(work_ref, unrelated_work_ref); - let deadline_report: Value = serde_json::from_str( - function_output_text(&requests[3].1, "wait-1-call") - .expect("first wait should return its deadline report"), - )?; assert_eq!(deadline_report["requestedTargets"], json!([work_ref])); assert_eq!(deadline_report["completionReason"], "deadline"); assert_eq!(deadline_report["conditionSatisfied"], false); From afa196df9b9ad5b8932edf3b7f9bebb9f28cccc6 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:50:24 +1100 Subject: [PATCH 51/54] test(mailbox): repeat serial adapter group for intermittent reports --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 16a9273c1cf5..1d988daffb1c 100644 --- a/justfile +++ b/justfile @@ -384,7 +384,7 @@ core-multi-agent-orchestration-targeted: cargo test --locked -p codex-rollout --lib mailbox -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo test --locked -p codex-core --lib mailbox -- --test-threads=1 RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents_spec::tests::spawn_agent_tool_v2_requires_task_name_and_lists_visible_models tools::handlers::multi_agents_spec::tests::followup_task_tool_requires_message_and_describes_model_receipt --exact - RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 + for mailbox_attempt in 1 2 3 4 5; do echo "Mailbox adapter regression pass ${mailbox_attempt}/5"; RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-core --test all suite::agent_mailbox:: -- --test-threads=1 || exit "$?"; done RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --lib -- tools::handlers::multi_agents::tests::multi_agent_v2_list_agents_returns_completed_status tools::handlers::multi_agents_v2::wait::tests::completion_rule_distinguishes_any_from_all session::input_queue::tests::input_queue_wait_input_allows_terminal_completion_after_mailbox_delivery_deferral agent::control::residency::tests::residency_slot_reservation_unloads_oldest_idle_v2_agent agent::control::residency::tests::interrupted_v2_agent_remains_known_and_reloads_after_residency_eviction agent::control::residency::tests::ephemeral_v2_agent_is_not_evicted_without_reloadable_history agent::control::residency::tests::spawned_v2_terminal_child_unloads_at_terminal_idle_deadline agent::registry::tests::cold_status_text_stays_compact_when_json_escaped agent::registry::tests::spawn_delivery_ownership_prevents_cancellation_from_claiming_abort agent::control::tests::delivery_owned_initial_input_failure_reconciles_and_preserves_the_delivery_error agent::control::tests::ensure_v2_agent_loaded_reloads_registered_unloaded_agent tools::parallel::tests::configured_v2_spawn_cancellation_after_new_thread_reconciles_before_runtime_abort tools::parallel::tests::v1_spawn_cancellation_after_initial_delivery_returns_published_result tools::parallel::tests::configured_v2_mailbox_spawn_fast_completion_cancellation_retry_reuses_native_work context::world_state::multi_agent_mode::tests::custom_mode_removal_replaces_retained_instructions context::world_state::multi_agent_mode::tests::snapshots --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo nextest run -p codex-core --no-fail-fast --no-tests=fail --test all -- suite::spawn_agent_description::spawn_agent_description_lists_visible_models_and_reasoning_efforts suite::agent_execution::v2_evicted_completed_agent_keeps_final_status suite::agent_execution::v2_cold_mailbox_allows_eviction_and_replays_on_followup suite::multi_agent_mode::changing_configured_mode_hint_to_empty_appends_explicit_reset suite::pending_input::queue_only_agent_mail_wakes_sleeping_root_and_persists_message --exact RUST_MIN_STACK="${RUST_MIN_STACK:-{{ rust_min_stack }}}" CODEX_JS_REPL_NODE_PATH="${CODEX_JS_REPL_NODE_PATH:-/tmp/codex-node22/bin/node}" cargo test --locked -p codex-exec --test all suite::completion_backfill::ignores_unrelated_turn_completion_before_backfilling_primary_turn -- --exact --test-threads=1 From 32fb3c35ab61b8dcf0308a3a08d2709b9d370e07 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:22:53 +1100 Subject: [PATCH 52/54] test: reproduce mailbox activation writer contention --- codex-rs/agent-mailbox/src/mailbox_tests.rs | 84 +++++++++++++++++++++ codex-rs/core/tests/suite/agent_mailbox.rs | 5 +- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index bb0f969a7f72..2d60cdc126cc 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1603,6 +1603,90 @@ async fn explicit_attention_is_one_shot_after_canonical_read_without_settling_re ); } +#[tokio::test] +async fn activation_claim_waits_for_an_existing_writer() { + let fixture = test_mailbox().await; + let (sender, recipient) = (thread_id(/*index*/ 20), thread_id(/*index*/ 21)); + let mut message_draft = draft( + "activation-writer", + sender, + recipient, + MessageKind::Request, + "activate", + /*legacy_attention*/ false, + ); + message_draft.activate_if_idle = true; + let message = send(&fixture.mailbox, message_draft).await; + let writer = fixture + .mailbox + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .unwrap(); + let claim = fixture.mailbox.begin_activation(sender, &message.id); + tokio::pin!(claim); + let while_locked = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut claim).await; + assert!( + while_locked.is_err(), + "activation must wait for the writer, not fail a read-to-write upgrade: {while_locked:?}" + ); + writer.commit().await.unwrap(); + assert_eq!(claim.await.unwrap(), ActivationDecision::Activate); +} + +#[tokio::test] +async fn activation_receipt_waits_for_an_existing_writer() { + let fixture = test_mailbox().await; + let (sender, recipient) = (thread_id(/*index*/ 20), thread_id(/*index*/ 21)); + let mut message_draft = draft( + "activation-receipt-writer", + sender, + recipient, + MessageKind::Request, + "activate", + /*legacy_attention*/ false, + ); + message_draft.activate_if_idle = true; + let message = send(&fixture.mailbox, message_draft).await; + assert_eq!( + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::Activate + ); + let writer = fixture + .mailbox + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .unwrap(); + let receipt = fixture.mailbox.record_activation( + sender, + &message.id, + ActivationDisposition::Denied, + ); + tokio::pin!(receipt); + let while_locked = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut receipt).await; + assert!( + while_locked.is_err(), + "activation receipt must wait for the writer, not fail a read-to-write upgrade: {while_locked:?}" + ); + writer.commit().await.unwrap(); + receipt.await.unwrap(); + assert_eq!( + fixture + .mailbox + .begin_activation(sender, &message.id) + .await + .unwrap(), + ActivationDecision::AlreadyDecided(ActivationDisposition::Denied) + ); +} + #[tokio::test] async fn activation_claim_is_single_use_across_a_lost_result() { let fixture = test_mailbox().await; diff --git a/codex-rs/core/tests/suite/agent_mailbox.rs b/codex-rs/core/tests/suite/agent_mailbox.rs index f2bf49bba2df..b418d72f1d43 100644 --- a/codex-rs/core/tests/suite/agent_mailbox.rs +++ b/codex-rs/core/tests/suite/agent_mailbox.rs @@ -565,7 +565,10 @@ async fn mailbox_read_keeps_encrypted_body_canonical_and_wait_joins_exact_ids() .as_str() .expect("request receipt should contain a native message ID"); assert_eq!(request_receipt["durable_state"], "accepted"); - assert_eq!(request_receipt["activation_disposition"], "denied"); + assert_eq!( + request_receipt["activation_disposition"], "denied", + "root activation receipt: {request_receipt}" + ); assert_eq!( request_receipt["activation_reason"], "root_activation_is_not_allowed" From 1203008b816f46ff2fcd66639d18c60e118b3d69 Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:25:39 +1100 Subject: [PATCH 53/54] fix: reserve mailbox writer before custody reads --- codex-rs/agent-mailbox/src/activation.rs | 4 ++-- codex-rs/agent-mailbox/src/message_delivery.rs | 6 +++--- codex-rs/agent-mailbox/src/messages.rs | 2 +- codex-rs/agent-mailbox/src/outcome.rs | 4 ++-- codex-rs/agent-mailbox/src/snapshot.rs | 2 +- codex-rs/agent-mailbox/src/store.rs | 2 ++ codex-rs/agent-mailbox/src/work.rs | 8 ++++---- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/codex-rs/agent-mailbox/src/activation.rs b/codex-rs/agent-mailbox/src/activation.rs index 4d3fac89dd16..1fa45b2f433a 100644 --- a/codex-rs/agent-mailbox/src/activation.rs +++ b/codex-rs/agent-mailbox/src/activation.rs @@ -14,7 +14,7 @@ impl Mailbox { sender: ThreadId, message_id: &str, ) -> MailboxResult { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let message = sqlx::query("SELECT id FROM mailbox_messages WHERE id = ? AND sender = ? LIMIT 1") .bind(message_id) @@ -64,7 +64,7 @@ impl Mailbox { message_id: &str, disposition: ActivationDisposition, ) -> MailboxResult<()> { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let current = sqlx::query( "SELECT sender, state, disposition FROM mailbox_activations WHERE message_id = ?", ) diff --git a/codex-rs/agent-mailbox/src/message_delivery.rs b/codex-rs/agent-mailbox/src/message_delivery.rs index 3e0ffd3defe9..d5457a635700 100644 --- a/codex-rs/agent-mailbox/src/message_delivery.rs +++ b/codex-rs/agent-mailbox/src/message_delivery.rs @@ -85,7 +85,7 @@ impl Mailbox { /// Withdrawal is durable and idempotent; replies cannot target it afterward. pub async fn withdraw_message(&self, sender: ThreadId, id: &str) -> MailboxResult<()> { let _input_arrival = self.begin_input_arrival(); - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let row = sqlx::query( "SELECT sender, kind, canonical, input_admitted, legacy_attention, CASE WHEN kind IN ('\"note\"', '\"progress\"') @@ -320,7 +320,7 @@ impl Mailbox { /// Mark canonical admission only after the adapter has appended and flushed /// the stable ID. This operation is idempotent and recipient-bound. pub async fn mark_canonical(&self, recipient: ThreadId, id: &str) -> MailboxResult<()> { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let current = sqlx::query( "SELECT canonical, input_admitted, legacy_attention, superseded_by, withdrawn, CASE WHEN kind IN ('\"note\"', '\"progress\"') @@ -382,7 +382,7 @@ impl Mailbox { /// Mark a no-response or typed outcome after the adapter has appended its /// exact `OutcomeReceipt.id` and flushed canonical history. pub async fn mark_outcome_canonical(&self, recipient: ThreadId, id: &str) -> MailboxResult<()> { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let result = sqlx::query( "UPDATE mailbox_work SET outcome_canonical = 1 WHERE outcome_id = ? AND issuer = ? AND outcome_json IS NOT NULL", diff --git a/codex-rs/agent-mailbox/src/messages.rs b/codex-rs/agent-mailbox/src/messages.rs index bc85500fb572..ae66b5d7641a 100644 --- a/codex-rs/agent-mailbox/src/messages.rs +++ b/codex-rs/agent-mailbox/src/messages.rs @@ -51,7 +51,7 @@ impl Mailbox { }; let inserted_id = id.clone(); let envelope = { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let envelope = insert_message_tx(&mut transaction, draft, Some(inserted_id)).await?; transaction.commit().await?; envelope diff --git a/codex-rs/agent-mailbox/src/outcome.rs b/codex-rs/agent-mailbox/src/outcome.rs index 2a839e9ba6cf..2c8aa9f99284 100644 --- a/codex-rs/agent-mailbox/src/outcome.rs +++ b/codex-rs/agent-mailbox/src/outcome.rs @@ -40,7 +40,7 @@ impl Mailbox { ) -> MailboxResult { let _input_arrival = self.begin_input_arrival(); validate_key(source_turn_id)?; - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let admitted_rows = sqlx::query( "SELECT work.* FROM mailbox_work AS work JOIN mailbox_admissions AS admission ON admission.work_id = work.id @@ -365,7 +365,7 @@ impl Mailbox { if let Continuation::Terminal(binding) = &continuation { validate_binding_for_work(binding, work_ref)?; } - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let mut groups = response_groups( &mut transaction, std::slice::from_ref(work_ref), diff --git a/codex-rs/agent-mailbox/src/snapshot.rs b/codex-rs/agent-mailbox/src/snapshot.rs index 8625bf4c7402..823801a11acf 100644 --- a/codex-rs/agent-mailbox/src/snapshot.rs +++ b/codex-rs/agent-mailbox/src/snapshot.rs @@ -181,7 +181,7 @@ impl Mailbox { "observation count must be between 1 and 100".into(), )); } - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; for target in targets { match target { ObservationTarget::Work(work_ref) => { diff --git a/codex-rs/agent-mailbox/src/store.rs b/codex-rs/agent-mailbox/src/store.rs index 400710c6723e..75f44add824e 100644 --- a/codex-rs/agent-mailbox/src/store.rs +++ b/codex-rs/agent-mailbox/src/store.rs @@ -9,6 +9,8 @@ use tokio::sync::watch; #[derive(Clone)] pub struct Mailbox { + // Read-before-write transactions use BEGIN IMMEDIATE so concurrent custody + // writes cannot invalidate a deferred WAL snapshot during its write upgrade. pub(crate) pool: SqlitePool, change_tx: watch::Sender, pub(crate) waits: Arc>>, diff --git a/codex-rs/agent-mailbox/src/work.rs b/codex-rs/agent-mailbox/src/work.rs index a1aa01c478f1..165b483a069c 100644 --- a/codex-rs/agent-mailbox/src/work.rs +++ b/codex-rs/agent-mailbox/src/work.rs @@ -74,7 +74,7 @@ impl Mailbox { })?; let input_json = serde_json::to_string(&work.input)?; - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; if let Some(row) = sqlx::query("SELECT * FROM mailbox_work WHERE issuer = ? AND native_key = ?") .bind(work.issuer.to_string()) @@ -245,7 +245,7 @@ impl Mailbox { /// Atomically claim native activation. Only the first accepted caller may /// send; dispatching work requires exact adapter recovery, never a retry send. pub async fn begin_dispatch(&self, work_ref: &WorkRef) -> MailboxResult { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let row = exact_work_tx(&mut transaction, work_ref).await?; let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; if state != DispatchState::Accepted { @@ -310,7 +310,7 @@ impl Mailbox { work_ref: &WorkRef, evidence: RecoveryEvidence, ) -> MailboxResult { - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let row = exact_work_tx(&mut transaction, work_ref).await?; let state = parse_dispatch_state(row.try_get::("dispatch_state")?.as_str())?; if state == DispatchState::Accepted @@ -457,7 +457,7 @@ impl Mailbox { )); } - let mut transaction = self.pool.begin().await?; + let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; let mut work_refs = Vec::with_capacity(ids.len()); let mut added = false; for input_id in &ids { From ae4c58cb0cde88343a146d249c2d62fee684204d Mon Sep 17 00:00:00 2001 From: GraciousGazelles <55840159+GraciousGazelles@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:45:19 +1100 Subject: [PATCH 54/54] style: format mailbox contention regression --- codex-rs/agent-mailbox/src/mailbox_tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/codex-rs/agent-mailbox/src/mailbox_tests.rs b/codex-rs/agent-mailbox/src/mailbox_tests.rs index 2d60cdc126cc..76137181f5df 100644 --- a/codex-rs/agent-mailbox/src/mailbox_tests.rs +++ b/codex-rs/agent-mailbox/src/mailbox_tests.rs @@ -1663,11 +1663,10 @@ async fn activation_receipt_waits_for_an_existing_writer() { .begin_with("BEGIN IMMEDIATE") .await .unwrap(); - let receipt = fixture.mailbox.record_activation( - sender, - &message.id, - ActivationDisposition::Denied, - ); + let receipt = + fixture + .mailbox + .record_activation(sender, &message.id, ActivationDisposition::Denied); tokio::pin!(receipt); let while_locked = tokio::time::timeout(std::time::Duration::from_millis(100), &mut receipt).await;