Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
9f0f795
fix: stop finite timeouts from killing legitimate work
asto18089 Sep 17, 2026
1117942
fix(tasks): keep the idle watchdog honest about in-flight tools
asto18089 Sep 17, 2026
3764d46
fix(vision): bound image_analyze with a real total envelope
asto18089 Sep 17, 2026
42a1f9e
fix(snapshot): drain git pipes while the child runs
asto18089 Sep 17, 2026
ac96d65
fix(mcp): widen the read wait per request instead of clamping the knob
asto18089 Sep 17, 2026
d31a2a3
fix(runtime): resolve pending approvals when the engine dies or the r…
asto18089 Sep 17, 2026
e2451f4
fix(runtime-api): publish approval interrupted flag over SSE and docu…
asto18089 Sep 17, 2026
a3b5197
fix(tools): close the remaining high-impact timeout outliers
asto18089 Sep 17, 2026
6f93013
fix(client): bound the non-streaming response body and the Anthropic …
asto18089 Sep 17, 2026
0321ec4
fix(tools): make the interpreter timeout actually kill the child
asto18089 Sep 17, 2026
64e6fa6
docs(mcp): refresh timeout defaults and explain the read/execute split
asto18089 Sep 17, 2026
149c3fd
style: apply rustfmt to the timeout-fix commits
asto18089 Sep 17, 2026
08353be
fix(config): follow the sub-agent heartbeat expectations past the new…
asto18089 Sep 17, 2026
ffae67b
fix(client): keep per-attempt request totals off the streaming open path
asto18089 Sep 17, 2026
70d9a5e
fix(runtime): exclude the user-input human wait from the turn wall clock
asto18089 Sep 18, 2026
5c81c81
fix(tools): return from timeouts even when grandchildren hold the pipes
asto18089 Sep 18, 2026
96dc5d4
fix(sandbox): bound the two unbounded external-request paths the audi…
asto18089 Sep 18, 2026
c952f44
fix(snapshot): clear a stale side-repo index.lock on open instead of …
asto18089 Sep 18, 2026
1ba281e
fix(tools): close the small gaps the review round flagged
asto18089 Sep 18, 2026
54d4112
style: satisfy clippy on the rlm recursion threading
asto18089 Sep 18, 2026
a3d6b79
fix(tasks): feed the worker idle watchdog from the in-flight tool window
asto18089 Sep 18, 2026
f7921ab
fix(tests): gate the code-execution pid test to unix
asto18089 Sep 18, 2026
c9b8b82
fix(tools): bound the interpreter pipe drain after a clean exit
asto18089 Sep 18, 2026
ed9c284
fix(snapshot): route every git call through the bounded drain core
asto18089 Sep 18, 2026
8412fc1
fix(app-server): bound the runtime bridge client and event stream
asto18089 Sep 18, 2026
64966da
docs: sync operator docs and comments with the current wait semantics
asto18089 Sep 18, 2026
4fdebaf
fix(mcp): give the send leg the same per-request budget
asto18089 Sep 18, 2026
bdd63e7
fix(client): widen the injected test envelope against parallel load
asto18089 Sep 18, 2026
dcd9f77
fix(sandbox): connect-bound the OpenSandbox exec client
asto18089 Sep 18, 2026
b97563d
fix(tools): restore the pandoc tool docs and complete the OCR disclosure
asto18089 Sep 18, 2026
818658c
fix(snapshot): make git pipe readers cancellable
asto18089 Sep 20, 2026
8d98535
fix(snapshot): pass git paths via env, not argv
asto18089 Sep 20, 2026
c801f85
fix(runtime-api): restore only known snapshot ids
asto18089 Sep 20, 2026
6949790
fix(tasks): keep liveness heartbeats out of the persist path
asto18089 Sep 20, 2026
464132a
fix(mcp): poison the connection on request-leg timeouts
asto18089 Sep 20, 2026
c9b601a
fix(snapshot): surface session failures through the stderr notice
asto18089 Sep 20, 2026
c744320
fix(snapshot): ignore ambient GIT_DIR for the side-repo init
asto18089 Sep 20, 2026
b3cc85a
fix(tools): let plugin callers detect a truncated drain
asto18089 Sep 20, 2026
7c909fb
fix(runtime): honor a queued approval decision over cancel
asto18089 Sep 20, 2026
38b7e64
fix(app-server): bound the runtime stream header wait
asto18089 Sep 20, 2026
b4b0510
fix(sandbox): correct the exec budget rationale
asto18089 Sep 20, 2026
11037e0
fix(tests): widen the human-wait wall-clock margin
asto18089 Sep 20, 2026
11e9b0f
docs: sync stale wait wording and the restore 404 contract
asto18089 Sep 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion crates/app-server/src/chat_completions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,20 @@ use serde_json::Value;

use super::AppState;

// ── Upstream deadlines ─────────────────────────────────────────────────

/// Connect budget for the upstream forward. Matches the connect family
/// used across the TUI client (vision, installs, DNS pre-flight).
const UPSTREAM_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);

/// Total budget for one upstream forward, connect through body end. The
/// handler rejects streaming (`stream: true`) and reads the full upstream
/// body, so without a client-level total a provider that accepts the
/// connection and stalls — or trickles the body — wedges this handler (and
/// the caller's connection) indefinitely. 1800s mirrors the TUI client's
/// non-streaming envelope for the same request class.
const UPSTREAM_TOTAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1800);

// ── Resolved endpoint ──────────────────────────────────────────────────

/// Everything needed to forward a single chat-completions request upstream.
Expand Down Expand Up @@ -373,8 +387,12 @@ pub(crate) async fn chat_completions_handler(
.into_response();
}

// Build upstream request.
// Build upstream request. The shared platform builder sets no timeouts,
// so the proxy would hang forever on an accept-and-stall upstream;
// bound both the connect and the whole non-streaming round trip.
let upstream_req = codewhale_release::platform_http_client_builder()
.connect_timeout(UPSTREAM_CONNECT_TIMEOUT)
.timeout(UPSTREAM_TOTAL_TIMEOUT)
.build()
.map_err(|e| {
(
Expand Down
61 changes: 54 additions & 7 deletions crates/app-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1399,6 +1399,30 @@ async fn invalidate_runtime_bridge(state: &AppState) {
*bridge = None;
}

// ── Runtime bridge deadlines ────────────────────────────────────────────

/// Connect budget for one bridge request. The bridge talks to a runtime
/// child this process spawned on loopback, so a connect that has not
/// completed in 10s means the child's listener is wedged.
const RUNTIME_BRIDGE_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);

/// Total budget for one bridging POST (`/v1/threads`, `/v1/threads/{id}/turns`).
/// These requests enqueue work and return; the turn itself streams over SSE.
/// Without a total, a runtime child that is alive but wedged (async workers
/// starved by a blocking tool call, a lock deadlock in the turn-start path)
/// hangs the request forever — and the inner bridge lock is held for the
/// whole turn, so one hung request queues every later JSON-RPC message
/// behind it. Mirrors the TUI client's non-streaming envelope.
const RUNTIME_BRIDGE_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1800);

/// Idle budget between SSE chunks on the event stream. The runtime emits a
/// keepalive every 15s, so silence beyond this means the child's event
/// stream wedged; the idle bound catches it without capping the total
/// duration of a live stream (a per-request total would ride the body and
/// hard-cut long turns — the same trap the model client's stream-open path
/// avoids).
const RUNTIME_BRIDGE_SSE_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);

impl RuntimeBridge {
async fn start(config_path: Option<&Path>) -> Result<Self> {
install_rustls_crypto_provider();
Expand All @@ -1410,6 +1434,7 @@ impl RuntimeBridge {
let mut bridge = Self {
base_url: format!("http://127.0.0.1:{port}"),
client: codewhale_release::platform_http_client_builder()
.connect_timeout(RUNTIME_BRIDGE_CONNECT_TIMEOUT)
.build()
.context("failed to build runtime API client")?,
auth_token: Some(auth_token),
Expand Down Expand Up @@ -1487,7 +1512,10 @@ impl RuntimeBridge {
}

async fn request_json(&self, builder: reqwest::RequestBuilder) -> Result<Value> {
let response = builder.send().await?;
let response = builder
.timeout(RUNTIME_BRIDGE_REQUEST_TIMEOUT)
.send()
.await?;
let status = response.status();
let body = response.text().await?;
if !status.is_success() {
Expand Down Expand Up @@ -1683,19 +1711,38 @@ impl RuntimeBridge {
since_seq: u64,
mut transcript: Option<&mut TurnTranscript>,
) -> Result<(u64, TurnTerminalStatus, Option<String>)> {
let mut response = self
.authed(self.client.get(format!(
// A runtime child that accepts the connection but never writes
// response headers would otherwise hold the bridge lock forever —
// the same accept-and-stall shape the chunk idle bound below covers
// for the body. The header wait shares that idle deadline; a per-
// request total is deliberately absent because reqwest's total would
// ride the returned body and hard-cut the live stream.
let mut response = tokio::time::timeout(
RUNTIME_BRIDGE_SSE_IDLE_TIMEOUT,
self.authed(self.client.get(format!(
"{}/v1/threads/{thread_id}/events?since_seq={since_seq}",
self.base_url
)))
.send()
.await?
.error_for_status()?;
.send(),
)
.await
.context("runtime event stream stalled before the first byte")??
.error_for_status()?;

let mut buffer = Vec::new();
let mut last_seq = since_seq;

while let Some(chunk) = response.chunk().await? {
// The event stream only ever pauses for the runtime's 15s
// keepalives; anything longer means the child wedged mid-stream.
// The idle bound re-arms per chunk, so a live stream of any length
// is never total-capped.
loop {
let chunk = tokio::time::timeout(RUNTIME_BRIDGE_SSE_IDLE_TIMEOUT, response.chunk())
.await
.context("runtime event stream stalled past the idle deadline")??;
let Some(chunk) = chunk else {
break;
};
buffer.extend_from_slice(&chunk);
if buffer.len() > MAX_SSE_FRAME_BYTES {
bail!(
Expand Down
8 changes: 5 additions & 3 deletions crates/cli/src/update.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@ const GITHUB_RELEASE_DOWNLOAD_BASE_URL: &str =
"https://github.com/Hmbown/CodeWhale/releases/download";
const UPDATE_HTTP_ATTEMPTS: usize = 3;
const UPDATE_HTTP_RETRY_DELAY_MS: u64 = 100;
/// Ceiling for one asset download. Generous, because release binaries are tens
/// of megabytes and some of the networks this exists for are slow.
const UPDATE_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(5 * 60);
/// Ceiling for one asset download. Release binaries are tens of megabytes
/// and some of the networks this exists for are slow: 600s covers a full
/// 60 MiB at ~100 KiB/s (the same download budget the audit gives skill
/// tarballs; the old 300s needed an implausible >1.6 Mbps to finish).
const UPDATE_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(600);
/// Ceiling for one checksum-manifest probe. The manifest is a few hundred
/// bytes, so this is only a backstop against a source that accepts the
/// connection and then stalls. GitHub gets the first attempt; an unavailable
Expand Down
8 changes: 5 additions & 3 deletions crates/core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,15 @@ use serde_json::{Value, json};
use tokio::time;
use uuid::Uuid;

/// Per-tool dispatch budget for the headless runtime. Matches the generous
/// subagent default so long-running tools are not cut off prematurely.
/// Per-tool dispatch budget for the headless runtime. 30 minutes: tools
/// legitimately run long (builds, test suites, MCP-backed calls), and this
/// wrapper is a runaway backstop, not an expected duration — the previous
/// 300s value cut off healthy in-flight tool work.
fn tool_dispatch_timeout() -> Duration {
if cfg!(test) {
Duration::from_millis(50)
} else {
Duration::from_secs(300)
Duration::from_secs(1800)
}
}

Expand Down
12 changes: 10 additions & 2 deletions crates/mcp/src/stdio_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,15 @@ const PROTOCOL_VERSION: &str = "2024-11-05";
/// because a first `npx`/`uvx` launch may download the server package.
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30);

/// Budget for a single request once the server is up.
/// Budget for a single request once the server is up. `tools/call` gets a
/// separate, much longer budget (`CALL_TOOL_TIMEOUT`): a tool legitimately
/// runs minutes (scrapes, remote jobs, builds), and killing it at 2 minutes
/// returned "timed out" to the model for healthy work.
const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
/// Budget for `tools/call` specifically. Matches the TUI pool's default
/// execute timeout; still bounded so a wedged server cannot hang a consumer
/// forever, but far past any legitimate tool run.
const CALL_TOOL_TIMEOUT: Duration = Duration::from_secs(1800);

/// How long a dropped client waits for a graceful exit after closing stdin
/// before it kills the child.
Expand Down Expand Up @@ -709,12 +716,13 @@ impl McpManagedClient for ChildProcessMcpClient {
// The server's result is returned verbatim, including an `isError`
// content payload: reinterpreting it here would replace what the
// server actually said with our guess about it.
self.request(
self.request_with_timeout(
"tools/call",
json!({
"name": tool_name,
"arguments": arguments
}),
CALL_TOOL_TIMEOUT,
)
}

Expand Down
Loading
Loading