From 27b07258d235412171564011a907c2577b467478 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:55:40 +0000 Subject: [PATCH 1/2] fix(daemon): do not report trace ingest enqueue during shutdown as a worker crash The trace ingest worker exits on the shutdown signal. A trace connection that is still open during a stop or restart can then send one more payload to a closed channel. enqueue_trace_payload logged this at error level as "worker may have crashed", and the tracing layer sent it to error tracking. When the send fails and the daemon is shutting down, log at debug and return an error without a second request_shutdown. A closed channel without a shutdown request still logs at error level and stops the daemon. Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2b9ec6bd-d43e-438e-83cf-fe138906df3a --- src/daemon.rs | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/daemon.rs b/src/daemon.rs index d59ac20..aaf680e 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -5400,6 +5400,20 @@ impl ActorDaemonCoordinator { "trace payload accounting rollback error" ); } + if self.is_shutting_down() { + // The worker exits on the shutdown signal, so a connection + // that is still open during stop or restart can deliver one + // more payload after the receiver is gone. + tracing::debug!( + component = "daemon", + phase = "enqueue_trace_payload", + reason = "ingest_worker_stopped_for_shutdown", + "trace payload dropped: daemon is shutting down" + ); + return Err(AutterError::Generic( + "trace payload dropped: daemon is shutting down".to_string(), + )); + } tracing::error!( component = "daemon", phase = "enqueue_trace_payload", @@ -10228,6 +10242,59 @@ mod tests { tokio::task::yield_now().await; } + /// A payload that arrives after the worker exited for shutdown is a + /// benign race, not a worker crash. + #[tokio::test] + async fn enqueue_after_shutdown_is_not_reported_as_worker_crash() { + let coord = Arc::new(ActorDaemonCoordinator::new()); + coord.start_trace_ingest_worker().unwrap(); + coord.request_shutdown(); + let tx = coord.trace_ingest_tx.get().unwrap(); + for _ in 0..100 { + if tx.is_closed() { + break; + } + tokio::task::yield_now().await; + } + assert!(tx.is_closed(), "ingest worker must exit on shutdown"); + + let payload = serde_json::json!({ + "event": "start", + "sid": "20260411T120000.000000-Ptest0002", + "__autter_ingest_seq": 1_u64, + "argv": ["git", "commit", "-m", "test"], + }); + let error = coord.enqueue_trace_payload(payload).unwrap_err(); + assert!( + error.to_string().contains("daemon is shutting down"), + "unexpected error: {error}" + ); + assert_eq!(coord.queued_trace_payloads.load(Ordering::Relaxed), 0); + } + + /// A closed ingest channel without a shutdown request is still a worker + /// crash, and it stops the daemon. + #[tokio::test] + async fn enqueue_after_unexpected_worker_exit_reports_crash() { + let coord = ActorDaemonCoordinator::new(); + let (tx, rx) = mpsc::channel::(1); + drop(rx); + coord.trace_ingest_tx.set(tx).unwrap(); + + let payload = serde_json::json!({ + "event": "start", + "sid": "20260411T120000.000000-Ptest0003", + "__autter_ingest_seq": 1_u64, + "argv": ["git", "commit", "-m", "test"], + }); + let error = coord.enqueue_trace_payload(payload).unwrap_err(); + assert!( + error.to_string().contains("worker may have crashed"), + "unexpected error: {error}" + ); + assert!(coord.is_shutting_down()); + } + /// Concurrent enqueues from multiple threads must never deadlock or /// corrupt the accounting counter. #[tokio::test] From 41642a6d9164cd4cd4e1827ed41c6fbf356aa4d5 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:57:24 +0000 Subject: [PATCH 2/2] test(daemon): reuse make_start_payload in shutdown enqueue tests Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2b9ec6bd-d43e-438e-83cf-fe138906df3a --- src/daemon.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index aaf680e..d8b4e46 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -10258,12 +10258,7 @@ mod tests { } assert!(tx.is_closed(), "ingest worker must exit on shutdown"); - let payload = serde_json::json!({ - "event": "start", - "sid": "20260411T120000.000000-Ptest0002", - "__autter_ingest_seq": 1_u64, - "argv": ["git", "commit", "-m", "test"], - }); + let payload = make_start_payload(&["git", "commit", "-m", "test"]); let error = coord.enqueue_trace_payload(payload).unwrap_err(); assert!( error.to_string().contains("daemon is shutting down"), @@ -10281,12 +10276,7 @@ mod tests { drop(rx); coord.trace_ingest_tx.set(tx).unwrap(); - let payload = serde_json::json!({ - "event": "start", - "sid": "20260411T120000.000000-Ptest0003", - "__autter_ingest_seq": 1_u64, - "argv": ["git", "commit", "-m", "test"], - }); + let payload = make_start_payload(&["git", "commit", "-m", "test"]); let error = coord.enqueue_trace_payload(payload).unwrap_err(); assert!( error.to_string().contains("worker may have crashed"),