From 3856144b650db3a53d8e7b28dfc9ef30d477d3fb Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 20:46:52 -0500 Subject: [PATCH] fix(worker): Unblock Observe's first snapshot and fix test races Observe registered its subscriber, released the lock, then sent to its own notify channel, which holds one signal. A status published in that window filled the channel first, so the handler blocked on a send only it could receive: the stream sent nothing, ignored cancellation and never removed its subscriber. The first send is now non-blocking, since pending already holds the latest snapshot. A nil-by-default hook lets TestObservePublishBeforeFirstSnapshot publish in that window; it fails with DeadlineExceeded without the fix. Four tests raced under CPU load: - TestAuthorizationRevoked revoked the claim as soon as the client's Observe call returned, before the server subscribed. It now receives the first snapshot before revoking. - TestCrashBeforeReleaseRecorded lost its record after the restart, when the new incarnation's first reconcile finished the release before the test read the status. The restart now holds that reconcile at Plan with the fake module's pause, checks the restored status and record, then resumes. It also waits for the injected Delete failure to be used instead of for an empty queue. - TestCrashBeforeProgramming: crash() cancels the context and then opens the fake's Plan gate, and select could pick the gate, letting Apply write rows after the crash. The fake's Plan and Apply now check the context again after the gate opens. - TestObserveDeliversCompleteSnapshots: the server ends a stream cleanly at its deadline, which can reach the client as EOF before the client's DeadlineExceeded. The test now accepts a clean end once the deadline has passed, when no message arrived. Fixes datum-cloud/galactic#1448 --- internal/worker/api.go | 11 +++++++- internal/worker/module/fake/fake.go | 12 +++++++-- internal/worker/recovery_test.go | 17 +++++++++++- internal/worker/worker.go | 5 ++++ internal/worker/worker_test.go | 40 ++++++++++++++++++++++++++++- 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/internal/worker/api.go b/internal/worker/api.go index 4033ec14..5367b2d7 100644 --- a/internal/worker/api.go +++ b/internal/worker/api.go @@ -149,10 +149,19 @@ func (w *Worker) Observe(req *workerv1alpha1.ObserveRequest, delete(w.subs, sub) w.mu.Unlock() }() + if w.subscribed != nil { + w.subscribed() + } if err := stream.SendHeader(metadata.MD{}); err != nil { return err } - sub.notify <- struct{}{} + // A status published since subscribing may already have filled notify. + // pending holds the latest snapshot either way, so one wake-up is enough, + // and a blocking send here would wait on the only reader. + select { + case sub.notify <- struct{}{}: + default: + } renew := time.NewTicker(w.cfg.RenewInterval) defer renew.Stop() diff --git a/internal/worker/module/fake/fake.go b/internal/worker/module/fake/fake.go index 5c6088d7..31982f68 100644 --- a/internal/worker/module/fake/fake.go +++ b/internal/worker/module/fake/fake.go @@ -223,7 +223,12 @@ func (m *Module) Plan(ctx context.Context, scope module.Scope, desired []module. select { case <-gate: case <-ctx.Done(): - return nil, ctx.Err() + } + // A crash cancels the context and then opens the gate. When both + // are ready, select picks either, so check the context again: a + // stopped incarnation must not go on to Apply. + if err := ctx.Err(); err != nil { + return nil, err } } want := map[module.ClaimKey]map[string]uint64{} @@ -263,7 +268,10 @@ func (m *Module) Apply(ctx context.Context, scope module.Scope, c module.Change) select { case <-gate: case <-ctx.Done(): - return ctx.Err() + } + // As in Plan: a stopped incarnation writes nothing. + if err := ctx.Err(); err != nil { + return err } } want := c.(change).want diff --git a/internal/worker/recovery_test.go b/internal/worker/recovery_test.go index e69e8d02..b7490d98 100644 --- a/internal/worker/recovery_test.go +++ b/internal/worker/recovery_test.go @@ -108,17 +108,25 @@ func TestCrashBeforeReleaseRecorded(t *testing.T) { e.store.failDelete.Store(1) e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) e.eventually("rows removed", func() bool { return len(e.host.Rows(vpcA)) == 0 }) - e.idle() + e.eventually("record delete attempted", func() bool { return e.store.failDelete.Load() == 0 }) if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing { t.Fatalf("records = %+v, want the releasing record kept", recs) } e.crash() e.cfg.RetryMin, e.cfg.RetryMax = 5*time.Millisecond, 50*time.Millisecond + // Hold the restart's first reconcile at Plan, before it can confirm the + // removal and delete the record. + e.pauseOnStart = true e.start() + <-e.startPaused if st := e.status(k); st == nil || condOf(st, workerv1alpha1.ConditionWithdrawing).GetStatus() != condTrue { t.Fatalf("restored status = %v, want Withdrawing", st) } + if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing { + t.Fatalf("records before the restart's reconcile = %+v, want the releasing record", recs) + } + e.local.ResumePlan() e.waitGone(k) if recs, _ := e.store.List(); len(recs) != 0 { t.Fatalf("records after restart = %+v", recs) @@ -249,6 +257,13 @@ func TestAuthorizationRevoked(t *testing.T) { e.waitReady(k, 1) stream, err := e.api.Observe(e.ctx(natSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) e.mustOK(err) + // Observe returns before the server subscribes. The first snapshot + // proves the subscription exists, so the revocation cannot be missed. + first, err := stream.Recv() + e.mustOK(err) + if !isTrue(first.GetStatus(), workerv1alpha1.ConditionReady) || first.GetStatus().GetReleased() { + t.Fatalf("first snapshot = %v, want the Ready claim", first.GetStatus()) + } a, _, _ := e.src.Attachment(t.Context(), "svc-a") a.Admitted = false diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 1009f1e6..f6b2424f 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -172,6 +172,11 @@ type Worker struct { shutting atomic.Bool started atomic.Bool wg sync.WaitGroup + + // subscribed, when set, runs after Observe registers a subscriber and + // before it queues the first snapshot. Tests use it to publish in + // between, and set it before any Observe starts. + subscribed func() } // New returns a Worker. Start loads desired state and starts reconciling. diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go index ce9513e2..d08add1b 100644 --- a/internal/worker/worker_test.go +++ b/internal/worker/worker_test.go @@ -6,7 +6,9 @@ package worker import ( "context" + "errors" "fmt" + "io" "maps" "net/netip" "slices" @@ -305,7 +307,16 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) { all, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{}) e.mustOK(err) e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1"))) - if r, err := all.Recv(); status.Code(err) != codes.DeadlineExceeded { + // When the deadline passes, the server ends the stream cleanly. That end + // can reach the client before its own deadline error does, and before + // ctx's timer runs, so a clean end at the deadline is correct as long as + // nothing was sent. The margin covers the deadline's encoding on the + // wire; an early end still fails. + r, err := all.Recv() + dl, _ := ctx.Deadline() + deadlineEnd := status.Code(err) == codes.DeadlineExceeded || + (errors.Is(err, io.EOF) && time.Until(dl) < 50*time.Millisecond) + if r != nil || !deadlineEnd { t.Fatalf("another caller's stream got %v, %v; want nothing until the deadline", r, err) } named, err := e.api.Observe(e.ctx(otherSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) @@ -315,6 +326,33 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) { } } +// TestObservePublishBeforeFirstSnapshot publishes a status after Observe +// registers its subscriber and before it queues the first snapshot. The +// stream still delivers the claim instead of blocking on its own wake-up. +func TestObservePublishBeforeFirstSnapshot(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.w.subscribed = func() { + e.w.mu.Lock() + defer e.w.mu.Unlock() + e.w.publish(e.w.claims[k]) + } + ctx, cancel := context.WithTimeout(e.ctx(natSA), 5*time.Second) + defer cancel() + stream, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) + e.mustOK(err) + r, err := stream.Recv() + if err != nil { + t.Fatalf("first snapshot: %v", err) + } + if !isTrue(r.GetStatus(), workerv1alpha1.ConditionReady) { + t.Fatalf("first snapshot = %v, want the Ready claim", r.GetStatus()) + } +} + // TestSessionsAndValidation covers the refusals that need no claim state. func TestSessionsAndValidation(t *testing.T) { e := newEnv(t)