diff --git a/internal/worker/api.go b/internal/worker/api.go index 4033ec14..5367b2d7 100644 --- a/internal/worker/api.go +++ b/internal/worker/api.go @@ -149,10 +149,19 @@ func (w *Worker) Observe(req *workerv1alpha1.ObserveRequest, delete(w.subs, sub) w.mu.Unlock() }() + if w.subscribed != nil { + w.subscribed() + } if err := stream.SendHeader(metadata.MD{}); err != nil { return err } - sub.notify <- struct{}{} + // A status published since subscribing may already have filled notify. + // pending holds the latest snapshot either way, so one wake-up is enough, + // and a blocking send here would wait on the only reader. + select { + case sub.notify <- struct{}{}: + default: + } renew := time.NewTicker(w.cfg.RenewInterval) defer renew.Stop() diff --git a/internal/worker/module/fake/fake.go b/internal/worker/module/fake/fake.go index 5c6088d7..31982f68 100644 --- a/internal/worker/module/fake/fake.go +++ b/internal/worker/module/fake/fake.go @@ -223,7 +223,12 @@ func (m *Module) Plan(ctx context.Context, scope module.Scope, desired []module. select { case <-gate: case <-ctx.Done(): - return nil, ctx.Err() + } + // A crash cancels the context and then opens the gate. When both + // are ready, select picks either, so check the context again: a + // stopped incarnation must not go on to Apply. + if err := ctx.Err(); err != nil { + return nil, err } } want := map[module.ClaimKey]map[string]uint64{} @@ -263,7 +268,10 @@ func (m *Module) Apply(ctx context.Context, scope module.Scope, c module.Change) select { case <-gate: case <-ctx.Done(): - return ctx.Err() + } + // As in Plan: a stopped incarnation writes nothing. + if err := ctx.Err(); err != nil { + return err } } want := c.(change).want diff --git a/internal/worker/recovery_test.go b/internal/worker/recovery_test.go index e69e8d02..b7490d98 100644 --- a/internal/worker/recovery_test.go +++ b/internal/worker/recovery_test.go @@ -108,17 +108,25 @@ func TestCrashBeforeReleaseRecorded(t *testing.T) { e.store.failDelete.Store(1) e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) e.eventually("rows removed", func() bool { return len(e.host.Rows(vpcA)) == 0 }) - e.idle() + e.eventually("record delete attempted", func() bool { return e.store.failDelete.Load() == 0 }) if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing { t.Fatalf("records = %+v, want the releasing record kept", recs) } e.crash() e.cfg.RetryMin, e.cfg.RetryMax = 5*time.Millisecond, 50*time.Millisecond + // Hold the restart's first reconcile at Plan, before it can confirm the + // removal and delete the record. + e.pauseOnStart = true e.start() + <-e.startPaused if st := e.status(k); st == nil || condOf(st, workerv1alpha1.ConditionWithdrawing).GetStatus() != condTrue { t.Fatalf("restored status = %v, want Withdrawing", st) } + if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing { + t.Fatalf("records before the restart's reconcile = %+v, want the releasing record", recs) + } + e.local.ResumePlan() e.waitGone(k) if recs, _ := e.store.List(); len(recs) != 0 { t.Fatalf("records after restart = %+v", recs) @@ -249,6 +257,13 @@ func TestAuthorizationRevoked(t *testing.T) { e.waitReady(k, 1) stream, err := e.api.Observe(e.ctx(natSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) e.mustOK(err) + // Observe returns before the server subscribes. The first snapshot + // proves the subscription exists, so the revocation cannot be missed. + first, err := stream.Recv() + e.mustOK(err) + if !isTrue(first.GetStatus(), workerv1alpha1.ConditionReady) || first.GetStatus().GetReleased() { + t.Fatalf("first snapshot = %v, want the Ready claim", first.GetStatus()) + } a, _, _ := e.src.Attachment(t.Context(), "svc-a") a.Admitted = false diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 1009f1e6..f6b2424f 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -172,6 +172,11 @@ type Worker struct { shutting atomic.Bool started atomic.Bool wg sync.WaitGroup + + // subscribed, when set, runs after Observe registers a subscriber and + // before it queues the first snapshot. Tests use it to publish in + // between, and set it before any Observe starts. + subscribed func() } // New returns a Worker. Start loads desired state and starts reconciling. diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go index ce9513e2..d08add1b 100644 --- a/internal/worker/worker_test.go +++ b/internal/worker/worker_test.go @@ -6,7 +6,9 @@ package worker import ( "context" + "errors" "fmt" + "io" "maps" "net/netip" "slices" @@ -305,7 +307,16 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) { all, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{}) e.mustOK(err) e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1"))) - if r, err := all.Recv(); status.Code(err) != codes.DeadlineExceeded { + // When the deadline passes, the server ends the stream cleanly. That end + // can reach the client before its own deadline error does, and before + // ctx's timer runs, so a clean end at the deadline is correct as long as + // nothing was sent. The margin covers the deadline's encoding on the + // wire; an early end still fails. + r, err := all.Recv() + dl, _ := ctx.Deadline() + deadlineEnd := status.Code(err) == codes.DeadlineExceeded || + (errors.Is(err, io.EOF) && time.Until(dl) < 50*time.Millisecond) + if r != nil || !deadlineEnd { t.Fatalf("another caller's stream got %v, %v; want nothing until the deadline", r, err) } named, err := e.api.Observe(e.ctx(otherSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) @@ -315,6 +326,33 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) { } } +// TestObservePublishBeforeFirstSnapshot publishes a status after Observe +// registers its subscriber and before it queues the first snapshot. The +// stream still delivers the claim instead of blocking on its own wake-up. +func TestObservePublishBeforeFirstSnapshot(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.w.subscribed = func() { + e.w.mu.Lock() + defer e.w.mu.Unlock() + e.w.publish(e.w.claims[k]) + } + ctx, cancel := context.WithTimeout(e.ctx(natSA), 5*time.Second) + defer cancel() + stream, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) + e.mustOK(err) + r, err := stream.Recv() + if err != nil { + t.Fatalf("first snapshot: %v", err) + } + if !isTrue(r.GetStatus(), workerv1alpha1.ConditionReady) { + t.Fatalf("first snapshot = %v, want the Ready claim", r.GetStatus()) + } +} + // TestSessionsAndValidation covers the refusals that need no claim state. func TestSessionsAndValidation(t *testing.T) { e := newEnv(t)