Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion internal/worker/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,19 @@ func (w *Worker) Observe(req *workerv1alpha1.ObserveRequest,
delete(w.subs, sub)
w.mu.Unlock()
}()
if w.subscribed != nil {
w.subscribed()
}
if err := stream.SendHeader(metadata.MD{}); err != nil {
return err
}
sub.notify <- struct{}{}
// A status published since subscribing may already have filled notify.
// pending holds the latest snapshot either way, so one wake-up is enough,
// and a blocking send here would wait on the only reader.
select {
case sub.notify <- struct{}{}:
default:
}

renew := time.NewTicker(w.cfg.RenewInterval)
defer renew.Stop()
Expand Down
12 changes: 10 additions & 2 deletions internal/worker/module/fake/fake.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,12 @@ func (m *Module) Plan(ctx context.Context, scope module.Scope, desired []module.
select {
case <-gate:
case <-ctx.Done():
return nil, ctx.Err()
}
// A crash cancels the context and then opens the gate. When both
// are ready, select picks either, so check the context again: a
// stopped incarnation must not go on to Apply.
if err := ctx.Err(); err != nil {
return nil, err
}
}
want := map[module.ClaimKey]map[string]uint64{}
Expand Down Expand Up @@ -263,7 +268,10 @@ func (m *Module) Apply(ctx context.Context, scope module.Scope, c module.Change)
select {
case <-gate:
case <-ctx.Done():
return ctx.Err()
}
// As in Plan: a stopped incarnation writes nothing.
if err := ctx.Err(); err != nil {
return err
}
}
want := c.(change).want
Expand Down
17 changes: 16 additions & 1 deletion internal/worker/recovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,17 +108,25 @@ func TestCrashBeforeReleaseRecorded(t *testing.T) {
e.store.failDelete.Store(1)
e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1))
e.eventually("rows removed", func() bool { return len(e.host.Rows(vpcA)) == 0 })
e.idle()
e.eventually("record delete attempted", func() bool { return e.store.failDelete.Load() == 0 })
if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing {
t.Fatalf("records = %+v, want the releasing record kept", recs)
}

e.crash()
e.cfg.RetryMin, e.cfg.RetryMax = 5*time.Millisecond, 50*time.Millisecond
// Hold the restart's first reconcile at Plan, before it can confirm the
// removal and delete the record.
e.pauseOnStart = true
e.start()
<-e.startPaused
if st := e.status(k); st == nil || condOf(st, workerv1alpha1.ConditionWithdrawing).GetStatus() != condTrue {
t.Fatalf("restored status = %v, want Withdrawing", st)
}
if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing {
t.Fatalf("records before the restart's reconcile = %+v, want the releasing record", recs)
}
e.local.ResumePlan()
e.waitGone(k)
if recs, _ := e.store.List(); len(recs) != 0 {
t.Fatalf("records after restart = %+v", recs)
Expand Down Expand Up @@ -249,6 +257,13 @@ func TestAuthorizationRevoked(t *testing.T) {
e.waitReady(k, 1)
stream, err := e.api.Observe(e.ctx(natSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")})
e.mustOK(err)
// Observe returns before the server subscribes. The first snapshot
// proves the subscription exists, so the revocation cannot be missed.
first, err := stream.Recv()
e.mustOK(err)
if !isTrue(first.GetStatus(), workerv1alpha1.ConditionReady) || first.GetStatus().GetReleased() {
t.Fatalf("first snapshot = %v, want the Ready claim", first.GetStatus())
}

a, _, _ := e.src.Attachment(t.Context(), "svc-a")
a.Admitted = false
Expand Down
5 changes: 5 additions & 0 deletions internal/worker/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@ type Worker struct {
shutting atomic.Bool
started atomic.Bool
wg sync.WaitGroup

// subscribed, when set, runs after Observe registers a subscriber and
// before it queues the first snapshot. Tests use it to publish in
// between, and set it before any Observe starts.
subscribed func()
}

// New returns a Worker. Start loads desired state and starts reconciling.
Expand Down
40 changes: 39 additions & 1 deletion internal/worker/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ package worker

import (
"context"
"errors"
"fmt"
"io"
"maps"
"net/netip"
"slices"
Expand Down Expand Up @@ -305,7 +307,16 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) {
all, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{})
e.mustOK(err)
e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1")))
if r, err := all.Recv(); status.Code(err) != codes.DeadlineExceeded {
// When the deadline passes, the server ends the stream cleanly. That end
// can reach the client before its own deadline error does, and before
// ctx's timer runs, so a clean end at the deadline is correct as long as
// nothing was sent. The margin covers the deadline's encoding on the
// wire; an early end still fails.
r, err := all.Recv()
dl, _ := ctx.Deadline()
deadlineEnd := status.Code(err) == codes.DeadlineExceeded ||
(errors.Is(err, io.EOF) && time.Until(dl) < 50*time.Millisecond)
if r != nil || !deadlineEnd {
t.Fatalf("another caller's stream got %v, %v; want nothing until the deadline", r, err)
}
named, err := e.api.Observe(e.ctx(otherSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")})
Expand All @@ -315,6 +326,33 @@ func TestObserveDeliversCompleteSnapshots(t *testing.T) {
}
}

// TestObservePublishBeforeFirstSnapshot publishes a status after Observe
// registers its subscriber and before it queues the first snapshot. The
// stream still delivers the claim instead of blocking on its own wake-up.
func TestObservePublishBeforeFirstSnapshot(t *testing.T) {
e := newEnv(t)
e.start()
k := mkey(vpcA, "svc-a")
e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1")))
e.waitReady(k, 1)
e.w.subscribed = func() {
e.w.mu.Lock()
defer e.w.mu.Unlock()
e.w.publish(e.w.claims[k])
}
ctx, cancel := context.WithTimeout(e.ctx(natSA), 5*time.Second)
defer cancel()
stream, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")})
e.mustOK(err)
r, err := stream.Recv()
if err != nil {
t.Fatalf("first snapshot: %v", err)
}
if !isTrue(r.GetStatus(), workerv1alpha1.ConditionReady) {
t.Fatalf("first snapshot = %v, want the Ready claim", r.GetStatus())
}
}

// TestSessionsAndValidation covers the refusals that need no claim state.
func TestSessionsAndValidation(t *testing.T) {
e := newEnv(t)
Expand Down
Loading