From 8b9cd3404be042446ef18eb3e277b7df0bc21900 Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Wed, 11 Nov 2026 10:39:57 +0000 Subject: [PATCH] feat: serve Post, Release and ChangelogEntry Postgres-backed storage for the three GitHub-synced kinds, with the field ownership rules from the design. - Posts: only feedback-sync creates and deletes. Status is merged per caller: sync owns GitHub-derived fields and delivery work items, the controller owns stage, history, highestNotified and part-shipped, the server computes vote counts. Staff can only change spec.hidden; their status writes are Forbidden - Hidden posts are NotFound outside the platform context - Release.status.posts and ChangelogEntry.status.posts are derived from the posts table on read, excluding hidden posts - A title or body change drops the post's embedding for re-embedding - Votes on deleted posts are kept for --vote-retention (30 days), then purged hourly; a returning post keeps its votes - Store hooks: before/after write in the request transaction, a read filter, and decoration of returned objects - --sync-users and --controller-users flags; pgtest package shared by integration tests Closes #4. --- README.md | 28 +- cmd/feedback/main.go | 21 +- internal/apiserver/apiserver.go | 71 ++++- internal/identity/identity.go | 68 +++++ internal/registry/post/storage.go | 229 ++++++++++++++++ internal/registry/post/storage_test.go | 316 ++++++++++++++++++++++ internal/registry/post/strategy.go | 201 ++++++++++++++ internal/registry/post/strategy_test.go | 149 ++++++++++ internal/registry/release/release.go | 316 ++++++++++++++++++++++ internal/registry/release/release_test.go | 175 ++++++++++++ internal/storage/pg/kind.go | 59 ++++ internal/storage/pg/pgtest/pgtest.go | 102 +++++++ internal/storage/pg/store.go | 32 +++ internal/storage/pg/store_test.go | 76 +----- migrations/00003_deleted_posts.sql | 13 + pkg/apis/feedback/v1alpha1/post_types.go | 9 + 16 files changed, 1788 insertions(+), 77 deletions(-) create mode 100644 internal/identity/identity.go create mode 100644 internal/registry/post/storage.go create mode 100644 internal/registry/post/storage_test.go create mode 100644 internal/registry/post/strategy.go create mode 100644 internal/registry/post/strategy_test.go create mode 100644 internal/registry/release/release.go create mode 100644 internal/registry/release/release_test.go create mode 100644 internal/storage/pg/pgtest/pgtest.go create mode 100644 migrations/00003_deleted_posts.sql diff --git a/README.md b/README.md index 097434e..1199b60 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,15 @@ entries. It runs as an aggregated API server under Milo, like ## Status Early. Every resource in the design has a v1alpha1 type in -`pkg/apis/feedback/v1alpha1` and a sample in `config/samples`, but the server -only serves `FeedbackSource` so far. Storage for the rest is tracked as +`pkg/apis/feedback/v1alpha1` and a sample in `config/samples`, and the server +serves `FeedbackSource`, `Post`, `Release` and `ChangelogEntry`. Storage for the rest is tracked as sub-issues of the epic. | Resource | Storage | State | | --- | --- | --- | | `FeedbackSource` | etcd | served | -| `Post`, `PostQuery`, `Release`, `ChangelogEntry` | Postgres | planned | +| `Post`, `Release`, `ChangelogEntry` | Postgres | served | +| `PostQuery` | Postgres | planned | | `Vote`, `Subscription` | Postgres | planned | | `Submission`, `SecurityReport`, `SecurityAcknowledgement` | Postgres | planned | @@ -59,6 +60,27 @@ runs it as an init container, and a Postgres session lock keeps replicas from migrating at the same time. Add a migration as the next numbered file in `migrations/`. +## Who writes what + +`feedback-sync` and `feedback-controller` are recognised by username +(`--sync-users`, `--controller-users`; defaults are the `feedback-system` +service accounts). On top of Milo's IAM: + +- Only `feedback-sync` creates and deletes posts, and writes releases and + changelog entries. +- Post status is split: `feedback-sync` owns the GitHub-derived fields and + delivery work items; `feedback-controller` owns the delivery stage, + history, `highestNotified` and part-shipped records; the server computes + vote counts. Fields a caller doesn't own are ignored, like spec on a + status update. Anyone else gets Forbidden on post status. +- Staff can change `spec.hidden` and metadata. Hidden posts are NotFound + outside the platform context and never listed on releases or changelog + entries. +- `Release.status.posts` and `ChangelogEntry.status.posts` are derived from + the posts table on every read. +- Votes on a deleted post are kept for `--vote-retention` (30 days) in case + the post comes back. + ## Development ```bash diff --git a/cmd/feedback/main.go b/cmd/feedback/main.go index 0df925d..f1065c8 100644 --- a/cmd/feedback/main.go +++ b/cmd/feedback/main.go @@ -26,6 +26,7 @@ import ( "k8s.io/kube-openapi/pkg/validation/spec" feedbackapiserver "go.miloapis.com/feedback/internal/apiserver" + "go.miloapis.com/feedback/internal/identity" "go.miloapis.com/feedback/internal/postgres" "go.miloapis.com/feedback/internal/version" "go.miloapis.com/feedback/pkg/generated/openapi" @@ -121,6 +122,8 @@ type serverOptions struct { RecommendedOptions *options.RecommendedOptions Logs *logsapi.LoggingConfiguration Postgres postgres.Config + Identities identity.Identities + VoteRetention time.Duration } func newServerOptions() *serverOptions { @@ -129,8 +132,10 @@ func newServerOptions() *serverOptions { "/registry/feedback.miloapis.com", feedbackapiserver.Codecs.LegacyCodec(feedbackapiserver.Scheme.PrioritizedVersionsAllGroups()...), ), - Logs: logsapi.NewLoggingConfiguration(), - Postgres: postgres.Config{MaxConns: 20, ConnectTimeout: 10 * time.Second}, + Logs: logsapi.NewLoggingConfiguration(), + Postgres: postgres.Config{MaxConns: 20, ConnectTimeout: 10 * time.Second}, + Identities: identity.Default, + VoteRetention: 30 * 24 * time.Hour, } // No admission plugins: validation lives in the registry strategies. o.RecommendedOptions.Admission = nil @@ -145,6 +150,12 @@ func (o *serverOptions) AddFlags(fs *pflag.FlagSet) { "Maximum number of connections in the Postgres pool.") fs.DurationVar(&o.Postgres.ConnectTimeout, "postgres-connect-timeout", o.Postgres.ConnectTimeout, "Timeout for the initial Postgres connection.") + fs.StringSliceVar(&o.Identities.Sync, "sync-users", o.Identities.Sync, + "Usernames of feedback-sync, the only writer of posts, releases and changelog entries.") + fs.StringSliceVar(&o.Identities.Controller, "controller-users", o.Identities.Controller, + "Usernames of feedback-controller, which owns delivery tracking on posts.") + fs.DurationVar(&o.VoteRetention, "vote-retention", o.VoteRetention, + "How long votes on a deleted post are kept in case it returns. 0 disables the purge.") } func (o *serverOptions) Complete() error { @@ -210,7 +221,11 @@ func (o *serverOptions) Config() (*feedbackapiserver.Config, error) { return &feedbackapiserver.Config{ GenericConfig: genericConfig, - ExtraConfig: feedbackapiserver.ExtraConfig{Postgres: o.Postgres}, + ExtraConfig: feedbackapiserver.ExtraConfig{ + Postgres: o.Postgres, + Identities: o.Identities, + VoteRetention: o.VoteRetention, + }, }, nil } diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 36c040c..14b9785 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -4,6 +4,7 @@ package apiserver import ( "context" "fmt" + "time" "github.com/jackc/pgx/v5/pgxpool" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,8 +15,12 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/klog/v2" + "go.miloapis.com/feedback/internal/identity" "go.miloapis.com/feedback/internal/postgres" "go.miloapis.com/feedback/internal/registry/feedbacksource" + "go.miloapis.com/feedback/internal/registry/post" + "go.miloapis.com/feedback/internal/registry/release" + "go.miloapis.com/feedback/internal/storage/pg" "go.miloapis.com/feedback/pkg/apis/feedback/install" feedbackv1alpha1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) @@ -50,6 +55,12 @@ func init() { // ExtraConfig holds feedback-specific configuration. type ExtraConfig struct { Postgres postgres.Config + // Identities are the usernames of feedback-sync and feedback-controller, + // which own particular fields. + Identities identity.Identities + // VoteRetention is how long votes on a deleted post are kept in case the + // post comes back. + VoteRetention time.Duration } // Config combines the generic and feedback-specific configuration. @@ -110,7 +121,35 @@ func (c completedConfig) New(ctx context.Context) (*FeedbackServer, error) { v1alpha1Storage["feedbacksources"] = sourceStorage.Store v1alpha1Storage["feedbacksources/status"] = sourceStorage.Status - // Postgres-backed resources (Post, PostQuery, Vote, Submission, ...) are + // Everything else is in Postgres. + db := &pg.DB{Pool: pool} + ids := c.ExtraConfig.Identities + + posts, err := post.NewREST(db, Scheme, ids) + if err != nil { + pool.Close() + return nil, fmt.Errorf("create Post storage: %w", err) + } + v1alpha1Storage["posts"] = posts.Store + v1alpha1Storage["posts/status"] = posts.Status + + releases, err := release.NewReleaseREST(db, Scheme, ids) + if err != nil { + pool.Close() + return nil, fmt.Errorf("create Release storage: %w", err) + } + v1alpha1Storage["releases"] = releases.Store + v1alpha1Storage["releases/status"] = releases.Status + + changelog, err := release.NewChangelogREST(db, Scheme, ids) + if err != nil { + pool.Close() + return nil, fmt.Errorf("create ChangelogEntry storage: %w", err) + } + v1alpha1Storage["changelogentries"] = changelog.Store + v1alpha1Storage["changelogentries/status"] = changelog.Status + + // Other Postgres-backed resources (PostQuery, Vote, Submission, ...) are // added here as they are implemented. See datum-cloud/enhancements#943. apiGroupInfo.VersionedResourcesStorageMap[feedbackv1alpha1.SchemeGroupVersion.Version] = v1alpha1Storage @@ -119,10 +158,40 @@ func (c completedConfig) New(ctx context.Context) (*FeedbackServer, error) { return nil, err } + retention := c.ExtraConfig.VoteRetention + genericServer.AddPostStartHookOrDie("feedback-vote-retention", func(hc genericapiserver.PostStartHookContext) error { + go runVoteRetention(hc.Context, db, retention) + return nil + }) + klog.Info("Feedback server initialized") return s, nil } +// runVoteRetention purges votes on long-deleted posts every hour. +func runVoteRetention(ctx context.Context, db *pg.DB, retention time.Duration) { + if retention <= 0 { + klog.Info("Vote retention purge disabled") + return + } + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + n, err := post.PurgeExpiredVotes(ctx, db, retention) + switch { + case err != nil && ctx.Err() == nil: + klog.ErrorS(err, "Purging votes on deleted posts failed") + case n > 0: + klog.InfoS("Purged votes on deleted posts", "votes", n, "retention", retention) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + // Run serves until ctx is cancelled, then closes the Postgres pool. func (s *FeedbackServer) Run(ctx context.Context) error { defer s.pool.Close() diff --git a/internal/identity/identity.go b/internal/identity/identity.go new file mode 100644 index 0000000..8391b41 --- /dev/null +++ b/internal/identity/identity.go @@ -0,0 +1,68 @@ +// Package identity recognises the feedback service's own writers. +// +// Milo's authorization decides which verbs a caller may use on a resource. +// Some rules are about fields within a resource (only feedback-sync writes a +// post's GitHub fields, only feedback-controller writes its delivery +// tracker), which IAM can't express, so strategies check the caller here. +package identity + +import ( + "context" + "slices" + + "k8s.io/apiserver/pkg/endpoints/request" +) + +// Identities holds the usernames of the service's own components. +type Identities struct { + // Sync are feedback-sync's usernames. It is the only writer of + // GitHub-derived content. + Sync []string + // Controller are feedback-controller's usernames. It writes triage and + // delivery tracking. + Controller []string +} + +// Default usernames, matching the Kubernetes service accounts in +// feedback-system. Override with flags where Milo names them differently. +var Default = Identities{ + Sync: []string{"system:serviceaccount:feedback-system:feedback-sync"}, + Controller: []string{"system:serviceaccount:feedback-system:feedback-controller"}, +} + +// Caller is who a request comes from, as far as field ownership goes. +type Caller int + +const ( + // Other is anyone else: staff, the website, users. + Other Caller = iota + Sync + Controller +) + +func (c Caller) String() string { + switch c { + case Sync: + return "feedback-sync" + case Controller: + return "feedback-controller" + default: + return "other" + } +} + +// Of returns the caller for ctx. +func (ids Identities) Of(ctx context.Context) Caller { + u, ok := request.UserFrom(ctx) + if !ok { + return Other + } + switch name := u.GetName(); { + case slices.Contains(ids.Sync, name): + return Sync + case slices.Contains(ids.Controller, name): + return Controller + default: + return Other + } +} diff --git a/internal/registry/post/storage.go b/internal/registry/post/storage.go new file mode 100644 index 0000000..87a4e7b --- /dev/null +++ b/internal/registry/post/storage.go @@ -0,0 +1,229 @@ +package post + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/apiserver/pkg/registry/rest" + + "go.miloapis.com/feedback/internal/identity" + "go.miloapis.com/feedback/internal/storage/pg" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +// REST is the storage for posts and posts/status. +type REST struct { + *pg.Store + Status *pg.StatusStore +} + +// NewREST returns Postgres-backed storage for Post. +func NewREST(db *pg.DB, typer runtime.ObjectTyper, ids identity.Identities) (*REST, error) { + s := newStrategy(typer, ids) + h := hooks{db: db, ids: ids} + store, err := pg.NewStore(db, pg.Kind{ + Table: "posts", + NewFunc: func() runtime.Object { return &v1.Post{} }, + NewListFunc: func() runtime.Object { return &v1.PostList{} }, + DefaultQualifiedResource: v1.Resource("posts"), + SingularQualifiedResource: v1.Resource("post"), + CreateStrategy: s, + UpdateStrategy: s, + DeleteStrategy: s, + Scope: pg.PlatformScoped, + Columns: columns, + FieldColumns: map[string]string{ + "spec.hidden": "hidden", + "status.state": "state", + "status.github.org": "github_org", + "status.github.repository": "github_repository", + "status.release": "release", + }, + TableConvertor: tableConvertor{}, + ReadFilter: readFilter, + BeforeWrite: h.beforeWrite, + AfterWrite: h.afterWrite, + }) + if err != nil { + return nil, err + } + return &REST{Store: store, Status: pg.NewStatusStore(store, statusStrategy{s})}, nil +} + +// readFilter hides hidden posts outside the platform context. Platform +// callers (staff, the website's service account, the service's own +// components) see them and filter with spec.hidden=false where needed. +func readFilter(ctx context.Context) string { + if kind, _ := pg.RequestContext(ctx); kind != pg.ScopePlatform { + return "NOT hidden" + } + return "" +} + +func columns(obj runtime.Object) []pg.Column { + p := obj.(*v1.Post) + var ( + org, repo string + number int32 + createdAt *time.Time + stage string + changedAt *time.Time + ) + if gh := p.Status.GitHub; gh != nil { + org, repo, number = gh.Org, gh.Repository, gh.Number + if !gh.CreatedAt.IsZero() { + t := gh.CreatedAt.Time + createdAt = &t + } + } + if d := p.Status.Delivery; d != nil { + stage = string(d.Stage) + if n := len(d.History); n > 0 { + t := d.History[n-1].At.Time + changedAt = &t + } + } + return []pg.Column{ + {Name: "hidden", Value: p.Spec.Hidden}, + {Name: "state", Value: string(p.Status.State)}, + {Name: "github_org", Value: org}, + {Name: "github_repository", Value: repo}, + {Name: "github_number", Value: number}, + {Name: "github_created_at", Value: createdAt}, + {Name: "release", Value: p.Status.Release}, + {Name: "votes_total", Value: p.Status.Votes.Total}, + {Name: "trending_score", Value: p.Status.Votes.TrendingScore}, + {Name: "delivery_stage", Value: stage}, + {Name: "delivery_changed_at", Value: changedAt}, + } +} + +type hooks struct { + db *pg.DB + ids identity.Identities +} + +func (h hooks) beforeWrite(ctx context.Context, op pg.Op, old, obj runtime.Object) error { + if op != pg.OpUpdate && h.ids.Of(ctx) != identity.Sync { + name := "" + if obj != nil { + name = obj.(*v1.Post).Name + } else { + name = old.(*v1.Post).Name + } + return apierrors.NewForbidden(v1.Resource("posts"), name, + fmt.Errorf("posts are created and deleted only by feedback-sync")) + } + if obj == nil { + return nil + } + return h.computeVotes(ctx, obj.(*v1.Post)) +} + +// computeVotes sets the Datum vote count and the total. Matching GitHub +// reactors to linked Datum accounts, and the trending score, come with vote +// storage. +func (h hooks) computeVotes(ctx context.Context, p *v1.Post) error { + var datum int32 + if err := h.db.Q(ctx).QueryRow(ctx, "SELECT count(*) FROM votes WHERE post_name = $1", p.Name).Scan(&datum); err != nil { + return apierrors.NewInternalError(fmt.Errorf("count votes: %w", err)) + } + p.Status.Votes.Datum = datum + p.Status.Votes.Total = datum + p.Status.Votes.GitHub + return nil +} + +func (h hooks) afterWrite(ctx context.Context, op pg.Op, old, obj runtime.Object) error { + q := h.db.Q(ctx) + var err error + switch op { + case pg.OpCreate: + // A post that comes back keeps the votes it had. + _, err = q.Exec(ctx, "DELETE FROM deleted_posts WHERE name = $1", obj.(*v1.Post).Name) + case pg.OpDelete: + name := old.(*v1.Post).Name + if _, err = q.Exec(ctx, `INSERT INTO deleted_posts (name, deleted_at) VALUES ($1, now()) + ON CONFLICT (name) DO UPDATE SET deleted_at = excluded.deleted_at`, name); err == nil { + _, err = q.Exec(ctx, "DELETE FROM post_embeddings WHERE post_name = $1", name) + } + case pg.OpUpdate: + // The embedder re-embeds posts that have no embedding. + o, n := old.(*v1.Post), obj.(*v1.Post) + if o.Status.Title != n.Status.Title || o.Status.BodyHTML != n.Status.BodyHTML { + _, err = q.Exec(ctx, "DELETE FROM post_embeddings WHERE post_name = $1", n.Name) + } + } + if err != nil { + return apierrors.NewInternalError(fmt.Errorf("post %s side effects: %w", op, err)) + } + return nil +} + +// PurgeExpiredVotes deletes votes and subscriptions for posts deleted more +// than retention ago, and forgets those posts. It returns the number of +// votes removed. +func PurgeExpiredVotes(ctx context.Context, db *pg.DB, retention time.Duration) (int64, error) { + var votes int64 + err := db.InTx(ctx, func(ctx context.Context) error { + return db.Q(ctx).QueryRow(ctx, ` + WITH expired AS ( + DELETE FROM deleted_posts WHERE deleted_at < now() - make_interval(secs => $1) RETURNING name + ), v AS ( + DELETE FROM votes WHERE post_name IN (SELECT name FROM expired) RETURNING 1 + ), s AS ( + DELETE FROM subscriptions WHERE post_name IN (SELECT name FROM expired) RETURNING 1 + ) + -- Data-modifying CTEs always run in full, so s runs without being read. + SELECT count(*) FROM v`, retention.Seconds()).Scan(&votes) + }) + return votes, err +} + +var errNotPost = errors.New("not a Post") + +type tableConvertor struct{} + +var _ rest.TableConvertor = tableConvertor{} + +func (tableConvertor) ConvertToTable(_ context.Context, obj runtime.Object, _ runtime.Object) (*metav1.Table, error) { + t := &metav1.Table{ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "State", Type: "string"}, + {Name: "Votes", Type: "integer"}, + {Name: "Stage", Type: "string"}, + {Name: "Hidden", Type: "boolean"}, + {Name: "Title", Type: "string"}, + {Name: "Age", Type: "string"}, + }} + add := func(p *v1.Post) { + stage := "" + if p.Status.Delivery != nil { + stage = string(p.Status.Delivery.Stage) + } + age := "" + if !p.CreationTimestamp.IsZero() { + age = duration.HumanDuration(time.Since(p.CreationTimestamp.Time)) + } + t.Rows = append(t.Rows, metav1.TableRow{ + Object: runtime.RawExtension{Object: p}, + Cells: []any{p.Name, string(p.Status.State), p.Status.Votes.Total, stage, p.Spec.Hidden, p.Status.Title, age}, + }) + } + switch o := obj.(type) { + case *v1.Post: + add(o) + case *v1.PostList: + for i := range o.Items { + add(&o.Items[i]) + } + default: + return nil, errNotPost + } + return t, nil +} diff --git a/internal/registry/post/storage_test.go b/internal/registry/post/storage_test.go new file mode 100644 index 0000000..7f0e1d8 --- /dev/null +++ b/internal/registry/post/storage_test.go @@ -0,0 +1,316 @@ +package post + +import ( + "context" + "os" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + "go.miloapis.com/feedback/internal/storage/pg" + "go.miloapis.com/feedback/internal/storage/pg/pgtest" + "go.miloapis.com/feedback/pkg/apis/feedback/install" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +var ( + syncCtx = pgtest.AsPlatform("sync") + ctrlCtx = pgtest.AsPlatform("controller") + staffCtx = pgtest.AsPlatform("staff@datum.net") + userCtx = pgtest.AsUser("u1") +) + +func setup(t *testing.T) (*pg.DB, *REST) { + t.Helper() + db := pgtest.Require(t, "posts", "votes", "subscriptions", "deleted_posts", "post_embeddings") + scheme := runtime.NewScheme() + install.Install(scheme) + r, err := NewREST(db, scheme, ids) + if err != nil { + t.Fatal(err) + } + return db, r +} + +func newPost(number int32, state v1.PostState) *v1.Post { + return &v1.Post{ + ObjectMeta: metav1.ObjectMeta{Name: v1.PostName("datum-cloud", "enhancements", number)}, + Status: v1.PostStatus{ + Title: "Post", BodyHTML: "

body

", State: state, + GitHub: &v1.GitHubIssue{Org: "datum-cloud", Repository: "enhancements", Number: number, State: "open", + URL: "https://github.com/datum-cloud/enhancements/issues/1"}, + Votes: v1.PostVotes{GitHub: 4, Datum: 50, Total: 50}, + }, + } +} + +func addVote(t *testing.T, db *pg.DB, userUID, postName string) { + t.Helper() + _, err := db.Pool.Exec(context.Background(), `INSERT INTO votes + (scope_type, scope_name, name, uid, resource_version, created_at, object, post_name) + VALUES ('User', $1, $2, gen_random_uuid(), nextval('resource_version_seq'), now(), '{}', $2)`, userUID, postName) + if err != nil { + t.Fatal(err) + } +} + +func get(t *testing.T, r *REST, ctx context.Context, name string) *v1.Post { + t.Helper() + obj, err := r.Get(ctx, name, &metav1.GetOptions{}) + if err != nil { + t.Fatalf("get %s: %v", name, err) + } + return obj.(*v1.Post) +} + +func update(r rest.Updater, ctx context.Context, p *v1.Post) (*v1.Post, error) { + out, _, err := r.Update(ctx, p.Name, rest.DefaultUpdatedObjectInfo(p), nil, nil, false, &metav1.UpdateOptions{}) + if err != nil { + return nil, err + } + return out.(*v1.Post), nil +} + +func TestOnlySyncCreatesAndDeletes(t *testing.T) { + _, r := setup(t) + if _, err := r.Create(staffCtx, newPost(1, ""), nil, &metav1.CreateOptions{}); !apierrors.IsForbidden(err) { + t.Fatalf("staff create: got %v, want Forbidden", err) + } + if _, err := r.Create(syncCtx, newPost(1, ""), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, _, err := r.Delete(staffCtx, newPost(1, "").Name, nil, &metav1.DeleteOptions{}); !apierrors.IsForbidden(err) { + t.Fatalf("staff delete: got %v, want Forbidden", err) + } + + wrong := newPost(2, "") + wrong.Name = "datum-cloud.enhancements.3" + if _, err := r.Create(syncCtx, wrong, nil, &metav1.CreateOptions{}); !apierrors.IsInvalid(err) { + t.Fatalf("name not matching github: got %v, want Invalid", err) + } +} + +func TestVotesComputed(t *testing.T) { + db, r := setup(t) + name := newPost(1, "").Name + addVote(t, db, "u1", name) + addVote(t, db, "u2", name) + + created, err := r.Create(syncCtx, newPost(1, ""), nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + if v := created.(*v1.Post).Status.Votes; v.Datum != 2 || v.GitHub != 4 || v.Total != 6 { + t.Fatalf("votes on create = %+v, want datum 2 github 4 total 6", v) + } + var col int32 + if err := db.Pool.QueryRow(context.Background(), "SELECT votes_total FROM posts WHERE name = $1", name).Scan(&col); err != nil || col != 6 { + t.Fatalf("votes_total column = %d, %v", col, err) + } + + // A sync update with a new 👍 count recomputes the total. + p := get(t, r, syncCtx, name) + p.Status.Votes.GitHub = 10 + out, err := update(r.Status, syncCtx, p) + if err != nil { + t.Fatal(err) + } + if v := out.Status.Votes; v.Total != 12 { + t.Fatalf("votes after sync update = %+v", v) + } +} + +func TestStaffHideAndForbiddenStatus(t *testing.T) { + _, r := setup(t) + created, _ := r.Create(syncCtx, newPost(1, v1.PostStateBacklog), nil, &metav1.CreateOptions{}) + name := created.(*v1.Post).Name + + p := get(t, r, staffCtx, name) + p.Spec.Hidden = true + p.Status.Title = "staff edit" + out, err := update(r, staffCtx, p) + if err != nil { + t.Fatal(err) + } + if !out.Spec.Hidden || out.Status.Title != "Post" { + t.Fatalf("staff main update: hidden=%v title=%q", out.Spec.Hidden, out.Status.Title) + } + + p = get(t, r, staffCtx, name) + p.Status.State = v1.PostStateShipped + if _, err := update(r.Status, staffCtx, p); !apierrors.IsInvalid(err) { + t.Fatalf("staff status update: got %v, want Invalid (forbidden field)", err) + } +} + +func TestSyncAndControllerOwnership(t *testing.T) { + _, r := setup(t) + created, _ := r.Create(syncCtx, newPost(1, v1.PostStateBacklog), nil, &metav1.CreateOptions{}) + name := created.(*v1.Post).Name + + // The controller sets the stage. + p := get(t, r, ctrlCtx, name) + p.Status.Delivery = &v1.DeliveryStatus{Stage: v1.DeliveryStagePlanned, HighestNotified: v1.DeliveryStagePlanned, + History: []v1.DeliveryEvent{{Stage: v1.DeliveryStagePlanned, At: metav1.Now()}}} + p.Status.Title = "controller edit" + if _, err := update(r.Status, ctrlCtx, p); err != nil { + t.Fatal(err) + } + + // Sync refreshes work items and tries to reset the stage. + p = get(t, r, syncCtx, name) + p.Status.State = v1.PostStateInProgress + p.Status.Delivery.Stage = v1.DeliveryStageRequested + p.Status.Delivery.Work = &v1.DeliveryWork{Total: 1, Items: []v1.WorkItem{{Kind: "issue", Ref: "datum-cloud/compute#9", State: "open"}}} + if _, err := update(r.Status, syncCtx, p); err != nil { + t.Fatal(err) + } + + got := get(t, r, staffCtx, name) + if got.Status.Title != "Post" || got.Status.State != v1.PostStateInProgress { + t.Errorf("sync fields: title=%q state=%s", got.Status.Title, got.Status.State) + } + if d := got.Status.Delivery; d.Stage != v1.DeliveryStagePlanned || len(d.History) != 1 || d.Work == nil || d.Work.Total != 1 { + t.Errorf("delivery = %+v", d) + } +} + +func TestHiddenOutsidePlatform(t *testing.T) { + _, r := setup(t) + visible, _ := r.Create(syncCtx, newPost(1, v1.PostStateBacklog), nil, &metav1.CreateOptions{}) + hidden, _ := r.Create(syncCtx, newPost(2, v1.PostStateBacklog), nil, &metav1.CreateOptions{}) + h := hidden.(*v1.Post) + h.Spec.Hidden = true + if _, err := update(r, staffCtx, h); err != nil { + t.Fatal(err) + } + + if _, err := r.Get(userCtx, h.Name, &metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("user get hidden: got %v, want NotFound", err) + } + list, err := r.List(userCtx, &metainternalversion.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if items := list.(*v1.PostList).Items; len(items) != 1 || items[0].Name != visible.(*v1.Post).Name { + t.Fatalf("user list = %d items", len(items)) + } + list, _ = r.List(staffCtx, &metainternalversion.ListOptions{}) + if n := len(list.(*v1.PostList).Items); n != 2 { + t.Fatalf("platform list = %d items, want 2", n) + } + list, _ = r.List(staffCtx, &metainternalversion.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.hidden", "false")}) + if n := len(list.(*v1.PostList).Items); n != 1 { + t.Fatalf("platform list spec.hidden=false = %d items, want 1", n) + } +} + +func TestFieldSelectors(t *testing.T) { + _, r := setup(t) + r.Create(syncCtx, newPost(1, v1.PostStateBacklog), nil, &metav1.CreateOptions{}) + r.Create(syncCtx, newPost(2, v1.PostStateInProgress), nil, &metav1.CreateOptions{}) + other := newPost(3, v1.PostStateInProgress) + other.Name = v1.PostName("milo-os", "search", 3) + other.Status.GitHub.Org, other.Status.GitHub.Repository = "milo-os", "search" + if _, err := r.Create(syncCtx, other, nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + for sel, want := range map[string]int{ + "status.state=in-progress": 2, + "status.github.org=milo-os": 1, + "status.state=in-progress,status.github.org=milo-os": 1, + "status.github.repository=enhancements": 2, + } { + fs, _ := fields.ParseSelector(sel) + list, err := r.List(staffCtx, &metainternalversion.ListOptions{FieldSelector: fs}) + if err != nil { + t.Fatal(err) + } + if n := len(list.(*v1.PostList).Items); n != want { + t.Errorf("%s: %d items, want %d", sel, n, want) + } + } +} + +func TestEmbeddingInvalidated(t *testing.T) { + db, r := setup(t) + created, _ := r.Create(syncCtx, newPost(1, ""), nil, &metav1.CreateOptions{}) + name := created.(*v1.Post).Name + embed := func() { + _, err := db.Pool.Exec(context.Background(), + "INSERT INTO post_embeddings (post_name, model, content_hash, embedding) VALUES ($1, 'm', 'h', '[1,2,3]')", name) + if err != nil { + t.Fatal(err) + } + } + has := func() bool { + var n int + _ = db.Pool.QueryRow(context.Background(), "SELECT count(*) FROM post_embeddings WHERE post_name = $1", name).Scan(&n) + return n == 1 + } + + embed() + p := get(t, r, syncCtx, name) + p.Status.State = v1.PostStateShipped + update(r.Status, syncCtx, p) + if !has() { + t.Fatal("state change dropped the embedding") + } + p = get(t, r, syncCtx, name) + p.Status.Title = "Renamed" + update(r.Status, syncCtx, p) + if has() { + t.Fatal("title change kept a stale embedding") + } +} + +func TestVoteRetention(t *testing.T) { + db, r := setup(t) + ctx := context.Background() + keep, gone := newPost(1, ""), newPost(2, "") + for _, p := range []*v1.Post{keep, gone} { + if _, err := r.Create(syncCtx, p.DeepCopy(), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + addVote(t, db, "u1", p.Name) + if _, _, err := r.Delete(syncCtx, p.Name, nil, &metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + } + // One was deleted long ago. + if _, err := db.Pool.Exec(ctx, "UPDATE deleted_posts SET deleted_at = now() - interval '31 days' WHERE name = $1", gone.Name); err != nil { + t.Fatal(err) + } + + n, err := PurgeExpiredVotes(ctx, db, 30*24*time.Hour) + if err != nil || n != 1 { + t.Fatalf("purged %d votes, %v; want 1", n, err) + } + var votes int + db.Pool.QueryRow(ctx, "SELECT count(*) FROM votes").Scan(&votes) + if votes != 1 { + t.Fatalf("%d votes left, want 1", votes) + } + + // The kept post comes back with its vote. + created, err := r.Create(syncCtx, keep.DeepCopy(), nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + if v := created.(*v1.Post).Status.Votes.Datum; v != 1 { + t.Fatalf("returned post has %d Datum votes, want 1", v) + } + var tracked int + db.Pool.QueryRow(ctx, "SELECT count(*) FROM deleted_posts").Scan(&tracked) + if tracked != 0 { + t.Fatalf("deleted_posts still has %d rows", tracked) + } +} diff --git a/internal/registry/post/strategy.go b/internal/registry/post/strategy.go new file mode 100644 index 0000000..cd86d7c --- /dev/null +++ b/internal/registry/post/strategy.go @@ -0,0 +1,201 @@ +// Package post implements storage for Post. +// +// Field ownership: +// - feedback-sync creates and deletes posts and owns everything in status +// except the fields below. +// - feedback-controller owns status.delivery's stage, highestNotified, +// history and work.partShipped. feedback-sync keeps work items, totals +// and pull requests, which it refreshes from GitHub. +// - The server computes status.votes.datum and status.votes.total on every +// write. status.votes.trendingScore is server-owned too. +// - Everyone else (staff) can change spec.hidden and metadata only. +package post + +import ( + "context" + "slices" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/storage/names" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" + + "go.miloapis.com/feedback/internal/identity" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +type strategy struct { + runtime.ObjectTyper + names.NameGenerator + ids identity.Identities +} + +func newStrategy(typer runtime.ObjectTyper, ids identity.Identities) strategy { + return strategy{ObjectTyper: typer, NameGenerator: names.SimpleNameGenerator, ids: ids} +} + +func (strategy) NamespaceScoped() bool { return false } + +// PrepareForCreate drops fields that feedback-sync doesn't own. The +// controller fills the delivery tracker and the server computes votes. +func (strategy) PrepareForCreate(_ context.Context, obj runtime.Object) { + p := obj.(*v1.Post) + keepServerVotes(&p.Status, v1.PostVotes{}) + keepControllerDelivery(&p.Status, nil) +} + +// PrepareForUpdate keeps status: it changes through the status subresource. +func (strategy) PrepareForUpdate(_ context.Context, obj, old runtime.Object) { + obj.(*v1.Post).Status = old.(*v1.Post).Status +} + +func (strategy) Validate(_ context.Context, obj runtime.Object) field.ErrorList { + p := obj.(*v1.Post) + errs := validate(p) + if p.Status.GitHub == nil { + errs = append(errs, field.Required(field.NewPath("status", "github"), "every post is backed by a GitHub issue")) + } + return errs +} + +func (strategy) ValidateUpdate(_ context.Context, obj, old runtime.Object) field.ErrorList { + return validate(obj.(*v1.Post)) +} + +func (strategy) WarningsOnCreate(context.Context, runtime.Object) []string { return nil } +func (strategy) WarningsOnUpdate(context.Context, runtime.Object, runtime.Object) []string { + return nil +} +func (strategy) AllowCreateOnUpdate(context.Context) bool { return false } +func (strategy) AllowUnconditionalUpdate(context.Context) bool { return true } +func (strategy) Canonicalize(runtime.Object) {} + +func (strategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set { + return map[fieldpath.APIVersion]*fieldpath.Set{ + fieldpath.APIVersion(v1.SchemeGroupVersion.String()): fieldpath.NewSet(fieldpath.MakePathOrDie("status")), + } +} + +// statusStrategy merges a status update field by field according to who is +// calling. Fields the caller doesn't own keep their stored value, like spec +// on any status update. +type statusStrategy struct { + strategy +} + +func (s statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + p, o := obj.(*v1.Post), old.(*v1.Post) + p.Spec = o.Spec + p.Labels, p.Annotations = o.Labels, o.Annotations + + switch s.ids.Of(ctx) { + case identity.Sync: + keepServerVotes(&p.Status, o.Status.Votes) + keepControllerDelivery(&p.Status, o.Status.Delivery) + case identity.Controller: + incoming := p.Status.Delivery + p.Status = *o.Status.DeepCopy() + keepControllerDelivery(&p.Status, incoming) + default: + p.Status = o.Status + } +} + +func (s statusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { + if s.ids.Of(ctx) == identity.Other { + return field.ErrorList{field.Forbidden(field.NewPath("status"), + "post status is written only by feedback-sync and feedback-controller")} + } + return validate(obj.(*v1.Post)) +} + +func (statusStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set { + return map[fieldpath.APIVersion]*fieldpath.Set{ + fieldpath.APIVersion(v1.SchemeGroupVersion.String()): fieldpath.NewSet(fieldpath.MakePathOrDie("spec")), + } +} + +// keepServerVotes resets the server-computed vote fields to from's. +func keepServerVotes(st *v1.PostStatus, from v1.PostVotes) { + st.Votes.Datum = from.Datum + st.Votes.Total = from.Total + st.Votes.TrendingScore = from.TrendingScore +} + +// keepControllerDelivery sets the controller-owned delivery fields in st to +// from's, leaving the sync-owned work items alone. +func keepControllerDelivery(st *v1.PostStatus, from *v1.DeliveryStatus) { + if from == nil { + from = &v1.DeliveryStatus{} + } + if st.Delivery == nil { + st.Delivery = &v1.DeliveryStatus{} + } + d := st.Delivery + d.Stage = from.Stage + d.HighestNotified = from.HighestNotified + d.History = slices.Clone(from.History) + var partShipped []v1.PartShipped + if from.Work != nil { + partShipped = slices.Clone(from.Work.PartShipped) + } + if d.Work == nil && len(partShipped) > 0 { + d.Work = &v1.DeliveryWork{} + } + if d.Work != nil { + d.Work.PartShipped = partShipped + } + if d.Stage == "" && d.HighestNotified == "" && len(d.History) == 0 && d.Work == nil { + st.Delivery = nil + } +} + +func validate(p *v1.Post) field.ErrorList { + var errs field.ErrorList + st := field.NewPath("status") + if gh := p.Status.GitHub; gh != nil { + ghp := st.Child("github") + if gh.Org == "" { + errs = append(errs, field.Required(ghp.Child("org"), "")) + } + if gh.Repository == "" { + errs = append(errs, field.Required(ghp.Child("repository"), "")) + } + if gh.Number <= 0 { + errs = append(errs, field.Invalid(ghp.Child("number"), gh.Number, "must be positive")) + } + if gh.State != "open" && gh.State != "closed" { + errs = append(errs, field.NotSupported(ghp.Child("state"), gh.State, []string{"open", "closed"})) + } + if want := v1.PostName(gh.Org, gh.Repository, gh.Number); len(errs) == 0 && p.Name != want { + errs = append(errs, field.Invalid(field.NewPath("metadata", "name"), p.Name, + "must be "+want+", from status.github")) + } + } + if s := p.Status.State; s != "" && !slices.Contains(v1.ValidPostStates, s) { + errs = append(errs, field.NotSupported(st.Child("state"), s, v1.ValidPostStates)) + } + if p.Status.Votes.GitHub < 0 { + errs = append(errs, field.Invalid(st.Child("votes", "github"), p.Status.Votes.GitHub, "must not be negative")) + } + if d := p.Status.Delivery; d != nil { + dp := st.Child("delivery") + for _, f := range []struct { + path *field.Path + stage v1.DeliveryStage + }{{dp.Child("stage"), d.Stage}, {dp.Child("highestNotified"), d.HighestNotified}} { + if f.stage != "" && !slices.Contains(v1.ValidDeliveryStages, f.stage) { + errs = append(errs, field.NotSupported(f.path, f.stage, v1.ValidDeliveryStages)) + } + } + for i, h := range d.History { + if !slices.Contains(v1.ValidDeliveryStages, h.Stage) { + errs = append(errs, field.NotSupported(dp.Child("history").Index(i).Child("stage"), h.Stage, v1.ValidDeliveryStages)) + } + } + } + if len(p.Status.PublicVoters) > 10 { + errs = append(errs, field.TooMany(st.Child("publicVoters"), len(p.Status.PublicVoters), 10)) + } + return errs +} diff --git a/internal/registry/post/strategy_test.go b/internal/registry/post/strategy_test.go new file mode 100644 index 0000000..810b6f8 --- /dev/null +++ b/internal/registry/post/strategy_test.go @@ -0,0 +1,149 @@ +package post + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + + "go.miloapis.com/feedback/internal/identity" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +var ids = identity.Identities{Sync: []string{"sync"}, Controller: []string{"controller"}} + +func as(name string) context.Context { + return request.WithUser(context.Background(), &user.DefaultInfo{Name: name}) +} + +func stored() *v1.Post { + return &v1.Post{ + ObjectMeta: metav1.ObjectMeta{Name: "datum-cloud.enhancements.1", Labels: map[string]string{"a": "1"}}, + Spec: v1.PostSpec{Hidden: false}, + Status: v1.PostStatus{ + Title: "old", + State: v1.PostStateBacklog, + Votes: v1.PostVotes{Datum: 3, GitHub: 2, Total: 5, TrendingScore: 1.5}, + Delivery: &v1.DeliveryStatus{ + Stage: v1.DeliveryStagePlanned, HighestNotified: v1.DeliveryStageRequested, + History: []v1.DeliveryEvent{{Stage: v1.DeliveryStagePlanned}}, + Work: &v1.DeliveryWork{Total: 2, Done: 1, Items: []v1.WorkItem{{Kind: "issue", Ref: "o/r#2", State: "open"}}, + PartShipped: []v1.PartShipped{{Changelog: "c1"}}}, + }, + }, + } +} + +// incoming is a status update that touches every owned area. +func incoming() *v1.Post { + p := stored() + p.Spec.Hidden = true + p.Labels = map[string]string{"b": "2"} + p.Status.Title = "new" + p.Status.Votes = v1.PostVotes{Datum: 99, GitHub: 7, Total: 99, TrendingScore: 99} + p.Status.Delivery = &v1.DeliveryStatus{ + Stage: v1.DeliveryStagePickedUp, HighestNotified: v1.DeliveryStagePickedUp, + History: []v1.DeliveryEvent{{Stage: v1.DeliveryStagePickedUp}}, + Work: &v1.DeliveryWork{Total: 3, Done: 3, Items: []v1.WorkItem{{Kind: "issue", Ref: "o/r#3", State: "closed"}}, + PartShipped: []v1.PartShipped{{Changelog: "c2"}}}, + } + return p +} + +func TestStatusMerge(t *testing.T) { + s := statusStrategy{newStrategy(runtime.NewScheme(), ids)} + + t.Run("sync", func(t *testing.T) { + p := incoming() + s.PrepareForUpdate(as("sync"), p, stored()) + st := p.Status + if st.Title != "new" || st.Votes.GitHub != 7 { + t.Errorf("sync-owned fields not applied: %+v", st) + } + if st.Votes.Datum != 3 || st.Votes.Total != 5 || st.Votes.TrendingScore != 1.5 { + t.Errorf("server-owned votes changed: %+v", st.Votes) + } + d := st.Delivery + if d.Stage != v1.DeliveryStagePlanned || d.HighestNotified != v1.DeliveryStageRequested || + d.History[0].Stage != v1.DeliveryStagePlanned || d.Work.PartShipped[0].Changelog != "c1" { + t.Errorf("controller-owned delivery changed: %+v", d) + } + if d.Work.Total != 3 || d.Work.Items[0].Ref != "o/r#3" { + t.Errorf("sync-owned work items not applied: %+v", d.Work) + } + if p.Spec.Hidden || p.Labels["a"] != "1" { + t.Error("status update changed spec or labels") + } + if errs := s.ValidateUpdate(as("sync"), p, stored()); len(errs) > 0 { + t.Errorf("validate: %v", errs) + } + }) + + t.Run("controller", func(t *testing.T) { + p := incoming() + s.PrepareForUpdate(as("controller"), p, stored()) + st := p.Status + if st.Title != "old" || st.Votes.GitHub != 2 { + t.Errorf("controller changed sync-owned fields: %+v", st) + } + d := st.Delivery + if d.Stage != v1.DeliveryStagePickedUp || d.HighestNotified != v1.DeliveryStagePickedUp || + d.History[0].Stage != v1.DeliveryStagePickedUp || d.Work.PartShipped[0].Changelog != "c2" { + t.Errorf("controller-owned delivery not applied: %+v", d) + } + if d.Work.Total != 2 || d.Work.Items[0].Ref != "o/r#2" { + t.Errorf("controller changed work items: %+v", d.Work) + } + }) + + t.Run("staff", func(t *testing.T) { + p := incoming() + s.PrepareForUpdate(as("staff@datum.net"), p, stored()) + if p.Status.Title != "old" { + t.Error("staff changed status") + } + if errs := s.ValidateUpdate(as("staff@datum.net"), p, stored()); len(errs) != 1 { + t.Errorf("staff status update should be forbidden, got %v", errs) + } + }) +} + +func TestMainUpdateKeepsStatus(t *testing.T) { + s := newStrategy(runtime.NewScheme(), ids) + p := incoming() + s.PrepareForUpdate(as("sync"), p, stored()) + if p.Status.Title != "old" || !p.Spec.Hidden { + t.Fatalf("main update: title=%q hidden=%v", p.Status.Title, p.Spec.Hidden) + } +} + +func TestCreateDropsUnownedFields(t *testing.T) { + s := newStrategy(runtime.NewScheme(), ids) + p := incoming() + s.PrepareForCreate(as("sync"), p) + if p.Status.Votes.Datum != 0 || p.Status.Votes.Total != 0 || p.Status.Votes.GitHub != 7 { + t.Errorf("votes: %+v", p.Status.Votes) + } + d := p.Status.Delivery + if d == nil || d.Stage != "" || d.HighestNotified != "" || len(d.History) != 0 || len(d.Work.PartShipped) != 0 || d.Work.Total != 3 { + t.Errorf("delivery: %+v", d) + } +} + +func TestValidate(t *testing.T) { + ok := stored() + ok.Status.GitHub = &v1.GitHubIssue{Org: "Datum-Cloud", Repository: "Enhancements", Number: 1, State: "open"} + if errs := validate(ok); len(errs) > 0 { + t.Fatalf("valid post rejected: %v", errs) + } + bad := ok.DeepCopy() + bad.Name = "datum-cloud.enhancements.2" + bad.Status.State = "someday" + bad.Status.Delivery.Stage = "nearly" + if errs := validate(bad); len(errs) != 3 { + t.Fatalf("want 3 errors (name, state, stage), got %v", errs) + } +} diff --git a/internal/registry/release/release.go b/internal/registry/release/release.go new file mode 100644 index 0000000..09c7028 --- /dev/null +++ b/internal/registry/release/release.go @@ -0,0 +1,316 @@ +// Package release implements storage for Release and ChangelogEntry. Both +// are synced from GitHub: only feedback-sync writes them and everyone can +// read them. Their status.posts lists are derived from the posts table on +// every read, so they always match the posts that exist and are visible. +package release + +import ( + "context" + "fmt" + "regexp" + "slices" + "strconv" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/storage/names" + + "go.miloapis.com/feedback/internal/identity" + "go.miloapis.com/feedback/internal/storage/pg" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +// REST is the storage for one of the kinds and its status subresource. +type REST struct { + *pg.Store + Status *pg.StatusStore +} + +// strategy is shared by both kinds: they have only status, so an update +// through either endpoint may change it. +type strategy struct { + runtime.ObjectTyper + names.NameGenerator + validate func(runtime.Object) field.ErrorList +} + +func (strategy) NamespaceScoped() bool { return false } +func (strategy) PrepareForCreate(context.Context, runtime.Object) {} +func (strategy) PrepareForUpdate(context.Context, runtime.Object, runtime.Object) {} +func (s strategy) Validate(_ context.Context, obj runtime.Object) field.ErrorList { + return s.validate(obj) +} +func (s strategy) ValidateUpdate(_ context.Context, obj, _ runtime.Object) field.ErrorList { + return s.validate(obj) +} +func (strategy) WarningsOnCreate(context.Context, runtime.Object) []string { return nil } +func (strategy) WarningsOnUpdate(context.Context, runtime.Object, runtime.Object) []string { + return nil +} +func (strategy) AllowCreateOnUpdate(context.Context) bool { return false } +func (strategy) AllowUnconditionalUpdate(context.Context) bool { return true } +func (strategy) Canonicalize(runtime.Object) {} + +// syncOnly rejects writes from anyone but feedback-sync. +func syncOnly(ids identity.Identities, gr schema.GroupResource) func(context.Context, pg.Op, runtime.Object, runtime.Object) error { + return func(ctx context.Context, op pg.Op, _, _ runtime.Object) error { + if ids.Of(ctx) != identity.Sync { + return apierrors.NewForbidden(gr, "", fmt.Errorf("%s is written only by feedback-sync", gr.Resource)) + } + return nil + } +} + +// NewReleaseREST returns storage for Release. +func NewReleaseREST(db *pg.DB, typer runtime.ObjectTyper, ids identity.Identities) (*REST, error) { + s := strategy{ObjectTyper: typer, NameGenerator: names.SimpleNameGenerator, validate: validateRelease} + gr := v1.Resource("releases") + guard := syncOnly(ids, gr) + store, err := pg.NewStore(db, pg.Kind{ + Table: "releases", + NewFunc: func() runtime.Object { return &v1.Release{} }, + NewListFunc: func() runtime.Object { return &v1.ReleaseList{} }, + DefaultQualifiedResource: gr, + SingularQualifiedResource: v1.Resource("release"), + CreateStrategy: s, UpdateStrategy: s, DeleteStrategy: s, + Scope: pg.PlatformScoped, + Columns: func(obj runtime.Object) []pg.Column { + return []pg.Column{{Name: "shipped", Value: obj.(*v1.Release).Status.Shipped}} + }, + FieldColumns: map[string]string{"status.shipped": "shipped"}, + BeforeWrite: func(ctx context.Context, op pg.Op, old, obj runtime.Object) error { + if err := guard(ctx, op, old, obj); err != nil { + return err + } + if r, ok := obj.(*v1.Release); ok { + r.Status.Posts = nil // derived on read + } + return nil + }, + Decorate: func(ctx context.Context, objs []runtime.Object) error { return decorateReleases(ctx, db, objs) }, + TableConvertor: releaseTable{}, + }) + if err != nil { + return nil, err + } + return &REST{Store: store, Status: pg.NewStatusStore(store, s)}, nil +} + +var dueOnRE = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) + +func validateRelease(obj runtime.Object) field.ErrorList { + r := obj.(*v1.Release) + var errs field.ErrorList + st := field.NewPath("status") + if r.Status.Title == "" { + errs = append(errs, field.Required(st.Child("title"), "")) + } + if d := r.Status.DueOn; d != "" { + if _, err := time.Parse(time.DateOnly, d); err != nil || !dueOnRE.MatchString(d) { + errs = append(errs, field.Invalid(st.Child("dueOn"), d, "must be YYYY-MM-DD")) + } + } + return errs +} + +// decorateReleases fills status.posts with the visible posts on each +// release's milestone. +func decorateReleases(ctx context.Context, db *pg.DB, objs []runtime.Object) error { + byName := map[string]*v1.Release{} + var keys []string + for _, o := range objs { + r := o.(*v1.Release) + r.Status.Posts = nil + byName[r.Name] = r + keys = append(keys, r.Name) + } + rows, err := db.Q(ctx).Query(ctx, + "SELECT release, name FROM posts WHERE release = ANY($1) AND NOT hidden ORDER BY name", keys) + if err != nil { + return apierrors.NewInternalError(fmt.Errorf("release posts: %w", err)) + } + defer rows.Close() + for rows.Next() { + var rel, name string + if err := rows.Scan(&rel, &name); err != nil { + return err + } + byName[rel].Status.Posts = append(byName[rel].Status.Posts, name) + } + return rows.Err() +} + +// NewChangelogREST returns storage for ChangelogEntry. +func NewChangelogREST(db *pg.DB, typer runtime.ObjectTyper, ids identity.Identities) (*REST, error) { + s := strategy{ObjectTyper: typer, NameGenerator: names.SimpleNameGenerator, validate: validateChangelog} + gr := v1.Resource("changelogentries") + guard := syncOnly(ids, gr) + store, err := pg.NewStore(db, pg.Kind{ + Table: "changelog_entries", + NewFunc: func() runtime.Object { return &v1.ChangelogEntry{} }, + NewListFunc: func() runtime.Object { return &v1.ChangelogEntryList{} }, + DefaultQualifiedResource: gr, + SingularQualifiedResource: v1.Resource("changelogentry"), + CreateStrategy: s, UpdateStrategy: s, DeleteStrategy: s, + Scope: pg.PlatformScoped, + Columns: func(obj runtime.Object) []pg.Column { + var published *time.Time + if t := obj.(*v1.ChangelogEntry).Status.PublishedAt; !t.IsZero() { + published = &t.Time + } + return []pg.Column{{Name: "published_at", Value: published}} + }, + BeforeWrite: func(ctx context.Context, op pg.Op, old, obj runtime.Object) error { + if err := guard(ctx, op, old, obj); err != nil { + return err + } + if c, ok := obj.(*v1.ChangelogEntry); ok { + c.Status.Posts = nil // derived on read + } + return nil + }, + Decorate: func(ctx context.Context, objs []runtime.Object) error { return decorateChangelog(ctx, db, objs) }, + TableConvertor: changelogTable{}, + }) + if err != nil { + return nil, err + } + return &REST{Store: store, Status: pg.NewStatusStore(store, s)}, nil +} + +// issueRefRE matches "/#". +var issueRefRE = regexp.MustCompile(`^([A-Za-z0-9-]+)/([A-Za-z0-9._-]+)#([1-9][0-9]*)$`) + +// PostNameForRef returns the post name for an issue reference, or false if +// it isn't one. +func PostNameForRef(ref string) (string, bool) { + m := issueRefRE.FindStringSubmatch(ref) + if m == nil { + return "", false + } + n, err := strconv.ParseInt(m[3], 10, 32) + if err != nil { + return "", false + } + return v1.PostName(m[1], m[2], int32(n)), true +} + +func validateChangelog(obj runtime.Object) field.ErrorList { + c := obj.(*v1.ChangelogEntry) + var errs field.ErrorList + st := field.NewPath("status") + if c.Status.Title == "" { + errs = append(errs, field.Required(st.Child("title"), "")) + } + if c.Status.PublishedAt.IsZero() { + errs = append(errs, field.Required(st.Child("publishedAt"), "")) + } + for i, ref := range c.Status.IssueRefs { + if _, ok := PostNameForRef(ref); !ok { + errs = append(errs, field.Invalid(st.Child("issueRefs").Index(i), ref, "must be /#")) + } + } + return errs +} + +// decorateChangelog fills status.posts with the visible posts the entry's +// issueRefs point at. References to other issues (work items) are left in +// issueRefs only. +func decorateChangelog(ctx context.Context, db *pg.DB, objs []runtime.Object) error { + wanted := map[string][]*v1.ChangelogEntry{} + var keys []string + for _, o := range objs { + c := o.(*v1.ChangelogEntry) + c.Status.Posts = nil + for _, ref := range c.Status.IssueRefs { + if name, ok := PostNameForRef(ref); ok { + if _, seen := wanted[name]; !seen { + keys = append(keys, name) + } + wanted[name] = append(wanted[name], c) + } + } + } + if len(keys) == 0 { + return nil + } + rows, err := db.Q(ctx).Query(ctx, "SELECT name FROM posts WHERE name = ANY($1) AND NOT hidden", keys) + if err != nil { + return apierrors.NewInternalError(fmt.Errorf("changelog posts: %w", err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return err + } + for _, c := range wanted[name] { + c.Status.Posts = append(c.Status.Posts, name) + } + } + if err := rows.Err(); err != nil { + return err + } + for _, o := range objs { + slices.Sort(o.(*v1.ChangelogEntry).Status.Posts) + } + return nil +} + +type releaseTable struct{} + +var _ rest.TableConvertor = releaseTable{} + +func (releaseTable) ConvertToTable(_ context.Context, obj runtime.Object, _ runtime.Object) (*metav1.Table, error) { + t := &metav1.Table{ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Title", Type: "string"}, + {Name: "Due", Type: "string"}, + {Name: "Shipped", Type: "boolean"}, + {Name: "Posts", Type: "integer"}, + }} + add := func(r *v1.Release) { + t.Rows = append(t.Rows, metav1.TableRow{Object: runtime.RawExtension{Object: r}, + Cells: []any{r.Name, r.Status.Title, r.Status.DueOn, r.Status.Shipped, len(r.Status.Posts)}}) + } + switch o := obj.(type) { + case *v1.Release: + add(o) + case *v1.ReleaseList: + for i := range o.Items { + add(&o.Items[i]) + } + } + return t, nil +} + +type changelogTable struct{} + +var _ rest.TableConvertor = changelogTable{} + +func (changelogTable) ConvertToTable(_ context.Context, obj runtime.Object, _ runtime.Object) (*metav1.Table, error) { + t := &metav1.Table{ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Title", Type: "string"}, + {Name: "Published", Type: "string"}, + {Name: "Posts", Type: "integer"}, + }} + add := func(c *v1.ChangelogEntry) { + t.Rows = append(t.Rows, metav1.TableRow{Object: runtime.RawExtension{Object: c}, + Cells: []any{c.Name, c.Status.Title, c.Status.PublishedAt.Format(time.DateOnly), len(c.Status.Posts)}}) + } + switch o := obj.(type) { + case *v1.ChangelogEntry: + add(o) + case *v1.ChangelogEntryList: + for i := range o.Items { + add(&o.Items[i]) + } + } + return t, nil +} diff --git a/internal/registry/release/release_test.go b/internal/registry/release/release_test.go new file mode 100644 index 0000000..5a871f6 --- /dev/null +++ b/internal/registry/release/release_test.go @@ -0,0 +1,175 @@ +package release + +import ( + "os" + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + "go.miloapis.com/feedback/internal/identity" + "go.miloapis.com/feedback/internal/registry/post" + "go.miloapis.com/feedback/internal/storage/pg/pgtest" + "go.miloapis.com/feedback/pkg/apis/feedback/install" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +var ( + ids = identity.Identities{Sync: []string{"sync"}} + syncCtx = pgtest.AsPlatform("sync") + staffCtx = pgtest.AsPlatform("staff@datum.net") + userCtx = pgtest.AsUser("u1") +) + +type stores struct { + posts *post.REST + releases *REST + changelog *REST +} + +func setup(t *testing.T) stores { + t.Helper() + db := pgtest.Require(t, "posts", "releases", "changelog_entries") + scheme := runtime.NewScheme() + install.Install(scheme) + p, err := post.NewREST(db, scheme, ids) + if err != nil { + t.Fatal(err) + } + r, err := NewReleaseREST(db, scheme, ids) + if err != nil { + t.Fatal(err) + } + c, err := NewChangelogREST(db, scheme, ids) + if err != nil { + t.Fatal(err) + } + return stores{p, r, c} +} + +// addPost creates datum-cloud/enhancements#n on a release, optionally hidden. +func (s stores) addPost(t *testing.T, n int32, release string, hidden bool) string { + t.Helper() + p := &v1.Post{ + ObjectMeta: metav1.ObjectMeta{Name: v1.PostName("datum-cloud", "enhancements", n)}, + Status: v1.PostStatus{Title: "p", Release: release, + GitHub: &v1.GitHubIssue{Org: "datum-cloud", Repository: "enhancements", Number: n, State: "open"}}, + } + out, err := s.posts.Create(syncCtx, p, nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + if hidden { + p = out.(*v1.Post) + p.Spec.Hidden = true + if _, _, err := s.posts.Update(staffCtx, p.Name, rest.DefaultUpdatedObjectInfo(p), nil, nil, false, &metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + return p.Name +} + +func TestReleasePostsDerived(t *testing.T) { + s := setup(t) + a := s.addPost(t, 1, "hedy-2-0", false) + b := s.addPost(t, 2, "hedy-2-0", false) + s.addPost(t, 3, "hedy-2-0", true) // hidden + s.addPost(t, 4, "other", false) + + rel := &v1.Release{ObjectMeta: metav1.ObjectMeta{Name: "hedy-2-0"}, + Status: v1.ReleaseStatus{Title: "Hedy: 2.0", DueOn: "2026-10-31", Posts: []string{"stale.from.sync"}}} + if _, err := s.releases.Create(staffCtx, rel.DeepCopy(), nil, &metav1.CreateOptions{}); !apierrors.IsForbidden(err) { + t.Fatalf("staff create: got %v, want Forbidden", err) + } + out, err := s.releases.Create(syncCtx, rel, nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + want := []string{a, b} + if got := out.(*v1.Release).Status.Posts; !equal(got, want) { + t.Fatalf("create returned posts %v, want %v", got, want) + } + got, err := s.releases.Get(userCtx, "hedy-2-0", &metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if p := got.(*v1.Release).Status.Posts; !equal(p, want) { + t.Fatalf("user get posts %v, want %v", p, want) + } + list, err := s.releases.List(userCtx, &metainternalversion.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if p := list.(*v1.ReleaseList).Items[0].Status.Posts; !equal(p, want) { + t.Fatalf("list posts %v, want %v", p, want) + } + + bad := &v1.Release{ObjectMeta: metav1.ObjectMeta{Name: "bad"}, Status: v1.ReleaseStatus{DueOn: "31/10/2026"}} + if _, err := s.releases.Create(syncCtx, bad, nil, &metav1.CreateOptions{}); !apierrors.IsInvalid(err) { + t.Fatalf("invalid release: got %v, want Invalid", err) + } +} + +func TestChangelogPostsDerived(t *testing.T) { + s := setup(t) + a := s.addPost(t, 913, "", false) + s.addPost(t, 914, "", true) // hidden + + entry := &v1.ChangelogEntry{ObjectMeta: metav1.ObjectMeta{Name: "datum-cloud.datum.303"}, + Status: v1.ChangelogEntryStatus{Title: "ALB wildcard hostnames", PublishedAt: metav1.Now(), + IssueRefs: []string{"datum-cloud/enhancements#913", "datum-cloud/enhancements#914", "datum-cloud/compute#311"}}} + out, err := s.changelog.Create(syncCtx, entry, nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + if got := out.(*v1.ChangelogEntry).Status.Posts; !equal(got, []string{a}) { + t.Fatalf("posts = %v, want [%s]", got, a) + } + if refs := out.(*v1.ChangelogEntry).Status.IssueRefs; len(refs) != 3 { + t.Fatalf("issueRefs changed: %v", refs) + } + + entry = out.(*v1.ChangelogEntry) + entry.Status.Title = "staff edit" + if _, _, err := s.changelog.Update(staffCtx, entry.Name, rest.DefaultUpdatedObjectInfo(entry), nil, nil, false, &metav1.UpdateOptions{}); !apierrors.IsForbidden(err) { + t.Fatalf("staff update: got %v, want Forbidden", err) + } + + bad := &v1.ChangelogEntry{ObjectMeta: metav1.ObjectMeta{Name: "bad"}, + Status: v1.ChangelogEntryStatus{Title: "x", PublishedAt: metav1.Now(), IssueRefs: []string{"#913"}}} + if _, err := s.changelog.Create(syncCtx, bad, nil, &metav1.CreateOptions{}); !apierrors.IsInvalid(err) { + t.Fatalf("bad issue ref: got %v, want Invalid", err) + } +} + +func TestPostNameForRef(t *testing.T) { + for ref, want := range map[string]string{ + "datum-cloud/enhancements#913": "datum-cloud.enhancements.913", + "Milo-OS/Search#7": "milo-os.search.7", + "datum-cloud/datum.net#12": "datum-cloud.datum.net.12", + "#913": "", + "datum-cloud/enhancements#0": "", + } { + got, ok := PostNameForRef(ref) + if got != want || ok != (want != "") { + t.Errorf("%s: got %q %v, want %q", ref, got, ok, want) + } + } +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/storage/pg/kind.go b/internal/storage/pg/kind.go index d0f22a8..3c0d8dc 100644 --- a/internal/storage/pg/kind.go +++ b/internal/storage/pg/kind.go @@ -1,6 +1,7 @@ package pg import ( + "context" "errors" "k8s.io/apimachinery/pkg/runtime" @@ -42,8 +43,36 @@ type Kind struct { // TableConvertor renders kubectl tables. Defaults to name and age. TableConvertor rest.TableConvertor + + // ReadFilter optionally returns an extra SQL condition (no parameters) + // that every read, update and delete must match. Rows it excludes are + // NotFound to the caller. Used to hide hidden posts outside the platform + // context. + ReadFilter func(ctx context.Context) string + + // BeforeWrite runs inside the write's transaction after validation and + // before the row is written. old is nil on create and obj is nil on + // delete. It may change obj (for server-computed fields); an error aborts + // the write. + BeforeWrite func(ctx context.Context, op Op, old, obj runtime.Object) error + + // AfterWrite runs inside the write's transaction after the row is + // written, for side effects that must commit with it. + AfterWrite func(ctx context.Context, op Op, old, obj runtime.Object) error + + // Decorate fills derived fields on objects about to be returned. + Decorate func(ctx context.Context, objs []runtime.Object) error } +// Op is a write operation passed to the write hooks. +type Op string + +const ( + OpCreate Op = "create" + OpUpdate Op = "update" + OpDelete Op = "delete" +) + func (k Kind) validate() error { var errs []error if k.Table == "" { @@ -70,3 +99,33 @@ func (k Kind) columns(obj runtime.Object) []Column { } return k.Columns(obj) } + +func (k Kind) beforeWrite(ctx context.Context, op Op, old, obj runtime.Object) error { + if k.BeforeWrite == nil { + return nil + } + return k.BeforeWrite(ctx, op, old, obj) +} + +func (k Kind) afterWrite(ctx context.Context, op Op, old, obj runtime.Object) error { + if k.AfterWrite == nil { + return nil + } + return k.AfterWrite(ctx, op, old, obj) +} + +func (k Kind) decorate(ctx context.Context, objs ...runtime.Object) error { + if k.Decorate == nil || len(objs) == 0 { + return nil + } + return k.Decorate(ctx, objs) +} + +func (k Kind) readFilter(ctx context.Context, w *where) { + if k.ReadFilter == nil { + return + } + if c := k.ReadFilter(ctx); c != "" { + w.conds = append(w.conds, "("+c+")") + } +} diff --git a/internal/storage/pg/pgtest/pgtest.go b/internal/storage/pg/pgtest/pgtest.go new file mode 100644 index 0000000..09b4d05 --- /dev/null +++ b/internal/storage/pg/pgtest/pgtest.go @@ -0,0 +1,102 @@ +// Package pgtest starts a throwaway pgvector Postgres for integration tests +// and applies the real migrations. +package pgtest + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + + "go.miloapis.com/feedback/internal/postgres" + "go.miloapis.com/feedback/internal/storage/pg" +) + +// DB is set by Main when Postgres started; tests call Require to skip +// without it. +var DB *pg.DB + +// Main starts Postgres, runs the package's tests and stops it. Call it from +// TestMain: os.Exit(pgtest.Main(m)). Without Docker the tests still run and +// skip themselves through Require. +func Main(m *testing.M) int { + ctx := context.Background() + ctr, err := tcpostgres.Run(ctx, "pgvector/pgvector:pg17", + tcpostgres.WithDatabase("feedback"), + tcpostgres.WithUsername("feedback"), + tcpostgres.WithPassword("feedback"), + testcontainers.WithWaitStrategy(wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2).WithStartupTimeout(90*time.Second)), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "skipping Postgres tests: %v\n", err) + return m.Run() + } + defer func() { _ = testcontainers.TerminateContainer(ctr) }() + + dsn, err := ctr.ConnectionString(ctx, "sslmode=disable") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + cfg := postgres.Config{DSN: dsn} + if err := postgres.Migrate(ctx, cfg); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + pool, err := postgres.NewPool(ctx, cfg) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + defer pool.Close() + DB = &pg.DB{Pool: pool} + return m.Run() +} + +// Require skips the test without Postgres and empties tables. +func Require(t *testing.T, tables ...string) *pg.DB { + t.Helper() + if DB == nil { + t.Skip("Postgres not available (needs Docker)") + } + if len(tables) > 0 { + if _, err := DB.Pool.Exec(context.Background(), "TRUNCATE "+strings.Join(tables, ", ")); err != nil { + t.Fatal(err) + } + } + return DB +} + +// requestContext is what the apiserver passes a cluster-scoped request. +func requestContext() context.Context { + return request.WithNamespace(context.Background(), "") +} + +// AsUser is a request from a user in the user context. +func AsUser(uid string) context.Context { + return request.WithUser(requestContext(), &user.DefaultInfo{ + Name: uid + "@example.com", UID: uid, + Extra: map[string][]string{pg.ParentKindExtraKey: {"User"}, pg.ParentNameExtraKey: {uid}}, + }) +} + +// AsPlatform is a request from name in the platform context. +func AsPlatform(name string) context.Context { + return request.WithUser(requestContext(), &user.DefaultInfo{Name: name, UID: name}) +} + +// AsContext is a request from a user in another parent context, e.g. +// Organization. +func AsContext(kind, name string) context.Context { + return request.WithUser(requestContext(), &user.DefaultInfo{Name: "someone", Extra: map[string][]string{ + pg.ParentKindExtraKey: {kind}, pg.ParentNameExtraKey: {name}}}) +} diff --git a/internal/storage/pg/store.go b/internal/storage/pg/store.go index 6c81e3e..cc330b9 100644 --- a/internal/storage/pg/store.go +++ b/internal/storage/pg/store.go @@ -130,6 +130,7 @@ type current struct { func (s *Store) load(ctx context.Context, sc Scope, name string, forUpdate bool) (*current, error) { w := &where{} w.scope(sc) + s.kind.readFilter(ctx, w) w.add("name = %s", name) q := fmt.Sprintf("SELECT scope_type, scope_name, resource_version, object FROM %s WHERE %s LIMIT 2", s.table(), w.sql()) if forUpdate { @@ -177,6 +178,9 @@ func (s *Store) Get(ctx context.Context, name string, _ *metav1.GetOptions) (run if err != nil { return nil, err } + if err := s.kind.decorate(ctx, c.obj); err != nil { + return nil, err + } return c.obj, nil } @@ -194,6 +198,7 @@ func (s *Store) List(ctx context.Context, opts *metainternalversion.ListOptions) w := &where{} w.scope(sc) + s.kind.readFilter(ctx, w) w.labelSelector(opts.LabelSelector) if err := w.fieldSelector(opts.FieldSelector, s.kind.FieldColumns); err != nil { return nil, apierrors.NewBadRequest(err.Error()) @@ -263,6 +268,9 @@ func (s *Store) List(ctx context.Context, opts *metainternalversion.ListOptions) return nil, s.dbErr(err, "") } + if err := s.kind.decorate(ctx, items...); err != nil { + return nil, err + } list := s.kind.NewListFunc() if err := meta.SetList(list, items); err != nil { return nil, err @@ -290,6 +298,9 @@ func (s *Store) Create(ctx context.Context, obj runtime.Object, createValidation if out, err = s.create(ctx, sc, obj, createValidation, false); err != nil { return err } + if err := s.kind.decorate(ctx, out); err != nil { + return err + } if opts != nil && len(opts.DryRun) > 0 { return errRollback } @@ -326,6 +337,9 @@ func (s *Store) create(ctx context.Context, sc Scope, obj runtime.Object, create return nil, err } } + if err := s.kind.beforeWrite(ctx, OpCreate, nil, obj); err != nil { + return nil, err + } data, err := encode(obj) if err != nil { @@ -352,6 +366,9 @@ func (s *Store) create(ctx context.Context, sc Scope, obj runtime.Object, create return nil, s.dbErr(err, m.GetName()) } m.SetResourceVersion(strconv.FormatInt(rv, 10)) + if err := s.kind.afterWrite(ctx, OpCreate, nil, obj); err != nil { + return nil, err + } return obj, nil } @@ -391,6 +408,9 @@ func (s *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj return err } } + if err := s.kind.decorate(ctx, out); err != nil { + return err + } if opts != nil && len(opts.DryRun) > 0 { return errRollback } @@ -432,6 +452,9 @@ func (s *Store) update(ctx context.Context, cur *current, objInfo rest.UpdatedOb return nil, err } } + if err := s.kind.beforeWrite(ctx, OpUpdate, old, obj); err != nil { + return nil, err + } data, err := encode(obj) if err != nil { @@ -466,6 +489,9 @@ func (s *Store) update(ctx context.Context, cur *current, objInfo rest.UpdatedOb return nil, s.dbErr(err, m.GetName()) } m.SetResourceVersion(strconv.FormatInt(rv, 10)) + if err := s.kind.afterWrite(ctx, OpUpdate, old, obj); err != nil { + return nil, err + } return obj, nil } @@ -502,11 +528,17 @@ func (s *Store) Delete(ctx context.Context, name string, deleteValidation rest.V return err } } + if err := s.kind.beforeWrite(ctx, OpDelete, cur.obj, nil); err != nil { + return err + } if _, err := s.db.Q(ctx).Exec(ctx, fmt.Sprintf("DELETE FROM %s WHERE scope_type = $1 AND scope_name = $2 AND name = $3", s.table()), cur.scope.Type, cur.scope.Name, name); err != nil { return s.dbErr(err, name) } + if err := s.kind.afterWrite(ctx, OpDelete, cur.obj, nil); err != nil { + return err + } out = cur.obj if opts != nil && len(opts.DryRun) > 0 { return errRollback diff --git a/internal/storage/pg/store_test.go b/internal/storage/pg/store_test.go index 3872510..f1f5154 100644 --- a/internal/storage/pg/store_test.go +++ b/internal/storage/pg/store_test.go @@ -6,11 +6,7 @@ import ( "fmt" "os" "testing" - "time" - "github.com/testcontainers/testcontainers-go" - tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" - "github.com/testcontainers/testcontainers-go/wait" apierrors "k8s.io/apimachinery/pkg/api/errors" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,68 +15,22 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage/names" - "go.miloapis.com/feedback/internal/postgres" "go.miloapis.com/feedback/internal/storage/pg" + "go.miloapis.com/feedback/internal/storage/pg/pgtest" "go.miloapis.com/feedback/pkg/apis/feedback/install" v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) var db *pg.DB -// TestMain starts one pgvector Postgres for the package and applies the real -// migrations. Without Docker the tests skip. -func TestMain(m *testing.M) { - os.Exit(func() int { - ctx := context.Background() - ctr, err := tcpostgres.Run(ctx, "pgvector/pgvector:pg17", - tcpostgres.WithDatabase("feedback"), - tcpostgres.WithUsername("feedback"), - tcpostgres.WithPassword("feedback"), - testcontainers.WithWaitStrategy(wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2).WithStartupTimeout(90*time.Second)), - ) - if err != nil { - fmt.Fprintf(os.Stderr, "skipping Postgres tests: %v\n", err) - return m.Run() // tests call requireDB and skip - } - defer func() { _ = testcontainers.TerminateContainer(ctr) }() - - dsn, err := ctr.ConnectionString(ctx, "sslmode=disable") - if err != nil { - fmt.Fprintln(os.Stderr, err) - return 1 - } - cfg := postgres.Config{DSN: dsn} - if err := postgres.Migrate(ctx, cfg); err != nil { - fmt.Fprintln(os.Stderr, err) - return 1 - } - pool, err := postgres.NewPool(ctx, cfg) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return 1 - } - defer pool.Close() - db = &pg.DB{Pool: pool} - return m.Run() - }()) -} +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } func requireDB(t *testing.T) { t.Helper() - if db == nil { - t.Skip("Postgres not available (needs Docker)") - } - // Each test starts from empty tables. - _, err := db.Pool.Exec(context.Background(), "TRUNCATE posts, votes, subscriptions") - if err != nil { - t.Fatal(err) - } + db = pgtest.Require(t, "posts", "votes", "subscriptions") } var scheme = func() *runtime.Scheme { @@ -192,21 +142,8 @@ func subscriptionStore(t *testing.T) *pg.Store { return s } -// requestContext is what the apiserver passes a cluster-scoped request. -func requestContext() context.Context { - return request.WithNamespace(context.Background(), "") -} - -func asUser(uid string) context.Context { - return request.WithUser(requestContext(), &user.DefaultInfo{ - Name: uid + "@example.com", UID: uid, - Extra: map[string][]string{pg.ParentKindExtraKey: {"User"}, pg.ParentNameExtraKey: {uid}}, - }) -} - -func asPlatform() context.Context { - return request.WithUser(requestContext(), &user.DefaultInfo{Name: "staff@datum.net", UID: "staff"}) -} +func asUser(uid string) context.Context { return pgtest.AsUser(uid) } +func asPlatform() context.Context { return pgtest.AsPlatform("staff@datum.net") } func vote(post string) *v1.Vote { return &v1.Vote{ObjectMeta: metav1.ObjectMeta{Name: post}, Spec: v1.VoteSpec{PostRef: v1.PostReference{Name: post}}} @@ -503,8 +440,7 @@ func TestUserScopes(t *testing.T) { t.Fatalf("platform create: got %v, want Forbidden", err) } - org := request.WithUser(requestContext(), &user.DefaultInfo{Name: "x", Extra: map[string][]string{ - pg.ParentKindExtraKey: {"Organization"}, pg.ParentNameExtraKey: {"acme"}}}) + org := pgtest.AsContext("Organization", "acme") if _, err := s.List(org, &metainternalversion.ListOptions{}); !apierrors.IsForbidden(err) { t.Fatalf("organization context: got %v, want Forbidden", err) } diff --git a/migrations/00003_deleted_posts.sql b/migrations/00003_deleted_posts.sql new file mode 100644 index 0000000..c6c92ae --- /dev/null +++ b/migrations/00003_deleted_posts.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- When a post is deleted (label removed, repo made private, issue deleted) +-- its votes and subscriptions are kept for a retention period in case it +-- comes back. A row here records when it went; recreating the post removes +-- the row, and the apiserver purges votes for rows past retention. +CREATE TABLE deleted_posts ( + name text PRIMARY KEY, + deleted_at timestamptz NOT NULL +); +CREATE INDEX deleted_posts_deleted_at_idx ON deleted_posts (deleted_at); + +-- +goose Down +SELECT 1; diff --git a/pkg/apis/feedback/v1alpha1/post_types.go b/pkg/apis/feedback/v1alpha1/post_types.go index 294b9d5..7d9fba1 100644 --- a/pkg/apis/feedback/v1alpha1/post_types.go +++ b/pkg/apis/feedback/v1alpha1/post_types.go @@ -1,6 +1,9 @@ package v1alpha1 import ( + "fmt" + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -249,3 +252,9 @@ type PostList struct { Items []Post `json:"items"` } + +// PostName is the name of the Post for a GitHub issue: +// "..", lower-cased. +func PostName(org, repository string, number int32) string { + return fmt.Sprintf("%s.%s.%d", strings.ToLower(org), strings.ToLower(repository), number) +}