From dd4a044c87882b3e6fc651221d726cbf3ac4d7ae Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Tue, 17 Nov 2026 13:25:19 -0500 Subject: [PATCH 1/5] feat: attachments through a feedback-uploads service CRDs can't take file uploads, so feedback-uploads accepts them with the caller's Milo token: a TokenReview says who they are and a SubjectAccessReview whether they may create (or get) the resource in the context they name. Files are type-checked and stored per user and purpose. The attachments webhook checks each uploadRef is the owner's upload and sets its name and type; the operator copies attached files under the submission, with a finalizer that deletes them with it. --- README.md | 29 +- cmd/feedback/main.go | 2 +- cmd/feedback/operator.go | 16 +- cmd/feedback/uploads.go | 96 +++++++ config/base/config.yaml | 8 + config/base/kustomization.yaml | 2 + config/base/operator/deployment.yaml | 6 + config/base/uploads/deployment.yaml | 69 +++++ config/base/uploads/kustomization.yaml | 7 + config/base/uploads/service.yaml | 13 + config/base/uploads/serviceaccount.yaml | 4 + .../milo-integration/kustomization.yaml | 44 ++- config/milo/rbac/feedback-operator.yaml | 4 +- config/milo/webhook/manifests.yaml | 20 ++ go.mod | 49 +++- go.sum | 163 ++++++++++- internal/attachments/attachments.go | 259 +++++++++++++++++ internal/attachments/attachments_test.go | 107 +++++++ internal/controller/controller_test.go | 53 +++- internal/controller/setup.go | 5 +- internal/controller/submission.go | 51 +++- internal/identity/identity.go | 7 +- internal/ratelimit/ratelimit.go | 78 ++++++ internal/ratelimit/ratelimit_test.go | 32 +++ internal/uploads/milo.go | 61 ++++ internal/uploads/server.go | 264 ++++++++++++++++++ internal/uploads/server_test.go | 138 +++++++++ internal/webhook/attachments.go | 74 +++++ internal/webhook/attachments_test.go | 72 +++++ internal/webhook/register.go | 5 + internal/webhook/submission.go | 5 - internal/webhook/submission_test.go | 3 - 32 files changed, 1706 insertions(+), 40 deletions(-) create mode 100644 cmd/feedback/uploads.go create mode 100644 config/base/config.yaml create mode 100644 config/base/uploads/deployment.yaml create mode 100644 config/base/uploads/kustomization.yaml create mode 100644 config/base/uploads/service.yaml create mode 100644 config/base/uploads/serviceaccount.yaml create mode 100644 internal/attachments/attachments.go create mode 100644 internal/attachments/attachments_test.go create mode 100644 internal/ratelimit/ratelimit.go create mode 100644 internal/ratelimit/ratelimit_test.go create mode 100644 internal/uploads/milo.go create mode 100644 internal/uploads/server.go create mode 100644 internal/uploads/server_test.go create mode 100644 internal/webhook/attachments.go create mode 100644 internal/webhook/attachments_test.go diff --git a/README.md b/README.md index d1730d7..7fa9457 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ webhooks. ## Status The CRDs are defined. The operator sets owners and reviewers in admission, -and reconciles votes, follows and submission reviews. Attachments, triage, -security reports and digests are tracked as sub-issues of the epic. +reconciles votes, follows and submission reviews, and stores attachments. +Triage, security reports and digests are tracked as sub-issues of the epic. | Resource | Who writes it | Who reads it | | --- | --- | --- | @@ -68,6 +68,24 @@ decided on a `SubmissionReview`. - `Approved` waits until feedback-sync creates the GitHub issue and syncs a post annotated `feedback.miloapis.com/submission: `. It then links the submission to that post and has the submitter follow (`reason: submitted`) and vote. - The `Applied` condition and `status.applied` record when the decision has been fully carried out. +## Attachments + +CRDs can't take file uploads, so `feedback uploads` (the `feedback-uploads` Deployment) handles them. Clients use the same bearer token they use for Milo. + +``` +POST /users/{user}/uploads/{submissions|securityreports}?name=shot.png (body: the file, with its Content-Type) +GET /users/{user}/{submissions|securityreports}/{name}/attachments/{ref} (the owner) +GET /{submissions|securityreports}/{name}/attachments/{ref} (staff) +``` + +- **Checking the caller.** A Milo `TokenReview` identifies the caller. A `SubjectAccessReview` checks that they may create (to upload) or get (to download) that resource in the context the path names. In a user's context, the service also checks that the object is theirs. +- **Files accepted.** Files are type-checked against their content: PNG, JPEG, GIF, WebP, PDF, text and JSON only. The size limit is 10 MiB by default, and each user gets a per-minute upload limit. +- **Storage.** Files land under `uploads///` in the bucket. +- **Attaching.** The `attachments` mutating webhook checks every `uploadRef` on a new submission or report is the owner's upload for that purpose, and sets its name and type from the upload. +- **Keeping and deleting.** The operator copies attached files under `submissions//` as soon as the submission exists, and adds the `feedback.miloapis.com/attachments` finalizer. Deleting the submission deletes its files. +- **Lifecycle rule.** The bucket's lifecycle rule should delete `uploads/` after a day. +- **Configuration.** Set the bucket in the `feedback-config` ConfigMap (`attachments-bucket`). The operator and the upload service share it. + ## Roles | Role | For | @@ -84,15 +102,18 @@ client certificates and get Kubernetes RBAC in Milo (`config/milo/rbac`). ## Layout ``` -cmd/feedback/ operator and version commands +cmd/feedback/ operator, uploads and version commands +internal/attachments/ the attachments bucket internal/controller/ reconcilers internal/identity/ who a request comes from, from Milo's user info internal/index/ cache field indexes shared by reconcilers and webhooks internal/milo/ voter lookups in Milo +internal/ratelimit/ per-user token bucket +internal/uploads/ the feedback-uploads HTTP service internal/webhook/ admission webhooks Milo calls pkg/apis/feedback/ feedback.miloapis.com/v1alpha1 types (CRDs) config/crd/ generated CRDs, installed in Milo -config/base/ operator Deployment, Services, leader election RBAC +config/base/ operator and uploads Deployments, Services, leader election RBAC config/components/ milo-integration: Milo kubeconfig, client and webhook certificates config/milo/ applied to Milo: IAM (iam/), RBAC for the service's components (rbac/), webhook config (webhook/), and diff --git a/cmd/feedback/main.go b/cmd/feedback/main.go index cb798d5..ae41861 100644 --- a/cmd/feedback/main.go +++ b/cmd/feedback/main.go @@ -25,7 +25,7 @@ func newRootCommand() *cobra.Command { roadmap posts synced from GitHub, votes, submissions and security reports.`, SilenceUsage: true, } - cmd.AddCommand(newOperatorCommand(), newVersionCommand()) + cmd.AddCommand(newOperatorCommand(), newUploadsCommand(), newVersionCommand()) return cmd } diff --git a/cmd/feedback/operator.go b/cmd/feedback/operator.go index 8911666..ca225f4 100644 --- a/cmd/feedback/operator.go +++ b/cmd/feedback/operator.go @@ -17,6 +17,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + "go.miloapis.com/feedback/internal/attachments" "go.miloapis.com/feedback/internal/controller" "go.miloapis.com/feedback/internal/index" "go.miloapis.com/feedback/internal/milo" @@ -42,6 +43,7 @@ type operatorOptions struct { webhookCertDir string submissionLimit int submissionWindow time.Duration + attachmentsBucket string } func newOperatorCommand() *cobra.Command { @@ -65,6 +67,7 @@ func newOperatorCommand() *cobra.Command { f.StringVar(&o.webhookCertDir, "webhook-cert-dir", "/etc/kubernetes/pki/webhook", "Directory holding tls.crt and tls.key for the webhook server.") f.IntVar(&o.submissionLimit, "submission-limit", 5, "Submissions a user may send per --submission-window; 0 disables the limit.") f.DurationVar(&o.submissionWindow, "submission-window", 10*time.Minute, "Window for --submission-limit.") + f.StringVar(&o.attachmentsBucket, "attachments-bucket", "", "Bucket URL for attachments, shared with feedback-uploads. Empty refuses attachments.") goFlags := flag.NewFlagSet("zap", flag.ExitOnError) zapOpts.BindFlags(goFlags) f.AddGoFlagSet(goFlags) @@ -105,13 +108,22 @@ func runOperator(o operatorOptions) error { if err := index.Add(ctx, mgr); err != nil { return fmt.Errorf("adding indexes: %w", err) } - webhook.Register(mgr, webhook.Options{SubmissionLimit: o.submissionLimit, SubmissionWindow: o.submissionWindow}) + var files *attachments.Store + if o.attachmentsBucket != "" { + if files, err = attachments.Open(ctx, o.attachmentsBucket); err != nil { + return err + } + defer files.Close() + } else { + log.Info("No --attachments-bucket; submissions and reports with attachments will be refused") + } + webhook.Register(mgr, webhook.Options{SubmissionLimit: o.submissionLimit, SubmissionWindow: o.submissionWindow, Files: files}) users, err := milo.NewClient(miloConfig) if err != nil { return fmt.Errorf("creating Milo user client: %w", err) } - if err := controller.Setup(ctx, mgr, controller.Options{Users: users}); err != nil { + if err := controller.Setup(ctx, mgr, controller.Options{Users: users, Files: files}); err != nil { return fmt.Errorf("setting up controllers: %w", err) } diff --git a/cmd/feedback/uploads.go b/cmd/feedback/uploads.go new file mode 100644 index 0000000..f31fb30 --- /dev/null +++ b/cmd/feedback/uploads.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/ratelimit" + "go.miloapis.com/feedback/internal/uploads" +) + +type uploadsOptions struct { + miloKubeconfig string + addr string + bucket string + maxSize int + perMinute float64 + burst int +} + +func newUploadsCommand() *cobra.Command { + o := uploadsOptions{} + zapOpts := zap.Options{} + cmd := &cobra.Command{ + Use: "uploads", + Short: "Serve attachment uploads and downloads", + RunE: func(cmd *cobra.Command, _ []string) error { + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) + return runUploads(ctrl.SetupSignalHandler(), o) + }, + } + f := cmd.Flags() + f.StringVar(&o.miloKubeconfig, "milo-kubeconfig", "", "Kubeconfig for Milo. Empty uses the in-cluster config.") + f.StringVar(&o.addr, "listen-address", ":8080", "Address to serve HTTP on. TLS is terminated in front of the service.") + f.StringVar(&o.bucket, "attachments-bucket", "", "Bucket URL for attachments, e.g. gs://feedback-attachments or file:///tmp/attachments.") + f.IntVar(&o.maxSize, "attachment-max-size", attachments.DefaultMaxSize, "Largest file accepted, in bytes.") + f.Float64Var(&o.perMinute, "uploads-per-minute", 2, "Uploads a user may make per minute, on average; 0 disables the limit.") + f.IntVar(&o.burst, "uploads-burst", 10, "Uploads a user may make at once.") + return cmd +} + +func runUploads(ctx context.Context, o uploadsOptions) error { + log := ctrl.Log.WithName("uploads") + if o.bucket == "" { + return errors.New("--attachments-bucket is required") + } + files, err := attachments.Open(ctx, o.bucket) + if err != nil { + return err + } + defer files.Close() + files.MaxSize = o.maxSize + + cfg, err := restConfig(o.miloKubeconfig) + if err != nil { + return fmt.Errorf("loading Milo config: %w", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + objects, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return err + } + milo := uploads.Milo{Client: kube} + s := &uploads.Server{ + Files: files, Authn: milo, Authz: milo, Objects: objects, + Limiter: ratelimit.NewLimiter(o.perMinute, o.burst), + } + srv := &http.Server{ + Addr: o.addr, + Handler: http.TimeoutHandler(s.Handler(), uploads.RequestTimeout, "request timed out"), + ReadHeaderTimeout: 10 * time.Second, + } + go func() { + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = srv.Shutdown(shutdown) + }() + log.Info("Serving uploads", "address", o.addr) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/config/base/config.yaml b/config/base/config.yaml new file mode 100644 index 0000000..9a51a4a --- /dev/null +++ b/config/base/config.yaml @@ -0,0 +1,8 @@ +# Settings the environment overrides. +apiVersion: v1 +kind: ConfigMap +metadata: + name: feedback-config +data: + # Bucket URL for attachments, shared by the operator and feedback-uploads. + attachments-bucket: mem:// diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 060d2bc..f4f4070 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -2,7 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - config.yaml - operator + - uploads images: - name: ghcr.io/milo-os/feedback diff --git a/config/base/operator/deployment.yaml b/config/base/operator/deployment.yaml index d5d33e7..97551cc 100644 --- a/config/base/operator/deployment.yaml +++ b/config/base/operator/deployment.yaml @@ -41,8 +41,14 @@ spec: - --leader-elect - --leader-election-namespace=$(POD_NAMESPACE) - --webhook-cert-dir=/etc/kubernetes/pki/webhook + - --attachments-bucket=$(ATTACHMENTS_BUCKET) - --zap-encoder=json env: + - name: ATTACHMENTS_BUCKET + valueFrom: + configMapKeyRef: + name: feedback-config + key: attachments-bucket - name: POD_NAMESPACE valueFrom: fieldRef: diff --git a/config/base/uploads/deployment.yaml b/config/base/uploads/deployment.yaml new file mode 100644 index 0000000..af56772 --- /dev/null +++ b/config/base/uploads/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: feedback-uploads +spec: + replicas: 2 + selector: + matchLabels: + app: feedback-uploads + template: + metadata: + labels: + app: feedback-uploads + spec: + serviceAccountName: feedback-uploads + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: uploads + image: ghcr.io/milo-os/feedback:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: + - ALL + command: + - /feedback + - uploads + args: + # The milo-integration component mounts this kubeconfig. + - --milo-kubeconfig=/etc/kubernetes/milo/kubeconfig + - --attachments-bucket=$(ATTACHMENTS_BUCKET) + - --zap-encoder=json + env: + # The environment sets the bucket, e.g. gs://feedback-attachments-staging. + # Its lifecycle rule should delete uploads/ objects after one day. + - name: ATTACHMENTS_BUCKET + valueFrom: + configMapKeyRef: + name: feedback-config + key: attachments-bucket + ports: + - containerPort: 8080 + name: http + protocol: TCP + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /healthz + port: http diff --git a/config/base/uploads/kustomization.yaml b/config/base/uploads/kustomization.yaml new file mode 100644 index 0000000..7b88772 --- /dev/null +++ b/config/base/uploads/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceaccount.yaml + - deployment.yaml + - service.yaml diff --git a/config/base/uploads/service.yaml b/config/base/uploads/service.yaml new file mode 100644 index 0000000..6286388 --- /dev/null +++ b/config/base/uploads/service.yaml @@ -0,0 +1,13 @@ +# The environment routes a public hostname to this Service; TLS is +# terminated in front of it. +apiVersion: v1 +kind: Service +metadata: + name: feedback-uploads +spec: + selector: + app: feedback-uploads + ports: + - name: http + port: 80 + targetPort: http diff --git a/config/base/uploads/serviceaccount.yaml b/config/base/uploads/serviceaccount.yaml new file mode 100644 index 0000000..19d6d15 --- /dev/null +++ b/config/base/uploads/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: feedback-uploads diff --git a/config/components/milo-integration/kustomization.yaml b/config/components/milo-integration/kustomization.yaml index c3746aa..3cf69ca 100644 --- a/config/components/milo-integration/kustomization.yaml +++ b/config/components/milo-integration/kustomization.yaml @@ -14,7 +14,9 @@ kind: Component # trusts the admission webhooks. # - Mounts the feedback-milo-kubeconfig ConfigMap, which consumers provide # in the feedback-system namespace. It points at Milo and at the mounted -# client certificate and trust bundle. +# client certificate (/etc/kubernetes/pki/client) and trust bundle +# (/etc/kubernetes/pki/trust). feedback-uploads mounts the same files, with +# its own certificate, system:feedback-uploads@feedback.miloapis.com. # # The namespace needs the label # `infra.datum.net/inject-datum-control-plane-trust-bundle: ""` so @@ -78,3 +80,43 @@ patches: csi.cert-manager.io/issuer-kind: ClusterIssuer csi.cert-manager.io/issuer-name: datum-control-plane csi.cert-manager.io/key-usages: client auth + - target: + kind: Deployment + name: feedback-uploads + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: feedback-uploads + spec: + template: + spec: + containers: + - name: uploads + volumeMounts: + - mountPath: /etc/kubernetes/pki/trust + name: trust-bundle + readOnly: true + - mountPath: /etc/kubernetes/pki/client + name: client-cert + readOnly: true + - mountPath: /etc/kubernetes/milo + name: milo-kubeconfig + readOnly: true + volumes: + - name: trust-bundle + configMap: + name: datum-control-plane-trust-bundle + - name: milo-kubeconfig + configMap: + name: feedback-milo-kubeconfig + - name: client-cert + csi: + driver: csi.cert-manager.io + readOnly: true + volumeAttributes: + csi.cert-manager.io/common-name: system:feedback-uploads@feedback.miloapis.com + csi.cert-manager.io/fs-group: "65532" + csi.cert-manager.io/issuer-kind: ClusterIssuer + csi.cert-manager.io/issuer-name: datum-control-plane + csi.cert-manager.io/key-usages: client auth diff --git a/config/milo/rbac/feedback-operator.yaml b/config/milo/rbac/feedback-operator.yaml index 2a473b9..85aa949 100644 --- a/config/milo/rbac/feedback-operator.yaml +++ b/config/milo/rbac/feedback-operator.yaml @@ -19,9 +19,11 @@ rules: - apiGroups: [feedback.miloapis.com] resources: [votes, subscriptions, submissionreviews, securityreports, securityacknowledgements, feedbackdigests] verbs: [create, update, patch, delete] + # delete for reports moved to the security track; update and patch only to + # add and remove the attachments finalizer. - apiGroups: [feedback.miloapis.com] resources: [submissions] - verbs: [delete] + verbs: [update, patch, delete] # Voters' organizations and linked GitHub usernames, for staff analytics # and so a 👍 from the same person isn't counted twice. - apiGroups: [resourcemanager.miloapis.com] diff --git a/config/milo/webhook/manifests.yaml b/config/milo/webhook/manifests.yaml index 3968506..c6ce0bc 100644 --- a/config/milo/webhook/manifests.yaml +++ b/config/milo/webhook/manifests.yaml @@ -4,6 +4,26 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: feedback-operator-webhook + namespace: feedback-system + path: /mutate-attachments + failurePolicy: Fail + name: attachments.feedback.miloapis.com + rules: + - apiGroups: + - feedback.miloapis.com + apiVersions: + - v1alpha1 + operations: + - CREATE + resources: + - submissions + - securityreports + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/go.mod b/go.mod index 7c3f1b3..db203ef 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ toolchain go1.27.4 require ( github.com/evanphx/json-patch/v5 v5.9.11 github.com/spf13/cobra v1.10.2 + gocloud.dev v0.48.0 + golang.org/x/time v0.16.0 k8s.io/api v0.37.2 k8s.io/apimachinery v0.37.2 k8s.io/client-go v0.37.2 @@ -15,13 +17,29 @@ require ( ) require ( + cel.dev/expr v0.25.3 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.24.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.10.0 // indirect + cloud.google.com/go/iam v1.14.0 // indirect + cloud.google.com/go/monitoring v1.31.0 // indirect + cloud.google.com/go/storage v1.69.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.6 // indirect github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/go-openapi/jsonreference v1.0.0 // indirect @@ -38,35 +56,56 @@ require ( github.com/go-openapi/swag/typeutils v0.27.1 // indirect github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/s2a-go v0.1.11 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/google/wire v0.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.24 // indirect + github.com/googleapis/gax-go/v2 v2.27.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.24.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.0 // indirect github.com/prometheus/procfs v0.23.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/testify v1.12.1 // indirect + github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect + go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.6 // indirect + golang.org/x/crypto v0.58.0 // indirect golang.org/x/mod v0.42.0 // indirect golang.org/x/net v0.62.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.25.0 // indirect golang.org/x/sys v0.50.0 // indirect golang.org/x/term v0.48.0 // indirect golang.org/x/text v0.44.0 // indirect - golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.52.1-0.20261106145633-d407941a339a // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/api v0.308.0 // indirect + google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b // indirect + google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 64147a6..ccbc4ea 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,77 @@ +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs= +cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c= +cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88= +cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk= +cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY= +cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk= +cloud.google.com/go/iam v1.14.0 h1:YRIB1/4xxwJ+NfI6RQr1WaOoHHeV0MizJEj940nBXJA= +cloud.google.com/go/iam v1.14.0/go.mod h1:JTPZfbHdGXJrOYREtkRoFzCDGaBdbA7ILLKO/DDuav0= +cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210HlXM= +cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w= +cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI= +cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0= +cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY= +cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= +github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= +github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,16 +79,31 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= +github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= +github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= @@ -59,17 +142,33 @@ github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAg github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-replayers/grpcreplay v1.5.1 h1:R90FMYaA6GQNhi/GG0D4zS14HEQ+wpO8bQAV0Qwyt08= +github.com/google/go-replayers/grpcreplay v1.5.1/go.mod h1:42YCe31fkUrhRo+haVAfLse+gNqGI1iO1wdNR+sWY9U= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.11 h1:7v1V1TraNGP63WptsAkNqr2w8jgm4vM6A6qx6Lsw9iM= +github.com/google/s2a-go v0.1.11/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= +github.com/googleapis/enterprise-certificate-proxy v0.3.24 h1:G9iCJIY+AYVmUskKdjX8FVy7sG+nbzjOQ41cvlmhcvg= +github.com/googleapis/enterprise-certificate-proxy v0.3.24/go.mod h1:hEjCmnoakQzv5ImLUp6alkY7XgLlNoAxwTXmhHHNIsI= +github.com/googleapis/gax-go/v2 v2.27.0 h1:qe2iQOUzxb3fd6x6ewpIEixPQhJ+0Ihc/RRZhKA/2g0= +github.com/googleapis/gax-go/v2 v2.27.0/go.mod h1:RofiqDw3ehcG4L0H1zO3D2Itpwd3/uS/7wdmbNDF1DE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -92,6 +191,8 @@ github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -109,6 +210,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4= +github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= @@ -117,23 +220,51 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg= +go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +gocloud.dev v0.48.0 h1:rEPS6euyhuDMlEKH7E1sztIHHwnABeiAiQmyiaHiPW0= +gocloud.dev v0.48.0/go.mod h1:uN+hz2x8ieyW5RjeMetvmpV/nRqFOr7DF0CKVY0Owp8= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= @@ -142,16 +273,30 @@ golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk= golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4= golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/tools v0.52.1-0.20261106145633-d407941a339a h1:Yqozs0FNPRoPGS4hNGlrNwRENWr5H5nVG9sN/f+5640= golang.org/x/tools v0.52.1-0.20261106145633-d407941a339a/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.308.0 h1:iGb+4qT3nVb3uuIzwor3yP2nyu+rMZr31PvLFiRNJQk= +google.golang.org/api v0.308.0/go.mod h1:4XMH+b/ZEfuTbK7FnrFRY9c4xfFHVflGTKuy/i5wUmE= +google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg= +google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI= +google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8= +google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -159,8 +304,6 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= k8s.io/api v0.37.2/go.mod h1:+NVRLft99kx+Jg9v34GmHw867Dapim89TSyCsn2hK+M= k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= diff --git a/internal/attachments/attachments.go b/internal/attachments/attachments.go new file mode 100644 index 0000000..168abaa --- /dev/null +++ b/internal/attachments/attachments.go @@ -0,0 +1,259 @@ +// Package attachments stores the files attached to submissions and security +// reports in a private bucket. feedback-uploads writes uploads and serves +// files to people Milo says may read them; the operator copies attached +// uploads under their parent and deletes them with it. +// +// Layout: +// +// uploads/{purpose}/{owner}/{ref} an upload, attached or not +// submissions/{name}/{ref} attached to a submission +// security-reports/{name}/{ref} attached to a security report +// +// The bucket's lifecycle rule deletes uploads/ objects after a day. A file +// is safe once the operator has copied it under its parent, which it does as +// soon as the parent is created. +package attachments + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strings" + "unicode/utf8" + + "gocloud.dev/blob" + _ "gocloud.dev/blob/fileblob" // file:// for local development + _ "gocloud.dev/blob/gcsblob" // gs:// in clusters + _ "gocloud.dev/blob/memblob" // mem:// for tests + "gocloud.dev/gcerrors" + + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +// DefaultMaxSize is the largest file accepted by default. +const DefaultMaxSize = 10 << 20 + +// AllowedTypes are the media types that can be uploaded. Nothing that a +// browser would run (HTML, SVG) is allowed. +var AllowedTypes = []string{ + "image/png", "image/jpeg", "image/gif", "image/webp", + "application/pdf", "text/plain", "application/json", +} + +// Purpose is what an upload is for. Uploads for one can't be attached to the +// other. +type Purpose string + +const ( + Submission Purpose = "submission" + SecurityReport Purpose = "security-report" +) + +// PurposeOf returns the purpose for a feedback resource, e.g. "submissions". +func PurposeOf(resource string) (Purpose, bool) { + switch resource { + case "submissions": + return Submission, true + case "securityreports": + return SecurityReport, true + } + return "", false +} + +// ErrNotFound is returned for an upload or attachment that doesn't exist, +// or belongs to someone else. +var ErrNotFound = errors.New("attachment not found") + +// Store is the attachments bucket. +type Store struct { + bucket *blob.Bucket + MaxSize int +} + +// Open opens the bucket at url, e.g. gs://feedback-attachments, file:///tmp/a +// or mem://. +func Open(ctx context.Context, bucketURL string) (*Store, error) { + b, err := blob.OpenBucket(ctx, bucketURL) + if err != nil { + return nil, fmt.Errorf("open attachments bucket: %w", err) + } + return &Store{bucket: b, MaxSize: DefaultMaxSize}, nil +} + +// Close closes the bucket. +func (s *Store) Close() error { return s.bucket.Close() } + +// ParentPrefix is where a parent's attachments live. +func ParentPrefix(p Purpose, parent string) string { + if p == SecurityReport { + return "security-reports/" + url.PathEscape(parent) + "/" + } + return "submissions/" + url.PathEscape(parent) + "/" +} + +func uploadKey(p Purpose, owner, ref string) string { + return "uploads/" + string(p) + "/" + url.PathEscape(owner) + "/" + url.PathEscape(ref) +} + +// Check validates a file before it is stored. It returns a message for the +// caller, or "". +func (s *Store) Check(name, contentType string, data []byte) string { + switch { + case name == "" || len(name) > 255 || strings.ContainsAny(name, "/\\\x00"): + return "the file name must be at most 255 characters, without slashes" + case !slices.Contains(AllowedTypes, contentType): + return "Content-Type must be one of " + strings.Join(AllowedTypes, ", ") + case len(data) == 0: + return "the file is empty" + case len(data) > s.MaxSize: + return fmt.Sprintf("the file is larger than %d bytes", s.MaxSize) + } + // The declared type must match the content, so a file served back to + // staff is what it claims to be. + switch { + case strings.HasPrefix(contentType, "text/") || contentType == "application/json": + if !utf8.Valid(data) { + return contentType + " must be UTF-8 text" + } + default: + if got := http.DetectContentType(data); got != contentType { + return fmt.Sprintf("the content looks like %s, not %s", got, contentType) + } + } + return "" +} + +// Upload stores data as an upload by owner and returns its ref and SHA-256. +// Call Check first. +func (s *Store) Upload(ctx context.Context, p Purpose, owner, name, contentType string, data []byte) (ref, sum string, err error) { + b := make([]byte, 12) + if _, err := rand.Read(b); err != nil { + return "", "", err + } + ref = "att-" + hex.EncodeToString(b) + h := sha256.Sum256(data) + sum = hex.EncodeToString(h[:]) + err = s.bucket.WriteAll(ctx, uploadKey(p, owner, ref), data, &blob.WriterOptions{ + ContentType: contentType, + Metadata: map[string]string{"name": name, "sha256": sum}, + }) + return ref, sum, err +} + +// Lookup returns references, with their stored names and types, for owner's +// uploads. A ref that isn't owner's upload for this purpose is ErrNotFound. +func (s *Store) Lookup(ctx context.Context, p Purpose, owner string, refs []string) ([]v1.AttachmentReference, error) { + out := make([]v1.AttachmentReference, 0, len(refs)) + for _, ref := range refs { + attrs, err := s.bucket.Attributes(ctx, uploadKey(p, owner, ref)) + if gcerrors.Code(err) == gcerrors.NotFound { + return nil, fmt.Errorf("%w: %s", ErrNotFound, ref) + } + if err != nil { + return nil, err + } + out = append(out, v1.AttachmentReference{Name: attrs.Metadata["name"], ContentType: attrs.ContentType, UploadRef: ref}) + } + return out, nil +} + +// Attach copies owner's uploads under parent. Copying again is harmless. +func (s *Store) Attach(ctx context.Context, p Purpose, owner, parent string, refs []string) error { + for _, ref := range refs { + dst := ParentPrefix(p, parent) + url.PathEscape(ref) + if ok, err := s.bucket.Exists(ctx, dst); err != nil { + return err + } else if ok { + continue + } + err := s.bucket.Copy(ctx, dst, uploadKey(p, owner, ref), nil) + if gcerrors.Code(err) == gcerrors.NotFound { + return fmt.Errorf("%w: %s", ErrNotFound, ref) + } + if err != nil { + return err + } + } + return nil +} + +// CopyParent copies a parent's attachments to another parent, e.g. when a +// submission becomes a security report. +func (s *Store) CopyParent(ctx context.Context, from, to Purpose, fromParent, toParent string) error { + src := ParentPrefix(from, fromParent) + it := s.bucket.List(&blob.ListOptions{Prefix: src}) + for { + obj, err := it.Next(ctx) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if err := s.bucket.Copy(ctx, ParentPrefix(to, toParent)+strings.TrimPrefix(obj.Key, src), obj.Key, nil); err != nil { + return err + } + } +} + +// File is an attached file. +type File struct { + Name string + ContentType string + Data []byte +} + +// Get reads an attachment of parent. Until the operator has copied it, it is +// read from owner's uploads. +func (s *Store) Get(ctx context.Context, p Purpose, parent, owner, ref string) (*File, error) { + f, err := s.read(ctx, ParentPrefix(p, parent)+url.PathEscape(ref)) + if errors.Is(err, ErrNotFound) && owner != "" { + return s.read(ctx, uploadKey(p, owner, ref)) + } + return f, err +} + +func (s *Store) read(ctx context.Context, key string) (*File, error) { + attrs, err := s.bucket.Attributes(ctx, key) + if gcerrors.Code(err) == gcerrors.NotFound { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + r, err := s.bucket.NewReader(ctx, key, nil) + if err != nil { + return nil, err + } + defer r.Close() + var buf bytes.Buffer + if _, err := io.Copy(&buf, io.LimitReader(r, int64(s.MaxSize)+1)); err != nil { + return nil, err + } + return &File{Name: attrs.Metadata["name"], ContentType: attrs.ContentType, Data: buf.Bytes()}, nil +} + +// DeleteParent deletes every attachment of a parent. +func (s *Store) DeleteParent(ctx context.Context, p Purpose, parent string) error { + it := s.bucket.List(&blob.ListOptions{Prefix: ParentPrefix(p, parent)}) + for { + obj, err := it.Next(ctx) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if err := s.bucket.Delete(ctx, obj.Key); err != nil && gcerrors.Code(err) != gcerrors.NotFound { + return err + } + } +} diff --git a/internal/attachments/attachments_test.go b/internal/attachments/attachments_test.go new file mode 100644 index 0000000..28508d1 --- /dev/null +++ b/internal/attachments/attachments_test.go @@ -0,0 +1,107 @@ +package attachments + +import ( + "context" + "errors" + "strings" + "testing" +) + +var png = []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 32)) + +func open(t *testing.T) *Store { + t.Helper() + s, err := Open(context.Background(), "mem://") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestCheck(t *testing.T) { + s := open(t) + s.MaxSize = 64 + for _, tc := range []struct { + name, ct string + data []byte + ok bool + }{ + {"shot.png", "image/png", png, true}, + {"notes.txt", "text/plain", []byte("hello"), true}, + {"page.html", "text/html", []byte(""), false}, + {"shot.png", "image/png", []byte("not a png"), false}, + {"a/b.png", "image/png", png, false}, + {"big.txt", "text/plain", []byte(strings.Repeat("x", 65)), false}, + {"bad.txt", "text/plain", []byte{0xff, 0xfe}, false}, + {"empty.txt", "text/plain", nil, false}, + } { + if msg := s.Check(tc.name, tc.ct, tc.data); (msg == "") != tc.ok { + t.Errorf("Check(%s, %s) = %q", tc.name, tc.ct, msg) + } + } +} + +func TestUploadAttachGetDelete(t *testing.T) { + ctx := context.Background() + s := open(t) + ref, sum, err := s.Upload(ctx, Submission, "u1", "shot.png", "image/png", png) + if err != nil || !strings.HasPrefix(ref, "att-") || len(sum) != 64 { + t.Fatalf("Upload = %q %q %v", ref, sum, err) + } + + got, err := s.Lookup(ctx, Submission, "u1", []string{ref}) + if err != nil || len(got) != 1 || got[0].Name != "shot.png" || got[0].ContentType != "image/png" { + t.Fatalf("Lookup = %+v, %v", got, err) + } + for _, tc := range []struct { + p Purpose + owner string + }{{Submission, "u2"}, {SecurityReport, "u1"}} { + if _, err := s.Lookup(ctx, tc.p, tc.owner, []string{ref}); !errors.Is(err, ErrNotFound) { + t.Errorf("Lookup as %s for %s: %v, want ErrNotFound", tc.owner, tc.p, err) + } + } + + // Before attaching, Get falls back to the owner's upload. + if f, err := s.Get(ctx, Submission, "sub-1", "u1", ref); err != nil || f.Name != "shot.png" { + t.Fatalf("Get before attach = %+v, %v", f, err) + } + for range 2 { // attaching again is harmless + if err := s.Attach(ctx, Submission, "u1", "sub-1", []string{ref}); err != nil { + t.Fatal(err) + } + } + if err := s.Attach(ctx, Submission, "u2", "sub-2", []string{ref}); !errors.Is(err, ErrNotFound) { + t.Errorf("attaching someone else's upload: %v", err) + } + if f, err := s.Get(ctx, Submission, "sub-1", "", ref); err != nil || string(f.Data) != string(png) { + t.Fatalf("Get after attach = %v", err) + } + + if err := s.CopyParent(ctx, Submission, SecurityReport, "sub-1", "sec-1"); err != nil { + t.Fatal(err) + } + if _, err := s.Get(ctx, SecurityReport, "sec-1", "", ref); err != nil { + t.Errorf("copied report attachment: %v", err) + } + + if err := s.DeleteParent(ctx, Submission, "sub-1"); err != nil { + t.Fatal(err) + } + if _, err := s.Get(ctx, Submission, "sub-1", "", ref); !errors.Is(err, ErrNotFound) { + t.Errorf("after delete: %v", err) + } +} + +func TestPurposeOf(t *testing.T) { + if p, ok := PurposeOf("submissions"); !ok || p != Submission { + t.Error("submissions") + } + if p, ok := PurposeOf("securityreports"); !ok || p != SecurityReport { + t.Error("securityreports") + } + if _, ok := PurposeOf("votes"); ok { + t.Error("votes") + } +} diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go index 54ac3ad..ea9d581 100644 --- a/internal/controller/controller_test.go +++ b/internal/controller/controller_test.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "go.miloapis.com/feedback/internal/attachments" "go.miloapis.com/feedback/internal/index" v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) @@ -46,7 +47,7 @@ func (c *clock) Set(t time.Time) { // start runs the reconcilers against an envtest API server with the CRDs, // and returns a client that reads straight from the API server. -func start(t *testing.T, now *clock) client.Client { +func start(t *testing.T, now *clock, files ...*attachments.Store) client.Client { t.Helper() if os.Getenv("KUBEBUILDER_ASSETS") == "" { t.Skip("KUBEBUILDER_ASSETS not set; run with task test") @@ -78,7 +79,11 @@ func start(t *testing.T, now *clock) client.Client { if err := index.Add(ctx, mgr); err != nil { t.Fatal(err) } - if err := Setup(ctx, mgr, Options{Users: fakeUsers{}, Now: now.Now}); err != nil { + o := Options{Users: fakeUsers{}, Now: now.Now} + if len(files) > 0 { + o.Files = files[0] + } + if err := Setup(ctx, mgr, o); err != nil { t.Fatal(err) } go func() { _ = mgr.Start(ctx) }() @@ -397,3 +402,47 @@ func TestSubmissionReviews(t *testing.T) { } }) } + +func TestSubmissionAttachments(t *testing.T) { + ctx := context.Background() + files, err := attachments.Open(ctx, "mem://") + if err != nil { + t.Fatal(err) + } + c := start(t, &clock{now: time.Now()}, files) + ref, _, err := files.Upload(ctx, attachments.Submission, "u1", "notes.txt", "text/plain", []byte("hi")) + if err != nil { + t.Fatal(err) + } + sub := &v1.Submission{ + ObjectMeta: metav1.ObjectMeta{Name: "sub-files"}, + Spec: v1.SubmissionSpec{UserRef: v1.UserReference{Name: "u1"}, Kind: v1.SubmissionKindBug, Title: "t", Body: "b", + Attachments: []v1.AttachmentReference{{UploadRef: ref}}}, + } + if err := c.Create(ctx, sub); err != nil { + t.Fatal(err) + } + eventually(t, "copied", func() error { + if err := c.Get(ctx, client.ObjectKeyFromObject(sub), sub); err != nil { + return err + } + if len(sub.Finalizers) != 1 { + return errorf("no finalizer") + } + // Read without the owner, so only the copy counts. + _, err := files.Get(ctx, attachments.Submission, sub.Name, "", ref) + return err + }) + if err := c.Delete(ctx, sub); err != nil { + t.Fatal(err) + } + eventually(t, "deleted", func() error { + if err := c.Get(ctx, client.ObjectKeyFromObject(sub), &v1.Submission{}); !apierrors.IsNotFound(err) { + return errorf("submission still there") + } + if _, err := files.Get(ctx, attachments.Submission, sub.Name, "", ref); err == nil { + return errorf("file still there") + } + return nil + }) +} diff --git a/internal/controller/setup.go b/internal/controller/setup.go index d3d15e7..8e21351 100644 --- a/internal/controller/setup.go +++ b/internal/controller/setup.go @@ -12,6 +12,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" + "go.miloapis.com/feedback/internal/attachments" "go.miloapis.com/feedback/internal/milo" ) @@ -25,6 +26,8 @@ type Options struct { Users milo.Users // Now returns the current time; tests replace it. Now func() time.Time + // Files is the attachments bucket. Nil leaves attachments alone. + Files *attachments.Store } func (o Options) now() time.Time { @@ -42,7 +45,7 @@ func Setup(ctx context.Context, mgr ctrl.Manager, o Options) error { &PostVotesReconciler{Client: c, Options: o}, &VoteReconciler{Client: c, Options: o}, &MergeReconciler{Client: c}, - &SubmissionReconciler{Client: c}, + &SubmissionReconciler{Client: c, Files: o.Files}, &ReviewReconciler{Client: c}, } { if err := r.SetupWithManager(mgr); err != nil { diff --git a/internal/controller/submission.go b/internal/controller/submission.go index a451487..b2814ee 100644 --- a/internal/controller/submission.go +++ b/internal/controller/submission.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "go.miloapis.com/feedback/internal/attachments" "go.miloapis.com/feedback/internal/index" v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) @@ -23,18 +24,29 @@ import ( // decision has been carried out. const ConditionApplied = "Applied" +// AttachmentsFinalizer is on a submission or report whose attachments the +// operator copied under it. Deleting the object deletes the files, then the +// finalizer. +const AttachmentsFinalizer = "feedback.miloapis.com/attachments" + // SubmissionReconciler gives every new submission its SubmissionReview, owned -// by the submission so it goes with it, and starts it in PendingReview. +// by the submission so it goes with it, and starts it in PendingReview. It +// copies attached uploads under the submission before the bucket's lifecycle +// rule deletes them, and deletes them with the submission. type SubmissionReconciler struct { client.Client + Files *attachments.Store } // Reconcile implements reconcile.Reconciler. func (r *SubmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var sub v1.Submission - if err := r.Get(ctx, req.NamespacedName, &sub); err != nil || sub.DeletionTimestamp != nil { + if err := r.Get(ctx, req.NamespacedName, &sub); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + if done, err := keepAttachments(ctx, r.Client, r.Files, attachments.Submission, &sub, sub.Spec.UserRef.Name, sub.Spec.Attachments); done || err != nil { + return ctrl.Result{}, err + } review := &v1.SubmissionReview{ObjectMeta: metav1.ObjectMeta{Name: sub.Name}} if err := controllerutil.SetOwnerReference(&sub, review, r.Scheme()); err != nil { return ctrl.Result{}, err @@ -49,6 +61,41 @@ func (r *SubmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } +// keepAttachments copies obj's attachments under it and adds +// AttachmentsFinalizer, or, when obj is being deleted, deletes them and +// removes the finalizer. done reports that obj is being deleted, so there is +// nothing else to do. +func keepAttachments(ctx context.Context, c client.Client, files *attachments.Store, p attachments.Purpose, obj client.Object, owner string, atts []v1.AttachmentReference) (done bool, err error) { + has := controllerutil.ContainsFinalizer(obj, AttachmentsFinalizer) + if obj.GetDeletionTimestamp() != nil { + if !has { + return true, nil + } + if files == nil { + return true, fmt.Errorf("can't delete the attachments of %s: no attachments bucket", obj.GetName()) + } + if err := files.DeleteParent(ctx, p, obj.GetName()); err != nil { + return true, err + } + patch := client.MergeFrom(obj.DeepCopyObject().(client.Object)) + controllerutil.RemoveFinalizer(obj, AttachmentsFinalizer) + return true, c.Patch(ctx, obj, patch) + } + if has || len(atts) == 0 || files == nil { + return false, nil + } + refs := make([]string, len(atts)) + for i, a := range atts { + refs[i] = a.UploadRef + } + if err := files.Attach(ctx, p, owner, obj.GetName(), refs); err != nil { + return false, fmt.Errorf("copying attachments: %w", err) + } + patch := client.MergeFrom(obj.DeepCopyObject().(client.Object)) + controllerutil.AddFinalizer(obj, AttachmentsFinalizer) + return false, c.Patch(ctx, obj, patch) +} + // SetupWithManager registers the reconciler. func (r *SubmissionReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr).Named("submission").For(&v1.Submission{}).Complete(r) diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 33f75b1..ec7bd6a 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -18,9 +18,10 @@ const ( // Keys Milo sets in user extras. const ( - ParentKindExtraKey = "iam.miloapis.com/parent-type" - ParentNameExtraKey = "iam.miloapis.com/parent-name" - EmailVerifiedExtraKey = "iam.miloapis.com/emailVerified" + ParentAPIGroupExtraKey = "iam.miloapis.com/parent-api-group" + ParentKindExtraKey = "iam.miloapis.com/parent-type" + ParentNameExtraKey = "iam.miloapis.com/parent-name" + EmailVerifiedExtraKey = "iam.miloapis.com/emailVerified" ) // ParentKindUser is the parent kind of requests in a user's own context. diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..817d20c --- /dev/null +++ b/internal/ratelimit/ratelimit.go @@ -0,0 +1,78 @@ +// Package ratelimit is a per-user token bucket. +package ratelimit + +import ( + "sync" + "time" + + "golang.org/x/time/rate" +) + +// Limiter is a token bucket per user. It is per replica, so the effective +// limit scales with the number of replicas. +type Limiter struct { + every time.Duration + burst int + now func() time.Time + + mu sync.Mutex + users map[string]*bucket + swept time.Time +} + +type bucket struct { + l *rate.Limiter + seen time.Time +} + +// idle is how long an unused bucket is kept. A bucket idle this long has +// refilled, so dropping it changes nothing. +const idle = 10 * time.Minute + +// NewLimiter allows perMinute requests a minute per user, with bursts of +// burst. perMinute <= 0 disables limiting (a nil Limiter allows everything). +func NewLimiter(perMinute float64, burst int) *Limiter { + if perMinute <= 0 { + return nil + } + if burst < 1 { + burst = 1 + } + return &Limiter{ + every: time.Duration(float64(time.Minute) / perMinute), + burst: burst, + now: time.Now, + users: map[string]*bucket{}, + } +} + +// Allow takes a token for user. When none is left it returns false and how +// long until one is. +func (l *Limiter) Allow(user string) (bool, time.Duration) { + if l == nil { + return true, 0 + } + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + if now.Sub(l.swept) > idle { + for u, b := range l.users { + if now.Sub(b.seen) > idle { + delete(l.users, u) + } + } + l.swept = now + } + b, ok := l.users[user] + if !ok { + b = &bucket{l: rate.NewLimiter(rate.Every(l.every), l.burst)} + l.users[user] = b + } + b.seen = now + r := b.l.ReserveN(now, 1) + if d := r.DelayFrom(now); d > 0 { + r.CancelAt(now) + return false, d + } + return true, 0 +} diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go new file mode 100644 index 0000000..ba328fb --- /dev/null +++ b/internal/ratelimit/ratelimit_test.go @@ -0,0 +1,32 @@ +package ratelimit + +import ( + "testing" + "time" +) + +func TestLimiter(t *testing.T) { + if ok, _ := (*Limiter)(nil).Allow("x"); !ok { + t.Fatal("nil limiter must allow") + } + now := time.Unix(0, 0) + l := NewLimiter(60, 2) // one a second, bursts of two + l.now = func() time.Time { return now } + for i, want := range []bool{true, true, false} { + if ok, _ := l.Allow("a"); ok != want { + t.Fatalf("call %d: %v", i, ok) + } + } + if ok, wait := l.Allow("a"); ok || wait <= 0 || wait > time.Second { + t.Fatalf("wait %v", wait) + } + now = now.Add(time.Second) + if ok, _ := l.Allow("a"); !ok { + t.Fatal("token not refilled") + } + now = now.Add(idle + time.Second) + l.Allow("b") + if _, ok := l.users["a"]; ok { + t.Error("idle bucket not swept") + } +} diff --git a/internal/uploads/milo.go b/internal/uploads/milo.go new file mode 100644 index 0000000..1fe1e3d --- /dev/null +++ b/internal/uploads/milo.go @@ -0,0 +1,61 @@ +package uploads + +import ( + "context" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "go.miloapis.com/feedback/internal/identity" +) + +// Milo authenticates and authorizes callers with Milo's TokenReview and +// SubjectAccessReview APIs. +type Milo struct { + Client kubernetes.Interface +} + +var ( + _ Authenticator = Milo{} + _ Authorizer = Milo{} +) + +// Authenticate implements Authenticator. +func (m Milo) Authenticate(ctx context.Context, token string) (authenticationv1.UserInfo, bool, error) { + tr, err := m.Client.AuthenticationV1().TokenReviews().Create(ctx, &authenticationv1.TokenReview{ + Spec: authenticationv1.TokenReviewSpec{Token: token}, + }, metav1.CreateOptions{}) + if err != nil { + return authenticationv1.UserInfo{}, false, err + } + return tr.Status.User, tr.Status.Authenticated, nil +} + +// Allowed implements Authorizer. In a user's context it adds the parent +// extras Milo adds to requests made there. +func (m Milo) Allowed(ctx context.Context, u authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes, inUserContext string) (bool, error) { + extra := map[string]authorizationv1.ExtraValue{} + for k, v := range u.Extra { + extra[k] = authorizationv1.ExtraValue(v) + } + if inUserContext != "" { + extra[identity.ParentAPIGroupExtraKey] = authorizationv1.ExtraValue{"iam.miloapis.com"} + extra[identity.ParentKindExtraKey] = authorizationv1.ExtraValue{identity.ParentKindUser} + extra[identity.ParentNameExtraKey] = authorizationv1.ExtraValue{inUserContext} + } + sar, err := m.Client.AuthorizationV1().SubjectAccessReviews().Create(ctx, &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &attrs, + User: u.Username, + UID: u.UID, + Groups: u.Groups, + Extra: extra, + }, + }, metav1.CreateOptions{}) + if err != nil { + return false, err + } + return sar.Status.Allowed, nil +} diff --git a/internal/uploads/server.go b/internal/uploads/server.go new file mode 100644 index 0000000..e4978b2 --- /dev/null +++ b/internal/uploads/server.go @@ -0,0 +1,264 @@ +// Package uploads is the feedback-uploads HTTP service. CRDs can't take file +// uploads, so clients send files here, with the same bearer token they use +// for Milo, and reference them from submissions and security reports. +// +// Milo decides who may do what: a TokenReview says who the caller is, and a +// SubjectAccessReview says whether they may create (to upload) or get (to +// download) the resource the file belongs to, in the context they name. For +// downloads in a user's own context the service also checks the object is +// theirs, which Milo's owner filter does for API requests but an access +// review can't. +// +// Routes: +// +// POST /users/{user}/uploads/{resource}?name={file name} +// GET /users/{user}/{resource}/{name}/attachments/{ref} +// GET /{resource}/{name}/attachments/{ref} +// +// {resource} is submissions or securityreports. Uploads send the file as the +// body with its Content-Type. The reply is the uploadRef to put in +// spec.attachments. +package uploads + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "slices" + "strconv" + "strings" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/identity" + "go.miloapis.com/feedback/internal/ratelimit" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +// Authenticator says who a bearer token belongs to. +type Authenticator interface { + Authenticate(ctx context.Context, token string) (authenticationv1.UserInfo, bool, error) +} + +// Authorizer asks Milo whether user may do attrs. A non-empty inUserContext +// asks as if the request were made in that user's own context. +type Authorizer interface { + Allowed(ctx context.Context, user authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes, inUserContext string) (bool, error) +} + +// Server serves uploads and downloads. +type Server struct { + Files *attachments.Store + Authn Authenticator + Authz Authorizer + // Objects reads submissions and security reports from Milo. + Objects client.Reader + // Limiter limits uploads per user. Nil doesn't limit. + Limiter *ratelimit.Limiter +} + +// Handler returns the service's HTTP handler. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST /users/{user}/uploads/{resource}", s.upload) + mux.HandleFunc("GET /users/{user}/{resource}/{name}/attachments/{ref}", s.download) + mux.HandleFunc("GET /{resource}/{name}/attachments/{ref}", s.download) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) + return mux +} + +// Uploaded is the reply to an upload. +type Uploaded struct { + UploadRef string `json:"uploadRef"` + Name string `json:"name"` + ContentType string `json:"contentType"` + SHA256 string `json:"sha256"` +} + +func (s *Server) upload(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + resource, user := r.PathValue("resource"), r.PathValue("user") + purpose, ok := attachments.PurposeOf(resource) + if !ok { + fail(w, http.StatusNotFound, "uploads are for submissions or securityreports") + return + } + caller, ok := s.authenticate(w, r) + if !ok { + return + } + if !s.authorize(w, r, caller, authorizationv1.ResourceAttributes{Group: v1.GroupName, Resource: resource, Verb: "create"}, user) { + return + } + if identity.EmailUnverified(caller) { + fail(w, http.StatusForbidden, "verify your email address to send files") + return + } + if ok, wait := s.Limiter.Allow(user); !ok { + w.Header().Set("Retry-After", strconv.Itoa(int(wait.Seconds())+1)) + fail(w, http.StatusTooManyRequests, "too many uploads; try again shortly") + return + } + contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + fail(w, http.StatusBadRequest, "set Content-Type to the file's media type") + return + } + data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(s.Files.MaxSize)+1)) + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + fail(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("the file is larger than %d bytes", s.Files.MaxSize)) + return + } + if err != nil { + fail(w, http.StatusBadRequest, "reading the file failed") + return + } + name := r.URL.Query().Get("name") + if msg := s.Files.Check(name, contentType, data); msg != "" { + fail(w, http.StatusBadRequest, msg) + return + } + ref, sum, err := s.Files.Upload(ctx, purpose, user, name, contentType, data) + if err != nil { + log.FromContext(ctx).Error(err, "Storing an upload failed") + fail(w, http.StatusServiceUnavailable, "storing the file failed; try again") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(Uploaded{UploadRef: ref, Name: name, ContentType: contentType, SHA256: sum}) +} + +func (s *Server) download(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + resource, name, ref, user := r.PathValue("resource"), r.PathValue("name"), r.PathValue("ref"), r.PathValue("user") + purpose, ok := attachments.PurposeOf(resource) + if !ok { + fail(w, http.StatusNotFound, "not found") + return + } + caller, ok := s.authenticate(w, r) + if !ok { + return + } + if !s.authorize(w, r, caller, authorizationv1.ResourceAttributes{Group: v1.GroupName, Resource: resource, Verb: "get", Name: name}, user) { + return + } + owner, refs, err := s.attachmentsOf(ctx, resource, name) + switch { + case apierrors.IsNotFound(err): + fail(w, http.StatusNotFound, "not found") + return + case err != nil: + log.FromContext(ctx).Error(err, "Reading the attachment's parent failed") + fail(w, http.StatusServiceUnavailable, "try again") + return + } + // In a user's own context the object must be theirs; staff ask in the + // platform context. + if (user != "" && owner != user) || !slices.Contains(refs, ref) { + fail(w, http.StatusNotFound, "not found") + return + } + f, err := s.Files.Get(ctx, purpose, name, owner, ref) + if errors.Is(err, attachments.ErrNotFound) { + fail(w, http.StatusNotFound, "not found") + return + } + if err != nil { + log.FromContext(ctx).Error(err, "Reading an attachment failed") + fail(w, http.StatusServiceUnavailable, "try again") + return + } + h := w.Header() + h.Set("Content-Type", f.ContentType) + h.Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": f.Name})) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Content-Security-Policy", "default-src 'none'; sandbox") + h.Set("Cache-Control", "private, no-store") + _, _ = w.Write(f.Data) +} + +// attachmentsOf returns the owner and attachment refs of a submission or +// report. +func (s *Server) attachmentsOf(ctx context.Context, resource, name string) (string, []string, error) { + key := types.NamespacedName{Name: name} + var owner string + var atts []v1.AttachmentReference + switch resource { + case "submissions": + var o v1.Submission + if err := s.Objects.Get(ctx, key, &o); err != nil { + return "", nil, err + } + owner, atts = o.Spec.UserRef.Name, o.Spec.Attachments + default: + var o v1.SecurityReport + if err := s.Objects.Get(ctx, key, &o); err != nil { + return "", nil, err + } + if o.Spec.UserRef != nil { + owner = o.Spec.UserRef.Name + } + atts = o.Spec.Attachments + } + refs := make([]string, len(atts)) + for i, a := range atts { + refs[i] = a.UploadRef + } + return owner, refs, nil +} + +func (s *Server) authenticate(w http.ResponseWriter, r *http.Request) (authenticationv1.UserInfo, bool) { + token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || token == "" { + fail(w, http.StatusUnauthorized, "send your Milo token as a bearer token") + return authenticationv1.UserInfo{}, false + } + u, ok, err := s.Authn.Authenticate(r.Context(), token) + if err != nil { + log.FromContext(r.Context()).Error(err, "TokenReview failed") + fail(w, http.StatusServiceUnavailable, "try again") + return u, false + } + if !ok { + fail(w, http.StatusUnauthorized, "the token isn't valid") + return u, false + } + return u, true +} + +func (s *Server) authorize(w http.ResponseWriter, r *http.Request, u authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes, inUserContext string) bool { + ok, err := s.Authz.Allowed(r.Context(), u, attrs, inUserContext) + if err != nil { + log.FromContext(r.Context()).Error(err, "SubjectAccessReview failed") + fail(w, http.StatusServiceUnavailable, "try again") + return false + } + if !ok { + // Don't say whether the object exists. + fail(w, http.StatusNotFound, "not found") + } + return ok +} + +func fail(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) +} + +// RequestTimeout bounds one request. +const RequestTimeout = 30 * time.Second diff --git a/internal/uploads/server_test.go b/internal/uploads/server_test.go new file mode 100644 index 0000000..797cab2 --- /dev/null +++ b/internal/uploads/server_test.go @@ -0,0 +1,138 @@ +package uploads + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/identity" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +// fakeMilo knows two tokens: "ada" (user u1) and "staff". Ada may act in +// her own context; staff may get submissions in the platform context. +type fakeMilo struct{} + +func (fakeMilo) Authenticate(_ context.Context, token string) (authenticationv1.UserInfo, bool, error) { + switch token { + case "ada": + return authenticationv1.UserInfo{Username: "ada@example.com", Extra: map[string]authenticationv1.ExtraValue{ + identity.EmailVerifiedExtraKey: {"true"}}}, true, nil + case "staff": + return authenticationv1.UserInfo{Username: "staff@datum.net"}, true, nil + } + return authenticationv1.UserInfo{}, false, nil +} + +func (fakeMilo) Allowed(_ context.Context, u authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes, inUserContext string) (bool, error) { + switch u.Username { + case "ada@example.com": + return inUserContext == "u1", nil + case "staff@datum.net": + return inUserContext == "" && attrs.Resource == "submissions" && attrs.Verb == "get", nil + } + return false, nil +} + +func newServer(t *testing.T) (*Server, http.Handler) { + t.Helper() + files, err := attachments.Open(context.Background(), "mem://") + if err != nil { + t.Fatal(err) + } + files.MaxSize = 1024 + scheme := runtime.NewScheme() + _ = v1.AddToScheme(scheme) + s := &Server{Files: files, Authn: fakeMilo{}, Authz: fakeMilo{}, Objects: fake.NewClientBuilder().WithScheme(scheme).Build()} + return s, s.Handler() +} + +func do(h http.Handler, method, path, token, contentType string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func TestUploadAndDownload(t *testing.T) { + s, h := newServer(t) + ctx := context.Background() + + for _, tc := range []struct { + name, path, token, ct string + body []byte + want int + }{ + {"no token", "/users/u1/uploads/submissions?name=a.txt", "", "text/plain", []byte("hi"), http.StatusUnauthorized}, + {"bad token", "/users/u1/uploads/submissions?name=a.txt", "nope", "text/plain", []byte("hi"), http.StatusUnauthorized}, + {"someone else's context", "/users/u2/uploads/submissions?name=a.txt", "ada", "text/plain", []byte("hi"), http.StatusNotFound}, + {"not a feedback resource", "/users/u1/uploads/votes?name=a.txt", "ada", "text/plain", []byte("hi"), http.StatusNotFound}, + {"html", "/users/u1/uploads/submissions?name=a.html", "ada", "text/html", []byte(""), http.StatusBadRequest}, + {"too big", "/users/u1/uploads/submissions?name=a.txt", "ada", "text/plain", bytes.Repeat([]byte("x"), 2000), http.StatusRequestEntityTooLarge}, + } { + if rec := do(h, http.MethodPost, tc.path, tc.token, tc.ct, tc.body); rec.Code != tc.want { + t.Errorf("%s: %d %s, want %d", tc.name, rec.Code, rec.Body, tc.want) + } + } + + rec := do(h, http.MethodPost, "/users/u1/uploads/submissions?name=notes.txt", "ada", "text/plain; charset=utf-8", []byte("hello")) + if rec.Code != http.StatusCreated { + t.Fatalf("upload: %d %s", rec.Code, rec.Body) + } + var up Uploaded + if err := json.NewDecoder(rec.Body).Decode(&up); err != nil || up.Name != "notes.txt" || up.ContentType != "text/plain" { + t.Fatalf("reply %+v, %v", up, err) + } + + sub := &v1.Submission{ObjectMeta: metav1.ObjectMeta{Name: "sub-1"}, Spec: v1.SubmissionSpec{ + UserRef: v1.UserReference{Name: "u1"}, Attachments: []v1.AttachmentReference{{UploadRef: up.UploadRef}}}} + theirs := &v1.Submission{ObjectMeta: metav1.ObjectMeta{Name: "sub-2"}, Spec: v1.SubmissionSpec{ + UserRef: v1.UserReference{Name: "u2"}, Attachments: []v1.AttachmentReference{{UploadRef: up.UploadRef}}}} + for _, o := range []*v1.Submission{sub, theirs} { + if err := s.Objects.(client.Client).Create(ctx, o); err != nil { + t.Fatal(err) + } + } + + get := func(path, token string) *httptest.ResponseRecorder { return do(h, http.MethodGet, path, token, "", nil) } + rec = get("/users/u1/submissions/sub-1/attachments/"+up.UploadRef, "ada") + if rec.Code != http.StatusOK || rec.Body.String() != "hello" { + t.Fatalf("own download: %d %s", rec.Code, rec.Body) + } + if h := rec.Header(); !strings.HasPrefix(h.Get("Content-Disposition"), "attachment") || h.Get("X-Content-Type-Options") != "nosniff" { + t.Errorf("headers %v", h) + } + for _, tc := range []struct{ name, path, token string }{ + {"another user's submission in my context", "/users/u1/submissions/sub-2/attachments/" + up.UploadRef, "ada"}, + {"a ref the submission doesn't have", "/users/u1/submissions/sub-1/attachments/att-other", "ada"}, + {"platform context without staff access", "/submissions/sub-1/attachments/" + up.UploadRef, "ada"}, + {"missing submission", "/users/u1/submissions/sub-9/attachments/" + up.UploadRef, "ada"}, + } { + if rec := get(tc.path, tc.token); rec.Code != http.StatusNotFound { + t.Errorf("%s: %d, want 404", tc.name, rec.Code) + } + } + if rec := get("/submissions/sub-1/attachments/"+up.UploadRef, "staff"); rec.Code != http.StatusOK { + b, _ := io.ReadAll(rec.Body) + t.Errorf("staff download: %d %s", rec.Code, b) + } +} diff --git a/internal/webhook/attachments.go b/internal/webhook/attachments.go new file mode 100644 index 0000000..c34b8b9 --- /dev/null +++ b/internal/webhook/attachments.go @@ -0,0 +1,74 @@ +package webhook + +import ( + "context" + "encoding/json" + "errors" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/identity" +) + +// +kubebuilder:webhook:path=/mutate-attachments,mutating=true,failurePolicy=fail,sideEffects=None,groups=feedback.miloapis.com,resources=submissions;securityreports,verbs=create,versions=v1alpha1,name=attachments.feedback.miloapis.com,admissionReviewVersions=v1 + +// Attachments checks that every attachment on a new submission or security +// report is one of the owner's uploads for that purpose, and sets each +// attachment's name and type from the upload rather than trusting the +// client. The operator's own creates (a report moved from a submission) +// carry files it has already copied, and pass. +type Attachments struct { + // Files is the attachments bucket. Nil refuses attachments. + Files *attachments.Store +} + +var _ admission.Handler = Attachments{} + +// Handle implements admission.Handler. +func (a Attachments) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.Operation != admissionv1.Create || identity.Of(req.UserInfo) == identity.Operator { + return admission.Allowed("") + } + obj := &unstructured.Unstructured{} + if err := json.Unmarshal(req.Object.Raw, &obj.Object); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + list, _, _ := unstructured.NestedSlice(obj.Object, "spec", "attachments") + if len(list) == 0 { + return admission.Allowed("") + } + if a.Files == nil { + return admission.Denied("attachments aren't available; send it without them") + } + purpose, _ := attachments.PurposeOf(req.Resource.Resource) + owner, ok := identity.User(req.UserInfo) + if !ok { + return admission.Denied("attachments can only be added by their uploader, in their own user context") + } + refs := make([]string, 0, len(list)) + for _, item := range list { + ref, _, _ := unstructured.NestedString(item.(map[string]any), "uploadRef") + refs = append(refs, ref) + } + found, err := a.Files.Lookup(ctx, purpose, owner, refs) + if errors.Is(err, attachments.ErrNotFound) { + return admission.Denied("every uploadRef must be one of your uploads for this from the last day") + } + if err != nil { + log.FromContext(ctx).Error(err, "Looking up uploads failed") + return admission.Errored(http.StatusServiceUnavailable, errors.New("attachments aren't available right now; try again or send it without them")) + } + out := make([]any, len(found)) + for i, f := range found { + out[i] = map[string]any{"name": f.Name, "contentType": f.ContentType, "uploadRef": f.UploadRef} + } + if err := unstructured.SetNestedSlice(obj.Object, out, "spec", "attachments"); err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return patch(req, obj) +} diff --git a/internal/webhook/attachments_test.go b/internal/webhook/attachments_test.go new file mode 100644 index 0000000..1045d3c --- /dev/null +++ b/internal/webhook/attachments_test.go @@ -0,0 +1,72 @@ +package webhook + +import ( + "context" + "strings" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" + + "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/identity" + v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" +) + +func TestAttachments(t *testing.T) { + ctx := context.Background() + files, err := attachments.Open(ctx, "mem://") + if err != nil { + t.Fatal(err) + } + ref, _, err := files.Upload(ctx, attachments.Submission, "u1", "notes.txt", "text/plain", []byte("hi")) + if err != nil { + t.Fatal(err) + } + sub := func(ref string) *v1.Submission { + return &v1.Submission{Spec: v1.SubmissionSpec{ + UserRef: v1.UserReference{Name: "u1"}, Kind: v1.SubmissionKindBug, Title: "t", Body: "b", + // The client's name and type are ignored. + Attachments: []v1.AttachmentReference{{UploadRef: ref, Name: "evil.html", ContentType: "text/html"}}, + }} + } + a := Attachments{Files: files} + + req := request(t, admissionv1.Create, "submissions", inUserContext("u1"), sub(ref), nil) + resp := a.Handle(ctx, req) + if !resp.Allowed { + t.Fatalf("own upload denied: %v", resp.Result.Message) + } + var got v1.Submission + patched(t, req, resp, &got) + if at := got.Spec.Attachments[0]; at.Name != "notes.txt" || at.ContentType != "text/plain" { + t.Errorf("attachment = %+v", at) + } + + for _, tc := range []struct { + name string + a Attachments + resource string + user authenticationv1.UserInfo + ref string + denied string + }{ + {"someone else's upload", a, "submissions", inUserContext("u2"), ref, "one of your uploads"}, + {"a made-up ref", a, "submissions", inUserContext("u1"), "att-nope", "one of your uploads"}, + {"an upload for the other purpose", a, "securityreports", inUserContext("u1"), ref, "one of your uploads"}, + {"no bucket", Attachments{}, "submissions", inUserContext("u1"), ref, "aren't available"}, + {"staff in the platform context", a, "submissions", authenticationv1.UserInfo{Username: "staff"}, ref, "their own user context"}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := tc.a.Handle(ctx, request(t, admissionv1.Create, tc.resource, tc.user, sub(tc.ref), nil)) + if resp.Allowed || !strings.Contains(resp.Result.Message, tc.denied) { + t.Errorf("allowed %v, %q; want denial containing %q", resp.Allowed, resp.Result.Message, tc.denied) + } + }) + } + + op := authenticationv1.UserInfo{Username: identity.OperatorUser} + if resp := a.Handle(ctx, request(t, admissionv1.Create, "securityreports", op, sub("att-moved"), nil)); !resp.Allowed { + t.Errorf("operator's moved report denied: %v", resp.Result.Message) + } +} diff --git a/internal/webhook/register.go b/internal/webhook/register.go index 4914b1e..02cd023 100644 --- a/internal/webhook/register.go +++ b/internal/webhook/register.go @@ -3,6 +3,8 @@ package webhook import ( "time" + "go.miloapis.com/feedback/internal/attachments" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -13,6 +15,8 @@ type Options struct { // disables the limit. SubmissionLimit int SubmissionWindow time.Duration + // Files is the attachments bucket. Nil refuses attachments. + Files *attachments.Store } // Register adds every admission webhook to mgr's webhook server. The indexes @@ -22,6 +26,7 @@ func Register(mgr manager.Manager, o Options) { c := mgr.GetClient() s.Register("/mutate-owner", &webhook.Admission{Handler: Owner{}}) s.Register("/mutate-submissionreview", &webhook.Admission{Handler: Reviewer{}}) + s.Register("/mutate-attachments", &webhook.Admission{Handler: Attachments{Files: o.Files}}) s.Register("/validate-vote", &webhook.Admission{Handler: VoteCheck{Posts: c}}) s.Register("/validate-submission", &webhook.Admission{Handler: SubmissionCheck{ Submissions: c, Limit: o.SubmissionLimit, Window: o.SubmissionWindow, diff --git a/internal/webhook/submission.go b/internal/webhook/submission.go index 92f1bdc..da743e7 100644 --- a/internal/webhook/submission.go +++ b/internal/webhook/submission.go @@ -30,8 +30,6 @@ type SubmissionCheck struct { Window time.Duration // Now returns the current time; tests replace it. Now func() time.Time - // Attachments reports whether attachments can be accepted. - Attachments bool } var _ admission.Handler = SubmissionCheck{} @@ -48,9 +46,6 @@ func (c SubmissionCheck) Handle(ctx context.Context, req admission.Request) admi if err := json.Unmarshal(req.Object.Raw, sub); err != nil { return admission.Errored(http.StatusBadRequest, err) } - if len(sub.Spec.Attachments) > 0 && !c.Attachments { - return admission.Denied("attachments aren't available yet; send the feedback without them") - } if c.Limit <= 0 { return admission.Allowed("") } diff --git a/internal/webhook/submission_test.go b/internal/webhook/submission_test.go index a815168..cb6a9bf 100644 --- a/internal/webhook/submission_test.go +++ b/internal/webhook/submission_test.go @@ -56,9 +56,6 @@ func TestSubmissionCheck(t *testing.T) { if resp := c.Handle(context.Background(), request(t, admissionv1.Create, "submissions", user("u2", "false"), newSub("u2", 0), nil)); resp.Allowed { t.Error("unverified email allowed") } - if resp := c.Handle(context.Background(), request(t, admissionv1.Create, "submissions", user("u2", "true"), newSub("u2", 1), nil)); resp.Allowed { - t.Error("attachments allowed without uploads") - } c.Limit = 0 if resp := c.Handle(context.Background(), request(t, admissionv1.Create, "submissions", user("u1", "true"), newSub("u1", 0), nil)); !resp.Allowed { t.Error("limit 0 still limits") From 8d6dfde5724aa165f47eb0802aaeafb816b50b12 Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Tue, 17 Nov 2026 15:51:42 -0500 Subject: [PATCH 2/5] feat: store attachments on S3 and S3-compatible object stores Register gocloud's s3blob driver, so --attachments-bucket accepts s3://BUCKET URLs, with endpoint, region and use_path_style for stores outside AWS such as Cloudflare R2 or MinIO. Add the s3-attachments kustomize component, which gives the operator and feedback-uploads credentials from the feedback-attachments-s3 Secret. Run the attachment store tests against an in-process S3-compatible fake as well as memory, and against FEEDBACK_TEST_S3_URL when it is set. --- cmd/feedback/uploads.go | 2 +- config/base/config.yaml | 4 +- config/base/uploads/deployment.yaml | 5 +- .../s3-attachments/kustomization.yaml | 53 ++++++++++++++++++ go.mod | 22 ++++++++ go.sum | 16 ++++++ internal/attachments/attachments.go | 18 +++++-- internal/attachments/attachments_test.go | 54 +++++++++++++++++-- 8 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 config/components/s3-attachments/kustomization.yaml diff --git a/cmd/feedback/uploads.go b/cmd/feedback/uploads.go index f31fb30..f34066b 100644 --- a/cmd/feedback/uploads.go +++ b/cmd/feedback/uploads.go @@ -41,7 +41,7 @@ func newUploadsCommand() *cobra.Command { f := cmd.Flags() f.StringVar(&o.miloKubeconfig, "milo-kubeconfig", "", "Kubeconfig for Milo. Empty uses the in-cluster config.") f.StringVar(&o.addr, "listen-address", ":8080", "Address to serve HTTP on. TLS is terminated in front of the service.") - f.StringVar(&o.bucket, "attachments-bucket", "", "Bucket URL for attachments, e.g. gs://feedback-attachments or file:///tmp/attachments.") + f.StringVar(&o.bucket, "attachments-bucket", "", "Bucket URL for attachments: gs://BUCKET, s3://BUCKET?region=REGION, or s3://BUCKET?endpoint=URL®ion=REGION&use_path_style=true for an S3-compatible store. file:///PATH for local development.") f.IntVar(&o.maxSize, "attachment-max-size", attachments.DefaultMaxSize, "Largest file accepted, in bytes.") f.Float64Var(&o.perMinute, "uploads-per-minute", 2, "Uploads a user may make per minute, on average; 0 disables the limit.") f.IntVar(&o.burst, "uploads-burst", 10, "Uploads a user may make at once.") diff --git a/config/base/config.yaml b/config/base/config.yaml index 9a51a4a..50c3e6e 100644 --- a/config/base/config.yaml +++ b/config/base/config.yaml @@ -4,5 +4,7 @@ kind: ConfigMap metadata: name: feedback-config data: - # Bucket URL for attachments, shared by the operator and feedback-uploads. + # Bucket URL for attachments, shared by the operator and feedback-uploads: + # gs://BUCKET, or s3://BUCKET?... for S3 and S3-compatible stores (see the + # s3-attachments component). attachments-bucket: mem:// diff --git a/config/base/uploads/deployment.yaml b/config/base/uploads/deployment.yaml index af56772..f7bb5b2 100644 --- a/config/base/uploads/deployment.yaml +++ b/config/base/uploads/deployment.yaml @@ -41,8 +41,9 @@ spec: - --attachments-bucket=$(ATTACHMENTS_BUCKET) - --zap-encoder=json env: - # The environment sets the bucket, e.g. gs://feedback-attachments-staging. - # Its lifecycle rule should delete uploads/ objects after one day. + # The environment sets the bucket, e.g. gs://feedback-attachments-staging + # or an S3-compatible store with the s3-attachments component. Its + # lifecycle rule should delete uploads/ objects after one day. - name: ATTACHMENTS_BUCKET valueFrom: configMapKeyRef: diff --git a/config/components/s3-attachments/kustomization.yaml b/config/components/s3-attachments/kustomization.yaml new file mode 100644 index 0000000..68ef686 --- /dev/null +++ b/config/components/s3-attachments/kustomization.yaml @@ -0,0 +1,53 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# s3-attachments gives the operator and feedback-uploads credentials for an +# attachments bucket on Amazon S3 or an S3-compatible store (Cloudflare R2, +# MinIO, Ceph and others), from the feedback-attachments-s3 Secret in the +# feedback-system namespace: +# +# AWS_ACCESS_KEY_ID: the store's access key +# AWS_SECRET_ACCESS_KEY: its secret key +# +# The key needs read, write, list and delete on the bucket. Set +# attachments-bucket in the feedback-config ConfigMap to the bucket URL, e.g. +# +# s3://feedback-attachments?endpoint=https://ACCOUNT.r2.cloudflarestorage.com®ion=auto&use_path_style=true +# +# Add request_checksum_calculation=when_required if the store rejects the AWS +# SDK's default checksums. Give the bucket a lifecycle rule that deletes +# objects under uploads/ after one day. + +patches: + - target: + kind: Deployment + name: feedback-operator + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: feedback-operator + spec: + template: + spec: + containers: + - name: operator + envFrom: + - secretRef: + name: feedback-attachments-s3 + - target: + kind: Deployment + name: feedback-uploads + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: feedback-uploads + spec: + template: + spec: + containers: + - name: uploads + envFrom: + - secretRef: + name: feedback-attachments-s3 diff --git a/go.mod b/go.mod index db203ef..8d294a1 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.27.4 require ( github.com/evanphx/json-patch/v5 v5.9.11 + github.com/johannesboyne/gofakes3 v1.2.0 github.com/spf13/cobra v1.10.2 gocloud.dev v0.48.0 golang.org/x/time v0.16.0 @@ -27,6 +28,25 @@ require ( cloud.google.com/go/storage v1.69.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.9 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect + github.com/aws/smithy-go v1.28.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -73,6 +93,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.0 // indirect github.com/prometheus/procfs v0.23.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -86,6 +107,7 @@ require ( go.opentelemetry.io/otel/sdk v1.47.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect go.opentelemetry.io/otel/trace v1.47.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect diff --git a/go.sum b/go.sum index ccbc4ea..27c1c9e 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFM github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= @@ -70,6 +72,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -171,6 +175,8 @@ github.com/googleapis/gax-go/v2 v2.27.0 h1:qe2iQOUzxb3fd6x6ewpIEixPQhJ+0Ihc/RRZh github.com/googleapis/gax-go/v2 v2.27.0/go.mod h1:RofiqDw3ehcG4L0H1zO3D2Itpwd3/uS/7wdmbNDF1DE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/johannesboyne/gofakes3 v1.2.0 h1:I9VEzPWvvAUAGzDlhYFoZjF0AXMlkcEyZlmBwiI6Oms= +github.com/johannesboyne/gofakes3 v1.2.0/go.mod h1:UHhRZRod9rENGFrUWTYnQHZqlNgSmjOq8DaD/ATQYRM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.20.4 h1:dD0v/bvxtjCBqQu6sx1TNWkalEY5lNTIe+HY9w8TeNI= @@ -205,6 +211,10 @@ github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5 github.com/prometheus/procfs v0.23.0 h1:ad+S6FTkzwaMgacKLM5h6bNKExS+DMH0YiUwXItH/o4= github.com/prometheus/procfs v0.23.0/go.mod h1:3AVXO0FqLafyYFU/V/IQRZlC8Rgqz8Dm/x6pKLK+RIk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -220,6 +230,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg= @@ -244,6 +256,8 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -304,6 +318,8 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= k8s.io/api v0.37.2/go.mod h1:+NVRLft99kx+Jg9v34GmHw867Dapim89TSyCsn2hK+M= k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= diff --git a/internal/attachments/attachments.go b/internal/attachments/attachments.go index 168abaa..327c313 100644 --- a/internal/attachments/attachments.go +++ b/internal/attachments/attachments.go @@ -31,8 +31,9 @@ import ( "gocloud.dev/blob" _ "gocloud.dev/blob/fileblob" // file:// for local development - _ "gocloud.dev/blob/gcsblob" // gs:// in clusters + _ "gocloud.dev/blob/gcsblob" // gs:// for Google Cloud Storage _ "gocloud.dev/blob/memblob" // mem:// for tests + _ "gocloud.dev/blob/s3blob" // s3:// for Amazon S3 and S3-compatible stores "gocloud.dev/gcerrors" v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" @@ -78,8 +79,19 @@ type Store struct { MaxSize int } -// Open opens the bucket at url, e.g. gs://feedback-attachments, file:///tmp/a -// or mem://. +// Open opens the bucket at bucketURL: +// +// - gs://BUCKET for Google Cloud Storage, with the workload's credentials. +// - s3://BUCKET?region=REGION for Amazon S3. +// - s3://BUCKET?endpoint=https://HOST®ion=REGION&use_path_style=true for +// an S3-compatible store (Cloudflare R2, MinIO, Ceph and others). Add +// request_checksum_calculation=when_required for stores that reject the +// AWS SDK's default checksums. +// - file:///PATH for local development and mem:// for tests. +// +// S3 credentials come from the AWS SDK's usual sources, normally the +// AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. See +// gocloud.dev/blob/s3blob and gocloud.dev/aws for every URL parameter. func Open(ctx context.Context, bucketURL string) (*Store, error) { b, err := blob.OpenBucket(ctx, bucketURL) if err != nil { diff --git a/internal/attachments/attachments_test.go b/internal/attachments/attachments_test.go index 28508d1..0b83b66 100644 --- a/internal/attachments/attachments_test.go +++ b/internal/attachments/attachments_test.go @@ -3,15 +3,23 @@ package attachments import ( "context" "errors" + "net/http/httptest" + "net/url" + "os" "strings" "testing" + + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" ) var png = []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 32)) -func open(t *testing.T) *Store { +func open(t *testing.T) *Store { return openURL(t, "mem://") } + +func openURL(t *testing.T, bucketURL string) *Store { t.Helper() - s, err := Open(context.Background(), "mem://") + s, err := Open(context.Background(), bucketURL) if err != nil { t.Fatal(err) } @@ -19,6 +27,41 @@ func open(t *testing.T) *Store { return s } +// fakeS3 serves an S3-compatible API with one bucket and returns its bucket +// URL, as an S3-compatible store outside AWS would be configured. +func fakeS3(t *testing.T) string { + t.Helper() + backend := s3mem.New() + if err := backend.CreateBucket("attachments"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(gofakes3.New(backend).Server()) + t.Cleanup(srv.Close) + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + t.Setenv("AWS_CONFIG_FILE", os.DevNull) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", os.DevNull) + q := url.Values{ + "endpoint": {srv.URL}, + "region": {"auto"}, + "use_path_style": {"true"}, + } + return "s3://attachments?" + q.Encode() +} + +// backends returns the bucket URLs every store test runs against: memory, a +// fake S3-compatible store, and FEEDBACK_TEST_S3_URL when it is set, to check +// a real store such as MinIO or R2 (with its credentials in the AWS_* +// environment variables). +func backends(t *testing.T) map[string]string { + t.Helper() + b := map[string]string{"mem": "mem://", "s3-compatible": fakeS3(t)} + if u := os.Getenv("FEEDBACK_TEST_S3_URL"); u != "" { + b["FEEDBACK_TEST_S3_URL"] = u + } + return b +} + func TestCheck(t *testing.T) { s := open(t) s.MaxSize = 64 @@ -43,8 +86,13 @@ func TestCheck(t *testing.T) { } func TestUploadAttachGetDelete(t *testing.T) { + for name, bucketURL := range backends(t) { + t.Run(name, func(t *testing.T) { testUploadAttachGetDelete(t, openURL(t, bucketURL)) }) + } +} + +func testUploadAttachGetDelete(t *testing.T, s *Store) { ctx := context.Background() - s := open(t) ref, sum, err := s.Upload(ctx, Submission, "u1", "shot.png", "image/png", png) if err != nil || !strings.HasPrefix(ref, "att-") || len(sum) != 64 { t.Fatalf("Upload = %q %q %v", ref, sum, err) From 289fbd50976ea891f485b2f58946700332a44974 Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Tue, 17 Nov 2026 16:42:41 -0500 Subject: [PATCH 3/5] fix(uploads): stream downloads, cap concurrent transfers, drop TimeoutHandler Downloads now stream from the bucket to the client instead of reading the whole file into memory, and http.TimeoutHandler, which buffered every reply again, is replaced by connection deadlines: ReadHeaderTimeout, ReadTimeout, WriteTimeout and IdleTimeout on the server, extended per transfer with http.ResponseController to a deadline sized for the file (30s plus the file at 64 KiB/s), with a matching context deadline on the bucket read or write. Uploads and downloads each take a slot from a per-replica semaphore (--max-concurrent-uploads, default 4; --max-concurrent-downloads, default 32) and get 503 with Retry-After when none is free. Uploads still buffer, up to the size limit via MaxBytesReader, because Check needs the bytes. The Deployment sets GOMEMLIMIT under its 256Mi limit. --- README.md | 1 + cmd/feedback/uploads.go | 11 +- config/base/uploads/deployment.yaml | 5 + internal/attachments/attachments.go | 39 ++-- internal/attachments/attachments_test.go | 95 ++++---- .../attachmentstest/attachmentstest.go | 47 ++++ internal/uploads/server.go | 206 +++++++++++++++-- internal/uploads/server_test.go | 207 +++++++++++++++++- 8 files changed, 528 insertions(+), 83 deletions(-) create mode 100644 internal/attachments/attachmentstest/attachmentstest.go diff --git a/README.md b/README.md index 7fa9457..21ce7c0 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ GET /{submissions|securityreports}/{name}/attachments/{ref} (staff) - **Checking the caller.** A Milo `TokenReview` identifies the caller. A `SubjectAccessReview` checks that they may create (to upload) or get (to download) that resource in the context the path names. In a user's context, the service also checks that the object is theirs. - **Files accepted.** Files are type-checked against their content: PNG, JPEG, GIF, WebP, PDF, text and JSON only. The size limit is 10 MiB by default, and each user gets a per-minute upload limit. - **Storage.** Files land under `uploads///` in the bucket. +- **Limits per replica.** Downloads stream from the bucket. Each upload is buffered, up to the size limit, to check its type. `--max-concurrent-uploads` (default 4) and `--max-concurrent-downloads` (default 32) cap transfers on one replica; past them it answers 503 with `Retry-After`. Each transfer gets a deadline sized for its file, at 64 KiB/s plus 30 seconds. - **Attaching.** The `attachments` mutating webhook checks every `uploadRef` on a new submission or report is the owner's upload for that purpose, and sets its name and type from the upload. - **Keeping and deleting.** The operator copies attached files under `submissions//` as soon as the submission exists, and adds the `feedback.miloapis.com/attachments` finalizer. Deleting the submission deletes its files. - **Lifecycle rule.** The bucket's lifecycle rule should delete `uploads/` after a day. diff --git a/cmd/feedback/uploads.go b/cmd/feedback/uploads.go index f34066b..8ac1f24 100644 --- a/cmd/feedback/uploads.go +++ b/cmd/feedback/uploads.go @@ -25,6 +25,8 @@ type uploadsOptions struct { maxSize int perMinute float64 burst int + maxUploads int + maxDownloads int } func newUploadsCommand() *cobra.Command { @@ -45,6 +47,8 @@ func newUploadsCommand() *cobra.Command { f.IntVar(&o.maxSize, "attachment-max-size", attachments.DefaultMaxSize, "Largest file accepted, in bytes.") f.Float64Var(&o.perMinute, "uploads-per-minute", 2, "Uploads a user may make per minute, on average; 0 disables the limit.") f.IntVar(&o.burst, "uploads-burst", 10, "Uploads a user may make at once.") + f.IntVar(&o.maxUploads, "max-concurrent-uploads", uploads.DefaultMaxUploads, "Uploads one replica handles at once; more get 503 with Retry-After. Each can hold up to twice --attachment-max-size, plus about 16 MiB of bucket client buffers, so lower this if you raise that. 0 disables the cap.") + f.IntVar(&o.maxDownloads, "max-concurrent-downloads", uploads.DefaultMaxDownloads, "Downloads one replica streams at once; more get 503 with Retry-After. 0 disables the cap.") return cmd } @@ -76,12 +80,9 @@ func runUploads(ctx context.Context, o uploadsOptions) error { s := &uploads.Server{ Files: files, Authn: milo, Authz: milo, Objects: objects, Limiter: ratelimit.NewLimiter(o.perMinute, o.burst), + Uploads: uploads.NewSlots(o.maxUploads), Downloads: uploads.NewSlots(o.maxDownloads), } - srv := &http.Server{ - Addr: o.addr, - Handler: http.TimeoutHandler(s.Handler(), uploads.RequestTimeout, "request timed out"), - ReadHeaderTimeout: 10 * time.Second, - } + srv := uploads.NewHTTPServer(o.addr, s.Handler()) go func() { <-ctx.Done() shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/config/base/uploads/deployment.yaml b/config/base/uploads/deployment.yaml index f7bb5b2..7bd1ea2 100644 --- a/config/base/uploads/deployment.yaml +++ b/config/base/uploads/deployment.yaml @@ -49,6 +49,11 @@ spec: configMapKeyRef: name: feedback-config key: attachments-bucket + # A soft limit under the container's 256Mi, so the GC works harder + # before the kernel kills the process. The transfer caps keep live + # memory under it; see DefaultMaxUploads in internal/uploads. + - name: GOMEMLIMIT + value: 220MiB ports: - containerPort: 8080 name: http diff --git a/internal/attachments/attachments.go b/internal/attachments/attachments.go index 327c313..bf9926a 100644 --- a/internal/attachments/attachments.go +++ b/internal/attachments/attachments.go @@ -15,7 +15,6 @@ package attachments import ( - "bytes" "context" "crypto/rand" "crypto/sha256" @@ -216,24 +215,28 @@ func (s *Store) CopyParent(ctx context.Context, from, to Purpose, fromParent, to } } -// File is an attached file. +// File is an attached file. Get returns its name, type and size; Open +// streams its content, so serving a file never holds it in memory. type File struct { Name string ContentType string - Data []byte + Size int64 + + bucket *blob.Bucket + key string } -// Get reads an attachment of parent. Until the operator has copied it, it is +// Get finds an attachment of parent. Until the operator has copied it, it is // read from owner's uploads. func (s *Store) Get(ctx context.Context, p Purpose, parent, owner, ref string) (*File, error) { - f, err := s.read(ctx, ParentPrefix(p, parent)+url.PathEscape(ref)) + f, err := s.stat(ctx, ParentPrefix(p, parent)+url.PathEscape(ref)) if errors.Is(err, ErrNotFound) && owner != "" { - return s.read(ctx, uploadKey(p, owner, ref)) + return s.stat(ctx, uploadKey(p, owner, ref)) } return f, err } -func (s *Store) read(ctx context.Context, key string) (*File, error) { +func (s *Store) stat(ctx context.Context, key string) (*File, error) { attrs, err := s.bucket.Attributes(ctx, key) if gcerrors.Code(err) == gcerrors.NotFound { return nil, ErrNotFound @@ -241,16 +244,24 @@ func (s *Store) read(ctx context.Context, key string) (*File, error) { if err != nil { return nil, err } - r, err := s.bucket.NewReader(ctx, key, nil) - if err != nil { - return nil, err + return &File{ + Name: attrs.Metadata["name"], ContentType: attrs.ContentType, Size: attrs.Size, + bucket: s.bucket, key: key, + }, nil +} + +// Open returns a reader for the file's content. ctx bounds the whole read, +// not just opening it. The caller must close the reader. A file deleted since +// Get is ErrNotFound. +func (f *File) Open(ctx context.Context) (io.ReadCloser, error) { + r, err := f.bucket.NewReader(ctx, f.key, nil) + if gcerrors.Code(err) == gcerrors.NotFound { + return nil, ErrNotFound } - defer r.Close() - var buf bytes.Buffer - if _, err := io.Copy(&buf, io.LimitReader(r, int64(s.MaxSize)+1)); err != nil { + if err != nil { return nil, err } - return &File{Name: attrs.Metadata["name"], ContentType: attrs.ContentType, Data: buf.Bytes()}, nil + return r, nil } // DeleteParent deletes every attachment of a parent. diff --git a/internal/attachments/attachments_test.go b/internal/attachments/attachments_test.go index 0b83b66..bb7d997 100644 --- a/internal/attachments/attachments_test.go +++ b/internal/attachments/attachments_test.go @@ -1,16 +1,15 @@ package attachments import ( + "bytes" "context" + "crypto/rand" "errors" - "net/http/httptest" - "net/url" - "os" + "io" "strings" "testing" - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" + "go.miloapis.com/feedback/internal/attachments/attachmentstest" ) var png = []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 32)) @@ -27,41 +26,6 @@ func openURL(t *testing.T, bucketURL string) *Store { return s } -// fakeS3 serves an S3-compatible API with one bucket and returns its bucket -// URL, as an S3-compatible store outside AWS would be configured. -func fakeS3(t *testing.T) string { - t.Helper() - backend := s3mem.New() - if err := backend.CreateBucket("attachments"); err != nil { - t.Fatal(err) - } - srv := httptest.NewServer(gofakes3.New(backend).Server()) - t.Cleanup(srv.Close) - t.Setenv("AWS_ACCESS_KEY_ID", "test") - t.Setenv("AWS_SECRET_ACCESS_KEY", "test") - t.Setenv("AWS_CONFIG_FILE", os.DevNull) - t.Setenv("AWS_SHARED_CREDENTIALS_FILE", os.DevNull) - q := url.Values{ - "endpoint": {srv.URL}, - "region": {"auto"}, - "use_path_style": {"true"}, - } - return "s3://attachments?" + q.Encode() -} - -// backends returns the bucket URLs every store test runs against: memory, a -// fake S3-compatible store, and FEEDBACK_TEST_S3_URL when it is set, to check -// a real store such as MinIO or R2 (with its credentials in the AWS_* -// environment variables). -func backends(t *testing.T) map[string]string { - t.Helper() - b := map[string]string{"mem": "mem://", "s3-compatible": fakeS3(t)} - if u := os.Getenv("FEEDBACK_TEST_S3_URL"); u != "" { - b["FEEDBACK_TEST_S3_URL"] = u - } - return b -} - func TestCheck(t *testing.T) { s := open(t) s.MaxSize = 64 @@ -86,7 +50,7 @@ func TestCheck(t *testing.T) { } func TestUploadAttachGetDelete(t *testing.T) { - for name, bucketURL := range backends(t) { + for name, bucketURL := range attachmentstest.Backends(t) { t.Run(name, func(t *testing.T) { testUploadAttachGetDelete(t, openURL(t, bucketURL)) }) } } @@ -123,8 +87,12 @@ func testUploadAttachGetDelete(t *testing.T, s *Store) { if err := s.Attach(ctx, Submission, "u2", "sub-2", []string{ref}); !errors.Is(err, ErrNotFound) { t.Errorf("attaching someone else's upload: %v", err) } - if f, err := s.Get(ctx, Submission, "sub-1", "", ref); err != nil || string(f.Data) != string(png) { - t.Fatalf("Get after attach = %v", err) + f, err := s.Get(ctx, Submission, "sub-1", "", ref) + if err != nil || f.Name != "shot.png" || f.ContentType != "image/png" || f.Size != int64(len(png)) { + t.Fatalf("Get after attach = %+v, %v", f, err) + } + if got := readAll(t, f); !bytes.Equal(got, png) { + t.Fatalf("content after attach = %q", got) } if err := s.CopyParent(ctx, Submission, SecurityReport, "sub-1", "sec-1"); err != nil { @@ -140,6 +108,47 @@ func testUploadAttachGetDelete(t *testing.T, s *Store) { if _, err := s.Get(ctx, Submission, "sub-1", "", ref); !errors.Is(err, ErrNotFound) { t.Errorf("after delete: %v", err) } + if _, err := f.Open(ctx); !errors.Is(err, ErrNotFound) { + t.Errorf("opening a file deleted since Get: %v", err) + } +} + +// TestStreamLargeFile reads a file bigger than the read buffers in pieces, +// as a download does. +func TestStreamLargeFile(t *testing.T) { + for name, bucketURL := range attachmentstest.Backends(t) { + t.Run(name, func(t *testing.T) { + s := openURL(t, bucketURL) + ctx := context.Background() + data := make([]byte, 3<<20+7) + _, _ = rand.Read(data) + ref, _, err := s.Upload(ctx, Submission, "u1", "big.bin", "application/pdf", data) + if err != nil { + t.Fatal(err) + } + f, err := s.Get(ctx, Submission, "sub-1", "u1", ref) + if err != nil || f.Size != int64(len(data)) { + t.Fatalf("Get = %+v, %v", f, err) + } + if got := readAll(t, f); !bytes.Equal(got, data) { + t.Fatalf("read %d bytes, want the %d uploaded", len(got), len(data)) + } + }) + } +} + +func readAll(t *testing.T, f *File) []byte { + t.Helper() + r, err := f.Open(context.Background()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return b } func TestPurposeOf(t *testing.T) { diff --git a/internal/attachments/attachmentstest/attachmentstest.go b/internal/attachments/attachmentstest/attachmentstest.go new file mode 100644 index 0000000..09ec233 --- /dev/null +++ b/internal/attachments/attachmentstest/attachmentstest.go @@ -0,0 +1,47 @@ +// Package attachmentstest provides attachment buckets for tests. +package attachmentstest + +import ( + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" +) + +// FakeS3 serves an S3-compatible API with one bucket, in process, and returns +// its bucket URL, as an S3-compatible store outside AWS would be configured. +func FakeS3(t testing.TB) string { + t.Helper() + backend := s3mem.New() + if err := backend.CreateBucket("attachments"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(gofakes3.New(backend).Server()) + t.Cleanup(srv.Close) + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + t.Setenv("AWS_CONFIG_FILE", os.DevNull) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", os.DevNull) + q := url.Values{ + "endpoint": {srv.URL}, + "region": {"auto"}, + "use_path_style": {"true"}, + } + return "s3://attachments?" + q.Encode() +} + +// Backends returns the bucket URLs store tests run against: memory, a fake +// S3-compatible store, and FEEDBACK_TEST_S3_URL when it is set, to check a +// real store such as MinIO or R2 (with its credentials in the AWS_* +// environment variables). +func Backends(t testing.TB) map[string]string { + t.Helper() + b := map[string]string{"mem": "mem://", "s3-compatible": FakeS3(t)} + if u := os.Getenv("FEEDBACK_TEST_S3_URL"); u != "" { + b["FEEDBACK_TEST_S3_URL"] = u + } + return b +} diff --git a/internal/uploads/server.go b/internal/uploads/server.go index e4978b2..9fbab90 100644 --- a/internal/uploads/server.go +++ b/internal/uploads/server.go @@ -18,9 +18,17 @@ // {resource} is submissions or securityreports. Uploads send the file as the // body with its Content-Type. The reply is the uploadRef to put in // spec.attachments. +// +// Memory and time are bounded per replica. Downloads stream from the bucket +// and never hold a file. Uploads buffer at most one file each, and Slots cap +// how many uploads and downloads run at once; a full replica answers 503 with +// Retry-After rather than queueing. NewHTTPServer's timeouts bound every +// request without buffering replies, and a transfer gets a deadline sized for +// its file (TransferTimeout). package uploads import ( + "bytes" "context" "encoding/json" "errors" @@ -66,6 +74,125 @@ type Server struct { Objects client.Reader // Limiter limits uploads per user. Nil doesn't limit. Limiter *ratelimit.Limiter + // Uploads and Downloads cap concurrent transfers on this replica. Nil + // doesn't cap. + Uploads, Downloads Slots +} + +// Default transfer caps, for the 256Mi container limit in +// config/base/uploads and the default 10 MiB attachment-max-size. +// +// Memory budget per replica: +// +// - An upload holds its body, up to MaxSize (10 MiB). When the client +// doesn't send Content-Length, growing the buffer can briefly need twice +// that. The bucket client then buffers up to about 16 MiB more while it +// sends the file (GCS uploads in 16 MiB chunks, S3 in 8 MiB parts). Call +// it 36 MiB at worst, so 4 uploads need about 144 MiB. +// - A download streams through a 32 KiB copy buffer plus the bucket +// client's and the connection's buffers, well under 1 MiB, so 32 +// downloads need a few MiB. +// - The process itself (runtime, Milo and bucket clients) needs about 40 +// MiB. +// +// That totals about 190 MiB, leaving headroom under 256Mi; GOMEMLIMIT in the +// Deployment makes the GC work harder before the limit rather than letting +// garbage push the process over it. Raising --attachment-max-size means +// lowering --max-concurrent-uploads to match. +const ( + DefaultMaxUploads = 4 + DefaultMaxDownloads = 32 +) + +// Slots is a semaphore that caps concurrent transfers. A nil Slots has no +// cap. +type Slots chan struct{} + +// NewSlots returns n slots, or nil, for no cap, if n <= 0. +func NewSlots(n int) Slots { + if n <= 0 { + return nil + } + return make(Slots, n) +} + +// take takes a slot if one is free. It never waits: callers answer 503 +// instead, so a busy replica can't build an unbounded queue of waiting +// requests and their connections. +func (s Slots) take() bool { + if s == nil { + return true + } + select { + case s <- struct{}{}: + return true + default: + return false + } +} + +func (s Slots) give() { + if s != nil { + <-s + } +} + +// RequestTimeout bounds each request apart from moving a file: reading it, +// asking Milo, and writing a short reply. It is the server's ReadTimeout and +// WriteTimeout. +const RequestTimeout = 30 * time.Second + +// MinTransferRate is the slowest average rate, in bytes per second, at which +// a client may send or receive a file; slower, and the deadline cuts it off. +const MinTransferRate = 64 << 10 + +// RetryAfter is how long a client is told to wait when every slot is taken. +const RetryAfter = 5 * time.Second + +// TransferTimeout is how long moving a file of size bytes may take: +// RequestTimeout, plus size at MinTransferRate. A 10 MiB file gets 3m10s. +func TransferTimeout(size int64) time.Duration { + return RequestTimeout + time.Duration(size)*time.Second/MinTransferRate +} + +// NewHTTPServer returns an HTTP server for h with timeouts that, unlike +// http.TimeoutHandler, don't buffer replies: they are deadlines on the +// connection. Transfers extend them for their own request (see transfer). +func NewHTTPServer(addr string, h http.Handler) *http.Server { + return &http.Server{ + Addr: addr, + Handler: h, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: RequestTimeout, + WriteTimeout: RequestTimeout, + IdleTimeout: 2 * time.Minute, + } +} + +// transfer moves this request's read and write deadlines to the time a file +// of size bytes may take, and returns a context that ends then too, so a +// stalled client or bucket can't hold a slot past it. +// +// The read deadline matters for downloads as well as uploads: once a request +// has no body left, net/http reads the connection in the background to notice +// the client leaving, and if that read hits the server's ReadTimeout it +// cancels the request's context, which would cut a long download short. +func transfer(w http.ResponseWriter, r *http.Request, size int64) (context.Context, context.CancelFunc) { + deadline := time.Now().Add(TransferTimeout(size)) + rc := http.NewResponseController(w) + for _, set := range []func(time.Time) error{rc.SetReadDeadline, rc.SetWriteDeadline} { + // Only test recorders don't support deadlines. + if err := set(deadline); err != nil && !errors.Is(err, http.ErrNotSupported) { + log.FromContext(r.Context()).Error(err, "Setting a transfer deadline failed") + } + } + return context.WithDeadline(r.Context(), deadline) +} + +// busy answers a request that found every slot taken. +func busy(w http.ResponseWriter) { + w.Header().Set("Retry-After", strconv.Itoa(int(RetryAfter.Seconds()))) + fail(w, http.StatusServiceUnavailable, "the service is busy; try again shortly") } // Handler returns the service's HTTP handler. @@ -87,7 +214,6 @@ type Uploaded struct { } func (s *Server) upload(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() resource, user := r.PathValue("resource"), r.PathValue("user") purpose, ok := attachments.PurposeOf(resource) if !ok { @@ -105,26 +231,52 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request) { fail(w, http.StatusForbidden, "verify your email address to send files") return } + contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + fail(w, http.StatusBadRequest, "set Content-Type to the file's media type") + return + } + maxSize := int64(s.Files.MaxSize) + tooBig := fmt.Sprintf("the file is larger than %d bytes", maxSize) + if r.ContentLength > maxSize { + fail(w, http.StatusRequestEntityTooLarge, tooBig) + return + } + if !s.Uploads.take() { + busy(w) + return + } + defer s.Uploads.give() + // After taking a slot, so a busy replica doesn't spend the user's quota. if ok, wait := s.Limiter.Allow(user); !ok { w.Header().Set("Retry-After", strconv.Itoa(int(wait.Seconds())+1)) fail(w, http.StatusTooManyRequests, "too many uploads; try again shortly") return } - contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil { - fail(w, http.StatusBadRequest, "set Content-Type to the file's media type") - return + size := maxSize + if r.ContentLength >= 0 { + size = r.ContentLength } - data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(s.Files.MaxSize)+1)) - var tooBig *http.MaxBytesError - if errors.As(err, &tooBig) { - fail(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("the file is larger than %d bytes", s.Files.MaxSize)) + ctx, cancel := transfer(w, r, size) + defer cancel() + // The upload is buffered, up to MaxSize, because Check needs all of it to + // verify its type and the stored SHA-256 covers it. MaxBytesReader stops + // reading past MaxSize, and the upload slot bounds how many buffers exist + // at once. With a Content-Length the buffer is allocated once. + var body bytes.Buffer + if r.ContentLength >= 0 { + body.Grow(int(r.ContentLength) + bytes.MinRead) + } + _, err = body.ReadFrom(http.MaxBytesReader(w, r.Body, maxSize)) + if _, ok := errors.AsType[*http.MaxBytesError](err); ok { + fail(w, http.StatusRequestEntityTooLarge, tooBig) return } if err != nil { fail(w, http.StatusBadRequest, "reading the file failed") return } + data := body.Bytes() name := r.URL.Query().Get("name") if msg := s.Files.Check(name, contentType, data); msg != "" { fail(w, http.StatusBadRequest, msg) @@ -172,23 +324,46 @@ func (s *Server) download(w http.ResponseWriter, r *http.Request) { fail(w, http.StatusNotFound, "not found") return } + if !s.Downloads.take() { + busy(w) + return + } + defer s.Downloads.give() f, err := s.Files.Get(ctx, purpose, name, owner, ref) - if errors.Is(err, attachments.ErrNotFound) { - fail(w, http.StatusNotFound, "not found") + if err != nil { + unreadable(ctx, w, err) return } + ctx, cancel := transfer(w, r, f.Size) + defer cancel() + body, err := f.Open(ctx) if err != nil { - log.FromContext(ctx).Error(err, "Reading an attachment failed") - fail(w, http.StatusServiceUnavailable, "try again") + unreadable(ctx, w, err) return } + defer func() { _ = body.Close() }() h := w.Header() h.Set("Content-Type", f.ContentType) + h.Set("Content-Length", strconv.FormatInt(f.Size, 10)) h.Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": f.Name})) h.Set("X-Content-Type-Options", "nosniff") h.Set("Content-Security-Policy", "default-src 'none'; sandbox") h.Set("Cache-Control", "private, no-store") - _, _ = w.Write(f.Data) + if _, err := io.Copy(w, body); err != nil { + // The status has been sent, so all that's left is to stop short of + // Content-Length, which tells the client the file is incomplete. + log.FromContext(ctx).V(1).Info("Sending an attachment failed", "error", err.Error()) + } +} + +// unreadable answers a download whose file couldn't be read. +func unreadable(ctx context.Context, w http.ResponseWriter, err error) { + if errors.Is(err, attachments.ErrNotFound) { + fail(w, http.StatusNotFound, "not found") + return + } + log.FromContext(ctx).Error(err, "Reading an attachment failed") + fail(w, http.StatusServiceUnavailable, "try again") } // attachmentsOf returns the owner and attachment refs of a submission or @@ -259,6 +434,3 @@ func fail(w http.ResponseWriter, code int, msg string) { w.WriteHeader(code) _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) } - -// RequestTimeout bounds one request. -const RequestTimeout = 30 * time.Second diff --git a/internal/uploads/server_test.go b/internal/uploads/server_test.go index 797cab2..683088d 100644 --- a/internal/uploads/server_test.go +++ b/internal/uploads/server_test.go @@ -3,12 +3,16 @@ package uploads import ( "bytes" "context" + "crypto/rand" "encoding/json" "io" "net/http" "net/http/httptest" + "net/url" + "strconv" "strings" "testing" + "time" authenticationv1 "k8s.io/api/authentication/v1" authorizationv1 "k8s.io/api/authorization/v1" @@ -18,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "go.miloapis.com/feedback/internal/attachments" + "go.miloapis.com/feedback/internal/attachments/attachmentstest" "go.miloapis.com/feedback/internal/identity" v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) @@ -47,13 +52,16 @@ func (fakeMilo) Allowed(_ context.Context, u authenticationv1.UserInfo, attrs au return false, nil } -func newServer(t *testing.T) (*Server, http.Handler) { +func newServer(t *testing.T) (*Server, http.Handler) { return newServerAt(t, "mem://", 1024) } + +func newServerAt(t *testing.T, bucketURL string, maxSize int) (*Server, http.Handler) { t.Helper() - files, err := attachments.Open(context.Background(), "mem://") + files, err := attachments.Open(context.Background(), bucketURL) if err != nil { t.Fatal(err) } - files.MaxSize = 1024 + t.Cleanup(func() { _ = files.Close() }) + files.MaxSize = maxSize scheme := runtime.NewScheme() _ = v1.AddToScheme(scheme) s := &Server{Files: files, Authn: fakeMilo{}, Authz: fakeMilo{}, Objects: fake.NewClientBuilder().WithScheme(scheme).Build()} @@ -113,7 +121,9 @@ func TestUploadAndDownload(t *testing.T) { } } - get := func(path, token string) *httptest.ResponseRecorder { return do(h, http.MethodGet, path, token, "", nil) } + get := func(path, token string) *httptest.ResponseRecorder { + return do(h, http.MethodGet, path, token, "", nil) + } rec = get("/users/u1/submissions/sub-1/attachments/"+up.UploadRef, "ada") if rec.Code != http.StatusOK || rec.Body.String() != "hello" { t.Fatalf("own download: %d %s", rec.Code, rec.Body) @@ -136,3 +146,192 @@ func TestUploadAndDownload(t *testing.T) { t.Errorf("staff download: %d %s", rec.Code, b) } } + +// attach records a submission by owner with refs attached. +func attach(t *testing.T, s *Server, name, owner string, refs ...string) { + t.Helper() + sub := &v1.Submission{ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: v1.SubmissionSpec{UserRef: v1.UserReference{Name: owner}}} + for _, ref := range refs { + sub.Spec.Attachments = append(sub.Spec.Attachments, v1.AttachmentReference{UploadRef: ref}) + } + if err := s.Objects.(client.Client).Create(context.Background(), sub); err != nil { + t.Fatal(err) + } +} + +func uploadRef(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + var up Uploaded + if rec.Code != http.StatusCreated { + t.Fatalf("upload: %d %s", rec.Code, rec.Body) + } + if err := json.NewDecoder(rec.Body).Decode(&up); err != nil { + t.Fatal(err) + } + return up.UploadRef +} + +// TestStreamingDownload uploads and downloads a file several times larger +// than the copy buffers, over a real HTTP server with the service's +// timeouts, from each kind of bucket. +func TestStreamingDownload(t *testing.T) { + for backend, bucketURL := range attachmentstest.Backends(t) { + t.Run(backend, func(t *testing.T) { + const maxSize = 4 << 20 + s, h := newServerAt(t, bucketURL, maxSize) + srv := httptest.NewUnstartedServer(nil) + srv.Config = NewHTTPServer("", h) + srv.Start() + t.Cleanup(srv.Close) + + data := make([]byte, maxSize) + copy(data, "%PDF-1.7\n") + _, _ = rand.Read(data[16:]) + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/users/u1/uploads/submissions?name="+url.QueryEscape("my report \"final\".pdf"), bytes.NewReader(data)) + req.Header.Set("Authorization", "Bearer ada") + req.Header.Set("Content-Type", "application/pdf") + res, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + var up Uploaded + err = json.NewDecoder(res.Body).Decode(&up) + _ = res.Body.Close() + if res.StatusCode != http.StatusCreated || err != nil { + t.Fatalf("upload: %d, %v", res.StatusCode, err) + } + attach(t, s, "sub-1", "u1", up.UploadRef) + + req, _ = http.NewRequest(http.MethodGet, srv.URL+"/users/u1/submissions/sub-1/attachments/"+up.UploadRef, nil) + req.Header.Set("Authorization", "Bearer ada") + res, err = srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = res.Body.Close() }() + got, err := io.ReadAll(res.Body) + if res.StatusCode != http.StatusOK || err != nil { + t.Fatalf("download: %d, %v", res.StatusCode, err) + } + if !bytes.Equal(got, data) { + t.Errorf("downloaded %d bytes that differ from the %d uploaded", len(got), len(data)) + } + for k, want := range map[string]string{ + "Content-Type": "application/pdf", + "Content-Length": strconv.Itoa(len(data)), + "Content-Disposition": `attachment; filename="my report \"final\".pdf"`, + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + } { + if got := res.Header.Get(k); got != want { + t.Errorf("%s = %q, want %q", k, got, want) + } + } + if res.ContentLength != int64(len(data)) { + t.Errorf("ContentLength = %d", res.ContentLength) + } + }) + } +} + +// TestTransferOutlivesReadTimeout checks a transfer isn't cut short by the +// server's ReadTimeout, which cancels a request's context once its +// background read of the connection times out. +func TestTransferOutlivesReadTimeout(t *testing.T) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := transfer(w, r, 1<<20) + defer cancel() + time.Sleep(300 * time.Millisecond) + if err := ctx.Err(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = io.WriteString(w, "ok") + })) + srv.Config.ReadTimeout = 100 * time.Millisecond + srv.Config.WriteTimeout = 100 * time.Millisecond + srv.Start() + t.Cleanup(srv.Close) + res, err := srv.Client().Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer func() { _ = res.Body.Close() }() + if b, err := io.ReadAll(res.Body); err != nil || string(b) != "ok" { + t.Fatalf("%d %q, %v", res.StatusCode, b, err) + } +} + +func TestBusy(t *testing.T) { + s, h := newServer(t) + s.Uploads, s.Downloads = NewSlots(1), NewSlots(1) + ref := uploadRef(t, do(h, http.MethodPost, "/users/u1/uploads/submissions?name=a.txt", "ada", "text/plain", []byte("hello"))) + attach(t, s, "sub-1", "u1", ref) + download := func() *httptest.ResponseRecorder { + return do(h, http.MethodGet, "/users/u1/submissions/sub-1/attachments/"+ref, "ada", "", nil) + } + upload := func() *httptest.ResponseRecorder { + return do(h, http.MethodPost, "/users/u1/uploads/submissions?name=b.txt", "ada", "text/plain", []byte("again")) + } + + // Fill every slot, as transfers in progress would. + if !s.Uploads.take() || !s.Downloads.take() { + t.Fatal("slots should be free") + } + for name, try := range map[string]func() *httptest.ResponseRecorder{"upload": upload, "download": download} { + rec := try() + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "5" { + t.Errorf("%s while busy: %d, Retry-After %q", name, rec.Code, rec.Header().Get("Retry-After")) + } + } + // Busy replies come after the access checks, so they say nothing to + // someone who may not see the file. + if rec := do(h, http.MethodGet, "/users/u1/submissions/sub-1/attachments/"+ref, "nope", "", nil); rec.Code != http.StatusUnauthorized { + t.Errorf("bad token while busy: %d", rec.Code) + } + + s.Uploads.give() + s.Downloads.give() + if rec := upload(); rec.Code != http.StatusCreated { + t.Errorf("upload once free: %d %s", rec.Code, rec.Body) + } + if rec := download(); rec.Code != http.StatusOK || rec.Body.String() != "hello" { + t.Errorf("download once free: %d %s", rec.Code, rec.Body) + } + // Finished transfers gave their slots back. + if len(s.Uploads) != 0 || len(s.Downloads) != 0 { + t.Errorf("slots in use after transfers finished: %d uploads, %d downloads", len(s.Uploads), len(s.Downloads)) + } +} + +func TestUploadSizeLimit(t *testing.T) { + _, h := newServer(t) // MaxSize 1024 + for _, tc := range []struct { + name string + size int + chunked bool + want int + }{ + {"at the limit", 1024, false, http.StatusCreated}, + {"over the limit", 1025, false, http.StatusRequestEntityTooLarge}, + {"at the limit, no Content-Length", 1024, true, http.StatusCreated}, + {"over the limit, no Content-Length", 1025, true, http.StatusRequestEntityTooLarge}, + {"far over the limit, no Content-Length", 1 << 20, true, http.StatusRequestEntityTooLarge}, + } { + var body io.Reader = bytes.NewReader(bytes.Repeat([]byte("x"), tc.size)) + if tc.chunked { + body = io.MultiReader(body) // hides the length + } + req := httptest.NewRequest(http.MethodPost, "/users/u1/uploads/submissions?name=a.txt", body) + if tc.chunked { + req.ContentLength = -1 + } + req.Header.Set("Authorization", "Bearer ada") + req.Header.Set("Content-Type", "text/plain") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != tc.want { + t.Errorf("%s: %d %s, want %d", tc.name, rec.Code, rec.Body, tc.want) + } + } +} From 154ef89bfa1cae274ee54823266a19edcc323a1d Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Tue, 17 Nov 2026 17:29:43 -0500 Subject: [PATCH 4/5] fix(uploads): accept the logging flags the Deployment passes feedback-uploads never bound the zap flags, so --zap-encoder=json from its Deployment made it exit with "unknown flag" and crash-loop. Bind them as the operator does, and test that every Deployment's args parse with its command. --- cmd/feedback/flags_test.go | 51 ++++++++++++++++++++++++++++++++++++++ cmd/feedback/uploads.go | 4 +++ 2 files changed, 55 insertions(+) create mode 100644 cmd/feedback/flags_test.go diff --git a/cmd/feedback/flags_test.go b/cmd/feedback/flags_test.go new file mode 100644 index 0000000..6ccc8aa --- /dev/null +++ b/cmd/feedback/flags_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "testing" + + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/yaml" +) + +// TestDeploymentArgsParse checks every Deployment in config/base passes only +// flags its command accepts, so a renamed or unbound flag fails here rather +// than as a crash loop in a cluster. +func TestDeploymentArgsParse(t *testing.T) { + files, err := filepath.Glob("../../config/base/*/deployment.yaml") + if err != nil || len(files) == 0 { + t.Fatalf("no deployments found: %v", err) + } + envRef := regexp.MustCompile(`\$\([A-Z_]+\)`) + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + var d appsv1.Deployment + if err := yaml.Unmarshal(data, &d); err != nil { + t.Fatalf("%s: %v", f, err) + } + for _, c := range d.Spec.Template.Spec.Containers { + if len(c.Command) < 2 || filepath.Base(c.Command[0]) != "feedback" { + continue + } + t.Run(d.Name+"/"+c.Name, func(t *testing.T) { + root := newRootCommand() + cmd, _, err := root.Find(c.Command[1:]) + if err != nil || cmd == root { + t.Fatalf("command %v not found: %v", c.Command[1:], err) + } + args := make([]string, len(c.Args)) + for i, a := range c.Args { + args[i] = envRef.ReplaceAllString(a, "x") + } + if err := cmd.ParseFlags(args); err != nil { + t.Errorf("%s args %q: %v", filepath.Base(f), c.Args, err) + } + }) + } + } +} diff --git a/cmd/feedback/uploads.go b/cmd/feedback/uploads.go index 8ac1f24..1d4f660 100644 --- a/cmd/feedback/uploads.go +++ b/cmd/feedback/uploads.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "flag" "fmt" "net/http" "time" @@ -49,6 +50,9 @@ func newUploadsCommand() *cobra.Command { f.IntVar(&o.burst, "uploads-burst", 10, "Uploads a user may make at once.") f.IntVar(&o.maxUploads, "max-concurrent-uploads", uploads.DefaultMaxUploads, "Uploads one replica handles at once; more get 503 with Retry-After. Each can hold up to twice --attachment-max-size, plus about 16 MiB of bucket client buffers, so lower this if you raise that. 0 disables the cap.") f.IntVar(&o.maxDownloads, "max-concurrent-downloads", uploads.DefaultMaxDownloads, "Downloads one replica streams at once; more get 503 with Retry-After. 0 disables the cap.") + goFlags := flag.NewFlagSet("zap", flag.ExitOnError) + zapOpts.BindFlags(goFlags) + f.AddGoFlagSet(goFlags) return cmd } From d8d8c501904037ebfb5dd564c5c3cfdf69cfc4c1 Mon Sep 17 00:00:00 2001 From: Matt Jenkinson Date: Wed, 18 Nov 2026 15:57:00 -0500 Subject: [PATCH 5/5] Cap each user's concurrent transfers The replica-wide transfer caps were shared by everyone, so one signed-in user could hold all four upload slots with slow requests and give every other user 503s until their deadlines ran out. Each user may now hold at most --max-concurrent-uploads-per-user (default 2) and --max-concurrent-downloads-per-user (default 8) transfers on a replica. Past that they get 429 with Retry-After, before taking a replica slot or spending rate-limit quota. The service refuses to start with a per-user cap that isn't below the replica's, so one user can never take every slot. TestOneUserCannotTakeEverySlot holds two of one user's uploads open mid-body and checks that user's third gets 429 while another user's upload succeeds. --- README.md | 2 +- cmd/feedback/flags_test.go | 17 +++++ cmd/feedback/uploads.go | 37 ++++++++--- internal/uploads/server.go | 106 +++++++++++++++++++++++++++++--- internal/uploads/server_test.go | 99 ++++++++++++++++++++++++++++- 5 files changed, 240 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 21ce7c0..606c458 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ GET /{submissions|securityreports}/{name}/attachments/{ref} (staff) - **Checking the caller.** A Milo `TokenReview` identifies the caller. A `SubjectAccessReview` checks that they may create (to upload) or get (to download) that resource in the context the path names. In a user's context, the service also checks that the object is theirs. - **Files accepted.** Files are type-checked against their content: PNG, JPEG, GIF, WebP, PDF, text and JSON only. The size limit is 10 MiB by default, and each user gets a per-minute upload limit. - **Storage.** Files land under `uploads///` in the bucket. -- **Limits per replica.** Downloads stream from the bucket. Each upload is buffered, up to the size limit, to check its type. `--max-concurrent-uploads` (default 4) and `--max-concurrent-downloads` (default 32) cap transfers on one replica; past them it answers 503 with `Retry-After`. Each transfer gets a deadline sized for its file, at 64 KiB/s plus 30 seconds. +- **Limits per replica.** Downloads stream from the bucket. Each upload is buffered, up to the size limit, to check its type. `--max-concurrent-uploads` (default 4) and `--max-concurrent-downloads` (default 32) cap transfers on one replica; past them it answers 503 with `Retry-After`. One user may hold at most `--max-concurrent-uploads-per-user` (default 2) and `--max-concurrent-downloads-per-user` (default 8) of them, so slow requests from one user can't keep everyone else out; past that they get 429 with `Retry-After`. Each transfer gets a deadline sized for its file, at 64 KiB/s plus 30 seconds. - **Attaching.** The `attachments` mutating webhook checks every `uploadRef` on a new submission or report is the owner's upload for that purpose, and sets its name and type from the upload. - **Keeping and deleting.** The operator copies attached files under `submissions//` as soon as the submission exists, and adds the `feedback.miloapis.com/attachments` finalizer. Deleting the submission deletes its files. - **Lifecycle rule.** The bucket's lifecycle rule should delete `uploads/` after a day. diff --git a/cmd/feedback/flags_test.go b/cmd/feedback/flags_test.go index 6ccc8aa..4061438 100644 --- a/cmd/feedback/flags_test.go +++ b/cmd/feedback/flags_test.go @@ -49,3 +49,20 @@ func TestDeploymentArgsParse(t *testing.T) { } } } + +func TestCheckUserCap(t *testing.T) { + for _, c := range []struct { + perUser, replica int + ok bool + }{ + {2, 4, true}, + {4, 4, false}, + {5, 4, false}, + {0, 4, true}, + {4, 0, true}, + } { + if err := checkUserCap("uploads", c.perUser, c.replica); (err == nil) != c.ok { + t.Errorf("per user %d, replica %d: %v", c.perUser, c.replica, err) + } + } +} diff --git a/cmd/feedback/uploads.go b/cmd/feedback/uploads.go index 1d4f660..43ec18b 100644 --- a/cmd/feedback/uploads.go +++ b/cmd/feedback/uploads.go @@ -20,14 +20,16 @@ import ( ) type uploadsOptions struct { - miloKubeconfig string - addr string - bucket string - maxSize int - perMinute float64 - burst int - maxUploads int - maxDownloads int + miloKubeconfig string + addr string + bucket string + maxSize int + perMinute float64 + burst int + maxUploads int + maxDownloads int + maxUserUploads int + maxUserDownloads int } func newUploadsCommand() *cobra.Command { @@ -50,6 +52,8 @@ func newUploadsCommand() *cobra.Command { f.IntVar(&o.burst, "uploads-burst", 10, "Uploads a user may make at once.") f.IntVar(&o.maxUploads, "max-concurrent-uploads", uploads.DefaultMaxUploads, "Uploads one replica handles at once; more get 503 with Retry-After. Each can hold up to twice --attachment-max-size, plus about 16 MiB of bucket client buffers, so lower this if you raise that. 0 disables the cap.") f.IntVar(&o.maxDownloads, "max-concurrent-downloads", uploads.DefaultMaxDownloads, "Downloads one replica streams at once; more get 503 with Retry-After. 0 disables the cap.") + f.IntVar(&o.maxUserUploads, "max-concurrent-uploads-per-user", uploads.DefaultMaxUploadsPerUser, "Uploads one user may run at once on one replica; more get 429 with Retry-After. Must be below --max-concurrent-uploads, so one user can't take every slot. 0 disables the cap.") + f.IntVar(&o.maxUserDownloads, "max-concurrent-downloads-per-user", uploads.DefaultMaxDownloadsPerUser, "Downloads one user may run at once on one replica; more get 429 with Retry-After. Must be below --max-concurrent-downloads. 0 disables the cap.") goFlags := flag.NewFlagSet("zap", flag.ExitOnError) zapOpts.BindFlags(goFlags) f.AddGoFlagSet(goFlags) @@ -61,6 +65,12 @@ func runUploads(ctx context.Context, o uploadsOptions) error { if o.bucket == "" { return errors.New("--attachments-bucket is required") } + if err := checkUserCap("uploads", o.maxUserUploads, o.maxUploads); err != nil { + return err + } + if err := checkUserCap("downloads", o.maxUserDownloads, o.maxDownloads); err != nil { + return err + } files, err := attachments.Open(ctx, o.bucket) if err != nil { return err @@ -85,6 +95,8 @@ func runUploads(ctx context.Context, o uploadsOptions) error { Files: files, Authn: milo, Authz: milo, Objects: objects, Limiter: ratelimit.NewLimiter(o.perMinute, o.burst), Uploads: uploads.NewSlots(o.maxUploads), Downloads: uploads.NewSlots(o.maxDownloads), + UploadsPerUser: uploads.NewUserSlots(o.maxUserUploads), + DownloadsPerUser: uploads.NewUserSlots(o.maxUserDownloads), } srv := uploads.NewHTTPServer(o.addr, s.Handler()) go func() { @@ -99,3 +111,12 @@ func runUploads(ctx context.Context, o uploadsOptions) error { } return nil } + +// checkUserCap refuses a per-user cap that would let one user take every one +// of the replica's slots. +func checkUserCap(kind string, perUser, replica int) error { + if perUser > 0 && replica > 0 && perUser >= replica { + return fmt.Errorf("--max-concurrent-%s-per-user (%d) must be below --max-concurrent-%s (%d), so one user can't take every slot", kind, perUser, kind, replica) + } + return nil +} diff --git a/internal/uploads/server.go b/internal/uploads/server.go index 9fbab90..9414989 100644 --- a/internal/uploads/server.go +++ b/internal/uploads/server.go @@ -22,7 +22,8 @@ // Memory and time are bounded per replica. Downloads stream from the bucket // and never hold a file. Uploads buffer at most one file each, and Slots cap // how many uploads and downloads run at once; a full replica answers 503 with -// Retry-After rather than queueing. NewHTTPServer's timeouts bound every +// Retry-After rather than queueing. UserSlots cap each user's share of them, +// so one user can't take every slot; a user at their cap gets 429. NewHTTPServer's timeouts bound every // request without buffering replies, and a transfer gets a deadline sized for // its file (TransferTimeout). package uploads @@ -39,6 +40,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" authenticationv1 "k8s.io/api/authentication/v1" @@ -77,6 +79,10 @@ type Server struct { // Uploads and Downloads cap concurrent transfers on this replica. Nil // doesn't cap. Uploads, Downloads Slots + // UploadsPerUser and DownloadsPerUser cap one user's concurrent + // transfers on this replica, below Uploads and Downloads, so slow + // requests from one user can't keep everyone else busy. Nil doesn't cap. + UploadsPerUser, DownloadsPerUser *UserSlots } // Default transfer caps, for the 256Mi container limit in @@ -99,9 +105,14 @@ type Server struct { // Deployment makes the GC work harder before the limit rather than letting // garbage push the process over it. Raising --attachment-max-size means // lowering --max-concurrent-uploads to match. +// +// One user may hold half the uploads and a quarter of the downloads, so +// others always have slots left. const ( - DefaultMaxUploads = 4 - DefaultMaxDownloads = 32 + DefaultMaxUploads = 4 + DefaultMaxDownloads = 32 + DefaultMaxUploadsPerUser = 2 + DefaultMaxDownloadsPerUser = 8 ) // Slots is a semaphore that caps concurrent transfers. A nil Slots has no @@ -137,6 +148,80 @@ func (s Slots) give() { } } +// UserSlots caps how many transfers each user runs at once. A nil UserSlots +// has no cap. +type UserSlots struct { + max int + mu sync.Mutex + held map[string]int +} + +// NewUserSlots returns a cap of n transfers per user, or nil, for no cap, if +// n <= 0. +func NewUserSlots(n int) *UserSlots { + if n <= 0 { + return nil + } + return &UserSlots{max: n, held: map[string]int{}} +} + +// take takes one of user's slots if they have one free. Like Slots.take, it +// never waits. +func (u *UserSlots) take(user string) bool { + if u == nil { + return true + } + u.mu.Lock() + defer u.mu.Unlock() + if u.held[user] >= u.max { + return false + } + u.held[user]++ + return true +} + +func (u *UserSlots) give(user string) { + if u == nil { + return + } + u.mu.Lock() + defer u.mu.Unlock() + if u.held[user]--; u.held[user] <= 0 { + delete(u.held, user) + } +} + +// inUse reports how many of user's slots are taken. +func (u *UserSlots) inUse(user string) int { + if u == nil { + return 0 + } + u.mu.Lock() + defer u.mu.Unlock() + return u.held[user] +} + +// claim takes a transfer slot for user: one of theirs, then one of the +// replica's. If either is full it answers the request, 429 for the user's +// cap and 503 for the replica's, and returns ok false. Otherwise the caller +// must call release when the transfer ends. +func claim(w http.ResponseWriter, perUser *UserSlots, replica Slots, user string) (release func(), ok bool) { + if !perUser.take(user) { + w.Header().Set("Retry-After", strconv.Itoa(int(RetryAfter.Seconds()))) + fail(w, http.StatusTooManyRequests, "you have too many transfers in progress; try again when one finishes") + return nil, false + } + if !replica.take() { + perUser.give(user) + busy(w) + return nil, false + } + return func() { + replica.give() + perUser.give(user) + }, true +} + // RequestTimeout bounds each request apart from moving a file: reading it, // asking Milo, and writing a short reply. It is the server's ReadTimeout and // WriteTimeout. @@ -146,7 +231,8 @@ const RequestTimeout = 30 * time.Second // a client may send or receive a file; slower, and the deadline cuts it off. const MinTransferRate = 64 << 10 -// RetryAfter is how long a client is told to wait when every slot is taken. +// RetryAfter is how long a client is told to wait when every slot, or every +// one of their own, is taken. const RetryAfter = 5 * time.Second // TransferTimeout is how long moving a file of size bytes may take: @@ -242,11 +328,11 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request) { fail(w, http.StatusRequestEntityTooLarge, tooBig) return } - if !s.Uploads.take() { - busy(w) + release, ok := claim(w, s.UploadsPerUser, s.Uploads, caller.Username) + if !ok { return } - defer s.Uploads.give() + defer release() // After taking a slot, so a busy replica doesn't spend the user's quota. if ok, wait := s.Limiter.Allow(user); !ok { w.Header().Set("Retry-After", strconv.Itoa(int(wait.Seconds())+1)) @@ -324,11 +410,11 @@ func (s *Server) download(w http.ResponseWriter, r *http.Request) { fail(w, http.StatusNotFound, "not found") return } - if !s.Downloads.take() { - busy(w) + release, ok := claim(w, s.DownloadsPerUser, s.Downloads, caller.Username) + if !ok { return } - defer s.Downloads.give() + defer release() f, err := s.Files.Get(ctx, purpose, name, owner, ref) if err != nil { unreadable(ctx, w, err) diff --git a/internal/uploads/server_test.go b/internal/uploads/server_test.go index 683088d..7d20f57 100644 --- a/internal/uploads/server_test.go +++ b/internal/uploads/server_test.go @@ -11,6 +11,7 @@ import ( "net/url" "strconv" "strings" + "sync" "testing" "time" @@ -27,8 +28,9 @@ import ( v1 "go.miloapis.com/feedback/pkg/apis/feedback/v1alpha1" ) -// fakeMilo knows two tokens: "ada" (user u1) and "staff". Ada may act in -// her own context; staff may get submissions in the platform context. +// fakeMilo knows three tokens: "ada" (user u1), "grace" (user u2) and +// "staff". Ada and Grace may act in their own contexts; staff may get +// submissions in the platform context. type fakeMilo struct{} func (fakeMilo) Authenticate(_ context.Context, token string) (authenticationv1.UserInfo, bool, error) { @@ -36,6 +38,9 @@ func (fakeMilo) Authenticate(_ context.Context, token string) (authenticationv1. case "ada": return authenticationv1.UserInfo{Username: "ada@example.com", Extra: map[string]authenticationv1.ExtraValue{ identity.EmailVerifiedExtraKey: {"true"}}}, true, nil + case "grace": + return authenticationv1.UserInfo{Username: "grace@example.com", Extra: map[string]authenticationv1.ExtraValue{ + identity.EmailVerifiedExtraKey: {"true"}}}, true, nil case "staff": return authenticationv1.UserInfo{Username: "staff@datum.net"}, true, nil } @@ -46,6 +51,8 @@ func (fakeMilo) Allowed(_ context.Context, u authenticationv1.UserInfo, attrs au switch u.Username { case "ada@example.com": return inUserContext == "u1", nil + case "grace@example.com": + return inUserContext == "u2", nil case "staff@datum.net": return inUserContext == "" && attrs.Resource == "submissions" && attrs.Verb == "get", nil } @@ -304,6 +311,94 @@ func TestBusy(t *testing.T) { } } +// TestOneUserCannotTakeEverySlot holds two of Ada's uploads open, as slow +// clients would, and checks she can't start a third while Grace still can. +func TestOneUserCannotTakeEverySlot(t *testing.T) { + s, h := newServer(t) + s.Uploads, s.UploadsPerUser = NewSlots(4), NewUserSlots(2) + const ada = "ada@example.com" + + var wg sync.WaitGroup + var bodies []*io.PipeWriter + codes := make(chan int, 2) + for range 2 { + r, w := io.Pipe() + bodies = append(bodies, w) + req := httptest.NewRequest(http.MethodPost, "/users/u1/uploads/submissions?name=slow.txt", r) + req.Header.Set("Authorization", "Bearer ada") + req.Header.Set("Content-Type", "text/plain") + wg.Go(func() { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + codes <- rec.Code + }) + } + defer func() { + for _, w := range bodies { + _ = w.Close() + } + }() + for deadline := time.Now().Add(5 * time.Second); s.UploadsPerUser.inUse(ada) < 2; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("slow uploads hold %d of Ada's slots, want 2", s.UploadsPerUser.inUse(ada)) + } + } + + rec := do(h, http.MethodPost, "/users/u1/uploads/submissions?name=third.txt", "ada", "text/plain", []byte("third")) + if rec.Code != http.StatusTooManyRequests || rec.Header().Get("Retry-After") != "5" { + t.Errorf("Ada's third upload: %d, Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) + } + if rec := do(h, http.MethodPost, "/users/u2/uploads/submissions?name=g.txt", "grace", "text/plain", []byte("hi")); rec.Code != http.StatusCreated { + t.Errorf("Grace's upload while Ada's are in progress: %d %s", rec.Code, rec.Body) + } + if len(s.Uploads) != 2 { + t.Errorf("replica slots in use: %d, want Ada's 2", len(s.Uploads)) + } + + for _, w := range bodies { + _, _ = w.Write([]byte("slow")) + _ = w.Close() + } + wg.Wait() + close(codes) + for code := range codes { + if code != http.StatusCreated { + t.Errorf("slow upload finished with %d", code) + } + } + if len(s.Uploads) != 0 || s.UploadsPerUser.inUse(ada) != 0 { + t.Errorf("slots in use after uploads finished: %d replica, %d Ada's", len(s.Uploads), s.UploadsPerUser.inUse(ada)) + } + if rec := do(h, http.MethodPost, "/users/u1/uploads/submissions?name=again.txt", "ada", "text/plain", []byte("again")); rec.Code != http.StatusCreated { + t.Errorf("Ada's upload once hers finished: %d %s", rec.Code, rec.Body) + } +} + +func TestUserDownloadCap(t *testing.T) { + s, h := newServer(t) + s.Downloads, s.DownloadsPerUser = NewSlots(4), NewUserSlots(1) + ref := uploadRef(t, do(h, http.MethodPost, "/users/u1/uploads/submissions?name=a.txt", "ada", "text/plain", []byte("hello"))) + attach(t, s, "sub-1", "u1", ref) + download := func() *httptest.ResponseRecorder { + return do(h, http.MethodGet, "/users/u1/submissions/sub-1/attachments/"+ref, "ada", "", nil) + } + + // As a download in progress would. + if !s.DownloadsPerUser.take("ada@example.com") { + t.Fatal("Ada's slot should be free") + } + if rec := download(); rec.Code != http.StatusTooManyRequests || rec.Header().Get("Retry-After") != "5" { + t.Errorf("download at Ada's cap: %d, Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) + } + if len(s.Downloads) != 0 { + t.Errorf("a refused download kept a replica slot") + } + s.DownloadsPerUser.give("ada@example.com") + if rec := download(); rec.Code != http.StatusOK || rec.Body.String() != "hello" { + t.Errorf("download once Ada's slot is free: %d %s", rec.Code, rec.Body) + } +} + func TestUploadSizeLimit(t *testing.T) { _, h := newServer(t) // MaxSize 1024 for _, tc := range []struct {