diff --git a/braintrust/README.md b/braintrust/README.md index 59b2fd9..d787679 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -217,6 +217,38 @@ Size the request for the pod's full local-storage usage: When you enable `tmpVolume`, make sure the `ephemeralStorage.request` still covers that extra space. +## GKE API Autoscaling + +The API can autoscale on GKE using a Horizontal Pod Autoscaler backed by GKE's native `AutoscalingMetric` resource. When enabled, each API pool scales on three signals - CPU (scoped to the `api` container via `ContainerResource`, so sidecars are excluded), Node.js event-loop utilization, and mean event-loop delay. + +This is underpinned by a **Preview (Pre-GA)** GKE feature. It requires: + +- Braintrust API / data plane **v2.9.0** or later (Prometheus `/metrics` on the API health server) +- GKE **1.35.1-gke.1396000** or later +- The Performance HPA profile and the Autoscaling API enabled on the cluster +- `roles/autoscaling.metricsWriter` granted to all node service accounts +- The Autoscaling API included in your service perimeter when using VPC Service Controls + +See [Expose custom metrics for autoscaling](https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling) for more details on `AutoscalingMetric` in GKE. + +Enable it in your values: + +```yaml +api: + autoscaling: + enabled: true + minReplicas: 4 + maxReplicas: 50 +``` + +When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. + +## EKS API Autoscaling + +On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`). + +Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background). + ## API workload isolation `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest` @@ -259,9 +291,9 @@ ingest, eval, function, and automation routes match `POST`; proxy routes match all methods. GKE Ingress cannot route by method, so its equivalent integration classifies matching paths for all methods. -This feature does not enable autoscaling. Configure fixed replica counts under -`api.replicas`, `api.workloadIsolation.ingest.replicas`, and -`api.workloadIsolation.background.replicas`. +Pools use fixed replica counts by default (`api.replicas` and +`api.workloadIsolation..replicas`). On GKE, enable `api.autoscaling` to +let each pool scale independently instead. ## Testing @@ -311,4 +343,5 @@ Example values files for different cloud providers and configurations are locate - `examples/google-autopilot/values.yaml`: GKE Autopilot deployment. - `examples/google-autopilot-cel/values.yaml`: GKE Autopilot deployment with CEL-friendly security settings. +- `examples/google-api-isolation-autoscaling/values.yaml`: Minimal example for API workload isolation and per-pool autoscaling on GKE (combine with an Autopilot or Standard values file). - `examples/google-standard/values.yaml`: GKE Standard deployment. diff --git a/braintrust/examples/google-api-isolation-autoscaling/values.yaml b/braintrust/examples/google-api-isolation-autoscaling/values.yaml new file mode 100644 index 0000000..7f98223 --- /dev/null +++ b/braintrust/examples/google-api-isolation-autoscaling/values.yaml @@ -0,0 +1,48 @@ +# Minimal GKE example with API workload isolation + per-pool autoscaling. +# +# Combine this with a normal google-autopilot (or google-standard) values +# file - only the fields below are specific to this pattern. +# +# Sizing: +# - api.autoscaling is shared by every pool +# - api.workloadIsolation..autoscaling deep-merges on top +# - With autoscaling enabled, set capacity via minReplicas/maxReplicas +# (Helm `replicas` is not applied to those Deployments) +# +# Also requires GKE 1.35.1+ with AutoscalingMetric, and ingress routes for the +# isolation contract (see README & files/contracts/api-workload-isolation-routes.yaml). + +api: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 40 + # Metric targets (inherited by every pool unless overridden below). + cpu: + targetAverageUtilization: 50 + eventLoopUtilization: + targetAverageValue: "0.4" # 0-1 ratio (0.4 is 40%) + eventLoopDelayMean: + targetAverageValue: "0.05" # seconds (0.05 is 50ms) + # HPA scale velocity (inherited by every pool unless overridden below). + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 + workloadIsolation: + enabled: true + ingest: + autoscaling: + minReplicas: 3 + maxReplicas: 60 + # Example pool-specific overrides: higher ELU target, faster scale-out. + eventLoopUtilization: + targetAverageValue: "0.5" # 0-1 ratio (0.5 is 50%) + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + background: + autoscaling: + minReplicas: 3 + maxReplicas: 50 diff --git a/braintrust/examples/google-autopilot-cel/values.yaml b/braintrust/examples/google-autopilot-cel/values.yaml index 2024a9d..380184e 100644 --- a/braintrust/examples/google-autopilot-cel/values.yaml +++ b/braintrust/examples/google-autopilot-cel/values.yaml @@ -25,6 +25,14 @@ api: service: networking.gke.io/load-balancer-type: "Internal" replicas: 4 + # Alternatively, autoscale the API on CPU and event-loop metrics. + # This is a Preview (Pre-GA) GKE feature requiring 1.35.1-gke.1396000 or later + # and the prerequisites documented in the main values.yaml. + # See api.autoscaling in the main values.yaml file for more details. + # autoscaling: + # enabled: true + # minReplicas: 4 + # maxReplicas: 50 service: type: LoadBalancer port: 8000 diff --git a/braintrust/examples/google-autopilot/values.yaml b/braintrust/examples/google-autopilot/values.yaml index dc9f482..9375209 100644 --- a/braintrust/examples/google-autopilot/values.yaml +++ b/braintrust/examples/google-autopilot/values.yaml @@ -33,6 +33,14 @@ api: service: networking.gke.io/load-balancer-type: "Internal" replicas: 4 + # Alternatively, autoscale the API on CPU and event-loop metrics. + # This is a Preview (Pre-GA) GKE feature requiring 1.35.1-gke.1396000 or later + # and the prerequisites documented in the main values.yaml. + # See api.autoscaling in the main values.yaml file for more details. + # autoscaling: + # enabled: true + # minReplicas: 4 + # maxReplicas: 50 # Uncomment the following section to use a different image or tag from the version in the Helm release #image: #repository: public.ecr.aws/braintrust/standalone-api diff --git a/braintrust/templates/_api-deployment.tpl b/braintrust/templates/_api-deployment.tpl index 8e2d39e..190c6ee 100644 --- a/braintrust/templates/_api-deployment.tpl +++ b/braintrust/templates/_api-deployment.tpl @@ -40,7 +40,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if not (dig "autoscaling" "enabled" false $api) }} replicas: {{ $api.replicas }} + {{- end }} strategy: type: {{ $api.strategy.type }} {{- with $api.strategy.rollingUpdate }} @@ -101,6 +103,9 @@ spec: {{- end }} ports: - containerPort: {{ $api.service.port }} + {{- if dig "autoscaling" "enabled" false $api }} + - containerPort: {{ $api.healthServer.port }} + {{- end }} resources: {{- toYaml $api.resources | nindent 12 }} {{- with $api.livenessProbe }} @@ -176,6 +181,10 @@ spec: {{- with $api.extraEnvVars }} {{- toYaml . | nindent 12 }} {{- end }} + {{- if dig "autoscaling" "enabled" false $api }} + - name: ENABLE_PROMETHEUS_METRICS + value: "true" + {{- end }} {{- if or $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled }} volumeMounts: {{- if $api.tmpVolume.enabled }} diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index d18b731..e7634f2 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -115,6 +115,21 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} +{{/* +Validate API autoscaling prerequisites. +GKE requires AutoscalingMetric (autoscaling.gke.io/v1beta1). +EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD). +*/}} +{{- define "braintrust.apiAutoscaling.validate" -}} +{{- if eq .Values.cloud "google" }} +{{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }} +{{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }} +{{- end }} +{{- else if ne .Values.cloud "aws" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }} +{{- end }} +{{- end -}} + {{/* Render Brainstore container resources with provider-specific ephemeral storage. diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml new file mode 100644 index 0000000..3600dc4 --- /dev/null +++ b/braintrust/templates/api-autoscaling-metric.yaml @@ -0,0 +1,55 @@ +{{- $root := . -}} +{{- if eq $root.Values.cloud "google" -}} +{{- $pools := include "braintrust.apiPools" . | fromYamlArray -}} +{{- $validated := false -}} +{{- $rendered := 0 -}} +{{- range $pool := $pools -}} +{{- $api := $pool.config -}} +{{- if dig "autoscaling" "enabled" false $api -}} +{{- if not $validated -}} +{{- include "braintrust.apiAutoscaling.validate" $root -}} +{{- $validated = true -}} +{{- end -}} +{{- if gt $rendered 0 }} +--- +{{- end }} +{{- $poolLabels := dict -}} +{{- if or $root.Values.api.workloadIsolation.enabled (ne $pool.role "default") -}} +{{- $_ := set $poolLabels "braintrust.dev/api-pool" $pool.role -}} +{{- end -}} +{{- $resourceLabels := mergeOverwrite (deepCopy $root.Values.global.labels) (deepCopy $api.labels) $poolLabels -}} +apiVersion: autoscaling.gke.io/v1beta1 +kind: AutoscalingMetric +metadata: + name: {{ $api.name }} + namespace: {{ include "braintrust.namespace" $root }} + {{- with $resourceLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $api.annotations.autoscalingMetric }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + metrics: + - pod: + selector: + matchLabels: + app: {{ $api.name }} + containers: + - endpoint: + port: {{ $api.healthServer.port }} + path: {{ $api.autoscaling.metricsPath }} + metrics: + - gauge: + # GKE gauge names must match ^[a-z]([-a-z0-9]*[a-z0-9])? + name: braintrust-api-event-loop-utilization-ratio + prometheusMetricName: braintrust_api_event_loop_utilization_ratio + - gauge: + name: braintrust-api-event-loop-delay-mean-seconds + prometheusMetricName: braintrust_api_event_loop_delay_mean_seconds +{{- $rendered = add1 $rendered -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml new file mode 100644 index 0000000..ca2d137 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml @@ -0,0 +1,159 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +rules: + - apiGroups: [""] + resources: ["namespaces", "pods", "services"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +data: + config.yaml: | + rules: + - seriesQuery: 'braintrust_api_event_loop_delay_mean_seconds{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: 'braintrust_api_event_loop_utilization_ratio{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + ports: + - name: https + port: 443 + targetPort: 6443 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} + port: 443 + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus-adapter + containers: + - name: prometheus-adapter + image: "{{ .Values.api.autoscaling.adapter.image.repository }}:{{ .Values.api.autoscaling.adapter.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.adapter.image.pullPolicy }} + args: + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --prometheus-url=http://{{ .Values.api.name }}-autoscaling-prometheus.{{ include "braintrust.namespace" . }}.svc:9090 + - --metrics-relist-interval={{ .Values.api.autoscaling.prometheus.scrapeInterval }} + - --config=/etc/adapter/config.yaml + - --v=2 + ports: + - containerPort: 6443 + name: https + resources: + {{- toYaml .Values.api.autoscaling.adapter.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/adapter + - name: tmp + mountPath: /tmp/cert + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + - name: tmp + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml new file mode 100644 index 0000000..be19129 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus.yaml @@ -0,0 +1,123 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +{{- $scrapeApps := list .Values.api.name -}} +{{- if .Values.api.workloadIsolation.enabled -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.background.name -}} +{{- end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +data: + prometheus.yml: | + global: + scrape_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + evaluation_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + + scrape_configs: + - job_name: braintrust-api-autoscaling + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - {{ include "braintrust.namespace" . }} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + regex: {{ join "|" $scrapeApps }} + action: keep + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: $1:{{ .Values.api.healthServer.port }} + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + metrics_path: {{ .Values.api.autoscaling.metricsPath }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus + ports: + - name: http + port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus + containers: + - name: prometheus + image: "{{ .Values.api.autoscaling.prometheus.image.repository }}:{{ .Values.api.autoscaling.prometheus.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.prometheus.image.pullPolicy }} + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ .Values.api.autoscaling.prometheus.retention }} + - --web.enable-lifecycle + ports: + - containerPort: 9090 + name: http + resources: + {{- toYaml .Values.api.autoscaling.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus + - name: data + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml new file mode 100644 index 0000000..10991f1 --- /dev/null +++ b/braintrust/templates/api-hpa.yaml @@ -0,0 +1,87 @@ +{{- $root := . -}} +{{- $pools := include "braintrust.apiPools" . | fromYamlArray -}} +{{- $validated := false -}} +{{- $rendered := 0 -}} +{{- range $pool := $pools -}} +{{- $api := $pool.config -}} +{{- if dig "autoscaling" "enabled" false $api -}} +{{- if not $validated -}} +{{- include "braintrust.apiAutoscaling.validate" $root -}} +{{- $validated = true -}} +{{- end -}} +{{- if gt $rendered 0 }} +--- +{{- end }} +{{- $poolLabels := dict -}} +{{- if or $root.Values.api.workloadIsolation.enabled (ne $pool.role "default") -}} +{{- $_ := set $poolLabels "braintrust.dev/api-pool" $pool.role -}} +{{- end -}} +{{- $resourceLabels := mergeOverwrite (deepCopy $root.Values.global.labels) (deepCopy $api.labels) $poolLabels -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $api.name }} + namespace: {{ include "braintrust.namespace" $root }} + {{- with $resourceLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $api.annotations.hpa }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $api.name }} + minReplicas: {{ $api.autoscaling.minReplicas }} + maxReplicas: {{ $api.autoscaling.maxReplicas }} + metrics: + # ContainerResource scopes CPU to the api container so sidecars / extraContainers + # do not skew utilization. Custom Pods metrics are already API-scoped via /metrics. + - type: ContainerResource + containerResource: + name: cpu + container: api + target: + type: Utilization + averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }} + {{- if eq $root.Values.cloud "google" }} + - type: Pods + pods: + metric: + name: autoscaling.gke.io|{{ $api.name }}|braintrust-api-event-loop-utilization-ratio + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }} + - type: Pods + pods: + metric: + name: autoscaling.gke.io|{{ $api.name }}|braintrust-api-event-loop-delay-mean-seconds + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- else if eq $root.Values.cloud "aws" }} + - type: Pods + pods: + metric: + name: event_loop_utilization_ratio + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }} + - type: Pods + pods: + metric: + name: event_loop_delay_mean_seconds + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- end }} + {{- with $api.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- $rendered = add1 $rendered -}} +{{- end -}} +{{- end }} diff --git a/braintrust/tests/api-autoscaling-crd_test.yaml b/braintrust/tests/api-autoscaling-crd_test.yaml new file mode 100644 index 0000000..ff7702b --- /dev/null +++ b/braintrust/tests/api-autoscaling-crd_test.yaml @@ -0,0 +1,17 @@ +suite: test API autoscaling CRD requirement +templates: + - api-hpa.yaml +# Do not advertise autoscaling.gke.io here — this suite asserts the fail path. +tests: + - it: should fail when AutoscalingMetric API is unavailable + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorPattern: "api\\.autoscaling requires the AutoscalingMetric API \\(autoscaling\\.gke\\.io/v1beta1\\)" diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml new file mode 100644 index 0000000..01dc574 --- /dev/null +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -0,0 +1,574 @@ +suite: test API autoscaling templates +templates: + - api-autoscaling-metric.yaml + - api-hpa.yaml + - api-autoscaling-prometheus.yaml + - api-autoscaling-prometheus-adapter.yaml + - api-deployment.yaml + - api-configmap.yaml +capabilities: + apiVersions: + - autoscaling.gke.io/v1beta1 +tests: + - it: should not render AutoscalingMetric when autoscaling is disabled + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should not render HPA when autoscaling is disabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should fail when autoscaling is enabled on an unsupported cloud + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" + + - it: should render AutoscalingMetric for GKE with event-loop gauges + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: AutoscalingMetric + - equal: + path: apiVersion + value: autoscaling.gke.io/v1beta1 + - equal: + path: metadata.name + value: braintrust-api + - equal: + path: metadata.namespace + value: braintrust + - equal: + path: spec.metrics[0].pod.selector.matchLabels.app + value: braintrust-api + - equal: + path: spec.metrics[0].pod.containers[0].endpoint.port + value: 8001 + - equal: + path: spec.metrics[0].pod.containers[0].endpoint.path + value: /metrics + - equal: + path: spec.metrics[0].pod.containers[0].metrics[0].gauge.name + value: braintrust-api-event-loop-utilization-ratio + - equal: + path: spec.metrics[0].pod.containers[0].metrics[0].gauge.prometheusMetricName + value: braintrust_api_event_loop_utilization_ratio + - equal: + path: spec.metrics[0].pod.containers[0].metrics[1].gauge.name + value: braintrust-api-event-loop-delay-mean-seconds + - equal: + path: spec.metrics[0].pod.containers[0].metrics[1].gauge.prometheusMetricName + value: braintrust_api_event_loop_delay_mean_seconds + + - it: should render HPA with CPU and event-loop metrics matching AWS ECS defaults + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: apiVersion + value: autoscaling/v2 + - equal: + path: metadata.name + value: braintrust-api + - equal: + path: spec.scaleTargetRef.name + value: braintrust-api + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 50 + - equal: + path: spec.metrics[0].type + value: ContainerResource + - equal: + path: spec.metrics[0].containerResource.name + value: cpu + - equal: + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[0].containerResource.target.averageUtilization + value: 50 + - equal: + path: spec.metrics[1].pods.metric.name + value: autoscaling.gke.io|braintrust-api|braintrust-api-event-loop-utilization-ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: autoscaling.gke.io|braintrust-api|braintrust-api-event-loop-delay-mean-seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + - lengthEqual: + path: spec.metrics + count: 3 + - equal: + path: spec.behavior.scaleDown.stabilizationWindowSeconds + value: 300 + - equal: + path: spec.behavior.scaleUp.stabilizationWindowSeconds + value: 60 + + - it: should omit Deployment replicas and expose metrics port when autoscaling is enabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.replicas: 4 + release: + namespace: "braintrust" + asserts: + - isNull: + path: spec.replicas + - contains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8000 + - contains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8001 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should set ENABLE_PROMETHEUS_METRICS after extraEnvVars so autoscaling wins + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.extraEnvVars: + - name: ENABLE_PROMETHEUS_METRICS + value: "false" + - name: SOME_OTHER_VAR + value: "x" + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.template.spec.containers[0].env[-1].name + value: ENABLE_PROMETHEUS_METRICS + - equal: + path: spec.template.spec.containers[0].env[-1].value + value: "true" + + - it: should keep Deployment replicas when autoscaling is disabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + api.replicas: 4 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.replicas + value: 4 + - notContains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8001 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should render three HPAs when workload isolation and autoscaling are enabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 3 + + - it: should scale the default pool with inherited autoscaling capacity + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 50 + - equal: + path: metadata.labels["braintrust.dev/api-pool"] + value: default + + - it: should scale the ingest pool with pool-specific capacity overrides + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + asserts: + - equal: + path: metadata.labels["braintrust.dev/api-pool"] + value: ingest + - equal: + path: spec.minReplicas + value: 3 + - equal: + path: spec.maxReplicas + value: 200 + - equal: + path: spec.metrics[1].pods.metric.name + value: autoscaling.gke.io|braintrust-api-ingest|braintrust-api-event-loop-utilization-ratio + + - it: should scale the background pool with pool-specific capacity overrides + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-background + asserts: + - equal: + path: metadata.labels["braintrust.dev/api-pool"] + value: background + - equal: + path: spec.minReplicas + value: 3 + - equal: + path: spec.maxReplicas + value: 50 + + - it: should scrape the ingest pool with a dedicated AutoscalingMetric + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + asserts: + - equal: + path: metadata.labels["braintrust.dev/api-pool"] + value: ingest + - equal: + path: spec.metrics[0].pod.selector.matchLabels.app + value: braintrust-api-ingest + + - it: should omit replicas on the ingest pool when autoscaling is enabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + asserts: + - isNull: + path: spec.replicas + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should not render an HPA for a pool with autoscaling disabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + filterAware: true + + - it: should still scale sibling pools when one pool disables autoscaling + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 2 + + - it: should keep fixed replicas on a pool with autoscaling disabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + api.workloadIsolation.ingest.replicas: 3 + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + asserts: + - equal: + path: spec.replicas + value: 3 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should not scrape a pool with autoscaling disabled + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + filterAware: true + + - it: should not render GKE AutoscalingMetric on AWS + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should not render Prometheus stack on GKE + template: api-autoscaling-prometheus.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should render HPA on AWS with prometheus-adapter metric names + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: spec.metrics[0].type + value: ContainerResource + - equal: + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[1].pods.metric.name + value: event_loop_utilization_ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: event_loop_delay_mean_seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + + - it: should scrape health server metrics on AWS Prometheus + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: ":8001" + - matchRegex: + path: data["prometheus.yml"] + pattern: "metrics_path: /metrics" + - matchRegex: + path: data["prometheus.yml"] + pattern: "regex: braintrust-api" + + - it: should scrape all API pools on AWS when workload isolation is enabled + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-ingest" + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-background" + + - it: should render three HPAs on AWS when workload isolation is enabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 3 + + - it: should render prometheus-adapter rules for event-loop gauges on AWS + template: api-autoscaling-prometheus-adapter.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_delay_mean_seconds" + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_utilization_ratio" diff --git a/braintrust/tests/labels-merge-autoscaling_test.yaml b/braintrust/tests/labels-merge-autoscaling_test.yaml new file mode 100644 index 0000000..a04fb5c --- /dev/null +++ b/braintrust/tests/labels-merge-autoscaling_test.yaml @@ -0,0 +1,86 @@ +suite: test label merge isolation for API autoscaling resources +templates: + - api-autoscaling-metric.yaml + - api-hpa.yaml + - brainstore-reader-configmap.yaml +capabilities: + apiVersions: + - autoscaling.gke.io/v1beta1 +tests: + # New templates that merge labels must prove that component labels remain + # isolated when Helm renders them alongside other chart resources. + - it: should give AutoscalingMetric only global and API-specific labels + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + api.labels.tier: api-tier + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: api-tier + - equal: + path: metadata.labels["api-specific"] + value: api-autoscaling + - isNull: + path: metadata.labels["reader-specific"] + + - it: should give HPA only global and API-specific labels + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + api.labels.tier: api-tier + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: api-tier + - equal: + path: metadata.labels["api-specific"] + value: api-autoscaling + - isNull: + path: metadata.labels["reader-specific"] + + - it: should not leak API labels into another rendered component + template: brainstore-reader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels["reader-specific"] + value: reader-config + - isNull: + path: metadata.labels["api-specific"] diff --git a/braintrust/values.yaml b/braintrust/values.yaml index fe14a1a..c4e6e4d 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -110,7 +110,68 @@ api: service: {} pod: {} serviceaccount: {} + hpa: {} + autoscalingMetric: {} replicas: 4 + # Autoscale API pools on CPU and Node.js event-loop metrics. + # Applies to every rendered API pool (default, and ingest/background when + # workloadIsolation is enabled). Pool overrides can set minReplicas/maxReplicas + # (or disable) under api.workloadIsolation..autoscaling. + # cloud=google (GKE): AutoscalingMetric CRD + HPA. Preview (Pre-GA); requires: + # - Braintrust API / data plane v2.9.0+ (Prometheus /metrics on the API health server) + # - GKE 1.35.1-gke.1396000 or later + # - the Performance HPA profile and Autoscaling API + # - roles/autoscaling.metricsWriter on all node service accounts + # - the Autoscaling API in the service perimeter when using VPC Service Controls + # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster. + # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling + # cloud=aws (EKS): in-chart Prometheus + prometheus-adapter + HPA (custom.metrics). + # When enabled for a pool, that pool's replicas setting is ignored. + autoscaling: + enabled: false + minReplicas: 4 + maxReplicas: 50 + # Prometheus metrics path on the API health server (api.healthServer.port). + metricsPath: /metrics + # HPA uses ContainerResource for the api container (sidecars excluded). + cpu: + targetAverageUtilization: 50 + # 0-1 ratio (0.4 = 40% event-loop utilization). + eventLoopUtilization: + targetAverageValue: "0.4" + # Seconds (0.05 = 50ms mean event-loop delay). + eventLoopDelayMean: + targetAverageValue: "0.05" + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 + # Used when cloud=aws (Prometheus + prometheus-adapter). + prometheus: + image: + repository: prom/prometheus + tag: v3.8.0 + pullPolicy: IfNotPresent + retention: 1h + scrapeInterval: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + adapter: + image: + repository: registry.k8s.io/prometheus-adapter/prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi # Optional fixed-capacity workload isolation. This creates dedicated API # Deployments and Services for ingestion and background work while retaining # api.name as the default pool for all remaining traffic. Public ingress must @@ -130,6 +191,10 @@ api: minAvailable: 1 topologySpread: enabled: true + # Inherited from api.autoscaling when enabled; override capacity per pool. + autoscaling: + minReplicas: 3 + maxReplicas: 200 background: name: "braintrust-api-background" replicas: 3 @@ -140,6 +205,9 @@ api: minAvailable: 1 topologySpread: enabled: true + autoscaling: + minReplicas: 3 + maxReplicas: 50 image: repository: public.ecr.aws/braintrust/standalone-api tag: v2.10.0