From d8ded2e79ae85eb7ee351e4c9393c9ed6ff40774 Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:51:01 -0700 Subject: [PATCH 01/12] Support autoscaling the API on GKE Adds optional Horizontal Pod Autoscaling on GKE using native AutoscalingMetric support. When turned on, the API scales on CPU, event-loop utilization, and mean event-loop delay (same signals we use on AWS ECS), and Prometheus metrics are enabled automatically. Requires GKE 1.35.1+. See https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling --- .../examples/google-autopilot-cel/values.yaml | 8 + .../examples/google-autopilot/values.yaml | 8 + .../templates/api-autoscaling-metric.yaml | 36 ++++ braintrust/templates/api-deployment.yaml | 9 + braintrust/templates/api-hpa.yaml | 50 +++++ braintrust/tests/api-autoscaling_test.yaml | 197 ++++++++++++++++++ .../tests/labels-merge-autoscaling_test.yaml | 83 ++++++++ braintrust/values.yaml | 29 +++ 8 files changed, 420 insertions(+) create mode 100644 braintrust/templates/api-autoscaling-metric.yaml create mode 100644 braintrust/templates/api-hpa.yaml create mode 100644 braintrust/tests/api-autoscaling_test.yaml create mode 100644 braintrust/tests/labels-merge-autoscaling_test.yaml diff --git a/braintrust/examples/google-autopilot-cel/values.yaml b/braintrust/examples/google-autopilot-cel/values.yaml index 2024a9d..4dc224d 100644 --- a/braintrust/examples/google-autopilot-cel/values.yaml +++ b/braintrust/examples/google-autopilot-cel/values.yaml @@ -25,6 +25,14 @@ api: service: networking.gke.io/load-balancer-type: "Internal" replicas: 4 + # Alternatively, autoscale the API on CPU and event-loop metrics. + # This is a Preview (Pre-GA) GKE feature requiring 1.35.1-gke.1396000 or later + # and the prerequisites documented in the main values.yaml. + # See api.autoscaling in the main values.yaml file for more details. + # autoscaling: + # enabled: true + # minReplicas: 3 + # maxReplicas: 50 service: type: LoadBalancer port: 8000 diff --git a/braintrust/examples/google-autopilot/values.yaml b/braintrust/examples/google-autopilot/values.yaml index dc9f482..4c7ffa5 100644 --- a/braintrust/examples/google-autopilot/values.yaml +++ b/braintrust/examples/google-autopilot/values.yaml @@ -33,6 +33,14 @@ api: service: networking.gke.io/load-balancer-type: "Internal" replicas: 4 + # Alternatively, autoscale the API on CPU and event-loop metrics. + # This is a Preview (Pre-GA) GKE feature requiring 1.35.1-gke.1396000 or later + # and the prerequisites documented in the main values.yaml. + # See api.autoscaling in the main values.yaml file for more details. + # autoscaling: + # enabled: true + # minReplicas: 3 + # maxReplicas: 50 # Uncomment the following section to use a different image or tag from the version in the Helm release #image: #repository: public.ecr.aws/braintrust/standalone-api diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml new file mode 100644 index 0000000..b96c94c --- /dev/null +++ b/braintrust/templates/api-autoscaling-metric.yaml @@ -0,0 +1,36 @@ +{{- if .Values.api.autoscaling.enabled }} +{{- if ne .Values.cloud "google" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} +{{- end }} +apiVersion: autoscaling.gke.io/v1beta1 +kind: AutoscalingMetric +metadata: + name: {{ .Values.api.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (merge (deepCopy .Values.api.labels) .Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.api.annotations.autoscalingMetric }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + metrics: + - pod: + selector: + matchLabels: + app: {{ .Values.api.name }} + containers: + - endpoint: + port: {{ .Values.api.healthServer.port }} + path: {{ .Values.api.autoscaling.metricsPath }} + metrics: + - gauge: + # GKE gauge names must match ^[a-z]([-a-z0-9]*[a-z0-9])? + name: braintrust-api-event-loop-utilization-ratio + prometheusMetricName: braintrust_api_event_loop_utilization_ratio + - gauge: + name: braintrust-api-event-loop-delay-mean-seconds + prometheusMetricName: braintrust_api_event_loop_delay_mean_seconds +{{- end }} diff --git a/braintrust/templates/api-deployment.yaml b/braintrust/templates/api-deployment.yaml index fbbde8b..e336cb8 100644 --- a/braintrust/templates/api-deployment.yaml +++ b/braintrust/templates/api-deployment.yaml @@ -12,7 +12,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if not .Values.api.autoscaling.enabled }} replicas: {{ .Values.api.replicas }} + {{- end }} strategy: type: RollingUpdate rollingUpdate: @@ -70,6 +72,9 @@ spec: {{- end }} ports: - containerPort: {{ .Values.api.service.port }} + {{- if .Values.api.autoscaling.enabled }} + - containerPort: {{ .Values.api.healthServer.port }} + {{- end }} resources: {{- toYaml .Values.api.resources | nindent 12 }} {{- with .Values.api.livenessProbe }} @@ -127,6 +132,10 @@ spec: value: {{ .Values.api.healthServer.host | quote }} - name: TS_API_HEALTHSERVER_PORT value: {{ .Values.api.healthServer.port | quote }} + {{- if .Values.api.autoscaling.enabled }} + - name: ENABLE_PROMETHEUS_METRICS + value: "true" + {{- end }} {{- if .Values.api.extraEnvVars }} {{- toYaml .Values.api.extraEnvVars | nindent 12 }} {{- end }} diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml new file mode 100644 index 0000000..f58fb46 --- /dev/null +++ b/braintrust/templates/api-hpa.yaml @@ -0,0 +1,50 @@ +{{- if .Values.api.autoscaling.enabled }} +{{- if ne .Values.cloud "google" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} +{{- end }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.api.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (merge (deepCopy .Values.api.labels) .Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.api.annotations.hpa }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.api.name }} + minReplicas: {{ .Values.api.autoscaling.minReplicas }} + maxReplicas: {{ .Values.api.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.api.autoscaling.cpu.targetAverageUtilization }} + - type: Pods + pods: + metric: + name: autoscaling.gke.io|{{ .Values.api.name }}|braintrust-api-event-loop-utilization-ratio + target: + type: AverageValue + averageValue: {{ .Values.api.autoscaling.eventLoopUtilization.targetAverageValue | quote }} + - type: Pods + pods: + metric: + name: autoscaling.gke.io|{{ .Values.api.name }}|braintrust-api-event-loop-delay-mean-seconds + target: + type: AverageValue + averageValue: {{ .Values.api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- with .Values.api.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml new file mode 100644 index 0000000..c37988f --- /dev/null +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -0,0 +1,197 @@ +suite: test API autoscaling templates +templates: + - api-autoscaling-metric.yaml + - api-hpa.yaml + - api-deployment.yaml + - api-configmap.yaml +tests: + - it: should not render AutoscalingMetric when autoscaling is disabled + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should not render HPA when autoscaling is disabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should fail when autoscaling is enabled on a non-google cloud + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE)" + + - it: should render AutoscalingMetric for GKE with event-loop gauges + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: AutoscalingMetric + - equal: + path: apiVersion + value: autoscaling.gke.io/v1beta1 + - equal: + path: metadata.name + value: braintrust-api + - equal: + path: metadata.namespace + value: braintrust + - equal: + path: spec.metrics[0].pod.selector.matchLabels.app + value: braintrust-api + - equal: + path: spec.metrics[0].pod.containers[0].endpoint.port + value: 8001 + - equal: + path: spec.metrics[0].pod.containers[0].endpoint.path + value: /metrics + - equal: + path: spec.metrics[0].pod.containers[0].metrics[0].gauge.name + value: braintrust-api-event-loop-utilization-ratio + - equal: + path: spec.metrics[0].pod.containers[0].metrics[0].gauge.prometheusMetricName + value: braintrust_api_event_loop_utilization_ratio + - equal: + path: spec.metrics[0].pod.containers[0].metrics[1].gauge.name + value: braintrust-api-event-loop-delay-mean-seconds + - equal: + path: spec.metrics[0].pod.containers[0].metrics[1].gauge.prometheusMetricName + value: braintrust_api_event_loop_delay_mean_seconds + + - it: should render HPA with CPU and event-loop metrics matching AWS ECS defaults + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: apiVersion + value: autoscaling/v2 + - equal: + path: metadata.name + value: braintrust-api + - equal: + path: spec.scaleTargetRef.name + value: braintrust-api + - equal: + path: spec.minReplicas + value: 3 + - equal: + path: spec.maxReplicas + value: 50 + - equal: + path: spec.metrics[0].type + value: Resource + - equal: + path: spec.metrics[0].resource.name + value: cpu + - equal: + path: spec.metrics[0].resource.target.averageUtilization + value: 50 + - equal: + path: spec.metrics[1].pods.metric.name + value: autoscaling.gke.io|braintrust-api|braintrust-api-event-loop-utilization-ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: autoscaling.gke.io|braintrust-api|braintrust-api-event-loop-delay-mean-seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + - lengthEqual: + path: spec.metrics + count: 3 + - equal: + path: spec.behavior.scaleDown.stabilizationWindowSeconds + value: 300 + - equal: + path: spec.behavior.scaleUp.stabilizationWindowSeconds + value: 60 + + - it: should omit Deployment replicas and expose metrics port when autoscaling is enabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.replicas: 4 + release: + namespace: "braintrust" + asserts: + - isNull: + path: spec.replicas + - contains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8000 + - contains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8001 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should keep Deployment replicas when autoscaling is disabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: false + api.replicas: 4 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.replicas + value: 4 + - notContains: + path: spec.template.spec.containers[0].ports + content: + containerPort: 8001 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" diff --git a/braintrust/tests/labels-merge-autoscaling_test.yaml b/braintrust/tests/labels-merge-autoscaling_test.yaml new file mode 100644 index 0000000..677093b --- /dev/null +++ b/braintrust/tests/labels-merge-autoscaling_test.yaml @@ -0,0 +1,83 @@ +suite: test label merge isolation for API autoscaling resources +templates: + - api-autoscaling-metric.yaml + - api-hpa.yaml + - brainstore-reader-configmap.yaml +tests: + # New templates that merge labels must prove that component labels remain + # isolated when Helm renders them alongside other chart resources. + - it: should give AutoscalingMetric only global and API-specific labels + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + api.labels.tier: api-tier + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: api-tier + - equal: + path: metadata.labels["api-specific"] + value: api-autoscaling + - isNull: + path: metadata.labels["reader-specific"] + + - it: should give HPA only global and API-specific labels + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + api.labels.tier: api-tier + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: api-tier + - equal: + path: metadata.labels["api-specific"] + value: api-autoscaling + - isNull: + path: metadata.labels["reader-specific"] + + - it: should not leak API labels into another rendered component + template: brainstore-reader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + global.labels.shared: global-autoscaling + api.labels.api-specific: api-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels["reader-specific"] + value: reader-config + - isNull: + path: metadata.labels["api-specific"] diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 1ba5758..a898893 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -110,7 +110,36 @@ api: service: {} pod: {} serviceaccount: {} + hpa: {} + autoscalingMetric: {} replicas: 4 + # Autoscale the API on CPU and Node.js event-loop metrics via AutoscalingMetric + HPA. + # This GKE feature is Preview (Pre-GA). It requires: + # - GKE 1.35.1-gke.1396000 or later + # - the Performance HPA profile and Autoscaling API + # - roles/autoscaling.metricsWriter on all node service accounts + # - the Autoscaling API in the service perimeter when using VPC Service Controls + # When enabled, replicas above is ignored. + # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling + autoscaling: + enabled: false + minReplicas: 3 + maxReplicas: 50 + # Prometheus metrics path on the API health server (api.healthServer.port). + metricsPath: /metrics + cpu: + targetAverageUtilization: 50 + # 0-1 ratio (0.4 = 40% event-loop utilization). + eventLoopUtilization: + targetAverageValue: "0.4" + # Seconds (0.05 = 50ms mean event-loop delay). + eventLoopDelayMean: + targetAverageValue: "0.05" + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 image: repository: public.ecr.aws/braintrust/standalone-api tag: v2.7.1 From f592b565b8c1a2b6f2da4816545a566bca72c24a Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:47:05 -0700 Subject: [PATCH 02/12] Inline docs for /metrics being added in v2.9.0 --- braintrust/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 72eacc7..3d11650 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -114,7 +114,7 @@ api: autoscalingMetric: {} replicas: 4 # Autoscale the API on CPU and Node.js event-loop metrics via AutoscalingMetric + HPA. - # This GKE feature is Preview (Pre-GA). It requires: + # The GKE feature for custom autoscaling metrics is in Preview (Pre-GA). It requires: # - GKE 1.35.1-gke.1396000 or later # - the Performance HPA profile and Autoscaling API # - roles/autoscaling.metricsWriter on all node service accounts @@ -126,6 +126,7 @@ api: minReplicas: 3 maxReplicas: 50 # Prometheus metrics path on the API health server (api.healthServer.port). + # This endpoint was introduced in v2.9.0+ of the Braintrust data plane. metricsPath: /metrics cpu: targetAverageUtilization: 50 From 44582852e355d105c046ee01f9e79f65cfdf2754 Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:04:13 -0700 Subject: [PATCH 03/12] Add gke autoscaling docs to README --- braintrust/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/braintrust/README.md b/braintrust/README.md index 4e0395e..d220bcc 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -217,6 +217,31 @@ Size the request for the pod's full local-storage usage: When you enable `tmpVolume`, make sure the `ephemeralStorage.request` still covers that extra space. +## GKE API Autoscaling + +The API can autoscale on GKE using a Horizontal Pod Autoscaler backed by GKE's native `AutoscalingMetric` resource. When enabled, the API scales on three signals - CPU, Node.js event-loop utilization, and mean event-loop delay. + +This is underpinned by a **Preview (Pre-GA)** GKE feature. It requires: + +- GKE **1.35.1-gke.1396000** or later +- The Performance HPA profile and the Autoscaling API enabled on the cluster +- `roles/autoscaling.metricsWriter` granted to all node service accounts +- The Autoscaling API included in your service perimeter when using VPC Service Controls + +See [Expose custom metrics for autoscaling](https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling) for more details on `AutoscalingMetric` in GKE. + +Enable it in your values: + +```yaml +api: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 50 +``` + +When enabled, `api.replicas` is ignored and the HPA controls the replica count. + ## Testing This Helm chart includes comprehensive automated unit tests. From fa1340748aa9333d1e2f20ef9218330af0cd4d9d Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:16:51 -0700 Subject: [PATCH 04/12] Force NABLE_PROMETHEUS_METRICS=true if autoscaled --- braintrust/templates/api-deployment.yaml | 6 +++--- braintrust/tests/api-autoscaling_test.yaml | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/braintrust/templates/api-deployment.yaml b/braintrust/templates/api-deployment.yaml index 20d3c78..ae3e6e5 100644 --- a/braintrust/templates/api-deployment.yaml +++ b/braintrust/templates/api-deployment.yaml @@ -158,13 +158,13 @@ spec: - name: PIP_CERT value: {{ $customCAPath | quote }} {{- end }} + {{- if .Values.api.extraEnvVars }} + {{- toYaml .Values.api.extraEnvVars | nindent 12 }} + {{- end }} {{- if .Values.api.autoscaling.enabled }} - name: ENABLE_PROMETHEUS_METRICS value: "true" {{- end }} - {{- if .Values.api.extraEnvVars }} - {{- toYaml .Values.api.extraEnvVars | nindent 12 }} - {{- end }} {{- if or .Values.api.tmpVolume.enabled (and (eq .Values.cloud "azure") .Values.azure.enableAzureKeyVaultDriver) $customCA.enabled }} volumeMounts: {{- if .Values.api.tmpVolume.enabled }} diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index c37988f..a11e94b 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -172,6 +172,28 @@ tests: name: ENABLE_PROMETHEUS_METRICS value: "true" + - it: should set ENABLE_PROMETHEUS_METRICS after extraEnvVars so autoscaling wins + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.extraEnvVars: + - name: ENABLE_PROMETHEUS_METRICS + value: "false" + - name: SOME_OTHER_VAR + value: "x" + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.template.spec.containers[0].env[-1].name + value: ENABLE_PROMETHEUS_METRICS + - equal: + path: spec.template.spec.containers[0].env[-1].value + value: "true" + - it: should keep Deployment replicas when autoscaling is disabled template: api-deployment.yaml values: From 86f0fae194255af2085ed20c1ce74abe7aaa2547 Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:21:17 -0700 Subject: [PATCH 05/12] bump minReplicas at 4 when autoscaling enabled --- braintrust/README.md | 2 +- braintrust/examples/google-autopilot-cel/values.yaml | 2 +- braintrust/examples/google-autopilot/values.yaml | 2 +- braintrust/tests/api-autoscaling_test.yaml | 2 +- braintrust/values.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index d220bcc..071ab15 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -236,7 +236,7 @@ Enable it in your values: api: autoscaling: enabled: true - minReplicas: 3 + minReplicas: 4 maxReplicas: 50 ``` diff --git a/braintrust/examples/google-autopilot-cel/values.yaml b/braintrust/examples/google-autopilot-cel/values.yaml index 4dc224d..380184e 100644 --- a/braintrust/examples/google-autopilot-cel/values.yaml +++ b/braintrust/examples/google-autopilot-cel/values.yaml @@ -31,7 +31,7 @@ api: # See api.autoscaling in the main values.yaml file for more details. # autoscaling: # enabled: true - # minReplicas: 3 + # minReplicas: 4 # maxReplicas: 50 service: type: LoadBalancer diff --git a/braintrust/examples/google-autopilot/values.yaml b/braintrust/examples/google-autopilot/values.yaml index 4c7ffa5..9375209 100644 --- a/braintrust/examples/google-autopilot/values.yaml +++ b/braintrust/examples/google-autopilot/values.yaml @@ -39,7 +39,7 @@ api: # See api.autoscaling in the main values.yaml file for more details. # autoscaling: # enabled: true - # minReplicas: 3 + # minReplicas: 4 # maxReplicas: 50 # Uncomment the following section to use a different image or tag from the version in the Helm release #image: diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index a11e94b..7f27a68 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -110,7 +110,7 @@ tests: value: braintrust-api - equal: path: spec.minReplicas - value: 3 + value: 4 - equal: path: spec.maxReplicas value: 50 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 3d11650..f6288e7 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -123,7 +123,7 @@ api: # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling autoscaling: enabled: false - minReplicas: 3 + minReplicas: 4 maxReplicas: 50 # Prometheus metrics path on the API health server (api.healthServer.port). # This endpoint was introduced in v2.9.0+ of the Braintrust data plane. From 7acb8189241db452290b1a253744fe5958947f6a Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:24:41 -0700 Subject: [PATCH 06/12] Ensure the autoscaling.gke.io/v1beta1 CRD exists --- braintrust/templates/_helpers.tpl | 12 ++++++++++++ .../templates/api-autoscaling-metric.yaml | 4 +--- braintrust/templates/api-hpa.yaml | 4 +--- braintrust/tests/api-autoscaling-crd_test.yaml | 17 +++++++++++++++++ braintrust/tests/api-autoscaling_test.yaml | 3 +++ .../tests/labels-merge-autoscaling_test.yaml | 3 +++ braintrust/values.yaml | 1 + 7 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 braintrust/tests/api-autoscaling-crd_test.yaml diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index d27e28f..2b91a98 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -39,6 +39,18 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} +{{/* +Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). +*/}} +{{- define "braintrust.apiAutoscaling.validate" -}} +{{- if ne .Values.cloud "google" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} +{{- end }} +{{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }} +{{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }} +{{- end }} +{{- end -}} + {{/* Render Brainstore container resources with provider-specific ephemeral storage. diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml index b96c94c..8870b83 100644 --- a/braintrust/templates/api-autoscaling-metric.yaml +++ b/braintrust/templates/api-autoscaling-metric.yaml @@ -1,7 +1,5 @@ {{- if .Values.api.autoscaling.enabled }} -{{- if ne .Values.cloud "google" }} -{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} -{{- end }} +{{- include "braintrust.apiAutoscaling.validate" . }} apiVersion: autoscaling.gke.io/v1beta1 kind: AutoscalingMetric metadata: diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml index f58fb46..e6ffaab 100644 --- a/braintrust/templates/api-hpa.yaml +++ b/braintrust/templates/api-hpa.yaml @@ -1,7 +1,5 @@ {{- if .Values.api.autoscaling.enabled }} -{{- if ne .Values.cloud "google" }} -{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} -{{- end }} +{{- include "braintrust.apiAutoscaling.validate" . }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/braintrust/tests/api-autoscaling-crd_test.yaml b/braintrust/tests/api-autoscaling-crd_test.yaml new file mode 100644 index 0000000..ff7702b --- /dev/null +++ b/braintrust/tests/api-autoscaling-crd_test.yaml @@ -0,0 +1,17 @@ +suite: test API autoscaling CRD requirement +templates: + - api-hpa.yaml +# Do not advertise autoscaling.gke.io here — this suite asserts the fail path. +tests: + - it: should fail when AutoscalingMetric API is unavailable + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorPattern: "api\\.autoscaling requires the AutoscalingMetric API \\(autoscaling\\.gke\\.io/v1beta1\\)" diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index 7f27a68..c0c2f6e 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -4,6 +4,9 @@ templates: - api-hpa.yaml - api-deployment.yaml - api-configmap.yaml +capabilities: + apiVersions: + - autoscaling.gke.io/v1beta1 tests: - it: should not render AutoscalingMetric when autoscaling is disabled template: api-autoscaling-metric.yaml diff --git a/braintrust/tests/labels-merge-autoscaling_test.yaml b/braintrust/tests/labels-merge-autoscaling_test.yaml index 677093b..a04fb5c 100644 --- a/braintrust/tests/labels-merge-autoscaling_test.yaml +++ b/braintrust/tests/labels-merge-autoscaling_test.yaml @@ -3,6 +3,9 @@ templates: - api-autoscaling-metric.yaml - api-hpa.yaml - brainstore-reader-configmap.yaml +capabilities: + apiVersions: + - autoscaling.gke.io/v1beta1 tests: # New templates that merge labels must prove that component labels remain # isolated when Helm renders them alongside other chart resources. diff --git a/braintrust/values.yaml b/braintrust/values.yaml index f6288e7..1b8d65f 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -119,6 +119,7 @@ api: # - the Performance HPA profile and Autoscaling API # - roles/autoscaling.metricsWriter on all node service accounts # - the Autoscaling API in the service perimeter when using VPC Service Controls + # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster. # When enabled, replicas above is ignored. # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling autoscaling: From a961e6704668e09ce7ff0949a6df11a4cfb3ef90 Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:30:10 -0700 Subject: [PATCH 07/12] Closer align the autoscaling max sizes to ECS --- braintrust/tests/api-autoscaling_test.yaml | 2 +- braintrust/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index c486064..a998f53 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -283,7 +283,7 @@ tests: value: 3 - equal: path: spec.maxReplicas - value: 100 + value: 200 - equal: path: spec.metrics[1].pods.metric.name value: autoscaling.gke.io|braintrust-api-ingest|braintrust-api-event-loop-utilization-ratio diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 0c0e496..a068b6e 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -167,7 +167,7 @@ api: # Inherited from api.autoscaling when enabled; override capacity per pool. autoscaling: minReplicas: 3 - maxReplicas: 100 + maxReplicas: 200 background: name: "braintrust-api-background" replicas: 3 From 71da05741220f9effc402f8dec5fa55cc87d30df Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:02:16 -0700 Subject: [PATCH 08/12] Add example values for autoscaling w/ dedicated api pools --- braintrust/README.md | 1 + .../values.yaml | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 braintrust/examples/google-api-isolation-autoscaling/values.yaml diff --git a/braintrust/README.md b/braintrust/README.md index ddce7c9..1f8fb2e 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -336,4 +336,5 @@ Example values files for different cloud providers and configurations are locate - `examples/google-autopilot/values.yaml`: GKE Autopilot deployment. - `examples/google-autopilot-cel/values.yaml`: GKE Autopilot deployment with CEL-friendly security settings. +- `examples/google-api-isolation-autoscaling/values.yaml`: Minimal example for API workload isolation and per-pool autoscaling on GKE (combine with an Autopilot or Standard values file). - `examples/google-standard/values.yaml`: GKE Standard deployment. diff --git a/braintrust/examples/google-api-isolation-autoscaling/values.yaml b/braintrust/examples/google-api-isolation-autoscaling/values.yaml new file mode 100644 index 0000000..43286e1 --- /dev/null +++ b/braintrust/examples/google-api-isolation-autoscaling/values.yaml @@ -0,0 +1,48 @@ +# Minimal GKE example with API workload isolation + per-pool autoscaling. +# +# Combine this with a normal google-autopilot (or google-standard) values +# file - only the fields below are specific to this pattern. +# +# Sizing: +# - api.autoscaling is shared by every pool +# - api.workloadIsolation..autoscaling deep-merges on top +# - With autoscaling enabled, set capacity via minReplicas/maxReplicas +# (Helm `replicas` is not applied to those Deployments) +# +# Also requires GKE 1.35.1+ with AutoscalingMetric, and ingress routes for the +# isolation contract (see README & files/contracts/api-workload-isolation-routes.yaml). + +api: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 40 + # Metric targets (inherited by every pool unless overridden below). + cpu: + targetAverageUtilization: 50 + eventLoopUtilization: + targetAverageValue: "0.4" # 0-1 ratio (0.4 is %40) + eventLoopDelayMean: + targetAverageValue: "0.05" # seconds (0.05 is 50ms) + # HPA scale velocity (inherited by every pool unless overridden below). + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 + workloadIsolation: + enabled: true + ingest: + autoscaling: + minReplicas: 3 + maxReplicas: 60 + # Example pool-specific overrides: higher ELU target, faster scale-out. + eventLoopUtilization: + targetAverageValue: "0.5" # 0-1 ratio (0.5 is %50) + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + background: + autoscaling: + minReplicas: 3 + maxReplicas: 50 From 8405cdef78aeb2979a0d9d5b28db7e299d891ee1 Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:12:11 -0700 Subject: [PATCH 09/12] formatting --- .../examples/google-api-isolation-autoscaling/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/braintrust/examples/google-api-isolation-autoscaling/values.yaml b/braintrust/examples/google-api-isolation-autoscaling/values.yaml index 43286e1..7f98223 100644 --- a/braintrust/examples/google-api-isolation-autoscaling/values.yaml +++ b/braintrust/examples/google-api-isolation-autoscaling/values.yaml @@ -21,7 +21,7 @@ api: cpu: targetAverageUtilization: 50 eventLoopUtilization: - targetAverageValue: "0.4" # 0-1 ratio (0.4 is %40) + targetAverageValue: "0.4" # 0-1 ratio (0.4 is 40%) eventLoopDelayMean: targetAverageValue: "0.05" # seconds (0.05 is 50ms) # HPA scale velocity (inherited by every pool unless overridden below). @@ -38,7 +38,7 @@ api: maxReplicas: 60 # Example pool-specific overrides: higher ELU target, faster scale-out. eventLoopUtilization: - targetAverageValue: "0.5" # 0-1 ratio (0.5 is %50) + targetAverageValue: "0.5" # 0-1 ratio (0.5 is 50%) behavior: scaleUp: stabilizationWindowSeconds: 30 From d45fb9bc71d22742ac8915e7f8a1c92d4b3ec82f Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:16:38 -0700 Subject: [PATCH 10/12] Expand tests and stop using brittle document indexes --- braintrust/tests/api-autoscaling_test.yaml | 115 +++++++++++++++++---- 1 file changed, 95 insertions(+), 20 deletions(-) diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index a998f53..af99cc6 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -245,11 +245,10 @@ tests: api.workloadIsolation.enabled: true release: namespace: "braintrust" - documentIndex: 0 + documentSelector: + path: metadata.name + value: braintrust-api asserts: - - equal: - path: metadata.name - value: braintrust-api - equal: path: spec.minReplicas value: 4 @@ -270,11 +269,10 @@ tests: api.workloadIsolation.enabled: true release: namespace: "braintrust" - documentIndex: 1 + documentSelector: + path: metadata.name + value: braintrust-api-ingest asserts: - - equal: - path: metadata.name - value: braintrust-api-ingest - equal: path: metadata.labels["braintrust.dev/api-pool"] value: ingest @@ -298,11 +296,10 @@ tests: api.workloadIsolation.enabled: true release: namespace: "braintrust" - documentIndex: 2 + documentSelector: + path: metadata.name + value: braintrust-api-background asserts: - - equal: - path: metadata.name - value: braintrust-api-background - equal: path: metadata.labels["braintrust.dev/api-pool"] value: background @@ -323,11 +320,10 @@ tests: api.workloadIsolation.enabled: true release: namespace: "braintrust" - documentIndex: 1 + documentSelector: + path: metadata.name + value: braintrust-api-ingest asserts: - - equal: - path: metadata.name - value: braintrust-api-ingest - equal: path: metadata.labels["braintrust.dev/api-pool"] value: ingest @@ -345,11 +341,10 @@ tests: api.workloadIsolation.enabled: true release: namespace: "braintrust" - documentIndex: 1 + documentSelector: + path: metadata.name + value: braintrust-api-ingest asserts: - - equal: - path: metadata.name - value: braintrust-api-ingest - isNull: path: spec.replicas - contains: @@ -357,3 +352,83 @@ tests: content: name: ENABLE_PROMETHEUS_METRICS value: "true" + + - it: should not render an HPA for a pool with autoscaling disabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + filterAware: true + + - it: should still scale sibling pools when one pool disables autoscaling + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 2 + + - it: should keep fixed replicas on a pool with autoscaling disabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + api.workloadIsolation.ingest.replicas: 3 + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + asserts: + - equal: + path: spec.replicas + value: 3 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ENABLE_PROMETHEUS_METRICS + value: "true" + + - it: should not scrape a pool with autoscaling disabled + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.autoscaling.enabled: false + release: + namespace: "braintrust" + documentSelector: + path: metadata.name + value: braintrust-api-ingest + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + filterAware: true From 6148261e459c6a81738318e0ecef8bf47848815f Mon Sep 17 00:00:00 2001 From: brianvans <1323225+brianvans@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:14:10 -0700 Subject: [PATCH 11/12] Narrow cpu scaling to just api container w/ ContainerResource Better document hard requirement on v2.9.0 for /metrics endpoint --- braintrust/README.md | 3 ++- braintrust/templates/api-hpa.yaml | 7 +++++-- braintrust/tests/api-autoscaling_test.yaml | 9 ++++++--- braintrust/values.yaml | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 1f8fb2e..dd2baac 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -219,10 +219,11 @@ When you enable `tmpVolume`, make sure the `ephemeralStorage.request` still cove ## GKE API Autoscaling -The API can autoscale on GKE using a Horizontal Pod Autoscaler backed by GKE's native `AutoscalingMetric` resource. When enabled, each API pool scales on three signals - CPU, Node.js event-loop utilization, and mean event-loop delay. +The API can autoscale on GKE using a Horizontal Pod Autoscaler backed by GKE's native `AutoscalingMetric` resource. When enabled, each API pool scales on three signals - CPU (scoped to the `api` container via `ContainerResource`, so sidecars are excluded), Node.js event-loop utilization, and mean event-loop delay. This is underpinned by a **Preview (Pre-GA)** GKE feature. It requires: +- Braintrust API / data plane **v2.9.0** or later (Prometheus `/metrics` on the API health server) - GKE **1.35.1-gke.1396000** or later - The Performance HPA profile and the Autoscaling API enabled on the cluster - `roles/autoscaling.metricsWriter` granted to all node service accounts diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml index eb167db..4b2d97e 100644 --- a/braintrust/templates/api-hpa.yaml +++ b/braintrust/templates/api-hpa.yaml @@ -38,9 +38,12 @@ spec: minReplicas: {{ $api.autoscaling.minReplicas }} maxReplicas: {{ $api.autoscaling.maxReplicas }} metrics: - - type: Resource - resource: + # ContainerResource scopes CPU to the api container so sidecars / extraContainers + # do not skew utilization. Custom Pods metrics are already API-scoped via /metrics. + - type: ContainerResource + containerResource: name: cpu + container: api target: type: Utilization averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }} diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index af99cc6..cb22f4c 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -119,12 +119,15 @@ tests: value: 50 - equal: path: spec.metrics[0].type - value: Resource + value: ContainerResource - equal: - path: spec.metrics[0].resource.name + path: spec.metrics[0].containerResource.name value: cpu - equal: - path: spec.metrics[0].resource.target.averageUtilization + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[0].containerResource.target.averageUtilization value: 50 - equal: path: spec.metrics[1].pods.metric.name diff --git a/braintrust/values.yaml b/braintrust/values.yaml index a068b6e..7cc5aab 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -118,6 +118,7 @@ api: # workloadIsolation is enabled). Pool overrides can set minReplicas/maxReplicas # (or disable) under api.workloadIsolation..autoscaling. # The GKE feature for custom autoscaling metrics is in Preview (Pre-GA). It requires: + # - Braintrust API / data plane v2.9.0+ (Prometheus /metrics on the API health server) # - GKE 1.35.1-gke.1396000 or later # - the Performance HPA profile and Autoscaling API # - roles/autoscaling.metricsWriter on all node service accounts @@ -130,8 +131,8 @@ api: minReplicas: 4 maxReplicas: 50 # Prometheus metrics path on the API health server (api.healthServer.port). - # This endpoint was introduced in v2.9.0+ of the Braintrust data plane. metricsPath: /metrics + # HPA uses ContainerResource for the api container (sidecars excluded). cpu: targetAverageUtilization: 50 # 0-1 ratio (0.4 = 40% event-loop utilization). From c0f2ea02895a3aaefc41402098e67adef8c21cf5 Mon Sep 17 00:00:00 2001 From: Erik Weathers <1111441+erikdw@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:37:34 -0700 Subject: [PATCH 12/12] Add AWS EKS autoscaling via Prometheus and prometheus-adapter. Rebase onto Brian's pool-based GKE autoscaling and ContainerResource CPU targeting. Scrapes all API pools and feeds per-pool HPAs through custom.metrics with the same event-loop targets. Co-authored-by: Cursor --- braintrust/README.md | 6 + braintrust/templates/_helpers.tpl | 11 +- .../templates/api-autoscaling-metric.yaml | 2 + .../api-autoscaling-prometheus-adapter.yaml | 159 ++++++++++++++++++ .../templates/api-autoscaling-prometheus.yaml | 123 ++++++++++++++ braintrust/templates/api-hpa.yaml | 17 ++ braintrust/tests/api-autoscaling_test.yaml | 143 +++++++++++++++- braintrust/values.yaml | 32 +++- 8 files changed, 483 insertions(+), 10 deletions(-) create mode 100644 braintrust/templates/api-autoscaling-prometheus-adapter.yaml create mode 100644 braintrust/templates/api-autoscaling-prometheus.yaml diff --git a/braintrust/README.md b/braintrust/README.md index dd2baac..d787679 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -243,6 +243,12 @@ api: When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. +## EKS API Autoscaling + +On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`). + +Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background). + ## API workload isolation `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest` diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 31e0c0a..e7634f2 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -116,15 +116,18 @@ http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ {{- end -}} {{/* -Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). +Validate API autoscaling prerequisites. +GKE requires AutoscalingMetric (autoscaling.gke.io/v1beta1). +EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD). */}} {{- define "braintrust.apiAutoscaling.validate" -}} -{{- if ne .Values.cloud "google" }} -{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} -{{- end }} +{{- if eq .Values.cloud "google" }} {{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }} {{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }} {{- end }} +{{- else if ne .Values.cloud "aws" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }} +{{- end }} {{- end -}} {{/* diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml index f04245f..3600dc4 100644 --- a/braintrust/templates/api-autoscaling-metric.yaml +++ b/braintrust/templates/api-autoscaling-metric.yaml @@ -1,4 +1,5 @@ {{- $root := . -}} +{{- if eq $root.Values.cloud "google" -}} {{- $pools := include "braintrust.apiPools" . | fromYamlArray -}} {{- $validated := false -}} {{- $rendered := 0 -}} @@ -50,4 +51,5 @@ spec: prometheusMetricName: braintrust_api_event_loop_delay_mean_seconds {{- $rendered = add1 $rendered -}} {{- end -}} +{{- end -}} {{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml new file mode 100644 index 0000000..ca2d137 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml @@ -0,0 +1,159 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +rules: + - apiGroups: [""] + resources: ["namespaces", "pods", "services"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +data: + config.yaml: | + rules: + - seriesQuery: 'braintrust_api_event_loop_delay_mean_seconds{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: 'braintrust_api_event_loop_utilization_ratio{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + ports: + - name: https + port: 443 + targetPort: 6443 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} + port: 443 + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus-adapter + containers: + - name: prometheus-adapter + image: "{{ .Values.api.autoscaling.adapter.image.repository }}:{{ .Values.api.autoscaling.adapter.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.adapter.image.pullPolicy }} + args: + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --prometheus-url=http://{{ .Values.api.name }}-autoscaling-prometheus.{{ include "braintrust.namespace" . }}.svc:9090 + - --metrics-relist-interval={{ .Values.api.autoscaling.prometheus.scrapeInterval }} + - --config=/etc/adapter/config.yaml + - --v=2 + ports: + - containerPort: 6443 + name: https + resources: + {{- toYaml .Values.api.autoscaling.adapter.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/adapter + - name: tmp + mountPath: /tmp/cert + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + - name: tmp + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml new file mode 100644 index 0000000..be19129 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus.yaml @@ -0,0 +1,123 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +{{- $scrapeApps := list .Values.api.name -}} +{{- if .Values.api.workloadIsolation.enabled -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.background.name -}} +{{- end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +data: + prometheus.yml: | + global: + scrape_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + evaluation_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + + scrape_configs: + - job_name: braintrust-api-autoscaling + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - {{ include "braintrust.namespace" . }} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + regex: {{ join "|" $scrapeApps }} + action: keep + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: $1:{{ .Values.api.healthServer.port }} + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + metrics_path: {{ .Values.api.autoscaling.metricsPath }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus + ports: + - name: http + port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus + containers: + - name: prometheus + image: "{{ .Values.api.autoscaling.prometheus.image.repository }}:{{ .Values.api.autoscaling.prometheus.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.prometheus.image.pullPolicy }} + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ .Values.api.autoscaling.prometheus.retention }} + - --web.enable-lifecycle + ports: + - containerPort: 9090 + name: http + resources: + {{- toYaml .Values.api.autoscaling.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus + - name: data + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml index 4b2d97e..10991f1 100644 --- a/braintrust/templates/api-hpa.yaml +++ b/braintrust/templates/api-hpa.yaml @@ -47,6 +47,7 @@ spec: target: type: Utilization averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }} + {{- if eq $root.Values.cloud "google" }} - type: Pods pods: metric: @@ -61,6 +62,22 @@ spec: target: type: AverageValue averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- else if eq $root.Values.cloud "aws" }} + - type: Pods + pods: + metric: + name: event_loop_utilization_ratio + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }} + - type: Pods + pods: + metric: + name: event_loop_delay_mean_seconds + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- end }} {{- with $api.autoscaling.behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index cb22f4c..01dc574 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -2,6 +2,8 @@ suite: test API autoscaling templates templates: - api-autoscaling-metric.yaml - api-hpa.yaml + - api-autoscaling-prometheus.yaml + - api-autoscaling-prometheus-adapter.yaml - api-deployment.yaml - api-configmap.yaml capabilities: @@ -34,18 +36,18 @@ tests: - hasDocuments: count: 0 - - it: should fail when autoscaling is enabled on a non-google cloud + - it: should fail when autoscaling is enabled on an unsupported cloud template: api-hpa.yaml values: - __fixtures__/base-values.yaml - - __fixtures__/aws-values.yaml + - __fixtures__/azure-values.yaml set: api.autoscaling.enabled: true release: namespace: "braintrust" asserts: - failedTemplate: - errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE)" + errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" - it: should render AutoscalingMetric for GKE with event-loop gauges template: api-autoscaling-metric.yaml @@ -435,3 +437,138 @@ tests: - hasDocuments: count: 0 filterAware: true + + - it: should not render GKE AutoscalingMetric on AWS + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should not render Prometheus stack on GKE + template: api-autoscaling-prometheus.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should render HPA on AWS with prometheus-adapter metric names + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: spec.metrics[0].type + value: ContainerResource + - equal: + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[1].pods.metric.name + value: event_loop_utilization_ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: event_loop_delay_mean_seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + + - it: should scrape health server metrics on AWS Prometheus + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: ":8001" + - matchRegex: + path: data["prometheus.yml"] + pattern: "metrics_path: /metrics" + - matchRegex: + path: data["prometheus.yml"] + pattern: "regex: braintrust-api" + + - it: should scrape all API pools on AWS when workload isolation is enabled + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-ingest" + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-background" + + - it: should render three HPAs on AWS when workload isolation is enabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 3 + + - it: should render prometheus-adapter rules for event-loop gauges on AWS + template: api-autoscaling-prometheus-adapter.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_delay_mean_seconds" + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_utilization_ratio" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 7cc5aab..c4e6e4d 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -113,19 +113,20 @@ api: hpa: {} autoscalingMetric: {} replicas: 4 - # Autoscale API pools on CPU and Node.js event-loop metrics via AutoscalingMetric + HPA. + # Autoscale API pools on CPU and Node.js event-loop metrics. # Applies to every rendered API pool (default, and ingest/background when # workloadIsolation is enabled). Pool overrides can set minReplicas/maxReplicas # (or disable) under api.workloadIsolation..autoscaling. - # The GKE feature for custom autoscaling metrics is in Preview (Pre-GA). It requires: + # cloud=google (GKE): AutoscalingMetric CRD + HPA. Preview (Pre-GA); requires: # - Braintrust API / data plane v2.9.0+ (Prometheus /metrics on the API health server) # - GKE 1.35.1-gke.1396000 or later # - the Performance HPA profile and Autoscaling API # - roles/autoscaling.metricsWriter on all node service accounts # - the Autoscaling API in the service perimeter when using VPC Service Controls # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster. - # When enabled for a pool, that pool's replicas setting is ignored. # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling + # cloud=aws (EKS): in-chart Prometheus + prometheus-adapter + HPA (custom.metrics). + # When enabled for a pool, that pool's replicas setting is ignored. autoscaling: enabled: false minReplicas: 4 @@ -146,6 +147,31 @@ api: stabilizationWindowSeconds: 300 scaleUp: stabilizationWindowSeconds: 60 + # Used when cloud=aws (Prometheus + prometheus-adapter). + prometheus: + image: + repository: prom/prometheus + tag: v3.8.0 + pullPolicy: IfNotPresent + retention: 1h + scrapeInterval: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + adapter: + image: + repository: registry.k8s.io/prometheus-adapter/prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi # Optional fixed-capacity workload isolation. This creates dedicated API # Deployments and Services for ingestion and background work while retaining # api.name as the default pool for all remaining traffic. Public ingress must