diff --git a/braintrust/README.md b/braintrust/README.md index d787679..940d485 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -243,9 +243,9 @@ api: When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. -## EKS API Autoscaling +## EKS / AKS API Autoscaling -On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`). +On AWS (`cloud: aws`) or Azure (`cloud: azure`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`). Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background). diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index e7634f2..00ce232 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -125,8 +125,8 @@ EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD). {{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }} {{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }} {{- end }} -{{- else if ne .Values.cloud "aws" }} -{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }} +{{- else if and (ne .Values.cloud "aws") (ne .Values.cloud "azure") }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE), aws (EKS), or azure (AKS)" }} {{- end }} {{- end -}} diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml index ca2d137..8c7f4cb 100644 --- a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml +++ b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +{{- if and .Values.api.autoscaling.enabled (or (eq .Values.cloud "aws") (eq .Values.cloud "azure")) }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml index be19129..f40ce77 100644 --- a/braintrust/templates/api-autoscaling-prometheus.yaml +++ b/braintrust/templates/api-autoscaling-prometheus.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +{{- if and .Values.api.autoscaling.enabled (or (eq .Values.cloud "aws") (eq .Values.cloud "azure")) }} {{- $scrapeApps := list .Values.api.name -}} {{- if .Values.api.workloadIsolation.enabled -}} {{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}} diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml index 10991f1..c3f4f00 100644 --- a/braintrust/templates/api-hpa.yaml +++ b/braintrust/templates/api-hpa.yaml @@ -62,7 +62,7 @@ spec: target: type: AverageValue averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} - {{- else if eq $root.Values.cloud "aws" }} + {{- else if or (eq $root.Values.cloud "aws") (eq $root.Values.cloud "azure") }} - type: Pods pods: metric: diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index 01dc574..8033ed9 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -40,14 +40,14 @@ tests: template: api-hpa.yaml values: - __fixtures__/base-values.yaml - - __fixtures__/azure-values.yaml set: + cloud: unsupported api.autoscaling.enabled: true release: namespace: "braintrust" asserts: - failedTemplate: - errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" + errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE), aws (EKS), or azure (AKS)" - it: should render AutoscalingMetric for GKE with event-loop gauges template: api-autoscaling-metric.yaml @@ -572,3 +572,87 @@ tests: - matchRegex: path: data["config.yaml"] pattern: "braintrust_api_event_loop_utilization_ratio" + + - it: should not render GKE AutoscalingMetric on Azure + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should render HPA on Azure with prometheus-adapter metric names + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: spec.metrics[0].type + value: ContainerResource + - equal: + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[1].pods.metric.name + value: event_loop_utilization_ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: event_loop_delay_mean_seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + + - it: should scrape health server metrics on Azure Prometheus + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: ":8001" + - matchRegex: + path: data["prometheus.yml"] + pattern: "metrics_path: /metrics" + + - it: should render prometheus-adapter rules for event-loop gauges on Azure + template: api-autoscaling-prometheus-adapter.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_delay_mean_seconds" + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_utilization_ratio" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index c4e6e4d..f9f9958 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -125,8 +125,8 @@ api: # - the Autoscaling API in the service perimeter when using VPC Service Controls # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster. # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling - # cloud=aws (EKS): in-chart Prometheus + prometheus-adapter + HPA (custom.metrics). - # When enabled for a pool, that pool's replicas setting is ignored. + # cloud=aws|azure (EKS/AKS): in-chart Prometheus + prometheus-adapter + HPA + # (custom.metrics). When enabled for a pool, that pool's replicas setting is ignored. autoscaling: enabled: false minReplicas: 4 @@ -147,7 +147,7 @@ api: stabilizationWindowSeconds: 300 scaleUp: stabilizationWindowSeconds: 60 - # Used when cloud=aws (Prometheus + prometheus-adapter). + # Used when cloud=aws or cloud=azure (Prometheus + prometheus-adapter). prometheus: image: repository: prom/prometheus