Add fixed-capacity API workload isolation pools

braintrust/README.md

@@ -217,6 +217,52 @@ Size the request for the pod's full local-storage usage:
 
 When you enable `tmpVolume`, make sure the `ephemeralStorage.request` still covers that extra space.
 
+## API workload isolation
+
+`api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest`
+and `braintrust-api-background` Deployments and Services alongside the existing
+default `braintrust-api` pool. The pools share the same image and base
+configuration, while allowing independent replicas, resources, probes, rollout
+settings, environment overrides, topology spreading, and disruption budgets.
+
+The product-owned route contract is defined in
+[`files/contracts/api-workload-isolation-routes.yaml`](files/contracts/api-workload-isolation-routes.yaml).
+An ingress or gateway must preserve `braintrust-api` as its default backend and
+route these paths:
+
+| Pool | Paths |
+| --- | --- |
+| `braintrust-api` (default) | All requests not matched by an explicit ingest or background route |
+| `braintrust-api-ingest` | `POST /logs3`, `POST /otel/v1/traces`, `POST /attachment`, `POST /attachment/status` |
+| `braintrust-api-background` | `POST /v1/eval`, `POST /v1/eval/*`, `POST /function/eval`, `POST /function/sandbox`, `POST /function/use`, `POST /function/invoke-async-batch`, `POST /function/insert-functions`, `POST /automation/logs/trigger`; all methods for `/v1/proxy/chat/completions`, `/v1/proxy/responses` |
+
+By default, Brainstore's internal `BRAINSTORE_AI_PROXY_URL` targets the
+background Service while isolation is enabled. For an existing deployment,
+first create the pools with
+`api.workloadIsolation.brainstoreAiProxyToBackground: false`, and keep public
+paths on the default Service. Verify the background pool is ready, then route
+the classified public paths and set `brainstoreAiProxyToBackground: true` in a
+later release. To roll back, first return both the public paths and
+`brainstoreAiProxyToBackground` to the default API Service and verify it is
+serving them. Only then disable workload isolation in the chart; the chart
+cannot update an external ingress or gateway on its own.
+
+When using the chart-managed Istio `VirtualService`, set
+`virtualService.workloadIsolation.enabled: true` during the second release.
+The chart then renders the same product-owned route contract ahead of the
+existing `virtualService.http` rules, which remain available for fallback or
+custom routing of non-classified paths. The product-owned routes take
+precedence, so do not use `virtualService.http` to override a classified path.
+This option requires both `virtualService.enabled: true` and
+`api.workloadIsolation.enabled: true`. It preserves the AWS route methods:
+ingest, eval, function, and automation routes match `POST`; proxy routes match
+all methods. GKE Ingress cannot route by method, so its equivalent integration
+classifies matching paths for all methods.
+
+This feature does not enable autoscaling. Configure fixed replica counts under
+`api.replicas`, `api.workloadIsolation.ingest.replicas`, and
+`api.workloadIsolation.background.replicas`.
+
 ## Testing
 
 This Helm chart includes comprehensive automated unit tests.

braintrust/files/contracts/api-workload-isolation-routes.yaml

@@ -0,0 +1,55 @@
+# Canonical public route ownership for API workload isolation.
+#
+# This is product-controlled rather than a chart value: changing it changes
+# which API pool receives customer traffic. Ingress and gateway integrations
+# must implement this contract and retain the default API Service for all
+# unmatched requests.
+version: v1
+pools:
+  ingest:
+    routes:
+      - path: /logs3
+        pathType: exact
+        method: POST
+      - path: /otel/v1/traces
+        pathType: exact
+        method: POST
+      - path: /attachment
+        pathType: exact
+        method: POST
+      - path: /attachment/status
+        pathType: exact
+        method: POST
+  background:
+    routes:
+      # Keep this distinct from the trailing-slash prefix. Istio URI
+      # prefixes are string prefixes, so /v1/eval would also match
+      # /v1/evaluate.
+      - path: /v1/eval
+        pathType: exact
+        method: POST
+      - path: /v1/eval/
+        pathType: prefix
+        method: POST
+      - path: /function/eval
+        pathType: exact
+        method: POST
+      - path: /function/sandbox
+        pathType: exact
+        method: POST
+      - path: /function/use
+        pathType: exact
+        method: POST
+      - path: /function/invoke-async-batch
+        pathType: exact
+        method: POST
+      - path: /function/insert-functions
+        pathType: exact
+        method: POST
+      - path: /automation/logs/trigger
+        pathType: exact
+        method: POST
+      - path: /v1/proxy/chat/completions
+        pathType: exact
+      - path: /v1/proxy/responses
+        pathType: exact

braintrust/templates/_api-deployment.tpl

@@ -0,0 +1,232 @@
+{{/* Render one API Deployment from a merged pool configuration. */}}
+{{- define "braintrust.apiDeployment" -}}
+{{- $root := .root -}}
+{{- $api := .api -}}
+{{- $role := .role -}}
+{{- $customCA := $api.customCA -}}
+{{- $customCAMountPath := "" -}}
+{{- $customCAFilename := "" -}}
+{{- $customCASecretName := "" -}}
+{{- $customCASecretKey := "" -}}
+{{- if $customCA.enabled -}}
+{{- $customCAMountPath = required "api.customCA.mountPath is required when api.customCA.enabled is true" $customCA.mountPath -}}
+{{- $customCAFilename = required "api.customCA.filename is required when api.customCA.enabled is true" $customCA.filename -}}
+{{- $customCASecretName = required "api.customCA.secretName is required when api.customCA.enabled is true" $customCA.secretName -}}
+{{- $customCASecretKey = required "api.customCA.secretKey is required when api.customCA.enabled is true" $customCA.secretKey -}}
+{{- end -}}
+{{- $poolLabels := dict -}}
+{{- if or $root.Values.api.workloadIsolation.enabled (ne $role "default") -}}
+{{- $_ := set $poolLabels "braintrust.dev/api-pool" $role -}}
+{{- end -}}
+{{- $resourceLabels := mergeOverwrite (deepCopy $root.Values.global.labels) (deepCopy $api.labels) $poolLabels -}}
+{{- $podLabels := mergeOverwrite (deepCopy $root.Values.global.labels) (deepCopy $api.labels) (deepCopy $api.podLabels) (dict "app" $api.name) $poolLabels -}}
+{{- if eq $root.Values.cloud "azure" -}}
+{{- $_ := set $podLabels "azure.workload.identity/use" "true" -}}
+{{- end -}}
+{{- if and (eq $root.Values.cloud "google") $api.enableGcsAuth -}}
+{{- $_ := set $podLabels "gke-workload-identity/use" "true" -}}
+{{- end -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $api.name }}
+  namespace: {{ include "braintrust.namespace" $root }}
+  {{- with $resourceLabels }}
+  labels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with $api.annotations.deployment }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: {{ $api.replicas }}
+  strategy:
+    type: {{ $api.strategy.type }}
+    {{- with $api.strategy.rollingUpdate }}
+    rollingUpdate:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  selector:
+    matchLabels:
+      app: {{ $api.name }}
+  template:
+    metadata:
+      labels:
+        {{- with $podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      annotations:
+        checksum/config: {{ include (print $root.Template.BasePath "/api-configmap.yaml") $root | sha256sum }}
+        {{- if and (eq $root.Values.cloud "google") $api.enableGcsAuth }}
+        iam.gke.io/gcp-service-account: {{ required "api.serviceAccount.googleServiceAccount is required when api.enableGcsAuth is true" $api.serviceAccount.googleServiceAccount }}
+        {{- end }}
+        {{- with $api.annotations.pod }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ $api.serviceAccount.name }}
+      {{- with $api.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $api.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $api.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if $api.topologySpread.enabled }}
+      topologySpreadConstraints:
+        - maxSkew: {{ $api.topologySpread.maxSkew }}
+          topologyKey: {{ $api.topologySpread.topologyKey | quote }}
+          whenUnsatisfiable: {{ $api.topologySpread.whenUnsatisfiable }}
+          labelSelector:
+            matchLabels:
+              app: {{ $api.name }}
+      {{- end }}
+      {{- with $api.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: api
+          image: "{{ $api.image.repository }}:{{ $api.image.tag }}"
+          imagePullPolicy: {{ $api.image.pullPolicy }}
+          {{- with $api.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - containerPort: {{ $api.service.port }}
+          resources:
+            {{- toYaml $api.resources | nindent 12 }}
+          {{- with $api.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with $api.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          envFrom:
+            - configMapRef:
+                name: {{ $root.Values.api.name }}
+          env:
+            - name: PG_URL
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: PG_URL
+            - name: REDIS_URL
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: REDIS_URL
+            - name: FUNCTION_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: FUNCTION_SECRET_KEY
+            - name: BRAINSTORE_LICENSE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: BRAINSTORE_LICENSE_KEY
+            {{- if eq $root.Values.cloud "azure" }}
+            - name: AZURE_STORAGE_CONNECTION_STRING
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: AZURE_STORAGE_CONNECTION_STRING
+            {{- end }}
+            {{- if and (eq $root.Values.cloud "google") (not $api.enableGcsAuth) }}
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: GCS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: braintrust-secrets
+                  key: GCS_SECRET_ACCESS_KEY
+            {{- end }}
+            - name: TS_API_HEALTHSERVER_HOST
+              value: {{ $api.healthServer.host | quote }}
+            - name: TS_API_HEALTHSERVER_PORT
+              value: {{ $api.healthServer.port | quote }}
+            {{- if $customCA.enabled }}
+            {{- $customCAPath := printf "%s/%s" $customCAMountPath $customCAFilename }}
+            - name: NODE_EXTRA_CA_CERTS
+              value: {{ $customCAPath | quote }}
+            - name: REQUESTS_CA_BUNDLE
+              value: {{ $customCAPath | quote }}
+            - name: SSL_CERT_FILE
+              value: {{ $customCAPath | quote }}
+            - name: CURL_CA_BUNDLE
+              value: {{ $customCAPath | quote }}
+            - name: AWS_CA_BUNDLE
+              value: {{ $customCAPath | quote }}
+            - name: PIP_CERT
+              value: {{ $customCAPath | quote }}
+            {{- end }}
+            {{- with $api.extraEnvVars }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- if or $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled }}
+          volumeMounts:
+            {{- if $api.tmpVolume.enabled }}
+            - name: tmp-volume
+              mountPath: /tmp
+            {{- end }}
+            {{- if and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver }}
+            - name: secrets-store-inline
+              mountPath: "/mnt/secrets-store"
+              readOnly: true
+            {{- end }}
+            {{- if $customCA.enabled }}
+            - name: custom-ca-bundle
+              mountPath: {{ $customCAMountPath | quote }}
+              readOnly: true
+            {{- end }}
+          {{- end }}
+      {{- with $api.extraContainers }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- if or $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }}
+        {{- if $api.tmpVolume.enabled }}
+        - name: tmp-volume
+          emptyDir:
+            {{- if $api.tmpVolume.sizeLimit }}
+            sizeLimit: {{ $api.tmpVolume.sizeLimit | quote }}
+            {{- else }}
+            {}
+            {{- end }}
+        {{- end }}
+        {{- if and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver }}
+        - name: secrets-store-inline
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: {{ $root.Values.azure.keyVaultName }}
+        {{- end }}
+        {{- if $customCA.enabled }}
+        - name: custom-ca-bundle
+          secret:
+            secretName: {{ $customCASecretName | quote }}
+            items:
+              - key: {{ $customCASecretKey | quote }}
+                path: {{ $customCAFilename | quote }}
+        {{- end }}
+        {{- with $api.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- else }}
+        []
+        {{- end }}
+{{- end -}}

braintrust/templates/_api-service.tpl

@@ -0,0 +1,33 @@
+{{/* Render one API Service from a merged pool configuration. */}}
+{{- define "braintrust.apiService" -}}
+{{- $root := .root -}}
+{{- $api := .api -}}
+{{- $role := .role -}}
+{{- $poolLabels := dict -}}
+{{- if or $root.Values.api.workloadIsolation.enabled (ne $role "default") -}}
+{{- $_ := set $poolLabels "braintrust.dev/api-pool" $role -}}
+{{- end -}}
+{{- $resourceLabels := mergeOverwrite (deepCopy $root.Values.global.labels) (deepCopy $api.labels) $poolLabels -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $api.service.name | default $api.name }}
+  namespace: {{ include "braintrust.namespace" $root }}
+  {{- with $resourceLabels }}
+  labels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with $api.annotations.service }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    app: {{ $api.name }}
+  ports:
+    - name: {{ $api.service.portName }}
+      protocol: TCP
+      port: {{ $api.service.port }}
+      targetPort: {{ $api.service.port }}
+  type: {{ $api.service.type }}
+{{- end -}}