Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions .env.template
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,13 @@
# GOMODEL_CACHE_DIR=.cache

# External model metadata registry (provides pricing, capabilities, context window, etc.)
# Default: ENTERPILOT/ai-model-list on GitHub. Point this at an internal mirror
# for air-gapped installs. Setting it to an empty string here does NOT disable
# the fetch (empty env values are skipped, so the default survives) -- to
# disable it, set cache.model.model_list.url: "" in config.yaml.
# Default: ENTERPILOT/ai-model-list on GitHub. Point this at an internal mirror,
# or set MODEL_LIST_URL=off to disable the download entirely for fully
# air-gapped installs -- models then keep provider-reported, configured,
# heuristic-derived, and any previously cached catalog metadata, so declare
# pricing in config.yaml if you need cost tracking and budgets. (Setting it
# to an empty string does NOT disable the fetch: empty env values are
# skipped, so the default survives.)
# MODEL_LIST_URL=https://raw.githubusercontent.com/ENTERPILOT/ai-model-list/refs/heads/main/models.min.json

# Model Access Configuration
Expand Down
72 changes: 71 additions & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func clearAllConfigEnvVars(t *testing.T) {
for _, key := range []string{
"CONFIG_STRICT",
"PORT", "BASE_PATH", "GOMODEL_MASTER_KEY", "BODY_SIZE_LIMIT", "SWAGGER_ENABLED", "PPROF_ENABLED", "ENABLE_PASSTHROUGH_ROUTES", "ALLOW_PASSTHROUGH_V1_ALIAS", "USER_PATH_HEADER", "ENABLED_PASSTHROUGH_PROVIDERS",
"GOMODEL_CACHE_DIR", "CACHE_REFRESH_INTERVAL",
"GOMODEL_CACHE_DIR", "CACHE_REFRESH_INTERVAL", "MODEL_LIST_URL",
"REDIS_URL", "REDIS_KEY_MODELS", "REDIS_KEY_RESPONSES", "REDIS_TTL_MODELS", "REDIS_TTL_RESPONSES",
"RESPONSE_CACHE_SIMPLE_ENABLED",
"SEMANTIC_CACHE_ENABLED", "SEMANTIC_CACHE_THRESHOLD", "SEMANTIC_CACHE_TTL", "SEMANTIC_CACHE_MAX_CONV_MESSAGES",
Expand Down Expand Up @@ -1433,6 +1433,76 @@ func TestLoad_EnvOverridesDefaults(t *testing.T) {
})
}

func TestLoad_ModelListURLEnv(t *testing.T) {
const defaultURL = "https://raw.githubusercontent.com/ENTERPILOT/ai-model-list/refs/heads/main/models.min.json"

tests := []struct {
name string
set bool
value string
want string
}{
{name: "UnsetKeepsDefault", set: false, want: defaultURL},
{name: "MirrorOverridesDefault", set: true, value: "https://mirror.internal/models.min.json", want: "https://mirror.internal/models.min.json"},
{name: "EmptyIsSkippedLikeAnyEnvVar", set: true, value: "", want: defaultURL},
{name: "OffDisablesDownloads", set: true, value: "off", want: ""},
{name: "OffIsCaseInsensitive", set: true, value: "OFF", want: ""},
{name: "OffTrimsWhitespace", set: true, value: " off ", want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clearAllConfigEnvVars(t)
withTempDir(t, func(_ string) {
if tt.set {
t.Setenv("MODEL_LIST_URL", tt.value)
}
result, err := Load()
if err != nil {
t.Fatalf("Load() failed: %v", err)
}
if got := result.Config.Cache.Model.ModelList.URL; got != tt.want {
t.Errorf("Cache.Model.ModelList.URL = %q, want %q", got, tt.want)
}
})
})
}

t.Run("EnvOffWinsOverConfigYAML", func(t *testing.T) {
clearAllConfigEnvVars(t)
withTempDir(t, func(dir string) {
yaml := "cache:\n model:\n model_list:\n url: \"https://mirror.internal/models.min.json\"\n"
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(yaml), 0o644); err != nil {
t.Fatalf("failed to write config.yaml: %v", err)
}
t.Setenv("MODEL_LIST_URL", "off")
result, err := Load()
if err != nil {
t.Fatalf("Load() failed: %v", err)
}
if got := result.Config.Cache.Model.ModelList.URL; got != "" {
t.Errorf("Cache.Model.ModelList.URL = %q, want empty (env off wins over config.yaml)", got)
}
})
})

t.Run("ConfigYAMLOffDisablesDownloads", func(t *testing.T) {
clearAllConfigEnvVars(t)
withTempDir(t, func(dir string) {
yaml := "cache:\n model:\n model_list:\n url: \"off\"\n"
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(yaml), 0o644); err != nil {
t.Fatalf("failed to write config.yaml: %v", err)
}
result, err := Load()
if err != nil {
t.Fatalf("Load() failed: %v", err)
}
if got := result.Config.Cache.Model.ModelList.URL; got != "" {
t.Errorf("Cache.Model.ModelList.URL = %q, want empty (yaml off disables)", got)
}
})
})
}

func TestLoad_ProviderFromYAML(t *testing.T) {
clearAllConfigEnvVars(t)

Expand Down
17 changes: 16 additions & 1 deletion config/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,22 @@ import (
// applyEnvOverrides walks cfg's struct fields and applies env var overrides
// based on `env` struct tags. Maps are skipped.
func applyEnvOverrides(cfg *Config) error {
return applyEnvOverridesValue(reflect.ValueOf(cfg).Elem())
if err := applyEnvOverridesValue(reflect.ValueOf(cfg).Elem()); err != nil {
return err
}
normalizeModelListURL(cfg)
return nil
}

// normalizeModelListURL maps the sentinel "off" (case-insensitive) to an empty
// model list URL, disabling catalog downloads for air-gapped installs. A
// sentinel is used because empty env values are skipped by the generic overlay
// (so MODEL_LIST_URL="" cannot override the default), and it works identically
// when set via config.yaml.
func normalizeModelListURL(cfg *Config) {
if strings.EqualFold(strings.TrimSpace(cfg.Cache.Model.ModelList.URL), "off") {
cfg.Cache.Model.ModelList.URL = ""
}
}

// hasEnvDescendants reports whether t (a struct type) contains any field (at
Expand Down
6 changes: 3 additions & 3 deletions docs/advanced/model-metadata.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,6 @@ flowchart LR
If the catalog fetch fails or the deployment is air-gapped, the gateway runs
normally — only the catalog-supplied defaults (including catalog pricing) are
missing. Pricing overrides, `config.yaml` metadata, provider discovery signals,
and the ID heuristic still apply. Mirror `MODEL_LIST_URL` internally or declare
metadata in `config.yaml`; see [Production guide](/guides/production) for
details.
and the ID heuristic still apply. Mirror `MODEL_LIST_URL` internally, or set
`MODEL_LIST_URL=off` to turn the download off entirely and declare metadata in
`config.yaml`; see [Production guide](/guides/production) for details.
15 changes: 9 additions & 6 deletions docs/guides/production.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,12 @@ normally. What you lose is metadata enrichment.
</Warning>

<Note>
Setting `MODEL_LIST_URL=""` does **not** disable the fetch. Empty environment
values are skipped when overrides are applied, so the compiled-in default URL
survives. To disable it, set `cache.model.model_list.url: ""` in
`config.yaml`. To redirect it, point `MODEL_LIST_URL` at an internal mirror.
To disable the fetch entirely — for fully air-gapped installs — set
`MODEL_LIST_URL=off` (or `cache.model.model_list.url: "off"` in
`config.yaml`). Setting the variable to an empty string does **not**
disable it: empty env values are skipped, so the default URL survives.
To redirect the fetch instead, point `MODEL_LIST_URL` at an internal
mirror.
</Note>

If a Bedrock provider is configured, the AWS SDK may also probe the link-local
Expand Down Expand Up @@ -377,5 +379,6 @@ if the file size matters.
`terminationGracePeriodSeconds` above 30.
- Enable metrics and alert on `gomodel_circuit_breaker_state` and on
`usage log buffer full`.
- For air-gapped installs, mirror `MODEL_LIST_URL` or declare model pricing in
config so cost tracking and budgets keep working.
- For air-gapped installs, mirror `MODEL_LIST_URL` (or disable it with
`MODEL_LIST_URL=off`) and declare model pricing in config so cost tracking
and budgets keep working.
4 changes: 3 additions & 1 deletion internal/providers/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,9 @@ func Init(ctx context.Context, result *config.LoadResult, factory *ProviderFacto

// Fetch model list in background (best-effort, non-blocking)
modelListURL := result.Config.Cache.Model.ModelList.URL
if modelListURL != "" {
if modelListURL == "" {
slog.Info("model list downloads disabled; models rely on provider-reported, configured, and any previously cached catalog metadata")
} else {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
go func() {
fetchCtx, cancel := context.WithTimeout(ctx, 45*time.Second)
defer cancel()
Expand Down