From 43fe66fc10b884ddd4d55b84eec0d348b010df9f Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 07:12:33 -0700 Subject: [PATCH 01/13] fix(llm): route an openai-provider model litellm cannot infer as openai/ resolve_model sent every --provider openai model to litellm bare, and litellm infers OpenAI only for names on its own lists. glm-5.3 behind OPENAI_API_BASE, an org/name id (meta-llama/Llama-3.1-8B) or a model newer than the installed litellm was refused before any request ("LLM Provider NOT provided"); a bare claude-* name went to Anthropic. So `eval cross-judge --judges openai/glm-5.3` scored nothing and read NOT MEASURED (#136), and answers, the judge, the panel and the question and persona generators failed the same way. Such names now go out as openai/. A name litellm already sends to OpenAI (its chat and text-completion lists) or one with a litellm provider prefix (openai/..., azure/...) keeps its route, so its response-cache key, which holds the litellm model string, is unchanged. llm.py is at its growth baseline: the module docstring's closing paragraph, which repeats the opening one, and a comment that repeats the legacy-section header are dropped, so the file stays at 1118 lines. - tests/test_openai_compatible_route.py: the production adapter with litellm's real provider lookup (mock_response, nothing sent), for LLM(...).ask and eval cross-judge end to end; fails on main. - tests/test_llm.py: route rows for the prefixed names and for the ones that must not change. Fixes #136 Signed-off-by: Vladislav Perevezentsev --- ...vezentsev_openai-compatible-model-route.md | 13 ++++ dokimi/dokimi/llm.py | 18 +++--- dokimi/tests/test_llm.py | 8 ++- dokimi/tests/test_openai_compatible_route.py | 62 +++++++++++++++++++ 4 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md create mode 100644 dokimi/tests/test_openai_compatible_route.py diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md new file mode 100644 index 000000000..3afa80bc3 --- /dev/null +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -0,0 +1,13 @@ +### Fixed + +- **`--provider openai` reaches an OpenAI-compatible endpoint with any model name** — a + model that litellm does not list as OpenAI's (`glm-5.3` behind `OPENAI_API_BASE`, an + `org/name` id such as `meta-llama/Llama-3.1-8B`, a model newer than the installed + litellm) was sent bare, and litellm refused it ("LLM Provider NOT provided") before any + request; a bare `claude-*` name went to Anthropic instead. Such names now go out as + `openai/`, on every path that takes the openai provider: answers, the judge and + panel, `eval cross-judge --judges openai/` (#136), whose arms read NOT MEASURED + before, and question and persona generation. A name litellm already sends to OpenAI + (`gpt-*`, `o*`), or one with a litellm provider prefix (`azure/…`), keeps its route and + its response-cache entries. The double-prefix workaround (`openai/openai/`) still + works. diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index bdd8067ed..4ab4de8e8 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -60,10 +60,6 @@ production; :class:`FakeAdapter` answers from a script and records every request, for tests (the ``fake_llm`` fixture). :func:`use_adapter` swaps the process default; ``LLM(..., adapter=...)`` overrides it for one object. - -``llm_call``, ``llm_call_with_usage``, ``chat_completion``, the -``ChatOpenAI``/``ChatAnthropic`` shims and ``utils.get_llm`` are delegates to -:class:`LLM` until their callers move. """ from __future__ import annotations @@ -136,17 +132,22 @@ def resolve_api_key(provider: str, api_key: Optional[str] = None) -> Optional[st def resolve_model(model: str, provider: str) -> str: """The litellm model string for *model* at *provider*. - A name that already carries a litellm route (contains ``/``) passes through, - except on OpenRouter and openai-codex, which add their own prefix. + A name with a ``/`` passes through, except on OpenRouter and openai-codex, which + add their own prefix, and on openai, where only a litellm provider prefix counts. """ name = _canonical(provider) if name == "openrouter": return model if model.startswith("openrouter/") else f"openrouter/{model}" if name == "openai-codex": return f"openai/{model}" - if "/" in model: - return model if name == "openai": + # litellm refuses a name off its OpenAI lists (glm-5.3 on OPENAI_API_BASE, meta-llama/x) + # or sends it elsewhere (claude-*): those get openai/. The rest keep route and cache key. + import litellm + known = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models + routed = model.split("/", 1)[0] in litellm.provider_list if "/" in model else model in known + return model if routed else f"openai/{model}" + if "/" in model: return model if name == "google-vertex": return f"vertex_ai/{model}" @@ -881,7 +882,6 @@ def parses(raw: str) -> bool: # Delegates for the callers that have not moved to :class:`LLM` yet; each is the # module's behaviour under the old call shape. They go once nothing imports them. -# Backward-compatibility shims for existing tests/modules. LANGCHAIN_AVAILABLE = True _build_litellm_model = resolve_model diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 2710b450a..0d931418e 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -56,7 +56,13 @@ def _no_ambient_bedrock_env(monkeypatch): @pytest.mark.parametrize("provider, model, expected", [ - ("openai", "gpt-6.1-sol", "gpt-6.1-sol"), + ("openai", "gpt-6.1-sol", "gpt-6.1-sol"), # on litellm's lists: bare, so its cache key holds + ("openai", "gpt-3.5-turbo-instruct", "gpt-3.5-turbo-instruct"), # litellm's text-completion route + ("openai", "glm-5.3", "openai/glm-5.3"), # off the lists: an OPENAI_API_BASE server's model + ("openai", "claude-sonnet-5-5", "openai/claude-sonnet-5-5"), # bare, litellm would pick anthropic + ("openai", "meta-llama/Llama-3.1-8B", "openai/meta-llama/Llama-3.1-8B"), # an org, not a route + ("openai", "openai/glm-5.3", "openai/glm-5.3"), + ("openai", "azure/gpt-4o", "azure/gpt-4o"), # an explicit litellm route passes through ("openai-codex", "gpt-5", "openai/gpt-5"), ("anthropic", "claude-sonnet-5-5", "anthropic/claude-sonnet-5-5"), ("openrouter", "openai/gpt-oss-120b", "openrouter/openai/gpt-oss-120b"), diff --git a/dokimi/tests/test_openai_compatible_route.py b/dokimi/tests/test_openai_compatible_route.py new file mode 100644 index 000000000..b1636661f --- /dev/null +++ b/dokimi/tests/test_openai_compatible_route.py @@ -0,0 +1,62 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A model on an OpenAI-compatible endpoint (``--provider openai``, ``OPENAI_API_BASE``) is +called, not refused by litellm's provider lookup (#136). + +The production adapter runs here with litellm's own routing: ``litellm.completion`` gets +``mock_response``, so it still resolves the provider from the model string, and refuses a name +it cannot route ("LLM Provider NOT provided"), but sends nothing. +""" + +import json + +import pytest + +from dokimi.llm import LLM + +JUDGE_REPLY = json.dumps({"correctness": 7, "completeness": 7, "specificity": 7, "code_quality": 7, + "actionability": 7, "aggregate": 0}) + + +@pytest.fixture +def litellm_routes(monkeypatch): + """``litellm.completion`` with its real provider lookup and a canned reply; the routed names land in the list.""" + import litellm + + from dokimi import caching + + real, routed = litellm.completion, [] + + def completion(**kwargs): + reply = real(**kwargs, mock_response=JUDGE_REPLY) # raises where litellm cannot route the name + routed.append(kwargs["model"]) + return reply + + monkeypatch.setattr(litellm, "completion", completion) + monkeypatch.delenv(caching.RESPONSE_CACHE_ENV, raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + return routed + + +@pytest.mark.parametrize("model", ["glm-5.3", "meta-llama/Llama-3.1-8B", "claude-sonnet-5-5"]) +def test_a_model_off_litellms_openai_lists_is_sent_to_the_openai_endpoint(litellm_routes, model): + assert LLM(model, "openai").ask("Reply with OK.") == JUDGE_REPLY + assert litellm_routes == [f"openai/{model}"] + + +def test_cross_judge_scores_with_an_openai_compatible_judge(litellm_routes, tmp_path): + """The reported command: before the fix every score failed and the arm read NOT MEASURED.""" + from dokimi.cli import build_parser + + answers = tmp_path / "run.json" + answers.write_text(json.dumps({"library_name": "L", "baseline_arm": "baseline", "answers": [ + {"question_id": f"Q{i}", "question_text": "How do I use X?", + "arms": {"baseline": {"answer": f"b{i}"}, "skill": {"answer": f"s{i}"}}} for i in range(3)]}), + encoding="utf-8") + out = tmp_path / "cj.json" + args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), "--no-stored", + "--judges", "openai/glm-5.3", "--output", str(out)]) + args.func(args) + assert litellm_routes == ["openai/glm-5.3"] * 6 # three questions, two arms + assert json.loads(out.read_text(encoding="utf-8"))["summary"]["arms"]["skill"]["label"] != "NOT MEASURED" From 98bd6a15846eb42afc0b6752714b542b8d5de9d6 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 07:23:58 -0700 Subject: [PATCH 02/13] docs(readme): run against an OpenAI-compatible server with provider openai Nothing in the docs said how to point dokimi at vLLM or a gateway; the #136 reporter found the openai/openai/ form by trial. The setup paragraph now names OPENAI_API_BASE / OPENAI_BASE_URL, says any served model name works (--judges openai/glm-5.3), that OPENAI_API_KEY must be set even when the server ignores it (the request fails with "Missing credentials" otherwise), and that cost is unknown for such a model, so a --max-usd run stops after its first call. Signed-off-by: Vladislav Perevezentsev --- README.md | 13 ++++++++++--- ...d-perevezentsev_openai-compatible-model-route.md | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5ddce8210..1f921fa24 100644 --- a/README.md +++ b/README.md @@ -147,9 +147,16 @@ Credentials are read from the environment: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `AWS_REGION` with `AWS_BEARER_TOKEN_BEDROCK` for Bedrock, and Google application-default credentials (`GOOGLE_APPLICATION_CREDENTIALS`, with `VERTEXAI_PROJECT` and -`VERTEXAI_LOCATION`) for Vertex. The judge must be a different model from the -one under measurement, because a model that grades its own answers inflates -the scores. `novelty` refuses a self-judge; `arms run` does not check, so name +`VERTEXAI_LOCATION`) for Vertex. For an OpenAI-compatible server (vLLM, a +gateway), use provider `openai` with `OPENAI_API_BASE` (or `OPENAI_BASE_URL`) +set to its `/v1` URL. Any model name the server serves works, such as +`--judges openai/glm-5.3` in `eval cross-judge`. `OPENAI_API_KEY` must be set +even when the server ignores it. litellm has no price for such a model, so cost +is reported as unknown and a `--max-usd` run stops after its first call. + +The judge must be a different model from the one under measurement, because a +model that grades its own answers inflates the scores. `novelty` refuses a +self-judge; `arms run` does not check, so name `--judge --judge-model --judge-provider ` yourself. ### A skill diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md index 3afa80bc3..fa6452d9e 100644 --- a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -11,3 +11,9 @@ (`gpt-*`, `o*`), or one with a litellm provider prefix (`azure/…`), keeps its route and its response-cache entries. The double-prefix workaround (`openai/openai/`) still works. + +### Documentation + +- **README: running against an OpenAI-compatible server** — provider `openai` with + `OPENAI_API_BASE` / `OPENAI_BASE_URL`, a placeholder `OPENAI_API_KEY` for a server that + ignores it, and why cost is unknown and `--max-usd` stops after one call. From 81c0d01419989729e5b079de179b20bc2c50009a Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 07:38:26 -0700 Subject: [PATCH 03/13] docs(changelog): name the models that keep their route by example, not as gpt-*/o* A gpt-* name off litellm's OpenAI lists (gpt-7-new) gets openai/ too, so the fragment's "(gpt-*, o*) keeps its route" overstated it. Signed-off-by: Vladislav Perevezentsev --- ...0000_vlad-perevezentsev_openai-compatible-model-route.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md index fa6452d9e..61a562950 100644 --- a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -7,9 +7,9 @@ request; a bare `claude-*` name went to Anthropic instead. Such names now go out as `openai/`, on every path that takes the openai provider: answers, the judge and panel, `eval cross-judge --judges openai/` (#136), whose arms read NOT MEASURED - before, and question and persona generation. A name litellm already sends to OpenAI - (`gpt-*`, `o*`), or one with a litellm provider prefix (`azure/…`), keeps its route and - its response-cache entries. The double-prefix workaround (`openai/openai/`) still + before, and question and persona generation. A name on litellm's OpenAI lists + (`gpt-6.1-sol`, `o3`), or one with a litellm provider prefix (`azure/…`), keeps its route + and its response-cache entries. The double-prefix workaround (`openai/openai/`) still works. ### Documentation From 93727368c34dd69da1b639a7b1c3264068df8bf5 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 28 Oct 2026 10:40:14 +0000 Subject: [PATCH 04/13] test(llm): pin an ollama/ prefix passing through on provider openai Signed-off-by: Anton Volkov --- dokimi/tests/test_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 0d931418e..c6e58cfb8 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -63,6 +63,7 @@ def _no_ambient_bedrock_env(monkeypatch): ("openai", "meta-llama/Llama-3.1-8B", "openai/meta-llama/Llama-3.1-8B"), # an org, not a route ("openai", "openai/glm-5.3", "openai/glm-5.3"), ("openai", "azure/gpt-4o", "azure/gpt-4o"), # an explicit litellm route passes through + ("openai", "ollama/llama3", "ollama/llama3"), # any litellm provider prefix passes through ("openai-codex", "gpt-5", "openai/gpt-5"), ("anthropic", "claude-sonnet-5-5", "anthropic/claude-sonnet-5-5"), ("openrouter", "openai/gpt-oss-120b", "openrouter/openai/gpt-oss-120b"), From 598fbfa16875ed103a02fac72b8127ab0c77338d Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 05:50:07 -0700 Subject: [PATCH 05/13] fix(llm): keep a bare ft:gpt-* fine-tune on its OpenAI route on provider openai resolve_model kept a bare name bare only when it was on litellm's OpenAI chat or text-completion lists. litellm also takes a fine-tune id (ft:gpt-4o-mini:org::id; its test is "ft:gpt-3.5-turbo" or "ft:gpt-4" in the name) for OpenAI's, so such a name went out as openai/ft:... : the same endpoint, but a new response-cache key, which the changelog said would not happen. The bare test now repeats litellm's, so the name keeps its route and its cache entries. litellm.get_llm_provider is not called instead: on a name it cannot place (glm-5.3, the case this fixes) it prints a "Provider List" banner to stdout. - tests/test_openai_compatible_route.py: names kept bare (gpt-4o, gpt-3.5-turbo-instruct, ft:gpt-4o-mini:org::id) still route through litellm's real lookup, and a fine-tune litellm does not place (ft:gpt-5-mini:org::id) gets openai/, so a looser ft: test fails. Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/llm.py | 6 +++--- dokimi/tests/test_llm.py | 1 + dokimi/tests/test_openai_compatible_route.py | 9 ++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 4ab4de8e8..20084a07a 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -141,11 +141,11 @@ def resolve_model(model: str, provider: str) -> str: if name == "openai-codex": return f"openai/{model}" if name == "openai": - # litellm refuses a name off its OpenAI lists (glm-5.3 on OPENAI_API_BASE, meta-llama/x) - # or sends it elsewhere (claude-*): those get openai/. The rest keep route and cache key. + # A name litellm does not take for OpenAI's (glm-5.3, meta-llama/x, claude-*) gets openai/. import litellm known = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models - routed = model.split("/", 1)[0] in litellm.provider_list if "/" in model else model in known + ours = model in known or "ft:gpt-3.5-turbo" in model or "ft:gpt-4" in model # as litellm + routed = model.split("/", 1)[0] in litellm.provider_list if "/" in model else ours return model if routed else f"openai/{model}" if "/" in model: return model diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index c6e58cfb8..737ebda7d 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -58,6 +58,7 @@ def _no_ambient_bedrock_env(monkeypatch): @pytest.mark.parametrize("provider, model, expected", [ ("openai", "gpt-6.1-sol", "gpt-6.1-sol"), # on litellm's lists: bare, so its cache key holds ("openai", "gpt-3.5-turbo-instruct", "gpt-3.5-turbo-instruct"), # litellm's text-completion route + ("openai", "ft:gpt-4o-mini:org::id", "ft:gpt-4o-mini:org::id"), # a fine-tune litellm takes as OpenAI's ("openai", "glm-5.3", "openai/glm-5.3"), # off the lists: an OPENAI_API_BASE server's model ("openai", "claude-sonnet-5-5", "openai/claude-sonnet-5-5"), # bare, litellm would pick anthropic ("openai", "meta-llama/Llama-3.1-8B", "openai/meta-llama/Llama-3.1-8B"), # an org, not a route diff --git a/dokimi/tests/test_openai_compatible_route.py b/dokimi/tests/test_openai_compatible_route.py index b1636661f..156612a27 100644 --- a/dokimi/tests/test_openai_compatible_route.py +++ b/dokimi/tests/test_openai_compatible_route.py @@ -39,12 +39,19 @@ def completion(**kwargs): return routed -@pytest.mark.parametrize("model", ["glm-5.3", "meta-llama/Llama-3.1-8B", "claude-sonnet-5-5"]) +@pytest.mark.parametrize("model", ["glm-5.3", "meta-llama/Llama-3.1-8B", "claude-sonnet-5-5", + "ft:gpt-5-mini:org::id"]) # a fine-tune litellm does not place def test_a_model_off_litellms_openai_lists_is_sent_to_the_openai_endpoint(litellm_routes, model): assert LLM(model, "openai").ask("Reply with OK.") == JUDGE_REPLY assert litellm_routes == [f"openai/{model}"] +@pytest.mark.parametrize("model", ["gpt-4o", "gpt-3.5-turbo-instruct", "ft:gpt-4o-mini:org::id"]) +def test_a_name_litellm_takes_for_openais_goes_out_bare_and_still_routes(litellm_routes, model): + assert LLM(model, "openai").ask("Reply with OK.") == JUDGE_REPLY + assert litellm_routes == [model] + + def test_cross_judge_scores_with_an_openai_compatible_judge(litellm_routes, tmp_path): """The reported command: before the fix every score failed and the arm read NOT MEASURED.""" from dokimi.cli import build_parser From 1f135976c7bbbd1bd818cac1416720b18ca2f392 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 05:50:07 -0700 Subject: [PATCH 06/13] test(llm): pin a bare Bedrock id going to the OPENAI_API_BASE endpoint on provider openai Review of #256: the openai rule also reroutes the bare names litellm assigns to another cloud (Bedrock, Vertex, Anthropic, Cohere, AI21). That is intended: on main such a name reached that cloud with OPENAI_API_KEY passed as its key, which Anthropic and Bedrock refuse. The row pins amazon.nova-pro-v1:0 -> openai/amazon.nova-pro-v1:0 next to the claude-* one. Signed-off-by: Vladislav Perevezentsev --- dokimi/tests/test_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 737ebda7d..6de162d7c 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -61,6 +61,7 @@ def _no_ambient_bedrock_env(monkeypatch): ("openai", "ft:gpt-4o-mini:org::id", "ft:gpt-4o-mini:org::id"), # a fine-tune litellm takes as OpenAI's ("openai", "glm-5.3", "openai/glm-5.3"), # off the lists: an OPENAI_API_BASE server's model ("openai", "claude-sonnet-5-5", "openai/claude-sonnet-5-5"), # bare, litellm would pick anthropic + ("openai", "amazon.nova-pro-v1:0", "openai/amazon.nova-pro-v1:0"), # bare, litellm would pick bedrock ("openai", "meta-llama/Llama-3.1-8B", "openai/meta-llama/Llama-3.1-8B"), # an org, not a route ("openai", "openai/glm-5.3", "openai/glm-5.3"), ("openai", "azure/gpt-4o", "azure/gpt-4o"), # an explicit litellm route passes through From feb0d7e3347f78d93f69f3024a5cfe518ec0db23 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 05:50:15 -0700 Subject: [PATCH 07/13] docs(readme): say which names reach an OpenAI-compatible server, and that only novelty takes --max-usd Review of #256. "Any model name the server serves works" was false for a name with a litellm provider prefix (deepseek/, groq/, ollama/): it keeps that provider's route and ignores OPENAI_API_BASE, so the text says to write it as openai/. It also says that a bare claude-*, Bedrock or Vertex id now goes to the server, and needs its cloud's provider to reach that cloud. --max-usd is a novelty flag (SpendCap), not one every --provider openai run takes. Signed-off-by: Vladislav Perevezentsev --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1f921fa24..9e03a8ea7 100644 --- a/README.md +++ b/README.md @@ -149,10 +149,15 @@ Credentials are read from the environment: `OPENAI_API_KEY`, credentials (`GOOGLE_APPLICATION_CREDENTIALS`, with `VERTEXAI_PROJECT` and `VERTEXAI_LOCATION`) for Vertex. For an OpenAI-compatible server (vLLM, a gateway), use provider `openai` with `OPENAI_API_BASE` (or `OPENAI_BASE_URL`) -set to its `/v1` URL. Any model name the server serves works, such as -`--judges openai/glm-5.3` in `eval cross-judge`. `OPENAI_API_KEY` must be set -even when the server ignores it. litellm has no price for such a model, so cost -is reported as unknown and a `--max-usd` run stops after its first call. +set to its `/v1` URL. A model name that litellm does not take for OpenAI's own +goes to that server, such as `glm-5.3` or `meta-llama/Llama-3.1-8B` +(`--judges openai/glm-5.3` in `eval cross-judge`); so does a bare `claude-*`, +Bedrock or Vertex id, which needs its cloud's provider to reach that cloud. A +name that starts with a litellm provider prefix (`deepseek/`, `groq/`, +`ollama/`) keeps that provider's route: write it as `openai/` to send it +to the server. `OPENAI_API_KEY` must be set even when the server ignores it. +litellm has no price for such a model, so cost is reported as unknown and +`novelty --max-usd` stops the run after its first call. The judge must be a different model from the one under measurement, because a model that grades its own answers inflates the scores. `novelty` refuses a From a6063b58a3ba0e755436bddd39916d88c829a4af Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 05:50:15 -0700 Subject: [PATCH 08/13] docs(changelog): name the bare names of other clouds that now go to the openai endpoint Review of #256. The fragment said a bare claude-* name "went to Anthropic instead" and left out the rest of that set: about 450 bare names in litellm 1.106 (Bedrock, Vertex, Anthropic, Cohere, AI21) that went to their cloud with OPENAI_API_KEY as the key and worked only with it unset. A Fixed entry now names the set, why it moves and which provider reaches those clouds. The first entry no longer says "any model name", lists a ft: fine-tune among the names that keep their route, and the README entry matches the reworded paragraph. Signed-off-by: Vladislav Perevezentsev --- ...vezentsev_openai-compatible-model-route.md | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md index 61a562950..ed8680483 100644 --- a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -1,19 +1,27 @@ ### Fixed -- **`--provider openai` reaches an OpenAI-compatible endpoint with any model name** — a - model that litellm does not list as OpenAI's (`glm-5.3` behind `OPENAI_API_BASE`, an - `org/name` id such as `meta-llama/Llama-3.1-8B`, a model newer than the installed - litellm) was sent bare, and litellm refused it ("LLM Provider NOT provided") before any - request; a bare `claude-*` name went to Anthropic instead. Such names now go out as - `openai/`, on every path that takes the openai provider: answers, the judge and - panel, `eval cross-judge --judges openai/` (#136), whose arms read NOT MEASURED - before, and question and persona generation. A name on litellm's OpenAI lists - (`gpt-6.1-sol`, `o3`), or one with a litellm provider prefix (`azure/…`), keeps its route - and its response-cache entries. The double-prefix workaround (`openai/openai/`) still - works. +- **`--provider openai` reaches an OpenAI-compatible endpoint with a model litellm does not + know as OpenAI's** — such a model (`glm-5.3` behind `OPENAI_API_BASE`, an `org/name` id + such as `meta-llama/Llama-3.1-8B`, a model newer than the installed litellm) was sent bare, + and litellm refused it ("LLM Provider NOT provided") before any request. Such names now go + out as `openai/`, on every path that takes the openai provider: answers, the judge + and panel, `eval cross-judge --judges openai/` (#136), whose arms read NOT MEASURED + before, and question and persona generation. A name litellm takes for OpenAI's + (`gpt-6.1-sol`, `o3`, a `ft:gpt-4o-mini:…` fine-tune), or one with a litellm provider prefix + (`azure/…`, `ollama/…`), keeps its route and its response-cache entries. The double-prefix + workaround (`openai/openai/`) still works. +- **`--provider openai` no longer hands a bare name to another cloud with the OpenAI key** — + a bare name that litellm assigns to another provider (about 450 in litellm 1.106: `claude-*` + to Anthropic, `amazon.nova-pro-v1:0` and the other Bedrock ids, `gemini-*` to Vertex, Cohere + and AI21 names) went to that provider with `OPENAI_API_KEY` as its key; Anthropic and + Bedrock refuse it, so such a run worked only while `OPENAI_API_KEY` was unset. It now goes + to the OpenAI endpoint like any other name litellm does not know as OpenAI's. To reach those + clouds, use `--provider anthropic`, `amazon-bedrock` or `google-vertex`. ### Documentation - **README: running against an OpenAI-compatible server** — provider `openai` with - `OPENAI_API_BASE` / `OPENAI_BASE_URL`, a placeholder `OPENAI_API_KEY` for a server that - ignores it, and why cost is unknown and `--max-usd` stops after one call. + `OPENAI_API_BASE` / `OPENAI_BASE_URL`, which names reach the server and which keep a + litellm provider prefix's route (write `openai/` for those), a placeholder + `OPENAI_API_KEY` for a server that ignores it, and why cost is unknown and + `novelty --max-usd` stops after one call. From ea2acaa54af538e825477d5b6fc7a0d690601db0 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 05:51:00 -0700 Subject: [PATCH 09/13] test(eval): re-pin the grader cache keys for gpt-x, now routed as openai/gpt-x test_grader_characterization (#251) pins the response-cache file of a judge and a panel rerun at model gpt-x, provider openai. gpt-x is off litellm's OpenAI lists, so this branch sends it as openai/gpt-x, and the key, which holds the litellm model string, changes with it. The new names are the keys main computes for model openai/gpt-x: only the route moved, the messages and draws behind the key are the same. Signed-off-by: Vladislav Perevezentsev --- dokimi/tests/test_grader_characterization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_grader_characterization.py b/dokimi/tests/test_grader_characterization.py index 006beed0f..c6a5d82f3 100644 --- a/dokimi/tests/test_grader_characterization.py +++ b/dokimi/tests/test_grader_characterization.py @@ -218,7 +218,7 @@ def test_judge_rerun_with_the_cache_on_replays_the_stored_redraw(provider_calls, provider_calls.push(BAD, REPLY) judge = _judge(**PINNED) _grade_judge(judge) - assert _files(cache_dir) == ["f0cd923dc4fd72e7e4746e77505f11ad80878e92ee2190c52df6c64fe78b531f.json"] + assert _files(cache_dir) == ["c97a0784894b371c59416967e36e4bb1a588852a38ac890c8f2b57cb827e7c79.json"] assert sent == [ (JUDGE_PROMPT, 0, False, (True, False), 0.0, 7), (JUDGE_REMINDER_1, 1, True, (True, False), 0.0, 7), @@ -346,7 +346,7 @@ def test_panel_rerun_with_the_cache_on_replays_the_stored_redraw(provider_calls, provider_calls.push(BAD, REPLY) panel, cfg = _panel(**PINNED) _grade_panel(panel, cfg) - assert _files(cache_dir) == ["50554ebc560c85cc138281ff31f5245b8208cc9f260ced6a82c763814cb7c883.json"] + assert _files(cache_dir) == ["02dfaa4529448c9110c789746538d8b9f6af6a310cd549fd5fa1a0a00276f3b0.json"] assert len(sent) == 5 # three replay probes, then the two live draws assert len(provider_calls.calls) == 2 From 4cc10a6402bc463139eaf83ac748fe0997be03d1 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 22:23:02 -0700 Subject: [PATCH 10/13] fix(llm): keep the route of a provider prefix litellm registers in JSON litellm also routes providers from its openai_like JSON registry (crusoe/, sarvam/, ...), which are not in provider_list. On provider openai such a name got openai/ and went to OPENAI_API_BASE instead. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/llm.py | 2 +- dokimi/tests/test_llm.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 667da276c..bd6620836 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -144,7 +144,7 @@ def resolve_model(model: str, provider: str) -> str: import litellm known = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models ours = model in known or "ft:gpt-3.5-turbo" in model or "ft:gpt-4" in model # as litellm - routed = model.split("/", 1)[0] in litellm.provider_list if "/" in model else ours + routed = (p := model.split("/", 1)[0]) in litellm.provider_list or litellm.JSONProviderRegistry.exists(p) if "/" in model else ours return model if routed else f"openai/{model}" if "/" in model: return model diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 6de162d7c..a2a266d1e 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -66,6 +66,7 @@ def _no_ambient_bedrock_env(monkeypatch): ("openai", "openai/glm-5.3", "openai/glm-5.3"), ("openai", "azure/gpt-4o", "azure/gpt-4o"), # an explicit litellm route passes through ("openai", "ollama/llama3", "ollama/llama3"), # any litellm provider prefix passes through + ("openai", "crusoe/x", "crusoe/x"), # a provider litellm registers in JSON, not provider_list ("openai-codex", "gpt-5", "openai/gpt-5"), ("anthropic", "claude-sonnet-5-5", "anthropic/claude-sonnet-5-5"), ("openrouter", "openai/gpt-oss-120b", "openrouter/openai/gpt-oss-120b"), From 01406a0f36b8c82115f09d842a30d9be276602f7 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 12:08:57 +0000 Subject: [PATCH 11/13] test(llm): pin the ft:gpt-3.5-turbo fine-tune route Signed-off-by: Anton Volkov --- dokimi/tests/test_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index a2a266d1e..03a13193d 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -59,6 +59,7 @@ def _no_ambient_bedrock_env(monkeypatch): ("openai", "gpt-6.1-sol", "gpt-6.1-sol"), # on litellm's lists: bare, so its cache key holds ("openai", "gpt-3.5-turbo-instruct", "gpt-3.5-turbo-instruct"), # litellm's text-completion route ("openai", "ft:gpt-4o-mini:org::id", "ft:gpt-4o-mini:org::id"), # a fine-tune litellm takes as OpenAI's + ("openai", "ft:gpt-3.5-turbo:org::id", "ft:gpt-3.5-turbo:org::id"), # the other fine-tune family litellm takes as OpenAI's ("openai", "glm-5.3", "openai/glm-5.3"), # off the lists: an OPENAI_API_BASE server's model ("openai", "claude-sonnet-5-5", "openai/claude-sonnet-5-5"), # bare, litellm would pick anthropic ("openai", "amazon.nova-pro-v1:0", "openai/amazon.nova-pro-v1:0"), # bare, litellm would pick bedrock From 5b6b90c916d9ce7679d2ca510b09f89db1847696 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 12:09:52 +0000 Subject: [PATCH 12/13] docs(changelog): name docs/reviewing.md, not README, in the entry Signed-off-by: Anton Volkov --- ...7_120000_vlad-perevezentsev_openai-compatible-model-route.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md index ed8680483..a705392ad 100644 --- a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -20,7 +20,7 @@ ### Documentation -- **README: running against an OpenAI-compatible server** — provider `openai` with +- **`docs/reviewing.md`: running against an OpenAI-compatible server** — provider `openai` with `OPENAI_API_BASE` / `OPENAI_BASE_URL`, which names reach the server and which keep a litellm provider prefix's route (write `openai/` for those), a placeholder `OPENAI_API_KEY` for a server that ignores it, and why cost is unknown and From 629669767a0b333161509c93aece99652efa17de Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 29 Oct 2026 06:04:23 -0700 Subject: [PATCH 13/13] fix(llm): key a provider-openai reply on the model name as given, not on its route Review of #256 (xaleryb): the openai route of a bare name depends on litellm's price map. litellm loads the remote map, or the installed backup under LITELLM_LOCAL_MODEL_COST_MAP=True or offline, and the two disagree on a new model: gpt-6.1-terra is on the remote one's OpenAI list (206 names) and not on the backup's (197), so it went out bare in one run and as openai/gpt-6.1-terra in the other. Both reach the same endpoint, but the response-cache key held the route, so one model and prompt had two keys, and a rerun on the other map paid for every call again and could give other numbers. On provider openai the key now holds self.model, which is what main keyed on, since main sent every openai name bare. Every key main wrote for that provider still hits, including those of the names this PR reroutes, and the map no longer moves it. Other providers are unchanged. OPENAI_API_BASE was never part of the key, so keying on the name loses nothing the route held. - test_grader_characterization.py: the two file names ea2acaa re-pinned for openai/gpt-x are back to main's, so the file matches main again. - test_llm.py: main's key for gpt-6.1-terra (routed openai/ under the installed map the tests use), recorded by running main's code. - test_openai_compatible_route.py: a reply stored with the model on the map replays once it is off the map and the route is openai/. - llm.py stays at 1118 lines: the response-cache paragraph of the module docstring, which names the key's parts, is rewrapped to the project's 100 columns. Signed-off-by: Vladislav Perevezentsev --- ...vezentsev_openai-compatible-model-route.md | 7 +++++-- dokimi/dokimi/llm.py | 18 +++++++++--------- dokimi/tests/test_grader_characterization.py | 4 ++-- dokimi/tests/test_llm.py | 5 ++++- dokimi/tests/test_openai_compatible_route.py | 19 +++++++++++++++++++ 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md index a705392ad..63ebbba20 100644 --- a/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md +++ b/dokimi/changelog.d/20261027_120000_vlad-perevezentsev_openai-compatible-model-route.md @@ -8,8 +8,11 @@ and panel, `eval cross-judge --judges openai/` (#136), whose arms read NOT MEASURED before, and question and persona generation. A name litellm takes for OpenAI's (`gpt-6.1-sol`, `o3`, a `ft:gpt-4o-mini:…` fine-tune), or one with a litellm provider prefix - (`azure/…`, `ollama/…`), keeps its route and its response-cache entries. The double-prefix - workaround (`openai/openai/`) still works. + (`azure/…`, `ollama/…`), keeps its route. Every name keeps its response-cache entries: on + provider openai the key holds the model name as given, not the route, so it does not move + with this fix or with litellm's price map (the remote one or the installed backup, which + can disagree on a new model). The double-prefix workaround (`openai/openai/`) still + works. - **`--provider openai` no longer hands a bare name to another cloud with the OpenAI key** — a bare name that litellm assigns to another provider (about 450 in litellm 1.106: `claude-*` to Anthropic, `amazon.nova-pro-v1:0` and the other Bedrock ids, `gemini-*` to Vertex, Cohere diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index bd6620836..b5afaca10 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -39,13 +39,13 @@ - **Retry.** Transient errors (:func:`is_retryable`) are retried ``max_retries`` times with doubling backoff; anything else is raised on the first attempt. -- **Response cache** (:mod:`dokimi.caching`, off unless - ``DOKIMI_RESPONSE_CACHE`` is set). ``complete`` is cached, ``chat`` never is: - replaying a completion would skip the tool calls an agent loop measures. The - key holds the litellm model, the messages, ``stream`` and ``max_tokens``, and - ``temperature``/``seed`` when set; ``draw=n`` is an independent sample with its - own entry (draw 0 adds nothing). ``cache_if(text)`` decides whether a reply may - be stored *or replayed*. A replay returns the stored usage record. +- **Response cache** (:mod:`dokimi.caching`, off unless ``DOKIMI_RESPONSE_CACHE`` is set). + ``complete`` is cached, ``chat`` never is: replaying a completion would skip the tool calls an + agent loop measures. The key holds the litellm model (on openai the caller's name, as the route + there moves with litellm's price map), the messages, ``stream`` and ``max_tokens``, and + ``temperature``/``seed`` when set; ``draw=n`` is an independent sample with its own entry + (draw 0 adds nothing). ``cache_if(text)`` decides whether a reply may be stored *or + replayed*. A replay returns the stored usage record. - **Usage.** Every reply carries a :class:`UsageRecord`; ``cost_usd`` is ``None`` when litellm has no price for the model, never a fake ``0.0``. - **Truncation.** A live ``complete``/``ask`` reply cut at the output ceiling @@ -707,8 +707,8 @@ def complete( model=self.model, provider=self.provider, ) key = caching.response_cache_key( - litellm_model=self.litellm_model, messages=messages, - extra=caching.draw_extra(self._cache_extra(stream, max_tokens), draw), + litellm_model=self.model if _canonical(self.provider) == "openai" else self.litellm_model, + messages=messages, extra=caching.draw_extra(self._cache_extra(stream, max_tokens), draw), ) hit = caching.load_response(key) if hit is not None and caching.keeps(cache_if, cast(str, hit[0])): diff --git a/dokimi/tests/test_grader_characterization.py b/dokimi/tests/test_grader_characterization.py index c6a5d82f3..006beed0f 100644 --- a/dokimi/tests/test_grader_characterization.py +++ b/dokimi/tests/test_grader_characterization.py @@ -218,7 +218,7 @@ def test_judge_rerun_with_the_cache_on_replays_the_stored_redraw(provider_calls, provider_calls.push(BAD, REPLY) judge = _judge(**PINNED) _grade_judge(judge) - assert _files(cache_dir) == ["c97a0784894b371c59416967e36e4bb1a588852a38ac890c8f2b57cb827e7c79.json"] + assert _files(cache_dir) == ["f0cd923dc4fd72e7e4746e77505f11ad80878e92ee2190c52df6c64fe78b531f.json"] assert sent == [ (JUDGE_PROMPT, 0, False, (True, False), 0.0, 7), (JUDGE_REMINDER_1, 1, True, (True, False), 0.0, 7), @@ -346,7 +346,7 @@ def test_panel_rerun_with_the_cache_on_replays_the_stored_redraw(provider_calls, provider_calls.push(BAD, REPLY) panel, cfg = _panel(**PINNED) _grade_panel(panel, cfg) - assert _files(cache_dir) == ["02dfaa4529448c9110c789746538d8b9f6af6a310cd549fd5fa1a0a00276f3b0.json"] + assert _files(cache_dir) == ["50554ebc560c85cc138281ff31f5245b8208cc9f260ced6a82c763814cb7c883.json"] assert len(sent) == 5 # three replay probes, then the two live draws assert len(provider_calls.calls) == 2 diff --git a/dokimi/tests/test_llm.py b/dokimi/tests/test_llm.py index 03a13193d..675b3a3f4 100644 --- a/dokimi/tests/test_llm.py +++ b/dokimi/tests/test_llm.py @@ -270,12 +270,15 @@ def broken(model): ("22510fd74bf01210757177f302f9b6728b81e9e3b31572bd6245f0c88af78dfd", lambda: get_llm("openrouter", "openai/gpt-oss-120b", "k", temperature=0.0).invoke( "ask", draw=1)), + # Off the installed price map, so routed as openai/gpt-6.1-terra here; main sent it bare. + ("c6dc142638019230bb9f5876e709302f8eafd63ea9a22a5990bc086fd7d07e3e", + lambda: LLM("gpt-6.1-terra", "openai", api_key="k").complete("ask")), ] @pytest.mark.parametrize("key, call", MAIN_WRITTEN, ids=["plain", "system+prefix", "pinned+max_tokens+draw", "stream", - "get_llm-bedrock", "get_llm-openrouter"]) + "get_llm-bedrock", "get_llm-openrouter", "openai-rerouted"]) def test_an_entry_main_wrote_is_hit(key, call, cache_dir, fake_llm): usage = UsageRecord(total_tokens=7, cost_usd=0.002, cost_known_calls=1) (cache_dir / f"{key}.json").write_text( diff --git a/dokimi/tests/test_openai_compatible_route.py b/dokimi/tests/test_openai_compatible_route.py index 156612a27..a0d283e33 100644 --- a/dokimi/tests/test_openai_compatible_route.py +++ b/dokimi/tests/test_openai_compatible_route.py @@ -52,6 +52,25 @@ def test_a_name_litellm_takes_for_openais_goes_out_bare_and_still_routes(litellm assert litellm_routes == [model] +def test_a_reply_cached_under_one_litellm_price_map_replays_under_the_other(litellm_routes, monkeypatch, tmp_path): + """A new model is on litellm's remote price map but not on its installed backup + (``LITELLM_LOCAL_MODEL_COST_MAP``): the route is bare under one and ``openai/`` under the + other, and the response-cache key must not follow it.""" + import litellm + + from dokimi import caching + + monkeypatch.setenv(caching.RESPONSE_CACHE_ENV, str(tmp_path)) + listed = set(litellm.open_ai_chat_completion_models) - {"gpt-6.1-terra"} + monkeypatch.setattr(litellm, "open_ai_chat_completion_models", listed | {"gpt-6.1-terra"}) + assert LLM("gpt-6.1-terra", "openai").ask("Reply with OK.") == JUDGE_REPLY + monkeypatch.setattr(litellm, "open_ai_chat_completion_models", listed) + off_map = LLM("gpt-6.1-terra", "openai") + assert off_map.litellm_model == "openai/gpt-6.1-terra" + assert off_map.ask("Reply with OK.") == JUDGE_REPLY + assert litellm_routes == ["gpt-6.1-terra"] # the second ask replayed, nothing sent + + def test_cross_judge_scores_with_an_openai_compatible_judge(litellm_routes, tmp_path): """The reported command: before the fix every score failed and the arm read NOT MEASURED.""" from dokimi.cli import build_parser