diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index f20abacb5..fd6e8d4e9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -305,7 +305,7 @@ def observe( run_name=run_name, dataset_item_id=dataset_item_id, trace_id=trace_id, - metadata=run_metadata or {"testing_framework": "tavern-e2e"}, + metadata=run_metadata or {}, run_description="", ) _log.debug( @@ -383,14 +383,31 @@ def build_run_context( dataset_name: str, run_timestamp: str | None, model_version_override: str | None, + run_metadata_extra: dict[str, Any] | None = None, ) -> tuple[str, dict[str, Any]]: - """Return (run_name_base, run_metadata) with model version resolved from workspace API.""" + """Return (run_name_base, run_metadata) with model version resolved from workspace API. + + Args: + host: GoodData host URL. + token: API token used to resolve the workspace model version. + workspace_id: Workspace whose active LLM provider is read when no override is given. + dataset_name: Langfuse dataset name; used as the run-name prefix. + run_timestamp: Shared run timestamp for the run name (falls back to the current time). + model_version_override: Explicit model-version tag; when set it wins over the + workspace lookup. + run_metadata_extra: Optional extra key/values added to the dataset-run metadata + (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps + behavior unchanged. The SDK-derived model_version is applied last and cannot + be overwritten by this dict. + """ model = get_model_version(host, token, workspace_id, model_version_override) ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S") base = f"{dataset_name}_{ts}" if model: base = f"{base}_{model}" - metadata: dict[str, Any] = {"testing_framework": "tavern-e2e"} + # Caller supplies its own run tags (e.g. testing_framework); model_version is applied + # last so the SDK-derived value cannot be overwritten by run_metadata_extra. + metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {} if model: metadata["model_version"] = model return base, metadata diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index c3a9df936..3a6b62d16 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -438,6 +438,7 @@ def evaluate_agentic_alert_skill( dataset_name: str = "alert_skill", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -475,6 +476,7 @@ def evaluate_agentic_alert_skill( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 5ae5668c5..8d18ad3ac 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -384,6 +384,7 @@ def evaluate_agentic_conversation( dataset_name: str = "conversation", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run conversation evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -419,6 +420,7 @@ def evaluate_agentic_conversation( dataset_name or fixture.dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index a50816f53..653a77956 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -152,6 +152,7 @@ def evaluate_agentic_general_question( dataset_name: str = "general_question", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run general-question evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -188,6 +189,7 @@ def evaluate_agentic_general_question( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index b6bc7ac81..cb61da24a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -149,6 +149,7 @@ def evaluate_agentic_guardrail( dataset_name: str = "guardrail", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run guardrail evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -185,6 +186,7 @@ def evaluate_agentic_guardrail( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2fca3eb5b..e15f6d0c0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -244,6 +244,7 @@ def evaluate_agentic_metric_skill( dataset_name: str = "metric_skill", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -281,6 +282,7 @@ def evaluate_agentic_metric_skill( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 0fe63664e..cdbce48ab 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -143,6 +143,7 @@ def evaluate_agentic_search_tool( dataset_name: str = "search", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -179,6 +180,7 @@ def evaluate_agentic_search_tool( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 6f2f510a2..12914157a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -263,6 +263,7 @@ def evaluate_agentic_visualization( dataset_name: str = "visualization", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, record_output_path: str | None = None, ) -> None: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.""" @@ -302,6 +303,7 @@ def evaluate_agentic_visualization( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) K = len(summary.run_results) traces_by_conv = find_traces_per_conversation( diff --git a/packages/gooddata-eval/tests/test_agentic_run_context.py b/packages/gooddata-eval/tests/test_agentic_run_context.py new file mode 100644 index 000000000..dc584ebf3 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_run_context.py @@ -0,0 +1,32 @@ +# (C) 2026 GoodData Corporation +from gooddata_eval.core.agentic._langfuse import build_run_context + +# model_version_override short-circuits get_model_version (no workspace API call). +_COMMON = { + "host": "h", + "token": "t", + "workspace_id": "w", + "dataset_name": "ds", + "run_timestamp": "2026-07-10_00-00-00", + "model_version_override": "m", +} + + +def test_build_run_context_includes_caller_extra_keys(): + base, metadata = build_run_context( + **_COMMON, run_metadata_extra={"testing_framework": "tavern-e2e", "github_run_id": "run-123"} + ) + assert base == "ds_2026-07-10_00-00-00_m" + assert metadata["testing_framework"] == "tavern-e2e" + assert metadata["github_run_id"] == "run-123" + assert metadata["model_version"] == "m" + + +def test_build_run_context_extra_cannot_override_model_version(): + _, metadata = build_run_context(**_COMMON, run_metadata_extra={"model_version": "hack"}) + assert metadata["model_version"] == "m" # SDK-derived value wins + + +def test_build_run_context_without_extra_has_only_model_version(): + _, metadata = build_run_context(**_COMMON) + assert metadata == {"model_version": "m"}