feat(gooddata-eval): optional run_metadata_extra on agentic evaluators

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -383,14 +383,23 @@ def build_run_context(
     dataset_name: str,
     run_timestamp: str | None,
     model_version_override: str | None,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> tuple[str, dict[str, Any]]:
-    """Return (run_name_base, run_metadata) with model version resolved from workspace API."""
+    """Return (run_name_base, run_metadata) with model version resolved from workspace API.
+
+    :param run_metadata_extra: optional extra key/values added to the dataset-run
+        metadata (e.g. a CI run id for scoping). Default None keeps behavior unchanged.
+        SDK-owned keys ("testing_framework", "model_version") always take precedence and
+        cannot be overwritten by this dict.
+    """
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
-    metadata: dict[str, Any] = {"testing_framework": "tavern-e2e"}
+    # Start from caller extras, then set SDK-owned keys last so they always win.
+    metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
+    metadata["testing_framework"] = "tavern-e2e"
     if model:
         metadata["model_version"] = model
     return base, metadata

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -438,6 +438,7 @@ def evaluate_agentic_alert_skill(
     dataset_name: str = "alert_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -475,6 +476,7 @@ def evaluate_agentic_alert_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -384,6 +384,7 @@ def evaluate_agentic_conversation(
     dataset_name: str = "conversation",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -419,6 +420,7 @@ def evaluate_agentic_conversation(
             dataset_name or fixture.dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -152,6 +152,7 @@ def evaluate_agentic_general_question(
     dataset_name: str = "general_question",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -188,6 +189,7 @@ def evaluate_agentic_general_question(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -149,6 +149,7 @@ def evaluate_agentic_guardrail(
     dataset_name: str = "guardrail",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -185,6 +186,7 @@ def evaluate_agentic_guardrail(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -244,6 +244,7 @@ def evaluate_agentic_metric_skill(
     dataset_name: str = "metric_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -281,6 +282,7 @@ def evaluate_agentic_metric_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -143,6 +143,7 @@ def evaluate_agentic_search_tool(
     dataset_name: str = "search",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -179,6 +180,7 @@ def evaluate_agentic_search_tool(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -263,6 +263,7 @@ def evaluate_agentic_visualization(
     dataset_name: str = "visualization",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
     record_output_path: str | None = None,
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
@@ -302,6 +303,7 @@ def evaluate_agentic_visualization(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         K = len(summary.run_results)
         traces_by_conv = find_traces_per_conversation(

packages/gooddata-eval/tests/test_agentic_run_context.py

@@ -0,0 +1,37 @@
+# (C) 2026 GoodData Corporation
+from gooddata_eval.core.agentic._langfuse import build_run_context
+
+# model_version_override short-circuits get_model_version (no workspace API call).
+_COMMON = {
+    "host": "h",
+    "token": "t",
+    "workspace_id": "w",
+    "dataset_name": "ds",
+    "run_timestamp": "2026-07-10_00-00-00",
+    "model_version_override": "m",
+}
+
+
+def test_build_run_context_merges_run_metadata_extra():
+    base, metadata = build_run_context(**_COMMON, run_metadata_extra={"github_run_id": "run-123"})
+    assert base == "ds_2026-07-10_00-00-00_m"
+    assert metadata["github_run_id"] == "run-123"
+    assert metadata["model_version"] == "m"
+    assert metadata["testing_framework"] == "tavern-e2e"
+
+
+def test_build_run_context_extra_cannot_override_reserved_keys():
+    _, metadata = build_run_context(
+        **_COMMON,
+        run_metadata_extra={"github_run_id": "run-123", "testing_framework": "hack", "model_version": "hack"},
+    )
+    # SDK-owned keys win; caller extras only add non-reserved keys.
+    assert metadata["testing_framework"] == "tavern-e2e"
+    assert metadata["model_version"] == "m"
+    assert metadata["github_run_id"] == "run-123"
+
+
+def test_build_run_context_without_extra_is_unchanged():
+    _, metadata = build_run_context(**_COMMON)
+    assert "github_run_id" not in metadata
+    assert metadata == {"testing_framework": "tavern-e2e", "model_version": "m"}