From d1f91891da37b3e3f407b58a1a82cbc518e5cdbb Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Fri, 10 Jul 2026 20:09:05 +0700 Subject: [PATCH] feat(gooddata-eval): optional run_metadata_extra on agentic evaluators Add an optional run_metadata_extra parameter to build_run_context and every evaluate_agentic_* entrypoint. When provided, its keys are added to the Langfuse dataset-run metadata; default None keeps behavior byte-for-byte identical, so existing callers are unaffected. SDK-owned keys ("testing_framework", "model_version") always take precedence and cannot be overwritten by the extra dict. Enables a caller (e.g. gdc-nas tavern-e2e CI) to stamp a github_run_id on runs for exact report isolation, without the SDK reading any ambient env itself. Includes unit coverage for the merge, reserved-key precedence, and default no-op paths. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../gooddata_eval/core/agentic/_langfuse.py | 13 ++++++- .../gooddata_eval/core/agentic/alert_skill.py | 2 + .../core/agentic/conversation.py | 2 + .../core/agentic/general_question.py | 2 + .../gooddata_eval/core/agentic/guardrail.py | 2 + .../core/agentic/metric_skill.py | 2 + .../gooddata_eval/core/agentic/search_tool.py | 2 + .../core/agentic/visualization.py | 2 + .../tests/test_agentic_run_context.py | 37 +++++++++++++++++++ 9 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_agentic_run_context.py diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index f20abacb5..5e760e009 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -383,14 +383,23 @@ def build_run_context( dataset_name: str, run_timestamp: str | None, model_version_override: str | None, + run_metadata_extra: dict[str, Any] | None = None, ) -> tuple[str, dict[str, Any]]: - """Return (run_name_base, run_metadata) with model version resolved from workspace API.""" + """Return (run_name_base, run_metadata) with model version resolved from workspace API. + + :param run_metadata_extra: optional extra key/values added to the dataset-run + metadata (e.g. a CI run id for scoping). Default None keeps behavior unchanged. + SDK-owned keys ("testing_framework", "model_version") always take precedence and + cannot be overwritten by this dict. + """ model = get_model_version(host, token, workspace_id, model_version_override) ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S") base = f"{dataset_name}_{ts}" if model: base = f"{base}_{model}" - metadata: dict[str, Any] = {"testing_framework": "tavern-e2e"} + # Start from caller extras, then set SDK-owned keys last so they always win. + metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {} + metadata["testing_framework"] = "tavern-e2e" if model: metadata["model_version"] = model return base, metadata diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index c3a9df936..3a6b62d16 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -438,6 +438,7 @@ def evaluate_agentic_alert_skill( dataset_name: str = "alert_skill", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -475,6 +476,7 @@ def evaluate_agentic_alert_skill( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 5ae5668c5..8d18ad3ac 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -384,6 +384,7 @@ def evaluate_agentic_conversation( dataset_name: str = "conversation", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run conversation evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -419,6 +420,7 @@ def evaluate_agentic_conversation( dataset_name or fixture.dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index a50816f53..653a77956 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -152,6 +152,7 @@ def evaluate_agentic_general_question( dataset_name: str = "general_question", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run general-question evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -188,6 +189,7 @@ def evaluate_agentic_general_question( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index b6bc7ac81..cb61da24a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -149,6 +149,7 @@ def evaluate_agentic_guardrail( dataset_name: str = "guardrail", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run guardrail evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -185,6 +186,7 @@ def evaluate_agentic_guardrail( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2fca3eb5b..e15f6d0c0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -244,6 +244,7 @@ def evaluate_agentic_metric_skill( dataset_name: str = "metric_skill", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -281,6 +282,7 @@ def evaluate_agentic_metric_skill( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 0fe63664e..cdbce48ab 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -143,6 +143,7 @@ def evaluate_agentic_search_tool( dataset_name: str = "search", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, ) -> None: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -179,6 +180,7 @@ def evaluate_agentic_search_tool( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 6f2f510a2..12914157a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -263,6 +263,7 @@ def evaluate_agentic_visualization( dataset_name: str = "visualization", run_timestamp: str | None = None, model_version_override: str | None = None, + run_metadata_extra: dict | None = None, record_output_path: str | None = None, ) -> None: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.""" @@ -302,6 +303,7 @@ def evaluate_agentic_visualization( dataset_name, run_timestamp, model_version_override, + run_metadata_extra, ) K = len(summary.run_results) traces_by_conv = find_traces_per_conversation( diff --git a/packages/gooddata-eval/tests/test_agentic_run_context.py b/packages/gooddata-eval/tests/test_agentic_run_context.py new file mode 100644 index 000000000..8bb6fec93 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_run_context.py @@ -0,0 +1,37 @@ +# (C) 2026 GoodData Corporation +from gooddata_eval.core.agentic._langfuse import build_run_context + +# model_version_override short-circuits get_model_version (no workspace API call). +_COMMON = { + "host": "h", + "token": "t", + "workspace_id": "w", + "dataset_name": "ds", + "run_timestamp": "2026-07-10_00-00-00", + "model_version_override": "m", +} + + +def test_build_run_context_merges_run_metadata_extra(): + base, metadata = build_run_context(**_COMMON, run_metadata_extra={"github_run_id": "run-123"}) + assert base == "ds_2026-07-10_00-00-00_m" + assert metadata["github_run_id"] == "run-123" + assert metadata["model_version"] == "m" + assert metadata["testing_framework"] == "tavern-e2e" + + +def test_build_run_context_extra_cannot_override_reserved_keys(): + _, metadata = build_run_context( + **_COMMON, + run_metadata_extra={"github_run_id": "run-123", "testing_framework": "hack", "model_version": "hack"}, + ) + # SDK-owned keys win; caller extras only add non-reserved keys. + assert metadata["testing_framework"] == "tavern-e2e" + assert metadata["model_version"] == "m" + assert metadata["github_run_id"] == "run-123" + + +def test_build_run_context_without_extra_is_unchanged(): + _, metadata = build_run_context(**_COMMON) + assert "github_run_id" not in metadata + assert metadata == {"testing_framework": "tavern-e2e", "model_version": "m"}