From 70dc70758f0d6cab5a0eb084ae08b38fc6aaa8e1 Mon Sep 17 00:00:00 2001 From: Saarthak Verma Date: Wed, 19 Aug 2026 18:17:46 +0530 Subject: [PATCH 1/4] create skill quality metric --- fasteval/__init__.py | 3 +- fasteval/core/__init__.py | 2 + fasteval/core/decorators.py | 25 +++++++ fasteval/core/evaluator.py | 2 + fasteval/metrics/conversation.py | 114 +++++++++++++++++++++++++++++ tests/test_conversation_metrics.py | 19 +++++ uv.lock | 2 +- 7 files changed, 165 insertions(+), 2 deletions(-) diff --git a/fasteval/__init__.py b/fasteval/__init__.py index 7b9d553..e4d8d3f 100644 --- a/fasteval/__init__.py +++ b/fasteval/__init__.py @@ -81,6 +81,7 @@ def test_qa(): relevance, rouge, safety_check, + skill_quality, speaker_diarization, stack, table_extraction, @@ -214,7 +215,7 @@ def test_qa(): "context_retention", "consistency", "topic_drift", - "metric", + "skill_quality" "metric", # Code-as-Judge "judge", # Data Decorators diff --git a/fasteval/core/__init__.py b/fasteval/core/__init__.py index 325d8c8..d676f29 100644 --- a/fasteval/core/__init__.py +++ b/fasteval/core/__init__.py @@ -16,6 +16,7 @@ metric, relevance, rouge, + skill_quality, topic_drift, toxicity, traces, @@ -41,6 +42,7 @@ "context_retention", "consistency", "topic_drift", + "skill_quality", # Data Decorators "csv", "conversation", diff --git a/fasteval/core/decorators.py b/fasteval/core/decorators.py index c22fe8b..2648fb8 100644 --- a/fasteval/core/decorators.py +++ b/fasteval/core/decorators.py @@ -1006,6 +1006,31 @@ async def test_on_topic(): ) +def skill_quality( + threshold: float = 0.7, + weight: float = 1.0, + name: Optional[str] = None, + skill: Optional[str] = None, + additional_skill_eval_criteria: Optional[str] = None, + llm_client: Optional[LLMClient] = None, + model: Optional[str] = None, + **config: Any, +) -> Callable[[F], F]: + + return _metric_decorator_factory( + "skill_quality", "skill_quality", default_threshold=0.7 + )( + threshold=threshold, + weight=weight, + name=name, + llm_client=llm_client, + model=model, + skill=skill, + additional_skill_eval_criteria=additional_skill_eval_criteria, + **config, + ) + + # === Vision Metric Decorators === diff --git a/fasteval/core/evaluator.py b/fasteval/core/evaluator.py index d880bb7..7f1fec6 100644 --- a/fasteval/core/evaluator.py +++ b/fasteval/core/evaluator.py @@ -11,6 +11,7 @@ from fasteval.metrics.conversation import ( ConsistencyMetric, ContextRetentionMetric, + SkillQualityMetric, TopicDriftMetric, ) from fasteval.metrics.deterministic import ( # Tool Trajectory Metrics @@ -82,6 +83,7 @@ "context_retention": ContextRetentionMetric, "consistency": ConsistencyMetric, "topic_drift": TopicDriftMetric, + "skill_quality": SkillQualityMetric, } # Register vision metrics if available diff --git a/fasteval/metrics/conversation.py b/fasteval/metrics/conversation.py index da47792..b677f5d 100644 --- a/fasteval/metrics/conversation.py +++ b/fasteval/metrics/conversation.py @@ -324,3 +324,117 @@ def get_evaluation_prompt(self, eval_input: EvalInput) -> str: ## OUTPUT FORMAT Respond with JSON only: {{"score": <0.0 to 1.0>, "reasoning": ""}}""" + + +class SkillQualityMetric(BaseLLMMetric): + def __init__( + self, + skill: str, + name: str = "skill_quality", + threshold: float = 0.8, + additional_skill_eval_criteria: str | None = None, + **kwargs: Any, + ) -> None: + super().__init__(name=name, threshold=threshold, **kwargs) + self.skill = skill + self.additional_skill_eval_criteria = additional_skill_eval_criteria + + def get_evaluation_prompt(self, eval_input: EvalInput) -> str: + + history_str = "" + if eval_input.history: + history_str = "\n".join( + f"**{turn.get('role', 'unknown').title()}**: {turn.get('content', '')}" + for turn in eval_input.history + ) + history_str = f"\n{history_str}\n" + expected_output = eval_input.expected_output or "N/A" + skill = self.skill + specialized_criteria = self.additional_skill_eval_criteria or "N/A" + + return f""" + +{history_str} + + + +{expected_output} + + + +{skill} + + + +{specialized_criteria} + + +You are a judge evaluating whether a SKILL (packaged metadata/instructions/resources) is +well designed — not whether the agent using it is competent. Your job is to find defects +in the SKILL for iterative improvement, not to grade the agent. + +## Root-cause rule (applies to every criterion below) +When the agent's behavior falls short, decide WHY before scoring: +- Traces back to something missing/ambiguous/wrong in the skill → skill defect → score 0. +- Skill was clear and adequate, agent just ignored/misread/hallucinated anyway → agent + failure, not the skill's fault → score 1 (note it in reasoning so it isn't mistaken for + a needed skill fix). +- If you cannot point to specific text in the skill that explains the failure, default to + 1 and say so — do not assume a skill defect without textual evidence. + +## Skill loading model (judge accordingly) +- Metadata (name + description): visible to the agent at ALL times, before + any decision to use the skill. +- Instructions (full SKILL.md body): loaded only once the skill is activated. +- Resources (files under scripts/, references/, assets/): loaded on demand, only if the + task needs that depth. +Judge each criterion against the layer the agent actually had access to at that point. + +## Input format (data supplied above, inside tags) +- : conversation between user and agent. +- (optional): reference conversation showing correct invocation and/or + correct final output. +- : skill under test, parseable into metadata / instructions / resources. +- (optional): additional use-case-specific criteria, one per entry. + +Evaluate each criterion independently. Do NOT sum or average scores yourself — return a +structured verdict per criterion; aggregation happens outside this prompt. + +## Criteria + +1. SKILL_MATCHING (always applicable) — judges the metadata layer only + Using ONLY name + description, was the skill discoverable for this user intent, and (if + relevant) activated without unnecessary delay? + - 1: description clearly matched intent and skill was activated promptly, OR skill was + irrelevant and correctly not activated, OR description was clear/adequate but the + agent still missed or delayed activation (agent-side miss, not a metadata defect). + - 0: description was vague, missing relevant keywords, misleading, or overly broad, and + THAT is what caused the wrong decision (missed, late, or false-positive activation). + +2. SKILL_FOLLOWING (applicable only if the skill was activated) — judges instructions + + resources + Did the agent's actions align with the skill's instructions, including resource-loading + discipline (loading a resource only when the task needed that depth; not skipping one + it was told to load; not hallucinating detail a resource would have supplied)? + - 1: agent followed instructions and handled resources correctly, OR agent deviated but + the instructions/resources were clear and adequate for the situation (agent's fault, + not the skill's). + - 0: instructions were silent, ambiguous, contradictory, or a needed resource was + missing for a situation that actually came up, and that gap caused the deviation. + - Not applicable if skill was never activated — mark applicable:false, do not score 0. + +3. EXPECTED_OUTPUT_MATCH (applicable only if expected_output is provided) + Does the actual conversation's outcome match the expected one in substance (key + facts/decisions/deliverable), not verbatim? + - 1: matches on key elements. 0: missing/contradicts key elements. + +4+. SPECIALIZED CRITERIA — evaluate exactly as described in the input, one per entry. + +## Output format — JSON only, no markdown fences: +{{ + "criteria": [ + {{"name": "SKILL_MATCHING", "applicable": true, "score": 1, "reasoning": "..."}}, + {{"name": "SKILL_FOLLOWING", "applicable": false, "score": null, "reasoning": "skill never activated"}} + ] +}} +""" diff --git a/tests/test_conversation_metrics.py b/tests/test_conversation_metrics.py index 08c0b46..5a2bb14 100644 --- a/tests/test_conversation_metrics.py +++ b/tests/test_conversation_metrics.py @@ -4,9 +4,11 @@ import pytest +from fasteval import metric from fasteval.metrics.conversation import ( ConsistencyMetric, ContextRetentionMetric, + SkillQualityMetric, TopicDriftMetric, ) from fasteval.models.evaluation import EvalInput @@ -103,3 +105,20 @@ async def test_evaluation(self): ) assert result.score == 0.7 assert result.passed is True + + +class TestSkillQualityMetric: + def test_defaults(self): + metric = SkillQualityMetric(llm_client=MockLLMClient()) + assert metric.threshold == 0.8 + assert metric.name == "skill_quality" + + def test_get_evaluation_prompt(self): + eval_input = EvalInput( + history=[ + {"role": "user", "content": "Let's discuss cooking"}, + {"role": "assistant", "content": "Sure, what dish?"}, + ] + ) + metric = SkillQualityMetric(llm_client=MockLLMClient()) + metric.get_evaluation_prompt(eval_input) diff --git a/uv.lock b/uv.lock index 790b580..0add259 100644 --- a/uv.lock +++ b/uv.lock @@ -541,7 +541,7 @@ wheels = [ [[package]] name = "fasteval-core" -version = "1.2.0" +version = "1.2.1" source = { editable = "." } dependencies = [ { name = "openai" }, From 636001b33b3d41cfe8c018c92b8695e1a8a937fe Mon Sep 17 00:00:00 2001 From: Saarthak Verma Date: Fri, 21 Aug 2026 19:18:09 +0530 Subject: [PATCH 2/4] add skill directory to string repr utility --- fasteval/utils/skill_parsing.py | 85 +++++++++++++++ tests/test_conversation_metrics.py | 4 +- tests/test_skill_parsing.py | 169 +++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 fasteval/utils/skill_parsing.py create mode 100644 tests/test_skill_parsing.py diff --git a/fasteval/utils/skill_parsing.py b/fasteval/utils/skill_parsing.py new file mode 100644 index 0000000..9349b00 --- /dev/null +++ b/fasteval/utils/skill_parsing.py @@ -0,0 +1,85 @@ +"""Utilities for reading Agent Skills directories.""" + +from pathlib import Path +from typing import List +from xml.sax.saxutils import escape, quoteattr + +DEFAULT_RESOURCE_DIRECTORIES = ("references", "scripts", "assets") + + +def flatten_skill_folder( + skill_path: str | Path, + additional_resource_paths: List[str] | List[Path] | None = None, +) -> str: + """Return an XML representation of an Agent Skills directory.""" + root = Path(skill_path) + if not root.is_dir(): + raise ValueError(f"Skill path must be a directory: {root}") + skill_md = root / "SKILL.md" + if not skill_md.is_file() or skill_md.is_symlink(): + raise ValueError(f"Skill directory must contain SKILL.md: {root}") + + resource_paths = [skill_md] + resource_paths.extend( + path + for directory in DEFAULT_RESOURCE_DIRECTORIES + if (path := root / directory).is_dir() and not path.is_symlink() + ) + + for additional_path in additional_resource_paths or []: + relative_path = Path(additional_path) + if relative_path.is_absolute() or ".." in relative_path.parts: + raise ValueError( + f"Additional resource path must be relative to the skill directory: " + f"{additional_path}" + ) + + resource_path = root / relative_path + if not resource_path.exists(): + raise ValueError( + f"Additional resource path does not exist: {additional_path}" + ) + if resource_path.is_symlink(): + raise ValueError( + f"Additional resource path must not be a symbolic link: " + f"{additional_path}" + ) + if not resource_path.resolve().is_relative_to(root.resolve()): + raise ValueError( + f"Additional resource path must resolve within the skill directory: " + f"{additional_path}" + ) + resource_paths.append(resource_path) + + files = set() + for resource_path in resource_paths: + if resource_path.is_file(): + files.add(resource_path) + elif resource_path.is_dir(): + files.update( + path + for path in resource_path.rglob("*") + if path.is_file() and not path.is_symlink() + ) + + sorted_files = sorted( + files, + key=lambda path: ( + path.relative_to(root).as_posix() != "SKILL.md", + path.relative_to(root).as_posix(), + ), + ) + + elements = [] + for path in sorted_files: + relative_posix = path.relative_to(root).as_posix() + path_attribute = quoteattr(relative_posix) + try: + content = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + elements.append(f" ") + continue + + elements.append(f" \n{escape(content)}\n ") + + return "\n" + "\n".join(elements) + "\n" diff --git a/tests/test_conversation_metrics.py b/tests/test_conversation_metrics.py index 5a2bb14..0851b17 100644 --- a/tests/test_conversation_metrics.py +++ b/tests/test_conversation_metrics.py @@ -109,7 +109,7 @@ async def test_evaluation(self): class TestSkillQualityMetric: def test_defaults(self): - metric = SkillQualityMetric(llm_client=MockLLMClient()) + metric = SkillQualityMetric(skill="Sample skill", llm_client=MockLLMClient()) assert metric.threshold == 0.8 assert metric.name == "skill_quality" @@ -120,5 +120,5 @@ def test_get_evaluation_prompt(self): {"role": "assistant", "content": "Sure, what dish?"}, ] ) - metric = SkillQualityMetric(llm_client=MockLLMClient()) + metric = SkillQualityMetric(skill="Sample skill", llm_client=MockLLMClient()) metric.get_evaluation_prompt(eval_input) diff --git a/tests/test_skill_parsing.py b/tests/test_skill_parsing.py new file mode 100644 index 0000000..df29d35 --- /dev/null +++ b/tests/test_skill_parsing.py @@ -0,0 +1,169 @@ +"""Tests for fasteval.utils.skill_parsing.""" + +from pathlib import Path + +import pytest + +from fasteval.utils.skill_parsing import flatten_skill_folder + + +@pytest.fixture +def skill_dir(tmp_path: Path) -> Path: + skill = tmp_path / "sample-skill" + outside_file = tmp_path / "outside.txt" + outside_file.write_text("secret", encoding="utf-8") + outside_directory = tmp_path / "outside" + outside_directory.mkdir() + (outside_directory / "secret.txt").write_text("secret", encoding="utf-8") + (skill / "references").mkdir(parents=True) + (skill / "scripts").mkdir() + (skill / "assets").mkdir() + (skill / ".claude").mkdir() + (skill / "examples").mkdir() + skill_md = """--- +name: sample-skill +description: Sample & test +--- + +Use + +""" + (skill / "SKILL.md").write_text( + skill_md, + encoding="utf-8", + ) + (skill / "references" / "guide.md").write_text("Read A & B.\n", encoding="utf-8") + (skill / "scripts" / "run.py").write_text('print("ok")\n', encoding="utf-8") + (skill / "assets" / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n\xff") + (skill / ".claude" / "settings.json").write_text("{}\n", encoding="utf-8") + (skill / "notes.md").write_text("Do not include.", encoding="utf-8") + (skill / "example.txt").write_text("Do not include.", encoding="utf-8") + (skill / "examples" / "demo.txt").write_text("Example.", encoding="utf-8") + (skill / "references" / "linked.txt").symlink_to(outside_file) + (skill / "linked-directory").symlink_to(outside_directory, target_is_directory=True) + return skill + + +def test_flattens_skill_directory_as_xml(skill_dir: Path): + path_result = flatten_skill_folder(skill_dir) + string_result = flatten_skill_folder(str(skill_dir)) + + assert path_result == string_result + assert path_result == """ + +--- +name: sample-skill +description: Sample & test +--- + +Use <carefully> + + + + + +Read A & B. + + + +print("ok") + + +""" + assert "linked.txt" not in path_result + assert "secret" not in path_result + + +def test_includes_additional_resources(skill_dir: Path): + string_result = flatten_skill_folder( + skill_dir, + additional_resource_paths=[ + "notes.md", + "examples", + "references", + "references/guide.md", + ], + ) + path_result = flatten_skill_folder( + skill_dir, + additional_resource_paths=[ + Path("notes.md"), + Path("examples"), + Path("references"), + Path("references/guide.md"), + ], + ) + + assert string_result == path_result + for result in (string_result, path_result): + assert '' in result + assert '' in result + assert result.count('') == 1 + + +@pytest.mark.parametrize( + ("additional_path", "message"), + [ + ("/tmp/outside.txt", "Additional resource path must be relative"), + ("../outside.txt", "Additional resource path must be relative"), + ("missing.txt", "Additional resource path does not exist"), + ( + "references/linked.txt", + "Additional resource path must not be a symbolic link", + ), + ( + "linked-directory/secret.txt", + "Additional resource path must resolve within the skill directory", + ), + ], +) +def test_rejects_invalid_additional_resource_paths( + skill_dir: Path, + additional_path: str, + message: str, +): + with pytest.raises(ValueError, match=message): + flatten_skill_folder( + skill_dir, + additional_resource_paths=[additional_path], + ) + + +def test_rejects_path_that_is_not_a_directory(tmp_path: Path): + missing_path = tmp_path / "missing" + + with pytest.raises( + ValueError, match=f"Skill path must be a directory: {missing_path}" + ): + flatten_skill_folder(missing_path) + + +def test_rejects_directory_without_root_skill_file(tmp_path: Path): + with pytest.raises( + ValueError, match=f"Skill directory must contain SKILL.md: {tmp_path}" + ): + flatten_skill_folder(tmp_path) + + +def test_rejects_symlinked_root_skill_file(skill_dir: Path): + outside = (skill_dir / "references" / "linked.txt").resolve() + (skill_dir / "SKILL.md").unlink() + (skill_dir / "SKILL.md").symlink_to(outside) + + with pytest.raises( + ValueError, match=f"Skill directory must contain SKILL.md: {skill_dir}" + ): + flatten_skill_folder(skill_dir) + + +def test_skips_symlinked_default_resource_directory(skill_dir: Path): + outside = (skill_dir / "linked-directory").resolve() + assets = skill_dir / "assets" + (assets / "logo.png").unlink() + assets.rmdir() + assets.symlink_to(outside, target_is_directory=True) + + result = flatten_skill_folder(skill_dir) + + assert "secret" not in result + assert "assets/secret.txt" not in result From 3c0274c560dac5f447812b8870fddec889a2c37c Mon Sep 17 00:00:00 2001 From: Saarthak Verma Date: Sat, 22 Aug 2026 18:50:04 +0530 Subject: [PATCH 3/4] fix typo --- fasteval/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fasteval/__init__.py b/fasteval/__init__.py index e4d8d3f..f63a99e 100644 --- a/fasteval/__init__.py +++ b/fasteval/__init__.py @@ -215,7 +215,8 @@ def test_qa(): "context_retention", "consistency", "topic_drift", - "skill_quality" "metric", + "skill_quality", + "metric", # Code-as-Judge "judge", # Data Decorators From a7fd25934f4ffb395f89fc6f20dd0c37ac9f2404 Mon Sep 17 00:00:00 2001 From: Saarthak Verma Date: Tue, 25 Aug 2026 22:34:24 +0530 Subject: [PATCH 4/4] override _parse_response for skill quality --- fasteval/metrics/conversation.py | 45 ++++++++++++++++++++- fasteval/utils/__init__.py | 2 + tests/test_conversation_metrics.py | 63 +++++++++++++++++++++++++++++- 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/fasteval/metrics/conversation.py b/fasteval/metrics/conversation.py index b677f5d..8d4b364 100644 --- a/fasteval/metrics/conversation.py +++ b/fasteval/metrics/conversation.py @@ -8,8 +8,11 @@ from typing import Any, Optional -from fasteval.metrics.llm import BaseLLMMetric +from pydantic import BaseModel, Field + +from fasteval.metrics.llm import BaseLLMMetric, LLMEvalResponse from fasteval.models.evaluation import EvalInput +from fasteval.utils.json_parsing import parse_json_response class ContextRetentionMetric(BaseLLMMetric): @@ -326,6 +329,21 @@ def get_evaluation_prompt(self, eval_input: EvalInput) -> str: {{"score": <0.0 to 1.0>, "reasoning": ""}}""" +class SkillQualityCriteria(BaseModel): + """A single criterion in a skill-quality evaluation response.""" + + name: str + applicable: bool + score: float | None = Field(default=None, ge=0.0, le=1.0) + reasoning: str = "" + + +class SkillQualityResponse(BaseModel): + """Structured response returned by the skill-quality evaluator.""" + + criteria: list[SkillQualityCriteria] + + class SkillQualityMetric(BaseLLMMetric): def __init__( self, @@ -339,6 +357,31 @@ def __init__( self.skill = skill self.additional_skill_eval_criteria = additional_skill_eval_criteria + def _parse_response(self, response: str) -> LLMEvalResponse: + parsed = parse_json_response(response, SkillQualityResponse) + applicable_criteria = [ + criterion for criterion in parsed.criteria if criterion.applicable + ] + if not applicable_criteria: + raise ValueError( + "Skill quality response must contain an applicable criterion" + ) + if any(criterion.score is None for criterion in applicable_criteria): + raise ValueError("Applicable skill quality criteria must include a score") + + scores = [ + criterion.score + for criterion in applicable_criteria + if criterion.score is not None + ] + reasoning = "\n".join( + f"{criterion.name}: {criterion.reasoning}" for criterion in parsed.criteria + ) + return LLMEvalResponse( + score=sum(scores) / len(scores), + reasoning=reasoning, + ) + def get_evaluation_prompt(self, eval_input: EvalInput) -> str: history_str = "" diff --git a/fasteval/utils/__init__.py b/fasteval/utils/__init__.py index 05772e8..6d5ae50 100644 --- a/fasteval/utils/__init__.py +++ b/fasteval/utils/__init__.py @@ -3,6 +3,7 @@ from fasteval.utils.async_helpers import run_async from fasteval.utils.formatting import format_evaluation_report from fasteval.utils.json_parsing import extract_json_from_text, parse_json_response +from fasteval.utils.skill_parsing import flatten_skill_folder from fasteval.utils.terminal_ui import ( HumanScore, is_interactive, @@ -24,4 +25,5 @@ "prompt_human_review", "render_human_review", "render_conversation_history", + "flatten_skill_folder", ] diff --git a/tests/test_conversation_metrics.py b/tests/test_conversation_metrics.py index 0851b17..138154f 100644 --- a/tests/test_conversation_metrics.py +++ b/tests/test_conversation_metrics.py @@ -15,8 +15,8 @@ class MockLLMClient: - def __init__(self, score=0.85): - self.response = json.dumps( + def __init__(self, score=0.85, response=None): + self.response = response or json.dumps( {"score": score, "reasoning": "Mock conversation eval"} ) @@ -122,3 +122,62 @@ def test_get_evaluation_prompt(self): ) metric = SkillQualityMetric(skill="Sample skill", llm_client=MockLLMClient()) metric.get_evaluation_prompt(eval_input) + + def test_parse_response_averages_applicable_criteria(self): + metric = SkillQualityMetric(skill="Sample skill", llm_client=MockLLMClient()) + response = ( + '{"criteria": [' + '{"name": "SKILL_MATCHING", "applicable": true, "score": 1, ' + '"reasoning": "Metadata matched."},' + '{"name": "SKILL_FOLLOWING", "applicable": true, "score": 0, ' + '"reasoning": "Instructions were ambiguous."},' + '{"name": "EXPECTED_OUTPUT_MATCH", "applicable": false, "score": null, ' + '"reasoning": "No expected output."}' + "]}" + ) + + parsed = metric._parse_response(response) + + assert parsed.score == 0.5 + assert parsed.reasoning == ( + "SKILL_MATCHING: Metadata matched.\n" + "SKILL_FOLLOWING: Instructions were ambiguous.\n" + "EXPECTED_OUTPUT_MATCH: No expected output." + ) + + @pytest.mark.asyncio + async def test_evaluate_method(self): + response = ( + '{"criteria": [{"name": "SKILL_MATCHING", "applicable": true, ' + '"score": 1, "reasoning": "Metadata matched."}]}' + ) + metric = SkillQualityMetric( + skill="Sample skill", + llm_client=MockLLMClient(response=response), + ) + + result = await metric.evaluate(EvalInput(history=[])) + + assert result.score == 1.0 + assert result.passed is True + + @pytest.mark.parametrize( + ("response", "message"), + [ + ( + '{"criteria": [{"name": "SKILL_MATCHING", "applicable": true, ' + '"score": null, "reasoning": ""}]}', + "Applicable skill quality criteria must include a score", + ), + ( + '{"criteria": [{"name": "SKILL_FOLLOWING", "applicable": false, ' + '"score": null, "reasoning": ""}]}', + "Skill quality response must contain an applicable criterion", + ), + ], + ) + def test_parse_response_rejects_invalid_criteria(self, response, message): + metric = SkillQualityMetric(skill="Sample skill", llm_client=MockLLMClient()) + + with pytest.raises(ValueError, match=message): + metric._parse_response(response)