create skill quality metric

fasteval/__init__.py

@@ -81,6 +81,7 @@ def test_qa():
     relevance,
     rouge,
     safety_check,
+    skill_quality,
     speaker_diarization,
     stack,
     table_extraction,
@@ -214,6 +215,7 @@ def test_qa():
     "context_retention",
     "consistency",
     "topic_drift",
+    "skill_quality",
     "metric",
     # Code-as-Judge
     "judge",

fasteval/core/__init__.py

@@ -16,6 +16,7 @@
     metric,
     relevance,
     rouge,
+    skill_quality,
     topic_drift,
     toxicity,
     traces,
@@ -41,6 +42,7 @@
     "context_retention",
     "consistency",
     "topic_drift",
+    "skill_quality",
     # Data Decorators
     "csv",
     "conversation",

fasteval/core/decorators.py

@@ -1006,6 +1006,31 @@ async def test_on_topic():
     )
 
 
+def skill_quality(
+    threshold: float = 0.7,
+    weight: float = 1.0,
+    name: Optional[str] = None,
+    skill: Optional[str] = None,
+    additional_skill_eval_criteria: Optional[str] = None,
+    llm_client: Optional[LLMClient] = None,
+    model: Optional[str] = None,
+    **config: Any,
+) -> Callable[[F], F]:
+
+    return _metric_decorator_factory(
+        "skill_quality", "skill_quality", default_threshold=0.7
+    )(
+        threshold=threshold,
+        weight=weight,
+        name=name,
+        llm_client=llm_client,
+        model=model,
+        skill=skill,
+        additional_skill_eval_criteria=additional_skill_eval_criteria,
+        **config,
+    )
+
+
 # === Vision Metric Decorators ===
 
 

fasteval/core/evaluator.py

@@ -11,6 +11,7 @@
 from fasteval.metrics.conversation import (
     ConsistencyMetric,
     ContextRetentionMetric,
+    SkillQualityMetric,
     TopicDriftMetric,
 )
 from fasteval.metrics.deterministic import (  # Tool Trajectory Metrics
@@ -82,6 +83,7 @@
     "context_retention": ContextRetentionMetric,
     "consistency": ConsistencyMetric,
     "topic_drift": TopicDriftMetric,
+    "skill_quality": SkillQualityMetric,
 }
 
 # Register vision metrics if available

fasteval/metrics/conversation.py

@@ -8,8 +8,11 @@
 
 from typing import Any, Optional
 
-from fasteval.metrics.llm import BaseLLMMetric
+from pydantic import BaseModel, Field
+
+from fasteval.metrics.llm import BaseLLMMetric, LLMEvalResponse
 from fasteval.models.evaluation import EvalInput
+from fasteval.utils.json_parsing import parse_json_response
 
 
 class ContextRetentionMetric(BaseLLMMetric):
@@ -324,3 +327,157 @@ def get_evaluation_prompt(self, eval_input: EvalInput) -> str:
 ## OUTPUT FORMAT
 Respond with JSON only:
 {{"score": <0.0 to 1.0>, "reasoning": "<identify main topic, assess response alignment, note any drift or tangents>"}}"""
+
+
+class SkillQualityCriteria(BaseModel):
+    """A single criterion in a skill-quality evaluation response."""
+
+    name: str
+    applicable: bool
+    score: float | None = Field(default=None, ge=0.0, le=1.0)
+    reasoning: str = ""
+
+
+class SkillQualityResponse(BaseModel):
+    """Structured response returned by the skill-quality evaluator."""
+
+    criteria: list[SkillQualityCriteria]
+
+
+class SkillQualityMetric(BaseLLMMetric):
+    def __init__(
+        self,
+        skill: str,
+        name: str = "skill_quality",
+        threshold: float = 0.8,
+        additional_skill_eval_criteria: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(name=name, threshold=threshold, **kwargs)
+        self.skill = skill
+        self.additional_skill_eval_criteria = additional_skill_eval_criteria
+
+    def _parse_response(self, response: str) -> LLMEvalResponse:
+        parsed = parse_json_response(response, SkillQualityResponse)
+        applicable_criteria = [
+            criterion for criterion in parsed.criteria if criterion.applicable
+        ]
+        if not applicable_criteria:
+            raise ValueError(
+                "Skill quality response must contain an applicable criterion"
+            )
+        if any(criterion.score is None for criterion in applicable_criteria):
+            raise ValueError("Applicable skill quality criteria must include a score")
+
+        scores = [
+            criterion.score
+            for criterion in applicable_criteria
+            if criterion.score is not None
+        ]
+        reasoning = "\n".join(
+            f"{criterion.name}: {criterion.reasoning}" for criterion in parsed.criteria
+        )
+        return LLMEvalResponse(
+            score=sum(scores) / len(scores),
+            reasoning=reasoning,
+        )
+
+    def get_evaluation_prompt(self, eval_input: EvalInput) -> str:
+
+        history_str = ""
+        if eval_input.history:
+            history_str = "\n".join(
+                f"**{turn.get('role', 'unknown').title()}**: {turn.get('content', '')}"
+                for turn in eval_input.history
+            )
+            history_str = f"\n{history_str}\n"
+        expected_output = eval_input.expected_output or "N/A"
+        skill = self.skill
+        specialized_criteria = self.additional_skill_eval_criteria or "N/A"
+
+        return f"""
+<conversation_history>
+{history_str}
+</conversation_history>
+
+<expected_output>
+{expected_output}
+</expected_output>
+
+<skill>
+{skill}
+</skill>
+
+<specialized_criteria>
+{specialized_criteria}
+</specialized_criteria>
+
+You are a judge evaluating whether a SKILL (packaged metadata/instructions/resources) is
+well designed — not whether the agent using it is competent. Your job is to find defects
+in the SKILL for iterative improvement, not to grade the agent.
+
+## Root-cause rule (applies to every criterion below)
+When the agent's behavior falls short, decide WHY before scoring:
+- Traces back to something missing/ambiguous/wrong in the skill → skill defect → score 0.
+- Skill was clear and adequate, agent just ignored/misread/hallucinated anyway → agent
+  failure, not the skill's fault → score 1 (note it in reasoning so it isn't mistaken for
+  a needed skill fix).
+- If you cannot point to specific text in the skill that explains the failure, default to
+  1 and say so — do not assume a skill defect without textual evidence.
+
+## Skill loading model (judge accordingly)
+- Metadata (name + description): visible to the agent at ALL times, before
+  any decision to use the skill.
+- Instructions (full SKILL.md body): loaded only once the skill is activated.
+- Resources (files under scripts/, references/, assets/): loaded on demand, only if the
+  task needs that depth.
+Judge each criterion against the layer the agent actually had access to at that point.
+
+## Input format (data supplied above, inside tags)
+- <conversation_history>: conversation between user and agent.
+- <expected_output> (optional): reference conversation showing correct invocation and/or
+  correct final output.
+- <skill>: skill under test, parseable into metadata / instructions / resources.
+- <specialized_criteria> (optional): additional use-case-specific criteria, one per entry.
+
+Evaluate each criterion independently. Do NOT sum or average scores yourself — return a
+structured verdict per criterion; aggregation happens outside this prompt.
+
+## Criteria
+
+1. SKILL_MATCHING (always applicable) — judges the metadata layer only
+   Using ONLY name + description, was the skill discoverable for this user intent, and (if
+   relevant) activated without unnecessary delay?
+   - 1: description clearly matched intent and skill was activated promptly, OR skill was
+     irrelevant and correctly not activated, OR description was clear/adequate but the
+     agent still missed or delayed activation (agent-side miss, not a metadata defect).
+   - 0: description was vague, missing relevant keywords, misleading, or overly broad, and
+     THAT is what caused the wrong decision (missed, late, or false-positive activation).
+
+2. SKILL_FOLLOWING (applicable only if the skill was activated) — judges instructions +
+   resources
+   Did the agent's actions align with the skill's instructions, including resource-loading
+   discipline (loading a resource only when the task needed that depth; not skipping one
+   it was told to load; not hallucinating detail a resource would have supplied)?
+   - 1: agent followed instructions and handled resources correctly, OR agent deviated but
+     the instructions/resources were clear and adequate for the situation (agent's fault,
+     not the skill's).
+   - 0: instructions were silent, ambiguous, contradictory, or a needed resource was
+     missing for a situation that actually came up, and that gap caused the deviation.
+   - Not applicable if skill was never activated — mark applicable:false, do not score 0.
+
+3. EXPECTED_OUTPUT_MATCH (applicable only if expected_output is provided)
+   Does the actual conversation's outcome match the expected one in substance (key
+   facts/decisions/deliverable), not verbatim?
+   - 1: matches on key elements. 0: missing/contradicts key elements.
+
+4+. SPECIALIZED CRITERIA — evaluate exactly as described in the input, one per entry.
+
+## Output format — JSON only, no markdown fences:
+{{
+  "criteria": [
+    {{"name": "SKILL_MATCHING", "applicable": true, "score": 1, "reasoning": "..."}},
+    {{"name": "SKILL_FOLLOWING", "applicable": false, "score": null, "reasoning": "skill never activated"}}
+  ]
+}}
+"""

fasteval/utils/__init__.py

@@ -3,6 +3,7 @@
 from fasteval.utils.async_helpers import run_async
 from fasteval.utils.formatting import format_evaluation_report
 from fasteval.utils.json_parsing import extract_json_from_text, parse_json_response
+from fasteval.utils.skill_parsing import flatten_skill_folder
 from fasteval.utils.terminal_ui import (
     HumanScore,
     is_interactive,
@@ -24,4 +25,5 @@
     "prompt_human_review",
     "render_human_review",
     "render_conversation_history",
+    "flatten_skill_folder",
 ]

fasteval/utils/skill_parsing.py

@@ -0,0 +1,85 @@
+"""Utilities for reading Agent Skills directories."""
+
+from pathlib import Path
+from typing import List
+from xml.sax.saxutils import escape, quoteattr
+
+DEFAULT_RESOURCE_DIRECTORIES = ("references", "scripts", "assets")
+
+
+def flatten_skill_folder(
+    skill_path: str | Path,
+    additional_resource_paths: List[str] | List[Path] | None = None,
+) -> str:
+    """Return an XML representation of an Agent Skills directory."""
+    root = Path(skill_path)
+    if not root.is_dir():
+        raise ValueError(f"Skill path must be a directory: {root}")
+    skill_md = root / "SKILL.md"
+    if not skill_md.is_file() or skill_md.is_symlink():
+        raise ValueError(f"Skill directory must contain SKILL.md: {root}")
+
+    resource_paths = [skill_md]
+    resource_paths.extend(
+        path
+        for directory in DEFAULT_RESOURCE_DIRECTORIES
+        if (path := root / directory).is_dir() and not path.is_symlink()
+    )
+
+    for additional_path in additional_resource_paths or []:
+        relative_path = Path(additional_path)
+        if relative_path.is_absolute() or ".." in relative_path.parts:
+            raise ValueError(
+                f"Additional resource path must be relative to the skill directory: "
+                f"{additional_path}"
+            )
+
+        resource_path = root / relative_path
+        if not resource_path.exists():
+            raise ValueError(
+                f"Additional resource path does not exist: {additional_path}"
+            )
+        if resource_path.is_symlink():
+            raise ValueError(
+                f"Additional resource path must not be a symbolic link: "
+                f"{additional_path}"
+            )
+        if not resource_path.resolve().is_relative_to(root.resolve()):
+            raise ValueError(
+                f"Additional resource path must resolve within the skill directory: "
+                f"{additional_path}"
+            )
+        resource_paths.append(resource_path)
+
+    files = set()
+    for resource_path in resource_paths:
+        if resource_path.is_file():
+            files.add(resource_path)
+        elif resource_path.is_dir():
+            files.update(
+                path
+                for path in resource_path.rglob("*")
+                if path.is_file() and not path.is_symlink()
+            )
+
+    sorted_files = sorted(
+        files,
+        key=lambda path: (
+            path.relative_to(root).as_posix() != "SKILL.md",
+            path.relative_to(root).as_posix(),
+        ),
+    )
+
+    elements = []
+    for path in sorted_files:
+        relative_posix = path.relative_to(root).as_posix()
+        path_attribute = quoteattr(relative_posix)
+        try:
+            content = path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            elements.append(f"  <resource path={path_attribute} />")
+            continue
+
+        elements.append(f"  <file path={path_attribute}>\n{escape(content)}\n  </file>")
+
+    return "<skill>\n" + "\n".join(elements) + "\n</skill>"