Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 24 additions & 27 deletions backend/api/scope/client/applications.py
Original file line number Diff line number Diff line change
Expand Up @@ -1920,44 +1920,41 @@ def _build_cognitive_metrics(
return [found[n] for n in COGNITIVE_METRIC_NAMES if n in found]


def _as_str_list(raw: object) -> list[str]:
"""Coerce a stored remediation property (list, JSON string or text) to ``list[str]``."""
if raw is None:
return []
if isinstance(raw, str):
try:
raw = json.loads(raw)
except (json.JSONDecodeError, TypeError):
return [raw] if raw.strip() else []
if isinstance(raw, str):
return [raw] if raw.strip() else []
if isinstance(raw, (list, tuple)):
return [str(r) for r in raw if r is not None and str(r).strip()]
return []
# Confidence levels of a failure report, lowest first.
_FAILURE_CONFIDENCE_RANK = {"low": 0, "medium": 1, "high": 2}


def _build_cognitive_failures(raw_failures: object) -> list[CognitiveFailure]:
"""Map ``CognitiveFailuresReport`` node properties to :class:`CognitiveFailure`."""
"""Map ``CognitiveFailureReport`` node properties to :class:`CognitiveFailure`.

The name is the challenge code. A failure detected at several calls of a session is reported
once, with its highest confidence. Most confident first.
"""
if not isinstance(raw_failures, (list, tuple)):
return []

failures: list[CognitiveFailure] = []
by_name: dict[str, CognitiveFailure] = {}
for props in raw_failures:
if not isinstance(props, dict):
continue
name = props.get("name")
confidence = props.get("presenceProbability")
if not name or isinstance(confidence, bool) or not isinstance(confidence, Real):
name = props.get("name") or props.get("code")
confidence = str(props.get("confidence") or "").lower()
if not name or confidence not in _FAILURE_CONFIDENCE_RANK:
continue
failures.append(
CognitiveFailure(
known = by_name.get(str(name))
if known is None or (
_FAILURE_CONFIDENCE_RANK[confidence] > _FAILURE_CONFIDENCE_RANK[known.confidence]
):
by_name[str(name)] = CognitiveFailure(
name=str(name),
confidence=float(confidence),
remediations=_as_str_list(props.get("remediation")),
description=str(props.get("description") or ""),
confidence=confidence,
reasoning=str(props.get("reasoning") or ""),
)
)
return sorted(failures, key=lambda f: f.confidence, reverse=True)
return sorted(
by_name.values(),
key=lambda f: _FAILURE_CONFIDENCE_RANK[f.confidence],
reverse=True,
)


def _get_application_sessions_with_cognitive_observability(
Expand All @@ -1972,7 +1969,7 @@ def _get_application_sessions_with_cognitive_observability(
"""Return the sessions of an application with their cognitive observability.

Metrics come from ``Session-[:hasMetric]->Metric`` nodes and failures from
the session's ``CognitiveFailuresReport`` nodes (one node per failure).
the session's ``CognitiveFailureReport`` nodes (one node per failure).
"""
del dialect # Neo4j-only query.
start_epoch, end_epoch = parse_epoch_range(start_time, end_time)
Expand Down
7 changes: 3 additions & 4 deletions backend/api/scope/models/otel_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,12 +334,11 @@ class CognitiveObservabilityMetric(BaseModel):


class CognitiveFailure(BaseModel):
"""A cognitive failure detected for a session (one ``CognitiveFailuresReport``)."""
"""A cognitive failure detected for a session (one ``CognitiveFailureReport``)."""

name: str
confidence: float
description: str = ""
remediations: list[str] = []
confidence: Literal["low", "medium", "high"]
reasoning: str = ""


class CognitiveSessionItem(CollectSessionItem):
Expand Down
10 changes: 3 additions & 7 deletions backend/api/scope/query_builders/ui.py
Original file line number Diff line number Diff line change
Expand Up @@ -3162,9 +3162,8 @@ def neo4j_sessions_with_cognitive_observability_query(

One row per session: ``sessionId, startTime, duration, cost, agents, tokens,
sessionMetrics, failures``. ``failures`` holds the raw properties of the
session's ``CognitiveFailuresReport`` / ``ChallengeDetectionReport`` nodes (one per failure). The
relationship is matched undirected/untyped, as for the other session-attached
nodes in this module.
session's ``CognitiveFailureReport`` nodes (one per failure), which the cognitive failure worker
attaches to the session by ``reportsOn``.
"""
stmt = """
MATCH (s:Session)-[:executesSession]->(ma:MAS) WHERE ma.id = $application_id
Expand Down Expand Up @@ -3202,10 +3201,7 @@ def neo4j_sessions_with_cognitive_observability_query(
source: sm.source,
reasoning: sm.reasoning
}) WHERE x.name IS NOT NULL] AS sessionMetrics
OPTIONAL MATCH (s)-[]-(f)
WHERE f:CognitiveFailuresReport
OR f:ChallengeDetectionReport
OR f.type = 'ChallengeDetectionReport'
OPTIONAL MATCH (f:CognitiveFailureReport)-[:reportsOn]->(s)
WITH
s,
cost,
Expand Down
26 changes: 10 additions & 16 deletions backend/api/tests/test_cognitive_observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,12 @@ def _row():
{"name": "Goal Alignment", "result": None, "value": None},
],
"failures": [
{
"name": "Incomplete Synthesis",
"presenceProbability": 0.6,
"remediation": ["A"],
"description": "desc A",
},
{
"name": "Task Delegation Failure",
"presenceProbability": 0.95,
"remediation": '["B", "C"]',
},
{"name": "No probability", "code": "X"},
{"name": "IR-1", "code": "IR-1", "confidence": "medium", "reasoning": "lost it"},
{"name": "IR-1", "code": "IR-1", "confidence": "high", "reasoning": "lost it again"},
{"name": "CC-3", "code": "CC-3", "confidence": "low", "reasoning": "weak"},
{"code": "DC-4", "confidence": "HIGH", "reasoning": "no name"},
{"name": "No confidence", "code": "X"},
{"name": "Bad confidence", "code": "Y", "confidence": "sure"},
],
}

Expand All @@ -67,11 +61,11 @@ def test_sessions_with_cognitive_observability_mapping():
("Task Completion", 0.5, "PERCENTAGE"),
]
assert [
(f.name, f.confidence, f.description, f.remediations)
for f in session.cognitiveFailures
(f.name, f.confidence, f.reasoning) for f in session.cognitiveFailures
] == [
("Task Delegation Failure", 0.95, "", ["B", "C"]),
("Incomplete Synthesis", 0.6, "desc A", ["A"]),
("IR-1", "high", "lost it again"),
("DC-4", "high", "no name"),
("CC-3", "low", "weak"),
]


Expand Down
Loading
Loading