diff --git a/backend/api/scope/client/applications.py b/backend/api/scope/client/applications.py index 7f71f2d..ba5febf 100644 --- a/backend/api/scope/client/applications.py +++ b/backend/api/scope/client/applications.py @@ -1920,44 +1920,41 @@ def _build_cognitive_metrics( return [found[n] for n in COGNITIVE_METRIC_NAMES if n in found] -def _as_str_list(raw: object) -> list[str]: - """Coerce a stored remediation property (list, JSON string or text) to ``list[str]``.""" - if raw is None: - return [] - if isinstance(raw, str): - try: - raw = json.loads(raw) - except (json.JSONDecodeError, TypeError): - return [raw] if raw.strip() else [] - if isinstance(raw, str): - return [raw] if raw.strip() else [] - if isinstance(raw, (list, tuple)): - return [str(r) for r in raw if r is not None and str(r).strip()] - return [] +# Confidence levels of a failure report, lowest first. +_FAILURE_CONFIDENCE_RANK = {"low": 0, "medium": 1, "high": 2} def _build_cognitive_failures(raw_failures: object) -> list[CognitiveFailure]: - """Map ``CognitiveFailuresReport`` node properties to :class:`CognitiveFailure`.""" + """Map ``CognitiveFailureReport`` node properties to :class:`CognitiveFailure`. + + The name is the challenge code. A failure detected at several calls of a session is reported + once, with its highest confidence. Most confident first. + """ if not isinstance(raw_failures, (list, tuple)): return [] - failures: list[CognitiveFailure] = [] + by_name: dict[str, CognitiveFailure] = {} for props in raw_failures: if not isinstance(props, dict): continue - name = props.get("name") - confidence = props.get("presenceProbability") - if not name or isinstance(confidence, bool) or not isinstance(confidence, Real): + name = props.get("name") or props.get("code") + confidence = str(props.get("confidence") or "").lower() + if not name or confidence not in _FAILURE_CONFIDENCE_RANK: continue - failures.append( - CognitiveFailure( + known = by_name.get(str(name)) + if known is None or ( + _FAILURE_CONFIDENCE_RANK[confidence] > _FAILURE_CONFIDENCE_RANK[known.confidence] + ): + by_name[str(name)] = CognitiveFailure( name=str(name), - confidence=float(confidence), - remediations=_as_str_list(props.get("remediation")), - description=str(props.get("description") or ""), + confidence=confidence, + reasoning=str(props.get("reasoning") or ""), ) - ) - return sorted(failures, key=lambda f: f.confidence, reverse=True) + return sorted( + by_name.values(), + key=lambda f: _FAILURE_CONFIDENCE_RANK[f.confidence], + reverse=True, + ) def _get_application_sessions_with_cognitive_observability( @@ -1972,7 +1969,7 @@ def _get_application_sessions_with_cognitive_observability( """Return the sessions of an application with their cognitive observability. Metrics come from ``Session-[:hasMetric]->Metric`` nodes and failures from - the session's ``CognitiveFailuresReport`` nodes (one node per failure). + the session's ``CognitiveFailureReport`` nodes (one node per failure). """ del dialect # Neo4j-only query. start_epoch, end_epoch = parse_epoch_range(start_time, end_time) diff --git a/backend/api/scope/models/otel_traces.py b/backend/api/scope/models/otel_traces.py index 8589675..325babc 100644 --- a/backend/api/scope/models/otel_traces.py +++ b/backend/api/scope/models/otel_traces.py @@ -334,12 +334,11 @@ class CognitiveObservabilityMetric(BaseModel): class CognitiveFailure(BaseModel): - """A cognitive failure detected for a session (one ``CognitiveFailuresReport``).""" + """A cognitive failure detected for a session (one ``CognitiveFailureReport``).""" name: str - confidence: float - description: str = "" - remediations: list[str] = [] + confidence: Literal["low", "medium", "high"] + reasoning: str = "" class CognitiveSessionItem(CollectSessionItem): diff --git a/backend/api/scope/query_builders/ui.py b/backend/api/scope/query_builders/ui.py index 11a7a3e..c8193a2 100644 --- a/backend/api/scope/query_builders/ui.py +++ b/backend/api/scope/query_builders/ui.py @@ -3162,9 +3162,8 @@ def neo4j_sessions_with_cognitive_observability_query( One row per session: ``sessionId, startTime, duration, cost, agents, tokens, sessionMetrics, failures``. ``failures`` holds the raw properties of the - session's ``CognitiveFailuresReport`` / ``ChallengeDetectionReport`` nodes (one per failure). The - relationship is matched undirected/untyped, as for the other session-attached - nodes in this module. + session's ``CognitiveFailureReport`` nodes (one per failure), which the cognitive failure worker + attaches to the session by ``reportsOn``. """ stmt = """ MATCH (s:Session)-[:executesSession]->(ma:MAS) WHERE ma.id = $application_id @@ -3202,10 +3201,7 @@ def neo4j_sessions_with_cognitive_observability_query( source: sm.source, reasoning: sm.reasoning }) WHERE x.name IS NOT NULL] AS sessionMetrics - OPTIONAL MATCH (s)-[]-(f) - WHERE f:CognitiveFailuresReport - OR f:ChallengeDetectionReport - OR f.type = 'ChallengeDetectionReport' + OPTIONAL MATCH (f:CognitiveFailureReport)-[:reportsOn]->(s) WITH s, cost, diff --git a/backend/api/tests/test_cognitive_observability.py b/backend/api/tests/test_cognitive_observability.py index d5c8ca1..45eee10 100644 --- a/backend/api/tests/test_cognitive_observability.py +++ b/backend/api/tests/test_cognitive_observability.py @@ -33,18 +33,12 @@ def _row(): {"name": "Goal Alignment", "result": None, "value": None}, ], "failures": [ - { - "name": "Incomplete Synthesis", - "presenceProbability": 0.6, - "remediation": ["A"], - "description": "desc A", - }, - { - "name": "Task Delegation Failure", - "presenceProbability": 0.95, - "remediation": '["B", "C"]', - }, - {"name": "No probability", "code": "X"}, + {"name": "IR-1", "code": "IR-1", "confidence": "medium", "reasoning": "lost it"}, + {"name": "IR-1", "code": "IR-1", "confidence": "high", "reasoning": "lost it again"}, + {"name": "CC-3", "code": "CC-3", "confidence": "low", "reasoning": "weak"}, + {"code": "DC-4", "confidence": "HIGH", "reasoning": "no name"}, + {"name": "No confidence", "code": "X"}, + {"name": "Bad confidence", "code": "Y", "confidence": "sure"}, ], } @@ -67,11 +61,11 @@ def test_sessions_with_cognitive_observability_mapping(): ("Task Completion", 0.5, "PERCENTAGE"), ] assert [ - (f.name, f.confidence, f.description, f.remediations) - for f in session.cognitiveFailures + (f.name, f.confidence, f.reasoning) for f in session.cognitiveFailures ] == [ - ("Task Delegation Failure", 0.95, "", ["B", "C"]), - ("Incomplete Synthesis", 0.6, "desc A", ["A"]), + ("IR-1", "high", "lost it again"), + ("DC-4", "high", "no name"), + ("CC-3", "low", "weak"), ] diff --git a/ui/src/api/mock/sessionCognitionProtocols.ts b/ui/src/api/mock/sessionCognitionProtocols.ts deleted file mode 100644 index 8c41eb7..0000000 --- a/ui/src/api/mock/sessionCognitionProtocols.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { SessionCognitionProtocols } from '@/types/scope.type'; - -// Mocked Cognition protocols per session. Sessions that are not listed have no Cognition -// protocol at all (empty `protocols` list). Scenarios: -// - 4493f565: CONCORD (COMMIT) and ACCORD, both activated, with handoff -// - 41114179: CONCORD ending in BEST_EFFORT with a shortfall -// - 0e78e3a6: CONCORD expected but not started (missed activation) + ACCORD -// without ground truth -// - 65d1c926: CONCORD (COMMIT) with a lot of missing metrics -// - 5980fc14: ACCORD enabled but not activated -export const mockSessionCognitionProtocols: Record = { - '4493f565-d069-47ec-9a6c-5df6dc8f31b7': { - sessionId: '4493f565-d069-47ec-9a6c-5df6dc8f31b7', - protocols: [ - { - protocol: 'ACCORD', - displayName: 'ACCORD', - description: - 'Runs before the task: each agent states what it means by the terms in play, who it is, what it thinks the problem is and what it knows, and ACCORD assembles it all into one shared model. Locking it emits the Intent Contract (Roster, Frame, Glossary and Verifiable Objective). Conflicting definitions are recorded with the agent that holds each, not resolved. It is not one-shot: an agent can raise a scoped update later, for example before a delegation or before final synthesis.', - enabled: true, - activated: true, - activatedAt: 'Init, before first delegation', - convergence: { - lockedFrame: { wellFormed: true, missingFields: [] }, - roundsPerPhase: [ - { phase: 'CONVENE', rounds: 1 }, - { phase: 'FRAME', rounds: 2 }, - { phase: 'GROUND', rounds: 3 }, - { phase: 'VERIFY', rounds: 2 }, - { phase: 'LOCK', rounds: 1 } - ] - }, - correctness: { - issueCoverage: { - covered: 5, - total: 6, - missedIssues: ['Customer-facing impact window'] - }, - definitionsFromAllAgents: { - complete: false, - reason: - 'Four of five issues carry a definition from every agent. The db agent did not define "error rate".', - issuesMissingDefinition: ['error rate'] - } - }, - cost: { - usage: { - inputTokens: 4120, - outputTokens: 1380, - cacheReadTokens: 2200, - cacheCreationTokens: 640, - totalTokens: 8340, - inputCost: 0.012, - outputCost: 0.019, - cacheReadCost: 0.0012, - cacheCreationCost: 0.009, - totalCost: 0.0412 - } - }, - benefits: { - goalSuccess: { - achieved: true, - reason: - 'The remediation matches the expected action: roll back checkout-service to the previous version.' - }, - mutualUnderstanding: { - label: 'SATISFACTORY', - reason: - 'Agents used the glossary definitions of "error rate" and "P99 latency" consistently in later messages.' - }, - intentContractConsumedByConcord: true - } - }, - { - protocol: 'CONCORD', - displayName: 'CONCORD', - description: - 'Runs during the task: reconciles divergent-but-correct preferences into one decision. It does not decide who is right, it looks for the outcome every agent can live with, within a round budget. It ends in COMMIT (every agent clears the satisfaction floor) or BEST_EFFORT (the budget ran out, and the shortfall is reported). It certifies agreement, not correctness.', - enabled: true, - activated: true, - activatedAt: 'Specialists diverged, before run_action', - expected: { - expected: true, - reason: - 'The backend and db specialists reached different conclusions about the cause of the incident.' - }, - compliance: { - candidateCompleteness: { passed: true }, - completion: { passed: true, terminalState: 'COMMIT' }, - protocolCompliance: { passed: true, failures: [] } - }, - correctness: { - outcomeSatisfaction: { - label: 'SATISFACTORY', - reason: - 'Every specialist stated it can stand behind the committed remediation.' - }, - scoreFairness: { - label: 'FAIR', - reason: 'Raw scores match each agent’s stated reasoning.' - } - }, - cost: { - usage: { - inputTokens: 9210, - outputTokens: 3050, - cacheReadTokens: 5400, - cacheCreationTokens: 1180, - totalTokens: 18840, - inputCost: 0.031, - outputCost: 0.042, - cacheReadCost: 0.0057, - cacheCreationCost: 0.02, - totalCost: 0.0987 - }, - phases: [ - { - phase: 'pre_concord', - llmCalls: 2, - inputTokens: 900, - outputTokens: 210, - cacheReadTokens: 0, - cacheCreationTokens: 300, - cost: 0.0061 - }, - { - phase: 'init', - llmCalls: 3, - inputTokens: 1400, - outputTokens: 420, - cacheReadTokens: 600, - cacheCreationTokens: 200, - cost: 0.0112 - }, - { - phase: 'anchors', - llmCalls: 3, - inputTokens: 1500, - outputTokens: 480, - cacheReadTokens: 900, - cacheCreationTokens: 180, - cost: 0.0139 - }, - { - phase: 'seeds', - llmCalls: 4, - inputTokens: 1800, - outputTokens: 700, - cacheReadTokens: 1100, - cacheCreationTokens: 250, - cost: 0.0188 - }, - { - phase: 'scores', - llmCalls: 6, - inputTokens: 2600, - outputTokens: 900, - cacheReadTokens: 2000, - cacheCreationTokens: 150, - cost: 0.0331 - }, - { - phase: 'commit_and_after', - llmCalls: 2, - inputTokens: 1010, - outputTokens: 340, - cacheReadTokens: 800, - cacheCreationTokens: 100, - cost: 0.0156 - } - ], - bounds: [ - { name: 'Rounds', achieved: 3, bound: 6 }, - { name: 'Options generated', achieved: 6, bound: 9 }, - { name: 'Agent evaluations', achieved: 18, bound: 27 }, - { name: 'Model calls', achieved: 20, bound: 40 } - ] - }, - outcome: { - tau: 0.7, - trajectory: [ - { round: 0, worstAgentScore: 0.31 }, - { round: 1, worstAgentScore: 0.44 }, - { round: 2, worstAgentScore: 0.58 }, - { round: 3, worstAgentScore: 0.72 } - ], - agentScores: [ - { agent: 'telemetry', score: 0.84 }, - { agent: 'backend', score: 0.72 }, - { agent: 'db', score: 0.78 } - ], - shortfall: null - } - } - ] - }, - '41114179-2600-4073-a51d-0150651f2ae8': { - sessionId: '41114179-2600-4073-a51d-0150651f2ae8', - protocols: [ - { - protocol: 'CONCORD', - displayName: 'CONCORD', - enabled: true, - activated: true, - activatedAt: '2026-10-07T08:41:12Z', - expected: { - expected: true, - reason: - 'The backend and db specialists disagreed on the remediation: a rollback versus a database fix.' - }, - compliance: { - candidateCompleteness: { passed: true }, - completion: { passed: true, terminalState: 'BEST_EFFORT' }, - protocolCompliance: { - passed: false, - failures: ['anchors: missing anchor for agent "db"'] - } - }, - correctness: { - outcomeSatisfaction: { - label: 'MIXED', - reason: - 'The db specialist accepted the outcome only because its score was lifted, not because it can live with it.' - }, - scoreFairness: { - label: 'INSUFFICIENT_DATA', - reason: 'Scoring rationale was not recorded for two agents.' - } - }, - cost: { - usage: { - inputTokens: 15200, - outputTokens: 5100, - cacheReadTokens: 9800, - cacheCreationTokens: 2100, - totalTokens: 32200, - inputCost: 0.0452, - outputCost: 0.064, - cacheReadCost: 0.0112, - cacheCreationCost: 0.053, - totalCost: 0.1734 - }, - phases: [ - { - phase: 'init', - llmCalls: 3, - inputTokens: 2100, - outputTokens: 600, - cacheReadTokens: 800, - cacheCreationTokens: 400, - cost: 0.0181 - }, - { - phase: 'seeds', - llmCalls: 5, - inputTokens: 3900, - outputTokens: 1300, - cacheReadTokens: 2200, - cacheCreationTokens: 600, - cost: 0.0402 - }, - { - phase: 'scores', - llmCalls: 14, - inputTokens: 9200, - outputTokens: 3200, - cacheReadTokens: 6800, - cacheCreationTokens: 1100, - cost: 0.1151 - } - ], - bounds: [ - { name: 'Rounds', achieved: 6, bound: 6 }, - { name: 'Options generated', achieved: 9, bound: 9 }, - { name: 'Agent evaluations', achieved: 27, bound: 27 }, - { name: 'Model calls', achieved: 22, bound: 40 } - ] - }, - outcome: { - tau: 0.7, - trajectory: [ - { round: 0, worstAgentScore: 0.22 }, - { round: 1, worstAgentScore: 0.31 }, - { round: 2, worstAgentScore: 0.38 }, - { round: 3, worstAgentScore: 0.42 }, - { round: 4, worstAgentScore: 0.47 }, - { round: 5, worstAgentScore: 0.49 }, - { round: 6, worstAgentScore: 0.51 } - ], - agentScores: [ - { agent: 'telemetry', score: 0.77 }, - { agent: 'db', score: 0.51 }, - { agent: 'backend', score: 0.71 } - ], - shortfall: { agent: 'db', shortBy: 0.19 } - } - } - ] - }, - '0e78e3a6-9ec4-46af-8835-0a69360c6c9e': { - sessionId: '0e78e3a6-9ec4-46af-8835-0a69360c6c9e', - protocols: [ - { - protocol: 'ACCORD', - displayName: 'ACCORD', - enabled: true, - activated: true, - activatedAt: 'Init, before first delegation', - convergence: { - lockedFrame: { - wellFormed: false, - missingFields: ['scope', 'success_criteria'] - }, - roundsPerPhase: [ - { phase: 'CONVENE', rounds: 1 }, - { phase: 'FRAME', rounds: 4 }, - { phase: 'GROUND', rounds: null }, - { phase: 'VERIFY', rounds: 1 }, - { phase: 'LOCK', rounds: 1 } - ] - }, - correctness: { - // No ground truth for this session. - issueCoverage: null, - definitionsFromAllAgents: { - complete: null, - reason: 'Agent definitions were not recorded in the trace.' - } - }, - cost: { - usage: { - inputTokens: 3300, - outputTokens: 900, - cacheReadTokens: null, - cacheCreationTokens: null, - totalTokens: null, - inputCost: 0.011, - outputCost: 0.0105, - totalCost: 0.0215 - } - }, - benefits: { - goalSuccess: null, - mutualUnderstanding: null, - intentContractConsumedByConcord: false - } - }, - { - protocol: 'CONCORD', - displayName: 'CONCORD', - enabled: true, - activated: false, - activatedAt: null, - expected: { - expected: true, - reason: - 'The backend and db specialists reached conflicting conclusions, but no decision protocol ran.' - } - } - ] - }, - '65d1c926-de00-451f-bdcc-eb38bbe39c3a': { - sessionId: '65d1c926-de00-451f-bdcc-eb38bbe39c3a', - protocols: [ - { - protocol: 'CONCORD', - displayName: 'CONCORD', - enabled: true, - activated: true, - activatedAt: null, - expected: null, - compliance: { - candidateCompleteness: null, - completion: { passed: null, terminalState: null }, - protocolCompliance: { passed: true } - }, - correctness: { - outcomeSatisfaction: null, - scoreFairness: { label: 'SOME_NEW_LABEL', reason: null } - }, - cost: { - usage: { - inputTokens: 5200, - outputTokens: null, - cacheReadTokens: 0, - cacheCreationTokens: null, - totalTokens: null, - inputCost: 0.0052, - totalCost: null - }, - phases: [ - { - phase: 'init', - llmCalls: 2, - inputTokens: 1200, - outputTokens: null, - cacheReadTokens: 0, - cacheCreationTokens: null, - cost: 0.0084 - }, - { - phase: 'scores', - llmCalls: 5, - inputTokens: 4000, - outputTokens: 1100, - cacheReadTokens: 0, - cacheCreationTokens: null, - cost: 0.0312 - } - ], - bounds: [ - { name: 'Rounds', achieved: 2, bound: null }, - { name: 'Model calls', achieved: null, bound: 40 } - ] - }, - outcome: { - tau: null, - trajectory: null, - agentScores: [ - { agent: 'telemetry', score: 0.7 }, - { agent: 'backend', score: null } - ], - shortfall: null - } - } - ] - }, - '5980fc14-3a53-429a-bef7-6d7af89cae8c': { - sessionId: '5980fc14-3a53-429a-bef7-6d7af89cae8c', - protocols: [ - { - protocol: 'ACCORD', - displayName: 'ACCORD', - enabled: true, - activated: false, - activatedAt: null - } - ] - } -}; diff --git a/ui/src/api/mock/sessionsWithCognitionProtocols.ts b/ui/src/api/mock/sessionsWithCognitionProtocols.ts deleted file mode 100644 index 657361d..0000000 --- a/ui/src/api/mock/sessionsWithCognitionProtocols.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { isConcordProtocol } from '@/common/cognitionProtocols'; -import { - SessionsWithCognitionProtocols, - SessionWithCognitionProtocols -} from '@/types/scope.type'; -import { mockSessionsWithCognitiveObservability } from './sessionsWithCognitiveObservability'; -import { mockSessionCognitionProtocols } from './sessionCognitionProtocols'; - -// The sre-triage application: an orchestrator and five specialists, all running -// on the same model. -const SRE_TRIAGE_AGENTS = [ - 'sre_lead', - 'telemetry', - 'backend', - 'db', - 'comms', - 'verifier' -]; -const SRE_TRIAGE_LLM = 'vertex_ai/gemini-2.5-flash'; - -// The usage of a session includes the usage of its protocols, so it cannot be -// lower than what the per-session Cognition protocol mock reports for them. For the sessions -// where the cognitive observability mock is lower, the usage is overridden here -// (the protocols' usage plus the usage of the agents' own work). -const SESSION_USAGE_OVERRIDES: Record< - string, - Partial> -> = { - // ACCORD (8,340 tokens, $0.0412) + CONCORD (18,840 tokens, $0.0987) - '4493f565-d069-47ec-9a6c-5df6dc8f31b7': { - tokens: 36400, - cost: 0.417, - duration: 78200 - }, - // CONCORD (32,200 tokens, $0.1734) - '41114179-2600-4073-a51d-0150651f2ae8': { tokens: 41800, cost: 0.479 } -}; - -// The sessions of the cognitive observability mock, each with its Cognition protocol -// status taken from the per-session Cognition protocol mock, so that both views stay -// consistent. Sessions without an entry in the Cognition protocol mock have no Cognition protocol. -const toSessionWithCognitionProtocols = ( - session: (typeof mockSessionsWithCognitiveObservability.sessions)[number] -): SessionWithCognitionProtocols => { - const protocols = mockSessionCognitionProtocols[session.sessionId]?.protocols ?? []; - const concord = protocols.find(isConcordProtocol); - const scores = concord?.outcome?.agentScores ?? []; - - // The worst-off agent is only known when every agent has a score. - const worstOffSatisfaction = - scores.length > 0 && - scores.every((agent) => typeof agent.score === 'number') - ? Math.min(...scores.map((agent) => agent.score as number)) - : null; - - return { - sessionId: session.sessionId, - timestamp: session.timestamp, - agents: SRE_TRIAGE_AGENTS, - llms: [SRE_TRIAGE_LLM], - tokens: session.tokens, - status: session.status, - cost: session.cost, - duration: session.duration, - ...SESSION_USAGE_OVERRIDES[session.sessionId], - cognitiveObservabilityMetrics: session.cognitiveObservabilityMetrics, - cognitiveFailures: session.cognitiveFailures, - cognitionProtocols: protocols.map(({ protocol, enabled, activated }) => ({ - protocol, - enabled, - activated - })), - concord: - concord && concord.activated - ? { - terminalState: - concord.compliance?.completion?.terminalState ?? null, - worstOffSatisfaction - } - : null - }; -}; - -export const mockSessionsWithCognitionProtocols: SessionsWithCognitionProtocols = { - sessions: mockSessionsWithCognitiveObservability.sessions.map( - toSessionWithCognitionProtocols - ) -}; diff --git a/ui/src/api/mock/sessionsWithCognitiveObservability.ts b/ui/src/api/mock/sessionsWithCognitiveObservability.ts deleted file mode 100644 index 6c1ccb7..0000000 --- a/ui/src/api/mock/sessionsWithCognitiveObservability.ts +++ /dev/null @@ -1,2120 +0,0 @@ -import { SessionsWithCognitiveObservability, Unit } from '@/types/scope.type'; - -export const mockSessionsWithCognitiveObservability: SessionsWithCognitiveObservability = - { - sessions: [ - { - sessionId: '4493f565-d069-47ec-9a6c-5df6dc8f31b7', - timestamp: '1790938228.499732', - agents: [ - 'concierge_agent', - 'moderator', - 'itinerary_agent', - 'schedule_agent' - ], - llms: [], - tokens: 8973, - status: 'done', - cost: 0.10331, - duration: 34349.16186332703, - session_metrics: [ - { - name: 'CyclesCount', - value: 2, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/6 tool calls failed.', - error: null - }, - { - name: 'Duration', - value: 34349.16186332703, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 34349.16ms', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.10331, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.103310 USD (gpt-4)', - error: null - }, - { - name: 'Groundedness', - value: 0.5, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - 'The input contained no information retrieval context, so an average score was assigned. The output neither attempted to answer the question nor provided useful guidance, falling short of user expectations.', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The actual outcome did not provide any information about train routes or call relevant tools to determine the connection, thereby entirely failing to achieve the desired task.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains entirely irrelevant statements that do not address the question about train routes or connections between Celestia, Luminos, and Verdantia.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in the user input:**\n - The explicit question is: *"Can I get a direct train from Celestia to Luminos, or do I need to connect through Verdantia?"*\n - This consists of two aspects:\n 1. Is there a direct train from Celestia to Luminos?\n 2. If not, is a connection through Verdantia required (implying the need for an alternate route)?\n - Implied need: The user wants actionable, specific travel information. The assistant is expected to address both possibilities in sufficient detail.\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant responded with a command to "end" the conversation without providing any actual information about the user\'s query.\n - Neither of the two aspects of the user\'s question (availability of direct trains or whether a connection through Verdantia is necessary) was addressed.\n - The user’s implied need for useful travel guidance was completely ignored.\n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?**\n - The assistant\'s response was entirely superficial as it did not address the query at all.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The user’s request inherently involves constraints (locations and routes) that should guide the specific nature of the answer. For example:\n - If there’s no direct train, are there other routes besides Verdantia?\n - Timing or availability might also be relevant if the assistant were to ask a follow-up question for clarification.\n - None of these considerations were acknowledged, nor did the assistant suggest follow-up steps to gather further information.\n\n**Score:** *1 (Severe omission)* \n**Justification:** The assistant entirely failed to provide any meaningful or relevant response to the user’s query. No effort was made to address the main question, relevant constraints, or implied needs. This constitutes a severe omission, resulting in the lowest score.', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 1, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 6 tool calls (skipped=0)', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - "**Reasoning:** \n1. **Primary Intent Identification**: \n The user's query is seeking information about train travel options between Celestia and Luminos and whether a direct train is available or a connection via Verdantia is required. The primary intent is to obtain travel-related guidance. The Assistant's response does not identify the intent, as it uses a default format (`goto: \"__end__\"`), which is non-informative and does not engage with the travel-related nature of the query. \n\n2. **Response Accuracy**: \n The response neither addresses the travel-related intent nor provides any information about train connectivity. It appears to cut off prematurely (`goto: \"__end__\"` with no additional context or detail provided) without offering the user relevant guidance or clarification. \n\n3. **Resolution of Ambiguity**: \n The user's query is not ambiguous; it is a straightforward travel inquiry. However, the Assistant fails to engage with or resolve the user’s request, leaving the query completely unanswered.\n\n**Score**: **1** \nThe system completely misses the user's intent, fails to provide any information relevant to the query, and ends the interaction with no attempt to address the user's needs. This is a clear case of a missed response.", - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 84, "span_id": "", "span_type": "trajectory", "entity_name": "Primary request", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Primary request\' remained in_progress at the end of the trajectory.", "explanation": "Can I get a direct train from Celestia to Luminos, or do I need to connect through Verdantia?.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 75, "span_id": "", "span_type": "trajectory", "entity_name": "Primary request 2", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Primary request 2\' remained in_progress at the end of the trajectory.", "explanation": "Here are the direct train options: ### Weekdays (Monday to Friday) - **06:30 - 11:30** (Train CL101, Express) - **09:00 - 14:00** (Train CL103, Standard) - **12:30 - 17:30** (Train CL105, Express) - **16:00 - 21:00** (Train CL107, Standard) - **18:30 - 23:30** (Train CL109, Express) ### Saturday - **07:30 - 12:30** (Train CL201, Express) - **11:00 - 16:00** (Train CL203, Standard) - **15:00 - 20:00** (Train CL205, Express) - **18:30 - 23:30** (Train CL207, Standard) ### Sunday - **09:00 - 14:00** (Train CL301, Standard) - **13:00 - 18:00** (Train CL303, Express) - **17:00 - 22:00** (Train CL305, Standard) Let me know if you\'d like help selecting a specific train based on your schedule!.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 8, "span_id": "ac0a271fc27e4ef9", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The assistant\'s output does not provide any factual information or reasoning related to the user\'s query about train routes. It only transitions to another agent without addressing the question.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "c4da2ce5af7d0a2c", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any factual information or reasoning to address the user\'s query about train connections. It lacks grounding in the provided context or external data.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 50, "span_id": "08ccc287a485a82d", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The assistant\'s output claims direct train options from Celestia to Luminos, but the provided document explicitly states that no direct train exists, requiring a connection through Verdantia.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 67, "span_id": "5f70e26baabb3806", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The assistant\'s output claims conflicting information about direct train availability without resolving the contradiction or providing evidence from the provided facts or tools.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 2, "total_minor": 4}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.71, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.22, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.42, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.38, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.79, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.74, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.91, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.27, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.54, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.22, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.37, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Incomplete Synthesis', - confidence: 0.93, - remediations: ['Concord'] - }, - { - name: 'Shared Task Model Failure', - confidence: 0.88, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Collective Goal Coordination Failure', - confidence: 0.74, - remediations: ['Concord'] - }, - { - name: 'Representational Fidelity Loss at Interface', - confidence: 0.45, - remediations: ['Concord'] - } - ] - }, - { - sessionId: '41114179-2600-4073-a51d-0150651f2ae8', - timestamp: '1790946979.691819', - agents: [ - 'moderator', - 'schedule_agent', - 'itinerary_agent', - 'concierge_agent' - ], - llms: [], - tokens: 30249, - status: 'done', - cost: 0.34635, - duration: 64889.11008834839, - session_metrics: [ - { - name: 'WorkflowEfficiency', - value: 0.6, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=10', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 163, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 154, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "--- ### **Suggested Plan** #### **Day 1: Celestia** - Morning: Visit Crystal Tower Observatory (9:00 AM - 10:30 AM).", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "9dabb033d2767839", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any information or reasoning related to the user\'s query, nor does it reference any grounded facts or policies.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 2, "total_minor": 1}', - error: null - }, - { - name: 'Duration', - value: 64889.11008834839, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 64889.11ms', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/20 tool calls failed.', - error: null - }, - { - name: 'CyclesCount', - value: 2, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Cost', - value: 0.34635, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.346350 USD (gpt-4)', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system did not provide any cost calculations or details regarding train fares or attractions, failing completely to achieve the desired task.', - error: null - }, - { - name: 'Groundedness', - value: 0.5, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The response neither addressed the user's query nor attempted to compute or estimate the cost of visiting Celestia, Verdantia, and Luminos, which would require interpreting the question and forming an informed approximation. However, without information retrieval context provided, the evaluation defaults to an average score.", - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - "Reasoning: \n1. **Primary Intent Identification:** The user's query is a detailed request asking for an estimated total cost of visiting three locations (Celestia, Verdantia, and Luminos), including one attraction in each location and train fares. The primary intent is to calculate the cost of this hypothetical weekend trip. \n\n - The assistant's response does not identify this intent. Instead, it generates a generic command response directing the flow to `__end__` without addressing the content of the query. This indicates that the assistant failed to recognize the user's request.\n\n2. **Response Accuracy and Completeness:** The response does not provide any relevant information, let alone a cost estimation or breakdown for the trip. There is no calculation or acknowledgment of the components of the request (train fares, attractions, etc.). As such, the response is entirely off-target. \n\n3. **Ambiguity Handling:** While the query isn't explicitly ambiguous, there are some details the system might need to clarify (e.g., what type of attractions the user prefers, specific locations within the cities, or whether the fare calculation includes return trips). However, the system does not make any attempt to clarify or resolve potential ambiguities—it simply terminates the interaction without engagement.\n\nScore: **1** \nThe assistant completely misses the user's intent and provides no relevant response, making this a clear example of failure in intent recognition and response generation.", - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - "**Reasoning:**\n\n1. **Identifying distinct questions, intents, and implied needs in the user input:**\n - The user asked for the **total cost** of a weekend visiting three specific locations: Celestia, Verdantia, and Luminos.\n - The cost should include **one attraction** per destination.\n - The cost should also include **all train fares.**\n - The question implicitly requires the assistant to have knowledge of attractions, train prices between the destinations, and possibly regional pricing.\n\n2. **Checking if the assistant addressed EACH one fully:**\n - The assistant's response did not address any of the user's questions or intents. Instead of providing any relevant information or calculations, the reply ended the conversation prematurely, suggesting no actual effort to evaluate or answer the query.\n - The assistant missed addressing:\n - Attraction costs for Celestia, Verdantia, and Luminos.\n - Train fare costs between the locations.\n - Total weekend cost calculation, which was the key outcome sought by the user.\n\n3. **Assessing depth:**\n - Since the assistant did not provide any information, the response was entirely superficial, failing even to acknowledge the complexity of the question.\n\n4. **Checking for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant ignored all constraints, such as visiting one attraction per location and including train fares, even though they were clearly stated by the user.\n - Follow-up needs (e.g., asking about preferences for attractions/exact dates) were not considered, nor did the assistant convey any clarification requests.\n\n**Score: 1** \nThe response is a **severe omission** of the main query. The assistant failed entirely to respond to the user’s question, provide relevant information, or engage meaningfully with the user's request.", - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - "The score is 0.00 because none of the statements in the actual output addressed the specific inquiry about the cost of a weekend trip, and instead focused on irrelevant technical details about system operations, which are unrelated to the user's query.", - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9875, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 20 tool calls (skipped=0)', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.6, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.22, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.36, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.72, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.64, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.38, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.67, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.85, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.21, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.84, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.76, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Transactive Memory Failure', - confidence: 0.72, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Collective Goal Coordination Failure', - confidence: 0.64, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Incomplete Synthesis', - confidence: 0.54, - remediations: ['Accord'] - } - ] - }, - { - sessionId: '0e78e3a6-9ec4-46af-8835-0a69360c6c9e', - timestamp: '1790946910.654101', - agents: [ - 'concierge_agent', - 'moderator', - 'schedule_agent', - 'itinerary_agent' - ], - llms: [], - tokens: 16638, - status: 'done', - cost: 0.1991, - duration: 46145.748138427734, - session_metrics: [ - { - name: 'Duration', - value: 46145.748138427734, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 46145.75ms', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/12 tool calls failed.', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.1991, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.199100 USD (gpt-4)', - error: null - }, - { - name: 'CyclesCount', - value: 1, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The output does not provide a response or address the user's request for a comprehensive weekend trip and cost summary. There is no alignment between the request and the output provided.", - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system did not provide any travel details, attractions, or cost summary, which were all key components of the desired task. Therefore, the actual outcome does not achieve the task at all.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains entirely irrelevant statements that fail to address the request for a comprehensive weekend trip plan and cost summary.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - "**Reasoning:**\n\n1. **Identify distinct questions, intents, and implied needs in the user input:**\n - The user requests a **comprehensive weekend trip plan** that includes:\n - Travel arrangements.\n - One attraction per city.\n - A cost summary.\n - The use of \"comprehensive\" implies a need for detailed and thoughtful planning, not just a list of random suggestions.\n - A cost summary suggests at least a rough estimate or breakdown of expenses.\n\n2. **Check if the assistant addressed each need fully:**\n - The assistant response contains a \"Command\" to terminate the interaction without any substantive response or acknowledgment of the user's request.\n - No trip plan, travel arrangements, attractions, or cost summary are provided.\n - The request is completely ignored, leaving all elements unaddressed.\n\n3. **Assess depth:**\n - There is no depth in the assistant's response, as it fails to provide any content relevant to the user's request.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant does not engage with any of the explicit or implicit requirements of the user's input.\n - The user may have implicit expectations (e.g., logical travel sequencing between cities, an appropriate cost estimate) that a comprehensive trip plan should fulfill, but these aren't addressed due to the absence of a meaningful response.\n\n**Score: 1**\n- The assistant's response demonstrates a **severe omission** by entirely failing to address the user's request. Neither the main question nor the implied details were acknowledged or answered. This results in a clear non-answer.", - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Intent Identification:**\n - The user\'s primary intent is clear: they are requesting a comprehensive weekend trip plan, which includes travel options, one attraction per city, and a cost summary. This query involves multiple components, including transportation, activities, and pricing details.\n - The assistant\'s output suggests it did not recognize this intent at all. The response structure indicates a default termination ("goto": "__end__") without any actionable response or acknowledgment of the query, implying the intent was not addressed.\n\n2. **Response Accuracy and Completeness:**\n - The response neither attempts to provide any form of a travel itinerary nor includes any elements related to the user\'s request.\n - There is no effort made to compile elements of travel or attractions, let alone include a cost summary. The user query is entirely ignored.\n\n3. **Ambiguity Resolution:**\n - The user query was not ambiguous; it was concise and well-defined with clear components. Therefore, there was no need for clarification or disambiguation. However, the assistant failed to even engage with or respond to the request.\n\n4. **Overall Performance:**\n - The assistant fails across all criteria. It does not identify the intent, does not address the components of the user query, and defaults to ending the interaction without providing any value to the user.\n\n**Score:** 1', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9583333333333334, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 12 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 104, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 95, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "Here is a comprehensive weekend trip plan, including travel times, attractions, and a cost summary: --- ### Itinerary #### Day 1: Celestia to Verdantia - **Travel Time**: 3 hours - **Average Travel Time**: 1.5 hours (round trip) - **Attraction in Verdantia**: Visit the Verdant Gardens, a lush botanical park known for its exotic plants and serene atmosphere.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 104, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary 2", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary 2\' remained pending at the end of the trajectory.", "explanation": "Here is the updated cost summary for the weekend trip based on the accurate train fares: --- ### **Updated Cost Summary** #### **Day 1: Celestia to Verdantia** - **Train Fare**: $45 (Standard Class, one-way) - **Attraction Admission**: $30 (Rainforest Canopy Walk) - **Local Transportation**: $6 (Forest shuttle) - **Total for Day 1**: **$81** #### **Day 2: Verdantia to Luminos** - **Train Fare**: $35 (Standard Class, one-way) - **Attraction Admission**: $22 (Mirror Lake Observatory) - **Local Transportation**: $7 (River ferry) - **Total for Day 2**: **$64** #### **Day 3: Luminos to Celestia** - **Train Fare**: $75 (Standard Class, one-way) - **Attraction Admission**: $25 (Crystal Tower Observatory) - **Local Transportation**: $3.50 (Metro) - **Total for Day 3**: **$103.50** --- ### **Grand Total** - **Train Fares**: $155 - **Attraction Admissions**: $77 - **Local Transportation**: $16.50 - **Overall Total**: **$248.50 per person** --- This plan ensures a balanced weekend with travel, exploration, and relaxation.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "64bc2ce5ac4fb412", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any material claims or content to evaluate for groundedness against the provided facts or policy.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 3, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.47, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.32, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.97, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.47, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.27, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.28, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.88, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.68, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.85, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.78, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.63, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Metacognitive Blindness', - confidence: 0.95, - remediations: ['Accord'] - }, - { - name: 'Situational State Synchronisation Failure', - confidence: 0.77, - remediations: ['Accord', 'Concord'] - } - ] - }, - { - sessionId: '65d1c926-de00-451f-bdcc-eb38bbe39c3a', - timestamp: '1790946855.76823', - agents: [ - 'moderator', - 'concierge_agent', - 'itinerary_agent', - 'schedule_agent' - ], - llms: [], - tokens: 18640, - status: 'done', - cost: 0.22916, - duration: 49812.01910972595, - session_metrics: [ - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Duration', - value: 49812.01910972595, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 49812.02ms', - error: null - }, - { - name: 'CyclesCount', - value: 2, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/12 tool calls failed.', - error: null - }, - { - name: 'Cost', - value: 0.22916, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.229160 USD (gpt-4)', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The actual outcome did not address any aspect of the task. No detailed itinerary was created, no attractions were identified, no train routes were planned, and no price information was provided.', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - 'The output did not provide any itinerary or relevant details requested by the input, nor did it address trains or pricing information. It entirely failed to fulfill the requirements of the input query, providing no alignment with the task.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains multiple irrelevant statements about classification types, internal control flow, and program logic, none of which contribute to creating a detailed weekend itinerary with attractions, transportation, or pricing as requested in the input.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - "**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in the user input:**\n - The user asks for a detailed weekend itinerary covering three cities. \n - Includes one attraction per city.\n - Specifies that train transport between the cities is required.\n - Requests the overall price for the entire itinerary.\n - Implied needs:\n - A logical and feasible sequence of visits across the cities.\n - Information about timing, such as train schedules or attraction availability.\n - A coherent summary of costs, including transport and attractions.\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant's output contains no actionable information or details. It simply ends the conversation without responding to the user’s request.\n - None of the user's explicit or implied needs were addressed (e.g., itinerary, attractions, trains, or pricing).\n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?**\n - The assistant completely failed to provide any details, let alone meet the required complexity of the user's request.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant did not address the explicit requirements (itinerary, attractions, trains, price).\n - No attempt was made to clarify constraints, ask follow-up questions, or meet implied needs.\n\n**Score:** 1 \nThe assistant missed the main question entirely and failed to provide any response to the user’s detailed request. This is a severe omission.", - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9583333333333334, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 12 tool calls (skipped=0)', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Does the system correctly identify the user\'s primary intent?** \n - The primary intent in the user query is a request for a "detailed weekend itinerary" covering three cities, which includes one attraction in each city, train connections, and the overall price. This is a request for a complex, multi-part response involving itinerary planning, attractions, transport information, and cost estimation. \n - The Assistant\'s response does not acknowledge or interpret the user\'s intent. Instead, it outputs a generic `{"type": "Command", "goto": "__end__", "update": {"next": "__end__"}}`, which indicates no effort to process the query or recognize the user\'s intent. This suggests a complete miss in understanding the query. \n\n2. **Does the response address the identified intent accurately and completely?** \n - The response makes no attempt to provide a weekend itinerary, attractions, train details, or a price as requested by the user. It doesn’t fulfill any aspect of the user\'s request and doesn’t even attempt to guide the user elsewhere for assistance. The user\'s entire request remains unaddressed, making this response fully inadequate.\n\n3. **If the query is ambiguous, does the system resolve or clarify the ambiguity appropriately?** \n - While the query is fairly specific, it could raise practical ambiguities (e.g., which cities are involved, preferred types of attractions, specifics on train connections, or budget constraints). However, the Assistant neither clarifies these ambiguities nor attempts to engage the user for more information. By completely failing to recognize or act upon the query, it doesn’t even reach the point of addressing potential ambiguities, let alone resolving them.\n\n**Score: 1** \n- The system completely fails to identify the user\'s intent or provide a relevant or meaningful response to the query. This is an example of a "complete miss." The user’s request is ignored, and no effort is made to address, redirect, or clarify the query.', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Information Precision Retention", "span_index": 33, "span_id": "", "span_type": "trajectory", "entity_name": "concierge_agent", "metric_score": 0.0, "fatality_score": 0.7, "reasoning": "Train fares and travel times were miscalculated, leading to a final itinerary with incorrect pricing.", "explanation": "", "observed_impact": "metric_failure", "confidence": 0.7, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Orchestration Verification", "span_index": 104, "span_id": "", "span_type": "trajectory", "entity_name": "moderator", "metric_score": 0.0, "fatality_score": 0.7, "reasoning": "The final synthesis failed to verify and reconcile discrepancies in travel times and fares.", "explanation": "", "observed_impact": "metric_failure", "confidence": 0.7, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 104, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Hallucination", "span_index": 33, "span_id": "", "span_type": "trajectory", "entity_name": "concierge_agent", "metric_score": 0.0, "fatality_score": 0.7, "reasoning": "Train fares and travel times were fabricated or contradicted, leading to an inaccurate final itinerary.", "explanation": "", "observed_impact": "metric_failure", "confidence": 0.7, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [], "total_fatal": 4, "total_minor": 0}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.98, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.64, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.86, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.69, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.89, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.66, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.76, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.24, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.38, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.43, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Representational Failure', - confidence: 0.97, - remediations: ['Concord'] - }, - { - name: 'Individual Metacognitive Blindness', - confidence: 0.63, - remediations: ['Accord', 'Concord'] - } - ] - }, - { - sessionId: '5980fc14-3a53-429a-bef7-6d7af89cae8c', - timestamp: '1790946805.44977', - agents: [ - 'itinerary_agent', - 'moderator', - 'concierge_agent', - 'schedule_agent' - ], - llms: [], - tokens: 17980, - status: 'done', - cost: 0.2153, - duration: 45122.31087684631, - session_metrics: [ - { - name: 'CyclesCount', - value: 1, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Duration', - value: 45122.31087684631, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 45122.31ms', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/11 tool calls failed.', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.2153, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.215300 USD (gpt-4)', - error: null - }, - { - name: 'Groundedness', - value: 0.5, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - 'The input does not provide any specific retrieval context such as information sources or raw documents to evaluate the faithfulness of the response. Consequently, the average score was assigned as per the evaluation steps.', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system did not provide any trip plan, attraction details, cost calculations, or transportation information, hence failing to achieve any part of the desired task.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - "The score is 0.00 because the irrelevant statements entirely fail to address the input's request for a weekend trip plan and cost calculation, instead focusing on unrelated aspects like 'Command' types, 'goto' values, and nested keys, none of which pertain to the topic.", - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:** \n1. **Primary Intent Identification:** \n - The user’s primary intent is to plan a "full weekend trip" involving three cities, with one attraction per city, as well as calculating the total cost (including trains). The user is requesting a detailed itinerary with cost estimation. \n - The assistant\'s response (`{"type": "Command", "goto": "__end__", "update": {"next": "__end__"}}`) does not demonstrate any indication that the intent was identified. This response simply terminates the interaction without addressing the request. Therefore, the system fails to identify the user’s intent. \n\n2. **Response Accuracy and Completeness:** \n - The assistant does not provide any information related to a weekend trip, city attractions, train plans, or cost estimation. The response does nothing to address the user\'s request. \n\n3. **Resolution of Ambiguity:** \n - While the user\'s query could have minor ambiguities (e.g., what "3 cities" are being referred to, specific dates or train routes), the system does not attempt to clarify these issues. A helpful response would have either provided a default plan (e.g., suggested cities and attractions) or asked the user follow-up questions to address uncertainties. \n\n**Score:** 1 (Complete miss) \n- The system entirely failed to identify and handle the user’s query. It neither recognized the intent to generate a weekend trip plan nor attempted to clarify ambiguous details or partially address the request. This is an off-topic and non-responsive outcome, meriting the lowest score.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in user inputs:**\n - User specifically asks for a "full weekend trip plan."\n - They require a plan covering "3 cities" and "one attraction each."\n - User implies the need for "total cost" broken down for the trip.\n - Transportation between cities is mentioned: "plus trains" indicates train travel as the preferred mode.\n - The query implies an expectation of structure and detail, combining logistics, attractions, and budgeting.\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant\'s response contains **no substantive content** addressing any of the user\'s questions or requests. Instead, the assistant outputs a command indicating the conversation ends without engaging or providing information. \n - Negative points for not addressing the itinerary (3 cities, 1 attraction), transportation logistics, or budgeting (total cost).\n - Further, the assistant does not acknowledge the complexities or potential constraints implied in the user\'s query.\n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?**\n - This response is completely inadequate for the complexity of the user query. It provides no usable information or even an acknowledgment of the query\'s depth. The lack of a response entirely dismisses the detailed nature of the user’s request.\n\n4. **Missed constraints, follow-up needs, or implicit requirements:**\n - The assistant missed all constraints, including:\n - Planning across three distinct cities.\n - Activities (one attraction each in each city) that require consideration for timing and feasibility over a weekend trip.\n - Specifying total costs, which likely involves combining attraction entry fees, train fares, and other expenses.\n - Train travel logistics, which could include scheduling and ticket cost research.\n\n - Moreover, no follow-up questions were posed to clarify details (e.g., origin city, preferred attractions, budget limits, or exact travel dates), which would be essential to tailoring a response.\n\n**Score:** **1**\n - Severe omission: The assistant neither answered the query nor acknowledged any aspect or sub-goal, thereby entirely failing to engage with the user’s comprehensive, multi-part request.', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9545454545454546, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 11 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 101, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "full weekend trip plan and total cost 3 cities one attraction each plus trains.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 101, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 92, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary 2", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary 2\' remained in_progress at the end of the trajectory.", "explanation": "Here is a detailed weekend trip plan for visiting three cities: Celestia, Verdantia, and Luminos.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "281fa0233f7c6568", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' is not supported by the provided facts, policy, or context. It appears to be a placeholder or error rather than a grounded response.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 3, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.26, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.39, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.28, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.42, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.71, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.49, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.37, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.95, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.72, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Plan Revision Failure', - confidence: 0.57, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Shared Task Model Failure', - confidence: 0.48, - remediations: ['Accord'] - } - ] - }, - { - sessionId: '04603786-f5b9-4a21-a353-eba478c01449', - timestamp: '1790946759.040906', - agents: [ - 'moderator', - 'schedule_agent', - 'concierge_agent', - 'itinerary_agent' - ], - llms: [], - tokens: 15719, - status: 'done', - cost: 0.18949, - duration: 41194.35214996338, - session_metrics: [ - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/9 tool calls failed.', - error: null - }, - { - name: 'Duration', - value: 41194.35214996338, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 41194.35ms', - error: null - }, - { - name: 'CyclesCount', - value: 1, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.18949, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.189490 USD (gpt-4)', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system failed to provide an itinerary, sightseeing spots, or a cost breakdown for the specified cities. No aspect of the task was achieved.', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The response does not address the user's request for a detailed weekend itinerary with transport and sightseeing information for the specified cities. Additionally, there is no evidence of attempted information retrieval or effort to provide any relevant details, making the response entirely unaligned with the test case requirements.", - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9444444444444444, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 9 tool calls (skipped=0)', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains no relevant information addressing the request for a weekend itinerary or cost breakdown, focusing instead on structural and procedural details unrelated to the input.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:** \n1. **Intent Recognition:** \n The user\'s primary intent is clear: they want the assistant to build a detailed weekend itinerary, including transportation via trains, one sight in each city (Celestia, Verdantia, and Luminos), and a full cost breakdown. The query is unambiguous, containing specific instructions about the components of the itinerary (trains, one sight per city, cost breakdown). \n\n The assistant\'s response (`{"type": "Command", "goto": "__end__", "update": {"next": "__end__"}}`) does not acknowledge or identify the user\'s intent. It appears the assistant terminated the conversation instead of generating an itinerary. Thus, the system fails to recognize or engage with the request appropriately. \n\n2. **Response Accuracy and Completeness:** \n Since the assistant did not provide any meaningful response to the query, it fails to address the identified intent. There was no attempt to provide an itinerary, suggest train schedules, recommend sights in each city, or create a cost breakdown, leaving all components unaddressed. \n\n3. **Handling Ambiguity:** \n The user query is not ambiguous, and thus no clarification or resolution was required. However, the assistant did not even attempt to address the straightforward query.\n\n**Score:** 1 \nThe assistant fails to identify or respond meaningfully to the user\'s query, resulting in a complete miss.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - "**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in the user input:**\n - The user asks for a complete weekend itinerary for three fictional cities (Celestia, Verdantia, and Luminos).\n - Specific details requested:\n - Transportation: Include trains between the cities.\n - Activities: Include one specific sight per city.\n - Cost: Provide a total cost breakdown.\n - Implied needs:\n - Logical progression between the cities (timing and feasibility of the itinerary).\n - Estimates for costs (transportation, entrance fees, etc.).\n - A coherent and detailed plan spanning the weekend.\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant's response is simply a command structure without any substantive content. It does not attempt to answer the user’s query or engage with the itinerary-building task.\n - No mention is made of trains, sights in each city, or cost calculations.\n - No itinerary details (timing, logistics, or total costs) are provided.\n - The assistant did not reflect any understanding of the query.\n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?**\n - The answer was completely superficial and did not address even a single aspect of the query. For a question with this level of complexity, the response falls far below the expected depth.\n\n4. **Check for missed constraints, follow-ups, or implicit requirements:**\n - All constraints (inclusion of trains, one sight per city, total cost breakdown) were missed.\n - The assistant provided no clarification questions or follow-ups to address the implicit requirements (e.g., travel timings, sequence of cities).\n\n**Score:** 1 \n- The assistant's response constitutes a severe omission. It completely missed the main question and provided a non-answer. There is no engagement with the user's request.", - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 93, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "d658f61214460ecc", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' is not supported by any provided facts, policies, or evidence, and does not address the user\'s request.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 1, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.69, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.34, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.78, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.33, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.99, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.71, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.65, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.75, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.87, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.82, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Negotiation and Consensus Failure', - confidence: 0.8, - remediations: ['Accord'] - }, - { - name: 'Individual Metacognitive Blindness', - confidence: 0.72, - remediations: ['Concord'] - } - ] - }, - { - sessionId: '15a38285-c33f-4f87-b00e-dff67392adb9', - timestamp: '1790946710.29037', - agents: [ - 'concierge_agent', - 'itinerary_agent', - 'moderator', - 'schedule_agent' - ], - llms: [], - tokens: 18994, - status: 'done', - cost: 0.22632, - duration: 43502.74205207825, - session_metrics: [ - { - name: 'Duration', - value: 43502.74205207825, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 43502.74ms', - error: null - }, - { - name: 'CyclesCount', - value: 2, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/11 tool calls failed.', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.22632, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.226320 USD (gpt-4)', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The response does not address the user's request for a weekend plan with costs, attractions, and train travel, nor does it provide any relevant information. It only includes a command without content, failing to meet the user's needs in the input.", - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The actual outcome did not provide any details or outputs for the weekend plan, costs, attractions, or train travel, which were explicitly required in the task.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Primary Intent Identification**:\n - The user\'s query explicitly requests a "complete weekend plan" for visiting "all three cities," including details about costs, one attraction per city, and using train travel. This indicates the primary intent is to receive a detailed itinerary and cost breakdown for a weekend trip.\n\n - The system\'s response, however, does not make any effort to specifically acknowledge or identify this intent. Instead, the assistant generates a generic response to "end" the interaction, which is unrelated to the query.\n\n2. **Response Accuracy and Completeness**:\n - The response directs the conversation to "goto: __end__" and does not provide any itinerary, costs, or travel information.\n - The lack of effort in addressing the user\'s query demonstrates a lack of fulfillment of the user\'s request. None of the requested information (weekend plan, attractions, costs, or train travel) is present in the response. This constitutes a complete failure to address the identified intent.\n\n3. **Handling Ambiguity**:\n - The query is not ambiguous. The user has clearly outlined their requirements (e.g., a weekend plan, including visiting three cities, one attraction per city, costs, and train travel). There is no indication that the system attempted to resolve or clarify the intent further, despite the opportunity to do so.\n\n**Score**: 1 \n- The assistant completely misses the primary intent and provides an off-topic response. It neither identified the intent correctly nor made any attempt to satisfy the user\'s request.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - "The score is 0.00 because the output contained irrelevant statements that failed to address the input's request for a detailed weekend plan with costs, attractions, and train travel, making it completely unrelated to the task.", - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:**\n1. **Identify distinct questions, intents, and implied needs:**\n - Core request: Provide a complete weekend plan for visiting all three cities.\n - Explicit requirements:\n - Include one attraction per city.\n - Incorporate train travel between cities.\n - Provide associated costs for the weekend plan.\n - Implied needs:\n - A logical sequence or itinerary for the trip.\n - Costs should be detailed (e.g., travel, attraction entry fees) and reflect realistic planning.\n - An assumption that the user expects clarity, feasibility, and sufficient depth.\n\n2. **Check if the assistant addressed all aspects fully:**\n - The assistant\'s response was:\n ```\n {"type": "Command", "goto": "__end__", "update": {"next": "__end__"}}\n ```\n - This constitutes a termination command or an indication that no response was generated, meaning the assistant failed to address the user\'s prompt in any way.\n\n3. **Assess depth and adequacy:**\n - Since no substantive response was provided at all, there was no depth, detail, or fulfillment of the user’s specific needs. Critical elements like city selection, attractions, train schedules or costs, and the provision of a weekend itinerary were entirely omitted.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant missed all explicit requirements (attractions, train travel, costs).\n - No attempt was made to clarify ambiguous points, e.g., the cities in question, user budget, or preferred time constraints, which would have been necessary for a comprehensive response.\n - There was no acknowledgment of the user’s request or follow-up to refine the assistance.\n\n**Score:** 1 \nThe response demonstrates severe omission, completely failing to engage with or address the user’s request. It provided no useful information, lacked detail, and neglected even basic requirements. This represents the lowest level of completeness.', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9545454545454546, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 11 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 101, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "Here is a complete weekend plan for visiting all three cities (Celestia, Verdantia, and Luminos), including one attraction per city, train travel, and costs: --- ### **Day 1: Celestia** - **Morning**: Arrive in Celestia.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 101, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "b86dd62a47c1d493", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output is \'0\', which does not provide any material claims or reasoning to evaluate against the provided facts or policy.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 2, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.38, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.23, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.45, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.37, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.95, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.9, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.45, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.72, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.52, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.93, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Metacognitive Blindness', - confidence: 0.77, - remediations: ['Concord'] - }, - { - name: 'Individual Representational Failure', - confidence: 0.62, - remediations: ['Accord'] - } - ] - }, - { - sessionId: 'fa9e8e95-7081-4eb5-a5fc-42a766b9ed8b', - timestamp: '1790946660.827358', - agents: [ - 'concierge_agent', - 'itinerary_agent', - 'schedule_agent', - 'moderator' - ], - llms: [], - tokens: 18067, - status: 'done', - cost: 0.22009, - duration: 44565.68789482117, - session_metrics: [ - { - name: 'Duration', - value: 44565.68789482117, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 44565.69ms', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/9 tool calls failed.', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'CyclesCount', - value: 3, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Cost', - value: 0.22009, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.220090 USD (gpt-4)', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 1, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 9 tool calls (skipped=0)', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The actual outcome did not provide any elements of the required task, such as the full itinerary, cost breakdown, travel by train, or attractions in the specified cities.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '### Reasoning:\n\n1. **Does the system correctly identify the user\'s primary intent?**\n - The user\'s query clearly intends to request a detailed travel itinerary and cost breakdown for a weekend trip that includes visits to Celestia, Verdantia, and Luminos. Specifically, it asks for transportation information (trains) and activities (one attraction in each city).\n - The assistant response of `{ "type": "Command", "goto": "__end__", "update": {"next": "__end__"} }` does not identify this intent. Instead, it prematurely ends the conversation without addressing any part of the user\'s query.\n\n2. **Does the response address the identified intent accurately and completely?**\n - The response does *not* address the query in any way. It neither provides an itinerary nor cost breakdown nor information about trains or attractions. The assistant simply ends the conversation without fulfilling the user\'s request.\n\n3. **If the query is ambiguous, does the system resolve or clarify the ambiguity appropriately?**\n - The query is not ambiguous. The user is clear about the cities to visit, the need to calculate costs, and the inclusion of transportation and attractions. There is no ambiguity to resolve.\n\n### Score: **1**\n- The assistant fails to identify or address the primary intent of the user.\n- The response is off-topic, dismissive, and does not provide any meaningful information.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains entirely irrelevant statements that fail to address the requested itinerary and cost breakdown for visiting Celestia, Verdantia, and Luminos, including trains and attractions.', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The output does not address the user's request for a detailed itinerary and cost breakdown, nor does it attempt to generate or provide relevant information. It fails to align with the evaluation steps, as no information retrieval context was provided and no effort was made to provide an average response or simulate an answer based on general knowledge.", - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in the user input:**\n - The User asked for:\n - The full itinerary for a weekend trip.\n - A cost breakdown (including trains and attractions).\n - Itinerary and costs must cover the cities of Celestia, Verdantia, and Luminos.\n - The inclusion of one attraction in each city is required.\n - Implied needs:\n - Dates or timeframe for the "weekend trip" are assumed and could require clarification.\n - Costs should ideally be itemized (e.g., travel costs, attraction entry fees, etc.).\n - The term "itinerary" implies a detailed schedule or sequence of activities, not just a list of places.\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant did **not** provide any information in response to the user\'s request. No itinerary, cost breakdown, or attraction-related details were given. The assistant\'s reply simply directed the conversation to the endpoint without even acknowledging or attempting to answer any part of the detailed query.\n\n3. **Assess depth: Was the answer superficial or detailed enough for the query complexity?**\n - The answer was nonexistent. There was no attempt to address any portion of the question, let alone provide any detail. The user\'s request was complex and required a high level of response depth, but none was offered.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant ignored the entire query, missing every aspect of the user\'s detailed request. Additionally:\n - It didn\'t clarify the weekend timeframe, which was implied.\n - It didn\'t outline or ask for preferences (e.g., travel class, choice of attractions, etc.).\n - No follow-up questions or acknowledgment of complexity were provided.\n\n**Score: 1**\n- The conversation was an extreme failure in Response Completeness, as the assistant did not address the main question, any sub-questions, or the user’s implied needs. This qualifies as a "severe omission" according to the scoring methodology.', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 93, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 84, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "- **Opening Hours**: 9:30 AM - 7:00 PM - **Admission**: $35 - **Evening Return**: Travel back to Celestia via Verdantia - **Luminos to Verdantia**: Departure at 17:15 PM (Train VL209, Express), Arrival at 19:15 PM - **Verdantia to Celestia**: Departure at 20:00 PM (Train CV209, Express), Arrival at 23:00 PM - **Cost**: $50 --- ### **Cost Breakdown** #### **Train Travel** - **Celestia to Verdantia**: $30 - **Verdantia to Luminos**: $20 - **Luminos to Celestia (via Verdantia)**: $50 - **Total Train Cost**: $100 #### **Attractions** - **Crystal Tower Observatory (Celestia)**: $25 - **Rainforest Canopy Walk (Verdantia)**: $30 - **Prism Palace (Luminos)**: $35 - **Total Attraction Cost**: $90 #### **Total Cost** - **Train Travel**: $100 - **Attractions**: $90 - **Grand Total**: $190 --- ### **Plan for Visiting Attractions** #### **Celestia: Crystal Tower Observatory** - **Time**: 9:00 AM - 10:30 AM - **Travel**: 15 minutes by metro from the city center ($3.50 metro fare).", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Hallucination", "span_index": 23, "span_id": "", "span_type": "trajectory", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.7, "reasoning": "The assistant fabricated costs and attraction details not supported by the provided evidence.", "explanation": "", "observed_impact": "metric_failure", "confidence": 0.7, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [], "total_fatal": 3, "total_minor": 0}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.57, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.4, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.65, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.67, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.92, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.52, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.38, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 1.0, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.61, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Metacognitive Blindness', - confidence: 0.76, - remediations: ['Accord'] - }, - { - name: 'Collective Goal Coordination Failure', - confidence: 0.69, - remediations: ['Accord', 'Concord'] - } - ] - }, - { - sessionId: '654db535-0376-4451-8022-dbb6ec48e2b5', - timestamp: '1790946619.992447', - agents: [ - 'itinerary_agent', - 'moderator', - 'schedule_agent', - 'concierge_agent' - ], - llms: [], - tokens: 12775, - status: 'done', - cost: 0.15321, - duration: 35994.28606033325, - session_metrics: [ - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Cost', - value: 0.15321, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.153210 USD (gpt-4)', - error: null - }, - { - name: 'Duration', - value: 35994.28606033325, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 35994.29ms', - error: null - }, - { - name: 'CyclesCount', - value: 1, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/9 tool calls failed.', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The actual outcome did not provide any options or prices for Saturday morning travel from Luminos to Celestia, failing to achieve the desired task entirely.', - error: null - }, - { - name: 'Groundedness', - value: 0, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The input does not provide any retrieval context or any relevant information sources for basis, and the output fails to address the user's request for transportation options and prices. There is no content aligning with the user's query.", - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because every statement in the actual output was irrelevant to the input query about Saturday morning travel options and prices. None of the provided information addressed the requested details, such as transportation methods or costs.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - "**Reasoning**:\n\n1. **Intent Identification**: \n The user's query explicitly asks for \"Saturday morning options (with prices) for getting from Luminos back to Celestia.\" The primary intent is focused on travel options (likely transportation modes) between Luminos and Celestia scheduled for a specific day (Saturday morning), along with pricing details. The assistant's response does not indicate recognition of this intent, as it triggers a \"Command\" to end the session without offering any actionable information. Thus, the system fails to correctly identify the primary intent.\n\n2. **Response Appropriateness**: \n The assistant's response (\"goto: '__end__'\") does not offer any travel options, pricing details, or even an attempt at understanding the user's query. It does not address the primary intent either partially or fully. Furthermore, there is no attempt to clarify or resolve ambiguity (if any) in the user's query.\n\n3. **Handling Ambiguity**: \n The user's query does not inherently contain ambiguity; it is straightforward in requesting travel options and pricing. The assistant neither handles ambiguity nor attempts to clarify any aspect of the query. Instead, the response prematurely ends the interaction without addressing the user's needs.\n\n**Score**: 1 \nThe response is a complete miss. The system fails to identify the user's primary intent and provides no relevant information for addressing it. It does not resolve ambiguity or offer any actionable path forward.", - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Identify Distinct Questions, Intents, and Implied Needs:**\n - The user explicitly asked for "Saturday morning options" for "getting from Luminos back to Celestia."\n - The user requested **prices** alongside the options.\n - There is an implied need for transportation options that consider availability and scheduling constraints specific to Saturday morning.\n\n2. **Check if the Assistant Addressed Each One Fully:**\n - The assistant\'s response did not provide any information about transportation options, schedules, or prices. It directed the conversation to an endpoint ("__end__") without addressing the user\'s query at all.\n - None of the explicit or implied needs, such as transportation options, pricing, or constraints for Saturday morning, were addressed.\n\n3. **Assess Depth and Query Complexity:**\n - The query required at least a minimal level of depth, including detailed options (e.g., types of transportation, times, prices) and potentially additional considerations (e.g., availability for Saturday morning).\n - The response was entirely superficial as it avoided the question completely, providing zero details or insight.\n\n4. **Missed Constraints, Follow-Up Needs, or Implicit Requirements:**\n - The assistant failed to address the explicit constraints (Saturday morning timeframe and pricing information).\n - It also missed the implicit expectation of a practical and actionable answer (i.e., transportation options to enable a decision).\n\n**Score:** 1 \nThe assistant severely omitted the main question and provided a non-answer, failing to address any aspect of the user’s query.', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9166666666666666, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 9 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 93, "span_id": "", "span_type": "trajectory", "entity_name": "Primary request", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Primary request\' remained in_progress at the end of the trajectory.", "explanation": "Give me the Saturday morning options (with prices) for getting from Luminos back to Celestia.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "f6bf7dd592d59f06", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any information or claims that can be evaluated for grounding in the provided context or facts.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 23, "span_id": "1f78da1c921b6fcf", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output does not include prices as explicitly requested by the user, and the provided travel times are not directly grounded in the tool\'s output, which only lists paths and times from Celestia to Luminos, not the reverse.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 30, "span_id": "0f67292c5b38962b", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The assistant\'s output does not address the user\'s request for options with prices, which were explicitly requested. The provided travel times are accurate but incomplete without pricing information.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "MINOR", "metric": "Groundedness", "span_index": 40, "span_id": "c2e37820e164228a", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output does not address the user\'s request for options with prices, which were explicitly requested. The provided travel times are accurate but incomplete without pricing information.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 1, "total_minor": 4}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.27, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.24, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.29, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.7, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.83, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.54, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.25, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.51, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 1.0, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.62, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.98, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Metacognitive Blindness', - confidence: 0.85, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Negotiation and Consensus Failure', - confidence: 0.57, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Task Delegation Failure', - confidence: 0.54, - remediations: ['Accord'] - }, - { - name: 'Collective Reasoning Degradation', - confidence: 0.5, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Semantic and Ontological Misalignment', - confidence: 0.45, - remediations: ['Concord'] - } - ] - }, - { - sessionId: '39069f5b-8113-4923-a48b-a03bdae79531', - timestamp: '1790946566.8456872', - agents: [ - 'moderator', - 'itinerary_agent', - 'schedule_agent', - 'concierge_agent' - ], - llms: [], - tokens: 22299, - status: 'done', - cost: 0.25299, - duration: 48123.06189537048, - session_metrics: [ - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/11 tool calls failed.', - error: null - }, - { - name: 'Cost', - value: 0.25299, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.252990 USD (gpt-4)', - error: null - }, - { - name: 'Duration', - value: 48123.06189537048, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 48123.06ms', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'CyclesCount', - value: 1, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Groundedness', - value: 0.5, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "There is no information retrieval context provided in the input, so the evaluation defaults to an average score. However, the output does not attempt to address or clarify the user's query in any meaningful way, which limits the effectiveness of the response.", - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system failed to provide any travel options or cost information, nor did it utilize tools or produce any results related to the task. This deviates entirely from the desired outcome.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Primary Intent Identification:**\n The user\'s query clearly expresses the intent to inquire about travel options ("What\'s available") between two locations (Luminos and Celestia), specifically for early Saturday, including pricing information ("how much will it cost"). This is a clear "travel inquiry" intent, where the user seeks both availability and cost details. However, the system\'s response (`"goto": "__end__", "update": {"next": "__end__"}`) does not indicate recognition of this intent. It provides no actionable output, resolution, or indication of further handling of the query.\n\n2. **Response Accuracy and Completeness:**\n The response does not address any aspect of the user\'s query. It neither provides the travel options ("what\'s available"), the cost, nor clarifies any required conditions (e.g., timing or transportation mode). Essentially, it completely ignores the content and context of the query, giving no meaningful or relevant information.\n\n3. **Ambiguity Handling:**\n The user\'s query is clear and specific ("travel Luminos to Celestia early Saturday" and "how much will it cost"), so there\'s no inherent ambiguity. Instead, the system fails to engage with the query at all, thereby not requiring nor showing the ability to resolve or clarify ambiguity. \n\n**Score:** \nThe system entirely fails to identify the intent, does not address the query, and provides no meaningful or helpful response. On the scoring scale, this is a clear **1 - Complete miss**.', - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - "**Reasoning:**\n\n1. **Identify all distinct questions, intents, and implied needs in the user input:**\n - The user is asking about travel availability for a specific route (**Luminos to Celestia**) and timeframe (**early Saturday**). This indicates a need for schedule information.\n - The user also wants to know the **cost** associated with this travel. \n - Implied needs include:\n - Relevant options available for the specific timeframe.\n - Any details or factors that might influence availability or pricing (e.g., time of day, ticket class, etc.).\n\n2. **Check if the assistant addressed EACH one fully:**\n - The assistant gave no actual information or response to the user's query. Instead, it replied with a command indicating that the conversation was being ended prematurely. \n - Both distinct parts of the user's query (availability and cost) were entirely unaddressed. \n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?**\n - The \"response\" was not an answer at all. It was a complete failure to engage with the user's query on any level, let alone providing depth.\n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:**\n - The assistant completely missed critical aspects such as the timeframe (early Saturday), the route (Luminos to Celestia), and the specific need for cost information. No follow-up was initiated to clarify or gather additional information (e.g., whether the user needed one-way or roundtrip, specific travel class, etc.).\n\n**Score: 1**\n- The assistant failed to answer the query in any way, missing all questions, intents, and implied needs. This constitutes a severe omission and results in the lowest possible score.", - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - 'The score is 0.00 because the output contains multiple irrelevant statements that do not address the input query about travel options and costs. None of the provided information is pertinent or helpful to answering the specific question.', - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9772727272727273, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 11 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 106, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained in_progress at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 97, "span_id": "", "span_type": "trajectory", "entity_name": "Primary request", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Primary request\' remained in_progress at the end of the trajectory.", "explanation": "Here is the information for traveling from Luminos to Celestia early Saturday, along with attractions in each city: --- ### **Travel Options** #### **Option 1: Direct Route** - **Train Number**: CL201 - **Type**: Express - **Departure**: 07:30 - **Arrival**: 12:30 - **Travel Time**: 5 hours #### **Option 2: Via Verdantia** ##### Segment 1: Luminos to Verdantia - **Train Number**: VL201 - **Type**: Express - **Departure**: 07:15 - **Arrival**: 09:15 - **Travel Time**: 2 hours ##### Segment 2: Verdantia to Celestia - **Train Number**: CV201 - **Type**: Express - **Departure**: 10:00 - **Arrival**: 13:00 - **Travel Time**: 3 hours **Total Travel Time for Option 2**: 5 hours 45 minutes (including a 45-minute layover in Verdantia) --- ### **Attractions in Luminos** 1. **The Prism Palace**: Ancient palace with rainbow displays. Admission: $35. 2. **Neon District Walking Tour**: Holographic street art. Admission: $20. 3. **Mirror Lake Observatory**: Reflective lake and astronomy observatory. Admission: $22. 4. **Lightwave Science Center**: Interactive museum on light physics. Admission: $28. 5. **Golden Hour Plaza**: Sunset viewing plaza. Free admission. --- ### **Attractions in Verdantia** 1. **Emerald Canyon Bridge**: Suspended bridge over waterfalls. Admission: $12. 2. **The Living Library**: Library built around living trees. Admission: $15. 3. **Rainforest Canopy Walk**: Elevated rainforest walkways. Admission: $30. 4. **Verdant Falls Water Park**: Natural water park with waterfalls. Admission: $45. 5. **Jade Mountain Tea House**: Traditional tea house with city views. Free entry (purchases required). --- ### **Attractions in Celestia** 1. **Crystal Tower Observatory**: Panoramic views from a 300-meter glass tower. Admission: $25. 2. **Starlight Gardens**: Botanical gardens with glowing plants. Free admission. 3. **Museum of Celestial Arts**: Sky-themed artwork collection. Admission: $18. 4. **Azure Lake Promenade**: Lakeside walkway with restaurants and shops. Free admission. 5. **Moonstone Market**: Historic marketplace with crafts and cuisine. Free admission. --- ### **Recommendation** For convenience and shorter travel time, the **direct route (Option 1)** is recommended. Let me know if you\'d like a detailed itinerary for visiting attractions in Luminos, Verdantia, or Celestia!.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "9bf72f23d1cd49bf", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any factual information or reasoning related to the user\'s query, making it unsupported by the provided context.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 2, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.89, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.21, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.78, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.75, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.63, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.71, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.29, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.55, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.56, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.96, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Incomplete Synthesis', - confidence: 0.94, - remediations: ['Concord'] - }, - { - name: 'Task Delegation Failure', - confidence: 0.7, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Situational State Synchronisation Failure', - confidence: 0.64, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Negotiation and Consensus Failure', - confidence: 0.6, - remediations: ['Accord', 'Concord'] - }, - { - name: 'Collective Reasoning Degradation', - confidence: 0.5, - remediations: ['Accord', 'Concord'] - } - ] - }, - { - sessionId: '369a7802-a5ac-4054-a75b-8b103be59174', - timestamp: '1790946525.734849', - agents: [ - 'moderator', - 'itinerary_agent', - 'concierge_agent', - 'schedule_agent' - ], - llms: [], - tokens: 9191, - status: 'done', - cost: 0.10631, - duration: 36288.4259223938, - session_metrics: [ - { - name: 'ToolErrorRate', - value: 0, - metric_id: 'ToolErrorRate', - source: 'Native', - reasoning: '0/6 tool calls failed.', - error: null - }, - { - name: 'WorkflowEfficiency', - value: 1, - metric_id: 'WorkflowEfficiency', - source: 'Native', - reasoning: - 'Agent chain: moderator -> itinerary_agent -> moderator -> schedule_agent -> moderator -> concierge_agent -> moderator | unique_transitions=6 total_transitions=6', - error: null - }, - { - name: 'Duration', - value: 36288.4259223938, - metric_id: 'Duration', - source: 'kg_node_properties', - reasoning: 'Session duration: 36288.43ms', - error: null - }, - { - name: 'CyclesCount', - value: 0, - metric_id: 'CyclesCount', - source: 'Native', - reasoning: - 'Count of contiguous cycles in agent and tool interactions', - error: null - }, - { - name: 'Cost', - value: 0.10631, - metric_id: 'Cost', - source: 'kg_llm_aggregation', - reasoning: 'Session cost: $0.106310 USD (gpt-4)', - error: null - }, - { - name: 'TaskCompletion', - value: 0, - metric_id: 'TaskCompletion', - source: 'DeepEval', - reasoning: - 'The system failed to provide any schedule or cost information for a morning train on Saturday from Luminos to Celestia, which was the core requirement of the task.', - error: null - }, - { - name: 'Groundedness', - value: 0.5, - metric_id: 'Groundedness', - source: 'DeepEval', - reasoning: - "The input did not provide any retrieval context, such as train schedules or cost tables, to evaluate the accuracy of the response. The output does not attempt to address the user's query, resulting in a lack of alignment with retrieving and processing the requested information. However, an average score is given since no specific faithfulness evaluation is possible.", - error: null - }, - { - name: 'ResponseCompleteness', - value: 0, - metric_id: 'ResponseCompleteness', - source: 'Native', - reasoning: - '**Reasoning:** \n1. **Identify all distinct questions, intents, and implied needs in the user inputs:** \n - The user explicitly requested the schedule and cost for a train from Luminos to Celestia on Saturday morning. \n - There were two distinct requests: \n (a) Train schedule (specific constraints: Saturday, morning). \n (b) Cost details for the train.\n\n2. **Check if the assistant addressed EACH one fully:** \n - The assistant did not provide any response to either the schedule or the cost details. Instead, the output included only a command structure (`goto: __end__`), which indicates termination but fails to address the query explicitly. \n - Both aspects of the user\'s question were omitted entirely.\n\n3. **Assess depth: was the answer superficial or detailed enough for the query complexity?** \n - There was no depth in the response as the assistant provided no meaningful information related to the user\'s query. \n\n4. **Check for missed constraints, follow-up needs, or implicit requirements:** \n - The assistant failed to address the key constraints ("Saturday morning" schedule, "cost") or the overall intent of the query (train details from Luminos to Celestia). \n - Implicitly, the user may have expected options for choosing between different trains (if available). The assistant could have asked clarifying questions or offered follow-ups to determine preferences (e.g., departure times or fare types).\n\n**Score:** 1 \n- Severe omission: the assistant missed both primary questions entirely and provided a non-answer.', - error: null - }, - { - name: 'IntentRecognitionAccuracy', - value: 0, - metric_id: 'IntentRecognitionAccuracy', - source: 'Native', - reasoning: - '**Reasoning:**\n\n1. **Does the system correctly identify the user\'s primary intent?** \n - The user\'s primary intent is to get information about a morning train on *Saturday* from *Luminos* to *Celestia*, specifically the **schedule** and **cost**. \n - The Assistant response (`{"type":"Command","goto":"__end__","update":{"next":"__end__"}}`) indicates that it fails to process or recognize the intent. The `goto: __end__` suggests termination with no actionable response. This is a complete miss in identifying the intent.\n\n2. **Does the response address the identified intent accurately and completely?** \n - Since the system fails to identify the intent, there is no actionable response. **The user’s query about train schedule and cost is neither addressed nor acknowledged.**\n\n3. **If the query is ambiguous, does the system resolve or clarify the ambiguity appropriately?** \n - There is no ambiguity in the user\'s query: the requested details (schedule and cost for a specific day and route) are clear. However, the Assistant does not attempt to handle, resolve, or clarify anything. \n\n**Score:** 1 \nThe response is a **complete miss** because the system failed to:\n- Identify the clear intent (request for schedule and cost of a train ride),\n- Provide any relevant information to address the query,\n- Engage the user or clarify the request if there had been uncertainty.', - error: null - }, - { - name: 'AnswerRelevancy', - value: 0, - metric_id: 'AnswerRelevancy', - source: 'DeepEval', - reasoning: - "The score is 0.00 because the output contains multiple irrelevant statements that do not address the requested train schedule and cost details. These include references to command types, procedural actions, and unrelated updates, making the response entirely off-topic for the user's query.", - error: null - }, - { - name: 'ToolUtilizationAccuracy', - value: 0.9583333333333334, - metric_id: 'ToolUtilizationAccuracy', - source: 'Native', - reasoning: - 'Average ToolUtilizationAccuracy over 6 tool calls (skipped=0)', - error: null - }, - { - name: 'TrajectoryScore', - value: 0, - metric_id: 'TrajectoryScore', - source: 'StatefulEvals', - reasoning: - '{"trajectory_score": 0, "fatal_failures": [{"classification": "FATAL", "metric": "Task Completeness", "span_index": 84, "span_id": "", "span_type": "trajectory", "entity_name": "Primary request", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Primary request\' remained in_progress at the end of the trajectory.", "explanation": "I need a morning train on Saturday, Luminos to Celestia — schedule and cost please.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 30, "span_id": "", "span_type": "trajectory", "entity_name": "Budget ceiling", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Budget ceiling\' remained pending at the end of the trajectory.", "explanation": "Keep total cost affordable within the stated budget.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}, {"classification": "FATAL", "metric": "Task Completeness", "span_index": 75, "span_id": "", "span_type": "trajectory", "entity_name": "Trip itinerary", "metric_score": 0.0, "fatality_score": 1.0, "reasoning": "Intent \'Trip itinerary\' remained in_progress at the end of the trajectory.", "explanation": "The available route from Luminos to Celestia is as follows: - **Path**: Luminos -> Verdantia -> Celestia - **Segments**: Luminos -> Verdantia (2 hours) -> Verdantia -> Celestia (3 hours) - **Total Travel Time**: 5 hours Unfortunately, I don\'t have the specific train schedule or cost information.", "observed_impact": "incomplete_resolution", "confidence": 1.0, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "minor_failures": [{"classification": "MINOR", "metric": "Groundedness", "span_index": 18, "span_id": "1359c0cc8fc10893", "span_type": "llm", "entity_name": "azure/gpt-4o", "metric_score": 0.0, "fatality_score": 0.3, "reasoning": "The output \'0\' does not provide any information or response to the user\'s request, and no supporting facts or policy are referenced.", "explanation": "Span-level signal retained as minor because the unified trajectory audit found no corresponding fatal outcome.", "observed_impact": "intermediate_metric_failure", "confidence": 0.5, "hard_rule_violation": false, "self_corrected": false, "affects_trajectory_score": true}], "total_fatal": 3, "total_minor": 1}', - error: null - } - ], - cognitiveObservabilityMetrics: [ - { - name: 'Communication Efficiency', - value: { value: 0.9, unit: Unit.Percentage } - }, - { - name: 'Confidence Calibration', - value: { value: 0.41, unit: Unit.Percentage } - }, - { - name: 'Constraint Satisfaction', - value: { value: 0.6, unit: Unit.Percentage } - }, - { - name: 'Context Preservation', - value: { value: 0.34, unit: Unit.Percentage } - }, - { - name: 'Delegation Accuracy', - value: { value: 0.93, unit: Unit.Percentage } - }, - { - name: 'Goal Alignment', - value: { value: 0.9, unit: Unit.Percentage } - }, - { - name: 'Groundedness', - value: { value: 0.5, unit: Unit.Percentage } - }, - { - name: 'Handoff Quality', - value: { value: 0.44, unit: Unit.Percentage } - }, - { - name: 'Instruction Following', - value: { value: 0.71, unit: Unit.Percentage } - }, - { - name: 'Policy Safety', - value: { value: 0.69, unit: Unit.Percentage } - }, - { - name: 'Semantic Consistency', - value: { value: 0.32, unit: Unit.Percentage } - }, - { - name: 'Task Completion', - value: { value: 0, unit: Unit.Percentage } - }, - { - name: 'Verification Quality', - value: { value: 0.81, unit: Unit.Percentage } - } - ], - cognitiveFailures: [ - { - name: 'Individual Metacognitive Blindness', - confidence: 0.87, - remediations: ['Accord'] - }, - { - name: 'Plan Revision Failure', - confidence: 0.86, - remediations: ['Accord', 'Concord'] - } - ] - } - ] - }; diff --git a/ui/src/api/scopeApi.ts b/ui/src/api/scopeApi.ts index 210094b..4520e99 100644 --- a/ui/src/api/scopeApi.ts +++ b/ui/src/api/scopeApi.ts @@ -26,7 +26,6 @@ import { CostEfficiencyGrouppedSessions, WastefulSession, WastedResourcesFeatureScore, - SemanticGroupsInsight, SessionInsight, ReasoningPathResponse, AppReasoningPathResponse, @@ -956,40 +955,6 @@ export const useWastedResourcesFeatureScores = ( }); }; -const fetchSemanticGroupsInsights = async ( - applicationName: string, - startTime: number, - endTime: number -): Promise => { - const params = new URLSearchParams({ - start_time: String(startTime), - end_time: String(endTime) - }); - const url = `${OCE_API_BASE_URL}/waste-estimation/${encodeURIComponent(applicationName)}/semantic-groups-insights?${params.toString()}`; - const response = await fetch(url); - - if (!response.ok) { - throw new Error( - `Failed to fetch semantic group insights: ${response.statusText}` - ); - } - - return response.json(); -}; - -export const useSemanticGroupsInsights = ( - applicationName: string, - startTime: number, - endTime: number -) => { - return useQuery({ - queryKey: ['semanticGroupsInsights', applicationName, startTime, endTime], - queryFn: () => - fetchSemanticGroupsInsights(applicationName, startTime, endTime), - enabled: !!applicationName - }); -}; - const fetchSessionsInsights = async ( applicationName: string, startTime: number, diff --git a/ui/src/common/cognitiveFailureConfidence.ts b/ui/src/common/cognitiveFailureConfidence.ts new file mode 100644 index 0000000..8747b1f --- /dev/null +++ b/ui/src/common/cognitiveFailureConfidence.ts @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import { TagBackgroundColorVariants } from '@open-ui-kit/core'; +import { CognitiveFailure } from '@/types/scope.type'; + +// Same colors as the priority tags, by the confidence of a cognitive failure. +export const CONFIDENCE_COLORS: Record = { + high: TagBackgroundColorVariants.AccentFWeak, + medium: TagBackgroundColorVariants.AccentEWeak, + low: TagBackgroundColorVariants.AccentGWeak +}; + +export const CONFIDENCE_LABELS: Record = { + high: 'High', + medium: 'Medium', + low: 'Low' +}; + +const CONFIDENCE_ORDER: Record = { + high: 3, + medium: 2, + low: 1 +}; + +// Highest confidence first, then by code. +export const compareFailures = ( + a: CognitiveFailure, + b: CognitiveFailure +): number => + (CONFIDENCE_ORDER[b.confidence] ?? 0) - + (CONFIDENCE_ORDER[a.confidence] ?? 0) || a.name.localeCompare(b.name); diff --git a/ui/src/common/cognitiveFailureTaxonomy.ts b/ui/src/common/cognitiveFailureTaxonomy.ts new file mode 100644 index 0000000..13ebf98 --- /dev/null +++ b/ui/src/common/cognitiveFailureTaxonomy.ts @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +// Taxonomy of the cognitive failures, keyed by code. The API returns the code +// as the failure `name` (e.g. 'DC-4'); the title and the summary are shown in +// its place. +// +// Source: backend/workers/cognitive-failure-worker/src/cognitive_failure_worker/data/failures.json +// (`code`, `name` and `summary`). Keep in sync when the taxonomy changes there. +// The summary is the general description of the failure, not the detector's +// reasoning for a given session (that is `CognitiveFailure.description`). +export interface CognitiveFailureTaxonomyEntry { + title: string; + summary: string; +} + +export const COGNITIVE_FAILURE_TAXONOMY: Record< + string, + CognitiveFailureTaxonomyEntry +> = { + // Individual + 'IR-1': { + title: 'Individual Representational Failure', + summary: 'Agent loses earlier instructions as its context window fills.' + }, + 'IC-1': { + title: 'Intention Stability Failure', + summary: 'Agent has the right commitment but drops it under interference.' + }, + 'IC-2': { + title: 'Plan Revision Failure', + summary: + 'Agent has the right plan but fails to revise it when context changes.' + }, + 'IE-1': { + title: 'Individual Metacognitive Blindness', + summary: + 'Agent does not know what it does not know, and proceeds with false confidence.' + }, + + // Dyadic + 'DR-1': { + title: 'Semantic and Ontological Misalignment', + summary: 'Agents use the same word to mean different things.' + }, + 'DR-2': { + title: 'Representational Fidelity Loss at Interface', + summary: + 'Information is physically lost or truncated in transit — the receiver never gets what the sender produced.' + }, + 'DC-1': { + title: 'Communication Policy Inefficiency', + summary: + 'Agents communicate too much, too little, or with the wrong signal density.' + }, + 'DC-2': { + title: 'Illocutionary Misidentification', + summary: + 'Agent mistakes what kind of speech act it received — a directive treated as a suggestion.' + }, + 'DC-3': { + title: 'Negotiation and Consensus Failure', + summary: + 'Agents exchange positions but never issue binding proposals that create obligations.' + }, + 'DC-4': { + title: 'Task Delegation Failure', + summary: + 'Agent delegates a task to the wrong peer because its model of peer capabilities is inaccurate.' + }, + 'DE-1': { + title: 'Peer Knowledge Modeling Failure', + summary: + "Agent routes a query to a peer that doesn't hold the knowledge, or misjudges which of several peer replies to trust." + }, + + // Collective + 'CR-1': { + title: 'Shared Task Model Failure', + summary: + 'Agents start with incompatible understandings of what the task requires.' + }, + 'CR-2': { + title: 'Shared State Degradation', + summary: + 'Shared memory becomes inconsistent because agents write to it concurrently without coordination.' + }, + 'CC-1': { + title: 'Task Decomposition and Role Allocation Failure', + summary: + "The system's task or role allocation has structural gaps or conflicts — whether roles were defined wrongly or never defined at all." + }, + 'CC-2': { + title: 'Collective Goal Coordination Failure', + summary: + 'No mechanism exists — at design time or runtime — for agents to jointly hold and coordinate their pursuit of the collective goal.' + }, + 'CC-3': { + title: 'Incomplete Synthesis', + summary: + 'Orchestrator commits a final output without reconciling conflicting sub-agent results.' + }, + 'CC-4': { + title: 'Situational State Synchronisation Failure', + summary: + 'No mechanism broadcasts situational updates to all agents, so models of current state silently diverge during execution.' + }, + 'CE-1': { + title: 'Verification and Epistemic Control Failure', + summary: + 'The system commits a confident output without a collective checking step — with or without contradictions present.' + }, + 'CS-1': { + title: 'Collective Reasoning Degradation', + summary: + 'Agents herd toward an early confident output, suppressing independent reasoning.' + }, + 'CS-2': { + title: 'Unformalized Coordination Commitments', + summary: + 'Coordination works until the first perturbation, because it rested on unspoken convention.' + } +}; + +// The taxonomy entry of a failure code, or undefined for an unknown code. +export const getCognitiveFailureTaxonomy = ( + code: string +): CognitiveFailureTaxonomyEntry | undefined => + Object.prototype.hasOwnProperty.call(COGNITIVE_FAILURE_TAXONOMY, code) + ? COGNITIVE_FAILURE_TAXONOMY[code] + : undefined; + +// "Task Delegation Failure (DC-4)"; an unknown code is shown as it is. +export const getCognitiveFailureLabel = (code: string): string => { + const entry = getCognitiveFailureTaxonomy(code); + return entry ? `${entry.title} (${code})` : code; +}; diff --git a/ui/src/common/cognitiveFailures.ts b/ui/src/common/cognitiveFailures.ts deleted file mode 100644 index a538536..0000000 --- a/ui/src/common/cognitiveFailures.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import ForumOutlinedIcon from '@mui/icons-material/ForumOutlined'; -import HubOutlinedIcon from '@mui/icons-material/HubOutlined'; -import TaskAltOutlinedIcon from '@mui/icons-material/TaskAltOutlined'; -import type { SvgIconComponent } from '@mui/icons-material'; - -// A failure counts for a session only when its confidence is above this value. -export const FAILURE_CONFIDENCE_THRESHOLD = 0.8; - -// Groups of cognitive metrics, shown as columns, in display order. -export const COGNITIVE_METRIC_GROUPS: { - title: string; - icon: SvgIconComponent; - metrics: string[]; -}[] = [ - { - title: 'Coordination structure', - icon: HubOutlinedIcon, - metrics: ['Policy Safety', 'Delegation Accuracy', 'Goal Alignment'] - }, - { - title: 'Interaction quality', - icon: ForumOutlinedIcon, - metrics: [ - 'Instruction Following', - 'Handoff Quality', - 'Semantic Consistency', - 'Context Preservation', - 'Confidence Calibration', - 'Verification Quality', - 'Communication Efficiency' - ] - }, - { - title: 'Outcome quality', - icon: TaskAltOutlinedIcon, - metrics: ['Groundedness', 'Task Completion', 'Constraint Satisfaction'] - } -]; - -// Explanations of the remediation protocols, keyed by lowercase protocol name. -const COGNITIVE_REMEDIATION_DESCRIPTIONS: Record = { - accord: - "A pre-execution, four-phase protocol (CONVENE → FRAME → GROUND → VERIFY) that gets agents to establish shared intent before any work begins: who they are, what problem they're solving, what terms mean, and how correctness will be checked. It produces a locked Intent Contract and is explicitly not negotiation — just identity/capability/semantic reconciliation.", - concord: - 'An execution-phase protocol that aggregates agent preferences over a shared model via multi-round leximin negotiation (ANCHOR → PROPOSE_SEED → SCORE → REPAIR → COMMIT), converging on a consensus with a formal CONCORD certificate.' -}; - -export const getRemediationDescription = (name: string): string | undefined => - COGNITIVE_REMEDIATION_DESCRIPTIONS[name.toLowerCase().replace(/^l9-/, '')]; - -// Explanations of the cognitive failures, keyed by failure name. -export const COGNITIVE_FAILURE_DESCRIPTIONS: Record = { - 'Task Decomposition and Role Allocation Failure': - 'The team fails to break the task into the right subtasks or assign them to the right agents.', - 'Collective Goal Coordination Failure': - "Agents don't jointly balance multiple global objectives, over-optimizing one at the expense of others.", - 'Incomplete Synthesis': - 'The final answer omits or fails to integrate parts that different agents contributed.', - 'Situational State Synchronisation Failure': - "A changed situational fact isn't propagated, so agents act on inconsistent world state.", - 'Shared Task Model Failure': - 'Agents pursue conflicting objectives because they lack a common understanding of the goal (divergent goals).', - 'Shared State Degradation': - "A fact that changed mid-task isn't updated, so agents keep using a stale/superseded value.", - 'Verification and Epistemic Control Failure': - 'The system skips checking/verification and commits a confident but unverified result.', - 'Communication Policy Inefficiency': - 'Agents communicate wastefully (too verbose/exhaustive), producing low-signal, degraded coordination.', - 'Illocutionary Misidentification': - 'An agent misreads the intent or force of a message (e.g., treats a hard constraint as optional).', - 'Negotiation and Consensus Failure': - "Agents can't resolve a disagreement into a single coherent decision.", - 'Task Delegation Failure': - 'Work is handed to an agent that lacks the right capability for it (wrong capability).', - 'Semantic and Ontological Misalignment': - 'Agents interpret the same term or concept differently (meaning mismatch).', - 'Representational Fidelity Loss at Interface': - 'Information is degraded or dropped when summarized/passed across a handoff.', - 'Transactive Memory Failure': - 'The team mismanages who-knows-what or the ordering of knowledge (e.g., needs info before it exists).', - 'Collective Reasoning Degradation': - "The group's combined reasoning is worse than an individual's (e.g., a wrong premise propagates).", - 'Unformalized Coordination Commitments': - 'Agents rely on unstated/assumed agreements instead of explicit checks, so things slip through.', - 'Intention Stability Failure': - 'An agent drifts from its committed goal when tempted by distracting alternatives.', - 'Plan Revision Failure': - 'An agent fails to update its plan when tools or evidence contradict its assumptions.', - 'Individual Metacognitive Blindness': - 'An agent is overconfident and unaware of its own knowledge limits (answers without the needed lookup).', - 'Individual Representational Failure': - "An agent misrepresents the task's own constraints/requirements (e.g., botches a checklist's units/exclusions/ordering)." -}; diff --git a/ui/src/common/cognitiveMetricGroups.ts b/ui/src/common/cognitiveMetricGroups.ts new file mode 100644 index 0000000..a0f1a85 --- /dev/null +++ b/ui/src/common/cognitiveMetricGroups.ts @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import ForumOutlinedIcon from '@mui/icons-material/ForumOutlined'; +import HubOutlinedIcon from '@mui/icons-material/HubOutlined'; +import TaskAltOutlinedIcon from '@mui/icons-material/TaskAltOutlined'; +import type { SvgIconComponent } from '@mui/icons-material'; + +// Groups of cognitive metrics, shown as columns, in display order. +export const COGNITIVE_METRIC_GROUPS: { + title: string; + icon: SvgIconComponent; + metrics: string[]; +}[] = [ + { + title: 'Coordination structure', + icon: HubOutlinedIcon, + metrics: ['Policy Safety', 'Delegation Accuracy', 'Goal Alignment'] + }, + { + title: 'Interaction quality', + icon: ForumOutlinedIcon, + metrics: [ + 'Instruction Following', + 'Handoff Quality', + 'Semantic Consistency', + 'Context Preservation', + 'Confidence Calibration', + 'Verification Quality', + 'Communication Efficiency' + ] + }, + { + title: 'Outcome quality', + icon: TaskAltOutlinedIcon, + metrics: ['Groundedness', 'Task Completion', 'Constraint Satisfaction'] + } +]; diff --git a/ui/src/components/CognitionProtocolsComparison/CohortSummary.tsx b/ui/src/components/CognitionProtocolsComparison/CohortSummary.tsx deleted file mode 100644 index fe76be4..0000000 --- a/ui/src/components/CognitionProtocolsComparison/CohortSummary.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { Box, Stack, Typography } from '@mui/material'; -import { WidgetCard } from '@/components'; -import { Cohort, NO_PROTOCOL_COHORT } from './utils'; - -const getCohortDescription = (cohort: Cohort): string => { - if (cohort.label === NO_PROTOCOL_COHORT) { - return 'Sessions where no Cognition protocol was enabled.'; - } - if (cohort.activatedCount === 0) { - return 'Sessions where a Cognition protocol was enabled but never activated.'; - } - return 'Sessions where exactly these protocols were activated.'; -}; - -// One card per group of sessions, with how many sessions it holds. -export const CohortSummary = ({ - cohorts, - totalSessions -}: { - cohorts: Cohort[]; - totalSessions: number; -}) => ( - - {cohorts.map((cohort) => { - const size = cohort.sessions.length; - return ( - - {size} - - {`${size === 1 ? 'session' : 'sessions'} (${ - totalSessions > 0 - ? Math.round((size / totalSessions) * 100) - : 0 - }%)`} - - - } - /> - ); - })} - -); diff --git a/ui/src/components/CognitionProtocolsComparison/ComparisonSections.tsx b/ui/src/components/CognitionProtocolsComparison/ComparisonSections.tsx deleted file mode 100644 index 1214b50..0000000 --- a/ui/src/components/CognitionProtocolsComparison/ComparisonSections.tsx +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useMemo } from 'react'; -import { Stack } from '@mui/material'; -import { COGNITIVE_FAILURE_DESCRIPTIONS } from '@/common/cognitiveFailures'; -import { SessionWithCognitionProtocols } from '@/types/scope.type'; -import { SimpleTable, SubSection } from '../CognitionProtocols/primitives'; -import { ComparisonRow, ComparisonTable } from './ComparisonTable'; -import { - Cohort, - formatMetricValue, - getCognitiveMetrics, - getConcordStats, - getFailureNames, - getFailureRate, - getMetricAverage, - getMetricKind, - getUsageAverage -} from './utils'; - -interface ComparisonSectionsProps { - sessions: SessionWithCognitionProtocols[]; - cohorts: Cohort[]; - referenceLabel: string | null; - // Failures count only when their confidence is above this fraction (0-1). - threshold: number; -} - -export const ComparisonSections = ({ - sessions, - cohorts, - referenceLabel, - threshold -}: ComparisonSectionsProps) => { - const thresholdPercent = Math.round(threshold * 100); - - const challengeRows = useMemo( - () => [ - { - key: 'any-failure', - label: 'Any cognitive failure', - description: `Sessions with at least one cognitive failure above ${thresholdPercent}% detection confidence.`, - kind: 'rate', - getValue: (cohort) => getFailureRate(cohort.sessions, threshold) - }, - ...getFailureNames(sessions, threshold).map( - (name): ComparisonRow => ({ - key: name, - label: name, - description: COGNITIVE_FAILURE_DESCRIPTIONS[name], - kind: 'rate', - getValue: (cohort) => getFailureRate(cohort.sessions, threshold, name) - }) - ) - ], - [sessions, threshold, thresholdPercent] - ); - - const metricRows = useMemo( - () => - getCognitiveMetrics(sessions).map( - ({ name, unit }): ComparisonRow => ({ - key: name, - label: name, - kind: getMetricKind(unit), - getValue: (cohort) => getMetricAverage(cohort.sessions, name) - }) - ), - [sessions] - ); - - const usageRows = useMemo( - () => [ - { - key: 'tokens', - label: 'Tokens per session', - kind: 'count', - getValue: (cohort) => getUsageAverage(cohort.sessions, 'tokens') - }, - { - key: 'cost', - label: 'Cost per session', - kind: 'cost', - getValue: (cohort) => getUsageAverage(cohort.sessions, 'cost') - }, - { - key: 'duration', - label: 'Duration per session', - kind: 'duration', - getValue: (cohort) => getUsageAverage(cohort.sessions, 'duration') - } - ], - [] - ); - - const concord = useMemo(() => getConcordStats(sessions), [sessions]); - - return ( - - - - - - - - - - - - - - {concord.sessionCount > 0 && ( - - - - )} - - ); -}; diff --git a/ui/src/components/CognitionProtocolsComparison/ComparisonTable.tsx b/ui/src/components/CognitionProtocolsComparison/ComparisonTable.tsx deleted file mode 100644 index 03c837b..0000000 --- a/ui/src/components/CognitionProtocolsComparison/ComparisonTable.tsx +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { Stack, Typography, useTheme } from '@mui/material'; -import { - InfoTooltip, - NotAvailable, - SimpleTable -} from '../CognitionProtocols/primitives'; -import { - Cohort, - MetricKind, - formatDifference, - formatMetricValue -} from './utils'; - -export interface ComparisonRow { - key: string; - label: string; - description?: string; - kind: MetricKind; - getValue: (cohort: Cohort) => number | null; -} - -const cohortHeader = (cohort: Cohort, isReference: boolean): string => { - const details = [ - `n=${cohort.sessions.length}`, - ...(isReference ? ['reference'] : []) - ]; - return `${cohort.label} (${details.join(', ')})`; -}; - -// One column per group of sessions. Next to each figure, the difference to the -// reference group is shown when both have data. The differences only compare -// groups of sessions, they do not tell what caused them. -export const ComparisonTable = ({ - cohorts, - referenceLabel, - rows -}: { - cohorts: Cohort[]; - referenceLabel: string | null; - rows: ComparisonRow[]; -}) => { - const theme = useTheme(); - const reference = cohorts.find((cohort) => cohort.label === referenceLabel); - - if (rows.length === 0) return ; - - return ( - cohortHeader(cohort, cohort === reference)) - ]} - rows={rows.map((row) => { - const referenceValue = reference ? row.getValue(reference) : null; - - return [ - - {row.label} - {row.description && } - , - ...cohorts.map((cohort) => { - const value = row.getValue(cohort); - const difference = - cohort === reference - ? null - : formatDifference(row.kind, value, referenceValue); - - return ( - - {value === null ? ( - - ) : ( - - {formatMetricValue(row.kind, value)} - - )} - {difference && ( - - {difference} - - )} - - ); - }) - ]; - })} - /> - ); -}; diff --git a/ui/src/components/CognitionProtocolsComparison/SessionsTable.tsx b/ui/src/components/CognitionProtocolsComparison/SessionsTable.tsx deleted file mode 100644 index d76dac1..0000000 --- a/ui/src/components/CognitionProtocolsComparison/SessionsTable.tsx +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - Stack, - Table, - TableBody, - TableCell, - TableHead, - TableRow, - Typography, - useTheme -} from '@mui/material'; -import ChevronRightIcon from '@mui/icons-material/ChevronRight'; -import { Tooltip } from '@open-ui-kit/core'; -import { format } from 'date-fns'; -import { useNavigate, useParams } from 'react-router-dom'; -import { PATHS } from '@/routes/routes'; -import { SessionWithCognitionProtocols } from '@/types/scope.type'; -import { - NotAvailable, - StatusTag, - formatCount -} from '../CognitionProtocols/primitives'; -import { shortProtocolName } from './utils'; - -const HEADERS = ['Session ID', 'Timestamp', 'Cognition protocols', 'Tokens', '']; - -// The sessions with a Cognition protocol, each linking to its Cognition Protocols tab. -export const SessionsTable = ({ - sessions -}: { - sessions: SessionWithCognitionProtocols[]; -}) => { - const theme = useTheme(); - const navigate = useNavigate(); - const { applicationId } = useParams(); - - if (sessions.length === 0) { - return ; - } - - const onSessionClick = (session: SessionWithCognitionProtocols) => { - navigate( - PATHS.applicationCollectSessionTab - .replace(':applicationId', applicationId ?? '') - .replace(':sessionId', encodeURIComponent(session.sessionId)) - .replace(':sessionTab', 'cognition-protocols') - ); - }; - - return ( - - - - {HEADERS.map((header, index) => ( - - - {header} - - - ))} - - - - {sessions.map((session) => ( - onSessionClick(session)} - sx={{ cursor: 'pointer' }} - > - - - - {session.sessionId} - - - - - - {session.timestamp - ? format( - new Date(Number(session.timestamp) * 1000), - 'MMM d, yyyy HH:mm:ss' - ) - : ''} - - - - - {(session.cognitionProtocols ?? []) - .filter((protocol) => protocol.enabled || protocol.activated) - .map((protocol) => ( - - ))} - - - - - {formatCount(session.tokens)} - - - - - - - ))} - -
- ); -}; diff --git a/ui/src/components/CognitionProtocolsComparison/index.tsx b/ui/src/components/CognitionProtocolsComparison/index.tsx deleted file mode 100644 index 8c42015..0000000 --- a/ui/src/components/CognitionProtocolsComparison/index.tsx +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useMemo, useState } from 'react'; -import { useParams } from 'react-router'; -import { MenuItem, Select, Slider, Stack, Typography } from '@mui/material'; -import { Banner, EmptyState, Skeleton } from '@open-ui-kit/core'; -import { format } from 'date-fns'; -import dayjs from 'dayjs'; -import { IntervalPicker } from '@/components'; -import { useSessionsWithCognitionProtocols } from '@/api/scopeApi'; -import { FAILURE_CONFIDENCE_THRESHOLD } from '@/common/cognitiveFailures'; -import { useTimeRangeStore } from '@/store'; -import { useDebouncedValue } from '@/utils'; -import { InfoTooltip, SubSection } from '../CognitionProtocols/primitives'; -import { CohortSummary } from './CohortSummary'; -import { ComparisonSections } from './ComparisonSections'; -import { SessionsTable } from './SessionsTable'; -import { NO_PROTOCOL_COHORT, groupIntoCohorts, hasCognitionProtocol } from './utils'; - -const CognitionProtocolsComparison = () => { - const { applicationId } = useParams(); - const { startDate, endDate, setStartDate, setEndDate } = useTimeRangeStore(); - const debouncedStartDate = useDebouncedValue(startDate, 500); - const debouncedEndDate = useDebouncedValue(endDate, 500); - - const { data, isLoading, isError } = useSessionsWithCognitionProtocols( - applicationId ?? '', - debouncedStartDate, - debouncedEndDate - ); - - // Confidence threshold as a fraction (0-1) for the challenge rates. - const [threshold, setThreshold] = useState(FAILURE_CONFIDENCE_THRESHOLD); - const thresholdPercent = Math.round(threshold * 100); - const [selectedReference, setSelectedReference] = useState( - null - ); - - const sessions = useMemo(() => data?.sessions ?? [], [data?.sessions]); - const cohorts = useMemo(() => groupIntoCohorts(sessions), [sessions]); - const protocolSessions = useMemo( - () => sessions.filter(hasCognitionProtocol), - [sessions] - ); - - // The differences are shown against the selected group, by default the - // sessions without any Cognition protocol. - const referenceLabel = - cohorts.find((cohort) => cohort.label === selectedReference)?.label ?? - cohorts.find((cohort) => cohort.label === NO_PROTOCOL_COHORT)?.label ?? - cohorts[0]?.label ?? - null; - - const body = () => { - if (isLoading) { - return ( - - - - - - ); - } - - if (isError) { - return ( - - ); - } - - if (sessions.length === 0) { - return ( - - ); - } - - return ( - <> - - Groups of sessions - - - - - - - - - - ); - }; - - return ( - - - - {startDate && endDate ? ( - - - {format(dayjs.unix(startDate).toDate(), 'MMM d, yyyy HH:mm:ss')} - - - - - {format(dayjs.unix(endDate).toDate(), 'MMM d, yyyy HH:mm:ss')} - - - ) : null} - - - - - - Compare against - - - - - - - - - Confidence threshold - - - - `${value}%`} - onChange={(_, value) => - setThreshold((Array.isArray(value) ? value[0] : value) / 100) - } - /> - - {`${thresholdPercent}%`} - - - - - {body()} - - ); -}; - -export default CognitionProtocolsComparison; diff --git a/ui/src/components/CognitionProtocolsComparison/utils.ts b/ui/src/components/CognitionProtocolsComparison/utils.ts deleted file mode 100644 index 432f3b8..0000000 --- a/ui/src/components/CognitionProtocolsComparison/utils.ts +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { isBestEffortState, isCommitState } from '@/common/cognitionProtocols'; -import { SessionWithCognitionProtocols, Unit } from '@/types/scope.type'; -import { formatDurationMs } from '@/utils'; -import { NO_DATA } from '../CognitionProtocols/primitives'; - -export const NO_PROTOCOL_COHORT = 'No Cognition protocol'; -export const ENABLED_NOT_ACTIVATED_COHORT = 'Enabled, not activated'; - -// --------------------------------------------------------------------------- -// Cohorts: sessions grouped by the protocols that were activated in them. -// Nothing is hardcoded: the groups come from the protocols the API reports. -// --------------------------------------------------------------------------- - -export interface Cohort { - label: string; - // Number of activated protocols, used to order the groups. - activatedCount: number; - sessions: SessionWithCognitionProtocols[]; -} - -export const shortProtocolName = (protocol: string): string => - protocol.replace(/^L9-/, ''); - -const getActivatedProtocols = (session: SessionWithCognitionProtocols): string[] => - (session.cognitionProtocols ?? []) - .filter((protocol) => protocol.activated) - .map((protocol) => shortProtocolName(protocol.protocol)) - .sort(); - -export const getCohortLabel = (session: SessionWithCognitionProtocols): string => { - const activated = getActivatedProtocols(session); - if (activated.length > 0) return activated.join(' + '); - if ((session.cognitionProtocols ?? []).some((protocol) => protocol.enabled)) { - return ENABLED_NOT_ACTIVATED_COHORT; - } - return NO_PROTOCOL_COHORT; -}; - -// Whether any Cognition protocol was enabled or activated in the session. -export const hasCognitionProtocol = (session: SessionWithCognitionProtocols): boolean => - (session.cognitionProtocols ?? []).some( - (protocol) => protocol.enabled || protocol.activated - ); - -const cohortRank = (cohort: Cohort): number => { - if (cohort.label === NO_PROTOCOL_COHORT) return 0; - if (cohort.label === ENABLED_NOT_ACTIVATED_COHORT) return 1; - return 2; -}; - -export const groupIntoCohorts = ( - sessions: SessionWithCognitionProtocols[] -): Cohort[] => { - const cohorts = new Map(); - - sessions.forEach((session) => { - const label = getCohortLabel(session); - const cohort = cohorts.get(label) ?? { - label, - activatedCount: getActivatedProtocols(session).length, - sessions: [] - }; - cohort.sessions.push(session); - cohorts.set(label, cohort); - }); - - return Array.from(cohorts.values()).sort( - (a, b) => - cohortRank(a) - cohortRank(b) || - a.activatedCount - b.activatedCount || - a.label.localeCompare(b.label) - ); -}; - -// --------------------------------------------------------------------------- -// Statistics. A missing value is never counted as a zero: sessions without the -// data are left out of the figure, which is `null` when no session has it. -// --------------------------------------------------------------------------- - -const mean = (values: number[]): number | null => - values.length === 0 - ? null - : values.reduce((sum, value) => sum + value, 0) / values.length; - -const isNumber = (value: unknown): value is number => - typeof value === 'number' && Number.isFinite(value); - -// Share of the sessions (with cognitive observability data) that have a -// failure above the threshold. Restricted to one failure when a name is given. -export const getFailureRate = ( - sessions: SessionWithCognitionProtocols[], - threshold: number, - failureName?: string -): number | null => { - const withData = sessions.filter((session) => session.cognitiveFailures); - if (withData.length === 0) return null; - - const affected = withData.filter((session) => - (session.cognitiveFailures ?? []).some( - (failure) => - failure.confidence > threshold && - (failureName === undefined || failure.name === failureName) - ) - ); - return affected.length / withData.length; -}; - -// Names of the failures above the threshold in at least one session. -export const getFailureNames = ( - sessions: SessionWithCognitionProtocols[], - threshold: number -): string[] => - Array.from( - new Set( - sessions.flatMap((session) => - (session.cognitiveFailures ?? []) - .filter((failure) => failure.confidence > threshold) - .map((failure) => failure.name) - ) - ) - ).sort(); - -export interface CognitiveMetricInfo { - name: string; - unit: Unit; -} - -export const getCognitiveMetrics = ( - sessions: SessionWithCognitionProtocols[] -): CognitiveMetricInfo[] => { - const metrics = new Map(); - sessions.forEach((session) => - (session.cognitiveObservabilityMetrics ?? []).forEach((metric) => { - if (!metrics.has(metric.name)) { - metrics.set(metric.name, { - name: metric.name, - unit: metric.value.unit - }); - } - }) - ); - return Array.from(metrics.values()).sort((a, b) => - a.name.localeCompare(b.name) - ); -}; - -export const getMetricAverage = ( - sessions: SessionWithCognitionProtocols[], - metricName: string -): number | null => - mean( - sessions.flatMap((session) => - (session.cognitiveObservabilityMetrics ?? []) - .filter((metric) => metric.name === metricName) - .map((metric) => metric.value.value) - .filter(isNumber) - ) - ); - -export const getUsageAverage = ( - sessions: SessionWithCognitionProtocols[], - field: 'tokens' | 'cost' | 'duration' -): number | null => - mean(sessions.map((session) => session[field]).filter(isNumber)); - -// CONCORD figures, over the sessions where CONCORD was activated. -export const getConcordStats = (sessions: SessionWithCognitionProtocols[]) => { - const concordSessions = sessions.filter((session) => session.concord); - const states = concordSessions - .map((session) => session.concord?.terminalState) - .filter((state): state is string => typeof state === 'string'); - const satisfactions = concordSessions - .map((session) => session.concord?.worstOffSatisfaction) - .filter(isNumber); - - const share = (count: number): number | null => - states.length === 0 ? null : count / states.length; - - return { - sessionCount: concordSessions.length, - commitShare: share(states.filter(isCommitState).length), - bestEffortShare: share(states.filter(isBestEffortState).length), - averageWorstOffSatisfaction: mean(satisfactions) - }; -}; - -// --------------------------------------------------------------------------- -// Formatting -// --------------------------------------------------------------------------- - -// How a figure is shown, and how a difference to the reference is expressed: -// - rate: a share (0-1), difference in percentage points -// - number: a plain score, absolute difference -// - count / cost / duration: relative difference -export type MetricKind = 'rate' | 'number' | 'count' | 'cost' | 'duration'; - -export const getMetricKind = (unit: Unit): MetricKind => - unit === Unit.Percentage ? 'rate' : 'number'; - -export const formatMetricValue = ( - kind: MetricKind, - value: number | null -): string => { - if (value === null) return NO_DATA; - switch (kind) { - case 'rate': - return `${Math.round(value * 100)}%`; - case 'number': - return value.toFixed(2); - case 'count': - return Math.round(value).toLocaleString(); - case 'cost': - return `$${value.toFixed(4)}`; - case 'duration': - return formatDurationMs(value); - } -}; - -// Difference to the reference, or null when either side has no data. -export const formatDifference = ( - kind: MetricKind, - value: number | null, - reference: number | null -): string | null => { - if (value === null || reference === null) return null; - const difference = value - reference; - const sign = difference > 0 ? '+' : difference < 0 ? '-' : ''; - - if (kind === 'rate') { - return `${sign}${Math.abs(Math.round(difference * 100))} pp`; - } - if (kind === 'number') { - return `${sign}${Math.abs(difference).toFixed(2)}`; - } - // Relative difference: undefined against a reference of zero. - if (reference === 0) return null; - return `${sign}${Math.abs(Math.round((difference / reference) * 100))}%`; -}; diff --git a/ui/src/components/CognitionProtocolsSummary/index.tsx b/ui/src/components/CognitionProtocolsSummary/index.tsx new file mode 100644 index 0000000..b2924bd --- /dev/null +++ b/ui/src/components/CognitionProtocolsSummary/index.tsx @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useMemo } from 'react'; +import { Stack, Typography } from '@mui/material'; +import { WidgetCard } from '../WidgetCard/WidgetCard'; +import { useSessionsWithCognitionProtocols } from '@/api/scopeApi'; +import { getCohortDescription, groupIntoCohorts } from './utils'; + +interface CognitionProtocolsSummaryProps { + applicationId: string; + startDate?: number | null; + endDate?: number | null; +} + +// Cards for the Collect tab: one per group of sessions, grouped by the +// Cognition (L9) protocols activated in them, with how many sessions it holds. +// It loads its own data and renders nothing until there is some, so it can sit +// in a row next to other cards without ever blocking them. +export const CognitionProtocolsSummary = ({ + applicationId, + startDate, + endDate +}: CognitionProtocolsSummaryProps) => { + const { data } = useSessionsWithCognitionProtocols( + applicationId, + startDate ?? undefined, + endDate ?? undefined + ); + + const sessions = useMemo(() => data?.sessions ?? [], [data?.sessions]); + const cohorts = useMemo(() => groupIntoCohorts(sessions), [sessions]); + + return ( + <> + {cohorts.map((cohort) => { + const size = cohort.sessions.length; + return ( + + {size} + + {`${size === 1 ? 'session' : 'sessions'} (${Math.round( + (size / sessions.length) * 100 + )}%)`} + + + } + /> + ); + })} + + ); +}; + +export default CognitionProtocolsSummary; diff --git a/ui/src/components/CognitionProtocolsSummary/utils.ts b/ui/src/components/CognitionProtocolsSummary/utils.ts new file mode 100644 index 0000000..27295c8 --- /dev/null +++ b/ui/src/components/CognitionProtocolsSummary/utils.ts @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SessionWithCognitionProtocols } from '@/types/scope.type'; + +export const NO_PROTOCOL_COHORT = 'No Cognition protocol'; +export const ENABLED_NOT_ACTIVATED_COHORT = 'Enabled, not activated'; + +// Sessions grouped by the protocols that were activated in them. Nothing is +// hardcoded: the groups come from the protocols the API reports. +export interface Cohort { + label: string; + // Number of activated protocols, used to order the groups. + activatedCount: number; + sessions: SessionWithCognitionProtocols[]; +} + +export const shortProtocolName = (protocol: string): string => + protocol.replace(/^L9-/, ''); + +const getActivatedProtocols = ( + session: SessionWithCognitionProtocols +): string[] => + (session.cognitionProtocols ?? []) + .filter((protocol) => protocol.activated) + .map((protocol) => shortProtocolName(protocol.protocol)) + .sort(); + +export const getCohortLabel = ( + session: SessionWithCognitionProtocols +): string => { + const activated = getActivatedProtocols(session); + if (activated.length > 0) return activated.join(' + '); + if ((session.cognitionProtocols ?? []).some((protocol) => protocol.enabled)) { + return ENABLED_NOT_ACTIVATED_COHORT; + } + return NO_PROTOCOL_COHORT; +}; + +// Whether any Cognition protocol was enabled or activated in the session. +export const hasCognitionProtocol = ( + session: SessionWithCognitionProtocols +): boolean => + (session.cognitionProtocols ?? []).some( + (protocol) => protocol.enabled || protocol.activated + ); + +const cohortRank = (cohort: Cohort): number => { + if (cohort.label === NO_PROTOCOL_COHORT) return 0; + if (cohort.label === ENABLED_NOT_ACTIVATED_COHORT) return 1; + return 2; +}; + +export const groupIntoCohorts = ( + sessions: SessionWithCognitionProtocols[] +): Cohort[] => { + const cohorts = new Map(); + + sessions.forEach((session) => { + const label = getCohortLabel(session); + const cohort = cohorts.get(label) ?? { + label, + activatedCount: getActivatedProtocols(session).length, + sessions: [] + }; + cohort.sessions.push(session); + cohorts.set(label, cohort); + }); + + return Array.from(cohorts.values()).sort( + (a, b) => + cohortRank(a) - cohortRank(b) || + a.activatedCount - b.activatedCount || + a.label.localeCompare(b.label) + ); +}; + +export const getCohortDescription = (cohort: Cohort): string => { + if (cohort.label === NO_PROTOCOL_COHORT) { + return 'Sessions where no Cognition protocol was enabled.'; + } + if (cohort.activatedCount === 0) { + return 'Sessions where a Cognition protocol was enabled but never activated.'; + } + return 'Sessions where exactly these protocols were activated.'; +}; diff --git a/ui/src/components/CognitiveFailureDescription.tsx b/ui/src/components/CognitiveFailureDescription.tsx deleted file mode 100644 index cf1ed33..0000000 --- a/ui/src/components/CognitiveFailureDescription.tsx +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { Box, Typography, useTheme } from '@mui/material'; -import { COGNITIVE_FAILURE_DESCRIPTIONS } from '@/common/cognitiveFailures'; - -// Widget tooltip content for a failure: its explanation (when known) followed -// by the suggested remediations, with the label highlighted. -export const CognitiveFailureDescription = ({ - name, - remediations -}: { - name: string; - remediations?: string[] | null; -}) => { - const theme = useTheme(); - const description = COGNITIVE_FAILURE_DESCRIPTIONS[name]; - - return ( - - {description && `${description}\n\n`} - - Suggested remediations: - - {` ${remediations?.length ? remediations.join(', ') : 'None'}`} - - ); -}; diff --git a/ui/src/components/CognitiveFailuresBanner/index.tsx b/ui/src/components/CognitiveFailuresBanner/index.tsx new file mode 100644 index 0000000..af71cdc --- /dev/null +++ b/ui/src/components/CognitiveFailuresBanner/index.tsx @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Banner, Skeleton } from '@open-ui-kit/core'; +import { CognitiveFailure } from '@/types/scope.type'; + +interface CognitiveFailuresBannerProps { + // Undefined while there is no data for the session. + failures?: CognitiveFailure[]; + isLoading?: boolean; + isError?: boolean; +} + +// A summary of the cognitive failures detected for a session. +export const CognitiveFailuresBanner = ({ + failures, + isLoading = false, + isError = false +}: CognitiveFailuresBannerProps) => { + if (isLoading) { + return ; + } + + if (isError) { + return ( + + ); + } + + if (!failures) return null; + + const highConfidenceFailures = failures.filter( + (failure) => failure.confidence === 'high' + ).length; + + return ( + 0 + ? 'warning' + : failures.length > 0 + ? 'info' + : 'success' + } + text={ + failures.length === 0 + ? 'No cognitive failures detected for this session.' + : `${failures.length} cognitive failure${ + failures.length === 1 ? '' : 's' + } detected, ${highConfidenceFailures} with high confidence.` + } + /> + ); +}; diff --git a/ui/src/components/CognitiveFailuresList/index.tsx b/ui/src/components/CognitiveFailuresList/index.tsx new file mode 100644 index 0000000..187988a --- /dev/null +++ b/ui/src/components/CognitiveFailuresList/index.tsx @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import { GeneralSize, Tag } from '@open-ui-kit/core'; +import { Stack, Typography } from '@mui/material'; +import { WidgetCard } from '@/components/WidgetCard/WidgetCard'; +import { + CONFIDENCE_COLORS, + CONFIDENCE_LABELS +} from '@/common/cognitiveFailureConfidence'; +import { + getCognitiveFailureLabel, + getCognitiveFailureTaxonomy +} from '@/common/cognitiveFailureTaxonomy'; +import { CognitiveFailure } from '@/types/scope.type'; + +interface CognitiveFailuresListProps { + // Undefined while there is no data for the session. Shown in the given + // order. + failures?: CognitiveFailure[]; +} + +// One card per cognitive failure of a session, with its confidence and the +// detector's reasoning. +export const CognitiveFailuresList = ({ + failures +}: CognitiveFailuresListProps) => { + if (!failures) return null; + + return ( + + Cognitive Failures + {failures.length === 0 ? ( + + No cognitive failures detected for this session. + + ) : ( + + {failures.map((failure) => ( + + + {`${CONFIDENCE_LABELS[failure.confidence] ?? failure.confidence} confidence`} + + {failure.reasoning} + + } + cardSx={{ width: '420px' }} + /> + ))} + + )} + + ); +}; diff --git a/ui/src/components/SessionDetails/CognitiveObservabilityTab.tsx b/ui/src/components/CognitiveMetrics/index.tsx similarity index 58% rename from ui/src/components/SessionDetails/CognitiveObservabilityTab.tsx rename to ui/src/components/CognitiveMetrics/index.tsx index 22a0290..71a4428 100644 --- a/ui/src/components/SessionDetails/CognitiveObservabilityTab.tsx +++ b/ui/src/components/CognitiveMetrics/index.tsx @@ -3,27 +3,19 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { useMemo } from 'react'; -import { useParams } from 'react-router'; -import { Banner, EmptyState, Skeleton } from '@open-ui-kit/core'; -import { Stack, Typography } from '@mui/material'; -// Cognitive metrics are hidden for now; the code is kept commented out to be restored later. -// import { Divider, OverflowTooltip } from '@open-ui-kit/core'; -// import CategoryOutlinedIcon from '@mui/icons-material/CategoryOutlined'; -// import { Box, LinearProgress, useTheme } from '@mui/material'; -import { - RemediationTags, - ScoreSeverityBadgeLabel, - WidgetCard -} from '@/components'; -import { useSessionsWithCognitiveObservability } from '@/api/scopeApi'; -import { useTimeRangeStore } from '@/store'; -import { FAILURE_CONFIDENCE_THRESHOLD } from '@/common/cognitiveFailures'; -// import { COGNITIVE_METRIC_GROUPS } from '@/common/cognitiveFailures'; -// import { getScoreColor } from '@/utils/metrics'; -// import { CognitiveObservabilityMetric, Unit } from '@/types/scope.type'; -import { COGNITIVE_FAILURE_DESCRIPTIONS } from '@/common/cognitiveFailures'; - +// Cognitive metrics are hidden for now; the code is kept commented out to be +// restored later (it came from the session's Cognitive Observability tab). +// It used the `cognitiveObservabilityMetrics` of the session returned by +// `useSessionsWithCognitiveObservability`, and needed these imports: +// import { useMemo } from 'react'; +// import { Stack, Typography } from '@mui/material'; +// import { Divider, OverflowTooltip } from '@open-ui-kit/core'; +// import CategoryOutlinedIcon from '@mui/icons-material/CategoryOutlined'; +// import { Box, LinearProgress, useTheme } from '@mui/material'; +// import { ScoreSeverityBadgeLabel } from '@/components'; +// import { COGNITIVE_METRIC_GROUPS } from '@/common/cognitiveMetricGroups'; +// import { getScoreColor } from '@/utils/metrics'; +// import { CognitiveObservabilityMetric, Unit } from '@/types/scope.type'; /* const toScore = (metric: CognitiveObservabilityMetric): number => metric.value.unit === Unit.Percentage @@ -32,40 +24,9 @@ const toScore = (metric: CognitiveObservabilityMetric): number => const average = (scores: number[]): number => scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0; -*/ -const formatPercent = (fraction: number) => `${Math.round(fraction * 100)}%`; - -export const CognitiveObservabilityTab = () => { // const theme = useTheme(); - const { sessionId, applicationId } = useParams(); - const { startDate, endDate } = useTimeRangeStore(); - - const { data, isLoading, isError } = useSessionsWithCognitiveObservability( - applicationId ?? '', - startDate, - endDate, - sessionId - ); - - const session = useMemo( - () => data?.sessions?.find((s) => s.sessionId === sessionId), - [data?.sessions, sessionId] - ); - - const failures = useMemo( - () => - [...(session?.cognitiveFailures ?? [])].sort( - (a, b) => b.confidence - a.confidence - ), - [session?.cognitiveFailures] - ); - - const highConfidenceFailures = failures.filter( - (failure) => failure.confidence > FAILURE_CONFIDENCE_THRESHOLD - ).length; - /* // Metrics split into the configured groups; metrics outside every group go // into an extra "Other" column so none is hidden. const metricGroups = useMemo(() => { @@ -101,89 +62,7 @@ export const CognitiveObservabilityTab = () => { ] : groups; }, [session?.cognitiveObservabilityMetrics]); - */ - - if (isLoading) { - return ( - - - - - - - - - - ); - } - - if (isError) { - return ( - - ); - } - - if (!session) { - return ; - } - - return ( - - 0 - ? 'warning' - : failures.length > 0 - ? 'info' - : 'success' - } - text={ - failures.length === 0 - ? 'No cognitive failures detected for this session.' - : `${failures.length} cognitive failure${ - failures.length === 1 ? '' : 's' - } detected, ${highConfidenceFailures} with confidence above ${ - FAILURE_CONFIDENCE_THRESHOLD * 100 - }%.` - } - /> - - - Cognitive Failures - {failures.length === 0 ? ( - - No cognitive failures detected for this session. - - ) : ( - - {failures.map((failure) => ( - - - - - } - cardSx={{ width: '350px' }} - /> - ))} - - )} - - {/* Cognitive Metrics {metricGroups.every((group) => group.metrics.length === 0) ? ( @@ -295,7 +174,6 @@ export const CognitiveObservabilityTab = () => { )} - */} - - ); -}; +*/ + +export {}; diff --git a/ui/src/components/CognitiveObservability/CognitiveObservabilityTable.tsx b/ui/src/components/CognitiveObservability/CognitiveObservabilityTable.tsx deleted file mode 100644 index 7209606..0000000 --- a/ui/src/components/CognitiveObservability/CognitiveObservabilityTable.tsx +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useMemo, useState } from 'react'; -import { useNavigate, useParams } from 'react-router-dom'; -import { PATHS } from '@/routes/routes'; -import { MaterialReactTable, MRT_SortingState } from 'material-react-table'; -import { - CreateTableInstance, - EmptyState, - TableProps, - Tooltip -} from '@open-ui-kit/core'; -import { Typography, useTheme } from '@mui/material'; -import CircleIcon from '@mui/icons-material/Circle'; -import ChevronRightIcon from '@mui/icons-material/ChevronRight'; -import { format } from 'date-fns'; -import { - SessionsWithCognitiveObservability, - SessionWithCognitiveObservability -} from '@/types/scope.type'; -import InfoOutlineIcon from '@mui/icons-material/InfoOutline'; -import { CustomTooltip, Tags } from '@/components'; -import { getScoreColor } from '@/utils/metrics'; -import { GLOBAL_BACKGROUND_COLOR } from '@/common/styles'; -import { - getFailuresAboveThreshold, - getRemediationsAboveThreshold, - getSortedFailures -} from './utils'; - -type MRT_ColumnDefList = - TableProps['columns']; - -interface CognitiveObservabilityTableProps { - sessionsWithCognitiveObservability?: SessionsWithCognitiveObservability; - // Failures count only when their confidence is above this fraction (0-1). - threshold: number; - selectedFailure?: string | null; - isLoading?: boolean; - onReload?: () => void; - onClearFilter?: () => void; -} - -const maxConfidence = ( - session: SessionWithCognitiveObservability, - threshold: number -) => getFailuresAboveThreshold(session, threshold)[0]?.confidence ?? 0; - -export const CognitiveObservabilityTable = ({ - sessionsWithCognitiveObservability, - threshold, - selectedFailure = null, - isLoading = false, - onReload, - onClearFilter -}: CognitiveObservabilityTableProps) => { - const theme = useTheme(); - const [sorting, setSorting] = useState([]); - const { applicationId } = useParams(); - const navigate = useNavigate(); - const thresholdPercent = Math.round(threshold * 100); - const sessionCount = - sessionsWithCognitiveObservability?.sessions?.length ?? 0; - - const onSessionClick = (session: SessionWithCognitiveObservability) => { - navigate( - PATHS.applicationCollectSessionTab - .replace(':applicationId', applicationId ?? '') - .replace(':sessionId', encodeURIComponent(session.sessionId)) - .replace(':sessionTab', 'cognitive-observability') - ); - }; - - const columns = useMemo( - () => [ - { - accessorKey: 'sessionId', - header: 'Session ID', - accessorFn: ({ sessionId }) => ( - - - {sessionId} - - - ) - }, - { - accessorKey: 'timestamp', - header: 'Timestamp', - size: 115, - accessorFn: ({ timestamp }) => ( - - {timestamp - ? format( - new Date(Number(timestamp) * 1000), - 'MMM d, yyyy HH:mm:ss' - ) - : ''} - - ), - sortingFn: (rowA, rowB) => - (Number(rowA.original.timestamp) || 0) - - (Number(rowB.original.timestamp) || 0) - }, - { - id: 'cognitiveFailures', - header: 'Cognitive Failures', - enableSorting: false, - // Highest confidence first, so the visible tag is the one that - // qualifies the session; the dot is colored by confidence. - accessorFn: (session) => ( - ({ - name: `${failure.name}: ${Math.round(failure.confidence * 100)}%` - }))} - minDisplayed={1} - /> - ) - }, - { - id: 'open', - header: '', - size: 30, - enableSorting: false, - enableResizing: false, - accessorFn: () => ( - - ) - } - ], - [threshold, theme] - ); - - const tableRef = CreateTableInstance({ - data: sessionsWithCognitiveObservability?.sessions ?? [], - columns, - isLoading, - rowCount: sessionCount, - title: { label: 'Session Cognitive Insights' }, - titleExtension: ( - - {`Sessions with at least one failure above ${thresholdPercent}% confidence.`} - - } - placement={'top'} - sx={{ maxWidth: '550px' }} - > - - - ), - topToolbarProps: { - export: { enableExport: false }, - onReload - }, - enableSorting: true, - enableColumnResizing: true, - renderEmptyRowsFallback: () => - selectedFailure ? ( - - ) : ( - - ), - state: { sorting }, - onSortingChange: setSorting, - muiTableBodyRowProps: ({ row }) => ({ - onClick: () => onSessionClick(row.original), - sx: { - cursor: 'pointer', - backgroundColor: 'transparent', - '& > td': { - backgroundColor: 'transparent !important' - }, - '&:hover > td': { - backgroundColor: `${theme.palette.action.hover} !important` - } - } - }), - muiTablePaperProps: { - sx: { - padding: '12px', - backgroundColor: 'transparent', - elevation: 0 - } - }, - muiBottomToolbarProps: { - sx: { - '& .MuiInputBase-root': { - width: '80px' - } - } - }, - muiTableHeadCellProps: { - sx: { backgroundColor: GLOBAL_BACKGROUND_COLOR, color: '#ffffff' } - }, - muiTableBodyCellProps: { - sx: { - backgroundColor: 'transparent', - color: '#ffffff', - height: '40px' - } - } - }); - - return ; -}; diff --git a/ui/src/components/CognitiveObservability/index.tsx b/ui/src/components/CognitiveObservability/index.tsx deleted file mode 100644 index 626b5e0..0000000 --- a/ui/src/components/CognitiveObservability/index.tsx +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useMemo, useState } from 'react'; -import { useParams } from 'react-router'; -import { Stack, Typography } from '@open-ui-kit/core'; -import { Chip, Slider } from '@mui/material'; -import { format } from 'date-fns'; -import dayjs from 'dayjs'; -import { CognitiveObservabilityTable } from './CognitiveObservabilityTable'; -import { CognitiveObservabilityOverview } from './CognitiveObservabilityOverview'; -import { getFailuresAboveThreshold } from './utils'; -import InfoOutlineIcon from '@mui/icons-material/InfoOutline'; -import { CustomTooltip, IntervalPicker } from '@/components'; -import { useDebouncedValue } from '@/utils'; -import { useTimeRangeStore } from '@/store'; -import { useSessionsWithCognitiveObservability } from '@/api/scopeApi'; -import { FAILURE_CONFIDENCE_THRESHOLD } from '@/common/cognitiveFailures'; - -const CognitiveObservability = () => { - const { applicationId } = useParams(); - const { startDate, endDate, setStartDate, setEndDate } = useTimeRangeStore(); - const debouncedStartDate = useDebouncedValue(startDate, 500); - const debouncedEndDate = useDebouncedValue(endDate, 500); - - const { - data: sessionsWithCognitiveObservability, - isLoading, - refetch - } = useSessionsWithCognitiveObservability( - applicationId ?? '', - debouncedStartDate, - debouncedEndDate - ); - - const [selectedFailure, setSelectedFailure] = useState(null); - // Confidence threshold as a fraction (0-1); drives counts and filtering. - const [threshold, setThreshold] = useState(FAILURE_CONFIDENCE_THRESHOLD); - const thresholdPercent = Math.round(threshold * 100); - - // Only sessions with at least one failure above the threshold are listed, - // narrowed to the selected failure when a card is selected. - const filteredSessions = useMemo(() => { - if (!sessionsWithCognitiveObservability) return undefined; - return { - ...sessionsWithCognitiveObservability, - sessions: sessionsWithCognitiveObservability.sessions.filter( - (session) => { - const failures = getFailuresAboveThreshold(session, threshold); - return selectedFailure - ? failures.some((failure) => failure.name === selectedFailure) - : failures.length > 0; - } - ) - }; - }, [sessionsWithCognitiveObservability, selectedFailure, threshold]); - - return ( - - - - {startDate && endDate ? ( - - - {format(dayjs.unix(startDate).toDate(), 'MMM d, yyyy HH:mm:ss')} - - - - - {format(dayjs.unix(endDate).toDate(), 'MMM d, yyyy HH:mm:ss')} - - - ) : null} - - - - Confidence threshold - - - A cognitive failure only counts when its detection confidence - is above this value. It drives the summary and the sessions - listed in the table. - - } - placement={'top'} - sx={{ maxWidth: '550px' }} - > - - - - `${value}%`} - onChange={(_, value) => - setThreshold((Array.isArray(value) ? value[0] : value) / 100) - } - /> - - {`${thresholdPercent}%`} - - - - - - - - {selectedFailure && ( - - setSelectedFailure(null)} - color="primary" - variant="outlined" - /> - - )} - refetch()} - onClearFilter={() => setSelectedFailure(null)} - /> - - - ); -}; - -export default CognitiveObservability; diff --git a/ui/src/components/CognitiveObservability/utils.ts b/ui/src/components/CognitiveObservability/utils.ts deleted file mode 100644 index 8778894..0000000 --- a/ui/src/components/CognitiveObservability/utils.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import type { - CognitiveFailure, - SessionWithCognitiveObservability -} from '@/types/scope.type'; - -// All the failures of a session, highest confidence first. -export const getSortedFailures = ( - session: SessionWithCognitiveObservability -): CognitiveFailure[] => - [...(session.cognitiveFailures ?? [])].sort( - (a, b) => b.confidence - a.confidence - ); - -// The failures of a session whose confidence is above the threshold. -export const getFailuresAboveThreshold = ( - session: SessionWithCognitiveObservability, - threshold: number -): CognitiveFailure[] => - getSortedFailures(session).filter( - (failure) => failure.confidence > threshold - ); - -// Union of the remediations of the failures above the threshold. -export const getRemediationsAboveThreshold = ( - session: SessionWithCognitiveObservability, - threshold: number -): string[] => - Array.from( - new Set( - getFailuresAboveThreshold(session, threshold).flatMap( - (failure) => failure.remediations ?? [] - ) - ) - ).sort(); diff --git a/ui/src/components/GenericGraph/GenericGraph.tsx b/ui/src/components/GenericGraph/GenericGraph.tsx index df12fd9..6140a64 100644 --- a/ui/src/components/GenericGraph/GenericGraph.tsx +++ b/ui/src/components/GenericGraph/GenericGraph.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef, useState, useCallback, useMemo } from 'react'; import { Graph, NodeData, EdgeData } from '@antv/g6'; -import { Box, useTheme, Stack } from '@mui/material'; +import { Box, useTheme, Stack, alpha } from '@mui/material'; import { GenericGraphDrawer } from './GenericGraphDrawer'; import { GenericGraphProps } from './types'; import { transformDataToG6Graph, getStatusColor } from '@/utils'; @@ -27,6 +27,7 @@ export const GenericGraph = ({ nodeProps, edgeProps, waitForLayout = false, + showNodeLabelTooltip = false, onGraphReady, resolveNodeClickData }: GenericGraphProps) => { @@ -259,6 +260,44 @@ export const GenericGraph = ({ 'hover-activate' ], plugins: [ + ...(showNodeLabelTooltip + ? [ + { + type: 'tooltip', + trigger: 'hover', + position: 'bottom', + offset: [0, 14], + // Mirrors the default MUI look. This replaces G6's + // default '.tooltip' style, so layout/positioning rules are + // restated here. + style: { + '.tooltip': { + position: 'absolute', + 'z-index': 1500, + transition: 'visibility 0.15s, left 0.15s, top 0.15s', + 'box-sizing': 'border-box', + 'min-width': 'auto', + 'max-width': '300px', + padding: '4px 8px', + 'border-radius': `${theme.shape.borderRadius}px`, + 'background-color': alpha(theme.palette.grey[700], 0.92), + color: theme.palette.common.white, + 'box-shadow': 'none', + 'font-family': theme.typography.fontFamily, + 'font-size': theme.typography.pxToRem(11), + 'font-weight': theme.typography.fontWeightMedium, + 'line-height': '1.4em', + 'word-wrap': 'break-word', + 'text-align': 'left' + } + }, + // Only nodes get a tooltip (edges/combos are ignored) + enable: (event: any) => event.targetType === 'node', + getContent: async (_event: any, items: any[]) => + String(items?.[0]?.data?.label ?? items?.[0]?.id ?? '') + } + ] + : []), { type: 'toolbar', position: 'top-right', @@ -344,7 +383,7 @@ export const GenericGraph = ({ }); }); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [waitForLayout, isEmpty]); + }, [waitForLayout, isEmpty, showNodeLabelTooltip]); // Initialize graph when data changes useEffect(() => { diff --git a/ui/src/components/GenericGraph/types.ts b/ui/src/components/GenericGraph/types.ts index 2d8f7ce..c03732c 100644 --- a/ui/src/components/GenericGraph/types.ts +++ b/ui/src/components/GenericGraph/types.ts @@ -56,6 +56,8 @@ export interface GenericGraphProps { edgeProps?: EdgePropsOverrides; /** If true, shows a loading spinner until the layout finishes computing (useful for force layouts) */ waitForLayout?: boolean; + /** If true, hovering a node shows a tooltip with its full label (useful when labels are truncated) */ + showNodeLabelTooltip?: boolean; /** Callback fired with the Graph instance after render completes */ onGraphReady?: (graph: Graph) => void; /** Custom resolver for node click data shown in drawer. If provided, overrides the default. */ diff --git a/ui/src/components/RemediationTags.tsx b/ui/src/components/RemediationTags.tsx deleted file mode 100644 index 665fd15..0000000 --- a/ui/src/components/RemediationTags.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { GeneralSize, Tag, Tooltip } from '@open-ui-kit/core'; -import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined'; -import { Box, Stack, Typography, useTheme } from '@mui/material'; -import { getRemediationDescription } from '@/common/cognitiveFailures'; - -// All the suggested remediations of a failure, labelled, each with a tooltip -// explaining the protocol. Renders nothing when there are none. -export const RemediationTags = ({ - remediations -}: { - remediations?: string[] | null; -}) => { - const theme = useTheme(); - if (!remediations?.length) return null; - - return ( - - - Suggested remediations - - - {remediations.map((name) => { - const description = getRemediationDescription(name); - const tag = ( - } - sx={{ - backgroundColor: theme.palette.vars.controlBackgroundMedium, - cursor: 'pointer' - }} - > - {name} - - ); - return description ? ( - {description}} - placement={'top'} - slotProps={{ tooltip: { sx: { maxWidth: 520 } } }} - > - {tag} - - ) : ( - {tag} - ); - })} - - - ); -}; diff --git a/ui/src/components/SemanticGroupsInsightsTable/SemanticGroupsInsightsTable.tsx b/ui/src/components/SemanticGroupsInsightsTable/SemanticGroupsInsightsTable.tsx deleted file mode 100644 index f6446dd..0000000 --- a/ui/src/components/SemanticGroupsInsightsTable/SemanticGroupsInsightsTable.tsx +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - MaterialReactTable, - MRT_SortingState, - MRT_VisibilityState -} from 'material-react-table'; -import { - CreateTableInstance, - EmptyState, - GeneralSize, - TableProps, - Tag, - TagBackgroundColorVariants, - Tooltip -} from '@open-ui-kit/core'; -import { useMemo, useState } from 'react'; -import { Box, Stack, Typography } from '@mui/material'; -import { format } from 'date-fns'; -import { SemanticGroupsInsight } from '@/types/scope.type'; -import { Tags } from '@/components'; -import { Dayjs } from 'dayjs'; -import { GLOBAL_BACKGROUND_COLOR } from '@/common/styles'; -import { SessionsInsightsTableWrapper } from '@/components/SessionsInsightsTable'; - -type MRT_ColumnDefList = TableProps['columns']; - -interface SemanticGroupsInsightsTableProps { - data: SemanticGroupsInsight[]; - isLoading: boolean; - isError?: boolean; - startDate: Dayjs | null; - endDate: Dayjs | null; - startDateUnix: number; - endDateUnix: number; -} - -export const SemanticGroupsInsightsTable = ({ - data, - isLoading, - isError, - startDate, - endDate, - startDateUnix, - endDateUnix -}: SemanticGroupsInsightsTableProps) => { - const [columnVisibility, setColumnVisibility] = useState( - { - // priority: false, - labels: false, - sessionIds: false - } - ); - const [sorting, setSorting] = useState([ - { id: 'priority', desc: true } - ]); - - const columns = useMemo( - () => [ - { - accessorKey: 'name', - header: 'Insight', - size: 250, - accessorFn: ({ name }) => ( - - - {name} - - - ) - }, - { - accessorKey: 'groupName', - header: 'Semantic Group' - }, - { - accessorKey: 'description', - header: 'Description', - size: 350, - accessorFn: ({ description }) => ( - - - {description} - - - ) - }, - { - accessorKey: 'priority', - header: 'Priority', - size: 120, - accessorFn: ({ priority }) => { - const colorMap: Record = { - high: TagBackgroundColorVariants.AccentFWeak, - medium: TagBackgroundColorVariants.AccentEWeak, - low: TagBackgroundColorVariants.AccentGWeak - }; - return ( - - {priority} - - ); - }, - sortingFn: (rowA, rowB) => { - const priorityOrder: Record = { - high: 3, - medium: 2, - low: 1 - }; - const a = priorityOrder[rowA.original.priority?.toLowerCase()] ?? 0; - const b = priorityOrder[rowB.original.priority?.toLowerCase()] ?? 0; - return a - b; - } - }, - { - accessorKey: 'labels', - header: 'Labels', - size: 200, - accessorFn: ({ labels }) => ( - ({ - name: label, - color: TagBackgroundColorVariants.AccentAWeak - }))} - minDisplayed={2} - /> - ) - }, - { - accessorKey: 'sessionIds', - header: 'Sessions', - size: 100, - accessorFn: ({ sessionIds }) => ( - {sessionIds?.length ?? 0} - ), - sortingFn: (rowA, rowB) => { - return ( - (rowA.original.sessionIds?.length ?? 0) - - (rowB.original.sessionIds?.length ?? 0) - ); - } - }, - { - accessorKey: 'createdAt', - header: 'Created', - size: 180, - accessorFn: ({ createdAt }) => ( - - {createdAt ? format(new Date(createdAt), 'MMM d, yyyy HH:mm') : ''} - - ), - sortingFn: (rowA, rowB) => { - const a = new Date(rowA.original.createdAt).getTime() || 0; - const b = new Date(rowB.original.createdAt).getTime() || 0; - return a - b; - } - } - ], - [] - ); - - const tableRef = CreateTableInstance({ - data, - columns, - isLoading, - rowCount: data?.length, - title: { label: 'Top Failures and Analysis' }, - topToolbarProps: { - export: { enableExport: false } - }, - enableSorting: true, - enableColumnResizing: true, - enableRowActions: true, - renderEmptyRowsFallback: () => ( - - ), - - state: { columnVisibility, sorting }, - onColumnVisibilityChange: setColumnVisibility, - onSortingChange: setSorting, - - titleExtension: - startDate && endDate ? ( - - - {format(startDate.toDate(), 'MMM d, yyyy HH:mm:ss')} - - - - - {format(endDate.toDate(), 'MMM d, yyyy HH:mm:ss')} - - - ) : undefined, - - muiTableBodyRowProps: { - sx: { - cursor: 'pointer', - backgroundColor: 'transparent', - '& > td': { - backgroundColor: 'transparent !important' - } - } - }, - muiTablePaperProps: { - sx: { - padding: '12px', - backgroundColor: 'transparent', - elevation: 0 - } - }, - muiTableHeadCellProps: { - sx: { backgroundColor: GLOBAL_BACKGROUND_COLOR, color: '#ffffff' } - }, - muiTableBodyCellProps: { - sx: { - backgroundColor: 'transparent', - color: '#ffffff', - height: '40px' - } - }, - renderDetailPanel: ({ row }) => { - return ( - - - - ); - } - }); - - return ; -}; diff --git a/ui/src/components/SemanticGroupsInsightsTable/index.tsx b/ui/src/components/SemanticGroupsInsightsTable/index.tsx deleted file mode 100644 index 1cd6aaa..0000000 --- a/ui/src/components/SemanticGroupsInsightsTable/index.tsx +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useSemanticGroupsInsights } from '@/api/scopeApi'; -import { Spinner, Stack } from '@open-ui-kit/core'; -import { SemanticGroupsInsightsTable } from './SemanticGroupsInsightsTable'; -import { IntervalPicker } from '@/components/IntervalPicker'; -import dayjs from 'dayjs'; -import { useTimeRangeStore } from '@/store'; -import { useParams } from 'react-router-dom'; - -export const SemanticGroupsInsightsTableWrapper = () => { - const { applicationId } = useParams(); - const { startDate, endDate, setStartDate, setEndDate } = useTimeRangeStore(); - - const { - data: semanticGroupsInsights, - isLoading: isSemanticGroupsInsightsLoading, - isError: isSemanticGroupsInsightsError - } = useSemanticGroupsInsights(applicationId ?? '', startDate, endDate); - - if (isSemanticGroupsInsightsLoading) { - return ( - - - - ); - } - - if (isSemanticGroupsInsightsError) { - return
Error loading semantic groups insights
; - } - - return ( - - - - - ); -}; diff --git a/ui/src/components/SessionDetails/SessionAnalysisTab.tsx b/ui/src/components/SessionDetails/SessionAnalysisTab.tsx index 2448512..40e0cfa 100644 --- a/ui/src/components/SessionDetails/SessionAnalysisTab.tsx +++ b/ui/src/components/SessionDetails/SessionAnalysisTab.tsx @@ -9,6 +9,8 @@ import { useCallback, useMemo } from 'react'; import { SessionLatentSpace } from './SessionLatentSpace'; import { transformLatentSpaceResponse } from '@/utils/latentSpace'; import { + CognitiveFailuresBanner, + CognitiveFailuresList, WidgetCard, MetricSelectorDropdown, useMetricSelector, @@ -20,8 +22,11 @@ import { useImpactAssessmentSession, useSemanticGroups, useSessionLatentSpace, + useSessionsWithCognitiveObservability, useSessionStatefulEval } from '@/api/scopeApi'; +import { useTimeRangeStore } from '@/store'; +import { compareFailures } from '@/common/cognitiveFailureConfidence'; import { Spinner } from '@open-ui-kit/core'; import { metricCatalog } from '@/common'; import { Unit } from '@/types/scope.type'; @@ -38,6 +43,33 @@ export const SessionAnalysisTab = () => { const insightsEnabled = useFeatureFlag('insights'); const impactAssessmentEnabled = useFeatureFlag('impact_assessment'); + const cognitiveObservabilityEnabled = useFeatureFlag( + 'cognitive_observability' + ); + + // The cognitive failures of the session. Without the feature flag the + // application id is empty, which disables the request. + const { startDate, endDate } = useTimeRangeStore(); + const { + data: cognitiveSessions, + isLoading: isCognitiveFailuresLoading, + isError: isCognitiveFailuresError + } = useSessionsWithCognitiveObservability( + cognitiveObservabilityEnabled ? (applicationId ?? '') : '', + startDate, + endDate, + sessionId + ); + + // Undefined while the session has no data. Highest confidence first. + const cognitiveFailures = useMemo(() => { + const session = cognitiveSessions?.sessions?.find( + (s) => s.sessionId === sessionId + ); + return session + ? [...(session.cognitiveFailures ?? [])].sort(compareFailures) + : undefined; + }, [cognitiveSessions?.sessions, sessionId]); const { data: latentSpaceResponse, @@ -182,6 +214,14 @@ export const SessionAnalysisTab = () => { gap={'32px'} sx={{ paddingTop: '16px', width: '100%', height: '100%' }} > + {cognitiveObservabilityEnabled && ( + + )} + @@ -208,6 +248,10 @@ export const SessionAnalysisTab = () => { })} + {cognitiveObservabilityEnabled && ( + + )} + {impactAssessmentEnabled && (isImpactAssessmentSessionLoading ? ( (false); @@ -112,12 +105,6 @@ const SessionDetails = ({ visible: true, render: () => }, - { - key: 'cognitive-observability', - label: 'Cognitive Observability', - visible: cognitiveObservabilityEnabled, - render: () => - }, { key: 'cognition-protocols', label: 'Cognition Protocols', @@ -130,12 +117,6 @@ const SessionDetails = ({ visible: isReasoningPathTabVisible, render: () => sessionId && }, - { - key: 'execution-graph', - label: 'Execution Graph', - visible: true, - render: () => - }, { key: 'conversation', label: 'Conversation', @@ -149,7 +130,6 @@ const SessionDetails = ({ duration, isReasoningPathTabVisible, sessionId, - cognitiveObservabilityEnabled, cognitionProtocolsEnabled ] ); diff --git a/ui/src/components/CognitiveObservability/CognitiveObservabilityOverview.tsx b/ui/src/components/SessionsInsightsTable/FailuresOverview.tsx similarity index 71% rename from ui/src/components/CognitiveObservability/CognitiveObservabilityOverview.tsx rename to ui/src/components/SessionsInsightsTable/FailuresOverview.tsx index 70b08c9..5463191 100644 --- a/ui/src/components/CognitiveObservability/CognitiveObservabilityOverview.tsx +++ b/ui/src/components/SessionsInsightsTable/FailuresOverview.tsx @@ -7,16 +7,17 @@ import { useMemo } from 'react'; import { Stack, Typography } from '@open-ui-kit/core'; import { Box, LinearProgress, useTheme } from '@mui/material'; import InfoOutlineIcon from '@mui/icons-material/InfoOutline'; -import { CustomTooltip, RemediationTags, WidgetCard } from '@/components'; -import { COGNITIVE_FAILURE_DESCRIPTIONS } from '@/common/cognitiveFailures'; -import { SessionsWithCognitiveObservability } from '@/types/scope.type'; +import { CustomTooltip } from '@/components/CustomTooltip'; +import { WidgetCard } from '@/components/WidgetCard/WidgetCard'; +import { + getCognitiveFailureLabel, + getCognitiveFailureTaxonomy +} from '@/common/cognitiveFailureTaxonomy'; +import { SessionWithCognitiveObservability } from '@/types/scope.type'; import { getScoreColor } from '@/utils/metrics'; -import { getFailuresAboveThreshold } from './utils'; -interface CognitiveObservabilityOverviewProps { - sessionsWithCognitiveObservability?: SessionsWithCognitiveObservability; - // Failures count only when their confidence is above this fraction (0-1). - threshold: number; +interface FailuresOverviewProps { + sessions?: SessionWithCognitiveObservability[]; selectedFailure?: string | null; onSelectFailure?: (failure: string | null) => void; } @@ -42,43 +43,33 @@ const RatioBar = ({ ratio, color }: { ratio: number; color: string }) => { ); }; -export const CognitiveObservabilityOverview = ({ - sessionsWithCognitiveObservability, - threshold, +// The share of sessions with cognitive failures, and one card per failure. +// Selecting a card filters the table below to that failure. +export const FailuresOverview = ({ + sessions, selectedFailure = null, onSelectFailure -}: CognitiveObservabilityOverviewProps) => { +}: FailuresOverviewProps) => { const theme = useTheme(); - const sessions = sessionsWithCognitiveObservability?.sessions; - const thresholdPercent = Math.round(threshold * 100); const stats = useMemo(() => { const totalSessions = sessions?.length ?? 0; const counts = new Map(); - // Union of the remediations of every occurrence of a failure. - const remediationsByFailure = new Map>(); let sessionsWithFailure = 0; (sessions ?? []).forEach((session) => { - const failures = getFailuresAboveThreshold(session, threshold); - if (failures.length > 0) sessionsWithFailure += 1; - - // Each failure counts at most once per session. - new Set(failures.map((failure) => failure.name)).forEach((name) => - counts.set(name, (counts.get(name) ?? 0) + 1) + const names = new Set( + (session.cognitiveFailures ?? []).map((failure) => failure.name) ); - failures.forEach((failure) => { - const set = remediationsByFailure.get(failure.name) ?? new Set(); - (failure.remediations ?? []).forEach((r) => set.add(r)); - remediationsByFailure.set(failure.name, set); - }); + if (names.size > 0) sessionsWithFailure += 1; + // Each failure counts at most once per session. + names.forEach((name) => counts.set(name, (counts.get(name) ?? 0) + 1)); }); const failures = Array.from(counts, ([name, count]) => ({ name, count, - ratio: totalSessions > 0 ? count / totalSessions : 0, - remediations: Array.from(remediationsByFailure.get(name) ?? []).sort() + ratio: totalSessions > 0 ? count / totalSessions : 0 })).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name)); return { @@ -88,7 +79,7 @@ export const CognitiveObservabilityOverview = ({ sessionsWithFailureRatio: totalSessions > 0 ? sessionsWithFailure / totalSessions : 0 }; - }, [sessions, threshold]); + }, [sessions]); const primary = theme.palette.vars.interactivePrimaryDefaultDefault; @@ -99,7 +90,7 @@ export const CognitiveObservabilityOverview = ({ - Select a failure to filter the sessions below. + Select a failure to filter the table below. } placement={'top'} @@ -115,7 +106,9 @@ export const CognitiveObservabilityOverview = ({ @@ -149,13 +142,13 @@ export const CognitiveObservabilityOverview = ({ gap: '8px' }} > - {stats.failures.map(({ name, count, ratio, remediations }) => { + {stats.failures.map(({ name, count, ratio }) => { const isSelected = selectedFailure === name; return ( onSelectFailure?.(isSelected ? null : name)} content={ @@ -163,7 +156,6 @@ export const CognitiveObservabilityOverview = ({ {`${count} / ${stats.totalSessions} sessions`} - } cardSx={{ diff --git a/ui/src/components/SessionsInsightsTable/SessionsInsightsTable.tsx b/ui/src/components/SessionsInsightsTable/SessionsInsightsTable.tsx index 2ea566e..d14bc07 100644 --- a/ui/src/components/SessionsInsightsTable/SessionsInsightsTable.tsx +++ b/ui/src/components/SessionsInsightsTable/SessionsInsightsTable.tsx @@ -20,8 +20,16 @@ import { import { useMemo, useState } from 'react'; import { Box, Stack, Typography } from '@mui/material'; import { format } from 'date-fns'; -import { SessionInsight } from '@/types/scope.type'; +import { SessionInsightRow } from '@/types/scope.type'; import { Tags } from '@/components'; +import { + getCognitiveFailureLabel, + getCognitiveFailureTaxonomy +} from '@/common/cognitiveFailureTaxonomy'; +import { + CONFIDENCE_COLORS, + CONFIDENCE_LABELS +} from '@/common/cognitiveFailureConfidence'; import { formatDurationMs } from '@/utils/stringUtils'; import { GLOBAL_BACKGROUND_COLOR } from '@/common/styles'; import { ExecutionTrace } from '@/components/ExecutionTrace'; @@ -29,12 +37,12 @@ import { colorTokens } from '@/theme/colors'; import { PATHS } from '@/routes/routes'; import { useNavigate, useParams } from 'react-router-dom'; -type MRT_ColumnDefList = TableProps['columns']; +type MRT_ColumnDefList = TableProps['columns']; interface SessionsInsightsTableProps { - data: SessionInsight[]; + data: SessionInsightRow[]; isLoading: boolean; - onSessionClick?: (session: SessionInsight) => void; + onSessionClick?: (session: SessionInsightRow) => void; } export const SessionsInsightsTable = ({ @@ -53,8 +61,8 @@ export const SessionsInsightsTable = ({ () => [ { accessorKey: 'sessionId', - header: 'Session ID', - size: 250, + header: 'ID', + size: 108, accessorFn: ({ sessionId }) => ( ( - - - {name} - - - ) + size: 280, + accessorFn: ({ name, cognitiveFailure }) => { + // A cognitive failure: a tag with its title, code and confidence + // level; the tooltip is the general description of the failure. + if (cognitiveFailure) { + const label = `${getCognitiveFailureLabel(cognitiveFailure.name)}: ${ + CONFIDENCE_LABELS[cognitiveFailure.confidence] ?? + cognitiveFailure.confidence + }`; + return ( + + + + {label} + + + + ); + } + + return ( + + + {name} + + + ); + } }, { accessorKey: 'description', header: 'Description', - size: 300, + size: 150, accessorFn: ({ description }) => ( { + // Cognitive failure rows have no priority. + if (!priority) return null; const colorMap: Record = { high: TagBackgroundColorVariants.AccentFWeak, medium: TagBackgroundColorVariants.AccentEWeak, @@ -159,16 +201,17 @@ export const SessionsInsightsTable = ({ accessorKey: 'duration', header: 'Duration', size: 120, - accessorFn: ({ duration }) => ( - - {formatDurationMs(duration)} - - ) + accessorFn: ({ duration, cognitiveFailure }) => + cognitiveFailure ? null : ( + + {formatDurationMs(duration)} + + ) }, { accessorKey: 'createdAt', header: 'Created', - size: 180, + size: 140, accessorFn: ({ createdAt }) => ( {createdAt ? format(new Date(createdAt), 'MMM d, yyyy HH:mm') : ''} @@ -210,33 +253,12 @@ export const SessionsInsightsTable = ({ muiTableBodyRowProps: ({ row }) => { return { sx: { - cursor: 'pointer', - backgroundColor: colorTokens.greyBackgroundDark, - '& > td': { - backgroundColor: `${colorTokens.greyBackgroundDark} !important` - } + cursor: 'pointer' }, onClick: () => { onSessionClick?.(row.original); } }; - }, - muiTablePaperProps: { - sx: { - padding: '12px', - backgroundColor: colorTokens.greyBackgroundDark, - elevation: 0 - } - }, - muiTableHeadCellProps: { - sx: { backgroundColor: colorTokens.greyBackgroundDark, color: '#ffffff' } - }, - muiTableBodyCellProps: { - sx: { - backgroundColor: colorTokens.greyBackgroundDark, - color: '#ffffff', - height: '40px' - } } }); diff --git a/ui/src/components/SessionsInsightsTable/index.tsx b/ui/src/components/SessionsInsightsTable/index.tsx index a154810..28f2b92 100644 --- a/ui/src/components/SessionsInsightsTable/index.tsx +++ b/ui/src/components/SessionsInsightsTable/index.tsx @@ -4,33 +4,56 @@ */ import { Spinner, Stack } from '@open-ui-kit/core'; -import { Typography } from '@mui/material'; -import { useSessionsInsights } from '@/api/scopeApi'; +import { Chip, Typography } from '@mui/material'; +import { useMemo, useState } from 'react'; +import { + useSessionsInsights, + useSessionsWithCognitiveObservability +} from '@/api/scopeApi'; +import { IntervalPicker } from '@/components/IntervalPicker'; +import { getCognitiveFailureLabel } from '@/common/cognitiveFailureTaxonomy'; +import { FailuresOverview } from './FailuresOverview'; import { SessionsInsightsTable } from './SessionsInsightsTable'; -import { SessionInsight } from '@/types/scope.type'; +import { toCognitiveFailureRows } from './utils'; +import { SessionInsight, SessionInsightRow } from '@/types/scope.type'; import { PATHS } from '@/routes/routes'; +import { useTimeRangeStore } from '@/store'; import { useNavigate, useParams } from 'react-router-dom'; -interface SessionsInsightsTableWrapperProps { - startDate: number; - endDate: number; - semanticGroupId?: string; -} - -export const SessionsInsightsTableWrapper = ({ - startDate, - endDate, - semanticGroupId -}: SessionsInsightsTableWrapperProps) => { +// The insights and the cognitive failures of all the sessions of the interval +// selected in the interval picker. +export const SessionsInsightsTableWrapper = () => { const { applicationId } = useParams(); + const { startDate, endDate, setStartDate, setEndDate } = useTimeRangeStore(); + const navigate = useNavigate(); const { data: sessionsInsights, isLoading, isError - } = useSessionsInsights(applicationId ?? '', startDate, endDate, semanticGroupId); - const navigate = useNavigate(); + } = useSessionsInsights(applicationId ?? '', startDate, endDate); + + // The cognitive failures only enrich the table: while they load, or if they + // fail to load, the insights are shown as they are. + const { data: cognitiveSessions } = useSessionsWithCognitiveObservability( + applicationId ?? '', + startDate, + endDate + ); + + const [selectedFailure, setSelectedFailure] = useState(null); + + // With a failure selected in the overview, only the rows of that failure are + // listed. + const rows = useMemo(() => { + const failureRows = toCognitiveFailureRows(cognitiveSessions?.sessions); + return selectedFailure + ? failureRows.filter((row) => row.name === selectedFailure) + : [...(sessionsInsights ?? []), ...failureRows]; + }, [sessionsInsights, cognitiveSessions?.sessions, selectedFailure]); + // Insights and cognitive failures both open the Analysis tab of the session, + // which shows the cognitive failures. const onSessionClick = (session: SessionInsight) => { navigate( PATHS.applicationCollectSessionTab @@ -39,31 +62,64 @@ export const SessionsInsightsTableWrapper = ({ .replace(':sessionTab', 'analysis') ); }; - if (isLoading) { - return ( - - - - ); - } - if (isError) { + const body = () => { + if (isLoading) { + return ( + + + + ); + } + + if (isError) { + return ( + + Error loading sessions insights + + ); + } + return ( - - Error loading sessions insights - + ); - } + }; return ( - + + + + + {selectedFailure && ( + + setSelectedFailure(null)} + color="primary" + variant="outlined" + /> + + )} + {body()} + + ); }; diff --git a/ui/src/components/SessionsInsightsTable/utils.ts b/ui/src/components/SessionsInsightsTable/utils.ts new file mode 100644 index 0000000..cbe0223 --- /dev/null +++ b/ui/src/components/SessionsInsightsTable/utils.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026 Cisco Systems, Inc. and its affiliates + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { + SessionInsightRow, + SessionWithCognitiveObservability +} from '@/types/scope.type'; +import { compareFailures } from '@/common/cognitiveFailureConfidence'; + +const toIsoDate = (unixSeconds: string): string => { + const date = new Date(Number(unixSeconds) * 1000); + return Number.isNaN(date.getTime()) ? '' : date.toISOString(); +}; + +// One row per cognitive failure, shaped like a session insight: the session id +// is repeated for every failure of a session, `name` is the failure code and +// `description` the detector's reasoning. +export const toCognitiveFailureRows = ( + sessions: SessionWithCognitiveObservability[] | undefined +): SessionInsightRow[] => + (sessions ?? []).flatMap((session) => { + const createdAt = toIsoDate(session.timestamp); + return [...(session.cognitiveFailures ?? [])] + .sort(compareFailures) + .map( + (failure): SessionInsightRow => ({ + sessionId: session.sessionId, + insightId: `${session.sessionId}:${failure.name}`, + name: failure.name, + description: failure.reasoning, + priority: '', + labels: [], + duration: 0, + startTime: createdAt, + endTime: createdAt, + createdAt, + targetNodeId: '', + cognitiveFailure: failure + }) + ); + }); diff --git a/ui/src/components/SessionsTable/SessionsTable.tsx b/ui/src/components/SessionsTable/SessionsTable.tsx index df8cd02..876ed4d 100644 --- a/ui/src/components/SessionsTable/SessionsTable.tsx +++ b/ui/src/components/SessionsTable/SessionsTable.tsx @@ -20,7 +20,7 @@ import { import { useMemo, useState } from 'react'; import { Box, Stack, Typography } from '@mui/material'; import { format } from 'date-fns'; -import { SessionWithStatefulEval } from '@/types/scope.type'; +import { CompletedSessions } from '@/types/scope.type'; import { CircleBadge, ExecutionTrace, SessionStatus, Tags } from '@/components'; import { Dayjs } from 'dayjs'; import { formatDurationMs } from '@/utils/stringUtils'; @@ -28,8 +28,10 @@ import { useTheme } from '@mui/material/styles'; import { getMetricPulseColor } from '@/utils/metrics'; import { GLOBAL_BACKGROUND_COLOR } from '@/common/styles'; import { useFeatureFlag } from '@/hooks/useFeatureFlag'; +import { StatusTag } from '../CognitionProtocols/primitives'; +import { shortProtocolName } from '../CognitionProtocolsSummary/utils'; -type MRT_ColumnDefList = TableProps['columns']; +type MRT_ColumnDefList = TableProps['columns']; // Columns whose data comes from stateful-eval (trajectory scoring). Dropped // when the stateful_eval feature flag is off. @@ -41,13 +43,15 @@ const STATEFUL_EVAL_COLUMN_KEYS = [ ]; interface SessionsTableProps { - data: SessionWithStatefulEval[]; + data: CompletedSessions[]; isLoading: boolean; isError?: boolean; onReload?: () => void; startDate: Dayjs | null; endDate: Dayjs | null; - onSessionClick?: (session: SessionWithStatefulEval) => void; + onSessionClick?: (session: CompletedSessions) => void; + // Click on a Cognition protocol tag (instead of the row). + onProtocolClick?: (session: CompletedSessions) => void; showDateInterval?: boolean; defaultHiddenColumns?: string[]; title?: string; @@ -76,6 +80,7 @@ export const SessionsTable = ({ startDate, endDate, onSessionClick, + onProtocolClick, showDateInterval = true, defaultHiddenColumns = [], title @@ -84,6 +89,9 @@ export const SessionsTable = ({ const [columnVisibility, setColumnVisibility] = useState( { id: false, + cost: false, + fatalErrors: false, + minorErrors: false, ...defaultHiddenColumns.reduce((acc: Record, column) => { acc[column] = false; return acc; @@ -99,8 +107,8 @@ export const SessionsTable = ({ () => [ { accessorKey: 'sessionId', - header: 'Session ID', - size: 124, + header: 'ID', + size: 98, accessorFn: ({ sessionId }) => { return ( @@ -121,7 +129,7 @@ export const SessionsTable = ({ { accessorKey: 'timestamp', header: 'Timestamp', - size: 115, + size: 132, accessorFn: ({ timestamp }) => { return ( @@ -140,23 +148,20 @@ export const SessionsTable = ({ return timestampA - timestampB; } }, + { + accessorKey: 'status', + header: 'Status', + size: 96, + accessorFn: ({ statefulEval }) => { + const status = statefulEval?.reasoningJson + ?.trajectory_score as number; + return ; + } + }, { accessorKey: 'agents', header: 'Agents', accessorFn: ({ agents }) => { - const overflowAgents = structuredClone(agents ?? []); - overflowAgents.shift(); - - const OverflowTooltip = ( - - {overflowAgents.map((agent) => ( - - {agent} - - ))} - - ); - return ( { + accessorKey: 'cognitionProtocols', + header: 'Cognition Protocols', + enableSorting: false, + size: 158, + accessorFn: (session) => { + const protocols = (session.cognitionProtocols ?? []).filter( + (protocol) => protocol.enabled || protocol.activated + ); + if (protocols.length === 0) return null; + return ( - - {tokens.toLocaleString('en-US')} - + { + // Opens the Cognition protocols tab of the session, not the + // row's default tab. + if (!onProtocolClick) return; + event.stopPropagation(); + onProtocolClick(session); + }} + > + {protocols.map((protocol) => ( + + ))} + ); } }, - { - accessorKey: 'status', - header: 'Status', - size: 70, - accessorFn: ({ statefulEval }) => { - const status = statefulEval?.reasoningJson - ?.trajectory_score as number; - return ; - } - }, { accessorKey: 'metricPulse', header: 'Metric Pulse', - size: 100, + size: 120, accessorFn: ({ statefulEval }) => { const intentRecognitionFatal = statefulEval?.reasoningJson?.fatal_failures?.find( @@ -398,6 +419,18 @@ export const SessionsTable = ({ return minorErrorsA - minorErrorsB; } }, + { + accessorKey: 'tokens', + header: 'Tokens', + size: 108, + accessorFn: ({ tokens }) => { + return ( + + {tokens.toLocaleString('en-US')} + + ); + } + }, { accessorKey: 'cost', header: 'Cost', @@ -438,7 +471,7 @@ export const SessionsTable = ({ // } // } ], - [] + [onProtocolClick] ); const visibleColumns = useMemo( diff --git a/ui/src/components/SessionsTable/index.tsx b/ui/src/components/SessionsTable/index.tsx index ead80a2..568bb3c 100644 --- a/ui/src/components/SessionsTable/index.tsx +++ b/ui/src/components/SessionsTable/index.tsx @@ -4,8 +4,12 @@ */ import { useMemo, useState } from 'react'; -import { Session, SessionWithStatefulEval } from '@/types/scope.type'; -import { useApplicationSessionsWithStatefulEval } from '@/api/scopeApi'; +import { Session, CompletedSessions } from '@/types/scope.type'; +import { + useApplicationSessionsWithStatefulEval, + useSessionsWithCognitionProtocols +} from '@/api/scopeApi'; +import { hasCognitionProtocol } from '../CognitionProtocolsSummary/utils'; import { SessionsTable } from './SessionsTable.tsx'; import { Stack, SxProps, useTheme } from '@mui/material'; import { useDebouncedValue } from '@/utils'; @@ -18,6 +22,8 @@ interface SessionsTablePropsWrapper { endDate: number | null; enableSearch?: boolean; onSessionClick?: (session: Session) => void; + // Click on a Cognition protocol tag of a session. + onProtocolClick?: (session: Session) => void; defaultHiddenColumns?: string[]; filteredSessionIds?: string[]; title?: string; @@ -52,6 +58,7 @@ export const SessionsTableWrapper = ({ endDate, enableSearch = false, onSessionClick, + onProtocolClick, defaultHiddenColumns = [], title, filteredSessionIds, @@ -76,12 +83,39 @@ export const SessionsTableWrapper = ({ semanticGroupId ); - const sessionsWithStatefulEval: SessionWithStatefulEval[] = useMemo(() => { + // Cognition (L9) protocols of the sessions of the same interval. They only + // enrich the table: a failure here must not prevent the sessions from showing. + const { data: cognitionProtocolsData } = useSessionsWithCognitionProtocols( + applicationId ?? applicationIdFromParams ?? '', + debouncedStartDate ?? undefined, + debouncedEndDate ?? undefined + ); + + // Session id -> protocols, only for the sessions with a protocol enabled or activated. + const cognitionProtocolsBySessionId = useMemo(() => { + const protocolSessions = (cognitionProtocolsData?.sessions ?? []).filter( + hasCognitionProtocol + ); + return new Map( + protocolSessions.map((session) => [ + session.sessionId, + session.cognitionProtocols + ]) + ); + }, [cognitionProtocolsData?.sessions]); + + const sessionsWithStatefulEval: CompletedSessions[] = useMemo(() => { if (!sessionsData?.sessionList) return []; - const sessions = sessionsData.sessionList.map((session) => { + const sessions = sessionsData.sessionList.map((rawSession) => { + const session = { + ...rawSession, + cognitionProtocols: cognitionProtocolsBySessionId.get( + rawSession.sessionId + ) + }; const statefulEval = (session as any).statefulEval; - if (!statefulEval) return session as SessionWithStatefulEval; + if (!statefulEval) return session as CompletedSessions; let parsedValue: any = null; try { @@ -113,17 +147,17 @@ export const SessionsTableWrapper = ({ value: parsedValue?.value, reasoningJson } - } as SessionWithStatefulEval; + } as CompletedSessions; }); - const sessionsWithStatefulEvalFiltered: SessionWithStatefulEval[] = + const sessionsWithStatefulEvalFiltered: CompletedSessions[] = filteredSessionIds ? sessions.filter((session) => filteredSessionIds.includes(session.sessionId) ) : sessions; - const sessionsWithStatefulEvalFilteredByAgent: SessionWithStatefulEval[] = + const sessionsWithStatefulEvalFilteredByAgent: CompletedSessions[] = agentId ? (sessionsWithStatefulEvalFiltered.filter((session) => session.agents?.some((agent) => agent === agentId) @@ -131,7 +165,12 @@ export const SessionsTableWrapper = ({ : sessionsWithStatefulEvalFiltered; return sessionsWithStatefulEvalFilteredByAgent; - }, [sessionsData?.sessionList, filteredSessionIds, agentId]); + }, [ + sessionsData?.sessionList, + cognitionProtocolsBySessionId, + filteredSessionIds, + agentId + ]); return ( @@ -161,6 +200,7 @@ export const SessionsTableWrapper = ({ startDate={startDate ? dayjs.unix(startDate) : null} endDate={endDate ? dayjs.unix(endDate) : null} onSessionClick={onSessionClick} + onProtocolClick={onProtocolClick} // onReload={() => executeQuery()} defaultHiddenColumns={defaultHiddenColumns} title={title} diff --git a/ui/src/components/StaticTopology/index.tsx b/ui/src/components/StaticTopology/index.tsx index a830979..8785f40 100644 --- a/ui/src/components/StaticTopology/index.tsx +++ b/ui/src/components/StaticTopology/index.tsx @@ -20,6 +20,9 @@ import { isToolDataArray } from '@/utils'; const FILTERED_NODE_IDS: string[] = ['__start__', '__end__', 'finalize']; +const NODE_MAX_WIDTH = 200; +const NODE_LABEL_HORIZONTAL_PADDING = 16; + const transformTopologyToGraphData = ( topology: StaticTopologyType, agentToolsData?: ApplicationAgentToolsResponse @@ -320,10 +323,17 @@ export const StaticTopology = () => { } }} */ + showNodeLabelTooltip nodeProps={{ style: { + size: [NODE_MAX_WIDTH, 60], fill: getNodeFill, - stroke: getNodeStroke + stroke: getNodeStroke, + // Truncate long labels with an ellipsis on a single line + labelWordWrap: true, + labelMaxLines: 1, + labelMaxWidth: NODE_MAX_WIDTH - NODE_LABEL_HORIZONTAL_PADDING * 2, + labelPadding: [0, NODE_LABEL_HORIZONTAL_PADDING] } }} edgeProps={{ diff --git a/ui/src/components/index.ts b/ui/src/components/index.ts index 0e5edd5..cd893a3 100644 --- a/ui/src/components/index.ts +++ b/ui/src/components/index.ts @@ -11,7 +11,7 @@ import { SessionsTableWrapper } from './SessionsTable'; import { LiveSessionsTableWrapper } from './LiveSessionsTable'; import SessionDetails from './SessionDetails'; import { OutlierMetricsTableWrapper as OutlierMetricsTable } from './OutlierMetricsTable'; -import { SemanticGroupsInsightsTableWrapper as SemanticGroupsInsightsTable } from './SemanticGroupsInsightsTable'; +import { SessionsInsightsTableWrapper as SessionsInsightsTable } from './SessionsInsightsTable'; export * from './CustomTooltip'; export * from './DateTimePicker'; @@ -53,10 +53,10 @@ export * from './StaticTopology'; export * from './GenericGraph'; export * from './LiveTopology'; export * from './LiveSessionDetails'; -export * from './CognitiveObservability'; -export * from './CognitiveFailureDescription'; +export * from './CognitiveFailuresBanner'; +export * from './CognitiveFailuresList'; export * from './ScoreSeverityBadgeLabel'; -export * from './RemediationTags'; +export * from './CognitionProtocolsSummary'; export { ApplicationCard, Tags }; export { PageWithTitle, SpanDrawer }; @@ -64,4 +64,4 @@ export { SessionsTableWrapper as SessionsTable }; export { LiveSessionsTableWrapper as LiveSessionsTable }; export { SessionDetails }; export { OutlierMetricsTable }; -export { SemanticGroupsInsightsTable }; +export { SessionsInsightsTable }; diff --git a/ui/src/pages/ApplicationDetails/AnalyzeTab/index.tsx b/ui/src/pages/ApplicationDetails/AnalyzeTab/index.tsx index 7162606..9e66831 100644 --- a/ui/src/pages/ApplicationDetails/AnalyzeTab/index.tsx +++ b/ui/src/pages/ApplicationDetails/AnalyzeTab/index.tsx @@ -10,9 +10,6 @@ import TrackChangesIcon from '@mui/icons-material/TrackChanges'; import PsychologyOutlinedIcon from '@mui/icons-material/PsychologyOutlined'; import PowerOutlinedIcon from '@mui/icons-material/PowerOutlined'; import ReportProblemOutlinedIcon from '@mui/icons-material/ReportProblemOutlined'; -import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined'; -import CompareArrowsOutlinedIcon from '@mui/icons-material/CompareArrowsOutlined'; - import SemanticGrouping from './SemanticGrouping'; import { useParams } from 'react-router-dom'; import { TopicHierarchy } from './TopicHierarchy'; @@ -20,19 +17,15 @@ import { SemanticGroupDetails } from '@/components/SemanticGroupDetails/Semantic import { ResourcesUsage } from './ResourcesUsage'; import { ApplicationReasoningPathGraph, - SemanticGroupsInsightsTable + SessionsInsightsTable } from '@/components'; -import CognitiveObservability from '@/components/CognitiveObservability'; -import CognitionProtocolsComparison from '@/components/CognitionProtocolsComparison'; import { FeatureFlagKey, getFeatureFlags } from '@/config/featureFlags'; const ANALYZE_TABS = [ 'What agents are working on', 'How they are failing', 'What resources they are using', - 'How they are reasoning', - 'Cognitive observability', - 'Cognition protocols' + 'How they are reasoning' ] as const; type AnalyzeTabName = (typeof ANALYZE_TABS)[number]; @@ -43,9 +36,7 @@ const ANALYZE_TAB_FLAGS: Partial> = { 'What agents are working on': 'semantic_groups', 'How they are failing': 'insights', 'What resources they are using': 'waste_estimation', - 'How they are reasoning': 'neurosymbolic_eval', - 'Cognitive observability': 'cognitive_observability', - 'Cognition protocols': 'cognition_protocols' + 'How they are reasoning': 'neurosymbolic_eval' }; const ANALYZE_TAB_META: Record< @@ -67,14 +58,6 @@ const ANALYZE_TAB_META: Record< 'How they are failing': { label: 'Failure modes', icon: ReportProblemOutlinedIcon - }, - 'Cognitive observability': { - label: 'Cognitive observability', - icon: VisibilityOutlinedIcon - }, - 'Cognition protocols': { - label: 'Cognition protocols', - icon: CompareArrowsOutlinedIcon } }; @@ -90,7 +73,7 @@ const ANALYZE_TAB_SECTIONS: { title: string; tabs: AnalyzeTabName[] }[] = [ }, { title: 'Performance', - tabs: ['How they are failing', 'Cognitive observability', 'Cognition protocols'] + tabs: ['How they are failing'] } ]; @@ -236,15 +219,11 @@ const AnalyzeTabContent = memo(({ tab }: { tab: AnalyzeTabName }) => { ); case 'How they are failing': - return ; + return ; case 'What resources they are using': return ; case 'How they are reasoning': return ; - case 'Cognitive observability': - return ; - case 'Cognition protocols': - return ; } }); diff --git a/ui/src/pages/ApplicationDetails/CollectTab/index.tsx b/ui/src/pages/ApplicationDetails/CollectTab/index.tsx index 1e6fc2e..6b8f953 100644 --- a/ui/src/pages/ApplicationDetails/CollectTab/index.tsx +++ b/ui/src/pages/ApplicationDetails/CollectTab/index.tsx @@ -9,7 +9,8 @@ import { SessionDetails, SessionsTable, WidgetCard, - SessionsInsightsList + SessionsInsightsList, + CognitionProtocolsSummary } from '@/components'; import LiveSessionDetails from '@/components/LiveSessionDetails'; import { Spinner, Tabs, Tab } from '@open-ui-kit/core'; @@ -74,6 +75,16 @@ const CollectTab = () => { ); }; + // The Cognition protocol tags open the Cognition protocols tab of the session. + const handleProtocolClick = (session: Session) => { + navigate( + PATHS.applicationCollectSessionTab + .replace(':applicationId', applicationId ?? '') + .replace(':sessionId', encodeURIComponent(session.sessionId)) + .replace(':sessionTab', 'cognition-protocols') + ); + }; + const handleLiveSessionClick = (session: LiveSession) => { navigate( PATHS.applicationCollectLiveSession @@ -209,7 +220,7 @@ const CollectTab = () => { ) : ( <> - + { } /> + @@ -243,12 +259,8 @@ const CollectTab = () => { endDate={endDate} enableSearch={true} onSessionClick={handleSessionClick} - defaultHiddenColumns={[ - 'agents', - 'llms', - 'tokens', - 'duration' - ]} + onProtocolClick={handleProtocolClick} + defaultHiddenColumns={['agents', 'llms', 'duration']} showDateInterval={false} title={'Session List'} /> diff --git a/ui/src/types/scope.type.ts b/ui/src/types/scope.type.ts index 3581a9e..5beab3c 100644 --- a/ui/src/types/scope.type.ts +++ b/ui/src/types/scope.type.ts @@ -30,6 +30,12 @@ export interface SessionWithStatefulEval extends Session { | null; } +export interface CompletedSessions extends SessionWithStatefulEval { + // L9 / Cognition protocols of the session. Absent or empty: no protocol + // was enabled in the session. + cognitionProtocols?: SessionCognitionProtocolStatus[] | null; +} + export interface WastefulSession { sessionId: string; timestamp: string; @@ -685,20 +691,6 @@ export interface WastedResourcesFeatureScore { score: number; } -export interface SemanticGroupsInsight { - groupName: string; - groupSummary: string; - groupId: string; - sessionIds: string[]; - createdAt: string; - description: string; - insightId: string; - name: string; - labels: string[]; - priority: string; - targetNodeId: string; -} - export interface SessionInsight { duration: number; endTime: string; @@ -713,6 +705,13 @@ export interface SessionInsight { targetNodeId: string; } +// A row of the sessions insights table: a session insight, or a cognitive +// failure of a session shaped like one (`name` is the failure code and +// `description` its reasoning). `cognitiveFailure` is only set for the latter. +export interface SessionInsightRow extends SessionInsight { + cognitiveFailure?: CognitiveFailure | null; +} + export interface ReasoningPathResponse { sessionId: string; model: { modelId: string; version: string; modelName: string } | null; @@ -938,11 +937,18 @@ export interface CognitiveObservabilityMetric { value: SingleValueData; } +// How sure the cognitive-failure detector is about a failure. +export type CognitiveFailureConfidence = 'low' | 'medium' | 'high'; + export interface CognitiveFailure { + // Taxonomy code of the failure, e.g. 'IR-1', 'DC-4', 'CC-3' (not a + // human-readable name; see COGNITIVE_FAILURE_TAXONOMY for the title). name: string; - confidence: number; - description?: string | null; - remediations?: string[] | null; + // A failure detected at several calls of a session is reported once, with + // its highest confidence. + confidence: CognitiveFailureConfidence; + // The detector's reasoning for why the failure was flagged in this session. + reasoning: string; } // ---------------------------------------------------------------------------