Classify observability errors

src/MCP-OpenTelemetry/MCPOpenTelemetryObservabilityBackend.class.st

@@ -119,25 +119,30 @@ MCPOpenTelemetryObservabilityBackend >> enrichSpan: span withRecord: aRecord att
 		attributeAt: 'mcp.output.tokens.approx' put: aRecord approximateOutputTokenCount;
 		attributeAt: 'mcp.output_budget.exceeded' put: aRecord outputBudgetExceeded.
 	aRecord outputBudgetExceeded ifTrue: [ span addEventNamed: 'mcp.output_budget_exceeded' attributes: attributes ].
-	anError ifNotNil: [
-			span
-				attributeAt: 'error.type' put: anError class name asString;
-				setStatus: OTSpan error description: anError description;
-				recordException: anError ].
+	aRecord errorKind ifNotEmpty: [ :kind | span attributeAt: 'mcp.error.kind' put: kind ].
+	aRecord errorCode ifNotEmpty: [ :code | span attributeAt: 'mcp.error.code' put: code ].
+	aRecord errorClass ifNotEmpty: [ :className | span attributeAt: 'error.type' put: className ].
+	aRecord causeClass ifNotEmpty: [ :className | span attributeAt: 'mcp.error.cause_class' put: className ].
+	aRecord schemaViolationCount > 0 ifTrue: [
+		span attributeAt: 'mcp.error.schema.violation_count' put: aRecord schemaViolationCount ].
+	aRecord status = 'error' ifTrue: [
+			span setStatus: OTSpan error description: aRecord displaySummary.
+			anError ifNotNil: [ span recordException: anError ] ].
 	aRecord status = 'ok' ifTrue: [ span setStatus: OTSpan ok description: '' ]
 ]
 
 { #category : 'private - metrics' }
 MCPOpenTelemetryObservabilityBackend >> errorAttributesForRecord: aRecord baseAttributes: attributes [
 
-	^ attributes copyWith: 'error.type' -> aRecord errorClass
-]
-
-{ #category : 'private' }
-MCPOpenTelemetryObservabilityBackend >> errorClassNameFor: anError [
-
-	anError ifNil: [ ^ '' ].
-	^ anError class name asString
+	| errorAttributes |
+	errorAttributes := OrderedCollection withAll: attributes.
+	aRecord errorClass ifNotEmpty: [ :className | errorAttributes add: 'error.type' -> className ].
+	aRecord errorKind ifNotEmpty: [ :kind | errorAttributes add: 'mcp.error.kind' -> kind ].
+	aRecord errorCode ifNotEmpty: [ :code | errorAttributes add: 'mcp.error.code' -> code ].
+	aRecord causeClass ifNotEmpty: [ :className | errorAttributes add: 'mcp.error.cause_class' -> className ].
+	aRecord schemaViolationCount > 0 ifTrue: [
+		errorAttributes add: 'mcp.error.schema.violation_count' -> aRecord schemaViolationCount ].
+	^ errorAttributes asArray
 ]
 
 { #category : 'private - instruments' }
@@ -155,7 +160,7 @@ MCPOpenTelemetryObservabilityBackend >> finishRecord: aRecord result: aToolResul
 	duration := (finishedAt - aRecord startedAt) asMilliSeconds.
 	statusString := self statusForResult: aToolResult error: anError.
 	aRecord finishAt: finishedAt durationMilliseconds: duration status: statusString.
-	aRecord errorClass: (self errorClassNameFor: anError).
+	self recordErrorDetailsFor: aRecord result: aToolResult error: anError.
 	self recordOutputBudgetFor: aRecord result: aToolResult error: anError.
 	aRecord inputObject: nil.
 	self updateEventKindsForRecord: aRecord

src/MCP-Tests-Resources/MCPExternalTestTool.class.st

@@ -50,6 +50,12 @@ MCPExternalTestTool >> executeWithRequest: aRequest [
 	^ self successResultText: 'External test tool executed.' data: Dictionary new
 ]
 
+{ #category : 'testing' }
+MCPExternalTestTool >> shouldRecordObservability [
+
+	^ false
+]
+
 { #category : 'metadata' }
 MCPExternalTestTool >> title [
 

src/MCP-Tests-Resources/MCPHaltingTestTool.class.st

@@ -56,6 +56,12 @@ MCPHaltingTestTool >> groupName [
 	^ 'plugins'
 ]
 
+{ #category : 'testing' }
+MCPHaltingTestTool >> shouldRecordObservability [
+
+	^ false
+]
+
 { #category : 'metadata' }
 MCPHaltingTestTool >> title [
 

src/MCP-UI-Tests/MCPLocalObservabilityBackendTest.class.st

@@ -152,6 +152,21 @@ MCPLocalObservabilityBackendTest >> testMCPRecordsToolCallsWhenObservabilityIsEn
 		self assert: mcp observability metricsFileReference exists ] ensure: [ directory exists ifTrue: [ directory deleteAll ] ]
 ]
 
+{ #category : 'tests' }
+MCPLocalObservabilityBackendTest >> testMCPTestToolsDoNotRecordObservability [
+
+	| mcp result |
+	mcp := MCP new.
+	mcp observabilityEnabled: true.
+
+	result := mcp executeTool: MCPExternalTestTool new named: MCPExternalTestTool toolName withParams: Dictionary new.
+
+	self deny: (result at: #isError ifAbsent: [ false ]).
+	self assert: mcp observability totalCallCount equals: 0.
+	self assert: mcp observabilityMetrics isEmpty.
+	self assert: mcp observability recentCallRecords isEmpty
+]
+
 { #category : 'tests' }
 MCPLocalObservabilityBackendTest >> testOutlierExplanationUsesPriorAverage [
 
@@ -224,6 +239,54 @@ MCPLocalObservabilityBackendTest >> testRecordsOutputBudgetMetadata [
 	self assert: (statsData at: #lastCallAt) isNotEmpty
 ]
 
+{ #category : 'tests' }
+MCPLocalObservabilityBackendTest >> testRecordsSchemaViolationErrorDetails [
+
+	| error record state stats |
+	state := MCPLocalObservabilityBackend new.
+	state enable.
+	error := MCPInvalidToolInput new
+		         toolName: 'class_search';
+		         violations: { (MCPInvalidToolInput missingRequiredArgumentViolationNamed: 'name') };
+		         messageText: 'Tool input does not match the input schema.';
+		         yourself.
+	record := state recordToolCallStart: 'class_search'.
+	state recordToolCallFinish: record result: nil error: error.
+	stats := state metrics first.
+
+	self assert: record errorKind equals: 'input_schema'.
+	self assert: record errorClass equals: 'MCPInvalidToolInput'.
+	self assert: record schemaViolationCount equals: 1.
+	self assert: (record schemaViolationKeywords includes: 'required').
+	self assert: (stats errorCountsByKind at: 'input_schema') equals: 1.
+	self assert: (stats errorCountsByClass at: 'MCPInvalidToolInput') equals: 1
+]
+
+{ #category : 'tests' }
+MCPLocalObservabilityBackendTest >> testRecordsStructuredToolErrorDetails [
+
+	| details record result state stats |
+	state := MCPLocalObservabilityBackend new.
+	state enable.
+	details := {
+		           (#errorClass -> 'MCPCommandError').
+		           (#errorCode -> 'MethodNotFound').
+		           (#causeClass -> 'KeyNotFound').
+		           (#message -> 'No method found.') } asDictionary.
+	record := state recordToolCallStart: 'method_get'.
+	result := MCPToolGetMethod new errorResultText: 'No method found.' details: details.
+	state recordToolCallFinish: record result: result error: nil.
+	stats := state metrics first.
+
+	self assert: record status equals: 'error'.
+	self assert: record errorKind equals: 'domain'.
+	self assert: record errorClass equals: 'MCPCommandError'.
+	self assert: record errorCode equals: 'MethodNotFound'.
+	self assert: record causeClass equals: 'KeyNotFound'.
+	self assert: (stats errorCountsByKind at: 'domain') equals: 1.
+	self assert: (stats errorCountsByCode at: 'MethodNotFound') equals: 1
+]
+
 { #category : 'tests' }
 MCPLocalObservabilityBackendTest >> testRecordsSuccessfulCallMetricsAndLogs [
 

src/MCP/MCP.class.st

@@ -147,11 +147,13 @@ MCP >> executeTool: tool named: toolName withParams: arguments [
 MCP >> executeTool: tool named: toolName withParams: arguments dispatchPath: dispatchPath [
 
 	| observabilityRecord request result toolResult |
-	observabilityRecord := self observability
-		                       recordToolCallStart: toolName
-		                       input: arguments
-		                       dispatchPath: dispatchPath
-		                       context: Dictionary new.
+	observabilityRecord := nil.
+	tool shouldRecordObservability ifTrue: [
+			observabilityRecord := self observability
+				                       recordToolCallStart: toolName
+				                       input: arguments
+				                       dispatchPath: dispatchPath
+				                       context: Dictionary new ].
 	^ [
 		  request := tool requestFromToolCallArguments: arguments.
 		  result := tool executeWithRequest: request.

src/MCP/MCPLocalObservabilityBackend.class.st

@@ -583,7 +583,7 @@ MCPLocalObservabilityBackend >> recordToolCallFinish: aRecord result: aToolResul
 	stats := self statsForToolNamed: aRecord toolName.
 	explanation := self outlierExplanationForDurationMilliseconds: duration stats: stats.
 	aRecord finishAt: finishTimestamp durationMilliseconds: duration status: statusString.
-	aRecord errorClass: (self errorClassNameFor: anError).
+	self recordErrorDetailsFor: aRecord result: aToolResult error: anError.
 	aRecord summary: (self summaryForResult: aToolResult).
 	aRecord outlierExplanation: explanation.
 	self recordOutputBudgetFor: aRecord result: aToolResult error: anError.
@@ -763,6 +763,11 @@ MCPLocalObservabilityBackend >> traceRecordDictionaryFor: aRecord [
 	dictionary at: #durationMilliseconds put: aRecord durationMilliseconds.
 	dictionary at: #status put: aRecord status.
 	aRecord errorClass ifNotEmpty: [ :errorClass | dictionary at: #errorClass put: errorClass ].
+	aRecord errorKind ifNotEmpty: [ :errorKind | dictionary at: #errorKind put: errorKind ].
+	aRecord errorCode ifNotEmpty: [ :errorCode | dictionary at: #errorCode put: errorCode ].
+	aRecord causeClass ifNotEmpty: [ :causeClass | dictionary at: #causeClass put: causeClass ].
+	aRecord schemaViolationKeywords ifNotEmpty: [ :keywords | dictionary at: #schemaViolationKeywords put: keywords ].
+	aRecord schemaViolationCount > 0 ifTrue: [ dictionary at: #schemaViolationCount put: aRecord schemaViolationCount ].
 	aRecord inputByteCount > 0 ifTrue: [ dictionary at: #inputBytes put: aRecord inputByteCount ].
 	aRecord outputByteCount > 0 ifTrue: [
 			dictionary at: #outputBytes put: aRecord outputByteCount.

src/MCP/MCPObservabilityBackend.class.st

@@ -47,6 +47,44 @@ MCPObservabilityBackend >> enabled: aBoolean [
 	self subclassResponsibility
 ]
 
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> errorDetailsForResult: aToolResult error: anError [
+
+	| details resultDetails |
+	details := Dictionary new.
+	anError ifNotNil: [
+			details at: #errorClass put: anError class name asString.
+			details at: #message put: (anError messageText ifNil: [ anError asString ]).
+			anError mcpAddToolErrorDetailsTo: details ].
+	resultDetails := self errorDetailsFromToolResult: aToolResult.
+	resultDetails keysAndValuesDo: [ :key :value | details at: key asSymbol put: value ].
+	^ details
+]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> errorDetailsFromToolResult: aToolResult [
+
+	| errorDetails structuredContent |
+	aToolResult ifNil: [ ^ Dictionary new ].
+	structuredContent := aToolResult structuredContent.
+	structuredContent ifNil: [ ^ Dictionary new ].
+	structuredContent isDictionary ifFalse: [ ^ Dictionary new ].
+	errorDetails := self valueAtKey: #error in: structuredContent ifAbsent: [ nil ].
+	errorDetails isDictionary ifFalse: [ ^ Dictionary new ].
+	^ errorDetails
+]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> errorKindForDetails: details error: anError violations: violations [
+
+	violations ifNotEmpty: [ ^ 'input_schema' ].
+	(self stringValueAtKey: #errorClass in: details) = 'MCPInvalidToolInput' ifTrue: [ ^ 'input_schema' ].
+	(self stringValueAtKey: #errorCode in: details) ifNotEmpty: [ ^ 'domain' ].
+	anError ifNil: [ ^ 'tool_result' ].
+	anError mcpIsJRPCError ifTrue: [ ^ 'json_rpc' ].
+	^ 'exception'
+]
+
 { #category : 'accessing' }
 MCPObservabilityBackend >> exportDirectory [
 
@@ -101,6 +139,21 @@ MCPObservabilityBackend >> recentCallRecords [
 	^ #(  )
 ]
 
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> recordErrorDetailsFor: aRecord result: aToolResult error: anError [
+
+	| details violations |
+	aRecord status = 'error' ifFalse: [ ^ self ].
+	details := self errorDetailsForResult: aToolResult error: anError.
+	violations := self violationsFromDetails: details.
+	aRecord errorClass: (self stringValueAtKey: #errorClass in: details).
+	aRecord errorCode: (self stringValueAtKey: #errorCode in: details).
+	aRecord causeClass: (self stringValueAtKey: #causeClass in: details).
+	aRecord schemaViolationKeywords: (self violationKeywordsFor: violations).
+	aRecord schemaViolationCount: violations size.
+	aRecord errorKind: (self errorKindForDetails: details error: anError violations: violations)
+]
+
 { #category : 'recording' }
 MCPObservabilityBackend >> recordSessionEndFor: anMCP [
 
@@ -165,6 +218,15 @@ MCPObservabilityBackend >> statusText [
 	self subclassResponsibility
 ]
 
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> stringValueAtKey: aKey in: aDictionary [
+
+	| value |
+	value := self valueAtKey: aKey in: aDictionary ifAbsent: [ '' ].
+	value ifNil: [ ^ '' ].
+	^ value asString
+]
+
 { #category : 'accessing' }
 MCPObservabilityBackend >> totalCallCount [
 
@@ -188,3 +250,32 @@ MCPObservabilityBackend >> traceRecords [
 
 	^ #(  )
 ]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> valueAtKey: aKey in: aDictionary ifAbsent: absentBlock [
+
+	(aDictionary includesKey: aKey) ifTrue: [ ^ aDictionary at: aKey ].
+	^ aDictionary at: aKey asString ifAbsent: absentBlock
+]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> violationKeywordFor: aViolation [
+
+	aViolation isDictionary ifTrue: [ ^ self stringValueAtKey: #keyword in: aViolation ].
+	^ ''
+]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> violationKeywordsFor: violations [
+
+	^ ((violations collect: [ :each | self violationKeywordFor: each ]) reject: [ :each | each isEmpty ]) asSet asArray
+]
+
+{ #category : 'private - errors' }
+MCPObservabilityBackend >> violationsFromDetails: details [
+
+	| violations |
+	violations := self valueAtKey: #violations in: details ifAbsent: [ #(  ) ].
+	violations ifNil: [ ^ #(  ) ].
+	^ violations asArray
+]

src/MCP/MCPTool.class.st

@@ -533,6 +533,12 @@ MCPTool >> shouldAdvertiseStringArrayItemDescription: anItemDescription [
 	^ true
 ]
 
+{ #category : 'testing' }
+MCPTool >> shouldRecordObservability [
+
+	^ true
+]
+
 { #category : 'testing' }
 MCPTool >> shouldSaveImageAfterSuccessfulExecution [