feat(mcp): capture agents feedback

.sampo/changesets/mcp-collect-feedback.md

@@ -0,0 +1,6 @@
+---
+pypi/posthog: minor
+---
+
+Add an opt-in `collect_feedback` option to MCP analytics. It injects a `send_feedback` virtual tool and captures every call as a `$mcp_feedback` event, so agents can report a missing capability, a tool problem, or praise.
+The option supports a custom tool name and description, host-declared extra schema fields, and an `on_feedback` handler that routes reports to a real backend. `PostHogMCP` gains the same option plus `capture_feedback` for custom dispatchers.

posthog/mcp/README.md

@@ -105,6 +105,85 @@ Model injection copies tool objects instead of changing their original schemas.
 Always advertise the returned list and pass the original application tool to
 `prepare_tool_call()`. Repeatedly preparing the original list preserves ownership.
 
+## Collect agent feedback
+
+Feedback collection is off by default. Enable it to advertise a `send_feedback`
+virtual tool. Agents use it to report a missing capability (the priority
+category), a tool that failed or confused them, or praise:
+
+```python
+from posthog.mcp import MCPAnalyticsOptions, instrument
+
+analytics = instrument(
+    server,
+    posthog,
+    MCPAnalyticsOptions(collect_feedback=True),
+)
+```
+
+Each call emits one `$mcp_feedback` event (never a `$mcp_tool_call`) with
+`$mcp_feedback_type`, `$mcp_feedback_summary`, `$mcp_feedback_details`,
+`$mcp_feedback_friction_points`, `$mcp_feedback_suggested_improvement`,
+`$mcp_feedback_tool`, `$mcp_feedback_sentiment`, and
+`$mcp_feedback_task_completed`. The summary and details also map to
+`$mcp_intent`. Free-text fields are sanitized, PII-redacted, and length-bounded;
+the raw arguments are never captured. An invalid `feedback_type` falls back to
+`other`. The agent receives an honest acknowledgement: the call records feedback
+and adds no tools.
+
+The tool covers what `report_missing` covers (as feedback_type
+`missing_capability`), so new integrations should enable only one of the two.
+
+If a real tool already uses the name, the SDK logs a warning, does not inject
+the virtual tool, and never intercepts the real tool.
+
+Use the object form to rename the tool, declare host-specific fields, or route
+reports to a real backend:
+
+```python
+from posthog.mcp import CollectFeedbackOptions, MCPAnalyticsOptions
+
+options = MCPAnalyticsOptions(
+    collect_feedback=CollectFeedbackOptions(
+        extra_properties={
+            "product_area": {
+                "type": "string",
+                "description": "The product the feedback is about.",
+            },
+        },
+        extra_required=["product_area"],
+        on_feedback=lambda report: feedback_backend.record(report),
+    ),
+)
+```
+
+Declared extras merge into the advertised schema and are captured as
+`$mcp_feedback_<key>`. Arguments the agent invents are never captured; the
+handler reads them from `report.raw`. A key that collides with a core field
+raises at configuration time. `on_feedback` may be sync or async; a returned
+non-blank string replaces the default reply, and a raised handler is logged and
+falls back to it. The event is captured either way.
+
+For a custom dispatcher, use the same option on `PostHogMCP`:
+
+```python
+from posthog.mcp import PostHogMCP, send_feedback_result
+
+posthog = PostHogMCP("phc_...", collect_feedback=True)
+
+# tools/list handler
+tools = posthog.prepare_tool_list(server_tools, collect_feedback=True)
+
+# tools/call dispatcher
+call = posthog.prepare_tool_call(tool_name, raw_args)
+if call.is_feedback:
+    posthog.capture_feedback(report=call.feedback_report)  # emits $mcp_feedback
+    return send_feedback_result()  # replies to the agent and stops dispatch
+```
+
+`on_feedback` is ignored on this path — the dispatcher routes reports itself via
+`call.feedback_report`.
+
 ## Stateless / multi-pod servers
 
 A stateless MCP server issues no session id, so `$session_id` fragments across pods

posthog/mcp/__init__.py

@@ -69,9 +69,17 @@
     get_mcp_session,
 )
 from ._sink import McpEventSink
+from .feedback import (
+    SEND_FEEDBACK_TOOL_NAME,
+    get_feedback_tool_descriptor,
+    resolve_collect_feedback_options,
+    send_feedback_result,
+)
 from .tools import get_more_tools_result
 from .types import (
     CaptureEventData,
+    CollectFeedbackOptions,
+    FeedbackReport,
     MCPAnalyticsContextOptions,
     MCPAnalyticsModelOptions,
     MCPAnalyticsModelSource,
@@ -91,8 +99,12 @@
     "MCPAnalyticsModelSource",
     "UserIdentity",
     "CaptureEventData",
+    "CollectFeedbackOptions",
+    "FeedbackReport",
     "PreparedToolCall",
     "get_more_tools_result",
+    "send_feedback_result",
+    "SEND_FEEDBACK_TOOL_NAME",
     # Read HTTP headers inside identify / intent_fallback /
     # event_properties callbacks on either SDK major: the per-request context
     # arrives as extra["ctx"] and its shape differs between them.
@@ -254,6 +266,13 @@ def instrument(
         )
     _warn_if_unsupported_mcp_version()
 
+    # Fail fast on a `collect_feedback` config error (reserved extra key,
+    # undeclared extra_required) — before the try below, so it raises instead of
+    # degrading to the no-op handle and first surfacing at tools/list time.
+    feedback_options = resolve_collect_feedback_options(opts.collect_feedback)
+    if feedback_options is not None:
+        get_feedback_tool_descriptor(feedback_options)
+
     key = _canonical_server(server)
 
     try:

posthog/mcp/_conversation_id.py

@@ -79,15 +79,21 @@ def resolve_conversation_id(
     args: Any,
     tool_name: Optional[str],
     missing_capability_tool_name: str,
+    feedback_tool_name: Optional[str] = None,
 ) -> Tuple[Optional[str], bool]:
-    """Return ``(conversation_id, minted)``. Disabled or get_more_tools → ``(None, False)``;
-    agent echoed a handle we could have minted → ``(value, False)``; anything
-    else (omitted, or a value the agent made up) → ``(new uuid, True)``.
+    """Return ``(conversation_id, minted)``. Disabled, get_more_tools, or
+    send_feedback → ``(None, False)``; agent echoed a handle we could have minted
+    → ``(value, False)``; anything else (omitted, or a value the agent made up)
+    → ``(new uuid, True)``.
 
     Lowercased on the way in: the shape test is case-insensitive but the hash
     behind ``$session_id`` is not, so an uppercased echo (some hosts normalise
     uuids) would land in a different session than the call that minted it."""
-    if not enabled or tool_name == missing_capability_tool_name:
+    if (
+        not enabled
+        or tool_name == missing_capability_tool_name
+        or (feedback_tool_name is not None and tool_name == feedback_tool_name)
+    ):
         return None, False
     supplied = extract_conversation_id(args)
     if supplied and _MINTED_CONVERSATION_ID.match(supplied):

posthog/mcp/_event_types.py

@@ -15,6 +15,7 @@ class MCPAnalyticsEventType:
 
     IDENTIFY = "posthog:identify"
     CUSTOM = "posthog:custom"
+    MCP_FEEDBACK = "mcp:feedback"
     MCP_MISSING_CAPABILITY = "mcp:missing_capability"
     MCP_INITIALIZE = "mcp:initialize"
     MCP_PROMPTS_GET = "mcp:prompts/get"

posthog/mcp/_instrument_fastmcp.py

@@ -31,9 +31,12 @@
 from ._instrumentation import (
     _to_jsonable,
     append_get_more_tools,
+    append_send_feedback,
     collect_listed_tools,
     extract_tools,
+    listing_has_next_page,
     mutate_tool_schema,
+    refresh_feedback_shadow,
     request_to_dict,
     resolve_session_and_client,
     start_tool_call_lifecycle,
@@ -118,6 +121,12 @@ async def wrapped(
                 mcp_types.TextContent(type="text", text=get_more_tools_result_text())
             ]
 
+        if lifecycle.is_feedback and not _feedback_name_owned_by_real_tool(
+            server, name
+        ):
+            reply = await lifecycle.record_feedback()
+            return [mcp_types.TextContent(type="text", text=reply)]
+
         # Strip each injected key independently. A tool can declare its own
         # `context` (kept) while `conversation_id` is still SDK-injected (stripped),
         # so coupling both to context-ownership leaked conversation_id into the tool.
@@ -219,7 +228,12 @@ async def list_handler(req: Any) -> Any:
         # advertise and write, the SDK rejects the customer's own tool result.
         if req is None:
             result = await original(req)
-            _inject_tool_schemas(server, data, extract_tools(result))
+            tools = extract_tools(result)
+            # Refresh the collision flag here too: this pass sees the real tool
+            # registry, so a real tool named like the feedback tool is detected
+            # before any client-facing listing.
+            refresh_feedback_shadow(data, tools)
+            _inject_tool_schemas(server, data, tools)
             return result
 
         client_name, client_version = _low_level_client_info(server)
@@ -258,6 +272,7 @@ async def list_handler(req: Any) -> Any:
         tools = extract_tools(result)
         # Empty is computed before adding the virtual missing-capability tool.
         names, empty = collect_listed_tools(data, tools)
+        feedback_name = refresh_feedback_shadow(data, tools)
 
         _inject_tool_schemas(server, data, tools)
 
@@ -267,6 +282,10 @@ async def list_handler(req: Any) -> Any:
                 append_get_more_tools(result, missing_name, data)
                 names.append(missing_name)
 
+        if feedback_name is not None and not listing_has_next_page(result):
+            append_send_feedback(result, data)
+            names.append(feedback_name)
+
         await lifecycle.record_result(
             names=names,
             response=_to_jsonable(result),
@@ -303,6 +322,16 @@ def _inject_prompt_back(result: Any, conversation_id: str) -> Any:
     return result
 
 
+def _feedback_name_owned_by_real_tool(server: Any, name: str) -> bool:
+    """Live registry probe so a real tool by the feedback tool's name is never
+    shadowed even before the first listing refreshes the collision flag."""
+    try:
+        tool_manager = getattr(server, "_tool_manager", None)
+        return tool_manager is not None and tool_manager.get_tool(name) is not None
+    except Exception:  # noqa: BLE001 - unknown tool -> the name is not owned
+        return False
+
+
 def _tool_owns_param(server: Any, name: str, param: str) -> bool:
     """True when the tool's own function declares ``param`` — then it's a real tool
     argument we must neither inject nor strip (the agent's value belongs to the tool)."""

posthog/mcp/_instrument_lowlevel.py

@@ -26,11 +26,14 @@
 from ._instrumentation import (
     _to_jsonable,
     append_get_more_tools,
+    append_send_feedback,
     collect_listed_tools,
     extract_tools,
+    listing_has_next_page,
     mutate_tool_schema,
     prepare_request,
     record_resource_request,
+    refresh_feedback_shadow,
     request_to_dict,
     resource_listing_response,
     resolve_session_and_client,
@@ -216,6 +219,17 @@ async def handler(req: Any) -> Any:
                 )
             )
 
+        if lifecycle.is_feedback and not await _feedback_name_owned_by_real_tool(
+            high_level, name
+        ):
+            reply = await lifecycle.record_feedback()
+            return mcp_types.ServerResult(
+                mcp_types.CallToolResult(
+                    content=[mcp_types.TextContent(type="text", text=reply)],
+                    isError=False,
+                )
+            )
+
         # On raw low-level servers `context`/`conversation_id` are injected as
         # *optional* schema properties and left in place (a (name, arguments)
         # handler ignores extra keys). FastMCP 2.0 validates against the function
@@ -340,9 +354,12 @@ async def handler(req: Any) -> Any:
         # `additionalProperties: false`.
         if req is None:
             result = await original(req)
-            _inject_tool_schemas(
-                data, extract_tools(result), context_required=context_required
-            )
+            tools = extract_tools(result)
+            # Refresh the collision flag here too: this pass sees the real tool
+            # registry, so a real tool named like the feedback tool is detected
+            # before any client-facing listing.
+            refresh_feedback_shadow(data, tools)
+            _inject_tool_schemas(data, tools, context_required=context_required)
             return result
 
         client_name, client_version = _client_info(server)
@@ -384,6 +401,7 @@ async def handler(req: Any) -> Any:
         # Zero advertised tools is treated as an errored tools/list before the
         # virtual missing-capability tool is appended.
         names, empty = collect_listed_tools(data, tools)
+        feedback_name = refresh_feedback_shadow(data, tools)
 
         _inject_tool_schemas(data, tools, context_required=context_required)
 
@@ -393,6 +411,10 @@ async def handler(req: Any) -> Any:
                 append_get_more_tools(result, missing_name, data)
                 names.append(missing_name)
 
+        if feedback_name is not None and not listing_has_next_page(result):
+            append_send_feedback(result, data)
+            names.append(feedback_name)
+
         await lifecycle.record_result(
             names=names,
             response=_to_jsonable(result),
@@ -406,6 +428,19 @@ async def handler(req: Any) -> Any:
     handlers[mcp_types.ListToolsRequest] = handler
 
 
+async def _feedback_name_owned_by_real_tool(high_level: Any, name: str) -> bool:
+    """Live registry probe on the standalone-fastmcp path, so a real tool by the
+    feedback tool's name is never shadowed even before the first listing refreshes
+    the collision flag. Raw low-level servers have no registry to probe; they rely
+    on the listing-derived flag alone."""
+    if high_level is None:
+        return False
+    try:
+        return await high_level.get_tool(name) is not None
+    except Exception:  # noqa: BLE001 - unknown tool -> the name is not owned
+        return False
+
+
 async def _tool_owned_injected_keys(high_level: Any, name: str) -> set:
     """Which of (``context``, ``conversation_id``) the jlowin FastMCP tool declares
     itself, read from its function signature. These are real tool arguments we must