feat(api): report semantic search freshness and identify the CLI to Cloud

src/basic_memory/alembic/versions/0044_entity_embedded_checksum.py

@@ -0,0 +1,66 @@
+"""Record which checksum each entity's embeddings were built from.
+
+Search responses now say how current semantic results are: the oldest last-modified
+time among notes whose embeddings lag their content. Basic Memory Cloud defers a
+workspace's embeddings past a daily fair-use budget while notes stay saved and
+full-text indexed, so a note can sit with new content and old vectors for a day.
+Nothing in the schema showed that: an edited note keeps its earlier `ready` chunks, which
+look the same as current ones.
+
+``entity.embedded_checksum`` holds the checksum a vector-sync pass finished embedding.
+A note is behind when it is NULL, differs from ``checksum``, or the last pass left the
+note part-way (``vector_sync_deferred_at``). A partial index on exactly those rows makes
+the per-search lookup read only the notes that are behind.
+
+The backfill treats every entity the last pass did not leave part-way as current. That
+is not provable from the manifest, but the alternative, NULL everywhere, would report
+notes that owe nothing (opted out of embeddings, or with no text to embed) as behind
+until something rewrites them. A note that was in fact behind at upgrade time reads as
+current until its pending embedding pass runs and stamps the real value.
+
+A plain ALTER TABLE, not batch mode: ``entity`` carries generated columns, and SQLite's
+batch recreation emits them twice (see x7d8e9f0a1b2).
+
+Revision ID: 0044
+Revises: 0043
+Create Date: 2026-11-04 20:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0044"
+down_revision: Union[str, None] = "0043"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+# Must match ENTITY_EMBEDDING_NOT_CURRENT_PREDICATE in models/knowledge.py. Copied, not
+# imported, so this revision keeps creating the index it was written with.
+EMBEDDING_NOT_CURRENT_PREDICATE = (
+    "content_type = 'text/markdown' AND ("
+    "embedded_checksum IS NULL "
+    "OR embedded_checksum <> checksum "
+    "OR vector_sync_deferred_at IS NOT NULL)"
+)
+
+
+def upgrade() -> None:
+    """Add entity.embedded_checksum, backfill it, and index the entities that are behind."""
+    op.add_column("entity", sa.Column("embedded_checksum", sa.String(), nullable=True))
+    op.execute(
+        "UPDATE entity SET embedded_checksum = checksum WHERE vector_sync_deferred_at IS NULL"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_entity_project_embedding_not_current "
+        "ON entity (project_id, updated_at) "
+        f"WHERE {EMBEDDING_NOT_CURRENT_PREDICATE}"
+    )
+
+
+def downgrade() -> None:
+    """Drop the index and the embedded checksum."""
+    op.execute("DROP INDEX IF EXISTS ix_entity_project_embedding_not_current")
+    op.drop_column("entity", "embedded_checksum")

src/basic_memory/api/v2/routers/scoped_search_router.py

@@ -21,6 +21,7 @@
 from basic_memory.deps import AppConfigDep, SessionMakerDep
 from basic_memory.repository.search_repository import create_search_reader
 from basic_memory.repository.search_scope import ProjectScope
+from basic_memory.repository.semantic_index_freshness import semantic_index_current_as_of
 from basic_memory.schemas.search import (
     MAX_SCOPED_SEARCH_PAGE_SIZE,
     ScopedSearchQuery,
@@ -119,4 +120,7 @@ async def search_scope(
             # None, not False, when nothing was asked: an ordinary search payload stays
             # exactly what it was before valid time existed.
             temporal_applied=True if temporal_requested else None,
+            semantic_index_current_as_of=await semantic_index_current_as_of(
+                session_maker, app_config, scope
+            ),
         )

src/basic_memory/api/v2/routers/search_router.py

@@ -19,6 +19,7 @@
 from basic_memory.read_cache.policy import SEARCH_READ_CACHE_TTL_SECONDS
 from basic_memory.read_cache.routes import ReadCacheRoute, cached_read, content_group
 from basic_memory.deps import (
+    AppConfigDep,
     EntityServiceV2ExternalDep,
     MemoryTimeIndexRepositoryV2ExternalDep,
     ProjectExternalIdPathDep,
@@ -28,6 +29,8 @@
 )
 from basic_memory.repository.metadata_filters import parse_metadata_filters
 from basic_memory.repository.search_index_row import SearchIndexRow
+from basic_memory.repository.search_scope import ProjectScope
+from basic_memory.repository.semantic_index_freshness import semantic_index_current_as_of
 from basic_memory.schemas.search import SearchQuery, SearchResponse, SearchRetrievalMode
 from basic_memory.services.search_service import SearchService
 from basic_memory.services.search_guidance import unspaced_script_query_hint
@@ -87,6 +90,7 @@ async def search(
     entity_service: EntityServiceV2ExternalDep,
     temporal_repository: MemoryTimeIndexRepositoryV2ExternalDep,
     session_maker: SessionMakerDep,
+    app_config: AppConfigDep,
     response: Response,
     internal_project_id: ProjectExternalIdPathDep,
     project_id: str = Path(..., description="Project external UUID"),
@@ -103,7 +107,8 @@ async def search(
         search_service: Search service scoped to project
         entity_service: Entity service scoped to project
         temporal_repository: Valid-time projection, read to explain temporal matches
-        session_maker: Session factory for the temporal hydration read
+        session_maker: Session factory for the temporal hydration and freshness reads
+        app_config: Configuration that says whether semantic search can run here
         page: Page number for pagination
         page_size: Number of results per page
 
@@ -216,6 +221,9 @@ async def search(
             phase="build_response",
             result_count=len(search_results),
         ):
+            current_as_of = await semantic_index_current_as_of(
+                session_maker, app_config, ProjectScope.single(internal_project_id)
+            )
             result = SearchResponse(
                 results=search_results,
                 current_page=page,
@@ -231,6 +239,7 @@ async def search(
                 # None, not False, when nothing was asked: an ordinary search
                 # payload stays exactly what it was before valid time existed.
                 temporal_applied=True if temporal_requested else None,
+                semantic_index_current_as_of=current_as_of,
             )
             return result
 

src/basic_memory/cli/auth.py

@@ -16,6 +16,7 @@
 
 from basic_memory.config import ConfigManager
 from basic_memory.cli.markup import literal
+from basic_memory.user_agent import BASIC_MEMORY_USER_AGENT
 
 console = Console()
 
@@ -50,7 +51,7 @@ async def _get_http_client(self) -> AsyncIterator[httpx.AsyncClient]:
             async with self._http_client_factory() as client:
                 yield client
         else:
-            async with httpx.AsyncClient() as client:
+            async with httpx.AsyncClient(headers={"User-Agent": BASIC_MEMORY_USER_AGENT}) as client:
                 yield client
 
     def generate_pkce_pair(self) -> tuple[str, str]:

src/basic_memory/cli/commands/cloud/api_client.py

@@ -11,6 +11,7 @@
 
 from basic_memory.cli.auth import CLIAuth
 from basic_memory.config import ConfigManager
+from basic_memory.user_agent import BASIC_MEMORY_USER_AGENT
 
 console = Console()
 
@@ -75,7 +76,9 @@ async def get_authenticated_headers(auth: CLIAuth | None = None) -> dict[str, st
 
 @asynccontextmanager
 async def _default_http_client(timeout: float) -> AsyncIterator[httpx.AsyncClient]:
-    async with httpx.AsyncClient(timeout=timeout) as client:
+    async with httpx.AsyncClient(
+        timeout=timeout, headers={"User-Agent": BASIC_MEMORY_USER_AGENT}
+    ) as client:
         yield client
 
 

src/basic_memory/man/man3/bm-search(3).md

@@ -113,6 +113,10 @@ bm_search(query="release", mode="text", page_size=1, output_format="json", proje
   a next-page call; follow it to check the remaining candidates.
 - Line results report `total_is_exact: false`; `total` counts ranked candidates,
   so page with `has_more` instead of using it as a result count.
+- Embeddings can lag a note's text. When some notes are not embedded yet, the
+  response carries `semantic_index_current_as_of` and the text output says
+  "Semantic results current as of ...". Notes changed after that time are found
+  by `mode="text"` but may be missing from semantic and hybrid matches.
 
 ## AUTHORED VALID TIME
 

src/basic_memory/mcp/async_client.py

@@ -11,6 +11,7 @@
 from loguru import logger
 
 import logfire
+from basic_memory.user_agent import BASIC_MEMORY_USER_AGENT
 from basic_memory.config import (
     BasicMemoryConfig,
     ConfigManager,
@@ -349,7 +350,7 @@ async def _cloud_client(
 
     token = await _resolve_cloud_token(config)
     proxy_base_url = f"{config.cloud_host}/proxy"
-    headers = {"Authorization": f"Bearer {token}"}
+    headers = {"Authorization": f"Bearer {token}", "User-Agent": BASIC_MEMORY_USER_AGENT}
     headers.update(workspace_permalink_headers())
     if workspace:
         headers["X-Workspace-ID"] = workspace
@@ -370,7 +371,7 @@ async def get_cloud_control_plane_client(
     config = ConfigManager().config
     timeout = _build_timeout()
     token = await _resolve_cloud_token(config)
-    headers = {"Authorization": f"Bearer {token}"}
+    headers = {"Authorization": f"Bearer {token}", "User-Agent": BASIC_MEMORY_USER_AGENT}
     if workspace:
         headers["X-Workspace-ID"] = workspace
     logger.info(f"Creating HTTP client for cloud control plane at: {config.cloud_host}")

src/basic_memory/mcp/tools/ranked_search.py

@@ -13,6 +13,7 @@
 
 import re
 from dataclasses import asdict
+from datetime import datetime, timezone
 from typing import Annotated, Any, Literal
 
 from fastmcp import Context
@@ -678,12 +679,14 @@ def _format_results(
         if next_call is not None
         else ""
     )
+    freshness = _semantic_freshness_note(payload)
     if not results:
         page_hint = " on this page" if next_call is not None else ""
         return (
             (f"No results for '{query}'" if query else "No results")
             + f"{page_hint} in {scope}."
             + (f"\n\n{payload['query_hint']}" if payload.get("query_hint") else "")
+            + freshness
             + continuation
         )
 
@@ -713,4 +716,20 @@ def _format_results(
                 lines.append(f"    (more matches from line {row['next_match_line']})")
         elif row.get("matched_chunk"):
             lines.append(f"    {row['matched_chunk'][:200]}")
-    return "\n".join(lines) + continuation
+    return "\n".join(lines) + freshness + continuation
+
+
+def _semantic_freshness_note(payload: dict[str, Any]) -> str:
+    """Say how current semantic results are, when embeddings lag some notes.
+
+    The field is absent when every note is embedded, so ordinary output is unchanged.
+    """
+    current_as_of = payload.get("semantic_index_current_as_of")
+    if current_as_of is None:
+        return ""
+    as_of_utc = datetime.fromisoformat(current_as_of).astimezone(timezone.utc)
+    return (
+        f"\n\nSemantic results current as of {as_of_utc:%Y-%m-%d %H:%M} UTC. "
+        "Notes changed since then are found by text search but may be missing "
+        "from semantic matches."
+    )

src/basic_memory/mcp/tools/search.py

@@ -7,6 +7,7 @@
 import re
 from collections.abc import Sequence
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Any, cast
 from uuid import UUID
 
@@ -589,6 +590,8 @@ async def _search_all_projects(
     databases_answered = 0
     failures: list[str] = []
     query_hint: str | None = None
+    # The merged page is only as current as its least current database.
+    semantic_current_as_of: list[datetime] = []
 
     for tenant_id, refs in databases.items():
         label = _database_label(tenant_id, refs)
@@ -619,6 +622,8 @@ async def _search_all_projects(
             response = _compact_search_response(response)
         if response.query_hint:
             query_hint = response.query_hint
+        if response.semantic_index_current_as_of is not None:
+            semantic_current_as_of.append(response.semantic_index_current_as_of)
         total += (
             response.total
             if response.total > 0
@@ -671,6 +676,9 @@ async def _search_all_projects(
             # Confirmed only because a database answered: `databases_answered` is
             # non-zero here for any temporal query, guarded immediately above.
             "temporal_applied": True if temporal_requested else None,
+            "semantic_index_current_as_of": min(semantic_current_as_of)
+            if semantic_current_as_of
+            else None,
         }
     )
 

src/basic_memory/models/knowledge.py

@@ -29,6 +29,18 @@
 from basic_memory.utils import file_path_conflict_keys, generate_permalink
 
 
+# A markdown entity whose semantic embeddings do not reflect its current content: never
+# embedded at this checksum, or a pass left part of it unembedded. Shared verbatim by the
+# partial index below and by the search-freshness query, because both engines use a
+# partial index only when the query repeats the index's predicate.
+ENTITY_EMBEDDING_NOT_CURRENT_PREDICATE = (
+    "content_type = 'text/markdown' AND ("
+    "embedded_checksum IS NULL "
+    "OR embedded_checksum <> checksum "
+    "OR vector_sync_deferred_at IS NOT NULL)"
+)
+
+
 def file_path_conflict_columns(file_path: str) -> dict[str, str]:
     """Return the derived entity columns that index ``file_path`` for conflict checks.
 
@@ -76,6 +88,15 @@ class Entity(Base):
         # File path conflict checks look up one project's rows by either conflict key.
         Index("ix_entity_project_file_path_key", "project_id", "file_path_key"),
         Index("ix_entity_project_file_path_permalink", "project_id", "file_path_permalink"),
+        # Search reports the oldest note whose embeddings are behind. Only those notes are
+        # in this index, so the lookup reads a handful of rows however large the project.
+        Index(
+            "ix_entity_project_embedding_not_current",
+            "project_id",
+            "updated_at",
+            sqlite_where=text(ENTITY_EMBEDDING_NOT_CURRENT_PREDICATE),
+            postgresql_where=text(ENTITY_EMBEDDING_NOT_CURRENT_PREDICATE),
+        ),
     )
 
     # Core identity
@@ -123,6 +144,12 @@ class Entity(Base):
     vector_sync_deferred_at: Mapped[Optional[datetime]] = mapped_column(
         DateTime(timezone=True), nullable=True, default=None
     )
+    # The entity checksum whose embeddings a vector-sync pass finished, or which owes no
+    # embeddings (opted out). It differs from `checksum` once the note changes and stays
+    # behind until a pass embeds the new content, which is how search can say how current
+    # semantic results are. Written only by vector sync, with the checksum read before
+    # that pass read the note's search rows.
+    embedded_checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
 
     # Metadata and tracking
     created_at: Mapped[datetime] = mapped_column(

src/basic_memory/repository/search_repository.py

@@ -6,7 +6,7 @@
 - PostgresSearchRepository: Uses tsvector/tsquery with GIN indexes
 """
 
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from typing import Any, Callable, List, Optional, Protocol
 
@@ -88,6 +88,10 @@ async def record_entity_vector_deferrals(
         """Record which entities a vector sync pass left unfinished."""
         ...
 
+    async def record_entity_embedded_checksums(self, checksums: Mapping[int, str | None]) -> None:
+        """Stamp the checksum each listed entity's embeddings now reflect."""
+        ...
+
     async def semantic_effectively_enabled(self) -> bool:
         """Return whether semantic retrieval can actually run right now."""
         ...