feat(maintenance): backfill stored connector documents into the memory tree

crates/tinymemory-api/src/null.rs

@@ -58,8 +58,9 @@ use crate::health::MemoryHealth;
 use crate::learning::LearningCandidate;
 use crate::operations::{AnswerRequest, AnswerResponse, RawMemoryEvent};
 use crate::provider::types::{
-    DiffReport, EntityHit, ExportPage, ExportRecord, FlushOutcome, ImportOutcome, IngestItem,
-    IngestOutcome, MaintenanceReport, ResetOutcome, SnapshotRef, SourceItem, SourceScope,
+    BackfillTreesOutcome, BackfillTreesRequest, DiffReport, EntityHit, ExportPage, ExportRecord,
+    FlushOutcome, ImportOutcome, IngestItem, IngestOutcome, MaintenanceReport, ResetOutcome,
+    SnapshotRef, SourceItem, SourceScope,
 };
 use crate::provider::{
     AddressBookSeedOutcome, ChunkDetail, ChunkEmbedding, ChunkQuery, CodingSessionIngestReport,
@@ -543,6 +544,13 @@ impl MemoryMaintenance for NullMemoryProvider {
         unsupported(Capability::Maintenance)
     }
 
+    async fn backfill_connector_trees(
+        &self,
+        _request: BackfillTreesRequest,
+    ) -> Result<BackfillTreesOutcome, MemoryError> {
+        unsupported(Capability::Maintenance)
+    }
+
     async fn reset_derived_index(&self) -> Result<ResetOutcome, MemoryError> {
         unsupported(Capability::Maintenance)
     }

crates/tinymemory-api/src/provider/mod.rs

@@ -131,8 +131,8 @@ pub use sync::{
     SyncAuditEntry, SyncFreshness, SyncRunOutcome,
 };
 pub use types::{
-    ChangeKind, ChunkEntityOccurrence, DiffReport, EntityHit, EntityOccurrence, EntityRef,
-    ExportPage, ExportRecord, FlushOutcome, ForgetOutcome, ForgetSelector, ImportOutcome,
-    IngestItem, IngestOutcome, MaintenanceReport, PurgeOutcome, ResetOutcome, SnapshotRef,
-    SourceChange, SourceItem, SourceScope,
+    BackfillTreesOutcome, BackfillTreesRequest, ChangeKind, ChunkEntityOccurrence, DiffReport,
+    EntityHit, EntityOccurrence, EntityRef, ExportPage, ExportRecord, FlushOutcome, ForgetOutcome,
+    ForgetSelector, ImportOutcome, IngestItem, IngestOutcome, MaintenanceReport, PurgeOutcome,
+    ResetOutcome, SnapshotRef, SourceChange, SourceItem, SourceScope,
 };

crates/tinymemory-api/src/provider/records.rs

@@ -21,8 +21,9 @@ use crate::error::MemoryError;
 use crate::goals::GoalsDoc;
 use crate::provider::diagnosis::{DegradedCapabilities, Diagnosis};
 use crate::provider::types::{
-    FlushOutcome, ForgetOutcome, ForgetSelector, IngestOutcome, MaintenanceReport, PurgeOutcome,
-    QueueFailure, QueueStats, ResetOutcome, SourceItem, StoreStats,
+    BackfillTreesOutcome, BackfillTreesRequest, FlushOutcome, ForgetOutcome, ForgetSelector,
+    IngestOutcome, MaintenanceReport, PurgeOutcome, QueueFailure, QueueStats, ResetOutcome,
+    SourceItem, StoreStats,
 };
 use crate::tool_memory::ToolMemoryRule;
 use crate::types::MemoryTaint;
@@ -338,6 +339,40 @@ pub trait MemoryMaintenance: Send + Sync {
         Ok(FlushOutcome::default())
     }
 
+    /// Re-file already-stored connector documents into the memory tree (#6012).
+    ///
+    /// openhuman#6007 fixed the routing for items synced *from now on*. It could
+    /// not fix the records already stored: the per-item sync gate treats an
+    /// ingested document as done, so re-syncing fetches nothing and creates no
+    /// tree rows. Those memories stay fully embedded in the document store and
+    /// invisible to every tree-backed surface until something re-files them.
+    ///
+    /// Idempotent, and by construction rather than by bookkeeping — the ingest
+    /// gate answers `already_ingested` for a document the tree already holds, so
+    /// a second pass writes nothing and an interrupted pass loses nothing. That
+    /// is also why `limit` bounds cost rather than carrying a cursor.
+    ///
+    /// Expensive on purpose to call explicitly: a pass is one read and one set
+    /// of chunk embeddings per document. A driver must not run this on its own
+    /// initiative — spending a user's embedding budget unasked is its own bug
+    /// (openhuman#5324).
+    ///
+    /// Defaulted to an empty outcome, matching [`Self::flush_pending`]: a driver
+    /// with no connector documents has nothing to re-file, which is a fact about
+    /// it rather than a refusal. A driver that stores nothing at all should
+    /// override and refuse instead, so "backfilled nothing" cannot read as work
+    /// done.
+    ///
+    /// # Errors
+    ///
+    /// Backend failures only.
+    async fn backfill_connector_trees(
+        &self,
+        _request: BackfillTreesRequest,
+    ) -> Result<BackfillTreesOutcome, MemoryError> {
+        Ok(BackfillTreesOutcome::default())
+    }
+
     /// Drop everything derived from stored content and schedule its
     /// re-derivation.
     ///

crates/tinymemory-bus/src/names.rs

@@ -212,6 +212,10 @@ pub mod methods {
     pub const RECALL_NAMESPACE_RECENT: &str = "RecallNamespaceRecent";
     /// `FlushPending` — flush buffered work old enough to be written out.
     pub const FLUSH_PENDING: &str = "FlushPending";
+    /// `BackfillConnectorTrees` — file already-stored connector documents into
+    /// the memory tree, for records synced before the routing fix
+    /// (openhuman#6007) reached them.
+    pub const BACKFILL_CONNECTOR_TREES: &str = "BackfillConnectorTrees";
     /// `ResetDerivedIndex` — drop derived state and schedule its rebuild.
     pub const RESET_DERIVED_INDEX: &str = "ResetDerivedIndex";
     /// `PurgeAll` — erase every row the driver holds.
@@ -373,7 +377,7 @@ pub mod methods {
 /// The order matters: `tinybus`'s `Interface::members()` returns declaration
 /// order, and the module compares the two sequences directly rather than as
 /// sets, so a reordering is caught alongside an addition or a removal.
-pub const METHODS: [&str; 142] = [
+pub const METHODS: [&str; 143] = [
     methods::DRIVER_ID,
     methods::CAPABILITIES,
     methods::HEALTH,
@@ -516,6 +520,11 @@ pub const METHODS: [&str; 142] = [
     methods::INGEST_EVENT,
     methods::ANSWER,
     methods::OVERRIDE_SCHEDULER_GATE,
+    // Connector-backfill round (openhuman#6012): appended at the tail, per
+    // this table's append-only rule — filing it beside `FlushPending`, where
+    // its family lives, would renumber every member after it and invoke the
+    // wrong method on a host built against an earlier release.
+    methods::BACKFILL_CONNECTOR_TREES,
 ];
 
 #[cfg(test)]

crates/tinymemory-bus/src/names_tests.rs

@@ -104,6 +104,12 @@ fn the_newest_members_are_appended_rather_than_filed_with_their_family() {
     // append-only rule.
     assert_eq!(METHODS[141], methods::OVERRIDE_SCHEDULER_GATE);
     assert_eq!(methods::OVERRIDE_SCHEDULER_GATE, "OverrideSchedulerGate");
+    // Connector-backfill round (openhuman#6012): slot 142. Its family sits at
+    // 137 (`FlushPending`), and filing it there is exactly what this test
+    // exists to catch — it renumbers 137 onward and a host built against
+    // v1.13.8 then invokes the wrong member.
+    assert_eq!(METHODS[142], methods::BACKFILL_CONNECTOR_TREES);
+    assert_eq!(methods::BACKFILL_CONNECTOR_TREES, "BackfillConnectorTrees");
 }
 
 #[test]
@@ -144,7 +150,7 @@ fn the_runtime_tree_doors_hold_the_wire_slots_they_were_released_in() {
     // reason the summariser-door test above gives: member order is wire order,
     // and an assertion measured from the end moves silently under the next
     // append — which is exactly the edit this exists to catch.
-    assert_eq!(METHODS.len(), 142);
+    assert_eq!(METHODS.len(), 143);
     assert_eq!(METHODS[131], methods::RUNTIME_BUFFER_WRITE);
     assert_eq!(METHODS[132], methods::RUNTIME_READ_NODE);
     assert_eq!(METHODS[133], methods::RUNTIME_READ_CHILDREN);

crates/tinymemory-bus/src/provider/types.rs

@@ -781,6 +781,49 @@ pub struct FlushOutcome {
     pub stale_buffers: u64,
 }
 
+/// What one connector-tree backfill pass examined and wrote (#6012).
+///
+/// Four counters rather than one, because "did nothing" has three very
+/// different causes a caller has to be able to tell apart: the tree already
+/// held everything (`already_present`), nothing could be addressed
+/// (`skipped`), or there was nothing to look at (`scanned: 0`). Collapsing
+/// them would make an account whose scope could not be resolved read exactly
+/// like one that is fully backfilled.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BackfillTreesOutcome {
+    /// Documents examined this pass.
+    pub scanned: u64,
+    /// Documents that produced new memory-tree rows.
+    pub ingested: u64,
+    /// Documents the tree already held. Not a failure: this is what makes a
+    /// repeated pass readable as "nothing left to do".
+    pub already_present: u64,
+    /// Documents left alone — no resolvable scope, or a tolerated failure.
+    /// Never filed under a guess.
+    pub skipped: u64,
+    /// Whether the pass stopped on its limit with documents still unexamined.
+    /// The caller resumes by calling again; there is no cursor to carry.
+    pub more_pending: bool,
+    /// Bounded, human-readable reasons behind `skipped`.
+    pub notes: Vec<String>,
+}
+
+/// How much of the backfill to attempt, and whether to write at all.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BackfillTreesRequest {
+    /// Documents to examine at most. `None` leaves the bound to the driver.
+    ///
+    /// A bound rather than a cursor because the work is idempotent: the pass
+    /// re-reads what it already treed and the ingest gate answers
+    /// `already_ingested`, so resuming is just calling again.
+    pub limit: Option<u64>,
+    /// Report what a real pass would examine, and write nothing.
+    ///
+    /// The honest way to show an operator the size of the job before they pay
+    /// for it — a full pass is one read and one embedding per document.
+    pub dry_run: bool,
+}
+
 /// What resetting the derived index deleted, requeued and scheduled.
 ///
 /// Three numbers rather than a `MaintenanceReport`'s two, because they are not