From 61158139fb044d694f1aca0ffb8de4b2f4d2c2cc Mon Sep 17 00:00:00 2001 From: hobertrand-hub Date: Sat, 25 Jul 2026 02:47:09 +0200 Subject: [PATCH 1/5] fix(memory): unblock legacy chunk reembedding --- src/memory/chunks/embeddings.rs | 9 +++- src/memory/chunks/store_embed_tests.rs | 68 ++++++++++++++++++++++++-- src/memory/store/content/read.rs | 22 +++++++-- src/memory/store/content/read_tests.rs | 10 ++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/src/memory/chunks/embeddings.rs b/src/memory/chunks/embeddings.rs index 0e07e25..ff04b42 100644 --- a/src/memory/chunks/embeddings.rs +++ b/src/memory/chunks/embeddings.rs @@ -67,6 +67,11 @@ fn upsert_chunk_embedding_conn( created_at = excluded.created_at", rusqlite::params![chunk_id, model_signature, bytes, dim, created_at], )?; + conn.execute( + "DELETE FROM mem_tree_chunk_reembed_skipped + WHERE chunk_id = ?1 AND model_signature = ?2", + rusqlite::params![chunk_id, model_signature], + )?; Ok(()) } @@ -385,7 +390,9 @@ pub fn has_uncovered_reembed_work( WHERE NOT EXISTS (SELECT 1 FROM mem_tree_chunk_embeddings e WHERE e.chunk_id = c.id AND e.model_signature = ?1) AND NOT EXISTS (SELECT 1 FROM mem_tree_chunk_reembed_skipped sk - WHERE sk.chunk_id = c.id AND sk.model_signature = ?1)) + WHERE sk.chunk_id = c.id AND sk.model_signature = ?1 + AND sk.reason NOT LIKE 'body read failed: no content pointer or raw refs for chunk %' + AND sk.reason NOT LIKE 'body read failed: empty content pointer and no raw refs for chunk %')) OR EXISTS( SELECT 1 FROM mem_tree_summaries s WHERE s.deleted = 0 diff --git a/src/memory/chunks/store_embed_tests.rs b/src/memory/chunks/store_embed_tests.rs index 4d364a8..d89c99a 100644 --- a/src/memory/chunks/store_embed_tests.rs +++ b/src/memory/chunks/store_embed_tests.rs @@ -19,9 +19,10 @@ use super::{ clear_summary_reembed_skipped, content_root, count_chunks, db_path_for, delete_chunks_by_owner, delete_chunks_by_source, extraction_coverage, get_chunk, get_chunk_embedding, get_chunk_embedding_for_signature, get_chunk_embeddings_for_signature_batch, get_chunks_batch, - is_source_ingested, list_chunks, mark_chunk_reembed_skipped, mark_summary_reembed_skipped, - set_chunk_embedding, set_chunk_embedding_for_signature, tree_active_signature, upsert_chunks, - ListChunksQuery, DB_DIR, GLOBAL_TOPIC_PURGE_MIGRATION_VERSION, + has_uncovered_reembed_work, is_source_ingested, list_chunks, mark_chunk_reembed_skipped, + mark_summary_reembed_skipped, set_chunk_embedding, set_chunk_embedding_for_signature, + tree_active_signature, upsert_chunks, ListChunksQuery, DB_DIR, + GLOBAL_TOPIC_PURGE_MIGRATION_VERSION, }; use crate::memory::config::MemoryConfig; use crate::memory::tree::store::{ @@ -81,6 +82,28 @@ fn clear_chunk_reembed_skipped_is_idempotent() { assert_eq!(count, 0); } +#[test] +fn setting_chunk_embedding_clears_matching_skip_marker() { + let (_tmp, cfg) = test_config(); + let c = sample_chunk("slack:#eng", 0, 1_700_000_000_000); + upsert_chunks(&cfg, std::slice::from_ref(&c)).unwrap(); + let sig = tree_active_signature(&cfg); + mark_chunk_reembed_skipped(&cfg, &c.id, &sig, "body read failed: no content pointer").unwrap(); + + set_chunk_embedding_for_signature(&cfg, &c.id, &sig, &[0.1, 0.2]).unwrap(); + + let count: i64 = with_connection(&cfg, |conn| { + Ok(conn.query_row( + "SELECT COUNT(*) FROM mem_tree_chunk_reembed_skipped + WHERE chunk_id = ?1 AND model_signature = ?2", + params![c.id, sig], + |r| r.get(0), + )?) + }) + .unwrap(); + assert_eq!(count, 0); +} + #[test] fn summary_reembed_tombstone_roundtrips_and_clears() { let (_tmp, cfg) = test_config(); @@ -318,6 +341,45 @@ fn batch_embedding_lookup_unknown_ids_absent_from_map() { assert_eq!(map.get(&c.id).cloned(), Some(vec![0.1])); } +#[test] +fn legacy_body_read_skip_does_not_hide_reembed_work() { + let (_tmp, cfg) = test_config(); + let c = sample_chunk("persona/communication", 0, 1_700_000_000_000); + upsert_chunks(&cfg, std::slice::from_ref(&c)).unwrap(); + let sig = "bge-m3@1024"; + mark_chunk_reembed_skipped( + &cfg, + &c.id, + sig, + &format!( + "body read failed: no content pointer or raw refs for chunk {}", + c.id + ), + ) + .unwrap(); + + with_connection(&cfg, |conn| { + assert!(has_uncovered_reembed_work(conn, sig)?); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn non_legacy_skip_still_hides_reembed_work() { + let (_tmp, cfg) = test_config(); + let c = sample_chunk("persona/communication", 0, 1_700_000_000_000); + upsert_chunks(&cfg, std::slice::from_ref(&c)).unwrap(); + let sig = "bge-m3@1024"; + mark_chunk_reembed_skipped(&cfg, &c.id, sig, "embed failed: provider rejected input").unwrap(); + + with_connection(&cfg, |conn| { + assert!(!has_uncovered_reembed_work(conn, sig)?); + Ok(()) + }) + .unwrap(); +} + #[test] fn batch_embedding_lookup_splits_id_list_above_per_batch_threshold() { let (_tmp, cfg) = test_config(); diff --git a/src/memory/store/content/read.rs b/src/memory/store/content/read.rs index fa3babf..fb09df7 100644 --- a/src/memory/store/content/read.rs +++ b/src/memory/store/content/read.rs @@ -8,8 +8,9 @@ use std::path::{Component, Path, PathBuf}; use super::atomic::sha256_hex; use super::compose::split_front_matter; use crate::memory::chunks::{ - content_root, get_chunk_content_pointers, get_chunk_raw_refs, get_summary_content_pointers, - update_chunk_content_sha256, update_summary_content_sha256, RawRef, + content_root, get_chunk, get_chunk_content_pointers, get_chunk_raw_refs, + get_summary_content_pointers, update_chunk_content_sha256, update_summary_content_sha256, + RawRef, }; use crate::memory::config::MemoryConfig; @@ -136,10 +137,11 @@ pub fn read_chunk_body(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result< } } - let (rel_path, expected_sha256) = get_chunk_content_pointers(config, chunk_id)? - .ok_or_else(|| anyhow::anyhow!("no content pointer or raw refs for chunk {chunk_id}"))?; + let Some((rel_path, expected_sha256)) = get_chunk_content_pointers(config, chunk_id)? else { + return read_legacy_chunk_preview(config, chunk_id); + }; if rel_path.is_empty() { - anyhow::bail!("empty content pointer and no raw refs for chunk {chunk_id}"); + return read_legacy_chunk_preview(config, chunk_id); } let abs_path = resolve_within_content_root(&content_root(config), &rel_path)?; let result = read_chunk_file(&abs_path)?; @@ -154,6 +156,16 @@ pub fn read_chunk_body(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result< Ok(result.body) } +fn read_legacy_chunk_preview(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result { + let Some(chunk) = get_chunk(config, chunk_id)? else { + anyhow::bail!("no content pointer or raw refs for chunk {chunk_id}"); + }; + if chunk.content.is_empty() { + anyhow::bail!("empty content pointer and no raw refs for chunk {chunk_id}"); + } + Ok(chunk.content) +} + fn read_chunk_body_from_raw(config: &MemoryConfig, refs: &[RawRef]) -> anyhow::Result { let root = content_root(config); let mut parts = Vec::with_capacity(refs.len()); diff --git a/src/memory/store/content/read_tests.rs b/src/memory/store/content/read_tests.rs index b0b850d..8c66917 100644 --- a/src/memory/store/content/read_tests.rs +++ b/src/memory/store/content/read_tests.rs @@ -199,6 +199,16 @@ fn high_level_chunk_reader_uses_custom_root_and_repairs_checksum() { assert_eq!(repaired, staged[0].content_sha256); } +#[test] +fn high_level_chunk_reader_falls_back_to_legacy_content_column() { + let dir = TempDir::new().unwrap(); + let config = crate::memory::MemoryConfig::new(dir.path()); + let chunk = sample_chunk(); + crate::memory::chunks::upsert_chunks(&config, std::slice::from_ref(&chunk)).unwrap(); + + assert_eq!(read_chunk_body(&config, &chunk.id).unwrap(), chunk.content); +} + #[test] fn high_level_chunk_reader_joins_clamped_raw_references() { let dir = TempDir::new().unwrap(); From 700bf8b7213dca38c9fd52e52636a95ff3a8bc6b Mon Sep 17 00:00:00 2001 From: hobertrand-hub Date: Sat, 25 Jul 2026 03:38:55 +0200 Subject: [PATCH 2/5] fix(memory): keep empty legacy chunks terminal --- docs/openhuman-memory-migration.md | 32 +++++++------ docs/openhuman-memory/README.md | 6 ++- .../openhuman-memory/sources-registry-sync.md | 6 ++- docs/plan/05-openhuman-compat-matrix.md | 9 ++-- src/memory/chunks/store_embed_tests.rs | 48 +++++++++++++++++++ src/memory/store/content/read.rs | 2 +- src/memory/store/content/read_tests.rs | 16 +++++++ 7 files changed, 95 insertions(+), 24 deletions(-) diff --git a/docs/openhuman-memory-migration.md b/docs/openhuman-memory-migration.md index 7c076a7..7c656d3 100644 --- a/docs/openhuman-memory-migration.md +++ b/docs/openhuman-memory-migration.md @@ -4,10 +4,10 @@ This repository now has a Rust crate rooted at the repository root. The first migration target is the memory core: stable contracts, storage primitives, and testable in-process behavior before API or UI integrations. -TinyCortex owns the generic sync engine and provider pipelines behind its -optional `sync` feature. OpenHuman retains scheduling, credentials, RPC, -source-scope/redaction policy, and event-bus publishing, and supplies those -product concerns through the sync adapter traits. +TinyCortex owns reusable provider fetch, pagination, and canonicalization +pipeline mechanics behind its optional `sync` feature and injected traits. +OpenHuman owns the live sync runner, credentials, scheduling, source policy, +callbacks, RPC, product events, and projections. ## Source Modules @@ -41,10 +41,10 @@ The memory engine now lives under `src/memory/` as cohesive modules: `conversations/`, `archivist/`: specialized memory surfaces. Future host adapters should keep OpenHuman's layer rule: orchestration depends -on storage, but storage does not depend upward on orchestration. Generic sync -fetch/pagination/canonical-record mechanics live in this crate behind injected -traits; OpenHuman retains scheduling, credentials, source policy, event-bus -translation, RPC, and product projections. +on storage, but storage does not depend upward on orchestration. Reusable sync +fetch, pagination, and canonical-record mechanics live in this crate behind +injected traits; OpenHuman retains the live runner, scheduling, credentials, +callbacks, source policy, event-bus translation, RPC, and product projections. ## Migration Order @@ -81,10 +81,12 @@ are the current validation gates). | `conversations` | `memory_conversations` | JSONL transcript store, inverted index, persistence bus. | | `archivist` | `memory_archivist` | Conversation turns → one tree leaf (tool-JSON stripped). Tree-leaf sink injected. | -Per the ownership boundary, the live sync scheduler, OAuth/webhook callbacks, -credentials, and real LLM/embedding/network backends remain host-owned -(OpenHuman) and are represented here as injectable traits. Generic provider -pipelines and workspace reconciliation are crate-owned. Known follow-ups: consolidate legacy -`score::store` entity-index helpers around `store::entity_index`; restore the -deferred peripheral surfaces (tree `health`/`nlp`, retrieval RPC/fast paths, -obsidian/wiki-git content, controller/tool registries) as host adapters land. +Per the ownership boundary, the live sync runner, OAuth/webhook callbacks, +credentials, scheduling, policy, RPC, product events, and real +LLM/embedding/network backends remain host-owned (OpenHuman) and are represented +here as injectable traits. Reusable provider fetch, pagination, and +canonicalization pipeline mechanics are crate-owned. Known follow-ups: +consolidate legacy `score::store` entity-index helpers around +`store::entity_index`; restore the deferred peripheral surfaces (tree +`health`/`nlp`, retrieval RPC/fast paths, obsidian/wiki-git content, +controller/tool registries) as host adapters land. diff --git a/docs/openhuman-memory/README.md b/docs/openhuman-memory/README.md index 4dd0c7b..bcef5d2 100644 --- a/docs/openhuman-memory/README.md +++ b/docs/openhuman-memory/README.md @@ -5,8 +5,10 @@ specifications for the TinyCortex migration. Each document captures the observed OpenHuman contract, the required data attributes, invariants, and the recommended TinyCortex landing area. -Boundary: TinyCortex does not own memory sync. The OpenHuman application -owns the sync module and decides when data is ingested on demand. These specs +Boundary: TinyCortex owns reusable provider fetch, pagination, and +canonicalization pipeline mechanics behind injected traits. OpenHuman owns the +live sync runner, credentials, scheduling, source policy, callbacks, RPC, +product events, and decides when data is ingested on demand. These specs describe the contracts TinyCortex exposes after OpenHuman supplies source data. Source checkout used for this pass: diff --git a/docs/openhuman-memory/sources-registry-sync.md b/docs/openhuman-memory/sources-registry-sync.md index d18e1b7..f807fd0 100644 --- a/docs/openhuman-memory/sources-registry-sync.md +++ b/docs/openhuman-memory/sources-registry-sync.md @@ -191,5 +191,7 @@ src/memory/ingest/canonicalize/ Port order: source kind/types/validation, registry patch semantics, reader trait and static reader contracts, canonicalizer pure functions, then -OpenHuman-facing ingest and sync adapters. The provider pipeline is crate-owned; -the live scheduler, credentials, policy, and product events stay in OpenHuman. +OpenHuman-facing ingest and sync adapters. Reusable provider fetch, pagination, +and canonicalization pipeline mechanics are crate-owned; the live sync runner, +credentials, scheduling, callbacks, policy, RPC, and product events stay in +OpenHuman. diff --git a/docs/plan/05-openhuman-compat-matrix.md b/docs/plan/05-openhuman-compat-matrix.md index 59dfe75..933345d 100644 --- a/docs/plan/05-openhuman-compat-matrix.md +++ b/docs/plan/05-openhuman-compat-matrix.md @@ -71,10 +71,11 @@ kind fields + sync budgets `types.rs:68-146`; discriminator validation reconciliation with `source_kind = raw_file` so interrupted syncs don't strand raw files (spec: sources-registry-sync.md §Raw Archive Coverage). -Generic Composio provider fetch/pagination pipelines are crate-owned behind -injected network and persistence traits. Provider credentials, the scheduler, -source policy, RPC, events, and non-Composio product integrations remain -host-owned. **Do not port the live sync scheduler.** +Generic provider fetch, pagination, and canonicalization pipeline mechanics are +crate-owned behind injected network and persistence traits. Provider +credentials, the live sync runner, scheduling, callbacks, source policy, RPC, +events, and non-Composio product integrations remain host-owned. **Do not port +the live sync scheduler.** --- diff --git a/src/memory/chunks/store_embed_tests.rs b/src/memory/chunks/store_embed_tests.rs index d89c99a..a25ba4f 100644 --- a/src/memory/chunks/store_embed_tests.rs +++ b/src/memory/chunks/store_embed_tests.rs @@ -365,6 +365,54 @@ fn legacy_body_read_skip_does_not_hide_reembed_work() { .unwrap(); } +#[test] +fn legacy_empty_pointer_skip_does_not_hide_reembed_work() { + let (_tmp, cfg) = test_config(); + let c = sample_chunk("persona/communication", 0, 1_700_000_000_000); + upsert_chunks(&cfg, std::slice::from_ref(&c)).unwrap(); + let sig = "bge-m3@1024"; + mark_chunk_reembed_skipped( + &cfg, + &c.id, + sig, + &format!( + "body read failed: empty content pointer and no raw refs for chunk {}", + c.id + ), + ) + .unwrap(); + + with_connection(&cfg, |conn| { + assert!(has_uncovered_reembed_work(conn, sig)?); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn empty_legacy_content_skip_hides_reembed_work() { + let (_tmp, cfg) = test_config(); + let c = sample_chunk("persona/communication", 0, 1_700_000_000_000); + upsert_chunks(&cfg, std::slice::from_ref(&c)).unwrap(); + let sig = "bge-m3@1024"; + mark_chunk_reembed_skipped( + &cfg, + &c.id, + sig, + &format!( + "body read failed: legacy chunk content empty for chunk {}", + c.id + ), + ) + .unwrap(); + + with_connection(&cfg, |conn| { + assert!(!has_uncovered_reembed_work(conn, sig)?); + Ok(()) + }) + .unwrap(); +} + #[test] fn non_legacy_skip_still_hides_reembed_work() { let (_tmp, cfg) = test_config(); diff --git a/src/memory/store/content/read.rs b/src/memory/store/content/read.rs index fb09df7..9f91cdb 100644 --- a/src/memory/store/content/read.rs +++ b/src/memory/store/content/read.rs @@ -161,7 +161,7 @@ fn read_legacy_chunk_preview(config: &MemoryConfig, chunk_id: &str) -> anyhow::R anyhow::bail!("no content pointer or raw refs for chunk {chunk_id}"); }; if chunk.content.is_empty() { - anyhow::bail!("empty content pointer and no raw refs for chunk {chunk_id}"); + anyhow::bail!("legacy chunk content empty for chunk {chunk_id}"); } Ok(chunk.content) } diff --git a/src/memory/store/content/read_tests.rs b/src/memory/store/content/read_tests.rs index 8c66917..332d432 100644 --- a/src/memory/store/content/read_tests.rs +++ b/src/memory/store/content/read_tests.rs @@ -209,6 +209,22 @@ fn high_level_chunk_reader_falls_back_to_legacy_content_column() { assert_eq!(read_chunk_body(&config, &chunk.id).unwrap(), chunk.content); } +#[test] +fn high_level_chunk_reader_empty_legacy_content_uses_terminal_error() { + let dir = TempDir::new().unwrap(); + let config = crate::memory::MemoryConfig::new(dir.path()); + let mut chunk = sample_chunk(); + chunk.content.clear(); + crate::memory::chunks::upsert_chunks(&config, std::slice::from_ref(&chunk)).unwrap(); + + let err = read_chunk_body(&config, &chunk.id).unwrap_err(); + + assert_eq!( + err.to_string(), + format!("legacy chunk content empty for chunk {}", chunk.id) + ); +} + #[test] fn high_level_chunk_reader_joins_clamped_raw_references() { let dir = TempDir::new().unwrap(); From b47bf899be361b6bec0e54f8fd6d5d37c38839da Mon Sep 17 00:00:00 2001 From: hobertrand-hub Date: Mon, 27 Jul 2026 00:23:14 +0200 Subject: [PATCH 3/5] Handle mem_src scopes in source retrieval --- src/memory/retrieval/fast.rs | 19 +++++++++++++++++-- src/memory/retrieval/source.rs | 3 +++ src/memory/retrieval/source_tests.rs | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/memory/retrieval/fast.rs b/src/memory/retrieval/fast.rs index 2322b9a..318552e 100644 --- a/src/memory/retrieval/fast.rs +++ b/src/memory/retrieval/fast.rs @@ -208,7 +208,7 @@ fn resolve_local( let mut hits = Vec::new(); for (id, _) in ordered { if let Some(mut hit) = by_id.remove(&id) { - if source_scope.is_some_and(|scope| !scope.contains(&hit.tree_scope)) { + if source_scope.is_some_and(|scope| !source_scope_allows(scope, &hit.tree_scope)) { continue; } hit.score = coverage.get(&id).copied().unwrap_or_default(); @@ -239,7 +239,9 @@ async fn dense( ) .await?; if let Some(scope) = source_scope { - response.hits.retain(|hit| scope.contains(&hit.tree_scope)); + response + .hits + .retain(|hit| source_scope_allows(scope, &hit.tree_scope)); } let total = response.hits.len(); response.hits.truncate(limit); @@ -282,6 +284,19 @@ fn dedup_ids(ids: impl Iterator) -> Vec { ids.filter(|id| seen.insert(id.clone())).collect() } +fn source_scope_allows(scope: &HashSet, tree_scope: &str) -> bool { + if scope.contains(tree_scope) { + return true; + } + extract_mem_src_id(tree_scope).is_some_and(|id| scope.contains(id)) +} + +fn extract_mem_src_id(value: &str) -> Option<&str> { + let rest = value.strip_prefix("mem_src:")?; + let (id, _) = rest.split_once(':')?; + (!id.is_empty()).then_some(id) +} + #[cfg(test)] #[path = "fast_tests.rs"] mod tests; diff --git a/src/memory/retrieval/source.rs b/src/memory/retrieval/source.rs index fffeb78..077f00f 100644 --- a/src/memory/retrieval/source.rs +++ b/src/memory/retrieval/source.rs @@ -236,6 +236,9 @@ fn scope_matches_kind(scope: &str, kind_prefix: &str) -> bool { if lower.starts_with(&format!("{kind_prefix}:")) { return true; } + if kind_prefix == SourceKind::Document.as_str() && lower.starts_with("mem_src:") { + return true; + } PLATFORM_KINDS .iter() .any(|(platform, kind)| *kind == kind_prefix && lower.starts_with(&format!("{platform}:"))) diff --git a/src/memory/retrieval/source_tests.rs b/src/memory/retrieval/source_tests.rs index 343aded..b8d0e3d 100644 --- a/src/memory/retrieval/source_tests.rs +++ b/src/memory/retrieval/source_tests.rs @@ -195,6 +195,10 @@ fn scope_prefix_matching_known_platforms() { assert!(scope_matches_kind("gmail:alice", "email")); assert!(scope_matches_kind("notion:page123", "document")); assert!(scope_matches_kind("linear:conn-1:issue-abc", "document")); + assert!(scope_matches_kind( + "mem_src:src-folder-9:Slides_Notes/example.md", + "document" + )); assert!(!scope_matches_kind("slack:#eng", "email")); assert!(scope_matches_kind("chat:custom", "chat")); } From 1fd4c1f56a69c2c9d275871937a7bdd50bbf6818 Mon Sep 17 00:00:00 2001 From: hobertrand-hub Date: Mon, 27 Jul 2026 04:26:20 +0200 Subject: [PATCH 4/5] Clarify mem_src source scope filtering --- src/memory/retrieval/fast.rs | 31 ++++++++++++++++++++++++++-- src/memory/retrieval/fast_tests.rs | 23 +++++++++++++++++++++ src/memory/retrieval/source.rs | 4 ++++ src/memory/retrieval/source_tests.rs | 4 ++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/memory/retrieval/fast.rs b/src/memory/retrieval/fast.rs index 318552e..000b565 100644 --- a/src/memory/retrieval/fast.rs +++ b/src/memory/retrieval/fast.rs @@ -288,15 +288,42 @@ fn source_scope_allows(scope: &HashSet, tree_scope: &str) -> bool { if scope.contains(tree_scope) { return true; } - extract_mem_src_id(tree_scope).is_some_and(|id| scope.contains(id)) + // Caller scopes are source selectors: either the exact tree scope, a bare + // source id such as `src-folder-9`, or the collection prefix + // `mem_src:src-folder-9`. Per-file trees store the full + // `mem_src::` scope, so extract the collection source id + // before applying source-level filtering. + let Some(id) = extract_mem_src_id(tree_scope) else { + return false; + }; + scope.contains(id) + || scope + .iter() + .any(|allowed| mem_src_scope_selects_id(allowed, id)) } fn extract_mem_src_id(value: &str) -> Option<&str> { - let rest = value.strip_prefix("mem_src:")?; + let prefix = value.get(..8)?; + if !prefix.eq_ignore_ascii_case("mem_src:") { + return None; + } + let rest = &value[8..]; let (id, _) = rest.split_once(':')?; (!id.is_empty()).then_some(id) } +fn mem_src_scope_selects_id(value: &str, expected_id: &str) -> bool { + let Some(prefix) = value.get(..8) else { + return false; + }; + if !prefix.eq_ignore_ascii_case("mem_src:") { + return false; + } + let rest = &value[8..]; + let id = rest.split_once(':').map_or(rest, |(id, _)| id); + !id.is_empty() && id == expected_id +} + #[cfg(test)] #[path = "fast_tests.rs"] mod tests; diff --git a/src/memory/retrieval/fast_tests.rs b/src/memory/retrieval/fast_tests.rs index 2b38a43..fa90981 100644 --- a/src/memory/retrieval/fast_tests.rs +++ b/src/memory/retrieval/fast_tests.rs @@ -17,6 +17,29 @@ fn options_and_ids_are_bounded_deterministically() { ); } +#[test] +fn mem_src_scope_filter_accepts_bare_id_collection_prefix_and_exact_scope() { + let tree_scope = "Mem_Src:src-folder-9:Slides_Notes/example.md"; + + assert_eq!(extract_mem_src_id(tree_scope), Some("src-folder-9")); + assert!(source_scope_allows( + &HashSet::from(["src-folder-9".to_string()]), + tree_scope + )); + assert!(source_scope_allows( + &HashSet::from(["mem_src:src-folder-9".to_string()]), + tree_scope + )); + assert!(source_scope_allows( + &HashSet::from([tree_scope.to_string()]), + tree_scope + )); + assert!(!source_scope_allows( + &HashSet::from(["src-other".to_string()]), + tree_scope + )); +} + #[tokio::test] async fn blank_query_is_empty_without_opening_storage() { let (_temp, config) = test_config(); diff --git a/src/memory/retrieval/source.rs b/src/memory/retrieval/source.rs index 077f00f..454ee93 100644 --- a/src/memory/retrieval/source.rs +++ b/src/memory/retrieval/source.rs @@ -236,6 +236,10 @@ fn scope_matches_kind(scope: &str, kind_prefix: &str) -> bool { if lower.starts_with(&format!("{kind_prefix}:")) { return true; } + // OpenHuman document sources encode per-file tree scopes as + // `mem_src::`, so kind filtering must classify the whole + // tree family as documents while exact source-id callers still use the + // source_id path above. if kind_prefix == SourceKind::Document.as_str() && lower.starts_with("mem_src:") { return true; } diff --git a/src/memory/retrieval/source_tests.rs b/src/memory/retrieval/source_tests.rs index b8d0e3d..deb3b7b 100644 --- a/src/memory/retrieval/source_tests.rs +++ b/src/memory/retrieval/source_tests.rs @@ -199,6 +199,10 @@ fn scope_prefix_matching_known_platforms() { "mem_src:src-folder-9:Slides_Notes/example.md", "document" )); + assert!(scope_matches_kind( + "Mem_Src:src-folder-9:Slides_Notes/example.md", + "document" + )); assert!(!scope_matches_kind("slack:#eng", "email")); assert!(scope_matches_kind("chat:custom", "chat")); } From b9e4f56c8f4408a38429f83c751239a26da3ad32 Mon Sep 17 00:00:00 2001 From: hobertrand-hub Date: Mon, 27 Jul 2026 04:30:08 +0200 Subject: [PATCH 5/5] Document legacy reembed skip reasons --- src/memory/chunks/embeddings.rs | 20 +++++++++++++--- src/memory/chunks/store_embed_tests.rs | 10 +++++--- src/memory/store/content/read.rs | 32 ++++++++++++++++++++++---- src/memory/store/content/read_tests.rs | 2 +- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/memory/chunks/embeddings.rs b/src/memory/chunks/embeddings.rs index ff04b42..e4a2e28 100644 --- a/src/memory/chunks/embeddings.rs +++ b/src/memory/chunks/embeddings.rs @@ -12,6 +12,9 @@ use rusqlite::{Connection, OptionalExtension}; use std::collections::HashMap; use crate::memory::config::MemoryConfig; +use crate::memory::store::content::read::{ + LEGACY_EMPTY_CONTENT_POINTER_REASON_PREFIX, LEGACY_NO_CONTENT_POINTER_REASON_PREFIX, +}; /// The active embedding vector dimension for `config`. Drives the legacy /// migration's dim-match decision. @@ -384,6 +387,13 @@ pub fn has_uncovered_reembed_work( conn: &Connection, model_signature: &str, ) -> rusqlite::Result { + let missing_pointer_retry_like = + format!("body read failed: {LEGACY_NO_CONTENT_POINTER_REASON_PREFIX}%"); + // Legacy-only: older readers emitted this for present-but-empty content + // pointers. Current readers filter those pointers before returning `Some`, + // but existing skip rows must still become retriable after the fallback fix. + let empty_pointer_retry_like = + format!("body read failed: {LEGACY_EMPTY_CONTENT_POINTER_REASON_PREFIX}%"); conn.query_row( "SELECT EXISTS( SELECT 1 FROM mem_tree_chunks c @@ -391,8 +401,8 @@ pub fn has_uncovered_reembed_work( WHERE e.chunk_id = c.id AND e.model_signature = ?1) AND NOT EXISTS (SELECT 1 FROM mem_tree_chunk_reembed_skipped sk WHERE sk.chunk_id = c.id AND sk.model_signature = ?1 - AND sk.reason NOT LIKE 'body read failed: no content pointer or raw refs for chunk %' - AND sk.reason NOT LIKE 'body read failed: empty content pointer and no raw refs for chunk %')) + AND sk.reason NOT LIKE ?2 + AND sk.reason NOT LIKE ?3)) OR EXISTS( SELECT 1 FROM mem_tree_summaries s WHERE s.deleted = 0 @@ -400,7 +410,11 @@ pub fn has_uncovered_reembed_work( WHERE e.summary_id = s.id AND e.model_signature = ?1) AND NOT EXISTS (SELECT 1 FROM mem_tree_summary_reembed_skipped sk WHERE sk.summary_id = s.id AND sk.model_signature = ?1))", - rusqlite::params![model_signature], + rusqlite::params![ + model_signature, + missing_pointer_retry_like, + empty_pointer_retry_like + ], |row| row.get(0), ) } diff --git a/src/memory/chunks/store_embed_tests.rs b/src/memory/chunks/store_embed_tests.rs index a25ba4f..a08f4e0 100644 --- a/src/memory/chunks/store_embed_tests.rs +++ b/src/memory/chunks/store_embed_tests.rs @@ -25,6 +25,10 @@ use super::{ GLOBAL_TOPIC_PURGE_MIGRATION_VERSION, }; use crate::memory::config::MemoryConfig; +use crate::memory::store::content::read::{ + LEGACY_EMPTY_CHUNK_CONTENT_REASON_PREFIX, LEGACY_EMPTY_CONTENT_POINTER_REASON_PREFIX, + LEGACY_NO_CONTENT_POINTER_REASON_PREFIX, +}; use crate::memory::tree::store::{ insert_summary_tx, insert_tree, SummaryNode, Tree, TreeKind, TreeStatus, }; @@ -352,7 +356,7 @@ fn legacy_body_read_skip_does_not_hide_reembed_work() { &c.id, sig, &format!( - "body read failed: no content pointer or raw refs for chunk {}", + "body read failed: {LEGACY_NO_CONTENT_POINTER_REASON_PREFIX}{}", c.id ), ) @@ -376,7 +380,7 @@ fn legacy_empty_pointer_skip_does_not_hide_reembed_work() { &c.id, sig, &format!( - "body read failed: empty content pointer and no raw refs for chunk {}", + "body read failed: {LEGACY_EMPTY_CONTENT_POINTER_REASON_PREFIX}{}", c.id ), ) @@ -400,7 +404,7 @@ fn empty_legacy_content_skip_hides_reembed_work() { &c.id, sig, &format!( - "body read failed: legacy chunk content empty for chunk {}", + "body read failed: {LEGACY_EMPTY_CHUNK_CONTENT_REASON_PREFIX}{}", c.id ), ) diff --git a/src/memory/store/content/read.rs b/src/memory/store/content/read.rs index 9f91cdb..5a27bda 100644 --- a/src/memory/store/content/read.rs +++ b/src/memory/store/content/read.rs @@ -14,6 +14,24 @@ use crate::memory::chunks::{ }; use crate::memory::config::MemoryConfig; +/// Prefix for retryable legacy rows that were written before staged content +/// files/raw references existed. `has_uncovered_reembed_work` matches this text +/// inside `body read failed: ...` skip reasons to keep those chunks eligible for +/// backfill once a legacy `content` column fallback is available. +pub(crate) const LEGACY_NO_CONTENT_POINTER_REASON_PREFIX: &str = + "no content pointer or raw refs for chunk "; + +/// Prefix preserved for pre-fix skip rows written by older readers when the +/// staged content pointer was present-but-empty. Current reads no longer emit +/// this text because `get_chunk_content_pointers` filters empty paths before +/// returning `Some`. +pub(crate) const LEGACY_EMPTY_CONTENT_POINTER_REASON_PREFIX: &str = + "empty content pointer and no raw refs for chunk "; + +/// Prefix for terminal legacy rows whose inline content column is empty. +pub(crate) const LEGACY_EMPTY_CHUNK_CONTENT_REASON_PREFIX: &str = + "legacy chunk content empty for chunk "; + /// Resolve a DB-stored relative forward-slash path against `content_root`, /// rejecting any traversal (`..`), absolute, or non-normal component. /// @@ -140,9 +158,6 @@ pub fn read_chunk_body(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result< let Some((rel_path, expected_sha256)) = get_chunk_content_pointers(config, chunk_id)? else { return read_legacy_chunk_preview(config, chunk_id); }; - if rel_path.is_empty() { - return read_legacy_chunk_preview(config, chunk_id); - } let abs_path = resolve_within_content_root(&content_root(config), &rel_path)?; let result = read_chunk_file(&abs_path)?; if result.sha256 != expected_sha256 { @@ -156,12 +171,19 @@ pub fn read_chunk_body(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result< Ok(result.body) } +/// Fallback reader for pre-content-store chunk rows. +/// +/// Its error text prefixes are part of the reembed-backfill contract: +/// `embeddings::has_uncovered_reembed_work` matches the wrapped +/// `body read failed: ...` skip reasons to decide which historical failures are +/// retryable after this fallback exists. Keep the constants above in sync with +/// that SQL before changing wording here. fn read_legacy_chunk_preview(config: &MemoryConfig, chunk_id: &str) -> anyhow::Result { let Some(chunk) = get_chunk(config, chunk_id)? else { - anyhow::bail!("no content pointer or raw refs for chunk {chunk_id}"); + anyhow::bail!("{LEGACY_NO_CONTENT_POINTER_REASON_PREFIX}{chunk_id}"); }; if chunk.content.is_empty() { - anyhow::bail!("legacy chunk content empty for chunk {chunk_id}"); + anyhow::bail!("{LEGACY_EMPTY_CHUNK_CONTENT_REASON_PREFIX}{chunk_id}"); } Ok(chunk.content) } diff --git a/src/memory/store/content/read_tests.rs b/src/memory/store/content/read_tests.rs index 332d432..8927fcd 100644 --- a/src/memory/store/content/read_tests.rs +++ b/src/memory/store/content/read_tests.rs @@ -221,7 +221,7 @@ fn high_level_chunk_reader_empty_legacy_content_uses_terminal_error() { assert_eq!( err.to_string(), - format!("legacy chunk content empty for chunk {}", chunk.id) + format!("{LEGACY_EMPTY_CHUNK_CONTENT_REASON_PREFIX}{}", chunk.id) ); }