diff --git a/CHANGELOG.md b/CHANGELOG.md index 6516988d..9fc88eda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **App and project scope identifiers now use a portable lowercase filesystem + contract.** Values are limited to 1–128 lowercase ASCII letters, digits, + underscore, dot, hyphen, `@`, or `+`. Trailing dots, Windows device names, + default-directory aliases, and runtime-owned app names are rejected with HTTP + 422. Public response IDs and sender/owner validation remain unchanged. + Existing roots containing nonconforming scope names require coordinated + migration of both source directories and retained scoped SQLite state before + upgrade; `cascade rebuild` does not perform that name migration. +- **LanceDB row identity is now scoped by app, project, and owner.** Existing + roots with conforming scope names require an offline + `everos cascade rebuild --yes`. A storage-generation marker, lifecycle locks, + and serialized bootstrap keep legacy or incomplete projections fail-closed. + +### Fixed + +- Prevented owner-local memory identifiers in separate projects from + overwriting each other in LanceDB. The repair covers episodes, atomic facts, + foresights, agent cases, agent skills, and user profiles while retaining the + historical public response identifiers. + ## [1.2.3] - 2026-08-07 **Background maintenance that fails loudly instead of quietly.** A soak run on diff --git a/QUICKSTART.md b/QUICKSTART.md index 52e5216e..04401ee3 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -235,8 +235,11 @@ Your extracted memory is a normal Markdown file under the memory root: └── lancedb/ ``` -Markdown is canonical; SQLite and LanceDB are derived indexes. You can read, -edit, diff, and version the memory files without a database client. +Markdown is canonical for extracted memory, and LanceDB is its rebuildable +search projection. SQLite also contains buffered and coordination state that is +not reconstructable from markdown. You can read, edit, diff, and version the +memory files without a database client, but should use `cascade rebuild` rather +than deleting `.index` manually. ## Upgrade capabilities when you need them diff --git a/docs/api.md b/docs/api.md index 7bb529b4..38ec07cc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -138,7 +138,7 @@ storage. This is the same rule users see when reading rendered output: #### Other conventions -- **Server-generated IDs** follow `___`, +- **Public response IDs** follow `___`, e.g. `alice_ep_20260528_00000001` for an episode, `alice_af_...` for an atomic fact. See [storage_layout.md §4](storage_layout.md) for the encoding. @@ -156,6 +156,20 @@ for the agent track). The default scope materialises on disk as added only for the literal id `"default"` so the default space stays visually distinct from user-named scopes). +Scope identifiers are lowercase because they are used as raw directory +segments. Rejecting case variants prevents two logical scopes from resolving +to the same directory on case-insensitive filesystems. + +This is a stricter contract than earlier releases. Before upgrading an existing +root, audit both its scope directories and retained SQLite state for uppercase, +reserved, trailing-dot, or device-name scopes. If any exist, do not start the +new binary or run `cascade rebuild` yet. This release has no general supported +in-place scope-name migration. Use a fresh-root markdown import or a separately +reviewed, deployment-specific migration that updates paths and retained SQLite +state together. Rebuild only reconstructs LanceDB; it does not rewrite retained +SQLite scope values. See the +[cascade runbook](cascade_runbook.md#legacy-nonconforming-scope-names). + A `/search` or `/get` query never crosses scopes — different `(app_id, project_id)` pairs are isolated. @@ -166,8 +180,14 @@ Both fields share the same validation: | Type | `string` | | Default | `"default"` | | Length | 1–128 chars | -| Charset | `^[a-zA-Z0-9_.-]+$` | -| Rejected literals | `"."` and `".."` (path-traversal guard) | +| Charset | `^[a-z0-9_.@+-]+$` | +| Rejected literals | `"."`, `".."`, trailing dots, Windows device names, and reserved storage/configuration names | + +For `app_id`, the reserved storage/configuration names are `.index`, `.tmp`, +`.lock`, `.projection.lock`, `default_app`, `everos.toml`, and `ome.toml`. +For `project_id`, `default_project` is reserved. These restrictions apply only +to app and project scopes; sender and owner identifiers retain their existing +mixed-case validation contract. ## Errors diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe3..d5c5af1b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,8 +203,13 @@ Three-piece observability: └── knowledge/ # global shared knowledge ``` -System-managed entries (`.index/`, `.tmp/`) and `ome.toml` live directly -under the memory root. +System-managed entries (`.index/`, `.tmp/`, `.lock`, and `.projection.lock`) +and the `everos.toml` / `ome.toml` configuration files live directly under +the memory root. + +`app_id` and `project_id` use a lowercase portable filesystem grammar because +they are stored as raw path segments. Runtime/configuration names are reserved +at the app level so user memory cannot overlap system-managed root entries. Full tree + frontmatter chassis: [storage_layout.md](storage_layout.md) and [how-memory-works.md](how-memory-works.md). Frontmatter has 4-tier field protection (L1 read-only / L2 system / L3 business / L4 user). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 3e0dcd90..8bf2057d 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -131,22 +131,83 @@ everos cascade rebuild # prompts for confirmation everos cascade rebuild --yes # non-interactive ``` -> **Stop the `everos server` first.** Unlike `cascade sync`, rebuild -> **drops and recreates** the LanceDB tables. A running daemon holds -> cached table handles that would keep pointing at (and writing to) the -> dropped dataset, corrupting the rebuild. This is the one cascade -> command that is **not** safe to run alongside a live server. +> **Stop the `everos server` and other mutating cascade commands first.** +> Unlike `cascade sync`, rebuild **drops and recreates** the LanceDB tables. +> A running daemon or CLI writer holds cached table handles that would keep +> pointing at the dropped dataset. Current servers and `sync`, `fix --apply`, +> and `backfill` hold the shared projection lifecycle lock for their complete +> LanceDB lifetime. Rebuild requires the exclusive side plus the legacy OME +> lock, and refuses to start while any such process is active. During an +> upgrade, do not launch an older binary concurrently with rebuild: an old +> process does not participate in the projection-lock protocol until its OME +> guard has been acquired. +> +> After a memory root has been rebuilt for storage generation 2, do not run an +> older EverOS binary against that root at any time. Older binaries ignore the +> generation marker and can write legacy row IDs while it still says +> `READY(2)`. If that happens, stop every EverOS process and run +> `everos cascade rebuild --yes` with the current binary before restarting. What it does, in order: -1. **Drops** every business LanceDB table (`drop_business_tables`) and - evicts them from the connection cache. -2. **Recreates** them empty from the current schema + FTS indexes - (`ensure_business_indexes`). -3. **Clears** the cascade queue (`md_change_state.reset_all`) so every - md file re-enqueues as `added` on the next scan. -4. **Re-scans + drains** (`sync_once`): re-embeds and re-inserts every - md entry. +1. **Establishes the exclusion precondition** described above. If either the + projection lifecycle lock or legacy OME lock is not free, rebuild exits + before changing the marker, queue, or LanceDB tables. +2. **Atomically publishes `REBUILDING` for storage generation 2** before + opening the stores or performing any destructive work. This invalidates + a previously valid `READY` marker before a crash can expose a partial + rebuild. +3. **Clears** the cascade queue (`md_change_state.reset_all`) so every md + file re-enqueues as `added` on the next scan. +4. **Drops** every business LanceDB table (`drop_business_tables`) and + evicts it from the connection cache. +5. **Recreates** the tables from the current schema and builds the FTS + indexes (`ensure_business_indexes`). +6. **Re-scans and drains** (`sync_once`, followed by additional drains + while work remains): re-embeds and re-inserts every md entry. +7. **Requires a clean completion state**: `pending == 0`, + `failed_retryable == 0`, and `failed_permanent == 0`. If any count is + nonzero, rebuild fails and leaves the marker as `REBUILDING`. +8. **Atomically publishes `READY(2)`** only after all completion checks + pass. Only then does the command print `rebuild complete`. + +If the process crashes, is interrupted, or encounters an error after +`REBUILDING` has been published but before `READY(2)` is published, the marker +remains `REBUILDING`. The API server and mutating cascade commands fail closed +rather than serving or changing a partial index. Correct the underlying error +and rerun `everos cascade rebuild` with the server stopped; the rebuild +recovery path deliberately does not require a `READY` marker. + +### Storage-generation gate on startup + +Generation 2 changes the value-level identity stored in LanceDB row primary +keys. A schema check cannot distinguish generation-1 keys from generation-2 +keys, so startup validates `.index/lancedb/.storage_identity.json` before it +opens LanceDB or runs schema/index migrations. + +Startup and mutating cascade commands acquire the shared projection lifecycle +lock before checking the marker and keep it until their LanceDB handles close. +They accept only `READY(2)` and refuse to proceed when the marker is: + +- missing on a memory root that contains source markdown or LanceDB artifacts; +- malformed or contains unknown fields or JSON types; +- `READY` for any generation other than 2; or +- `REBUILDING`, including after an interrupted or failed rebuild. + +A marker-less root is initialized directly as `READY(2)` only when its +extracted-memory source and projection are empty: it has neither source +markdown nor LanceDB artifacts. Retained SQLite state is not proof of a fresh +root, so existing installations must still perform the scope audit below. A +bootstrap lock serializes first-time marker, schema, and index creation across +API and mutating CLI processes. For every blocked state above, stop the server +and rerun `everos cascade rebuild`. The read-only `cascade status` and +`cascade fix` listing remain available through SQLite so operators can inspect +the queue without opening LanceDB. + +Scope directories must use the lowercase portable grammar documented in +[`api.md`](api.md#scopeid-app_id-and-project_id). Existing roots with only +conforming scope names can proceed directly to the generation-2 rebuild. +Nonconforming roots require the separate migration described below. It deliberately **skips `verify_business_schemas`** — the drift it recovers from would otherwise trip that guard on startup before the @@ -154,11 +215,45 @@ rebuild could run (chicken-and-egg). Why not a bare `rm`: -| Recovery | Re-populates `done` entries | Preserves `unprocessed_buffer` | +| Recovery | Current result | Preserves `unprocessed_buffer` | |---|---|---| -| `rm -rf .index/lancedb` | ❌ scanner skips `done` rows → empty index | ✅ | -| `rm -rf .index` | ✅ | ❌ deletes un-extracted messages | -| `everos cascade rebuild` | ✅ | ✅ | +| Remove `.index/lancedb` | Unsupported; existing-root startup fails the generation gate | Yes | +| Remove `.index` | Unsupported; startup fails the generation gate and SQLite-only state is lost | No | +| `everos cascade rebuild` | Rebuilds all source markdown and publishes `READY(2)` | Yes | + +### Legacy nonconforming scope names + +Earlier releases accepted scope names that are unsafe or ambiguous on common +filesystems, including uppercase letters, trailing dots, Windows device names, +and names now reserved for runtime state such as `.index` and `.tmp`. + +Before upgrading an existing root, inspect both the source tree and retained +SQLite data. If any `app_id` or `project_id` is nonconforming: + +1. Stop every EverOS process and take a complete backup of the memory root. +2. Do not run the new server or `cascade rebuild` against that root yet. +3. Do not hand-edit only the directory names or only the database. This release + does not provide a general supported in-place scope-name migration. +4. Either import the extracted markdown into a fresh root with unique accepted + scope names, or use a separately reviewed, deployment-specific migration + that updates the business-scope directories and every matching `app_id`, + `project_id`, stored markdown-path reference, and serialized scope value in + all SQLite databases as one transactionally planned maintenance operation. + This includes scope values embedded in OME `run_record.event_payload` JSON, + not only ordinary columns in `system.db`. +5. After that migration completes, run `everos cascade rebuild --yes` with the + current binary, then restart. + +`cascade rebuild` resets the cascade queue and reconstructs LanceDB, but it +intentionally preserves `unprocessed_buffer`, `memcell`, +`conversation_status`, knowledge, cluster, and reflection state in SQLite. It +therefore cannot perform the scope-name migration by itself. There is no +automatic migration for these legacy names in this release. + +For a legacy app literally named `.index` or `.tmp`, do not rename the whole +directory: those paths also contain runtime-owned state. Prefer a fresh-root +import; any attempt to separate business content from those directories needs +its own reviewed migration rather than an ad hoc filesystem move. ## Recovery paths @@ -186,14 +281,13 @@ and it would detonate later inside `merge_insert` as an opaque `LanceError(IO): Spill has sent an error` (EverOS #337). The type check turns that into this clean startup error. -Recover with **`everos cascade rebuild`** (documented above). Do **not** just -`rm -rf ~/.everos/.index/lancedb`: that clears the vectors but leaves -`md_change_state` marked `done`, so the scanner skips every already- -indexed file and the index comes back **empty**. And do **not** -`rm -rf ~/.everos/.index`: that also deletes `unprocessed_buffer` -(messages received but not yet extracted — not rebuildable from md). -`cascade rebuild` is correct on both counts. Markdown is the source of -truth, so no memory content is lost. +Recover with **`everos cascade rebuild`** (documented above). Do **not** remove +`~/.everos/.index/lancedb`: that also removes the storage-generation marker, +so current startup fails closed when the root contains source markdown or +projection artifacts. Do **not** remove `~/.everos/.index`: that additionally +deletes `unprocessed_buffer` messages that have not yet become markdown. +`cascade rebuild` preserves that SQLite-only state, rebuilds every source +entry, and publishes `READY(2)` only after completion checks pass. ### inotify watch-limit exhaustion (Linux) diff --git a/docs/how-memory-works.md b/docs/how-memory-works.md index be6fda71..defc0a09 100644 --- a/docs/how-memory-works.md +++ b/docs/how-memory-works.md @@ -24,19 +24,22 @@ This is the narrative companion to the reference docs: see ## The storage stack Three embedded pieces, each owning what it is best at. Markdown is the -**source of truth**; the other two are **derived and rebuildable**. +**source of truth for extracted memory**. LanceDB is rebuildable; SQLite also +contains operational state that markdown does not contain. | Layer | Backed by | Holds | Rebuildable? | |---|---|---|---| | **Markdown + YAML frontmatter** | plain `.md` files | the memory content itself — the only portable, human-editable asset | — (it *is* the truth) | -| **SQLite** (`aiosqlite`) | `.index/sqlite/*.db` | system state, audit log, the cascade queue, the boundary buffer, OME engine state | ✅ from markdown | +| **SQLite** (`aiosqlite`) | `.index/sqlite/*.db` | system state, audit log, the cascade queue, the boundary buffer, OME engine state | No, not completely | | **LanceDB** (Arrow) | `.index/lancedb/*.lance` | vector + BM25 + scalar columns for retrieval | ✅ from markdown | !!! note "The one rule that follows from this" - Delete the entire `.index/` directory and **no memory is lost** — it - rebuilds from the `.md` tree. There is no separate "export"; the - markdown *is* the export. (How to trigger a rebuild: - [Operating it](#operating-it).) + Markdown is the source of truth for extracted memory, but `.index/` also + contains SQLite-only state such as unprocessed buffered messages. Never + delete `.index/` or `.index/lancedb` manually. Use + `everos cascade rebuild`, which preserves SQLite state and rebuilds the + LanceDB projection from markdown. There is no separate markdown export; + the markdown tree is the extracted-memory record. ## Storage paths @@ -45,11 +48,17 @@ The default memory root is **`~/.everos/`** (override with inside the memory root as `everos.toml` (generated by `everos init`). Memory is partitioned by **`/`** *before* the -user-visible directories, so different `(app, project)` spaces never share -a directory or cross in search. The reserved id `"default"` materialises as +user-visible directories. In the managed layout, without operator-created +symlink aliases, different `(app, project)` spaces do not share a directory or +cross in search. The reserved id `"default"` materialises as `default_app` / `default_project` on disk (so a default space stays visually distinct from a user-named one). +App and project identifiers use lowercase portable filesystem names. This +prevents case variants from collapsing onto one directory on macOS or Windows. +Reserved runtime/configuration names are rejected for the app component; see +the [ScopeId contract](api.md#scopeid-app_id-and-project_id). + ``` ~/.everos/ ← memory root (EVEROS_ROOT) ├── default_app/ ← ("default" → default_app) @@ -71,7 +80,7 @@ visually distinct from a user-named one). │ │ └── skill_/SKILL.md (+ references/ scripts/) │ └── knowledge/ ← shared / global │ -├── .index/ ← system-managed, rebuildable (gitignore) +├── .index/ ← system-managed runtime state (gitignore; do not delete) │ ├── sqlite/ │ │ ├── system.db state / audit / cascade queue (md_change_state) / buffer / LSN │ │ ├── ome.db Offline Memory Engine state @@ -304,9 +313,10 @@ The CLI ([cli.md](cli.md)) is intentionally small: the whole index, run `everos cascade rebuild` — it drops the LanceDB tables and re-indexes from md, re-populating even entries the queue already marked `done` and preserving un-extracted - buffered messages. (A bare `rm -rf /.index/lancedb` - is **not** enough: the cascade queue still shows those files - `done`, so the scanner skips them and the index comes back empty.) + buffered messages. Do not remove `.index/lancedb` manually: current + startup fails closed because that also removes the required storage + generation marker. Do not remove `.index`, which additionally deletes + SQLite-only state that markdown cannot reconstruct. For an incremental catch-up, use `everos cascade sync`. - **Flush** is an HTTP endpoint (`POST /api/v2/memory/flush`), not a CLI command — it forces *extraction* of the session buffer, which is diff --git a/docs/knowledge.md b/docs/knowledge.md index d3ad92dd..ab22e631 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -47,9 +47,10 @@ Each level corresponds to a different granularity of API: ## Storage layout -Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +Every document is a self-contained directory. Markdown files are the source of +truth for extracted document content, and the cascade daemon builds its search +projection automatically. SQLite also holds metadata and broader operational +state, so the `.index` tree is not disposable as a whole. ``` ~/.everos///knowledge/ diff --git a/docs/openapi.json b/docs/openapi.json index ebe35b82..b2772000 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -313,6 +313,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -323,6 +326,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -497,6 +503,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -507,6 +516,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -557,6 +569,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -567,6 +582,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -673,6 +691,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -683,6 +704,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -769,6 +793,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -779,6 +806,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1073,6 +1103,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -1083,6 +1116,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1257,6 +1293,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -1267,6 +1306,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1317,6 +1359,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -1327,6 +1372,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1433,6 +1481,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -1443,6 +1494,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1529,6 +1583,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "App Id" } @@ -1539,6 +1596,9 @@ "required": false, "schema": { "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-z0-9_.@+-]+$", "default": "default", "title": "Project Id" } @@ -1630,11 +1690,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -1683,11 +1749,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -1736,11 +1808,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -1789,11 +1867,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -2252,11 +2336,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -2679,11 +2769,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" }, @@ -2890,11 +2986,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -2945,7 +3047,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.@+-]+$", + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, @@ -2953,7 +3055,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.@+-]+$", + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" }, @@ -2986,7 +3088,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.@+-]+$", + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, @@ -2994,7 +3096,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.@+-]+$", + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -3590,11 +3692,17 @@ }, "app_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "App Id", "default": "default" }, "project_id": { "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": "^[a-z0-9_.@+-]+$", "title": "Project Id", "default": "default" }, diff --git a/docs/overview.md b/docs/overview.md index 07297673..9117076c 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -30,8 +30,9 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md -delete any md file → memory is gone +delete LanceDB through cascade rebuild → extracted memory is re-indexed from md +delete SQLite → buffered and coordination state is lost +delete any md file → extracted memory content is gone ``` User trust comes from physical visibility — the user can `cat` / `vim` / `grep` their own memory at any time. diff --git a/docs/reflection.md b/docs/reflection.md index 63c09f1f..b54a0d0a 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,8 +147,9 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Markdown is the source of truth for extracted memory, and LanceDB is its +rebuildable search projection. SQLite also retains coordination and audit state +that cannot be reconstructed completely from markdown. | Store | What it holds | Role | |---|---|---| diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 86044664..93e42418 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +Markdown is the **source of truth for extracted memory content**. LanceDB is a +rebuildable projection. SQLite also contains coordination and in-flight state, +including buffered messages, that cannot be reconstructed from markdown alone. ## 1. Memory-root tree @@ -13,14 +14,18 @@ default location is `~/.everos/`; override via the `EVEROS_ROOT` env var or `--root` on the CLI. Memory is partitioned by **`/`** *before* the -user-visible scope dirs, so different `(app, project)` spaces never share -a directory. The reserved id `"default"` materialises as `default_app` / -`default_project` on disk. The scope is encoded **in the path**, not in -the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). +user-visible scope dirs. In the managed layout, without operator-created +symlink aliases, different accepted `(app, project)` spaces do not share a +directory. App and project identifiers use a portable lowercase filesystem +grammar; uppercase and reserved system names are rejected before path +construction. The reserved id `"default"` materialises as `default_app` / +`default_project` on disk. The scope is encoded **in the path**, not in the +frontmatter (see [§3](#3-frontmatter-chassis-yaml)). ``` / default ~/.everos │ +├── .projection.lock shared server/CLI vs exclusive rebuild lifecycle lock ├── / user-visible; "default" → default_app │ └── / "default" → default_project │ ├── users/ @@ -43,13 +48,15 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ └── scripts/ (optional) │ └── knowledge/ user-visible (shared / global) │ -├── .index/ system-managed, rebuildable (gitignore) +├── .index/ system-managed runtime state (gitignore) +│ ├── .projection.bootstrap.lock serializes first-time marker/schema/index setup │ ├── sqlite/ │ │ ├── system.db state / cascade queue (md_change_state) / buffer / audit / LSN (+ -wal / -shm) │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) │ └── lancedb/ +│ ├── .storage_identity.json READY/REBUILDING storage-key generation gate │ └── .lance/ one directory per LanceDB table │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) @@ -62,6 +69,15 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). > `.lock` anchor for the `memory_root_lock` primitive; there is no > `.cascade.log` / `.manifest.json`.) +`.projection.lock` is retained in shared mode by API servers and mutating +cascade commands, and in exclusive mode by `cascade rebuild`. First-time +marker, schema, and index setup is serialized by a separate bootstrap lock. +The storage-identity marker is published as `READY(2)` after a successful +rebuild or when a marker-less root has neither source markdown nor LanceDB +artifacts. Retained SQLite state is separate and still requires the upgrade +audit described in the cascade runbook. Missing, malformed, stale, or +`REBUILDING` state blocks normal writers. + The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the @@ -167,10 +183,12 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). -> Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> Cross-table joins must therefore key on **`(app_id, project_id, owner_id, +> entry_id)`** rather than `entry_id` alone. LanceDB uses a separate opaque, +> injective storage key for this complete identity while public response IDs +> retain their historical shape. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite runtime state + LanceDB projection ``` .index/ @@ -179,6 +197,7 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) └── lancedb/ + ├── .storage_identity.json fail-closed storage-key generation marker └── .lance/ one Arrow table per business kind — the per-kind rows (text / vector / tokens / metadata) live here ``` @@ -191,19 +210,21 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). - **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. + holds the per-kind business rows. Owner-scoped tables use an opaque, + injective storage key over app, project, owner, and the type-specific + logical id. Logical references remain in their explicit columns; each + table's `Vector(N)` dimension matches the embedding model output. Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` column. When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. -Both layers are **fully derivable from markdown** — wipe `.index/` -and the in-process cascade subsystem re-builds everything by scanning the -user-visible tree (the durable `md_change_state` SQLite queue covers -crash-recovery replay). +LanceDB business rows are a rebuildable projection of markdown. SQLite is not +fully disposable: it also contains `unprocessed_buffer` messages that have not +yet become markdown. Do not wipe `.index/` or `.index/lancedb` manually. +Use `everos cascade rebuild`, which preserves SQLite state, resets the durable +queue, rebuilds LanceDB, and publishes the required storage-identity marker. ## 6. Atomic write semantics diff --git a/pyproject.toml b/pyproject.toml index 991d9c91..17c886ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,6 +249,8 @@ ignore_imports = [ "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", + "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.projection_lock", + "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.storage_identity", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", diff --git a/scripts/e2e_memorize/README.md b/scripts/e2e_memorize/README.md index 7cd9373d..04a90cff 100644 --- a/scripts/e2e_memorize/README.md +++ b/scripts/e2e_memorize/README.md @@ -91,11 +91,18 @@ After `flush` the buffer should be empty for the test session. ### 6. Reset between runs -The fixture's session_id is randomised per invocation, so previous runs -don't pollute the new one. To wipe everything: +The fixture's session_id is randomised per invocation, so previous runs do not +pollute the new one. Do not delete `system.db` inside an active or valuable +memory root: it contains buffered and coordination state, and leaving the +LanceDB generation marker behind would produce an inconsistent root. + +For a fully disposable test run, stop every EverOS process and point +`EVEROS_ROOT` at a dedicated temporary directory. Remove that entire dedicated +test root between runs, never the default `~/.everos` root: ```bash -rm -rf ~/.everos/users ~/.everos/agents ~/.everos/.index/sqlite/system.db +export EVEROS_ROOT="$(mktemp -d)" +# run the fixture, stop EverOS, then remove only this dedicated test directory ``` ## Boundary expectations cheat sheet diff --git a/src/everos/core/persistence/lancedb/repository.py b/src/everos/core/persistence/lancedb/repository.py index de79702c..d1e65321 100644 --- a/src/everos/core/persistence/lancedb/repository.py +++ b/src/everos/core/persistence/lancedb/repository.py @@ -156,9 +156,8 @@ def _q(value: str) -> str: LanceDB has no parameterised query API; predicates are strings. Doubling the quote (``'`` → ``''``) is the SQL-standard way to keep - a literal single quote inside a single-quoted string. everos's PK - convention (``_``) never carries quotes — this - is defensive. + a literal single quote inside a single-quoted string. Current storage keys + never carry quotes; this remains defensive for generic callers. """ return value.replace("'", "''") @@ -663,8 +662,8 @@ async def get_by_id( Uses LanceDB scalar filter `` = ''``. Single quotes in ``id_value`` are doubled to avoid breaking the SQL-like - predicate; everos's PK convention is ``_`` - which never contains quotes, so the escape is defensive. + predicate. Current storage keys never contain quotes, so the escape is + defensive for them and required for arbitrary ``id_field`` callers. """ async with self._deadline(_READ_TIMEOUT_SECONDS, "get_by_id"): table = await self._table() diff --git a/src/everos/core/persistence/lancedb/row_id.py b/src/everos/core/persistence/lancedb/row_id.py new file mode 100644 index 00000000..220dd9b0 --- /dev/null +++ b/src/everos/core/persistence/lancedb/row_id.py @@ -0,0 +1,62 @@ +"""LanceDB storage identities and stable public wire identities. + +LanceDB upserts need an identifier that is unique across the complete +``(app, project, owner, logical id)`` partition. Public API identifiers keep +their historical shape for compatibility and are deliberately constructed +separately at the response boundary. +""" + +from __future__ import annotations + +STORAGE_ID_GENERATION = 2 + + +def make_storage_id(*parts: str) -> str: + """Return an injective length-prefixed encoding of UTF-8 string parts.""" + return "".join(f"{len(part.encode('utf-8'))}:{part}" for part in parts) + + +def daily_log_storage_id( + *, app_id: str, project_id: str, owner_id: str, entry_id: str +) -> str: + """Storage primary key for episode/fact/foresight/case rows.""" + return make_storage_id(app_id, project_id, owner_id, entry_id) + + +def agent_skill_storage_id( + *, app_id: str, project_id: str, owner_id: str, name: str +) -> str: + """Storage primary key for one named agent skill.""" + return make_storage_id(app_id, project_id, owner_id, name) + + +def user_profile_storage_id(*, app_id: str, project_id: str, owner_id: str) -> str: + """Storage primary key for one scoped user profile.""" + return make_storage_id(app_id, project_id, owner_id) + + +def daily_log_wire_id(*, owner_id: str, entry_id: str) -> str: + """Historical HTTP identifier for a daily-log row.""" + return f"{owner_id}_{entry_id}" + + +def agent_skill_wire_id(*, owner_id: str, name: str) -> str: + """Historical HTTP identifier for an agent skill.""" + return f"{owner_id}_{name}" + + +def user_profile_wire_id(*, owner_id: str) -> str: + """Historical HTTP identifier for a user profile.""" + return owner_id + + +__all__ = [ + "STORAGE_ID_GENERATION", + "agent_skill_storage_id", + "agent_skill_wire_id", + "daily_log_storage_id", + "daily_log_wire_id", + "make_storage_id", + "user_profile_storage_id", + "user_profile_wire_id", +] diff --git a/src/everos/core/persistence/markdown/entries.py b/src/everos/core/persistence/markdown/entries.py index 31f83c81..66d95752 100644 --- a/src/everos/core/persistence/markdown/entries.py +++ b/src/everos/core/persistence/markdown/entries.py @@ -44,9 +44,9 @@ strings, and manually-typed timestamps; the strong-typed model lives in business writers + the SQLite/LanceDB indexes. -Cross-user uniqueness is handled at the database layer via a composite -``_`` field; it is *not* encoded into the -:class:`EntryId` string itself. +Database storage uniqueness is handled separately from :class:`EntryId` by +an injective key over app, project, owner, and logical entry id. The public +wire id remains ``_`` for compatibility. """ from __future__ import annotations diff --git a/src/everos/core/persistence/markdown/frontmatter.py b/src/everos/core/persistence/markdown/frontmatter.py index 97eb77f9..82552c22 100644 --- a/src/everos/core/persistence/markdown/frontmatter.py +++ b/src/everos/core/persistence/markdown/frontmatter.py @@ -293,10 +293,11 @@ def sanitize_skill_name(cls, skill_name: str) -> str: filesystem the same pair simply stays two independent skills. Because ``AgentSkillWriter.write_main`` is a full-file replace and - the LanceDB primary key is ``f"{agent_id}_{sanitized_name}"``, a - collision means the later skill silently overwrites the earlier - one — its accumulated ``source_case_ids``, ``maturity_score``, and - body are lost, not merged. + both the public identity and scoped LanceDB storage key derive from + ``(agent_id, sanitized_name)``, a collision means the later skill + silently overwrites the earlier one. Its accumulated + ``source_case_ids``, ``maturity_score``, and body are lost, not + merged. This is deliberate, not an oversight — but not because a collision "usually reads as an intended update". ``_persist_skill`` sanitizes diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 701ff001..2caa03bf 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -8,9 +8,9 @@ knowledge/ global shared knowledge System-managed (dotfile prefix, hidden by default in ls / Finder): - .index/ derived indexes (rebuildable from markdown) - sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db - lancedb/ LanceDB tables + .index/ runtime state (do not delete manually) + sqlite/ coordination, buffer, and OME state + lancedb/ rebuildable LanceDB projection .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -32,14 +32,20 @@ from dataclasses import dataclass from pathlib import Path +from everos.core.scope_ids import validate_app_id, validate_project_id + # ── app / project directory-name convention ────────────────────────────────── # # A memory root is partitioned by ``/`` *before* the user-visible # scope dirs (``agents`` / ``users`` / ``knowledge``), so memory for different -# (app, project) pairs never shares a directory. The reserved id ``"default"`` +# (app, project) pairs do not share a directory in the managed layout. As with +# any filesystem-backed contract, operators must not add symlink aliases. The +# reserved id ``"default"`` # materialises as ``default_app`` / ``default_project`` on disk (rather than a # bare ``default``) so a default space is visually distinct from a user-named -# directory; every other id maps to itself. +# directory; every other accepted id maps to itself. App/project validators use +# a lowercase portable filesystem grammar and reserve runtime-owned names so +# this raw mapping remains injective across common local filesystems. # # The mapping is symmetric: the cascade path parser reverses it (see # :func:`app_id_from_dir`) to recover the ids from an on-disk path. The write @@ -53,22 +59,26 @@ def app_dir_name(app_id: str) -> str: """Map an ``app_id`` to its on-disk directory name.""" + validate_app_id(app_id) return _DEFAULT_APP_DIR if app_id == _DEFAULT_SCOPE_ID else app_id def project_dir_name(project_id: str) -> str: """Map a ``project_id`` to its on-disk directory name.""" + validate_project_id(project_id) return _DEFAULT_PROJECT_DIR if project_id == _DEFAULT_SCOPE_ID else project_id def app_id_from_dir(dir_name: str) -> str: """Inverse of :func:`app_dir_name` — recover the ``app_id`` from a dir name.""" - return _DEFAULT_SCOPE_ID if dir_name == _DEFAULT_APP_DIR else dir_name + app_id = _DEFAULT_SCOPE_ID if dir_name == _DEFAULT_APP_DIR else dir_name + return validate_app_id(app_id) def project_id_from_dir(dir_name: str) -> str: """Inverse of :func:`project_dir_name` — recover the ``project_id``.""" - return _DEFAULT_SCOPE_ID if dir_name == _DEFAULT_PROJECT_DIR else dir_name + project_id = _DEFAULT_SCOPE_ID if dir_name == _DEFAULT_PROJECT_DIR else dir_name + return validate_project_id(project_id) @dataclass(frozen=True, init=False) @@ -157,7 +167,7 @@ def knowledge_dir( @property def index_dir(self) -> Path: - """``/.index/`` — derived index root.""" + """``/.index/`` — system-managed runtime-state root.""" return self.root / ".index" @property diff --git a/src/everos/core/scope_ids.py b/src/everos/core/scope_ids.py new file mode 100644 index 00000000..55f74698 --- /dev/null +++ b/src/everos/core/scope_ids.py @@ -0,0 +1,107 @@ +"""Validated identifiers used as filesystem path segments. + +Sender/owner identifiers retain the historical mixed-case path-safe grammar. +App/project scopes use a stricter portable lowercase grammar because their raw +values become directory names and must not alias across common filesystems. +""" + +from __future__ import annotations + +import re +from typing import Annotated + +from pydantic import AfterValidator, StringConstraints + +PATH_SAFE_CHARSET = r"^[a-zA-Z0-9_.@+-]+$" +SCOPE_ID_CHARSET = r"^[a-z0-9_.@+-]+$" +SCOPE_ID_MIN_LENGTH = 1 +SCOPE_ID_MAX_LENGTH = 128 + +_PATH_SAFE_RE = re.compile(PATH_SAFE_CHARSET) +_SCOPE_ID_RE = re.compile(SCOPE_ID_CHARSET) +_PATH_TRAVERSAL_TOKENS = frozenset({".", ".."}) +_RESERVED_APP_IDS = frozenset( + { + ".index", + ".lock", + ".projection.lock", + ".tmp", + "default_app", + "everos.toml", + "ome.toml", + } +) +_RESERVED_PROJECT_IDS = frozenset({"default_project"}) +_WINDOWS_DEVICE_BASENAMES = frozenset( + { + "aux", + "con", + "nul", + "prn", + *(f"com{index}" for index in range(1, 10)), + *(f"lpt{index}" for index in range(1, 10)), + } +) + + +def _validate_path_safe(value: str) -> str: + if value in _PATH_TRAVERSAL_TOKENS: + raise ValueError("'.' and '..' are reserved (path traversal)") + if not _PATH_SAFE_RE.match(value): + raise ValueError( + "Only alphanumerics, underscore, dot, hyphen, @, and + are allowed" + ) + return value + + +def _validate_scope_id(value: str) -> str: + if not SCOPE_ID_MIN_LENGTH <= len(value) <= SCOPE_ID_MAX_LENGTH: + raise ValueError("Scope identifiers must contain between 1 and 128 characters") + if value in _PATH_TRAVERSAL_TOKENS: + raise ValueError("'.' and '..' are reserved (path traversal)") + if not _SCOPE_ID_RE.fullmatch(value): + raise ValueError( + "Scope identifiers must use lowercase ASCII letters, digits, " + "underscore, dot, hyphen, @, or +" + ) + if value.endswith("."): + raise ValueError("Scope identifiers must not end with a dot") + if value.split(".", 1)[0] in _WINDOWS_DEVICE_BASENAMES: + raise ValueError(f"{value!r} is a reserved filesystem device name") + return value + + +def validate_app_id(value: str) -> str: + value = _validate_scope_id(value) + if value in _RESERVED_APP_IDS: + raise ValueError(f"{value!r} is a reserved app identifier") + return value + + +def validate_project_id(value: str) -> str: + value = _validate_scope_id(value) + if value in _RESERVED_PROJECT_IDS: + raise ValueError(f"{value!r} is a reserved project identifier") + return value + + +PathSafeId = Annotated[str, AfterValidator(_validate_path_safe)] +_SCOPE_ID_CONSTRAINTS = StringConstraints( + min_length=SCOPE_ID_MIN_LENGTH, + max_length=SCOPE_ID_MAX_LENGTH, + pattern=SCOPE_ID_CHARSET, +) +AppId = Annotated[str, _SCOPE_ID_CONSTRAINTS, AfterValidator(validate_app_id)] +ProjectId = Annotated[str, _SCOPE_ID_CONSTRAINTS, AfterValidator(validate_project_id)] + +__all__ = [ + "PATH_SAFE_CHARSET", + "SCOPE_ID_CHARSET", + "SCOPE_ID_MAX_LENGTH", + "SCOPE_ID_MIN_LENGTH", + "AppId", + "PathSafeId", + "ProjectId", + "validate_app_id", + "validate_project_id", +] diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index d20e05b6..cbab97b2 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,14 +1,16 @@ """LanceDB lifespan provider (HTTP API entrypoint). Startup: - Open the connection via ``get_connection`` (lazy, idempotent). + Acquire the shared projection lock, then serialize marker, connection, + schema, and index bootstrap under the exclusive bootstrap lock. Importing :mod:`everos.infra.persistence.lancedb` also triggers the side-effect import of ``tables`` so business schemas are loaded (future: preflight registration). Log hint if unbackfilled (vector IS NULL) rows exist. Shutdown: - Close the connection (also clears the table cache). + Close the connection (also clears the table cache), then release the + shared projection lock. Unbackfilled hint: The informational "you have unbackfilled memory rows" banner runs @@ -25,24 +27,57 @@ from __future__ import annotations +from contextlib import AbstractAsyncContextManager from typing import Any +import anyio from fastapi import FastAPI from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import ( BUSINESS_SCHEMAS_WITH_VECTOR, dispose_connection, ensure_business_indexes, get_connection, get_table, + projection_bootstrap_lock, + projection_server_lock, verify_business_schemas, + verify_storage_identity_ready, ) logger = get_logger(__name__) +async def _dispose_connection_before_unlock( + *, preserve_active_exception: bool = False +) -> None: + """Finish idempotent LanceDB cleanup before lifecycle locks can release. + + The shield prevents an outer shutdown cancellation from interrupting + cleanup. A second attempt keeps a transient or explicitly injected first + failure inside the same lock boundary instead of retrying after bootstrap + exclusion has already been released. + """ + with anyio.CancelScope(shield=True): + first_error: BaseException | None = None + for attempt in range(2): + try: + await dispose_connection() + return + except BaseException as exc: + if first_error is None: + first_error = exc + logger.exception( + "lancedb_dispose_retry", + attempt=attempt + 1, + ) + if first_error is not None and not preserve_active_exception: + raise first_error + + async def _log_unbackfilled_hint() -> None: """Warn at startup if there are unbackfilled memory rows. @@ -85,27 +120,80 @@ async def _log_unbackfilled_hint() -> None: class LanceDBLifespanProvider(LifespanProvider): """Manage the LanceDB connection + table cache for the app lifecycle. - Startup runs four steps: + Startup runs seven steps: - 1. ``get_connection`` — lazy-open the async connection. - 2. ``verify_business_schemas`` — fail loud if an on-disk table's + 1. Acquire and retain the shared projection lock. + 2. Acquire the exclusive projection-bootstrap lock while retaining the + shared lock. This serializes fresh multi-process initialization. + 3. Require the current storage-identity generation. A missing marker is + initialized only when source markdown and LanceDB artifacts are absent; + an existing projection must be rebuilt. + 4. ``get_connection`` — lazy-open the async connection. + 5. ``verify_business_schemas`` — fail loud if an on-disk table's columns drift from the current Pydantic schema. LanceDB has no online migration; cascade is rebuildable from md so the recovery is ``everos cascade rebuild`` (see ``docs/cascade_runbook.md``). - 3. ``ensure_business_indexes`` — idempotent FTS index creation. - 4. ``_log_unbackfilled_hint`` — warn if unbackfilled rows exist. + 6. ``ensure_business_indexes`` — idempotent FTS index creation, then + release the bootstrap lock. + 7. ``_log_unbackfilled_hint`` — warn if unbackfilled rows exist. """ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) + self._projection_lock: AbstractAsyncContextManager[None] | None = None async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - await _log_unbackfilled_hint() + # Retain the shared lock from before marker verification until after + # shutdown has disposed every cached table handle. An offline rebuild + # owns the exclusive side, so the marker cannot change between this + # verification and later server writes. + memory_root = MemoryRoot.resolve() + lock = projection_server_lock(memory_root) + await lock.__aenter__() + cleanup_completed = False + try: + # All runtimes acquire locks in one order: projection SH, then + # bootstrap EX. Rebuild acquires projection EX and therefore never + # overlaps this block. The bootstrap lock is released before the + # steady-state lifespan so multiple server processes can coexist. + async with projection_bootstrap_lock(memory_root): + try: + # This gate must run before opening LanceDB or running any + # migration. Otherwise startup itself could mutate a legacy + # projection before proving the row-id generation current. + await verify_storage_identity_ready() + conn = await get_connection() + await verify_business_schemas() + await ensure_business_indexes() + except BaseException: + # Keep partial-bootstrap cleanup serialized. A waiting + # process must not enter while this process still owns + # half-initialized connection or table handles. + cleanup_completed = True + await _dispose_connection_before_unlock( + preserve_active_exception=True + ) + raise + await _log_unbackfilled_hint() + except BaseException as exc: + # Cleanup normally completed while bootstrap exclusion was still + # held. If failure happened outside that block, finish cleanup + # while retaining the shared projection lock. + if not cleanup_completed: + await _dispose_connection_before_unlock(preserve_active_exception=True) + await lock.__aexit__(type(exc), exc, exc.__traceback__) + raise + self._projection_lock = lock logger.info("lancedb_ready", uri=conn.uri) return conn async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + lock = self._projection_lock + try: + # Shutdown cancellation must not release projection exclusion + # while the process still owns cached LanceDB handles. + await _dispose_connection_before_unlock() + finally: + self._projection_lock = None + if lock is not None: + await lock.__aexit__(None, None, None) diff --git a/src/everos/entrypoints/api/routes/knowledge.py b/src/everos/entrypoints/api/routes/knowledge.py index d64d4998..b50c3eac 100644 --- a/src/everos/entrypoints/api/routes/knowledge.py +++ b/src/everos/entrypoints/api/routes/knowledge.py @@ -40,6 +40,7 @@ UnsupportedModalityError, ) from everos.core.persistence import MemoryRoot +from everos.core.scope_ids import AppId, ProjectId from everos.entrypoints.api.utils import extract_request_id from everos.service import ( CreateDocumentResult, @@ -58,9 +59,7 @@ search_knowledge, ) -# PathSafeId and SuccessEnvelope are imported from memorize routes; -# a shared module would be cleaner but is out of scope for this PR. -from .memorize import PathSafeId, SuccessEnvelope +from .memorize import SuccessEnvelope _KNOWLEDGE_FEATURE = "knowledge" @@ -124,8 +123,10 @@ def _require_knowledge_capabilities() -> None: _FormTitle = Annotated[str, Form(min_length=1, pattern=r"\w")] _FormOptStr = Annotated[str | None, Form()] -_FormPathSafe = Annotated[PathSafeId, Form()] -_QueryPathSafe = Annotated[PathSafeId, Query()] +_FormAppId = Annotated[AppId, Form()] +_FormProjectId = Annotated[ProjectId, Form()] +_QueryAppId = Annotated[AppId, Query()] +_QueryProjectId = Annotated[ProjectId, Query()] _QueryOptStr = Annotated[str | None, Query()] _QueryPage = Annotated[int, Query(ge=1)] _QueryPageSize = Annotated[int, Query(ge=1, le=100)] @@ -291,8 +292,8 @@ class KnowledgeSearchRequest(BaseModel): top_k: int = Field(default=10, ge=1, le=100) score_threshold: float | None = None include_content: bool = False - app_id: PathSafeId = "default" - project_id: PathSafeId = "default" + app_id: AppId = "default" + project_id: ProjectId = "default" class DocumentPatchRequest(BaseModel): @@ -300,8 +301,8 @@ class DocumentPatchRequest(BaseModel): title: str | None = Field(default=None, min_length=1, pattern=r"\w") category_id: str | None = Field(default=None, min_length=1) - app_id: PathSafeId = "default" - project_id: PathSafeId = "default" + app_id: AppId = "default" + project_id: ProjectId = "default" # ── Extractor builder ─────────────────────────────────────────────────────── @@ -553,8 +554,8 @@ async def create_document_route( title: _FormTitle, source_type: _FormOptStr = None, category_id: _FormOptStr = None, - app_id: _FormPathSafe = "default", - project_id: _FormPathSafe = "default", + app_id: _FormAppId = "default", + project_id: _FormProjectId = "default", ) -> SuccessEnvelope[DocumentCreateResponse]: """Upload a new knowledge document.""" rid = extract_request_id(request) @@ -592,8 +593,8 @@ async def replace_document_route( title: _FormTitle, source_type: _FormOptStr = None, category_id: _FormOptStr = None, - app_id: _FormPathSafe = "default", - project_id: _FormPathSafe = "default", + app_id: _FormAppId = "default", + project_id: _FormProjectId = "default", ) -> SuccessEnvelope[DocumentCreateResponse]: """Replace an existing knowledge document (atomic backup/restore on failure).""" rid = extract_request_id(request) @@ -622,8 +623,8 @@ async def replace_document_route( async def delete_document_route( request: Request, doc_id: _PathDocId, - app_id: _QueryPathSafe = "default", - project_id: _QueryPathSafe = "default", + app_id: _QueryAppId = "default", + project_id: _QueryProjectId = "default", ) -> SuccessEnvelope[DocumentDeleteResponse] | Response: """Remove a knowledge document.""" rid = extract_request_id(request) @@ -645,8 +646,8 @@ async def delete_document_route( # FastAPI requires flat Form/Query params — ≤5 positional rule exempted. async def list_documents_route( request: Request, - app_id: _QueryPathSafe = "default", - project_id: _QueryPathSafe = "default", + app_id: _QueryAppId = "default", + project_id: _QueryProjectId = "default", category_id: _QueryOptStr = None, page: _QueryPage = 1, page_size: _QueryPageSize = 20, @@ -671,8 +672,8 @@ async def list_documents_route( async def get_document_route( request: Request, doc_id: _PathDocId, - app_id: _QueryPathSafe = "default", - project_id: _QueryPathSafe = "default", + app_id: _QueryAppId = "default", + project_id: _QueryProjectId = "default", ) -> SuccessEnvelope[DocumentDetailResponse]: """Fetch a single document with its topic list.""" rid = extract_request_id(request) @@ -684,8 +685,8 @@ async def get_document_route( async def get_topic_route( request: Request, topic_id: _PathTopicId, - app_id: _QueryPathSafe = "default", - project_id: _QueryPathSafe = "default", + app_id: _QueryAppId = "default", + project_id: _QueryProjectId = "default", ) -> SuccessEnvelope[TopicDetailResponse]: """Fetch a single topic with full content.""" rid = extract_request_id(request) @@ -718,8 +719,8 @@ async def search_knowledge_route( @router.get("/categories") async def list_categories_route( request: Request, - app_id: _QueryPathSafe = "default", - project_id: _QueryPathSafe = "default", + app_id: _QueryAppId = "default", + project_id: _QueryProjectId = "default", ) -> SuccessEnvelope[CategoryListResponse]: """List taxonomy categories from ``.taxonomy.md``.""" rid = extract_request_id(request) diff --git a/src/everos/entrypoints/api/routes/memorize.py b/src/everos/entrypoints/api/routes/memorize.py index 2f5a750b..f3134c25 100644 --- a/src/everos/entrypoints/api/routes/memorize.py +++ b/src/everos/entrypoints/api/routes/memorize.py @@ -10,54 +10,24 @@ from __future__ import annotations -import re from typing import Annotated, Any, Literal from fastapi import APIRouter, Request -from pydantic import AfterValidator, BaseModel, ConfigDict, Field - +from pydantic import BaseModel, ConfigDict, Field + +from everos.core.scope_ids import ( + PATH_SAFE_CHARSET, + SCOPE_ID_CHARSET, + AppId, + PathSafeId, + ProjectId, +) from everos.entrypoints.api.utils import extract_request_id from everos.service import memorize router = APIRouter(prefix="/memory", tags=["memory"]) -# ── Path-safe identifier ──────────────────────────────────────────────────── -# ``app_id`` / ``project_id`` / ``sender_id`` all become directory segments -# under the memory root (``sender_id`` flows through to ``owner_id`` and is -# joined into the daily-log write path), so they must reject ``.`` and ``..`` -# (path traversal). The basic character whitelist is enforced via ``pattern`` -# (pydantic_core uses the Rust regex engine, which does NOT support -# lookaround), and the two reserved tokens are filtered out with a follow-up -# ``AfterValidator``. -# -# ``@`` and ``+`` are admitted so real-world ids survive (email-style -# ``user@example.com``, plus-addressing ``user+tag``); both are legal, -# non-separator filename chars on every target filesystem (incl. NTFS, whose -# reserved set is ``< > : " / \ | ? *``). The genuinely path-dangerous chars -# (``/`` ``\`` NUL) stay out of the whitelist, and ``.``/``..`` stay blocked -# by the token filter; the markdown writer's ``_ensure_within_root`` is the -# final backstop regardless. -_PATH_SAFE_CHARSET = r"^[a-zA-Z0-9_.@+-]+$" -_PATH_TRAVERSAL_TOKENS = frozenset({".", ".."}) - - -_PATH_SAFE_RE = re.compile(_PATH_SAFE_CHARSET) - - -def _reject_path_traversal(value: str) -> str: - if value in _PATH_TRAVERSAL_TOKENS: - raise ValueError("'.' and '..' are reserved (path traversal)") - if not _PATH_SAFE_RE.match(value): - raise ValueError( - "Only alphanumerics, underscore, dot, hyphen, @, and + are allowed" - ) - return value - - -PathSafeId = Annotated[str, AfterValidator(_reject_path_traversal)] - - # DTOs ──────────────────────────────────────────────────────────────────────── @@ -88,13 +58,14 @@ class ContentItemDTO(BaseModel): class MessageItemDTO(BaseModel): # ``sender_id`` becomes ``owner_id`` and then a directory segment on the - # episode write path, so it carries the same path-safety guard as - # ``app_id`` / ``project_id`` (charset whitelist + ``.``/``..`` rejection). + # episode write path, so it retains its historical path-safety guard + # (charset whitelist + ``.``/``..`` rejection). App/project scopes apply + # an additional lowercase portability contract independently. sender_id: PathSafeId = Field( ..., min_length=1, max_length=128, - pattern=_PATH_SAFE_CHARSET, + pattern=PATH_SAFE_CHARSET, ) sender_name: str | None = None role: Literal["user", "assistant", "tool"] @@ -114,17 +85,17 @@ class MessageItemDTO(BaseModel): class MemorizeAddRequest(BaseModel): session_id: str = Field(..., min_length=1, max_length=128) - app_id: PathSafeId = Field( + app_id: AppId = Field( default="default", min_length=1, max_length=128, - pattern=_PATH_SAFE_CHARSET, + pattern=SCOPE_ID_CHARSET, ) - project_id: PathSafeId = Field( + project_id: ProjectId = Field( default="default", min_length=1, max_length=128, - pattern=_PATH_SAFE_CHARSET, + pattern=SCOPE_ID_CHARSET, ) messages: list[MessageItemDTO] = Field(..., min_length=1, max_length=500) @@ -136,17 +107,17 @@ class AddResponseData(BaseModel): class MemorizeFlushRequest(BaseModel): session_id: str = Field(..., min_length=1, max_length=128) - app_id: PathSafeId = Field( + app_id: AppId = Field( default="default", min_length=1, max_length=128, - pattern=_PATH_SAFE_CHARSET, + pattern=SCOPE_ID_CHARSET, ) - project_id: PathSafeId = Field( + project_id: ProjectId = Field( default="default", min_length=1, max_length=128, - pattern=_PATH_SAFE_CHARSET, + pattern=SCOPE_ID_CHARSET, ) diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8710f947..f2f90432 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -32,10 +32,11 @@ import asyncio import enum import os -from contextlib import asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager from pathlib import Path from typing import Annotated +import anyio import typer from sqlmodel import SQLModel @@ -47,11 +48,18 @@ from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill from everos.infra.persistence.lancedb import ( + ProjectionLockUnavailableError, dispose_connection, drop_business_tables, ensure_business_indexes, get_connection, + mark_storage_identity_ready, + mark_storage_identity_rebuilding, + projection_bootstrap_lock, + projection_rebuild_lock, + projection_server_lock, verify_business_schemas, + verify_storage_identity_ready, ) from everos.infra.persistence.sqlite import ( dispose_engine, @@ -61,7 +69,6 @@ from everos.memory.cascade import ( CascadeOrchestrator, match_kind, - ome_lock_is_free, ) logger = get_logger(__name__) @@ -136,7 +143,11 @@ def _apply_verbose_logging(verbose: bool | None) -> None: @asynccontextmanager async def _runtime( # type: ignore[no-untyped-def] - *, verify: bool = True, ensure: bool = True + *, + verify: bool = True, + ensure: bool = True, + identity_gate: bool = True, + lifecycle_lock: bool = True, ): """Stand up sqlite + lancedb the same way the API lifespan would. @@ -144,8 +155,9 @@ async def _runtime( # type: ignore[no-untyped-def] does. They are **per-process**: a running daemon has its own connection and table-handle cache, so read/write traffic interleaves safely, but a change to the table *set* made here (drop / recreate) - is invisible to the daemon's cached handles — which is why - ``rebuild`` refuses to run while a server holds the OME lock. + is invisible to the daemon's cached handles. ``rebuild`` therefore takes + the exclusive projection lifecycle lock and the legacy OME lock before it + changes any projection state. ``verify=False`` skips :func:`verify_business_schemas` — required by ``cascade rebuild``, whose whole purpose is to recover from a table @@ -159,19 +171,99 @@ async def _runtime( # type: ignore[no-untyped-def] happen — the recovery path dying on the damage it was invoked to fix. Rebuild recreates the tables and their indexes itself after dropping, so skipping the pre-drop pass loses nothing. + + Normal runtimes acquire locks in the fixed order projection shared, then + bootstrap exclusive. The bootstrap lock covers only marker, connection, + schema, and index initialization; the shared lock remains held through the + command body and store disposal. ``lifecycle_lock=False`` bypasses both + locks for rebuild, which already owns the exclusive projection lock. + """ + async with AsyncExitStack() as stack: + memory_root = MemoryRoot.resolve() + if lifecycle_lock: + # Every mutating CLI path participates in the same projection + # lifecycle protocol as the API server. Acquire the shared side + # before checking READY and retain it until both stores are closed, + # so an offline rebuild cannot race a writer that already passed + # the generation gate. + await stack.enter_async_context(projection_server_lock(memory_root)) + cleanup_completed = False + + async def dispose_stores(*, preserve_active_exception: bool = False) -> None: + """Attempt both disposals, optionally preserving an earlier error.""" + + async def dispose_with_retry(operation): # type: ignore[no-untyped-def] + try: + await operation() + except BaseException: + await operation() + + first_disposal_error: BaseException | None = None + try: + await dispose_with_retry(dispose_connection) + except BaseException as exc: + first_disposal_error = exc + try: + await dispose_with_retry(dispose_engine) + except BaseException as exc: + if first_disposal_error is None: + first_disposal_error = exc + if first_disposal_error is not None and not preserve_active_exception: + raise first_disposal_error + + async def initialize() -> None: + if identity_gate: + # Gate before opening either store or running schema/index + # migrations. A legacy or interrupted projection must not be + # mutated by setup. + await verify_storage_identity_ready() + engine = get_engine() + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + await get_connection() + if verify: + await verify_business_schemas() + if ensure: + await ensure_business_indexes() + + try: + if lifecycle_lock: + # The shared lock is already held. Serialize bootstrap only, + # then release this exclusive lock before running the command. + async with projection_bootstrap_lock(memory_root): + try: + await initialize() + except BaseException: + # A waiting process cannot observe partial bootstrap + # while this process still has open store handles. Both + # disposals are attempted, but the initialization error + # remains the reported failure if cleanup also fails. + with anyio.CancelScope(shield=True): + await dispose_stores(preserve_active_exception=True) + cleanup_completed = True + raise + else: + await initialize() + yield + finally: + if not cleanup_completed: + with anyio.CancelScope(shield=True): + await dispose_stores() + + +@asynccontextmanager +async def _sqlite_runtime(): # type: ignore[no-untyped-def] + """Open only queue metadata for read-only status/failure inspection. + + This deliberately does not open or migrate LanceDB, so diagnostics remain + available in REBUILDING state without bypassing the projection gate. """ engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() - if verify: - await verify_business_schemas() - if ensure: - await ensure_business_indexes() try: yield finally: - await dispose_connection() await dispose_engine() @@ -221,18 +313,21 @@ def sync( _apply_verbose_logging(verbose) async def _run() -> None: + rel: str | None = None + spec = None + if path is not None: + rel = _resolve_relative(path) + spec = match_kind(rel) + if spec is None: + typer.echo( + f"error: path does not match any registered cascade kind: {rel}", + err=True, + ) + raise typer.Exit(code=1) + async with _runtime(): orchestrator = _build_orchestrator() - if path is not None: - rel = _resolve_relative(path) - spec = match_kind(rel) - if spec is None: - typer.echo( - f"error: path does not match any registered cascade " - f"kind: {rel}", - err=True, - ) - raise typer.Exit(code=1) + if rel is not None and spec is not None: await md_change_state_repo.force_enqueue(rel, spec.name) typer.echo(f"force-enqueued {rel} (kind={spec.name})") processed = await orchestrator.sync_once() @@ -260,7 +355,7 @@ def status( _apply_verbose_logging(verbose) async def _run() -> None: - async with _runtime(): + async with _sqlite_runtime(): summary = await md_change_state_repo.queue_summary() lag = max(0, summary.max_lsn - summary.last_processed_lsn) typer.echo("queue:") @@ -316,7 +411,8 @@ def fix( _apply_verbose_logging(verbose) async def _run() -> None: - async with _runtime(): + runtime = _runtime() if apply else _sqlite_runtime() + async with runtime: rows = await md_change_state_repo.list_failed() if not rows: typer.echo("no failed rows") @@ -453,37 +549,26 @@ def rebuild( ) -> None: """Rebuild the LanceDB index from markdown (recover from schema drift). - **Stop the ``everos server`` first** — this is the one cascade command + **Stop every ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the - tables, and the daemon's cached table handles would keep writing to - the dropped dataset; the command refuses to start while a server holds - the OME lock. + tables, and a daemon's cached table handles would keep writing to the + dropped dataset. The command refuses to start while a current process + holds the projection lock or an older server holds the legacy OME lock. Drops every business LanceDB table and re-indexes all md from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates - already-indexed entries (that command leaves the cascade queue - marked ``done``, so nothing re-indexes and the index comes back - empty); - - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that - is NOT rebuildable from md — notably ``unprocessed_buffer`` - (messages received but not yet extracted). + - do not remove ``~/.everos/.index/lancedb`` manually: that also removes + the storage-generation marker, so an existing root fails closed; + - do not remove ``~/.everos/.index``: it also deletes SQLite state that + is NOT rebuildable from md, notably ``unprocessed_buffer`` (messages + received but not yet extracted); + - after generation 2 has been published, do not run an older EverOS binary + against the root. If that happened, rerun this rebuild with the current + binary before restarting. """ - if not ome_lock_is_free(): - typer.echo( - "error: a server (or another exclusive CLI phase) is running on " - "this memory root.\n" - " cascade rebuild drops and recreates the LanceDB tables; a live " - "daemon holds cached\n" - " table handles and would keep writing to the dropped dataset. " - "Stop `everos server`\n" - " first, then re-run.", - err=True, - ) - raise typer.Exit(code=3) if not yes: typer.confirm( "Drop all LanceDB business tables and re-index from markdown? " @@ -491,39 +576,86 @@ def rebuild( abort=True, ) + memory_root = MemoryRoot.resolve() + async def _run() -> None: - # verify=False: the on-disk schema may be exactly what we're here - # to fix; the startup guard would abort before we could rebuild. - # ensure=False: the pre-drop migration pass would raise on exactly - # the damage we are here to repair (see _runtime). - async with _runtime(verify=False, ensure=False): - # Reset the queue FIRST so every crash window converges on - # "queue pending → next scan re-indexes". Doing it after the - # drop leaves a window where a crash yields empty tables with - # a fully-`done` queue: nothing re-indexes, the schema guard - # passes, and the deployment comes up silently empty — the - # exact state this command exists to avoid. - cleared = await md_change_state_repo.reset_all() - typer.echo(f"reset {cleared} cascade queue row(s)") - dropped = await drop_business_tables() - typer.echo( - f"dropped {len(dropped)} LanceDB table(s): " - f"{', '.join(dropped) or '(none)'}" - ) - # Recreate the tables (current schema) + FTS indexes. - await ensure_business_indexes() - # Re-scan + drain: re-embed and re-insert every md entry. - orchestrator = _build_orchestrator() - processed = await orchestrator.sync_once() - typer.echo(f"rebuild complete — re-indexed {processed} md file(s)") + # Acquire both exclusions before publishing REBUILDING. The projection + # lock excludes current servers/rebuilds; the retained OME portalocker + # detects an already-running older server which holds its legacy guard. + async with projection_rebuild_lock(memory_root): + memory_root.ensure() + # Publish REBUILDING before opening LanceDB or SQLite. Any failure + # after this point must leave startup blocked until recovery succeeds. + mark_storage_identity_rebuilding(memory_root) + + # verify=False: the on-disk schema may be exactly what we're here + # to fix; the startup guard would abort before we could rebuild. + # ensure=False: the pre-drop migration pass would raise on exactly + # the damage we are here to repair (see _runtime). + async with _runtime( + verify=False, + ensure=False, + identity_gate=False, + lifecycle_lock=False, + ): + # Reset the queue FIRST so every crash window converges on + # "queue pending → next scan re-indexes". Doing it after the + # drop leaves a window where a crash yields empty tables with + # a fully-`done` queue: nothing re-indexes, the schema guard + # passes, and the deployment comes up silently empty — the + # exact state this command exists to avoid. + cleared = await md_change_state_repo.reset_all() + typer.echo(f"reset {cleared} cascade queue row(s)") + dropped = await drop_business_tables() + typer.echo( + f"dropped {len(dropped)} LanceDB table(s): " + f"{', '.join(dropped) or '(none)'}" + ) + # Recreate the tables (current schema) + FTS indexes. + await ensure_business_indexes() + # Re-scan + drain: re-embed and re-insert every md entry. + orchestrator = _build_orchestrator() + processed = await orchestrator.sync_once() + summary = await orchestrator.queue_summary() + while summary.pending: + drained = await orchestrator.drain_once() + if drained == 0: + break + processed += drained + summary = await orchestrator.queue_summary() + if ( + summary.pending + or summary.failed_retryable + or summary.failed_permanent + ): + raise RuntimeError( + "cascade rebuild incomplete: " + f"pending={summary.pending}, " + f"failed_retryable={summary.failed_retryable}, " + f"failed_permanent={summary.failed_permanent}. " + "The storage identity remains REBUILDING; correct the " + "failed inputs and rerun `everos cascade rebuild`." + ) + mark_storage_identity_ready(memory_root) + typer.echo(f"rebuild complete — re-indexed {processed} md file(s)") try: asyncio.run(_run()) + except ProjectionLockUnavailableError as exc: + typer.echo( + "error: a server, rebuild, or exclusive OME phase is running on " + "this memory root.\n" + " cascade rebuild did not modify the storage generation or index. " + f"Lock detail: {exc}\n" + " Stop `everos server` or the conflicting command, then re-run.", + err=True, + ) + raise typer.Exit(code=3) from None except KeyboardInterrupt: typer.echo( - "\ninterrupted — the cascade queue is reset, so re-running " - "`everos cascade rebuild` (or starting the server) resumes the " - "re-index from where it stopped.", + "\ninterrupted; the storage identity remains REBUILDING. Re-run " + "`everos cascade rebuild --yes`. The server will refuse to start " + "until a rebuild completes.", err=True, ) raise typer.Exit(code=130) from None diff --git a/src/everos/infra/persistence/lancedb/__init__.py b/src/everos/infra/persistence/lancedb/__init__.py index 78ea873e..dc667a50 100644 --- a/src/everos/infra/persistence/lancedb/__init__.py +++ b/src/everos/infra/persistence/lancedb/__init__.py @@ -37,6 +37,12 @@ from .lancedb_manager import drop_tables as _drop_tables from .lancedb_manager import get_connection as get_connection from .lancedb_manager import get_table as get_table +from .projection_lock import ( + ProjectionLockUnavailableError as ProjectionLockUnavailableError, +) +from .projection_lock import projection_bootstrap_lock as projection_bootstrap_lock +from .projection_lock import projection_rebuild_lock as projection_rebuild_lock +from .projection_lock import projection_server_lock as projection_server_lock from .repos import agent_case_repo as agent_case_repo from .repos import agent_skill_repo as agent_skill_repo from .repos import atomic_fact_repo as atomic_fact_repo @@ -44,6 +50,19 @@ from .repos import foresight_repo as foresight_repo from .repos import knowledge_topic_repo as knowledge_topic_repo from .repos import user_profile_repo as user_profile_repo +from .storage_identity import ( + StorageIdentityMigrationRequiredError as StorageIdentityMigrationRequiredError, +) +from .storage_identity import ( + ensure_storage_identity_ready as ensure_storage_identity_ready, +) +from .storage_identity import mark_storage_identity_ready as mark_storage_identity_ready +from .storage_identity import ( + mark_storage_identity_rebuilding as mark_storage_identity_rebuilding, +) +from .storage_identity import ( + read_storage_identity_state as read_storage_identity_state, +) from .tables import AgentCase as AgentCase from .tables import AgentSkill as AgentSkill from .tables import AtomicFact as AtomicFact @@ -210,9 +229,9 @@ async def migrate_table_schemas() -> None: ``vector=None`` (soft-dependency embedding) into a still NOT-NULL column and every row would silently fail. Recovery escalates from a plain restart (transient hiccup) to ``everos cascade rebuild``, - which re-indexes from md (the SoT) *and* re-enqueues every file — - unlike deleting the index dir, which leaves the queue ``done`` and - yields an empty index. + which re-indexes from md (the SoT) *and* re-enqueues every file. + Manually deleting the LanceDB directory also removes the required storage + generation marker, so current startup fails closed for an existing root. """ logger = get_logger(__name__) memory_root = MemoryRoot.resolve() @@ -258,10 +277,10 @@ async def migrate_table_schemas() -> None: f"filesystem or LanceDB-side hiccup may resolve. (2) If " f"the error persists, run `everos cascade rebuild` (with " f"the server stopped) — it re-indexes from source markdown " - f"and preserves un-extracted buffered messages. Do NOT " - f"just delete `{memory_root.lancedb_dir}`: that leaves the " - f"cascade queue marked done, so nothing re-indexes and the " - f"index comes back empty." + f"and preserves un-extracted buffered messages. Do NOT just " + f"delete `{memory_root.lancedb_dir}` manually: that also " + f"removes the storage-generation marker, so current startup " + f"fails closed for an existing root." ) marker.parent.mkdir(parents=True, exist_ok=True) @@ -306,9 +325,9 @@ async def verify_business_schemas() -> None: turns that into a clean startup error pointing the user at the recovery path (``everos cascade rebuild`` — re-indexes from md, preserving un-extracted buffered messages; see - ``docs/cascade_runbook.md``). A bare ``rm -rf`` of the index dir is - *not* the recovery — it leaves the cascade queue marked ``done`` so - nothing re-indexes and the index comes back empty. + ``docs/cascade_runbook.md``). Manually removing the LanceDB directory is + not the recovery because it removes the required storage-generation + marker; current startup then fails closed for an existing root. Both dimensions are checked against ``schema.to_arrow_schema()`` — the exact schema ``get_table`` builds the table from, so a healthy @@ -355,6 +374,22 @@ async def verify_business_schemas() -> None: ) +async def verify_storage_identity_ready(*, initialize_empty: bool = True) -> None: + """Require a ready marker for the current storage-key generation. + + A missing marker is accepted only for a provably empty affected index, + which is the fresh-install case. Any existing row without a marker is + treated as legacy data and requires an offline cascade rebuild. + """ + memory_root = MemoryRoot.resolve() + if not initialize_empty and read_storage_identity_state(memory_root) is None: + raise StorageIdentityMigrationRequiredError( + "LanceDB storage identity marker is missing. " + "Run `everos cascade rebuild` with the server stopped." + ) + ensure_storage_identity_ready(memory_root) + + async def drop_business_tables() -> list[str]: """Drop every business LanceDB table; return the names dropped. @@ -377,6 +412,8 @@ async def drop_business_tables() -> list[str]: "LanceDBMigrationError", "LanceDBSchemaMismatchError", "ParentType", + "ProjectionLockUnavailableError", + "StorageIdentityMigrationRequiredError", "UserProfile", "agent_case_repo", "agent_skill_repo", @@ -384,13 +421,21 @@ async def drop_business_tables() -> list[str]: "dispose_connection", "drop_business_tables", "ensure_business_indexes", + "ensure_storage_identity_ready", "episode_repo", "foresight_repo", "get_connection", "get_table", "knowledge_topic_repo", + "mark_storage_identity_ready", + "mark_storage_identity_rebuilding", "migrate_fts_indexes", "migrate_table_schemas", + "projection_bootstrap_lock", + "projection_rebuild_lock", + "projection_server_lock", + "read_storage_identity_state", "user_profile_repo", "verify_business_schemas", + "verify_storage_identity_ready", ] diff --git a/src/everos/infra/persistence/lancedb/projection_lock.py b/src/everos/infra/persistence/lancedb/projection_lock.py new file mode 100644 index 00000000..25bce347 --- /dev/null +++ b/src/everos/infra/persistence/lancedb/projection_lock.py @@ -0,0 +1,210 @@ +"""Cross-process lifecycle lock for the rebuildable LanceDB projection. + +Normal servers hold a shared lock for the complete LanceDB lifespan. An +offline rebuild requires the exclusive side of the same lock, so it cannot +invalidate or replace tables while a current server has open table handles. +After taking the shared side, servers and mutating CLI commands briefly take +an exclusive bootstrap lock to serialize marker, schema, and index setup. + +Rebuild also holds the OfflineEngine's existing portalocker anchor. That +second lock detects an already-running older server which predates the +projection lock but already owns the OME lock. It cannot coordinate the +startup window of an old binary that does not implement this protocol, so +operators must stop old servers before rebuilding during an upgrade. +""" + +from __future__ import annotations + +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TextIO + +import anyio +import portalocker + +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot + +logger = get_logger(__name__) + +_PROJECTION_LOCK_NAME = ".projection.lock" +_PROJECTION_BOOTSTRAP_LOCK_NAME = ".projection.bootstrap.lock" +_POLL_INTERVAL_SECONDS = 0.25 +_SERVER_LOCK_TIMEOUT_SECONDS = 1800.0 + + +class ProjectionLockUnavailableError(RuntimeError): + """Raised when a projection lifecycle lock cannot be acquired.""" + + +def projection_lock_path(memory_root: MemoryRoot) -> Path: + """Return the stable shared/exclusive projection-lock anchor.""" + return memory_root.root / _PROJECTION_LOCK_NAME + + +def projection_bootstrap_lock_path(memory_root: MemoryRoot) -> Path: + """Return the exclusive projection-bootstrap lock anchor.""" + return memory_root.index_dir / _PROJECTION_BOOTSTRAP_LOCK_NAME + + +def ome_lock_path(memory_root: MemoryRoot) -> Path: + """Return the exact lock anchor used by :class:`OfflineEngine`.""" + return Path(str(memory_root.ome_db) + ".lock") + + +@asynccontextmanager +async def projection_server_lock( + memory_root: MemoryRoot, + *, + timeout_seconds: float | None = _SERVER_LOCK_TIMEOUT_SECONDS, +) -> AsyncIterator[None]: + """Hold a shared projection lock for one server's LanceDB lifespan. + + Shared holders coexist, so deployments that intentionally run more than + one API process are not serialized. An active rebuild holds the exclusive + side; startup waits for it to finish and verifies the generation marker + only after the rebuild has released the lock. + """ + async with _portalocker_lock( + projection_lock_path(memory_root), + flags=portalocker.LOCK_SH | portalocker.LOCK_NB, + blocking=True, + timeout_seconds=timeout_seconds, + label="projection shared", + ): + yield + + +@asynccontextmanager +async def projection_bootstrap_lock( + memory_root: MemoryRoot, + *, + timeout_seconds: float | None = _SERVER_LOCK_TIMEOUT_SECONDS, +) -> AsyncIterator[None]: + """Serialize projection bootstrap after acquiring the shared lock. + + Callers must acquire :func:`projection_server_lock` first and retain it + while this lock is held. The fixed order is therefore projection shared, + then bootstrap exclusive. It prevents concurrent fresh processes from + racing marker publication, connection creation, schema verification, or + index creation while still allowing their steady-state runtimes to + coexist after bootstrap completes. + + Rebuild deliberately does not acquire this lock. Its exclusive projection + lock excludes every correctly ordered bootstrap before any destructive + work starts. + """ + async with _portalocker_lock( + projection_bootstrap_lock_path(memory_root), + flags=portalocker.LOCK_EX | portalocker.LOCK_NB, + blocking=True, + timeout_seconds=timeout_seconds, + label="projection bootstrap exclusive", + ): + yield + + +@asynccontextmanager +async def projection_rebuild_lock(memory_root: MemoryRoot) -> AsyncIterator[None]: + """Hold all cross-version exclusions required by an offline rebuild. + + Acquisition is non-blocking and completes before the caller may publish a + ``REBUILDING`` marker or mutate either store. The projection lock excludes + current servers and concurrent rebuilds. The OME lock excludes an older + server once that process has acquired its legacy OME guard; it does not + make a concurrently starting old binary participate in this protocol. + """ + async with ( + _portalocker_lock( + projection_lock_path(memory_root), + flags=portalocker.LOCK_EX | portalocker.LOCK_NB, + blocking=False, + timeout_seconds=None, + label="projection exclusive", + ), + _portalocker_lock( + ome_lock_path(memory_root), + flags=portalocker.LOCK_EX | portalocker.LOCK_NB, + blocking=False, + timeout_seconds=None, + label="OME exclusive", + ), + ): + yield + + +@asynccontextmanager +async def _portalocker_lock( + path: Path, + *, + flags: int, + blocking: bool, + timeout_seconds: float | None, + label: str, +) -> AsyncIterator[None]: + """Acquire one retained portalocker handle with bounded async polling.""" + handle = await anyio.to_thread.run_sync(_open_lock_handle, path) + started = time.monotonic() + deadline = None if timeout_seconds is None else started + timeout_seconds + announced = False + try: + while True: + try: + await anyio.to_thread.run_sync(portalocker.lock, handle, flags) + break + except portalocker.LockException as exc: + if not blocking: + raise ProjectionLockUnavailableError( + f"{label} lock is held at {path}" + ) from exc + if not announced: + logger.info( + "projection_lock_waiting", + label=label, + path=str(path), + timeout_seconds=timeout_seconds, + ) + announced = True + if deadline is not None and time.monotonic() >= deadline: + raise ProjectionLockUnavailableError( + f"timed out waiting for {label} lock at {path}" + ) from exc + await anyio.sleep(_POLL_INTERVAL_SECONDS) + except BaseException: + await anyio.to_thread.run_sync(handle.close) + raise + + if announced: + logger.info( + "projection_lock_acquired_after_wait", + label=label, + path=str(path), + waited_seconds=round(time.monotonic() - started, 2), + ) + + try: + yield + finally: + with anyio.CancelScope(shield=True): + try: + await anyio.to_thread.run_sync(portalocker.unlock, handle) + finally: + await anyio.to_thread.run_sync(handle.close) + + +def _open_lock_handle(path: Path) -> TextIO: + path.parent.mkdir(parents=True, exist_ok=True) + return path.open("a+", encoding="utf-8") + + +__all__ = [ + "ProjectionLockUnavailableError", + "ome_lock_path", + "projection_bootstrap_lock", + "projection_bootstrap_lock_path", + "projection_lock_path", + "projection_rebuild_lock", + "projection_server_lock", +] diff --git a/src/everos/infra/persistence/lancedb/storage_identity.py b/src/everos/infra/persistence/lancedb/storage_identity.py new file mode 100644 index 00000000..b981adec --- /dev/null +++ b/src/everos/infra/persistence/lancedb/storage_identity.py @@ -0,0 +1,173 @@ +"""Fail-closed lifecycle marker for LanceDB storage-key generations. + +The shape of ``row.id`` is value-level state, so LanceDB's schema checker +cannot distinguish a legacy index from one rebuilt with the current storage +identity. This marker binds the on-disk projection to the implementation that +created it and prevents a partially rebuilt index from being served. +""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import STORAGE_ID_GENERATION + +_MARKER_NAME = ".storage_identity.json" + + +class StorageIdentityMigrationRequiredError(RuntimeError): + """Raised when an index is not proven ready for this key generation.""" + + +@dataclass(frozen=True) +class StorageIdentityState: + generation: int + state: Literal["READY", "REBUILDING"] + + +def marker_path(memory_root: MemoryRoot) -> Path: + return memory_root.lancedb_dir / _MARKER_NAME + + +def read_storage_identity_state( + memory_root: MemoryRoot, +) -> StorageIdentityState | None: + """Read and strictly validate the marker, returning ``None`` if absent.""" + path = marker_path(memory_root) + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise StorageIdentityMigrationRequiredError( + f"Storage identity marker {path} is unreadable or malformed. " + "Run `everos cascade rebuild` with the server stopped." + ) from exc + if type(payload) is not dict or set(payload) != {"generation", "state"}: + raise StorageIdentityMigrationRequiredError( + f"Storage identity marker {path} has an unknown JSON shape. " + "Run `everos cascade rebuild` with the server stopped." + ) + generation = payload["generation"] + state = payload["state"] + # ``bool`` is an ``int`` subclass in Python. Accepting ``true`` as + # generation 1/2 would turn malformed JSON into a valid migration gate. + if ( + type(generation) is not int + or type(state) is not str + or state not in {"READY", "REBUILDING"} + ): + raise StorageIdentityMigrationRequiredError( + f"Storage identity marker {path} has invalid fields. " + "Run `everos cascade rebuild` with the server stopped." + ) + return StorageIdentityState(generation=generation, state=state) + + +def mark_storage_identity_rebuilding(memory_root: MemoryRoot) -> None: + """Invalidate any prior ready marker before destructive rebuild work.""" + _write_state( + memory_root, + StorageIdentityState( + generation=STORAGE_ID_GENERATION, + state="REBUILDING", + ), + ) + + +def mark_storage_identity_ready(memory_root: MemoryRoot) -> None: + """Atomically publish readiness after every rebuild gate has passed.""" + _write_state( + memory_root, + StorageIdentityState( + generation=STORAGE_ID_GENERATION, + state="READY", + ), + ) + + +def ensure_storage_identity_ready(memory_root: MemoryRoot) -> None: + """Require READY(current generation), initializing only an empty projection. + + A missing marker is safe only when there is no source markdown and no + existing LanceDB artifact. This prevents a legacy projection, or a source + tree awaiting its first generation-2 rebuild, from being mistaken for an + empty projection. Retained SQLite state is outside this marker's scope. + """ + state = read_storage_identity_state(memory_root) + if state is None: + if _projection_is_provably_empty(memory_root): + mark_storage_identity_ready(memory_root) + return + raise StorageIdentityMigrationRequiredError( + "Storage identity marker is missing for an existing memory root. " + "Run `everos cascade rebuild` with the server stopped." + ) + if state.state != "READY" or state.generation != STORAGE_ID_GENERATION: + raise StorageIdentityMigrationRequiredError( + "LanceDB storage identity is not ready for this EverOS build " + f"(state={state.state!r}, generation={state.generation}, " + f"required={STORAGE_ID_GENERATION}). Run `everos cascade rebuild` " + "with the server stopped." + ) + + +def _projection_is_provably_empty(memory_root: MemoryRoot) -> bool: + root = memory_root.root + # Any markdown makes a marker-less root non-empty. App identifiers now + # reject the system-managed ``.index`` / ``.tmp`` namespaces, but a + # legacy or manually created source file there must still fail closed + # rather than be mistaken for a fresh installation. + if root.exists() and any(root.rglob("*.md")): + return False + + lancedb_dir = memory_root.lancedb_dir + if not lancedb_dir.exists(): + return True + return not any(path.is_file() for path in lancedb_dir.rglob("*")) + + +def _write_state(memory_root: MemoryRoot, state: StorageIdentityState) -> None: + path = marker_path(memory_root) + path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps( + {"generation": state.generation, "state": state.state}, + sort_keys=True, + separators=(",", ":"), + ) + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + fd = -1 + stream.write(payload) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temp_path, path) + dir_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + finally: + if fd >= 0: + os.close(fd) + temp_path.unlink(missing_ok=True) + + +__all__ = [ + "StorageIdentityMigrationRequiredError", + "StorageIdentityState", + "ensure_storage_identity_ready", + "mark_storage_identity_ready", + "mark_storage_identity_rebuilding", + "marker_path", + "read_storage_identity_state", +] diff --git a/src/everos/infra/persistence/lancedb/tables/agent_case.py b/src/everos/infra/persistence/lancedb/tables/agent_case.py index 6618f1fd..ed601dc3 100644 --- a/src/everos/infra/persistence/lancedb/tables/agent_case.py +++ b/src/everos/infra/persistence/lancedb/tables/agent_case.py @@ -25,7 +25,7 @@ class AgentCase(BaseLanceTable): BM25_FIELDS: ClassVar[list[str]] = ["task_intent_tokens", "approach_tokens"] id: str - """PK = ``_``.""" + """Opaque storage PK scoped by app, project, owner, and entry id.""" entry_id: str """md-side seq id ``ac__``.""" diff --git a/src/everos/infra/persistence/lancedb/tables/agent_skill.py b/src/everos/infra/persistence/lancedb/tables/agent_skill.py index 0e0ee634..c9390b61 100644 --- a/src/everos/infra/persistence/lancedb/tables/agent_skill.py +++ b/src/everos/infra/persistence/lancedb/tables/agent_skill.py @@ -1,8 +1,11 @@ """LanceDB ``agent_skill`` table schema. Field set for the agent-skill LanceDB row. AgentSkill is a *named -entity* rather than a daily-log entry — PK is ``_`` -(no date / seq), and same agent + same name is the same row (upsert). +entity* rather than a daily-log entry. Its opaque storage PK is the +generation-2 length-prefixed encoding of ``(app_id, project_id, owner_id, +skill_name)``. The historical public wire id remains +``_``; same scope + same agent + same name is the +same row (upsert). ``content`` is cascade-assembled from ``SKILL.md`` body plus every ``references/*.md`` sibling; ``scripts/`` is not indexed. @@ -24,7 +27,7 @@ class AgentSkill(BaseLanceTable): BM25_FIELDS: ClassVar[list[str]] = ["description_tokens", "content_tokens"] id: str - """PK = ``_``.""" + """Opaque storage PK scoped by app, project, owner, and skill name.""" owner_id: str """The owning ``agent_id``.""" @@ -37,7 +40,7 @@ class AgentSkill(BaseLanceTable): """App / project scope (default ``"default"``); cascade fills from md path.""" name: str - """Skill identifier; half of the PK.""" + """Logical skill identifier and one input to the scoped storage PK.""" description: str """When-to-use / purpose — original surface form (Tier-1 ad copy).""" diff --git a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py index c839f6c6..41788b23 100644 --- a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py +++ b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py @@ -24,7 +24,7 @@ class AtomicFact(BaseLanceTable): BM25_FIELDS: ClassVar[list[str]] = ["fact_tokens"] id: str - """PK = ``_``.""" + """Opaque storage PK scoped by app, project, owner, and entry id.""" entry_id: str """md-side seq id ``af__``.""" diff --git a/src/everos/infra/persistence/lancedb/tables/episode.py b/src/everos/infra/persistence/lancedb/tables/episode.py index f09ba1dd..2d0ff37f 100644 --- a/src/everos/infra/persistence/lancedb/tables/episode.py +++ b/src/everos/infra/persistence/lancedb/tables/episode.py @@ -26,7 +26,7 @@ class Episode(BaseLanceTable): BM25_FIELDS: ClassVar[list[str]] = ["episode_tokens"] id: str - """PK = ``_`` (scalar PK).""" + """Opaque storage PK scoped by app, project, owner, and entry id.""" entry_id: str """md-side seq id ``ep__`` (cascade reverse-lookup).""" diff --git a/src/everos/infra/persistence/lancedb/tables/foresight.py b/src/everos/infra/persistence/lancedb/tables/foresight.py index 51a9de94..c3c3668e 100644 --- a/src/everos/infra/persistence/lancedb/tables/foresight.py +++ b/src/everos/infra/persistence/lancedb/tables/foresight.py @@ -25,7 +25,7 @@ class Foresight(BaseLanceTable): BM25_FIELDS: ClassVar[list[str]] = ["foresight_tokens", "evidence_tokens"] id: str - """PK = ``_``.""" + """Opaque storage PK scoped by app, project, owner, and entry id.""" entry_id: str """md-side seq id ``fs__``.""" diff --git a/src/everos/infra/persistence/lancedb/tables/user_profile.py b/src/everos/infra/persistence/lancedb/tables/user_profile.py index 87654b9d..ae451fa2 100644 --- a/src/everos/infra/persistence/lancedb/tables/user_profile.py +++ b/src/everos/infra/persistence/lancedb/tables/user_profile.py @@ -28,7 +28,7 @@ class UserProfile(BaseLanceTable): # No BM25 columns: profile recall is KV-by-owner today. id: str - """PK = ``owner_id`` (one row per user).""" + """Opaque storage PK scoped by app, project, and owner.""" owner_id: str owner_type: str diff --git a/src/everos/infra/persistence/markdown/mds/agent_skill.py b/src/everos/infra/persistence/markdown/mds/agent_skill.py index fcff0139..deb18d15 100644 --- a/src/everos/infra/persistence/markdown/mds/agent_skill.py +++ b/src/everos/infra/persistence/markdown/mds/agent_skill.py @@ -5,9 +5,11 @@ the frontmatter contract). Skills are *named entities* rather than daily-log entries: the -LanceDB primary key is ``_`` (no date / seq). -Upserts replace the file wholesale; the cascade daemon recomputes the -``content`` index column by concatenating ``SKILL.md`` body with every +LanceDB primary key is the generation-2 length-prefixed encoding of +``(app_id, project_id, owner_id, skill_name)`` (no date / seq). The +historical public wire id remains ``_``. Upserts +replace the file wholesale; the cascade daemon recomputes the ``content`` +index column by concatenating ``SKILL.md`` body with every ``references/*.md`` sibling. Five directory-shape ClassVars pin the layout in one place so the diff --git a/src/everos/infra/persistence/markdown/writers/episode_writer.py b/src/everos/infra/persistence/markdown/writers/episode_writer.py index 3d885cf2..7ff8682c 100644 --- a/src/everos/infra/persistence/markdown/writers/episode_writer.py +++ b/src/everos/infra/persistence/markdown/writers/episode_writer.py @@ -35,8 +35,8 @@ class EpisodeWriter(BaseDailyWriter): ``append_entry`` / ``append_entries`` come from :class:`BaseDailyWriter`; the ``entry_id`` (``ep__``) is the in-file identity allocated under the per-path lock. Callers - can derive a globally-unique id from ``(owner_id, entry_id)`` - without persisting any algo-side uuid. + retain the stable public identity ``(owner_id, entry_id)``. Cascade derives + a separate app/project-scoped storage key for LanceDB. """ schema = EpisodeDailyFrontmatter diff --git a/src/everos/infra/persistence/sqlite/repos/cluster.py b/src/everos/infra/persistence/sqlite/repos/cluster.py index f84eea6e..0d66d766 100644 --- a/src/everos/infra/persistence/sqlite/repos/cluster.py +++ b/src/everos/infra/persistence/sqlite/repos/cluster.py @@ -122,12 +122,11 @@ async def find_cluster_id_for_member( to ``(app_id, project_id, owner_id)``. ``member_id`` (e.g. episode ``entry_id`` like - ``ep_20260517_00000001``) is only per-owner unique — see - ``core/persistence/markdown/entries.py``: "Cross-user uniqueness - is handled at the database layer via a composite - ``_`` field; it is not encoded into the - EntryId string itself." Without the scope filter, two owners - writing on the same day would share ``entry_id`` and either + ``ep_20260517_00000001``) is only unique inside one owner's app/project + scope. The logical EntryId deliberately does not encode those scope + fields; callers supply app, project, and owner for the reverse lookup. + Without that scope filter, two owners writing on the same day would + share ``entry_id`` and either collide on the reverse index (false hit → the second owner's row silently drops from a cluster it was never part of) or find a foreign cluster. diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index a6fa9c5b..8b6aa512 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -30,6 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp @@ -77,7 +78,12 @@ async def _build_row( reason="embedding_capability_unavailable", ) return AgentCase( - id=f"{owner_id}_{entry.entry_id}", + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=entry.entry_id, + ), entry_id=entry.entry_id, owner_id=owner_id, owner_type=owner_type, diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 88ae8395..69e37a05 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,6 +42,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader +from everos.core.persistence.lancedb.row_id import agent_skill_storage_id from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo from everos.infra.persistence.markdown import AgentSkillFrontmatter @@ -112,7 +113,12 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: ) # Skip when an existing row has the same digest. - skill_id = f"{owner_id}_{name}" + skill_id = agent_skill_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + name=name, + ) prior = await agent_skill_repo.get_by_id(skill_id) if prior is not None and prior.content_sha256 == digest: return HandlerOutcome( diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 6dea35e4..606bebca 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -23,6 +23,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp @@ -62,7 +63,12 @@ async def _build_row( reason="embedding_capability_unavailable", ) return AtomicFact( - id=f"{owner_id}_{entry.entry_id}", + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=entry.entry_id, + ), entry_id=entry.entry_id, owner_id=owner_id, owner_type=owner_type, diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 84878d30..5d2584cd 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -38,6 +38,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp @@ -102,7 +103,12 @@ async def _build_row( tokens = self._deps.tokenizer.tokenize(tokenize_source) return Episode( - id=f"{owner_id}_{entry.entry_id}", + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=entry.entry_id, + ), entry_id=entry.entry_id, owner_id=owner_id, owner_type=owner_type, diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index adbefe02..07035ffb 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -30,6 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo from ._common import ( @@ -86,7 +87,12 @@ async def _build_row( " ".join(self._deps.tokenizer.tokenize(evidence)) if evidence else None ) return Foresight( - id=f"{owner_id}_{entry.entry_id}", + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=entry.entry_id, + ), entry_id=entry.entry_id, owner_id=owner_id, owner_type=owner_type, diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d..ff167f87 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,6 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader +from everos.core.persistence.lancedb.row_id import user_profile_storage_id from everos.infra.persistence.lancedb import UserProfile, user_profile_repo from ..types import HandlerOutcome @@ -80,7 +81,11 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: } ) - row_id = owner_id + row_id = user_profile_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + ) prior = await user_profile_repo.get_by_id(row_id) if prior is not None and prior.content_sha256 == digest: return HandlerOutcome( diff --git a/src/everos/memory/extract/pipeline/user_memory.py b/src/everos/memory/extract/pipeline/user_memory.py index b401f6be..676a6e33 100644 --- a/src/everos/memory/extract/pipeline/user_memory.py +++ b/src/everos/memory/extract/pipeline/user_memory.py @@ -252,8 +252,9 @@ def _episode_to_entry_body( back so the LanceDB ``episode`` row keeps its back-link to the source. The md entry's ``entry_id`` (managed by the chassis writer) is the - single source of *entry* identity; cascade derives a global episode - id from ``_`` on the fly. + single source of *entry* identity. Cascade derives an app/project-scoped + storage key, while response shaping preserves the historical + ``_`` wire id. """ ts_iso = ( to_iso_format(from_timestamp(episode.timestamp)) diff --git a/src/everos/memory/get/dto.py b/src/everos/memory/get/dto.py index df9804c6..9cfd76d7 100644 --- a/src/everos/memory/get/dto.py +++ b/src/everos/memory/get/dto.py @@ -29,6 +29,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from everos.core.scope_ids import AppId, ProjectId from everos.memory.search import FilterNode @@ -69,8 +70,8 @@ class GetRequest(BaseModel): agent_id: str | None = Field(default=None, min_length=1) """Memory owner — provide ``user_id`` for ``episode`` / ``profile`` or ``agent_id`` for ``agent_case`` / ``agent_skill``; exactly one must be set.""" - app_id: str = "default" - project_id: str = "default" + app_id: AppId = "default" + project_id: ProjectId = "default" """App / project scope (default ``"default"``). Pinned into the query ``where`` so a listing never crosses into another space's rows.""" memory_type: GetMemoryType diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index c42edd7e..136b4e2a 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -23,6 +23,12 @@ from everos.component.utils.datetime import to_display_tz from everos.core.context import resolve_request_id from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import ( + agent_skill_wire_id, + daily_log_wire_id, + user_profile_storage_id, + user_profile_wire_id, +) from .dto import ( GetAgentCaseItem, @@ -94,7 +100,11 @@ async def get(self, req: GetRequest) -> GetResponse: count=len(items), ) case GetMemoryType.PROFILE: - profiles = await self._fetch_profile(req.owner_id) + profiles = await self._fetch_profile( + req.owner_id, + app_id=req.app_id, + project_id=req.project_id, + ) data = GetData( profiles=profiles, total_count=len(profiles), @@ -140,7 +150,7 @@ async def get(self, req: GetRequest) -> GetResponse: @staticmethod def _shape_episode(row: Episode) -> GetEpisodeItem: return GetEpisodeItem( - id=row.id, + id=daily_log_wire_id(owner_id=row.owner_id, entry_id=row.entry_id), user_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, @@ -156,7 +166,7 @@ def _shape_episode(row: Episode) -> GetEpisodeItem: @staticmethod def _shape_agent_case(row: AgentCase) -> GetAgentCaseItem: return GetAgentCaseItem( - id=row.id, + id=daily_log_wire_id(owner_id=row.owner_id, entry_id=row.entry_id), agent_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, @@ -171,7 +181,7 @@ def _shape_agent_case(row: AgentCase) -> GetAgentCaseItem: @staticmethod def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: return GetAgentSkillItem( - id=row.id, + id=agent_skill_wire_id(owner_id=row.owner_id, name=row.name), agent_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, @@ -185,7 +195,13 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: # ── Profile ────────────────────────────────────────────────────── - async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: + async def _fetch_profile( + self, + owner_id: str, + *, + app_id: str, + project_id: str, + ) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` LanceDB table (kept in sync with ``users//user.md`` by cascade). @@ -197,7 +213,12 @@ async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """ if not owner_id: return [] - row = await self._profile.get_by_id(owner_id) + storage_id = user_profile_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + ) + row = await self._profile.get_by_id(storage_id) if row is None: logger.debug("get_profile_miss", owner_id=owner_id) return [] @@ -209,7 +230,7 @@ async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: } return [ GetProfileItem( - id=row.id, + id=user_profile_wire_id(owner_id=row.owner_id), user_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 547adc64..1e491df5 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -824,6 +824,8 @@ async def _apply_deprecation_writes( deprecated_fact_count = await self._deprecate_lance_facts( parent_ids=to_deprecate, owner_id=owner_id, + app_id=app_id, + project_id=project_id, merged_entry_id=merged_entry_id, ) return deprecated_ep_count, deprecated_fact_count @@ -883,6 +885,8 @@ async def _deprecate_lance_facts( *, parent_ids: set[str], owner_id: str, + app_id: str, + project_id: str, merged_entry_id: str, ) -> int: """Mark deprecated atomic facts in LanceDB. @@ -890,6 +894,8 @@ async def _deprecate_lance_facts( Args: parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. owner_id: Target owner identifier. + app_id: Application scope. + project_id: Project scope. merged_entry_id: Entry ID of the replacement merged episode. Returns: @@ -904,6 +910,8 @@ async def _deprecate_lance_facts( where=( f"parent_id = '{_escape_sql(pid)}' " f"AND owner_id = '{_escape_sql(owner_id)}' " + f"AND app_id = '{_escape_sql(app_id)}' " + f"AND project_id = '{_escape_sql(project_id)}' " f"AND deprecated_by IS NULL" ), ) diff --git a/src/everos/memory/search/dto.py b/src/everos/memory/search/dto.py index 53d37fa5..f3d10cd5 100644 --- a/src/everos/memory/search/dto.py +++ b/src/everos/memory/search/dto.py @@ -27,6 +27,8 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from everos.core.scope_ids import AppId, ProjectId + class SearchMethod(StrEnum): """Public method enum. RRF / LR / vector_anchored are hidden under HYBRID.""" @@ -72,8 +74,8 @@ class SearchRequest(BaseModel): """Memory owner — provide ``user_id`` for user-memory (episodes / profiles) or ``agent_id`` for agent-memory (cases / skills); exactly one must be set.""" - app_id: str = "default" - project_id: str = "default" + app_id: AppId = "default" + project_id: ProjectId = "default" """App / project scope (default ``"default"``). Pinned into the LanceDB ``where`` so a search never crosses into another space's rows.""" query: str = Field(min_length=1) diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd9..bec15069 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -309,10 +309,15 @@ async def _search_cases_and_skills( produced high-quality case scores to inherit. """ if _effective_llm_rerank(req): - cases = await self._search_agent_cases(req, where) - bridge_cases = [ - Candidate(id=c.id, score=c.score, source="vector", metadata={}) - for c in cases + # Preserve the storage candidate and its logical ``entry_id`` + # through the bridge. Public DTO ids are compatibility wire ids + # and cannot safely be reverse-engineered into lineage ids. + bridge_cases = await self._rank_agent_case_candidates(req, where) + cases = [ + item + for candidate in bridge_cases + for item in [shape_agent_case_from_candidate(candidate)] + if item is not None ] skills = await self._search_agent_skills( req, where, bridge_cases=bridge_cases @@ -448,6 +453,19 @@ async def _search_agent_cases( llm=self._llm, # type: ignore[arg-type] top_k=self._top_k(req.top_k), ) + case_candidates = await self._rank_agent_case_candidates(req, where) + shaped = (shape_agent_case_from_candidate(c) for c in case_candidates) + return [item for item in shaped if item is not None] + + async def _rank_agent_case_candidates( + self, req: SearchRequest, where: str + ) -> list[Candidate]: + """Recall and rank cases while retaining storage-side metadata. + + This internal form is consumed by the case-to-skill lineage bridge. + Public wire shaping happens only after the bridge has read the logical + ``entry_id`` carried in each candidate's metadata. + """ fusion_mode, _ = resolve_pipeline(req.method, "agent_case") enable_rerank = _effective_llm_rerank(req) top_k = self._top_k(req.top_k, cap=_AGENT_TOP_K_CAP) @@ -456,8 +474,7 @@ async def _search_agent_cases( cands = await self._single_route_recall( self._case, req, where, top_k, cap=_AGENT_TOP_K_CAP ) - shaped = (shape_agent_case_from_candidate(c) for c in cands[:top_k]) - return [item for item in shaped if item is not None] + return cands[:top_k] sparse, dense, _ = await self._recall_sparse_dense( self._case, req, where, top_k, cap=_AGENT_TOP_K_CAP @@ -483,9 +500,7 @@ async def _search_agent_cases( enable_rerank=enable_rerank, rerank_top_k=top_k, ) - case_candidates = (_scored_as_candidate(s) for s in output.items) - shaped = (shape_agent_case_from_candidate(c) for c in case_candidates) - return [item for item in shaped if item is not None] + return [_scored_as_candidate(s) for s in output.items] # ── Agent skills ──────────────────────────────────────────────── @@ -576,7 +591,11 @@ async def _search_agent_skills( async def _fetch_profile(self, req: SearchRequest) -> list[SearchProfileItem]: if not req.include_profile or req.owner_type != "user": return [] - return await self._profile.fetch(req.owner_id) + return await self._profile.fetch( + req.owner_id, + app_id=req.app_id, + project_id=req.project_id, + ) # ── Recall helpers ────────────────────────────────────────────── @@ -694,7 +713,21 @@ async def _case_bridged_skills( """ if not bridge_cases: return [] - case_score = {c.id: c.score for c in bridge_cases} + case_score: dict[str, float] = {} + for candidate in bridge_cases: + entry_id = candidate.metadata.get("entry_id") + if not isinstance(entry_id, str) or not entry_id: + logger.warning( + "case_bridge_missing_entry_id", + candidate_id=candidate.id, + ) + continue + case_score[entry_id] = max( + candidate.score, + case_score.get(entry_id, float("-inf")), + ) + if not case_score: + return [] # Bound the reverse fetch by the matched-case count; one case can map # to several skills, so allow a small fan-out per case. skill_cands = await self._skill.fetch_by_case_ids( diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b92..1b1b1c1b 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,6 +19,10 @@ from typing import Any from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import ( + user_profile_storage_id, + user_profile_wire_id, +) from everos.infra.persistence.lancedb import user_profile_repo from ..dto import SearchProfileItem @@ -29,7 +33,13 @@ class ProfileRecaller: """Fetch the owner's profile row from LanceDB, return at most one item.""" - async def fetch(self, owner_id: str) -> list[SearchProfileItem]: + async def fetch( + self, + owner_id: str, + *, + app_id: str, + project_id: str, + ) -> list[SearchProfileItem]: """Return ``[item]`` if a profile row exists, otherwise ``[]``. Empty list (rather than 404) lets the caller emit a normal @@ -38,7 +48,12 @@ async def fetch(self, owner_id: str) -> list[SearchProfileItem]: """ if not owner_id: return [] - row = await user_profile_repo.get_by_id(owner_id) + storage_id = user_profile_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + ) + row = await user_profile_repo.get_by_id(storage_id) if row is None: logger.debug("profile_fetch_miss", owner_id=owner_id) return [] @@ -50,7 +65,7 @@ async def fetch(self, owner_id: str) -> list[SearchProfileItem]: } return [ SearchProfileItem( - id=row.id, + id=user_profile_wire_id(owner_id=row.owner_id), user_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, diff --git a/src/everos/memory/search/shaper.py b/src/everos/memory/search/shaper.py index 14bfce6e..8ea6a435 100644 --- a/src/everos/memory/search/shaper.py +++ b/src/everos/memory/search/shaper.py @@ -27,6 +27,10 @@ from everos.component.utils.datetime import to_display_tz from everos.core.observability.logging import get_logger +from everos.core.persistence.lancedb.row_id import ( + agent_skill_wire_id, + daily_log_wire_id, +) from .dto import ( SearchAgentCaseItem, @@ -66,12 +70,20 @@ def shape_episode_from_candidate( return None session_id = md.get("session_id") episode = md.get("episode") - if not isinstance(episode, str): + owner_id = md.get("owner_id") + entry_id = md.get("entry_id") + if not ( + isinstance(episode, str) + and isinstance(owner_id, str) + and owner_id + and isinstance(entry_id, str) + and entry_id + ): logger.warning("shape_episode_missing_required_field", id=candidate.id) return None return SearchEpisodeItem( - id=candidate.id, - user_id=_as_optional_str(md.get("owner_id")), + id=daily_log_wire_id(owner_id=owner_id, entry_id=entry_id), + user_id=owner_id, app_id=_as_str(md.get("app_id")) or "default", project_id=_as_str(md.get("project_id")) or "default", session_id=session_id, @@ -89,21 +101,43 @@ def shape_episode_from_candidate( # ── Atomic fact shaping ───────────────────────────────────────────────── -def shape_atomic_fact_from_candidate(candidate: Candidate) -> SearchAtomicFactItem: +def shape_atomic_fact_from_candidate( + candidate: Candidate, +) -> SearchAtomicFactItem | None: """Cast a LanceDB ``atomic_fact`` candidate row into the nested fact DTO.""" + owner_id = candidate.metadata.get("owner_id") + entry_id = candidate.metadata.get("entry_id") + if not ( + isinstance(owner_id, str) + and owner_id + and isinstance(entry_id, str) + and entry_id + ): + logger.warning("shape_fact_missing_identity", id=candidate.id) + return None content = _as_str(candidate.metadata.get("fact")) return SearchAtomicFactItem( - id=candidate.id, + id=daily_log_wire_id(owner_id=owner_id, entry_id=entry_id), content=content, score=float(candidate.score), ) -def shape_atomic_fact_from_scored(scored: ScoredItem) -> SearchAtomicFactItem: +def shape_atomic_fact_from_scored(scored: ScoredItem) -> SearchAtomicFactItem | None: """Cast an everalgo ``ScoredItem(item_type='atomic_fact')`` into a fact DTO.""" + owner_id = scored.metadata.get("owner_id") + entry_id = scored.metadata.get("entry_id") + if not ( + isinstance(owner_id, str) + and owner_id + and isinstance(entry_id, str) + and entry_id + ): + logger.warning("shape_fact_missing_identity", id=scored.id) + return None content = _as_str(scored.metadata.get("fact")) return SearchAtomicFactItem( - id=scored.id, + id=daily_log_wire_id(owner_id=owner_id, entry_id=entry_id), content=content, score=float(scored.score), ) @@ -126,12 +160,16 @@ def shape_agent_case_from_candidate(candidate: Candidate) -> SearchAgentCaseItem logger.warning("shape_case_missing_timestamp", id=candidate.id) return None owner_id = md.get("owner_id") + entry_id = md.get("entry_id") session_id = md.get("session_id") task_intent = md.get("task_intent") approach = md.get("approach") quality = md.get("quality_score") if not ( isinstance(owner_id, str) + and owner_id + and isinstance(entry_id, str) + and entry_id and isinstance(session_id, str) and isinstance(task_intent, str) and isinstance(approach, str) @@ -140,7 +178,7 @@ def shape_agent_case_from_candidate(candidate: Candidate) -> SearchAgentCaseItem logger.warning("shape_case_missing_required_field", id=candidate.id) return None return SearchAgentCaseItem( - id=candidate.id, + id=daily_log_wire_id(owner_id=owner_id, entry_id=entry_id), agent_id=owner_id, app_id=_as_str(md.get("app_id")) or "default", project_id=_as_str(md.get("project_id")) or "default", @@ -185,7 +223,7 @@ def shape_agent_skill_from_candidate( logger.warning("shape_skill_missing_required_field", id=candidate.id) return None return SearchAgentSkillItem( - id=candidate.id, + id=agent_skill_wire_id(owner_id=owner_id, name=name), agent_id=owner_id, app_id=_as_str(md.get("app_id")) or "default", project_id=_as_str(md.get("project_id")) or "default", @@ -267,8 +305,10 @@ def reshape_hybrid_output( def _build_fact_items(scoreds: list[ScoredItem]) -> list[SearchAtomicFactItem]: return [ - shape_atomic_fact_from_scored(s) + item for s in sorted(scoreds, key=lambda s: s.score, reverse=True) + for item in [shape_atomic_fact_from_scored(s)] + if item is not None ] diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 2040ace1..7ddef2c0 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -44,6 +44,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MemoryRoot from everos.core.persistence.markdown import dump_frontmatter, parse_frontmatter +from everos.core.scope_ids import validate_app_id, validate_project_id from everos.infra.persistence.markdown import ( KnowledgeWriter, ensure_taxonomy, @@ -67,7 +68,6 @@ _DOC_ID_PREFIX = "d_" _DOC_ID_HEX_LEN = 12 _MAX_MINT_RETRIES = 5 -_SCOPE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_.\-@+]+$") _ORIGINAL_DIR_NAME = "_original" _KNOWLEDGE_FEATURE = "knowledge" @@ -366,6 +366,8 @@ async def get_document( Raises: DocumentNotFoundError: When no row exists for ``doc_id``. """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") row = await knowledge_document_repo.get_by_doc_id(doc_id) if row is None: raise DocumentNotFoundError(f"Document {doc_id!r} not found") @@ -422,6 +424,8 @@ async def list_documents( Returns: DocumentListResult with items, total, page, and page_size. """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") page_result = await knowledge_document_repo.list_documents( app_id=app_id, project_id=project_id, @@ -475,6 +479,8 @@ async def get_topic( Raises: TopicNotFoundError: When no row exists for ``topic_id``. """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") rows = await knowledge_topic_sqlite_repo.get_topics_by_ids([topic_id]) if not rows: raise TopicNotFoundError(f"Topic {topic_id!r} not found") @@ -517,6 +523,8 @@ async def delete_document( Returns: DeleteResult with the topic count that was present before deletion. """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") row = await knowledge_document_repo.get_by_doc_id(doc_id) if row is None: return DeleteResult(doc_id=doc_id, deleted_topics=0) @@ -848,6 +856,8 @@ async def patch_document( Raises: DocumentNotFoundError: When neither SQLite nor md files contain ``doc_id``. """ + _validate_scope_id(app_id, "app_id") + _validate_scope_id(project_id, "project_id") memory_root = MemoryRoot.resolve() current = await _resolve_current_doc(doc_id, app_id, project_id, memory_root) @@ -1135,17 +1145,12 @@ class SearchKnowledgeResult: def _validate_scope_id(value: str, name: str) -> None: - """Reject scope ids with characters that could break LanceDB SQL. - - Args: - value: The identifier value to validate. - name: The parameter name (for error messages). - - Raises: - ValueError: If the value is empty or contains invalid characters. - """ - if not value or not _SCOPE_ID_PATTERN.match(value): - raise ValueError(f"{name} contains invalid characters: {value!r}") + """Apply the shared field-specific scope contract with local context.""" + validator = validate_app_id if name == "app_id" else validate_project_id + try: + validator(value) + except ValueError as exc: + raise ValueError(f"{name} is invalid: {exc}") from exc def compile_knowledge_where(app_id: str, project_id: str) -> str: diff --git a/src/everos/service/memorize.py b/src/everos/service/memorize.py index 3ce96139..969ff93e 100644 --- a/src/everos/service/memorize.py +++ b/src/everos/service/memorize.py @@ -33,6 +33,7 @@ from everos.core.context import resolve_request_id from everos.core.observability.tracing import memory_span from everos.core.persistence import MemoryRoot +from everos.core.scope_ids import validate_app_id, validate_project_id from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.persistence.markdown import EpisodeWriter @@ -186,8 +187,8 @@ async def memorize( """Execute one add cycle. Dispatched concurrently across pipelines. Args: - payload: ``{"session_id", "messages": [...]}`` — entrypoints DTO - dumped to dict. + payload: ``{"session_id", "messages": [...], "app_id"?, + "project_id"?}`` from an entrypoint DTO or a direct Python caller. is_final: ``True`` only for flush (algo guarantees ``tail=[]``). Concurrency: serialised per ``session_id`` via @@ -199,6 +200,17 @@ async def memorize( LLM cannot hold the lock indefinitely — on timeout the task is cancelled and ``async with`` auto-releases the lock. """ + # ``memorize`` is also a public Python entrypoint, so it must enforce the + # same scope contract as the HTTP DTO before tracing, locking, ingestion, + # or any SQLite/markdown side effect can occur. + payload = dict(payload) + app_id = payload.get("app_id", "default") + project_id = payload.get("project_id", "default") + if not isinstance(app_id, str) or not isinstance(project_id, str): + raise ValueError("app_id and project_id must be strings") + payload["app_id"] = validate_app_id(app_id) + payload["project_id"] = validate_project_id(project_id) + settings = load_settings() mode = settings.memorize.mode boundary_cfg = settings.boundary_detection diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 55af0989..fd691b03 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -20,6 +20,11 @@ from httpx import ASGITransport, AsyncClient from everos.config import load_settings +from everos.core.persistence.lancedb.row_id import ( + agent_skill_storage_id, + daily_log_storage_id, + user_profile_storage_id, +) from everos.entrypoints.api.app import create_app from everos.infra.persistence.lancedb import ( AgentCase, @@ -54,7 +59,12 @@ def _episode( day: int = 1, ) -> Episode: return Episode( - id=f"{owner}_{entry}", + id=daily_log_storage_id( + app_id="default", + project_id="default", + owner_id=owner, + entry_id=entry, + ), entry_id=entry, owner_id=owner, owner_type="user", @@ -81,7 +91,12 @@ def _agent_case( day: int = 1, ) -> AgentCase: return AgentCase( - id=f"{owner}_{entry}", + id=daily_log_storage_id( + app_id="default", + project_id="default", + owner_id=owner, + entry_id=entry, + ), entry_id=entry, owner_id=owner, owner_type="agent", @@ -107,7 +122,12 @@ def _agent_skill( owner: str = "a1", ) -> AgentSkill: return AgentSkill( - id=f"{owner}_{name}", + id=agent_skill_storage_id( + app_id="default", + project_id="default", + owner_id=owner, + name=name, + ), owner_id=owner, owner_type="agent", name=name, @@ -117,7 +137,7 @@ def _agent_skill( content_tokens=f"content {name}", confidence=0.9, maturity_score=0.7, - source_case_ids=[f"{owner}_ac_1"], + source_case_ids=["ac_1"], md_path=f"agents/{owner}/skills/{name}/SKILL.md", content_sha256="abc", vector=[0.0] * 1024, @@ -252,7 +272,11 @@ async def test_get_profile_returns_seeded_row(client: AsyncClient) -> None: await user_profile_repo.add( [ UserProfile( - id="u1", + id=user_profile_storage_id( + app_id="default", + project_id="default", + owner_id="u1", + ), owner_id="u1", owner_type="user", app_id="default", diff --git a/tests/e2e/test_search_endpoint_e2e.py b/tests/e2e/test_search_endpoint_e2e.py index b329b549..1be78ac3 100644 --- a/tests/e2e/test_search_endpoint_e2e.py +++ b/tests/e2e/test_search_endpoint_e2e.py @@ -42,6 +42,11 @@ from everos.component.embedding import get_embedding_capability from everos.config import load_settings +from everos.core.persistence.lancedb.row_id import ( + agent_skill_storage_id, + daily_log_storage_id, + user_profile_storage_id, +) from everos.entrypoints.api.app import create_app from everos.infra.persistence.lancedb import ( AgentCase, @@ -149,7 +154,19 @@ async def _seed_atomic_facts(rows: list[dict[str, Any]]) -> list[AtomicFact]: async def _seed_user_profiles(rows: list[dict[str, Any]]) -> list[UserProfile]: - profiles = [UserProfile.model_validate(r) for r in rows] + profiles = [ + UserProfile.model_validate( + { + **row, + "id": user_profile_storage_id( + app_id=str(row.get("app_id") or "default"), + project_id=str(row.get("project_id") or "default"), + owner_id=str(row["owner_id"]), + ), + } + ) + for row in rows + ] await user_profile_repo.add(profiles) # profile table has no FTS — no index rebuild needed. return profiles @@ -212,7 +229,12 @@ def _agent_case( intent = task_intent if task_intent is not None else f"intent {entry}" appr = approach if approach is not None else f"approach {entry}" return AgentCase( - id=f"{owner}_{entry}", + id=daily_log_storage_id( + app_id="default", + project_id="default", + owner_id=owner, + entry_id=entry, + ), entry_id=entry, owner_id=owner, owner_type="agent", @@ -244,7 +266,12 @@ def _agent_skill( desc = description if description is not None else f"desc {name}" body = content if content is not None else f"content {name}" return AgentSkill( - id=f"{owner}_{name}", + id=agent_skill_storage_id( + app_id="default", + project_id="default", + owner_id=owner, + name=name, + ), owner_id=owner, owner_type="agent", name=name, @@ -255,7 +282,7 @@ def _agent_skill( confidence=0.9, maturity_score=0.7, source_case_ids=( - list(source_case_ids) if source_case_ids is not None else [f"{owner}_ac_1"] + list(source_case_ids) if source_case_ids is not None else ["ac_1"] ), md_path=f"agents/{owner}/skills/{name}/SKILL.md", content_sha256="abc", @@ -580,7 +607,7 @@ async def test_hybrid_rerank_bridges_skill_via_case_lineage( surface it on its own; the bridge is the path that does, and LLM rerank keeps it because the topic is genuinely relevant. """ - case_id_with_owner = "a_bridge_ac_1" # mirrors AgentCase.id = "_" + logical_case_id = "ac_1" case_intent = "refactor authentication middleware" case_approach = "split provider lookup from session decode" # Skill is the *generalised lesson* from this case — same domain (auth / @@ -619,7 +646,7 @@ async def test_hybrid_rerank_bridges_skill_via_case_lineage( owner="a_bridge", description=skill_desc, content=skill_body, - source_case_ids=[case_id_with_owner], + source_case_ids=[logical_case_id], vector=skill_vec, ), ] @@ -639,7 +666,7 @@ async def test_hybrid_rerank_bridges_skill_via_case_lineage( case_ids = [c["id"] for c in data["agent_cases"]] skill_ids = [s["id"] for s in data["agent_skills"]] - assert case_id_with_owner in case_ids, ( + assert f"a_bridge_{logical_case_id}" in case_ids, ( f"case should match the query directly; got {case_ids}" ) assert "a_bridge_graphql_resolver_patterns" in skill_ids, ( diff --git a/tests/integration/test_cascade_all_kinds_consistency.py b/tests/integration/test_cascade_all_kinds_consistency.py index c2310ecf..19dcf74f 100644 --- a/tests/integration/test_cascade_all_kinds_consistency.py +++ b/tests/integration/test_cascade_all_kinds_consistency.py @@ -323,3 +323,75 @@ async def test_md_lance_strict_consistency_per_kind( ) finally: await orchestrator.stop() + + +async def test_issue_320_same_owner_entry_survives_across_projects( + cascade_runtime: MemoryRoot, +) -> None: + """Distinct project scopes must not overwrite the same daily entry id.""" + memory_root = cascade_runtime + orchestrator = CascadeOrchestrator( + memory_root=memory_root, + tokenizer=build_tokenizer(), + config=CascadeConfig( + scan_interval_seconds=60.0, + worker_batch_size=20, + worker_max_retry=1, + worker_poll_interval_seconds=0.05, + worker_retry_backoff_seconds=0.0, + ), + ) + await orchestrator.start() + await asyncio.sleep(0.3) + + try: + writer = EpisodeWriter(root=memory_root) + owner_id = "u_issue_320" + bucket = _dt.date(2026, 7, 1) + entry_ids: list[str] = [] + + for project_id, content in ( + ("project-a-test", "project A memory"), + ("project-b-test", "project B memory"), + ): + entry_id = await writer.append_entry( + owner_id, + inline={ + "owner_id": owner_id, + "session_id": f"session-{project_id}", + "timestamp": "2026-07-01T12:00:00+00:00", + "parent_id": f"parent-{project_id}", + "sender_ids": [owner_id], + }, + sections={"Content": content}, + date=bucket, + app_id="demo-app", + project_id=project_id, + ) + entry_ids.append(entry_id.format()) + md_path = ( + f"demo-app/{project_id}/users/{owner_id}/episodes/episode-2026-07-01.md" + ) + await _wait_path_done(md_path) + + assert entry_ids[0] == entry_ids[1] + entry_id = entry_ids[0] + row_a = await episode_repo.find_by_owner_entry( + owner_id, + entry_id, + app_id="demo-app", + project_id="project-a-test", + ) + row_b = await episode_repo.find_by_owner_entry( + owner_id, + entry_id, + app_id="demo-app", + project_id="project-b-test", + ) + + assert row_a is not None + assert row_b is not None + assert row_a.episode == "project A memory" + assert row_b.episode == "project B memory" + finally: + await orchestrator.stop() diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 22511ddf..fd82bb03 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -18,6 +18,7 @@ import datetime as _dt import re from collections.abc import Iterator +from contextlib import asynccontextmanager from pathlib import Path import pytest @@ -26,6 +27,9 @@ from everos.config import load_settings from everos.entrypoints.cli.commands import cascade as cascade_mod from everos.infra.persistence.lancedb import dispose_connection +from everos.infra.persistence.lancedb.projection_lock import ( + ProjectionLockUnavailableError, +) from everos.infra.persistence.sqlite import dispose_engine @@ -268,9 +272,11 @@ async def _atomic_fact_row_count() -> int: cascade_mod, "_build_orchestrator", _fake_orchestrator_factory() ) - # Contrast: a normal command boots via _runtime() → verify trips on the drift. + # Read-only status uses SQLite only, so operators can inspect the queue even + # while the LanceDB projection is drifted or requires a rebuild. status_result = CliRunner().invoke(cascade_mod.app, ["status"]) - assert status_result.exit_code != 0 + assert status_result.exit_code == 0 + assert "queue:" in status_result.stdout asyncio.run(_dispose_all()) # rebuild skips verify, recreates the table, and re-indexes md. @@ -293,11 +299,17 @@ def test_rebuild_refuses_to_run_while_a_server_holds_the_lock( It drops and recreates the LanceDB tables; a live daemon holds cached table handles and would keep writing to the dropped dataset, leaving a - corrupted rebuild plus a permanent-failure backlog. Detection reuses the - OME jobstore lock that ``backfill`` already gates on, and the exit code - matches backfill's ``3`` (SERVER_RUNNING). + corrupted rebuild plus a permanent-failure backlog. Detection uses the + retained projection/OME rebuild lock, and the exit code matches + backfill's ``3`` (SERVER_RUNNING). """ - monkeypatch.setattr(cascade_mod, "ome_lock_is_free", lambda: False) + + @asynccontextmanager + async def unavailable(_root): # type: ignore[no-untyped-def] + raise ProjectionLockUnavailableError("OME exclusive lock is held") + yield + + monkeypatch.setattr(cascade_mod, "projection_rebuild_lock", unavailable) result = CliRunner().invoke(cascade_mod.app, ["rebuild", "--yes"]) diff --git a/tests/integration/test_cascade_integration.py b/tests/integration/test_cascade_integration.py index d59e1d7b..98bef399 100644 --- a/tests/integration/test_cascade_integration.py +++ b/tests/integration/test_cascade_integration.py @@ -27,6 +27,7 @@ from everos.component.embedding import EmbeddingProvider from everos.component.tokenizer import build_tokenizer from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import ( dispose_connection, ensure_business_indexes, @@ -186,7 +187,12 @@ async def _state_done(): # type: ignore[no-untyped-def] assert done_row.error is None # 3. LanceDB carries the typed episode row. - episode_id = f"u_integration_{eid.format()}" + episode_id = daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u_integration", + entry_id=eid.format(), + ) ep_row = await episode_repo.get_by_id(episode_id) assert ep_row is not None assert ep_row.episode == "the user mentioned dark mode preference" @@ -242,7 +248,14 @@ async def test_delete_md_wipes_lancedb_row( absolute = memory_root.root / md_path async def _ep_present(): # type: ignore[no-untyped-def] - return await episode_repo.get_by_id(f"u_del_{eid.format()}") + return await episode_repo.get_by_id( + daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u_del", + entry_id=eid.format(), + ) + ) await _poll(_ep_present, deadline_seconds=10.0) @@ -250,7 +263,14 @@ async def _ep_present(): # type: ignore[no-untyped-def] absolute.unlink() async def _ep_gone(): # type: ignore[no-untyped-def] - row = await episode_repo.get_by_id(f"u_del_{eid.format()}") + row = await episode_repo.get_by_id( + daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u_del", + entry_id=eid.format(), + ) + ) return row is None assert await _poll(_ep_gone, deadline_seconds=10.0) diff --git a/tests/integration/test_issue_320_runtime_matrix.py b/tests/integration/test_issue_320_runtime_matrix.py new file mode 100644 index 00000000..294f06d7 --- /dev/null +++ b/tests/integration/test_issue_320_runtime_matrix.py @@ -0,0 +1,423 @@ +"""Real-LanceDB regression matrix for owner-scoped storage identities. + +The six owner-scoped business schemas must use the complete logical scope as +their physical primary key. These tests intentionally exercise the concrete +repositories against a temporary on-disk LanceDB rather than mocking the +storage boundary. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest +from everalgo.types import Candidate + +from everos.core.persistence.lancedb.repository import LanceRepoBase +from everos.core.persistence.lancedb.row_id import ( + agent_skill_storage_id, + agent_skill_wire_id, + daily_log_storage_id, + daily_log_wire_id, + user_profile_storage_id, + user_profile_wire_id, +) +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + UserProfile, + agent_case_repo, + agent_skill_repo, + atomic_fact_repo, + dispose_connection, + episode_repo, + foresight_repo, + user_profile_repo, +) +from everos.memory.search.recall.base import row_to_candidate +from everos.memory.search.recall.profile import ProfileRecaller +from everos.memory.search.shaper import ( + shape_agent_case_from_candidate, + shape_agent_skill_from_candidate, + shape_atomic_fact_from_candidate, + shape_episode_from_candidate, +) + +_TS = dt.datetime(2026, 8, 12, 12, 0, tzinfo=dt.UTC) +_VECTOR = [0.0] * 1024 + + +@pytest.fixture(autouse=True) +async def _fresh_lancedb() -> Any: + """Keep singleton tables and event-loop-bound write locks test-local.""" + LanceRepoBase._reset_locks_for_tests() + await dispose_connection() + yield + await dispose_connection() + + +def _episode( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> Episode: + return Episode( + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=logical_id, + ), + entry_id=logical_id, + owner_id=owner_id, + owner_type="user", + app_id=app_id, + project_id=project_id, + session_id="session", + timestamp=_TS, + parent_id="mc_1", + sender_ids=[owner_id], + subject="subject", + summary="summary", + episode=payload, + episode_tokens=payload, + md_path=f"{app_id}/{project_id}/users/{owner_id}/episodes/day.md", + content_sha256=payload, + vector=_VECTOR, + ) + + +def _atomic_fact( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> AtomicFact: + return AtomicFact( + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=logical_id, + ), + entry_id=logical_id, + owner_id=owner_id, + owner_type="user", + app_id=app_id, + project_id=project_id, + session_id="session", + timestamp=_TS, + parent_id="mc_1", + sender_ids=[owner_id], + fact=payload, + fact_tokens=payload, + md_path=f"{app_id}/{project_id}/users/{owner_id}/facts/day.md", + content_sha256=payload, + vector=_VECTOR, + ) + + +def _foresight( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> Foresight: + return Foresight( + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=logical_id, + ), + entry_id=logical_id, + owner_id=owner_id, + owner_type="user", + app_id=app_id, + project_id=project_id, + session_id="session", + timestamp=_TS, + parent_id="mc_1", + sender_ids=[owner_id], + foresight=payload, + foresight_tokens=payload, + md_path=f"{app_id}/{project_id}/users/{owner_id}/foresights/day.md", + content_sha256=payload, + vector=_VECTOR, + ) + + +def _agent_case( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> AgentCase: + return AgentCase( + id=daily_log_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + entry_id=logical_id, + ), + entry_id=logical_id, + owner_id=owner_id, + owner_type="agent", + app_id=app_id, + project_id=project_id, + session_id="session", + timestamp=_TS, + parent_id="mc_1", + quality_score=0.9, + task_intent=payload, + task_intent_tokens=payload, + approach="approach", + approach_tokens="approach", + md_path=f"{app_id}/{project_id}/agents/{owner_id}/cases/day.md", + content_sha256=payload, + vector=_VECTOR, + ) + + +def _agent_skill( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> AgentSkill: + return AgentSkill( + id=agent_skill_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + name=logical_id, + ), + owner_id=owner_id, + owner_type="agent", + app_id=app_id, + project_id=project_id, + name=logical_id, + description=payload, + description_tokens=payload, + content="content", + content_tokens="content", + confidence=0.9, + maturity_score=0.8, + source_case_ids=[], + md_path=f"{app_id}/{project_id}/agents/{owner_id}/skills/{logical_id}.md", + content_sha256=payload, + vector=_VECTOR, + ) + + +def _user_profile( + app_id: str, project_id: str, owner_id: str, logical_id: str, payload: str +) -> UserProfile: + del logical_id + return UserProfile( + id=user_profile_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + ), + owner_id=owner_id, + owner_type="user", + app_id=app_id, + project_id=project_id, + summary=payload, + explicit_info_json="[]", + implicit_traits_json="[]", + profile_timestamp_ms=1, + md_path=f"{app_id}/{project_id}/users/{owner_id}/user.md", + content_sha256=payload, + ) + + +def _daily_wire(owner_id: str, logical_id: str) -> str: + return daily_log_wire_id(owner_id=owner_id, entry_id=logical_id) + + +def _skill_wire(owner_id: str, logical_id: str) -> str: + return agent_skill_wire_id(owner_id=owner_id, name=logical_id) + + +def _profile_wire(owner_id: str, logical_id: str) -> str: + del logical_id + return user_profile_wire_id(owner_id=owner_id) + + +def _shape_episode(candidate: Candidate) -> Any: + return shape_episode_from_candidate(candidate) + + +def _shape_fact(candidate: Candidate) -> Any: + return shape_atomic_fact_from_candidate(candidate) + + +def _shape_case(candidate: Candidate) -> Any: + return shape_agent_case_from_candidate(candidate) + + +def _shape_skill(candidate: Candidate) -> Any: + return shape_agent_skill_from_candidate(candidate) + + +@dataclass(frozen=True) +class _SchemaCase: + name: str + repo: Any + build: Callable[[str, str, str, str, str], Any] + payload_field: str + wire_id: Callable[[str, str], str] + shaper: Callable[[Candidate], Any] | None + profile_identity: bool = False + + async def fetch( + self, *, app_id: str, project_id: str, owner_id: str, logical_id: str + ) -> Any: + if self.profile_identity: + return await self.repo.get_by_id( + user_profile_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + ) + ) + if self.name == "agent_skill": + return await self.repo.get_by_id( + agent_skill_storage_id( + app_id=app_id, + project_id=project_id, + owner_id=owner_id, + name=logical_id, + ) + ) + return await self.repo.find_by_owner_entry( + owner_id, + logical_id, + app_id=app_id, + project_id=project_id, + ) + + +_CASES = [ + _SchemaCase( + "episode", episode_repo, _episode, "episode", _daily_wire, _shape_episode + ), + _SchemaCase( + "atomic_fact", atomic_fact_repo, _atomic_fact, "fact", _daily_wire, _shape_fact + ), + _SchemaCase( + "foresight", foresight_repo, _foresight, "foresight", _daily_wire, None + ), + _SchemaCase( + "agent_case", + agent_case_repo, + _agent_case, + "task_intent", + _daily_wire, + _shape_case, + ), + _SchemaCase( + "agent_skill", + agent_skill_repo, + _agent_skill, + "description", + _skill_wire, + _shape_skill, + ), + _SchemaCase( + "user_profile", + user_profile_repo, + _user_profile, + "summary", + _profile_wire, + None, + profile_identity=True, + ), +] + + +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case.name) +async def test_owner_scoped_identity_runtime_matrix(case: _SchemaCase) -> None: + """Prove coexistence, update isolation, injectivity, and wire containment.""" + app_id = "app-main" + owner_id = "owner_same" + logical_id = "entry_same" + + row_a = case.build(app_id, "project-a", owner_id, logical_id, "payload-a") + row_b = case.build(app_id, "project-b", owner_id, logical_id, "payload-b") + await case.repo.upsert([row_a]) + await case.repo.upsert([row_b]) + + # Cross-project coexistence for an otherwise identical logical identity. + assert await case.repo.count() == 2 + stored_a = await case.fetch( + app_id=app_id, + project_id="project-a", + owner_id=owner_id, + logical_id=logical_id, + ) + stored_b = await case.fetch( + app_id=app_id, + project_id="project-b", + owner_id=owner_id, + logical_id=logical_id, + ) + assert stored_a is not None and stored_b is not None + assert stored_a.id != stored_b.id + assert getattr(stored_a, case.payload_field) == "payload-a" + assert getattr(stored_b, case.payload_field) == "payload-b" + + # Replaying an unchanged same-scope row is idempotent. + await case.repo.upsert([row_a]) + assert await case.repo.count() == 2 + + # Updating project A replaces only project A, never its project B sibling. + updated_a = case.build( + app_id, "project-a", owner_id, logical_id, "payload-a-updated" + ) + await case.repo.upsert([updated_a]) + assert await case.repo.count() == 2 + stored_a = await case.fetch( + app_id=app_id, + project_id="project-a", + owner_id=owner_id, + logical_id=logical_id, + ) + stored_b = await case.fetch( + app_id=app_id, + project_id="project-b", + owner_id=owner_id, + logical_id=logical_id, + ) + assert getattr(stored_a, case.payload_field) == "payload-a-updated" + assert getattr(stored_b, case.payload_field) == "payload-b" + + # These tuples collide under naive underscore joining. Length-prefixing + # must remain injective for delimiter-heavy and Unicode scope segments. + if case.profile_identity: + injective_a = case.build("应用", "项目_甲", "乙_丙", "", "unicode-a") + injective_b = case.build("应用", "项目", "甲_乙_丙", "", "unicode-b") + else: + injective_a = case.build("应用", "项目_甲", "乙", "丙_丁", "unicode-a") + injective_b = case.build("应用", "项目", "甲_乙", "丙_丁", "unicode-b") + assert injective_a.id != injective_b.id + await case.repo.upsert([injective_a, injective_b]) + assert await case.repo.count() == 4 + assert await case.repo.get_by_id(injective_a.id) is not None + assert await case.repo.get_by_id(injective_b.id) is not None + + # Every currently exposed DTO is shaped from logical identity fields, not + # the physical storage key. Foresight has no public response shaper today. + expected_wire_id = case.wire_id(owner_id, logical_id) + assert stored_a.id != expected_wire_id + if case.shaper is not None: + candidate = row_to_candidate( + stored_a.model_dump(mode="python"), source="vector", score=0.5 + ) + public_item = case.shaper(candidate) + assert public_item is not None + assert public_item.id == expected_wire_id + assert stored_a.id not in public_item.model_dump_json() + elif case.profile_identity: + public_items = await ProfileRecaller().fetch( + owner_id, + app_id=app_id, + project_id="project-a", + ) + assert len(public_items) == 1 + assert public_items[0].id == expected_wire_id + assert stored_a.id not in public_items[0].model_dump_json() diff --git a/tests/integration/test_memorize_integration.py b/tests/integration/test_memorize_integration.py index 0f1fc0ed..7d2bc0ef 100644 --- a/tests/integration/test_memorize_integration.py +++ b/tests/integration/test_memorize_integration.py @@ -259,6 +259,10 @@ def _episode_paths(tmp_path: Path) -> list[Path]: return sorted(base.rglob("episode-*.md")) +def _markdown_paths(tmp_path: Path) -> list[Path]: + return sorted(tmp_path.rglob("*.md")) + + # --------------------------------------------------------------------------- # Happy path baseline # --------------------------------------------------------------------------- @@ -301,6 +305,26 @@ async def test_chat_baseline_two_msgs_one_cell( assert "Test body" in body +async def test_direct_memorize_invalid_scope_has_no_persistence_side_effects( + tmp_path: Path, + memorize_env: Callable[..., Any], +) -> None: + await memorize_env(mode="chat", fake_llm=_make_fake_llm()) + payload = { + "session_id": "invalid_scope", + "app_id": "DEFAULT_APP", + "project_id": "project-a", + "messages": [_user("must not persist", 1_700_000_000_000)], + } + + with pytest.raises(ValueError): + await memorize(payload) + + assert _buffer_count(tmp_path) == 0 + assert _memcell_rows(tmp_path) == [] + assert _markdown_paths(tmp_path) == [] + + # --------------------------------------------------------------------------- # Input-shape boundary cases (6) # --------------------------------------------------------------------------- diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 63b91c2f..e1247de0 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -33,6 +33,7 @@ open_lancedb_connection, ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.ome.testing import FakeStrategyContext from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode @@ -224,6 +225,95 @@ async def _teardown_sqlite() -> None: # --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_fact_deprecation_isolated_by_project( + memory_root: MemoryRoot, +) -> None: + """Deprecating a fact in one project must not mutate its scoped sibling.""" + conn = await open_lancedb_connection(memory_root.lancedb_dir, LanceDBSettings()) + try: + table = await conn.create_table("atomic_fact", schema=AtomicFact) + fact_repo = _AtomicFactRepo(table=table) + + owner_id = "u_shared" + parent_id = "ep_shared" + entry_id = "af_20260610_0001" + app_id = "app_shared" + project_a = "project_a" + project_b = "project_b" + timestamp = _dt.datetime(2026, 6, 10, 10, 0, tzinfo=_dt.UTC) + + fact_a = _make_lance_fact( + entry_id=entry_id, + owner_id=owner_id, + fact="Project A fact", + parent_id=parent_id, + timestamp=timestamp, + ).model_copy( + update={ + "id": daily_log_storage_id( + app_id=app_id, + project_id=project_a, + owner_id=owner_id, + entry_id=entry_id, + ), + "app_id": app_id, + "project_id": project_a, + } + ) + fact_b = _make_lance_fact( + entry_id=entry_id, + owner_id=owner_id, + fact="Project B fact", + parent_id=parent_id, + timestamp=timestamp, + ).model_copy( + update={ + "id": daily_log_storage_id( + app_id=app_id, + project_id=project_b, + owner_id=owner_id, + entry_id=entry_id, + ), + "app_id": app_id, + "project_id": project_b, + } + ) + await fact_repo.add([fact_a, fact_b]) + + orchestrator = ReflectionOrchestrator( + cluster_repo=object(), + episode_store=object(), + atomic_fact_store=fact_repo, + episode_writer=object(), + report_repo=object(), + reflector=object(), + embedder=object(), + ) + update_count = await orchestrator._deprecate_lance_facts( + parent_ids={parent_id}, + owner_id=owner_id, + app_id=app_id, + project_id=project_a, + merged_entry_id="ep_merged", + ) + + assert update_count == 1 + rows_a = await fact_repo.find_where( + f"owner_id = '{owner_id}' AND parent_id = '{parent_id}' " + f"AND app_id = '{app_id}' AND project_id = '{project_a}'" + ) + rows_b = await fact_repo.find_where( + f"owner_id = '{owner_id}' AND parent_id = '{parent_id}' " + f"AND app_id = '{app_id}' AND project_id = '{project_b}'" + ) + assert len(rows_a) == len(rows_b) == 1 + assert rows_a[0].deprecated_by == "ep_merged" + assert rows_b[0].deprecated_by is None + finally: + conn.close() + + @pytest.mark.asyncio async def test_reflection_init_merges_cluster_episodes( tmp_path: Path, diff --git a/tests/unit/test_core/test_persistence/test_lancedb/test_row_id.py b/tests/unit/test_core/test_persistence/test_lancedb/test_row_id.py new file mode 100644 index 00000000..bb1b2bff --- /dev/null +++ b/tests/unit/test_core/test_persistence/test_lancedb/test_row_id.py @@ -0,0 +1,73 @@ +"""Storage identity and public wire compatibility contracts.""" + +from __future__ import annotations + +from everos.core.persistence.lancedb.row_id import ( + agent_skill_storage_id, + agent_skill_wire_id, + daily_log_storage_id, + daily_log_wire_id, + make_storage_id, + user_profile_storage_id, + user_profile_wire_id, +) + + +def test_storage_id_is_injective_across_part_boundaries() -> None: + assert make_storage_id("a_b", "c") != make_storage_id("a", "b_c") + + +def test_storage_id_counts_utf8_bytes_and_is_deterministic() -> None: + assert make_storage_id("é", "x") == "2:é1:x" + assert make_storage_id("é", "x") == make_storage_id("é", "x") + + +def test_daily_log_storage_id_changes_with_every_scope_dimension() -> None: + base = daily_log_storage_id( + app_id="app_a", project_id="project_a", owner_id="owner", entry_id="ep_1" + ) + variants = { + daily_log_storage_id( + app_id="app_b", + project_id="project_a", + owner_id="owner", + entry_id="ep_1", + ), + daily_log_storage_id( + app_id="app_a", + project_id="project_b", + owner_id="owner", + entry_id="ep_1", + ), + daily_log_storage_id( + app_id="app_a", + project_id="project_a", + owner_id="other", + entry_id="ep_1", + ), + daily_log_storage_id( + app_id="app_a", + project_id="project_a", + owner_id="owner", + entry_id="ep_2", + ), + } + assert base not in variants + assert len(variants) == 4 + + +def test_kind_specific_storage_ids_include_scope() -> None: + assert agent_skill_storage_id( + app_id="app", project_id="one", owner_id="agent", name="skill" + ) != agent_skill_storage_id( + app_id="app", project_id="two", owner_id="agent", name="skill" + ) + assert user_profile_storage_id( + app_id="app", project_id="one", owner_id="user" + ) != user_profile_storage_id(app_id="app", project_id="two", owner_id="user") + + +def test_wire_ids_keep_the_historical_public_shape() -> None: + assert daily_log_wire_id(owner_id="owner", entry_id="ep_1") == "owner_ep_1" + assert agent_skill_wire_id(owner_id="agent", name="skill") == "agent_skill" + assert user_profile_wire_id(owner_id="user") == "user" diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index 6da6ccd9..ee6ca6f3 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -6,7 +6,11 @@ import pytest -from everos.core.persistence import MemoryRoot +from everos.core.persistence import ( + MemoryRoot, + app_id_from_dir, + project_id_from_dir, +) def test_resolve_returns_home_everos( @@ -87,6 +91,47 @@ def test_user_visible_dirs_named_scope(tmp_path: Path) -> None: assert mr.knowledge_dir("claude_code", "oss") == base / "knowledge" +def test_reserved_default_directory_aliases_are_rejected(tmp_path: Path) -> None: + """Caller scope ids cannot alias the reserved default directories.""" + mr = MemoryRoot(tmp_path) + + with pytest.raises(ValueError, match="reserved app"): + mr.users_dir("default_app", "project") + with pytest.raises(ValueError, match="reserved project"): + mr.users_dir("app", "default_project") + + +@pytest.mark.parametrize( + "app_id, project_id", + [ + ("DEFAULT_APP", "project"), + ("app", "DEFAULT_PROJECT"), + ("app", "Project-A"), + (".index", "project"), + (".tmp", "project"), + ], +) +def test_nonportable_scopes_are_rejected_before_path_resolution( + tmp_path: Path, + app_id: str, + project_id: str, +) -> None: + mr = MemoryRoot(tmp_path) + + with pytest.raises(ValueError): + mr.users_dir(app_id, project_id) + + +def test_scope_recovery_rejects_nonportable_existing_directories() -> None: + with pytest.raises(ValueError): + app_id_from_dir("DEFAULT_APP") + with pytest.raises(ValueError): + project_id_from_dir("Project-A") + + assert app_id_from_dir("default_app") == "default" + assert project_id_from_dir("default_project") == "default" + + def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" diff --git a/tests/unit/test_core/test_scope_ids.py b/tests/unit/test_core/test_scope_ids.py new file mode 100644 index 00000000..5e0b8ed9 --- /dev/null +++ b/tests/unit/test_core/test_scope_ids.py @@ -0,0 +1,89 @@ +"""Portable filesystem contracts for app and project scope identifiers.""" + +from __future__ import annotations + +import pytest + +from everos.core.scope_ids import validate_app_id, validate_project_id + + +@pytest.mark.parametrize( + "value", + [ + "default", + "app", + "project-1", + "team_one", + "release.2", + "user@example.com", + "tag+value", + "x" * 128, + ], +) +def test_scope_ids_accept_portable_lowercase_values(value: str) -> None: + assert validate_app_id(value) == value + assert validate_project_id(value) == value + + +@pytest.mark.parametrize( + "value", + [ + "", + ".", + "..", + "Project-A", + "safe\n", + "contains space", + "contains/slash", + "trailing.", + "x" * 129, + ], +) +def test_scope_ids_reject_nonportable_values(value: str) -> None: + with pytest.raises(ValueError): + validate_app_id(value) + with pytest.raises(ValueError): + validate_project_id(value) + + +@pytest.mark.parametrize( + "value", + [ + "con", + "con.txt", + "prn", + "aux.log", + "nul", + "com1", + "com9.json", + "lpt1", + "lpt9.txt", + ], +) +def test_scope_ids_reject_windows_device_names(value: str) -> None: + with pytest.raises(ValueError, match="filesystem device"): + validate_app_id(value) + with pytest.raises(ValueError, match="filesystem device"): + validate_project_id(value) + + +@pytest.mark.parametrize( + "value", + [ + ".index", + ".lock", + ".projection.lock", + ".tmp", + "default_app", + "everos.toml", + "ome.toml", + ], +) +def test_app_ids_reject_memory_root_managed_names(value: str) -> None: + with pytest.raises(ValueError, match="reserved app"): + validate_app_id(value) + + +def test_project_id_rejects_default_directory_alias() -> None: + with pytest.raises(ValueError, match="reserved project"): + validate_project_id("default_project") diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py index 312bbfe4..34a595c2 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_storage.py @@ -2,16 +2,25 @@ from __future__ import annotations +from contextlib import asynccontextmanager from pathlib import Path +from types import SimpleNamespace +import anyio import pytest from fastapi import FastAPI +import everos.entrypoints.api.lifespans.lancedb as lancedb_lifespan +from everos.core.persistence import MemoryRoot from everos.entrypoints.api.lifespans import ( LanceDBLifespanProvider, SqliteLifespanProvider, ) from everos.infra.persistence.lancedb import lancedb_manager +from everos.infra.persistence.lancedb.projection_lock import ( + ProjectionLockUnavailableError, + projection_rebuild_lock, +) from everos.infra.persistence.sqlite import sqlite_manager @@ -60,6 +69,7 @@ async def test_lancedb_provider_startup_opens_connection(tmp_path: Path) -> None assert conn is await lancedb_manager.get_connection() # singleton wired assert (tmp_path / ".index" / "lancedb").is_dir() + await provider.shutdown(app) async def test_lancedb_provider_shutdown_disposes_singleton() -> None: @@ -70,3 +80,302 @@ async def test_lancedb_provider_shutdown_disposes_singleton() -> None: await provider.shutdown(app) assert lancedb_manager._conn is None + + +async def test_lancedb_provider_holds_shared_projection_lock_through_disposal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider = LanceDBLifespanProvider() + app = FastAPI() + real_verify = lancedb_lifespan.verify_storage_identity_ready + real_dispose = lancedb_lifespan.dispose_connection + observed: list[str] = [] + + async def verify_while_locked() -> None: + with pytest.raises(ProjectionLockUnavailableError): + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + raise AssertionError("rebuild lock entered during marker verification") + observed.append("verify_locked") + await real_verify() + + async def dispose_while_locked() -> None: + with pytest.raises(ProjectionLockUnavailableError): + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + raise AssertionError("rebuild lock entered before connection disposal") + observed.append("dispose_locked") + await real_dispose() + + monkeypatch.setattr( + lancedb_lifespan, "verify_storage_identity_ready", verify_while_locked + ) + monkeypatch.setattr(lancedb_lifespan, "dispose_connection", dispose_while_locked) + + await provider.startup(app) + await provider.shutdown(app) + + assert observed == ["verify_locked", "dispose_locked"] + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + pass + + +async def test_lancedb_provider_serializes_only_bootstrap_in_fixed_lock_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + @asynccontextmanager + async def server_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("projection_shared_enter") + try: + yield + finally: + events.append("projection_shared_exit") + + @asynccontextmanager + async def bootstrap_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + assert events == ["projection_shared_enter"] + events.append("bootstrap_exclusive_enter") + try: + yield + finally: + events.append("bootstrap_exclusive_exit") + + async def record(name: str, result=None): # type: ignore[no-untyped-def] + events.append(name) + return result + + monkeypatch.setattr(lancedb_lifespan, "projection_server_lock", server_lock) + monkeypatch.setattr(lancedb_lifespan, "projection_bootstrap_lock", bootstrap_lock) + monkeypatch.setattr( + lancedb_lifespan, + "verify_storage_identity_ready", + lambda: record("identity_gate"), + ) + monkeypatch.setattr( + lancedb_lifespan, + "get_connection", + lambda: record("connection", SimpleNamespace(uri="test://lancedb")), + ) + monkeypatch.setattr( + lancedb_lifespan, + "verify_business_schemas", + lambda: record("schema_verify"), + ) + monkeypatch.setattr( + lancedb_lifespan, + "ensure_business_indexes", + lambda: record("index_ensure"), + ) + monkeypatch.setattr( + lancedb_lifespan, + "_log_unbackfilled_hint", + lambda: record("unbackfilled_hint"), + ) + monkeypatch.setattr( + lancedb_lifespan, + "dispose_connection", + lambda: record("dispose"), + ) + + provider = LanceDBLifespanProvider() + await provider.startup(FastAPI()) + + assert events == [ + "projection_shared_enter", + "bootstrap_exclusive_enter", + "identity_gate", + "connection", + "schema_verify", + "index_ensure", + "bootstrap_exclusive_exit", + "unbackfilled_hint", + ] + + await provider.shutdown(FastAPI()) + assert events[-2:] == ["dispose", "projection_shared_exit"] + + +async def test_lancedb_bootstrap_failure_cleans_up_before_unlocking( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + @asynccontextmanager + async def server_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("projection_shared_enter") + try: + yield + finally: + events.append("projection_shared_exit") + + @asynccontextmanager + async def bootstrap_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("bootstrap_exclusive_enter") + try: + yield + finally: + events.append("bootstrap_exclusive_exit") + + async def pass_gate() -> None: + events.append("identity_gate") + + async def open_connection(): + events.append("connection") + return SimpleNamespace(uri="test://lancedb") + + async def fail_schema() -> None: + events.append("schema_verify") + raise RuntimeError("schema rejected") + + dispose_calls = 0 + + async def dispose() -> None: + nonlocal dispose_calls + dispose_calls += 1 + events.append(f"dispose_{dispose_calls}") + if dispose_calls == 1: + raise RuntimeError("transient cleanup failure") + + monkeypatch.setattr(lancedb_lifespan, "projection_server_lock", server_lock) + monkeypatch.setattr(lancedb_lifespan, "projection_bootstrap_lock", bootstrap_lock) + monkeypatch.setattr(lancedb_lifespan, "verify_storage_identity_ready", pass_gate) + monkeypatch.setattr(lancedb_lifespan, "get_connection", open_connection) + monkeypatch.setattr(lancedb_lifespan, "verify_business_schemas", fail_schema) + monkeypatch.setattr(lancedb_lifespan, "dispose_connection", dispose) + + with pytest.raises(RuntimeError, match="schema rejected"): + await LanceDBLifespanProvider().startup(FastAPI()) + + assert events == [ + "projection_shared_enter", + "bootstrap_exclusive_enter", + "identity_gate", + "connection", + "schema_verify", + "dispose_1", + "dispose_2", + "bootstrap_exclusive_exit", + "projection_shared_exit", + ] + + +async def test_lancedb_bootstrap_preserves_primary_error_when_cleanup_exhausts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + @asynccontextmanager + async def server_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("projection_shared_enter") + try: + yield + finally: + events.append("projection_shared_exit") + + @asynccontextmanager + async def bootstrap_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("bootstrap_exclusive_enter") + try: + yield + finally: + events.append("bootstrap_exclusive_exit") + + async def pass_gate() -> None: + return None + + async def open_connection(): + return SimpleNamespace(uri="test://lancedb") + + async def fail_schema() -> None: + raise ValueError("PRIMARY_INIT_FAILURE") + + async def fail_dispose() -> None: + events.append("dispose_failed") + raise RuntimeError("SECONDARY_DISPOSE_FAILURE") + + monkeypatch.setattr(lancedb_lifespan, "projection_server_lock", server_lock) + monkeypatch.setattr(lancedb_lifespan, "projection_bootstrap_lock", bootstrap_lock) + monkeypatch.setattr(lancedb_lifespan, "verify_storage_identity_ready", pass_gate) + monkeypatch.setattr(lancedb_lifespan, "get_connection", open_connection) + monkeypatch.setattr(lancedb_lifespan, "verify_business_schemas", fail_schema) + monkeypatch.setattr(lancedb_lifespan, "dispose_connection", fail_dispose) + + with pytest.raises(ValueError, match="PRIMARY_INIT_FAILURE"): + await LanceDBLifespanProvider().startup(FastAPI()) + + assert events == [ + "projection_shared_enter", + "bootstrap_exclusive_enter", + "dispose_failed", + "dispose_failed", + "bootstrap_exclusive_exit", + "projection_shared_exit", + ] + + +async def test_lancedb_shutdown_cancellation_keeps_projection_lock_until_disposed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider = LanceDBLifespanProvider() + app = FastAPI() + await provider.startup(app) + real_dispose = lancedb_lifespan.dispose_connection + dispose_started = anyio.Event() + allow_dispose = anyio.Event() + shutdown_done = anyio.Event() + cancel_scope: anyio.CancelScope | None = None + + async def delayed_dispose() -> None: + dispose_started.set() + await allow_dispose.wait() + await real_dispose() + + monkeypatch.setattr(lancedb_lifespan, "dispose_connection", delayed_dispose) + + async def run_shutdown() -> None: + nonlocal cancel_scope + with anyio.CancelScope() as scope: + cancel_scope = scope + try: + await provider.shutdown(app) + finally: + shutdown_done.set() + + async with anyio.create_task_group() as tasks: + tasks.start_soon(run_shutdown) + await dispose_started.wait() + assert cancel_scope is not None + cancel_scope.cancel() + await anyio.lowlevel.checkpoint() + + assert not shutdown_done.is_set() + with pytest.raises(ProjectionLockUnavailableError): + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + raise AssertionError("rebuild entered during shielded cleanup") + + allow_dispose.set() + await shutdown_done.wait() + + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + pass + + +async def test_lancedb_startup_failure_releases_projection_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fail_verification() -> None: + raise RuntimeError("marker rejected") + + monkeypatch.setattr( + lancedb_lifespan, "verify_storage_identity_ready", fail_verification + ) + provider = LanceDBLifespanProvider() + + with pytest.raises(RuntimeError, match="marker rejected"): + await provider.startup(FastAPI()) + + async with projection_rebuild_lock(MemoryRoot(tmp_path)): + pass diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py index 11f0587b..84ef7242 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_get_route_validation.py @@ -127,6 +127,32 @@ async def test_owner_memory_type_mismatch_returns_422(client: AsyncClient) -> No assert resp.status_code == 422 +@pytest.mark.parametrize( + "field,value", + [ + ("app_id", "DEFAULT_APP"), + ("app_id", ".index"), + ("project_id", "Project-A"), + ("project_id", "default_project"), + ], +) +async def test_nonportable_scope_returns_422( + client: AsyncClient, + field: str, + value: str, +) -> None: + """Filesystem scope aliases are rejected by the public route.""" + payload = { + "user_id": "u1", + "memory_type": "episode", + "app_id": "default", + "project_id": "default", + } + payload[field] = value + resp = await client.post("/api/v1/memory/get", json=payload) + assert resp.status_code == 422 + + # ── service.compile_filters_for_get 422 ─────────────────────────────── diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py b/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py index 5890e95a..22729448 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_knowledge_api.py @@ -519,6 +519,72 @@ async def test_pathsafe_rejects_traversal_in_body(client: AsyncClient) -> None: assert resp.status_code == 422 +@pytest.mark.parametrize( + "params", + [{"app_id": "default_app"}, {"project_id": "default_project"}], +) +async def test_knowledge_query_rejects_reserved_scope_aliases( + client: AsyncClient, + params: dict[str, str], +) -> None: + resp = await client.get("/api/v1/knowledge/documents", params=params) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "scope", + [{"app_id": "default_app"}, {"project_id": "default_project"}], +) +async def test_knowledge_body_rejects_reserved_scope_aliases( + client: AsyncClient, + scope: dict[str, str], +) -> None: + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", **scope}, + ) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "params", + [ + {"project_id": "Project-A"}, + {"app_id": "DEFAULT_APP"}, + {"app_id": ".index"}, + {"app_id": ".tmp"}, + {"project_id": "safe\n"}, + ], +) +async def test_knowledge_query_rejects_nonportable_scopes( + client: AsyncClient, + params: dict[str, str], +) -> None: + resp = await client.get("/api/v1/knowledge/documents", params=params) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "scope", + [ + {"project_id": "Project-A"}, + {"app_id": "DEFAULT_APP"}, + {"app_id": ".index"}, + {"app_id": ".tmp"}, + {"project_id": "safe\n"}, + ], +) +async def test_knowledge_body_rejects_nonportable_scopes( + client: AsyncClient, + scope: dict[str, str], +) -> None: + resp = await client.post( + "/api/v1/knowledge/search", + json={"query": "hello", **scope}, + ) + assert resp.status_code == 422 + + # ── _parse_upload: binary file rejection ──────────────────────────────────── diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py index 045bae2d..7a2c5321 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py @@ -15,6 +15,7 @@ from everos.entrypoints.api.routes.memorize import ( MemorizeAddRequest, + MemorizeFlushRequest, MessageItemDTO, ) @@ -56,6 +57,7 @@ def test_message_item_rejects_unsafe_sender_id(bad_sender_id: str) -> None: "user@example.com", # email-style id (``@`` + dotted domain) "user+tag", # plus-addressing "user+tag@example.com", # both, combined + "User-A", # scope restrictions do not change sender identities ], ) def test_message_item_accepts_path_safe_sender_id(good_sender_id: str) -> None: @@ -78,3 +80,66 @@ def test_add_request_rejects_traversal_sender_id_in_messages() -> None: } ], ) + + +@pytest.mark.parametrize("sender_id", ["default_app", "default_project"]) +def test_reserved_scope_aliases_remain_valid_sender_ids(sender_id: str) -> None: + """Scope aliases are reserved by field, not globally across identities.""" + assert _message(sender_id).sender_id == sender_id + + +@pytest.mark.parametrize( + "request_type, field, value", + [ + (MemorizeAddRequest, "app_id", "default_app"), + (MemorizeAddRequest, "project_id", "default_project"), + (MemorizeFlushRequest, "app_id", "default_app"), + (MemorizeFlushRequest, "project_id", "default_project"), + ], +) +def test_memory_requests_reject_reserved_scope_aliases( + request_type: type[MemorizeAddRequest] | type[MemorizeFlushRequest], + field: str, + value: str, +) -> None: + payload = { + "session_id": "s1", + "app_id": "default", + "project_id": "default", + } + payload[field] = value + if request_type is MemorizeAddRequest: + payload["messages"] = [_message("u1")] + with pytest.raises(ValidationError): + request_type.model_validate(payload) + + +@pytest.mark.parametrize( + "request_type, field, value", + [ + (MemorizeAddRequest, "project_id", "Project-A"), + (MemorizeFlushRequest, "project_id", "Project-A"), + (MemorizeAddRequest, "app_id", "DEFAULT_APP"), + (MemorizeFlushRequest, "app_id", "DEFAULT_APP"), + (MemorizeAddRequest, "app_id", ".index"), + (MemorizeFlushRequest, "app_id", ".tmp"), + (MemorizeAddRequest, "app_id", "x" * 129), + (MemorizeFlushRequest, "project_id", "safe\n"), + ], +) +def test_memory_requests_reject_nonportable_scopes( + request_type: type[MemorizeAddRequest] | type[MemorizeFlushRequest], + field: str, + value: str, +) -> None: + payload = { + "session_id": "s1", + "app_id": "default", + "project_id": "default", + } + payload[field] = value + if request_type is MemorizeAddRequest: + payload["messages"] = [_message("User-A")] + + with pytest.raises(ValidationError): + request_type.model_validate(payload) diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py index d83fa965..1b22f496 100644 --- a/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_search_route_validation.py @@ -120,6 +120,28 @@ async def test_radius_above_one_returns_422(client: AsyncClient) -> None: assert resp.status_code == 422 +@pytest.mark.parametrize( + "field,value", + [ + ("app_id", "DEFAULT_APP"), + ("app_id", ".index"), + ("project_id", "Project-A"), + ("project_id", "default_project"), + ], +) +async def test_nonportable_scope_returns_422( + client: AsyncClient, + field: str, + value: str, +) -> None: + """Filesystem scope aliases are rejected by the public route.""" + resp = await client.post( + "/api/v1/memory/search", + json=_body(**{field: value}), + ) + assert resp.status_code == 422 + + # ── service.compile_filters 422 ─────────────────────────────────────── diff --git a/tests/unit/test_entrypoints/test_cli/test_cascade_storage_identity.py b/tests/unit/test_entrypoints/test_cli/test_cascade_storage_identity.py new file mode 100644 index 00000000..dc22440d --- /dev/null +++ b/tests/unit/test_entrypoints/test_cli/test_cascade_storage_identity.py @@ -0,0 +1,569 @@ +"""Storage-generation gates around mutating cascade CLI commands.""" + +from __future__ import annotations + +import subprocess +import sys +from collections.abc import Iterator +from contextlib import asynccontextmanager, contextmanager +from pathlib import Path +from types import SimpleNamespace + +import anyio +import pytest +from typer.testing import CliRunner + +from everos.config import load_settings +from everos.core.persistence import MemoryRoot +from everos.entrypoints.cli.commands import cascade as cascade_mod +from everos.infra.persistence.lancedb.projection_lock import ( + ProjectionLockUnavailableError, + projection_rebuild_lock, +) +from everos.infra.persistence.lancedb.storage_identity import ( + StorageIdentityMigrationRequiredError, + ensure_storage_identity_ready, + mark_storage_identity_ready, + mark_storage_identity_rebuilding, + marker_path, + read_storage_identity_state, +) + +_MUTATING_RUNTIME_HOLDER = """ +import asyncio +import os +import sys + +os.environ["EVEROS_ROOT"] = sys.argv[1] + +from everos.entrypoints.cli.commands.cascade import _runtime + +async def main(): + async with _runtime(verify=False, ensure=False): + print("LOCKED", flush=True) + sys.stdin.readline() + +asyncio.run(main()) +""" + + +@contextmanager +def _external_mutating_runtime(root: Path) -> Iterator[None]: + process = subprocess.Popen( + [sys.executable, "-c", _MUTATING_RUNTIME_HOLDER, str(root)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdout is not None + startup_lines: list[str] = [] + while True: + ready = process.stdout.readline() + if not ready: + break + startup_lines.append(ready.rstrip()) + if ready.strip() == "LOCKED": + break + if not startup_lines or startup_lines[-1] != "LOCKED": + _out, err = process.communicate(timeout=10) + raise AssertionError( + f"mutating runtime helper failed: stdout={startup_lines!r}, stderr={err!r}" + ) + try: + yield + finally: + if process.stdin is not None: + process.stdin.write("\n") + process.stdin.flush() + process.communicate(timeout=10) + + +def _set_root(monkeypatch: pytest.MonkeyPatch, root: Path) -> None: + monkeypatch.setenv("EVEROS_ROOT", str(root)) + load_settings.cache_clear() + + +def test_sync_is_blocked_while_rebuild_is_incomplete( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + mark_storage_identity_rebuilding(MemoryRoot(tmp_path)) + + result = CliRunner().invoke(cascade_mod.app, ["sync"]) + + assert result.exit_code != 0 + assert isinstance(result.exception, StorageIdentityMigrationRequiredError) + + +def test_status_remains_available_without_opening_lancedb( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + mark_storage_identity_rebuilding(MemoryRoot(tmp_path)) + + async def forbidden_connection(): # type: ignore[no-untyped-def] + raise AssertionError("read-only status must not open LanceDB") + + monkeypatch.setattr(cascade_mod, "get_connection", forbidden_connection) + + result = CliRunner().invoke(cascade_mod.app, ["status"]) + + assert result.exit_code == 0, result.output + assert "pending:" in result.output + + +async def test_runtime_initialization_failure_disposes_under_shared_lock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failed CLI startup closes both stores before releasing exclusion.""" + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + mark_storage_identity_ready(root) + disposed: list[str] = [] + + async def fail_after_open() -> None: + raise RuntimeError("schema verification failed") + + async def dispose_lance() -> None: + with pytest.raises(ProjectionLockUnavailableError, match="projection"): + async with projection_rebuild_lock(root): + raise AssertionError("rebuild entered before LanceDB cleanup") + disposed.append("lancedb") + raise RuntimeError("LanceDB cleanup also failed") + + async def dispose_sqlite() -> None: + with pytest.raises(ProjectionLockUnavailableError, match="projection"): + async with projection_rebuild_lock(root): + raise AssertionError("rebuild entered before SQLite cleanup") + disposed.append("sqlite") + + monkeypatch.setattr(cascade_mod, "verify_business_schemas", fail_after_open) + monkeypatch.setattr(cascade_mod, "dispose_connection", dispose_lance) + monkeypatch.setattr(cascade_mod, "dispose_engine", dispose_sqlite) + + with pytest.raises(RuntimeError, match="schema verification failed"): + async with cascade_mod._runtime(): + raise AssertionError("runtime body must not be entered") + + assert disposed == ["lancedb", "lancedb", "sqlite"] + async with projection_rebuild_lock(root): + pass + + +async def test_runtime_shutdown_attempts_both_disposals_when_first_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Steady-state teardown must not strand SQLite after a LanceDB failure.""" + _set_root(monkeypatch, tmp_path) + mark_storage_identity_ready(MemoryRoot(tmp_path)) + real_dispose_lancedb = cascade_mod.dispose_connection + real_dispose_sqlite = cascade_mod.dispose_engine + disposed: list[str] = [] + + async def fail_lancedb_disposal() -> None: + disposed.append("lancedb") + await real_dispose_lancedb() + raise RuntimeError("LanceDB cleanup failed") + + async def dispose_sqlite() -> None: + disposed.append("sqlite") + await real_dispose_sqlite() + + monkeypatch.setattr(cascade_mod, "dispose_connection", fail_lancedb_disposal) + monkeypatch.setattr(cascade_mod, "dispose_engine", dispose_sqlite) + + with pytest.raises(RuntimeError, match="LanceDB cleanup failed"): + async with cascade_mod._runtime(verify=False, ensure=False): + pass + + assert disposed == ["lancedb", "lancedb", "sqlite"] + + +async def test_runtime_shutdown_cancellation_keeps_rebuild_excluded_until_cleanup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + mark_storage_identity_ready(root) + real_dispose_lancedb = cascade_mod.dispose_connection + dispose_started = anyio.Event() + allow_dispose = anyio.Event() + runtime_done = anyio.Event() + cancel_scope: anyio.CancelScope | None = None + + async def delayed_dispose() -> None: + dispose_started.set() + await allow_dispose.wait() + await real_dispose_lancedb() + + monkeypatch.setattr(cascade_mod, "dispose_connection", delayed_dispose) + + async def run_runtime() -> None: + nonlocal cancel_scope + with anyio.CancelScope() as scope: + cancel_scope = scope + try: + async with cascade_mod._runtime(verify=False, ensure=False): + pass + finally: + runtime_done.set() + + async with anyio.create_task_group() as tasks: + tasks.start_soon(run_runtime) + await dispose_started.wait() + assert cancel_scope is not None + cancel_scope.cancel() + await anyio.lowlevel.checkpoint() + + assert not runtime_done.is_set() + with pytest.raises(ProjectionLockUnavailableError): + async with projection_rebuild_lock(root): + raise AssertionError("rebuild entered during shielded cleanup") + + allow_dispose.set() + await runtime_done.wait() + + async with projection_rebuild_lock(root): + pass + + +async def test_runtime_uses_shared_then_exclusive_bootstrap_lock_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + events: list[str] = [] + + @asynccontextmanager + async def server_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + events.append("projection_shared_enter") + try: + yield + finally: + events.append("projection_shared_exit") + + @asynccontextmanager + async def bootstrap_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + assert events == ["projection_shared_enter"] + events.append("bootstrap_exclusive_enter") + try: + yield + finally: + events.append("bootstrap_exclusive_exit") + + class FakeConnection: + async def run_sync(self, _operation) -> None: # type: ignore[no-untyped-def] + events.append("sqlite_schema") + + class FakeBegin: + async def __aenter__(self) -> FakeConnection: + return FakeConnection() + + async def __aexit__(self, *_args) -> None: # type: ignore[no-untyped-def] + return None + + class FakeEngine: + def begin(self) -> FakeBegin: + return FakeBegin() + + async def record(name: str) -> None: + events.append(name) + + def get_engine() -> FakeEngine: + events.append("sqlite_engine") + return FakeEngine() + + monkeypatch.setattr(cascade_mod, "projection_server_lock", server_lock) + monkeypatch.setattr(cascade_mod, "projection_bootstrap_lock", bootstrap_lock) + monkeypatch.setattr( + cascade_mod, + "verify_storage_identity_ready", + lambda: record("identity_gate"), + ) + monkeypatch.setattr(cascade_mod, "get_engine", get_engine) + monkeypatch.setattr(cascade_mod, "get_connection", lambda: record("lancedb")) + monkeypatch.setattr( + cascade_mod, + "verify_business_schemas", + lambda: record("schema_verify"), + ) + monkeypatch.setattr( + cascade_mod, + "ensure_business_indexes", + lambda: record("index_ensure"), + ) + monkeypatch.setattr( + cascade_mod, + "dispose_connection", + lambda: record("dispose_lancedb"), + ) + monkeypatch.setattr( + cascade_mod, + "dispose_engine", + lambda: record("dispose_sqlite"), + ) + + async with cascade_mod._runtime(): + events.append("runtime_body") + + assert events == [ + "projection_shared_enter", + "bootstrap_exclusive_enter", + "identity_gate", + "sqlite_engine", + "sqlite_schema", + "lancedb", + "schema_verify", + "index_ensure", + "bootstrap_exclusive_exit", + "runtime_body", + "dispose_lancedb", + "dispose_sqlite", + "projection_shared_exit", + ] + + +async def test_rebuild_runtime_bypasses_both_lifecycle_locks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + events: list[str] = [] + + @asynccontextmanager + async def forbidden_lock(_root: MemoryRoot): # type: ignore[no-untyped-def] + raise AssertionError("rebuild runtime must not acquire lifecycle locks") + yield + + class FakeConnection: + async def run_sync(self, _operation) -> None: # type: ignore[no-untyped-def] + events.append("sqlite_schema") + + class FakeBegin: + async def __aenter__(self) -> FakeConnection: + return FakeConnection() + + async def __aexit__(self, *_args) -> None: # type: ignore[no-untyped-def] + return None + + class FakeEngine: + def begin(self) -> FakeBegin: + return FakeBegin() + + monkeypatch.setattr(cascade_mod, "projection_server_lock", forbidden_lock) + monkeypatch.setattr(cascade_mod, "projection_bootstrap_lock", forbidden_lock) + monkeypatch.setattr(cascade_mod, "get_engine", FakeEngine) + + async def record(name: str) -> None: + events.append(name) + + monkeypatch.setattr(cascade_mod, "get_connection", lambda: record("lancedb")) + monkeypatch.setattr( + cascade_mod, + "dispose_connection", + lambda: record("dispose_lancedb"), + ) + monkeypatch.setattr( + cascade_mod, + "dispose_engine", + lambda: record("dispose_sqlite"), + ) + + async with cascade_mod._runtime( + verify=False, + ensure=False, + identity_gate=False, + lifecycle_lock=False, + ): + events.append("runtime_body") + + assert events == [ + "sqlite_schema", + "lancedb", + "runtime_body", + "dispose_lancedb", + "dispose_sqlite", + ] + + +def test_rebuild_sets_rebuilding_before_runtime_and_ready_only_after_clean_queue( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + + @asynccontextmanager + async def fake_runtime(**kwargs): # type: ignore[no-untyped-def] + assert kwargs == { + "verify": False, + "ensure": False, + "identity_gate": False, + "lifecycle_lock": False, + } + state = read_storage_identity_state(root) + assert state is not None and state.state == "REBUILDING" + yield + + class FakeOrchestrator: + async def sync_once(self) -> int: + return 7 + + async def drain_once(self) -> int: + return 0 + + async def queue_summary(self): # type: ignore[no-untyped-def] + return SimpleNamespace( + pending=0, + failed_retryable=0, + failed_permanent=0, + ) + + async def zero() -> int: + return 0 + + async def no_tables() -> list[str]: + return [] + + async def noop() -> None: + return None + + monkeypatch.setattr(cascade_mod, "_runtime", fake_runtime) + monkeypatch.setattr(cascade_mod.md_change_state_repo, "reset_all", zero) + monkeypatch.setattr(cascade_mod, "drop_business_tables", no_tables) + monkeypatch.setattr(cascade_mod, "ensure_business_indexes", noop) + monkeypatch.setattr(cascade_mod, "_build_orchestrator", FakeOrchestrator) + + result = CliRunner().invoke(cascade_mod.app, ["rebuild", "--yes"]) + + assert result.exit_code == 0, result.output + assert "rebuild complete" in result.output + state = read_storage_identity_state(root) + assert state is not None + assert state.state == "READY" + assert state.generation == 2 + + +def test_failed_rebuild_leaves_rebuilding_marker( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + + @asynccontextmanager + async def fake_runtime(**kwargs): # type: ignore[no-untyped-def] + yield + + class FailedOrchestrator: + async def sync_once(self) -> int: + return 1 + + async def drain_once(self) -> int: + return 0 + + async def queue_summary(self): # type: ignore[no-untyped-def] + return SimpleNamespace( + pending=0, + failed_retryable=0, + failed_permanent=1, + ) + + async def zero() -> int: + return 0 + + async def no_tables() -> list[str]: + return [] + + async def noop() -> None: + return None + + monkeypatch.setattr(cascade_mod, "_runtime", fake_runtime) + monkeypatch.setattr(cascade_mod.md_change_state_repo, "reset_all", zero) + monkeypatch.setattr(cascade_mod, "drop_business_tables", no_tables) + monkeypatch.setattr(cascade_mod, "ensure_business_indexes", noop) + monkeypatch.setattr(cascade_mod, "_build_orchestrator", FailedOrchestrator) + + result = CliRunner().invoke(cascade_mod.app, ["rebuild", "--yes"]) + + assert result.exit_code != 0 + assert isinstance(result.exception, RuntimeError) + assert "failed_permanent=1" in str(result.exception) + state = read_storage_identity_state(root) + assert state is not None and state.state == "REBUILDING" + + +def test_interrupted_rebuild_leaves_rebuilding_and_blocks_normal_startup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + + @asynccontextmanager + async def fake_runtime(**kwargs): # type: ignore[no-untyped-def] + state = read_storage_identity_state(root) + assert state is not None and state.state == "REBUILDING" + yield + + async def zero() -> int: + return 0 + + async def interrupt_after_queue_reset() -> list[str]: + raise KeyboardInterrupt + + monkeypatch.setattr(cascade_mod, "_runtime", fake_runtime) + monkeypatch.setattr(cascade_mod.md_change_state_repo, "reset_all", zero) + monkeypatch.setattr( + cascade_mod, "drop_business_tables", interrupt_after_queue_reset + ) + + result = CliRunner().invoke(cascade_mod.app, ["rebuild", "--yes"]) + + assert result.exit_code == 130 + assert "remains REBUILDING" in result.stderr + state = read_storage_identity_state(root) + assert state is not None and state.state == "REBUILDING" + with pytest.raises(StorageIdentityMigrationRequiredError, match="REBUILDING"): + ensure_storage_identity_ready(root) + + +def test_rebuild_lock_refusal_happens_before_generation_or_index_mutation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_root(monkeypatch, tmp_path) + root = MemoryRoot(tmp_path) + mutation_called = False + + @asynccontextmanager + async def unavailable(_root: MemoryRoot): # type: ignore[no-untyped-def] + raise ProjectionLockUnavailableError("projection exclusive lock is held") + yield + + async def forbidden_drop() -> list[str]: + nonlocal mutation_called + mutation_called = True + return [] + + monkeypatch.setattr(cascade_mod, "projection_rebuild_lock", unavailable) + monkeypatch.setattr(cascade_mod, "drop_business_tables", forbidden_drop) + + result = CliRunner().invoke(cascade_mod.app, ["rebuild", "--yes"]) + + assert result.exit_code == 3 + assert "did not modify" in result.stderr + assert mutation_called is False + assert not marker_path(root).exists() + + +async def test_mutating_cli_runtime_excludes_rebuild_across_processes( + tmp_path: Path, +) -> None: + """A CLI writer holds the shared lifecycle lock until its stores close.""" + root = MemoryRoot(tmp_path) + mark_storage_identity_ready(root) + with _external_mutating_runtime(tmp_path): + with pytest.raises(ProjectionLockUnavailableError, match="projection"): + async with projection_rebuild_lock(root): + raise AssertionError("rebuild entered while CLI writer was active") + state = read_storage_identity_state(root) + assert state is not None and state.state == "READY" + + async with projection_rebuild_lock(root): + pass diff --git a/tests/unit/test_infra/test_lancedb/test_projection_lock.py b/tests/unit/test_infra/test_lancedb/test_projection_lock.py new file mode 100644 index 00000000..562eb8fe --- /dev/null +++ b/tests/unit/test_infra/test_lancedb/test_projection_lock.py @@ -0,0 +1,329 @@ +"""Cross-process exclusion contracts for LanceDB projection lifecycle locks.""" + +from __future__ import annotations + +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import portalocker +import pytest + +from everos.core.persistence import MemoryRoot +from everos.infra.persistence.lancedb.projection_lock import ( + ProjectionLockUnavailableError, + ome_lock_path, + projection_bootstrap_lock, + projection_bootstrap_lock_path, + projection_lock_path, + projection_rebuild_lock, + projection_server_lock, +) +from everos.infra.persistence.lancedb.storage_identity import ( + read_storage_identity_state, +) + +_LOCK_HOLDER = """ +import portalocker +import sys + +path = sys.argv[1] +flags = int(sys.argv[2]) | portalocker.LOCK_NB +handle = open(path, "a+") +portalocker.lock(handle, flags) +print("LOCKED", flush=True) +sys.stdin.readline() +portalocker.unlock(handle) +handle.close() +""" + +_FRESH_BOOTSTRAP_WORKER = """ +import asyncio +import sys +from pathlib import Path + +from everos.core.persistence import MemoryRoot +from everos.infra.persistence.lancedb.projection_lock import ( + projection_bootstrap_lock, + projection_server_lock, +) +from everos.infra.persistence.lancedb.storage_identity import ( + ensure_storage_identity_ready, +) + +root = MemoryRoot(Path(sys.argv[1])) +role = sys.argv[2] +entered_path = Path(sys.argv[3]) + +async def main(): + async with projection_server_lock(root): + print(f"{role}:SHARED", flush=True) + async with projection_bootstrap_lock(root): + entered_path.write_text("entered", encoding="utf-8") + print(f"{role}:BOOTSTRAP", flush=True) + if role == "first": + sys.stdin.readline() + ensure_storage_identity_ready(root) + print(f"{role}:READY", flush=True) + +asyncio.run(main()) +""" + +_FRESH_PROVIDER_WORKER = """ +import asyncio +import os +import sys +import time +from pathlib import Path + +os.environ["EVEROS_ROOT"] = sys.argv[1] + +from fastapi import FastAPI + +from everos.config import load_settings +from everos.entrypoints.api.lifespans.lancedb import LanceDBLifespanProvider + +load_settings.cache_clear() +ready_path = Path(sys.argv[2]) +go_path = Path(sys.argv[3]) +ready_path.write_text("ready", encoding="utf-8") +while not go_path.exists(): + time.sleep(0.01) + +async def main(): + provider = LanceDBLifespanProvider() + app = FastAPI() + await provider.startup(app) + await provider.shutdown(app) + +asyncio.run(main()) +""" + + +@contextmanager +def _external_lock(path: Path, flags: int) -> Iterator[None]: + path.parent.mkdir(parents=True, exist_ok=True) + process = subprocess.Popen( + [sys.executable, "-c", _LOCK_HOLDER, str(path), str(flags)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdout is not None + ready = process.stdout.readline().strip() + if ready != "LOCKED": + _out, err = process.communicate(timeout=5) + raise AssertionError(f"external lock helper failed: {err}") + try: + yield + finally: + if process.stdin is not None: + process.stdin.write("\n") + process.stdin.flush() + process.communicate(timeout=5) + + +async def test_external_server_shared_lock_allows_shared_and_blocks_rebuild( + tmp_path: Path, +) -> None: + root = MemoryRoot(tmp_path) + + with _external_lock(projection_lock_path(root), int(portalocker.LOCK_SH)): + async with projection_server_lock(root, timeout_seconds=0.2): + pass + with pytest.raises(ProjectionLockUnavailableError, match="projection"): + async with projection_rebuild_lock(root): + raise AssertionError("exclusive rebuild lock must not be entered") + + async with projection_rebuild_lock(root): + pass + + +async def test_external_rebuild_exclusive_lock_blocks_server_startup( + tmp_path: Path, +) -> None: + root = MemoryRoot(tmp_path) + + with ( + _external_lock(projection_lock_path(root), int(portalocker.LOCK_EX)), + pytest.raises(ProjectionLockUnavailableError, match="timed out"), + ): + async with projection_server_lock(root, timeout_seconds=0.01): + raise AssertionError("server lock entered during active rebuild") + + async with projection_server_lock(root, timeout_seconds=0.2): + pass + + +async def test_bootstrap_lock_is_exclusive_beneath_shared_projection_lock( + tmp_path: Path, +) -> None: + root = MemoryRoot(tmp_path) + assert projection_bootstrap_lock_path(root) == ( + root.index_dir / ".projection.bootstrap.lock" + ) + + async with projection_server_lock(root, timeout_seconds=0.2): + with ( + _external_lock( + projection_bootstrap_lock_path(root), int(portalocker.LOCK_EX) + ), + pytest.raises(ProjectionLockUnavailableError, match="bootstrap"), + ): + async with projection_bootstrap_lock(root, timeout_seconds=0.01): + raise AssertionError("concurrent bootstrap lock must not be entered") + + with pytest.raises(RuntimeError, match="bootstrap failed"): + async with projection_bootstrap_lock(root, timeout_seconds=0.2): + raise RuntimeError("bootstrap failed") + + # A failed bootstrap body releases the exclusive side for a retry while + # the caller continues to retain its shared lifecycle exclusion. + async with projection_bootstrap_lock(root, timeout_seconds=0.2): + pass + + +def test_fresh_initialization_is_serialized_across_processes(tmp_path: Path) -> None: + """A second fresh process waits and observes the first READY marker.""" + first_entered = tmp_path / "first-entered" + second_entered = tmp_path / "second-entered" + + def start(role: str, entered_path: Path) -> subprocess.Popen[str]: + return subprocess.Popen( + [ + sys.executable, + "-c", + _FRESH_BOOTSTRAP_WORKER, + str(tmp_path), + role, + str(entered_path), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + first = start("first", first_entered) + second: subprocess.Popen[str] | None = None + try: + assert first.stdout is not None + assert first.stdout.readline().strip() == "first:SHARED" + assert first.stdout.readline().strip() == "first:BOOTSTRAP" + assert first_entered.exists() + + second = start("second", second_entered) + assert second.stdout is not None + assert second.stdout.readline().strip() == "second:SHARED" + + # The first process has conclusively entered the exclusive bootstrap + # section. Give the second process multiple polling intervals; it must + # remain outside until the first publishes READY and releases the lock. + deadline = time.monotonic() + 0.6 + while time.monotonic() < deadline and not second_entered.exists(): + time.sleep(0.01) + assert not second_entered.exists() + assert second.poll() is None + + assert first.stdin is not None + first.stdin.write("\n") + first.stdin.flush() + first_out, first_err = first.communicate(timeout=10) + second_out, second_err = second.communicate(timeout=10) + + assert first.returncode == 0, f"stdout={first_out!r}, stderr={first_err!r}" + assert second.returncode == 0, f"stdout={second_out!r}, stderr={second_err!r}" + assert "first:READY" in first_out + assert "second:BOOTSTRAP" in second_out + assert "second:READY" in second_out + assert second_entered.exists() + state = read_storage_identity_state(MemoryRoot(tmp_path)) + assert state is not None + assert state.generation == 2 + assert state.state == "READY" + finally: + for process in (first, second): + if process is not None and process.poll() is None: + process.terminate() + process.communicate(timeout=5) + + +def test_fresh_provider_startup_is_serialized_across_processes( + tmp_path: Path, +) -> None: + """Real marker, table, and index bootstrap succeeds for concurrent APIs.""" + memory_path = tmp_path / "memory" + coordination_path = tmp_path / "coordination" + coordination_path.mkdir() + go_path = coordination_path / "go" + processes: list[subprocess.Popen[str]] = [] + + try: + for index in range(4): + ready_path = coordination_path / f"ready-{index}" + processes.append( + subprocess.Popen( + [ + sys.executable, + "-c", + _FRESH_PROVIDER_WORKER, + str(memory_path), + str(ready_path), + str(go_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + ) + + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if len(list(coordination_path.glob("ready-*"))) == len(processes): + break + assert all(process.poll() is None for process in processes) + time.sleep(0.02) + else: + raise AssertionError("provider workers did not reach the start barrier") + + go_path.write_text("go", encoding="utf-8") + for process in processes: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, ( + f"returncode={process.returncode}, stdout={stdout!r}, stderr={stderr!r}" + ) + + memory_root = MemoryRoot(memory_path) + state = read_storage_identity_state(memory_root) + assert state is not None + assert state.generation == 2 + assert state.state == "READY" + assert len(list(memory_root.lancedb_dir.glob("*.lance"))) == 7 + assert not list(memory_root.lancedb_dir.glob("..storage_identity.json.*")) + finally: + for process in processes: + if process.poll() is None: + process.terminate() + process.communicate(timeout=5) + + +async def test_external_legacy_ome_lock_blocks_rebuild_before_body( + tmp_path: Path, +) -> None: + root = MemoryRoot(tmp_path) + entered = False + + with ( + _external_lock(ome_lock_path(root), int(portalocker.LOCK_EX)), + pytest.raises(ProjectionLockUnavailableError, match="OME"), + ): + async with projection_rebuild_lock(root): + entered = True + + assert entered is False + async with projection_rebuild_lock(root): + pass diff --git a/tests/unit/test_infra/test_lancedb/test_storage_identity.py b/tests/unit/test_infra/test_lancedb/test_storage_identity.py new file mode 100644 index 00000000..cced39e5 --- /dev/null +++ b/tests/unit/test_infra/test_lancedb/test_storage_identity.py @@ -0,0 +1,117 @@ +"""Fail-closed storage-identity generation marker contracts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import STORAGE_ID_GENERATION +from everos.infra.persistence.lancedb.storage_identity import ( + StorageIdentityMigrationRequiredError, + ensure_storage_identity_ready, + mark_storage_identity_ready, + mark_storage_identity_rebuilding, + marker_path, + read_storage_identity_state, +) + + +def test_fresh_empty_root_initializes_current_ready_marker(tmp_path: Path) -> None: + root = MemoryRoot(tmp_path) + + ensure_storage_identity_ready(root) + + state = read_storage_identity_state(root) + assert state is not None + assert state.state == "READY" + assert state.generation == STORAGE_ID_GENERATION == 2 + + +def test_existing_source_without_marker_fails_closed(tmp_path: Path) -> None: + root = MemoryRoot(tmp_path) + source = tmp_path / "default_app/default_project/users/u1/episodes/day.md" + source.parent.mkdir(parents=True) + source.write_text("# existing memory\n", encoding="utf-8") + + with pytest.raises(StorageIdentityMigrationRequiredError, match="missing"): + ensure_storage_identity_ready(root) + + assert not marker_path(root).exists() + + +@pytest.mark.parametrize("managed_dir", [".index", ".tmp"]) +def test_markerless_root_with_markdown_in_managed_namespace_fails_closed( + tmp_path: Path, + managed_dir: str, +) -> None: + root = MemoryRoot(tmp_path) + source = tmp_path / managed_dir / "legacy-source.md" + source.parent.mkdir(parents=True) + source.write_text("# existing memory\n", encoding="utf-8") + + with pytest.raises(StorageIdentityMigrationRequiredError, match="missing"): + ensure_storage_identity_ready(root) + + assert not marker_path(root).exists() + + +def test_existing_lancedb_artifact_without_marker_fails_closed(tmp_path: Path) -> None: + root = MemoryRoot(tmp_path) + artifact = root.lancedb_dir / "episode.lance/data/legacy.lance" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"legacy") + + with pytest.raises(StorageIdentityMigrationRequiredError, match="missing"): + ensure_storage_identity_ready(root) + + +@pytest.mark.parametrize( + "payload", + [ + {"generation": True, "state": "READY"}, + {"generation": 2.0, "state": "READY"}, + {"generation": 2, "state": "ready"}, + {"generation": 2, "state": "READY", "extra": "ignored?"}, + ["READY", 2], + ], +) +def test_marker_json_types_and_shape_are_strict( + tmp_path: Path, payload: object +) -> None: + root = MemoryRoot(tmp_path) + path = marker_path(root) + path.parent.mkdir(parents=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(StorageIdentityMigrationRequiredError): + read_storage_identity_state(root) + + +def test_rebuilding_and_old_generation_both_block_startup(tmp_path: Path) -> None: + root = MemoryRoot(tmp_path) + mark_storage_identity_rebuilding(root) + + with pytest.raises(StorageIdentityMigrationRequiredError, match="REBUILDING"): + ensure_storage_identity_ready(root) + + marker_path(root).write_text( + json.dumps({"generation": 1, "state": "READY"}), encoding="utf-8" + ) + with pytest.raises(StorageIdentityMigrationRequiredError, match="required=2"): + ensure_storage_identity_ready(root) + + +def test_ready_replaces_rebuilding_atomically_at_same_path(tmp_path: Path) -> None: + root = MemoryRoot(tmp_path) + mark_storage_identity_rebuilding(root) + assert read_storage_identity_state(root).state == "REBUILDING" # type: ignore[union-attr] + + mark_storage_identity_ready(root) + + state = read_storage_identity_state(root) + assert state is not None + assert state.state == "READY" + assert list(root.lancedb_dir.glob(f".{marker_path(root).name}.*")) == [] diff --git a/tests/unit/test_infra/test_markdown/test_writers/test_daily_log_writers.py b/tests/unit/test_infra/test_markdown/test_writers/test_daily_log_writers.py index 20372fb3..0068340b 100644 --- a/tests/unit/test_infra/test_markdown/test_writers/test_daily_log_writers.py +++ b/tests/unit/test_infra/test_markdown/test_writers/test_daily_log_writers.py @@ -16,11 +16,13 @@ import pytest from everos.core.persistence import MarkdownReader, MemoryRoot +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.markdown import ( AgentCaseReader, AgentCaseWriter, AtomicFactReader, AtomicFactWriter, + EpisodeWriter, ForesightReader, ForesightWriter, ) @@ -33,6 +35,79 @@ def memory_root(tmp_path: Path) -> MemoryRoot: return mr +@pytest.mark.parametrize( + "app_id, project_id", + [ + ("app", "Project-A"), + ("DEFAULT_APP", "project"), + (".index", "project"), + (".tmp", "project"), + ], +) +async def test_episode_writer_rejects_nonportable_scope_before_writing( + memory_root: MemoryRoot, + app_id: str, + project_id: str, +) -> None: + writer = EpisodeWriter(memory_root) + + with pytest.raises(ValueError): + await writer.append_entry( + "u1", + inline={ + "owner_id": "u1", + "session_id": "s1", + "timestamp": "2026-05-15T10:00:00+00:00", + "parent_id": "mc_1", + "sender_ids": ["u1"], + }, + sections={"Content": "must not be written"}, + date=_dt.date(2026, 5, 15), + app_id=app_id, + project_id=project_id, + ) + + assert list(memory_root.root.rglob("*.md")) == [] + + +async def test_episode_writer_rejects_case_alias_without_changing_existing_scope( + memory_root: MemoryRoot, +) -> None: + writer = EpisodeWriter(memory_root) + kwargs = { + "inline": { + "owner_id": "u1", + "session_id": "s1", + "timestamp": "2026-05-15T10:00:00+00:00", + "parent_id": "mc_1", + "sender_ids": ["u1"], + }, + "date": _dt.date(2026, 5, 15), + "app_id": "app", + } + await writer.append_entry( + "u1", + sections={"Content": "preserved"}, + project_id="project-a", + **kwargs, + ) + + with pytest.raises(ValueError): + await writer.append_entry( + "u1", + sections={"Content": "must not overwrite"}, + project_id="Project-A", + **kwargs, + ) + + files = list(memory_root.root.rglob("episode-*.md")) + assert len(files) == 1 + parsed = await MarkdownReader.read(files[0]) + assert len(parsed.entries) == 1 + assert "preserved" in parsed.body + assert "must not overwrite" not in parsed.body + + # ── AtomicFact ──────────────────────────────────────────────────────────── @@ -256,7 +331,12 @@ async def embed_batch(self, ts): # type: ignore[no-untyped-def] row = await handler._build_row( owner_id="u1", owner_type="user", md_path=rel, entry=pe ) - assert row.id == f"u1_{eid.format()}" + assert row.id == daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u1", + entry_id=eid.format(), + ) assert row.fact == "Alice prefers Italian." assert row.parent_id == "mc_1" assert row.sender_ids == ["u1"] diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 9f79cc56..f61027ec 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -23,6 +23,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import AgentCase from everos.infra.persistence.markdown import AgentCaseWriter from everos.memory.cascade.handlers import HandlerDeps @@ -162,7 +163,12 @@ async def test_added_entry_builds_the_agent_track_row( row = fake_repo.upserts[0][0] assert row.owner_id == _AGENT assert row.owner_type == "agent" - assert row.id.startswith(f"{_AGENT}_") + assert row.id == daily_log_storage_id( + app_id="default", + project_id="default", + owner_id=_AGENT, + entry_id=row.entry_id, + ) assert row.session_id == "s1" assert row.parent_type == "memcell" assert row.parent_id == "mc_case_parent" diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index 8ed467d4..5c01ed88 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -18,6 +18,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import agent_skill_storage_id from everos.infra.persistence.lancedb import AgentSkill from everos.infra.persistence.markdown import AgentSkillWriter from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -152,7 +153,12 @@ async def test_handle_added_or_modified_upserts_typed_row( assert outcome.upserted == 1 assert outcome.deleted == 0 row = fake_repo.upserts[0][0] - assert row.id == "a1_contract_scan" + assert row.id == agent_skill_storage_id( + app_id="default", + project_id="default", + owner_id="a1", + name="contract_scan", + ) assert row.owner_id == "a1" assert row.owner_type == "agent" assert row.name == "contract_scan" @@ -201,7 +207,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( ) -> None: """User edits SKILL.md frontmatter.name; the LanceDB row id changes. - skill_id is derived from ``frontmatter.name`` (``_``). + skill_id is derived from the complete app/project/owner/name identity. When the user edits the name in place — common when refining a skill title without moving the file — the new id differs from the old, so a plain ``upsert([new_row])`` would leave the old row behind and a @@ -217,7 +223,13 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( ) ) await handler.handle_added_or_modified(md_path) - assert fake_repo.rows == {"a1_old_name": fake_repo.rows["a1_old_name"]} + old_id = agent_skill_storage_id( + app_id="default", + project_id="default", + owner_id="a1", + name="old_name", + ) + assert fake_repo.rows == {old_id: fake_repo.rows[old_id]} # Second pass: simulate the user editing frontmatter.name in place # (md_path unchanged, only the name field flips). @@ -230,10 +242,16 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert outcome.upserted == 1 assert outcome.deleted == 1 # Old id is gone, new id is present, exactly one row survives. - assert list(fake_repo.rows.keys()) == ["a1_new_name"] + new_id = agent_skill_storage_id( + app_id="default", + project_id="default", + owner_id="a1", + name="new_name", + ) + assert list(fake_repo.rows.keys()) == [new_id] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"md_path = '{md_path}' AND id != 'a1_new_name'" + f"md_path = '{md_path}' AND id != '{new_id}'" ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index a8af7fe1..3d9f7576 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -18,6 +18,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.infra.persistence.lancedb import Episode from everos.infra.persistence.markdown import EpisodeWriter from everos.memory.cascade.handlers import HandlerDeps @@ -182,7 +183,12 @@ async def test_added_entry_upserts_typed_row( assert row.subject == "Test" assert row.md_path == rel assert row.entry_id.startswith("ep_") - assert row.id == f"u1_{row.entry_id}" + assert row.id == daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u1", + entry_id=row.entry_id, + ) assert len(row.vector) == 1024 assert row.subject_vector is not None assert len(row.subject_vector) == 1024 diff --git a/tests/unit/test_memory/test_cascade/test_handler_user_profile.py b/tests/unit/test_memory/test_cascade/test_handler_user_profile.py index 6805afe6..85d77bd6 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_user_profile.py +++ b/tests/unit/test_memory/test_cascade/test_handler_user_profile.py @@ -17,6 +17,7 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot +from everos.core.persistence.lancedb.row_id import user_profile_storage_id from everos.infra.persistence.lancedb import UserProfile from everos.infra.persistence.markdown import ProfileWriter, UserProfileFrontmatter from everos.memory.cascade.handlers import HandlerDeps, UserProfileHandler @@ -113,7 +114,11 @@ async def test_first_pass_upserts_typed_row( assert outcome.upserted == 1 assert outcome.skipped == 0 row = fake_repo.upserts[0][0] - assert row.id == "u_alice" + assert row.id == user_profile_storage_id( + app_id="default", + project_id="default", + owner_id="u_alice", + ) assert row.owner_id == "u_alice" assert row.owner_type == "user" assert row.summary.startswith("Alice") @@ -234,9 +239,14 @@ async def test_handle_deleted_drops_row( ) handler = _handler(memory_root) await handler.handle_added_or_modified(md_path) - assert "u_alice" in fake_repo.rows + row_id = user_profile_storage_id( + app_id="default", + project_id="default", + owner_id="u_alice", + ) + assert row_id in fake_repo.rows outcome = await handler.handle_deleted(md_path) assert outcome.deleted == 1 assert fake_repo.deletes == [md_path] - assert "u_alice" not in fake_repo.rows + assert row_id not in fake_repo.rows diff --git a/tests/unit/test_memory/test_cascade/test_handlers_daily_log_mapping.py b/tests/unit/test_memory/test_cascade/test_handlers_daily_log_mapping.py index b3fcb4a4..0690c6bb 100644 --- a/tests/unit/test_memory/test_cascade/test_handlers_daily_log_mapping.py +++ b/tests/unit/test_memory/test_cascade/test_handlers_daily_log_mapping.py @@ -20,6 +20,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot, StructuredEntry +from everos.core.persistence.lancedb.row_id import daily_log_storage_id from everos.memory.cascade.handlers import ( AgentCaseHandler, AtomicFactHandler, @@ -111,7 +112,12 @@ async def test_atomic_fact_build_row_maps_inline_and_section(tmp_path) -> None: sections={"Fact": "the user prefers dark mode"}, ), ) - assert row.id == "u1_af_20260514_0001" + assert row.id == daily_log_storage_id( + app_id="default", + project_id="default", + owner_id="u1", + entry_id="af_20260514_0001", + ) assert row.fact == "the user prefers dark mode" assert row.fact_tokens == "the user prefers dark mode" assert row.parent_id == "mc_1" diff --git a/tests/unit/test_memory/test_get/test_dto.py b/tests/unit/test_memory/test_get/test_dto.py index e9b81cf9..92f24b53 100644 --- a/tests/unit/test_memory/test_get/test_dto.py +++ b/tests/unit/test_memory/test_get/test_dto.py @@ -136,6 +136,38 @@ def test_get_request_rejects_invalid_sort_order() -> None: ) +@pytest.mark.parametrize( + "field, value", + [("app_id", "default_app"), ("project_id", "default_project")], +) +def test_get_request_rejects_reserved_scope_aliases(field: str, value: str) -> None: + with pytest.raises(ValidationError): + GetRequest( + user_id="u1", + memory_type=GetMemoryType.EPISODE, + **{field: value}, + ) + + +@pytest.mark.parametrize( + "field, value", + [ + ("project_id", "Project-A"), + ("app_id", "DEFAULT_APP"), + ("app_id", ".index"), + ("app_id", "x" * 129), + ("project_id", "safe\n"), + ], +) +def test_get_request_rejects_nonportable_scopes(field: str, value: str) -> None: + with pytest.raises(ValidationError): + GetRequest( + user_id="u1", + memory_type=GetMemoryType.EPISODE, + **{field: value}, + ) + + # ── owner_type × memory_type pairing ───────────────────────────────────── diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 35cec6b7..378333c8 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -21,6 +21,11 @@ import pytest +from everos.core.persistence.lancedb.row_id import ( + agent_skill_storage_id, + daily_log_storage_id, + user_profile_storage_id, +) from everos.infra.persistence.lancedb import ( AgentCase, AgentSkill, @@ -95,7 +100,9 @@ def _ts(day: int = 1) -> _dt.datetime: def _episode_row(entry: str) -> Episode: return Episode( - id=f"u1_{entry}", + id=daily_log_storage_id( + app_id="default", project_id="default", owner_id="u1", entry_id=entry + ), entry_id=entry, owner_id="u1", owner_type="user", @@ -116,7 +123,9 @@ def _episode_row(entry: str) -> Episode: def _agent_case_row(entry: str) -> AgentCase: return AgentCase( - id=f"a1_{entry}", + id=daily_log_storage_id( + app_id="default", project_id="default", owner_id="a1", entry_id=entry + ), entry_id=entry, owner_id="a1", owner_type="agent", @@ -138,7 +147,9 @@ def _agent_case_row(entry: str) -> AgentCase: def _agent_skill_row(name: str) -> AgentSkill: return AgentSkill( - id=f"a1_{name}", + id=agent_skill_storage_id( + app_id="default", project_id="default", owner_id="a1", name=name + ), owner_id="a1", owner_type="agent", name=name, @@ -157,7 +168,9 @@ def _agent_skill_row(name: str) -> AgentSkill: def _user_profile_row(owner: str = "u1") -> UserProfile: return UserProfile( - id=owner, + id=user_profile_storage_id( + app_id="default", project_id="default", owner_id=owner + ), owner_id=owner, owner_type="user", app_id="default", @@ -301,8 +314,10 @@ async def test_profile_hit_shapes_row_into_item( item = resp.data.profiles[0] assert item.id == "u1" assert item.user_id == "u1" - # KV fetch keys on owner_id. - assert profile_repo.last_id == "u1" + # KV fetch keys on the complete app/project/owner partition. + assert profile_repo.last_id == user_profile_storage_id( + app_id="default", project_id="default", owner_id="u1" + ) # json buckets are decoded back into structured profile_data. assert item.profile_data["summary"] == "u1 loves climbing in Yosemite" assert item.profile_data["explicit_info"] == [ diff --git a/tests/unit/test_memory/test_search/test_agentic.py b/tests/unit/test_memory/test_search/test_agentic.py index 9ab4896d..21c2dc7b 100644 --- a/tests/unit/test_memory/test_search/test_agentic.py +++ b/tests/unit/test_memory/test_search/test_agentic.py @@ -41,6 +41,7 @@ def _mc_candidate(mc_id: str, ep_id: str, score: float = 0.8) -> Candidate: source="vector", metadata={ "episode_id": ep_id, + "entry_id": ep_id, "owner_id": "alice", "owner_type": "user", "session_id": "sess_a", @@ -324,9 +325,9 @@ async def fake_embed(q: str) -> list[float]: assert len(result) == 1 assert isinstance(result[0], SearchEpisodeItem) - assert result[0].id == "ep_1", ( - f"Expected episode_id='ep_1' but got {result[0].id!r}. " - "Shaper must remap from memcell_id via metadata['episode_id']." + assert result[0].id == "alice_ep_1", ( + f"Expected wire id='alice_ep_1' but got {result[0].id!r}. " + "Shaper must reconstruct it from owner_id and entry_id." ) diff --git a/tests/unit/test_memory/test_search/test_agentic_agent.py b/tests/unit/test_memory/test_search/test_agentic_agent.py index 0fa6be0f..b2e876a8 100644 --- a/tests/unit/test_memory/test_search/test_agentic_agent.py +++ b/tests/unit/test_memory/test_search/test_agentic_agent.py @@ -53,6 +53,7 @@ def _case_candidate(cid: str, score: float = 0.8) -> Candidate: metadata={ "owner_id": "agent_a", "owner_type": "agent", + "entry_id": cid, "session_id": "sess_b", "timestamp": _ts(), "task_intent": f"intent {cid}", @@ -257,7 +258,7 @@ async def fake_aagentic( assert len(result) == 1 assert isinstance(result[0], SearchAgentCaseItem) - assert result[0].id == "c_1" + assert result[0].id == "agent_a_c_1" assert result[0].task_intent == "intent c_1" @@ -283,7 +284,7 @@ async def fake_aagentic( assert len(result) == 1 assert isinstance(result[0], SearchAgentSkillItem) - assert result[0].id == "s_1" + assert result[0].id == "agent_a_skill_s_1" assert result[0].name == "skill_s_1" diff --git a/tests/unit/test_memory/test_search/test_dto.py b/tests/unit/test_memory/test_search/test_dto.py index f1a5a567..30098f51 100644 --- a/tests/unit/test_memory/test_search/test_dto.py +++ b/tests/unit/test_memory/test_search/test_dto.py @@ -90,6 +90,30 @@ def test_min_score_in_range_accepted() -> None: assert req.min_score == 0.4 +@pytest.mark.parametrize( + "field, value", + [("app_id", "default_app"), ("project_id", "default_project")], +) +def test_search_request_rejects_reserved_scope_aliases(field: str, value: str) -> None: + with pytest.raises(ValidationError): + SearchRequest(**_minimal_request_kwargs(), **{field: value}) + + +@pytest.mark.parametrize( + "field, value", + [ + ("project_id", "Project-A"), + ("app_id", "DEFAULT_APP"), + ("app_id", ".index"), + ("app_id", "x" * 129), + ("project_id", "safe\n"), + ], +) +def test_search_request_rejects_nonportable_scopes(field: str, value: str) -> None: + with pytest.raises(ValidationError): + SearchRequest(**_minimal_request_kwargs(), **{field: value}) + + def test_neither_user_id_nor_agent_id_rejected() -> None: """The xor validator requires exactly one of user_id / agent_id.""" with pytest.raises(ValidationError, match="exactly one of"): diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 73dd6661..f31a53e5 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -68,6 +68,7 @@ def _case_row(cid: str) -> Candidate: metadata={ "owner_id": "agent_a", "owner_type": "agent", + "entry_id": cid, "session_id": "sess_b", "timestamp": _ts(), "task_intent": f"intent {cid}", @@ -195,6 +196,7 @@ def __init__( # Bridge recall fixture: reverse-resolved skills (``fetch_by_case_ids``). # Default empty — only the bridge tests populate this. self._by_case = by_case or [] + self.last_case_ids: list[str] | None = None async def sparse_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._sparse) @@ -205,11 +207,12 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def fetch_by_case_ids( self, case_ids: Sequence[str], where: str, *, limit: int ) -> list[Candidate]: + self.last_case_ids = list(case_ids) return list(self._by_case) class _StubProfileRecaller: - async def fetch(self, owner_id: str) -> list: + async def fetch(self, owner_id: str, *, app_id: str, project_id: str) -> list: return [] @@ -296,7 +299,7 @@ async def test_user_keyword_returns_episodes_only() -> None: c in "0123456789abcdef" for c in resp.request_id ) assert len(resp.data.episodes) == 1 - assert resp.data.episodes[0].id == "ep_1" + assert resp.data.episodes[0].id == "alice_ep_1" assert resp.data.episodes[0].user_id == "alice" assert resp.data.episodes[0].type == "Conversation" # Agent paths stay empty. @@ -469,7 +472,7 @@ async def test_vector_method_returns_episodes_via_maxsim() -> None: embedding=_StubEmbedding(), ) resp = await mgr.search(_user_req(method=SearchMethod.VECTOR)) - assert [e.id for e in resp.data.episodes] == ["ep_dense"] + assert [e.id for e in resp.data.episodes] == ["alice_ep_dense"] async def test_vector_radius_filter_drops_below_threshold() -> None: @@ -485,7 +488,7 @@ async def test_vector_radius_filter_drops_below_threshold() -> None: embedding=_StubEmbedding(), ) resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, radius=0.5)) - assert [e.id for e in resp.data.episodes] == ["ep_high"] + assert [e.id for e in resp.data.episodes] == ["alice_ep_high"] async def test_unlimited_mode_applies_default_radius_for_vector() -> None: @@ -509,7 +512,7 @@ async def test_unlimited_mode_applies_default_radius_for_vector() -> None: ) resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1)) # Ordered by max-pooled fact score descending. - assert [e.id for e in resp.data.episodes] == ["ep_high", "ep_mid"] + assert [e.id for e in resp.data.episodes] == ["alice_ep_high", "alice_ep_mid"] async def test_unlimited_mode_explicit_radius_overrides_default() -> None: @@ -527,7 +530,7 @@ async def test_unlimited_mode_explicit_radius_overrides_default() -> None: ) resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=-1, radius=0.1)) # 0.1 threshold keeps both rows (the default 0.5 would have dropped ep_low). - assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"} + assert {e.id for e in resp.data.episodes} == {"alice_ep_low", "alice_ep_high"} async def test_normal_mode_keeps_full_pool_when_no_radius() -> None: @@ -545,7 +548,7 @@ async def test_normal_mode_keeps_full_pool_when_no_radius() -> None: ) resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=10)) # No radius default in normal mode -> both kept. - assert {e.id for e in resp.data.episodes} == {"ep_low", "ep_high"} + assert {e.id for e in resp.data.episodes} == {"alice_ep_low", "alice_ep_high"} async def test_vector_maxsim_max_pools_facts_to_episodes() -> None: @@ -566,7 +569,7 @@ async def test_vector_maxsim_max_pools_facts_to_episodes() -> None: resp = await mgr.search(_user_req(method=SearchMethod.VECTOR, top_k=5)) eps = resp.data.episodes # Both episodes returned, ordered by max-pool score desc. - assert [e.id for e in eps] == ["ep_A", "ep_B"] + assert [e.id for e in eps] == ["alice_ep_A", "alice_ep_B"] assert eps[0].score == pytest.approx(0.95) # max(0.95, 0.40) assert eps[1].score == pytest.approx(0.75) @@ -630,7 +633,11 @@ async def test_user_hybrid_episode_fuses_and_evicts_facts() -> None: id="f1", parent_episode_id="ep_1", score=0.95, - metadata={"fact": "Alice prefers oat milk"}, + metadata={ + "owner_id": "alice", + "entry_id": "af_1", + "fact": "Alice prefers oat milk", + }, ) mgr = _build_manager( episode_sparse=[ep1, ep2], @@ -641,10 +648,10 @@ async def test_user_hybrid_episode_fuses_and_evicts_facts() -> None: resp = await mgr.search(_user_req(method=SearchMethod.HYBRID, top_k=10)) eps = resp.data.episodes assert len(eps) >= 1 - ep1_result = next((e for e in eps if e.id == "ep_1"), None) + ep1_result = next((e for e in eps if e.id == "alice_ep_1"), None) assert ep1_result is not None assert len(ep1_result.atomic_facts) == 1 - assert ep1_result.atomic_facts[0].id == "f1" + assert ep1_result.atomic_facts[0].id == "alice_af_1" async def test_agentic_requires_reranker_and_llm( @@ -759,8 +766,8 @@ async def test_agent_keyword_returns_cases_and_skills_only() -> None: resp = await mgr.search(_agent_req()) assert resp.data.episodes == [] assert resp.data.profiles == [] - assert [c.id for c in resp.data.agent_cases] == ["c_1"] - assert [s.id for s in resp.data.agent_skills] == ["s_1"] + assert [c.id for c in resp.data.agent_cases] == ["agent_a_c_1"] + assert [s.id for s in resp.data.agent_skills] == ["agent_a_skill_s_1"] async def test_agent_owner_ignores_include_profile() -> None: @@ -777,7 +784,11 @@ async def test_top_k_truncates_results() -> None: rows = [_episode_row(f"ep_{i}", score=1.0 - i * 0.01) for i in range(10)] mgr = _build_manager(episode_sparse=rows) resp = await mgr.search(_user_req(top_k=3)) - assert [e.id for e in resp.data.episodes] == ["ep_0", "ep_1", "ep_2"] + assert [e.id for e in resp.data.episodes] == [ + "alice_ep_0", + "alice_ep_1", + "alice_ep_2", + ] async def test_top_k_minus_one_caps_at_100() -> None: @@ -911,9 +922,24 @@ async def test_case_bridged_skills_max_pools_score_across_source_cases() -> None ) mgr = _build_manager(skill_by_case=[skill_row]) bridge_cases = [ - Candidate(id="c1", score=0.4, source="vector", metadata={}), - Candidate(id="c2", score=0.9, source="vector", metadata={}), # max wins - Candidate(id="c_other", score=0.7, source="vector", metadata={}), + Candidate( + id="storage:c1", + score=0.4, + source="vector", + metadata={"entry_id": "c1"}, + ), + Candidate( + id="storage:c2", + score=0.9, + source="vector", + metadata={"entry_id": "c2"}, + ), # max wins + Candidate( + id="storage:other", + score=0.7, + source="vector", + metadata={"entry_id": "c_other"}, + ), ] bridged = await mgr._case_bridged_skills(bridge_cases, where="", top_k=5) assert len(bridged) == 1 @@ -923,6 +949,7 @@ async def test_case_bridged_skills_max_pools_score_across_source_cases() -> None # Metadata (incl. ``source_case_ids``) rides through so downstream # shaping doesn't need a second fetch. assert bridged[0].metadata["source_case_ids"] == ["c1", "c2", "c3"] + assert mgr._skill.last_case_ids == ["c1", "c2", "c_other"] async def test_case_bridged_skills_returns_empty_for_none_or_empty_input() -> None: @@ -1014,7 +1041,7 @@ async def test_agent_hybrid_llm_rerank_merges_bridged_skills_into_dense_pool( from everalgo.types import RankOutput, ScoredItem case_result = ScoredItem( - id="agent_a_c1", + id="storage:case:c1", score=0.85, item_type="case", # Shaper requires owner_type="agent" + timestamp + intent/approach; @@ -1022,6 +1049,7 @@ async def test_agent_hybrid_llm_rerank_merges_bridged_skills_into_dense_pool( metadata={ "owner_id": "agent_a", "owner_type": "agent", + "entry_id": "c1", "session_id": "sess_b", "timestamp": _ts(), "task_intent": "intent c1", @@ -1034,7 +1062,7 @@ async def test_agent_hybrid_llm_rerank_merges_bridged_skills_into_dense_pool( id="s_bridged", score=0.0, source="vector", - metadata={"source_case_ids": ["agent_a_c1"], "name": "s_bridged"}, + metadata={"source_case_ids": ["c1"], "name": "s_bridged"}, ) seen_skill_dense: dict[str, list[Candidate]] = {} @@ -1280,7 +1308,7 @@ async def test_search_captures_returned_hits_when_content_on( attrs = _span_index(_search_spans)["everos.memory.search"].attributes out = json.loads(attrs["langfuse.observation.output"]) - assert out["episodes"] == ["ep_1"] + assert out["episodes"] == ["alice_ep_1"] assert out["agent_cases"] == [] and out["agent_skills"] == [] diff --git a/tests/unit/test_memory/test_search/test_recall_profile.py b/tests/unit/test_memory/test_search/test_recall_profile.py index 0563d003..bea04dfa 100644 --- a/tests/unit/test_memory/test_search/test_recall_profile.py +++ b/tests/unit/test_memory/test_search/test_recall_profile.py @@ -13,6 +13,7 @@ import pytest +from everos.core.persistence.lancedb.row_id import user_profile_storage_id from everos.infra.persistence.lancedb import ( UserProfile, lancedb_manager, @@ -24,15 +25,21 @@ def _profile_row( *, owner_id: str, + app_id: str = "default", + project_id: str = "default", summary: str = "summary text", explicit_info: list | None = None, implicit_traits: list | None = None, profile_timestamp_ms: int = 1_700_000_000_000, ) -> UserProfile: return UserProfile( - id=owner_id, + id=user_profile_storage_id( + app_id=app_id, project_id=project_id, owner_id=owner_id + ), owner_id=owner_id, owner_type="user", + app_id=app_id, + project_id=project_id, summary=summary, explicit_info_json=json.dumps(explicit_info or [], ensure_ascii=False), implicit_traits_json=json.dumps(implicit_traits or [], ensure_ascii=False), @@ -64,7 +71,9 @@ async def test_fetch_returns_dto_when_row_exists() -> None: ] ) - items = await ProfileRecaller().fetch("u_alice") + items = await ProfileRecaller().fetch( + "u_alice", app_id="default", project_id="default" + ) assert len(items) == 1 item = items[0] assert item.id == "u_alice" @@ -78,7 +87,9 @@ async def test_fetch_returns_dto_when_row_exists() -> None: async def test_fetch_returns_empty_when_row_missing() -> None: - items = await ProfileRecaller().fetch("u_cold_start") + items = await ProfileRecaller().fetch( + "u_cold_start", app_id="default", project_id="default" + ) assert items == [] @@ -86,7 +97,7 @@ async def test_fetch_returns_empty_for_blank_owner() -> None: """Blank ``owner_id`` short-circuits — never hit LanceDB with an empty-string PK (which would otherwise return any row whose id was persisted as the empty string).""" - items = await ProfileRecaller().fetch("") + items = await ProfileRecaller().fetch("", app_id="default", project_id="default") assert items == [] @@ -97,7 +108,9 @@ async def test_fetch_isolates_by_owner() -> None: _profile_row(owner_id="u_bob", summary="Bob"), ] ) - bob_items = await ProfileRecaller().fetch("u_bob") + bob_items = await ProfileRecaller().fetch( + "u_bob", app_id="default", project_id="default" + ) assert len(bob_items) == 1 assert bob_items[0].profile_data["summary"] == "Bob" @@ -108,7 +121,9 @@ async def test_fetch_tolerates_malformed_json_columns() -> None: await user_profile_repo.upsert( [ UserProfile( - id="u_broken", + id=user_profile_storage_id( + app_id="default", project_id="default", owner_id="u_broken" + ), owner_id="u_broken", owner_type="user", summary="ok", @@ -121,8 +136,38 @@ async def test_fetch_tolerates_malformed_json_columns() -> None: ] ) - items = await ProfileRecaller().fetch("u_broken") + items = await ProfileRecaller().fetch( + "u_broken", app_id="default", project_id="default" + ) assert len(items) == 1 assert items[0].profile_data["explicit_info"] == [] assert items[0].profile_data["implicit_traits"] == [] assert items[0].profile_data["summary"] == "ok" + + +async def test_fetch_isolates_same_owner_across_projects() -> None: + await user_profile_repo.upsert( + [ + _profile_row( + owner_id="u_same", + app_id="app_a", + project_id="project_a", + summary="Project A", + ), + _profile_row( + owner_id="u_same", + app_id="app_a", + project_id="project_b", + summary="Project B", + ), + ] + ) + + items = await ProfileRecaller().fetch( + "u_same", app_id="app_a", project_id="project_b" + ) + + assert len(items) == 1 + assert items[0].id == "u_same" + assert items[0].project_id == "project_b" + assert items[0].profile_data["summary"] == "Project B" diff --git a/tests/unit/test_memory/test_search/test_shaper.py b/tests/unit/test_memory/test_search/test_shaper.py index 77c4bd3b..6de3f447 100644 --- a/tests/unit/test_memory/test_search/test_shaper.py +++ b/tests/unit/test_memory/test_search/test_shaper.py @@ -24,7 +24,12 @@ def _ts(year: int = 2026) -> _dt.datetime: return _dt.datetime(year, 1, 1, tzinfo=_dt.UTC) -def _episode_candidate(*, id: str = "alice_ep_1", score: float = 0.9) -> Candidate: +def _episode_candidate( + *, + id: str = "storage:episode:1", + entry_id: str = "ep_1", + score: float = 0.9, +) -> Candidate: return Candidate( id=id, score=score, @@ -32,6 +37,7 @@ def _episode_candidate(*, id: str = "alice_ep_1", score: float = 0.9) -> Candida metadata={ "owner_id": "alice", "owner_type": "user", + "entry_id": entry_id, "session_id": "sess_a", "timestamp": _ts(), "sender_ids": ["alice", "assistant_1"], @@ -44,12 +50,13 @@ def _episode_candidate(*, id: str = "alice_ep_1", score: float = 0.9) -> Candida def _agent_case_candidate() -> Candidate: return Candidate( - id="agent_a_case_1", + id="storage:case:1", score=0.8, source="keyword", metadata={ "owner_id": "agent_a", "owner_type": "agent", + "entry_id": "case_1", "session_id": "sess_a", "timestamp": _ts(), "task_intent": "Draft a follow-up email", @@ -62,7 +69,7 @@ def _agent_case_candidate() -> Candidate: def _agent_skill_candidate() -> Candidate: return Candidate( - id="agent_a_skill_1", + id="storage:skill:1", score=0.7, source="keyword", metadata={ @@ -105,17 +112,20 @@ def test_shape_episode_drops_when_timestamp_missing() -> None: def test_shape_episode_attaches_facts() -> None: - facts = [ - shape_atomic_fact_from_candidate( - Candidate( - id="f1", - score=0.5, - source="other", - metadata={"fact": "Alice prefers oat milk"}, - ) + fact = shape_atomic_fact_from_candidate( + Candidate( + id="storage:fact:1", + score=0.5, + source="other", + metadata={ + "owner_id": "alice", + "entry_id": "af_1", + "fact": "Alice prefers oat milk", + }, ) - ] - item = shape_episode_from_candidate(_episode_candidate(), atomic_facts=facts) + ) + assert fact is not None + item = shape_episode_from_candidate(_episode_candidate(), atomic_facts=[fact]) assert item is not None assert len(item.atomic_facts) == 1 assert item.atomic_facts[0].content == "Alice prefers oat milk" @@ -127,6 +137,7 @@ def test_shape_episode_attaches_facts() -> None: def test_shape_agent_case_basic() -> None: item = shape_agent_case_from_candidate(_agent_case_candidate()) assert item is not None + assert item.id == "agent_a_case_1" assert item.agent_id == "agent_a" assert item.task_intent == "Draft a follow-up email" assert item.quality_score == 0.92 @@ -142,6 +153,7 @@ def test_shape_agent_case_drops_when_owner_type_wrong() -> None: def test_shape_agent_skill_basic() -> None: item = shape_agent_skill_from_candidate(_agent_skill_candidate()) assert item is not None + assert item.id == "agent_a_contract_redline" assert item.name == "contract_redline" assert item.maturity_score == 0.5 assert item.source_case_ids == ["agent_a_case_1"] @@ -158,6 +170,7 @@ def _scored_episode(eid: str, score: float) -> ScoredItem: metadata={ "owner_id": "alice", "owner_type": "user", + "entry_id": eid, "session_id": "s1", "timestamp": _ts(), "sender_ids": ["alice"], @@ -174,7 +187,11 @@ def _scored_fact(fid: str, parent: str, score: float) -> ScoredItem: score=score, item_type="atomic_fact", parent_episode_id=parent, - metadata={"fact": f"fact text {fid}"}, + metadata={ + "owner_id": "alice", + "entry_id": fid, + "fact": f"fact text {fid}", + }, ) @@ -186,9 +203,9 @@ def test_reshape_hybrid_nests_facts_under_kept_episode() -> None: ] out = reshape_hybrid_output(scored, episode_pool={}) assert len(out) == 1 - assert out[0].id == "ep_1" + assert out[0].id == "alice_ep_1" # Facts sorted descending by score. - assert [f.id for f in out[0].atomic_facts] == ["f_1", "f_2"] + assert [f.id for f in out[0].atomic_facts] == ["alice_f_1", "alice_f_2"] def test_reshape_hybrid_backfills_evicted_episode_from_pool() -> None: @@ -198,14 +215,14 @@ def test_reshape_hybrid_backfills_evicted_episode_from_pool() -> None: _scored_episode("ep_1", 0.7), _scored_fact("f_a", "ep_2", 0.95), ] - pool_episode = _episode_candidate(id="ep_2", score=0.0) + pool_episode = _episode_candidate(id="ep_2", entry_id="ep_2", score=0.0) out = reshape_hybrid_output(scored, episode_pool={"ep_2": pool_episode}) assert len(out) == 2 # Output sorted by score descending — ep_2 takes fact's max score (0.95). - assert out[0].id == "ep_2" + assert out[0].id == "alice_ep_2" assert out[0].score == 0.95 assert len(out[0].atomic_facts) == 1 - assert out[1].id == "ep_1" + assert out[1].id == "alice_ep_1" def test_reshape_hybrid_drops_orphan_facts_with_no_pool_parent() -> None: diff --git a/tests/unit/test_memory/test_search/test_skill_hybrid.py b/tests/unit/test_memory/test_search/test_skill_hybrid.py index b07c7014..3944436d 100644 --- a/tests/unit/test_memory/test_search/test_skill_hybrid.py +++ b/tests/unit/test_memory/test_search/test_skill_hybrid.py @@ -95,8 +95,8 @@ async def test_returns_shaped_items_up_to_top_k(self) -> None: assert len(result) == 2 assert all(isinstance(item, SearchAgentSkillItem) for item in result) - assert result[0].id == "s1" - assert result[1].id == "s2" + assert result[0].id == "agent_a_skill_s1" + assert result[1].id == "agent_a_skill_s2" async def test_reranker_receives_skill_instruction_and_shaped_passages( self, diff --git a/tests/unit/test_service/test_knowledge_crud.py b/tests/unit/test_service/test_knowledge_crud.py index e808a5ff..8a5cc750 100644 --- a/tests/unit/test_service/test_knowledge_crud.py +++ b/tests/unit/test_service/test_knowledge_crud.py @@ -343,3 +343,27 @@ async def test_patch_document_not_found_raises() -> None: with pytest.raises(DocumentNotFoundError): await patch_document("d_missing", "app1", "proj1", title="New") + + +@pytest.mark.parametrize( + "operation", + [ + lambda: get_document("d_test", "DEFAULT_APP", "project"), + lambda: list_documents("app", "Project-A"), + lambda: get_topic("n_test", ".index", "project"), + lambda: delete_document("d_test", "app", "default_project"), + lambda: patch_document("d_test", "app", "project."), + ], +) +async def test_exported_knowledge_crud_rejects_invalid_scope_before_repository( + operation, +) -> None: # type: ignore[no-untyped-def] + with ( + patch(f"{_MOD}.knowledge_document_repo") as mock_doc_repo, + patch(f"{_MOD}.knowledge_topic_sqlite_repo") as mock_topic_repo, + pytest.raises(ValueError), + ): + await operation() + + assert mock_doc_repo.mock_calls == [] + assert mock_topic_repo.mock_calls == [] diff --git a/tests/unit/test_service/test_memorize_span.py b/tests/unit/test_service/test_memorize_span.py index 1e7fdccb..900386c7 100644 --- a/tests/unit/test_service/test_memorize_span.py +++ b/tests/unit/test_service/test_memorize_span.py @@ -65,3 +65,27 @@ async def test_flush_emits_memory_flush_span(_patch: InMemorySpanExporter) -> No force_flush() names = {s.name for s in _patch.get_finished_spans()} assert "everos.memory.flush" in names + + +@pytest.mark.parametrize( + "field,value", + [ + ("app_id", "DEFAULT_APP"), + ("app_id", ".index"), + ("project_id", "Project-A"), + ("project_id", "default_project"), + ], +) +async def test_direct_memorize_rejects_invalid_scope_before_side_effects( + _patch: InMemorySpanExporter, + field: str, + value: str, +) -> None: + payload = {"session_id": "s1", "messages": [], field: value} + + with pytest.raises(ValueError): + await mm.memorize(payload) + + mm._memorize_locked.assert_not_awaited() + force_flush() + assert _patch.get_finished_spans() == ()