diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d54c7d64..7d839615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,12 +107,12 @@ jobs: - run: cargo build --workspace - run: cargo test --workspace --lib --no-fail-fast - name: Persist and recover authored memory through the CLI - run: cargo test --locked -p terraphim_agent --test memory_capture_cli_tests --test memory_apply_cli_tests --test memory_rubric_cli_tests + run: cargo test --locked -p terraphim_agent --test memory_capture_cli_tests --test memory_apply_cli_tests --test memory_rubric_cli_tests --test agent_store_contract --test agent_learning_store_contract # Exercise the released grep command against real files and exported # synonym graphs, including Linux-only hostile filesystem names. - run: cargo test --locked -p terraphim_grep --test synonym_retrieval --test answer_request --test path_guard_hostile_tree --test single_file_search - name: Verify authored synonyms and role boundaries over real MCP stdio - run: cargo test --locked -p terraphim_mcp_server --test test_mcp_role_contract + run: cargo test --locked -p terraphim_mcp_server --test test_mcp_role_contract --test test_mcp_agent_contract # GitHub-side home for the updater integration contracts (managed-mode # receipt/refusal, strict v2 manifest readers, R2 checksum fail-closed). # On Gitea these lanes live in .gitea/workflows/native-ci.yml, which is diff --git a/.github/workflows/pkgbuild-contract.yml b/.github/workflows/pkgbuild-contract.yml index bd4ba4f3..b46a3cb8 100644 --- a/.github/workflows/pkgbuild-contract.yml +++ b/.github/workflows/pkgbuild-contract.yml @@ -11,11 +11,15 @@ on: - "pkgbuilds/terraphim-clients-bin/**" - "tests/test_pkgbuild_release_contract.py" - ".github/workflows/pkgbuild-contract.yml" + - "scripts/prepare-pkgbuild-stage.py" + - "tests/test_pkgbuild_stage.py" pull_request: paths: - "pkgbuilds/terraphim-clients-bin/**" - "tests/test_pkgbuild_release_contract.py" - ".github/workflows/pkgbuild-contract.yml" + - "scripts/prepare-pkgbuild-stage.py" + - "tests/test_pkgbuild_stage.py" permissions: contents: read @@ -32,6 +36,9 @@ jobs: arch-package: name: arch build + install runs-on: ubuntu-latest + permissions: + contents: read + actions: read container: image: archlinux:base-devel options: --privileged @@ -39,5 +46,80 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install namcap run: pacman -Sy --noconfirm --needed namcap + # Before public promotion the exact signed archives exist only in the + # sealed stage. Once all public archives exist, use makepkg's normal URL + # download path; do not depend on retained Actions artefacts indefinitely. + - name: Resolve reviewed stage identity + id: stage + run: python3 scripts/prepare-pkgbuild-stage.py pkgbuilds/terraphim-clients-bin/release-stage.json identity >> "$GITHUB_OUTPUT" + - name: Check whether the exact public release is complete + id: source + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.stage.outputs.version }} + run: | + set -euo pipefail + status="$(curl --silent --show-error -H "Authorization: Bearer $GH_TOKEN" \ + -o "$RUNNER_TEMP/release.json" -w '%{http_code}' \ + "https://api.github.com/repos/terraphim/terraphim-clients/releases/tags/v$VERSION")" + python3 scripts/prepare-pkgbuild-stage.py pkgbuilds/terraphim-clients-bin/release-stage.json \ + mode "$status" "$RUNNER_TEMP/release.json" >> "$GITHUB_OUTPUT" + - name: Read exact producer and artefact metadata + if: steps.source.outputs.mode == 'stage' + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.stage.outputs.run_id }} + ARTIFACT_ID: ${{ steps.stage.outputs.artifact_id }} + run: | + set -euo pipefail + curl --fail --silent --show-error -H "Authorization: Bearer $GH_TOKEN" \ + "https://api.github.com/repos/terraphim/terraphim-clients/actions/runs/$RUN_ID" > "$RUNNER_TEMP/stage-run.json" + curl --fail --silent --show-error -H "Authorization: Bearer $GH_TOKEN" \ + "https://api.github.com/repos/terraphim/terraphim-clients/actions/artifacts/$ARTIFACT_ID" > "$RUNNER_TEMP/stage-artifact.json" + python3 scripts/prepare-pkgbuild-stage.py pkgbuilds/terraphim-clients-bin/release-stage.json \ + metadata "$RUNNER_TEMP/stage-run.json" "$RUNNER_TEMP/stage-artifact.json" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: steps.source.outputs.mode == 'stage' + with: + artifact-ids: ${{ steps.stage.outputs.artifact_id }} + merge-multiple: true + path: ${{ runner.temp }}/${{ steps.stage.outputs.artifact_name }} + repository: terraphim/terraphim-clients + run-id: ${{ steps.stage.outputs.run_id }} + github-token: ${{ github.token }} + - name: Verify sealed provenance and cache exact native archives + if: steps.source.outputs.mode == 'stage' + env: + STAGE_NAME: ${{ steps.stage.outputs.artifact_name }} + run: | + set -euo pipefail + python3 scripts/prepare-pkgbuild-stage.py pkgbuilds/terraphim-clients-bin/release-stage.json \ + prepare "$RUNNER_TEMP/$STAGE_NAME" "$RUNNER_TEMP/stage-run.json" \ + "$RUNNER_TEMP/stage-artifact.json" pkgbuilds/terraphim-clients-bin "$(uname -m)" \ + > "$RUNNER_TEMP/pkgbuild-stage-readback.json" + PKGBUILD_STAGE_TEST_DIR="$RUNNER_TEMP/$STAGE_NAME" \ + PKGBUILD_STAGE_RUN_JSON="$RUNNER_TEMP/stage-run.json" \ + PKGBUILD_STAGE_ARTIFACT_JSON="$RUNNER_TEMP/stage-artifact.json" \ + python3 -W error::ResourceWarning -m unittest tests.test_pkgbuild_stage -v - name: Build, lint, install, smoke and remove run: ./pkgbuilds/terraphim-clients-bin/verify.sh + - name: Collect source-selection and cache readback + if: always() + run: | + set -euo pipefail + receipt_dir="$RUNNER_TEMP/pkgbuild-source-readback" + mkdir -p "$receipt_dir" + for name in release.json stage-run.json stage-artifact.json pkgbuild-stage-readback.json; do + if [ -f "$RUNNER_TEMP/$name" ]; then + cp "$RUNNER_TEMP/$name" "$receipt_dir/$name" + fi + done + - name: Retain source-selection and cache readback + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: pkgbuild-source-readback-${{ github.run_id }}-${{ github.run_attempt }} + # One path lets Actions translate the host mount into the job container. + path: ${{ runner.temp }}/pkgbuild-source-readback/ + if-no-files-found: error + overwrite: false diff --git a/Cargo.lock b/Cargo.lock index ed566135..05fa7a8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6315,7 +6315,7 @@ dependencies = [ [[package]] name = "terraphim-cli" -version = "1.21.20" +version = "1.21.22" dependencies = [ "anyhow", "assert_cmd", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "terraphim-session-analyzer" -version = "1.21.20" +version = "1.21.22" dependencies = [ "aho-corasick", "anyhow", @@ -6395,7 +6395,7 @@ dependencies = [ [[package]] name = "terraphim_agent" -version = "1.21.20" +version = "1.21.22" dependencies = [ "ahash", "anyhow", @@ -6580,7 +6580,7 @@ dependencies = [ [[package]] name = "terraphim_grep" -version = "1.21.20" +version = "1.21.22" dependencies = [ "anyhow", "async-trait", @@ -6609,7 +6609,7 @@ dependencies = [ [[package]] name = "terraphim_hooks" -version = "1.21.20" +version = "1.21.22" dependencies = [ "dirs 5.0.1", "serde", @@ -6623,7 +6623,7 @@ dependencies = [ [[package]] name = "terraphim_lsp" -version = "1.21.20" +version = "1.21.22" dependencies = [ "log", "serde", @@ -6650,7 +6650,7 @@ dependencies = [ [[package]] name = "terraphim_mcp_server" -version = "1.21.20" +version = "1.21.22" dependencies = [ "ahash", "anyhow", @@ -6661,9 +6661,12 @@ dependencies = [ "fff-search", "regex", "rmcp", + "schemars 1.2.1", + "serde", "serde_json", "serial_test", "tempfile", + "terraphim_agent", "terraphim_automata", "terraphim_config", "terraphim_file_search", @@ -6713,7 +6716,7 @@ dependencies = [ [[package]] name = "terraphim_negative_contribution" -version = "1.21.20" +version = "1.21.22" dependencies = [ "log", "terraphim_automata", diff --git a/Cargo.toml b/Cargo.toml index 4f0c63d3..2a751c00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ ] [workspace.package] -version = "1.21.20" +version = "1.21.22" edition = "2024" authors = ["Terraphim Team "] documentation = "https://terraphim.ai" diff --git a/README.md b/README.md index 19eddf4d..b40a4501 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Client + integration crates extracted from terraphim-ai (#1910): - `terraphim-cli` / `terraphim_agent` -- CLI + agent REPL -- `terraphim_mcp_server` -- MCP server +- `terraphim_mcp_server` -- MCP server (graph tools and optional [local agent memory/learning tools](docs/mcp/agent-tools.md)) - `terraphim_sessions` -- session import/search - `terraphim_lsp` -- LSP server (EDM diagnostics) - `terraphim_grep`, `terraphim_hooks`, `terraphim_update`, `terraphim_command_runtime`, `terraphim_negative_contribution` diff --git a/crates/terraphim_agent/Cargo.toml b/crates/terraphim_agent/Cargo.toml index 899af2cf..47ce987e 100644 --- a/crates/terraphim_agent/Cargo.toml +++ b/crates/terraphim_agent/Cargo.toml @@ -68,6 +68,7 @@ jiff = { version = "0.2", features = ["serde"] } strsim = "0.11" # For edit distance / fuzzy matching in forgiving CLI uuid = { workspace = true } dialoguer = "0.12" # Interactive CLI prompts for onboarding wizard +tempfile = { workspace = true } sha2 = "0.10" # Corpus and thesaurus hashes for the judge-free memory benchmark (#260) # REPL dependencies - only compiled with features diff --git a/crates/terraphim_agent/src/agent_store.rs b/crates/terraphim_agent/src/agent_store.rs new file mode 100644 index 00000000..06bb0b99 --- /dev/null +++ b/crates/terraphim_agent/src/agent_store.rs @@ -0,0 +1,343 @@ +//! Explicit local storage shared by the agent CLI and MCP adapter. +//! +//! Preserves the existing evolution JSON and learning Markdown formats using +//! Terraphim types. Never executes commands, installs hooks or activates rules. +use crate::memory_retrieve::collect_memory_items; +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + fs, + io::Write, + path::{Component, Path, PathBuf}, +}; +use terraphim_agent_evolution::{ + AgentEvolutionSystem, ImportanceLevel, LessonsState, MemoryItem, MemoryItemType, MemoryState, +}; + +mod learnings; +mod transaction_lock; +pub use learnings::{CompiledCorrections, KgExport, KgNote, LearningRecord, SavedLearning}; +use transaction_lock::TransactionLock; + +const MAX_STORE_BYTES: u64 = 128 * 1024 * 1024; + +// Both the legacy CLI learning APIs and explicit MCP transactions hold this +// same lock. Staged core operations acquire an independent staging-directory +// lock; they cannot unlock or overwrite the destination transaction. +pub(crate) fn lock_learning_directory(directory: &Path) -> Result { + let absolute = if directory.is_absolute() { + directory.to_owned() + } else { + std::env::current_dir()?.join(directory) + }; + AgentStore::open(&absolute)?.lock(Path::new(".write.lock")) +} + +#[derive(Serialize, Deserialize)] +struct Snapshot { + #[serde(default = "agent_id")] + agent_id: String, + #[serde(default)] + saved_at: String, + memory: MemoryState, + lessons: LessonsState, +} +fn agent_id() -> String { + "cli-agent".into() +} +impl Default for Snapshot { + fn default() -> Self { + Self { + agent_id: agent_id(), + saved_at: String::new(), + memory: MemoryState::default(), + lessons: LessonsState::default(), + } + } +} + +/// Explicit local data root shared with `TERRAPHIM_DEFAULT_DATA_PATH`. +#[derive(Debug, Clone)] +pub struct AgentStore { + root: PathBuf, +} +impl AgentStore { + /// Select an absolute data root, creating that directory when absent. + /// + /// # Errors + /// Rejects relative/traversing roots, symlink roots and inaccessible paths. + pub fn open(root: &Path) -> Result { + ensure!(root.is_absolute(), "Agent data root must be absolute"); + ensure!( + !root.components().any(|c| matches!(c, Component::ParentDir)), + "Agent data root must not contain '..'" + ); + if let Ok(meta) = fs::symlink_metadata(root) { + ensure!( + !meta.file_type().is_symlink(), + "Agent data root must not be a symlink" + ); + } + fs::create_dir_all(root).context("Cannot create agent data root")?; + let root = root + .canonicalize() + .context("Cannot resolve agent data root")?; + ensure!(root.is_dir(), "Agent data root must be a directory"); + Ok(Self { root }) + } + + /// Use the CLI's existing environment override or OS configuration root. + /// + /// # Errors + /// Returns an error for a missing default location or invalid explicit root. + pub fn for_cli() -> Result { + let root = std::env::var_os("TERRAPHIM_DEFAULT_DATA_PATH") + .map(PathBuf::from) + .or_else(dirs::config_dir) + .context("No local agent configuration directory")?; + Self::open(&root) + } + + /// Effective canonical root; the working directory does not scope it. + pub fn root(&self) -> &Path { + &self.root + } + /// Existing CLI evolution-file location inside this root. + pub fn memory_path(&self) -> PathBuf { + self.root.join("terraphim/evolution/cli-agent.json") + } + + fn checked(&self, relative: &Path, create_parents: bool) -> Result { + ensure!( + self.root.canonicalize()? == self.root, + "Agent data root changed" + ); + ensure!( + relative + .components() + .all(|c| matches!(c, Component::Normal(_))), + "Invalid store-relative path" + ); + let path = self.root.join(relative); + let mut cursor = self.root.clone(); + let components: Vec<_> = relative.components().collect(); + for (i, component) in components.iter().enumerate() { + cursor.push(component); + let final_component = i + 1 == components.len(); + match fs::symlink_metadata(&cursor) { + Ok(meta) => { + ensure!( + !meta.file_type().is_symlink(), + "Refusing symlink in agent store: {}", + cursor.display() + ); + if !final_component { + ensure!( + meta.is_dir(), + "Store parent is not a directory: {}", + cursor.display() + ); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if create_parents && !final_component { + match fs::create_dir(&cursor) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => (), + Err(error) => return Err(error.into()), + } + let meta = fs::symlink_metadata(&cursor)?; + ensure!( + meta.is_dir() && !meta.file_type().is_symlink(), + "Store parent changed during creation" + ); + } + } + Err(error) => return Err(error.into()), + } + } + Ok(path) + } + + fn read(&self, relative: &Path) -> Result>> { + let path = self.checked(relative, false)?; + let metadata = match fs::symlink_metadata(&path) { + Ok(meta) => meta, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + ensure!( + metadata.is_file() && metadata.len() <= MAX_STORE_BYTES, + "Agent store must be a regular file of at most 128 MiB" + ); + let data = fs::read(&path).with_context(|| format!("Cannot read {}", path.display()))?; + ensure!( + data.len() as u64 <= MAX_STORE_BYTES, + "Agent store exceeds 128 MiB" + ); + self.checked(relative, false)?; + Ok(Some(data)) + } + + fn memory_snapshot(&self) -> Result { + match self.read(Path::new("terraphim/evolution/cli-agent.json"))? { + None => Ok(Snapshot::default()), + Some(data) => { + let snapshot: Snapshot = serde_json::from_slice(&data) + .context("Agent memory store is corrupt; existing bytes were preserved")?; + ensure!( + snapshot.agent_id == "cli-agent", + "Unexpected agent identity in memory store" + ); + Ok(snapshot) + } + } + } + + fn lock(&self, relative: &Path) -> Result { + let path = self.checked(relative, true)?; + let file = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path)?; + ensure!( + file.metadata()?.is_file(), + "Store lock is not a regular file" + ); + let transaction = + TransactionLock::acquire(file).context("Agent store is busy; retry this request")?; + self.checked(relative, false)?; + Ok(transaction) + } + + fn write_atomic(&self, relative: &Path, bytes: &[u8]) -> Result<()> { + ensure!( + bytes.len() as u64 <= MAX_STORE_BYTES, + "Agent store exceeds 128 MiB" + ); + let path = self.checked(relative, true)?; + if let Ok(metadata) = fs::symlink_metadata(&path) { + ensure!( + metadata.is_file(), + "Store destination is not a regular file" + ); + } + let mut temporary = + tempfile::NamedTempFile::new_in(path.parent().context("Missing store parent")?)?; + temporary.write_all(bytes)?; + temporary.as_file().sync_all()?; + self.checked(relative, false)?; + temporary + .persist(&path) + .map_err(|error| error.error) + .context("Cannot commit agent store")?; + ensure!( + self.read(relative)?.as_deref() == Some(bytes), + "Agent store readback differs from committed bytes" + ); + Ok(()) + } + + /// Load the existing typed snapshot without replacing corrupt data. + /// + /// # Errors + /// Returns filesystem, scope and JSON errors instead of an empty replacement. + pub fn load_evolution(&self) -> Result { + let snapshot = self.memory_snapshot()?; + let mut evolution = AgentEvolutionSystem::new(agent_id()); + evolution.memory.current_state = snapshot.memory; + evolution.lessons.current_state = snapshot.lessons; + Ok(evolution) + } + + /// Capture authored UTF-8 content and read the saved item back. + /// + /// Existing `MemoryState` semantics retain the latest 100 short-term items. + /// The advisory transaction lock prevents cooperating CLI/MCP writers losing + /// acknowledged records. A busy store returns an error, never a success. + /// + /// # Errors + /// Rejects blank/oversized content, oversized tags, busy/corrupt stores and I/O failures. + pub fn memory_capture( + &self, + content: &str, + provenance_tag: Option<&str>, + ) -> Result { + validate_text("Memory lesson", content, 1_000_000)?; + if let Some(tag) = provenance_tag { + validate_text("Provenance tag", tag, 512)?; + } + let _lock = self.lock(Path::new("terraphim/evolution/cli-agent.lock"))?; + let mut snapshot = self.memory_snapshot()?; + let memory = MemoryItem { + id: uuid::Uuid::new_v4().to_string(), + item_type: MemoryItemType::Experience, + content: content.to_owned(), + created_at: chrono::Utc::now(), + last_accessed: None, + access_count: 0, + importance: ImportanceLevel::Medium, + tags: provenance_tag + .map(|tag| vec![format!("provenance:{tag}")]) + .unwrap_or_default(), + associations: HashMap::new(), + }; + let id = memory.id.clone(); + snapshot.memory.add_memory(memory); + snapshot.saved_at = chrono::Utc::now().to_rfc3339(); + self.write_atomic( + Path::new("terraphim/evolution/cli-agent.json"), + &serde_json::to_vec_pretty(&snapshot)?, + )?; + self.memory_show(&id) + } + + /// Read one exact stored ID. Prefixes and path-like IDs are not accepted. + /// + /// # Errors + /// Returns an error for invalid/unknown IDs or unreadable/corrupt storage. + pub fn memory_show(&self, id: &str) -> Result { + validate_id(id)?; + collect_memory_items(&self.memory_snapshot()?.memory) + .into_iter() + .find(|item| item.id == id) + .with_context(|| format!("Memory ID '{id}' not found in the configured store")) + } + + /// List stored items in stable creation-time/ID order with bounded pagination. + /// + /// # Errors + /// Rejects limits outside 1..=100 and unreadable/corrupt storage. + pub fn memory_list(&self, limit: usize, offset: usize) -> Result> { + ensure!( + (1..=100).contains(&limit), + "Limit must be between 1 and 100" + ); + let mut items = collect_memory_items(&self.memory_snapshot()?.memory); + items.sort_by(|a, b| { + b.created_at + .cmp(&a.created_at) + .then_with(|| a.id.cmp(&b.id)) + }); + Ok(items.into_iter().skip(offset).take(limit).collect()) + } +} +fn validate_text(name: &str, value: &str, max: usize) -> Result<()> { + ensure!(!value.trim().is_empty(), "{name} must not be blank"); + ensure!(value.len() <= max, "{name} exceeds {max} UTF-8 bytes"); + Ok(()) +} +fn validate_id(id: &str) -> Result<()> { + ensure!( + !id.is_empty() + && id.len() <= 200 + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')), + "ID must be a complete identifier, not a path or prefix" + ); + Ok(()) +} diff --git a/crates/terraphim_agent/src/agent_store/learnings.rs b/crates/terraphim_agent/src/agent_store/learnings.rs new file mode 100644 index 00000000..6934f38f --- /dev/null +++ b/crates/terraphim_agent/src/agent_store/learnings.rs @@ -0,0 +1,416 @@ +//! Scoped application operations over the existing learning/correction APIs. +use super::{AgentStore, validate_id, validate_text}; +use crate::learnings::{ + self, CapturedLearning, CorrectionEvent, CorrectionType, CorrectionTypeFilter, + LearningCaptureConfig, +}; +use anyhow::{Context, Result, ensure}; +use serde::Serialize; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// Parsed bytes actually saved in a learning Markdown record. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "kind", content = "record", rename_all = "snake_case")] +pub enum LearningRecord { + /// Recorded failure; no command was executed by capture. + Failure(CapturedLearning), + /// Explicit correction; no hook was activated. + Correction(CorrectionEvent), +} +impl LearningRecord { + fn id(&self) -> &str { + match self { + Self::Failure(record) => &record.id, + Self::Correction(record) => &record.id, + } + } +} +/// A stored record and its effective location. +#[derive(Debug, Clone, Serialize)] +pub struct SavedLearning { + /// Existing-format Markdown file inside the selected data root. + pub path: PathBuf, + /// Parsed persisted record. + pub record: LearningRecord, +} +/// A fresh compiled mapping artefact, distinct from hook activation. +#[derive(Debug, Serialize)] +pub struct CompiledCorrections { + /// Generated JSON path inside a fresh export directory. + pub path: PathBuf, + /// Actual number of compiled mappings. + pub mappings: usize, + /// Actual decoded Terraphim thesaurus. + pub thesaurus: terraphim_types::Thesaurus, +} +/// A fresh immutable KG export and the notes read back from it. +#[derive(Debug, Serialize)] +pub struct KgExport { + /// Unique generation directory inside the explicit data root. + pub directory: PathBuf, + /// Actual saved Markdown notes and their paths. + pub files: Vec, +} +/// One generated Logseq-compatible concept note. +#[derive(Debug, Serialize)] +pub struct KgNote { + /// Actual saved path. + pub path: PathBuf, + /// Actual saved UTF-8 Markdown including `synonyms::`. + pub markdown: String, +} + +impl AgentStore { + fn learning_relative(name: &str) -> PathBuf { + Path::new("terraphim/learnings").join(name) + } + + fn learning_records(&self) -> Result> { + let directory = self.checked(Path::new("terraphim/learnings"), false)?; + if !directory.exists() { + return Ok(Vec::new()); + } + ensure!(directory.is_dir(), "Learning store is not a directory"); + let mut records = Vec::new(); + let mut total = 0usize; + for entry in fs::read_dir(directory)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().context("Learning filename is not UTF-8")?; + if !name.ends_with(".md") { + continue; + } + ensure!( + records.len() < 1_000, + "Learning store exceeds 1,000 records" + ); + let relative = Self::learning_relative(name); + let bytes = self + .read(&relative)? + .context("Learning record disappeared")?; + total = total + .checked_add(bytes.len()) + .context("Learning store size overflow")?; + ensure!( + bytes.len() <= 1_048_576 && total <= 16 * 1024 * 1024, + "Learning store exceeds record or total byte limit" + ); + let text = std::str::from_utf8(&bytes).context("Learning record is not UTF-8")?; + let record = if name.starts_with("correction-") { + LearningRecord::Correction( + CorrectionEvent::from_markdown(text).context("Corrupt correction record")?, + ) + } else { + ensure!( + name.starts_with("learning-"), + "Unknown Markdown record in learning store" + ); + LearningRecord::Failure( + CapturedLearning::from_markdown(text).context("Corrupt learning record")?, + ) + }; + validate_id(record.id())?; + let prefix = match record { + LearningRecord::Failure(_) => "learning", + LearningRecord::Correction(_) => "correction", + }; + ensure!( + name == format!("{prefix}-{}.md", record.id()), + "Learning filename/ID mismatch" + ); + records.push(SavedLearning { + path: self.checked(&relative, false)?, + record, + }); + } + records.sort_by(|a, b| a.record.id().cmp(b.record.id())); + Ok(records) + } + + fn stage_learnings(&self) -> Result { + let stage_parent = self.checked(Path::new("terraphim/staging/placeholder"), true)?; + let stage = tempfile::tempdir_in(stage_parent.parent().context("No staging parent")?)?; + for record in self.learning_records()? { + let name = record + .path + .file_name() + .context("Learning record has no name")?; + let relative = Path::new("terraphim/learnings").join(name); + let bytes = self + .read(&relative)? + .context("Learning record disappeared")?; + fs::write(stage.path().join(name), bytes)?; + } + Ok(stage) + } + + fn stage_config(stage: &Path) -> LearningCaptureConfig { + // Both with and without the environment override, select only this + // explicit staging directory. The nonexistent project parent cannot win. + LearningCaptureConfig::new(stage.join("unselected-project/learnings"), stage.to_owned()) + } + + fn commit_staged_learning(&self, path: &Path, replace: bool) -> Result { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .context("Invalid generated record filename")?; + let relative = Self::learning_relative(name); + let destination = self.checked(&relative, true)?; + ensure!( + replace || !destination.exists(), + "Refusing to replace an existing learning ID" + ); + let bytes = fs::read(path)?; + ensure!(bytes.len() <= 1_048_576, "Learning record exceeds 1 MiB"); + let existing = self.learning_records()?; + ensure!( + replace || existing.len() < 1_000, + "Learning store exceeds 1,000 records" + ); + let mut total = bytes.len() as u64; + for record in existing { + if record.path != destination { + total = total + .checked_add(fs::metadata(record.path)?.len()) + .context("Learning store size overflow")?; + } + } + ensure!(total <= 16 * 1024 * 1024, "Learning store exceeds 16 MiB"); + let text = std::str::from_utf8(&bytes)?; + let record = if name.starts_with("correction-") { + LearningRecord::Correction( + CorrectionEvent::from_markdown(text) + .context("Generated correction cannot be decoded")?, + ) + } else { + LearningRecord::Failure( + CapturedLearning::from_markdown(text) + .context("Generated failure cannot be decoded")?, + ) + }; + validate_id(record.id())?; + self.write_atomic(&relative, &bytes)?; + self.learning_records()? + .into_iter() + .find(|saved| saved.path == destination) + .context("Saved learning was not readable") + } + + /// Record a supplied failed-command observation; does not execute it. + /// + /// # Errors + /// Rejects blank/oversized observations, success exit codes, busy or corrupt storage. + pub fn learning_capture( + &self, + command: &str, + error: &str, + exit_code: i32, + ) -> Result { + validate_text("Command", command, 65_536)?; + validate_text("Error output", error, 524_288)?; + ensure!( + exit_code != 0, + "Learning capture requires a non-zero exit code" + ); + let _lock = + super::lock_learning_directory(&self.checked(Path::new("terraphim/learnings"), true)?)?; + let stage = self.stage_learnings()?; + let path = learnings::capture_failed_command( + &learnings::redact_secrets(command), + error, + exit_code, + &Self::stage_config(stage.path()), + )?; + let saved = CapturedLearning::from_markdown(&fs::read_to_string(&path)?) + .context("Generated failure cannot be decoded")?; + ensure!( + saved.command == learnings::redact_secrets(command) + && saved.error_output == learnings::redact_secrets(error) + && saved.exit_code == exit_code, + "Existing learning Markdown cannot preserve this observation exactly; supply one failing command and representable error text" + ); + self.commit_staged_learning(&path, false) + } + + /// Query existing learning/correction records using the existing text query API. + /// + /// # Errors + /// Rejects blank/oversized queries, invalid limits and corrupt or unreadable records. + pub fn learning_query(&self, query: &str, limit: usize) -> Result> { + validate_text("Query", query, 4_096)?; + ensure!( + (1..=100).contains(&limit), + "Limit must be between 1 and 100" + ); + let records = self.learning_records()?; + let directory = self.checked(Path::new("terraphim/learnings"), false)?; + let matches = learnings::query_all_entries(&directory, query, false)?; + let ids: std::collections::HashSet<_> = matches.iter().map(|entry| entry.id()).collect(); + Ok(records + .into_iter() + .filter(|entry| ids.contains(entry.record.id())) + .take(limit) + .collect()) + } + + /// Annotate one exact failure ID with a correction; this adds no compiled rule. + /// + /// # Errors + /// Rejects invalid/unknown IDs, invalid correction text and storage errors. + pub fn learning_correct(&self, id: &str, correction: &str) -> Result { + validate_id(id)?; + validate_text("Correction", correction, 65_536)?; + let _lock = + super::lock_learning_directory(&self.checked(Path::new("terraphim/learnings"), true)?)?; + let selected = self + .learning_records()? + .into_iter() + .find( + |entry| matches!(&entry.record, LearningRecord::Failure(record) if record.id == id), + ) + .context("Exact learning ID not found")?; + let parent = self.checked(Path::new("terraphim/staging/placeholder"), true)?; + let stage = tempfile::tempdir_in(parent.parent().context("No staging parent")?)?; + fs::copy( + &selected.path, + stage + .path() + .join(selected.path.file_name().context("Missing record name")?), + )?; + let path = learnings::correct_learning( + &stage.path().to_owned(), + id, + &learnings::redact_secrets(correction), + )?; + let saved = CapturedLearning::from_markdown(&fs::read_to_string(&path)?) + .context("Generated annotation cannot be decoded")?; + ensure!( + saved.id == id + && saved.correction.as_deref() + == Some(learnings::redact_secrets(correction).as_str()), + "Existing learning Markdown cannot preserve this annotation exactly" + ); + self.commit_staged_learning(&path, true) + } + + /// Add an explicit ToolPreference correction; compilation is a separate operation. + /// + /// # Errors + /// Rejects blank/oversized text and storage errors. Context may be empty. + pub fn correction_add( + &self, + original: &str, + corrected: &str, + context: &str, + ) -> Result { + validate_text("Original", original, 65_536)?; + validate_text("Corrected", corrected, 65_536)?; + ensure!( + context.len() <= 65_536, + "Context exceeds 65,536 UTF-8 bytes" + ); + let _lock = + super::lock_learning_directory(&self.checked(Path::new("terraphim/learnings"), true)?)?; + let stage = self.stage_learnings()?; + let path = learnings::capture_correction( + CorrectionType::ToolPreference, + original, + corrected, + context, + &Self::stage_config(stage.path()), + )?; + let saved = CorrectionEvent::from_markdown(&fs::read_to_string(&path)?) + .context("Generated correction cannot be decoded")?; + ensure!( + saved.original == learnings::redact_secrets(original) + && saved.corrected == learnings::redact_secrets(corrected) + && saved.context_description == learnings::redact_secrets(context), + "Existing correction Markdown cannot preserve these fields exactly" + ); + self.commit_staged_learning(&path, false) + } + + fn new_export(&self) -> Result { + let relative = Path::new("terraphim/exports").join(uuid::Uuid::new_v4().to_string()); + let path = self.checked(&relative, true)?; + fs::create_dir(&path)?; + Ok(path) + } + + /// Compile explicit corrections into a fresh JSON artefact, without activating it. + /// + /// # Errors + /// Returns storage, decoding and compilation errors; never fabricates a count. + pub fn correction_compile(&self) -> Result { + let _lock = + super::lock_learning_directory(&self.checked(Path::new("terraphim/learnings"), true)?)?; + let stage = self.stage_learnings()?; + let thesaurus = learnings::compile_corrections_to_thesaurus(stage.path())?; + let output = self.new_export()?.join("compiled-corrections.json"); + let relative = output.strip_prefix(&self.root)?; + self.write_atomic(relative, &serde_json::to_vec_pretty(&thesaurus)?)?; + let saved = self.read(relative)?.context("Compiled output missing")?; + let thesaurus: terraphim_types::Thesaurus = serde_json::from_slice(&saved)?; + Ok(CompiledCorrections { + path: output, + mappings: thesaurus.len(), + thesaurus, + }) + } + + /// Export explicit corrections as fresh `synonyms::` Markdown concept notes. + /// + /// # Errors + /// Refuses originals that cannot be represented by the existing comma-separated + /// synonym syntax and any read, write or export failure. Does not register a role. + pub fn correction_export_kg(&self) -> Result { + let _lock = + super::lock_learning_directory(&self.checked(Path::new("terraphim/learnings"), true)?)?; + for record in self.learning_records()? { + if let LearningRecord::Correction(record) = record.record { + ensure!( + !record.original.contains([',', '\n', '\r']) + && !record.corrected.contains(['\n', '\r']), + "This correction cannot be represented by single-line comma-separated synonyms:: syntax; compiled JSON remains supported" + ); + } + } + let stage = self.stage_learnings()?; + let generated = stage.path().join("export"); + let count = learnings::export_corrections_as_kg( + stage.path(), + &generated, + CorrectionTypeFilter::ToolPreference, + )?; + let directory = self.new_export()?; + let mut files = Vec::new(); + if generated.exists() { + for entry in fs::read_dir(generated)? { + let entry = entry?; + let output = directory.join(entry.file_name()); + let relative = output.strip_prefix(&self.root)?; + self.write_atomic(relative, &fs::read(entry.path())?)?; + let markdown = + String::from_utf8(self.read(relative)?.context("Exported note missing")?)?; + ensure!( + markdown.contains("synonyms::"), + "Exported note lacks synonym syntax" + ); + files.push(KgNote { + path: output, + markdown, + }); + } + } + ensure!( + files.len() == count, + "Export count differs from saved notes" + ); + files.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(KgExport { directory, files }) + } +} diff --git a/crates/terraphim_agent/src/agent_store/transaction_lock.rs b/crates/terraphim_agent/src/agent_store/transaction_lock.rs new file mode 100644 index 00000000..9932ea39 --- /dev/null +++ b/crates/terraphim_agent/src/agent_store/transaction_lock.rs @@ -0,0 +1,124 @@ +//! Transaction ownership must end before inherited descriptors are closed. +use std::fs::{File, TryLockError}; + +#[derive(Debug)] +pub(crate) struct TransactionLock { + file: File, +} +impl TransactionLock { + pub(super) fn acquire(file: File) -> Result { + file.try_lock()?; + Ok(Self { file }) + } +} + +impl Drop for TransactionLock { + fn drop(&mut self) { + // Closing this descriptor alone can leave an inherited/duplicated open + // description holding the lock. End ownership explicitly before close. + // Drop cannot report an unlock error; closing the file remains the fallback. + let _ = self.file.unlock(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + fs::OpenOptions, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, + }; + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + Self(std::env::temp_dir().join(format!( + "terraphim-transaction-lock-{}-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(), + NEXT.fetch_add(1, Ordering::Relaxed) + ))) + } + fn create(&self) -> File { + OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&self.0) + .unwrap() + } + fn reopen(&self) -> File { + OpenOptions::new() + .read(true) + .write(true) + .open(&self.0) + .unwrap() + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + + #[test] + fn completed_transaction_releases_lock_while_duplicate_is_alive() { + let fixture = Fixture::new(); + let transaction = TransactionLock::acquire(fixture.create()).unwrap(); + let duplicate = transaction.file.try_clone().unwrap(); + assert!(TransactionLock::acquire(fixture.reopen()).is_err()); + drop(transaction); + let next = TransactionLock::acquire(fixture.reopen()); + drop(duplicate); + assert!( + next.is_ok(), + "completed transaction must release its lock: {next:?}" + ); + } + + #[cfg(unix)] + #[test] + fn completed_transaction_releases_lock_while_forked_child_is_alive() { + use std::{ + io::Write, + os::{fd::AsRawFd, unix::net::UnixStream}, + }; + unsafe extern "C" { + fn fork() -> i32; + fn read(fd: i32, buf: *mut std::ffi::c_void, count: usize) -> isize; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + fn _exit(status: i32) -> !; + } + let fixture = Fixture::new(); + let transaction = TransactionLock::acquire(fixture.create()).unwrap(); + let (mut parent, child) = UnixStream::pair().unwrap(); + let child_fd = child.as_raw_fd(); + // SAFETY: after fork the child uses only async-signal-safe POSIX calls + // and _exit, never Rust allocation, locks or destructors. The live + // stream descriptor and byte buffer remain valid until _exit. + let pid = unsafe { fork() }; + assert!(pid >= 0); + if pid == 0 { + let mut byte = 0_u8; + unsafe { + read(child_fd, (&mut byte as *mut u8).cast(), 1); + _exit(0) + } + } + drop(transaction); + let next = TransactionLock::acquire(fixture.reopen()); + // Always reap the child before asserting, including a red regression. + parent.write_all(&[1]).unwrap(); + let mut status = 0; + // SAFETY: pid is this test's child; status points to a valid integer. + assert_eq!(unsafe { waitpid(pid, &mut status, 0) }, pid); + assert_eq!(status, 0); + assert!( + next.is_ok(), + "child lifetime must not extend transaction ownership: {next:?}" + ); + } +} diff --git a/crates/terraphim_agent/src/learnings/capture.rs b/crates/terraphim_agent/src/learnings/capture.rs index 54e069c2..0d1fb6ca 100644 --- a/crates/terraphim_agent/src/learnings/capture.rs +++ b/crates/terraphim_agent/src/learnings/capture.rs @@ -1031,6 +1031,8 @@ pub fn capture_failed_command( // Determine storage location let storage_dir = config.storage_location(); + let _transaction = crate::agent_store::lock_learning_directory(&storage_dir) + .map_err(|error| LearningError::StorageError(format!("{error:#}")))?; // Create storage directory if it doesn't exist fs::create_dir_all(&storage_dir) @@ -1143,6 +1145,8 @@ pub fn capture_correction( let redacted_context = redact_secrets(context_description); let storage_dir = config.storage_location(); + let _transaction = crate::agent_store::lock_learning_directory(&storage_dir) + .map_err(|error| LearningError::StorageError(format!("{error:#}")))?; fs::create_dir_all(&storage_dir) .map_err(|e| LearningError::StorageError(format!("Cannot create storage dir: {}", e)))?; @@ -1304,6 +1308,8 @@ pub fn correct_learning( return Err(LearningError::NotFound(id.to_string())); } + let _transaction = crate::agent_store::lock_learning_directory(storage_dir) + .map_err(|error| LearningError::StorageError(format!("{error:#}")))?; let entries = fs::read_dir(storage_dir)?; for entry in entries.flatten() { diff --git a/crates/terraphim_agent/src/lib.rs b/crates/terraphim_agent/src/lib.rs index 55b9ea89..21b9ba67 100644 --- a/crates/terraphim_agent/src/lib.rs +++ b/crates/terraphim_agent/src/lib.rs @@ -3,6 +3,8 @@ //! Bundles the interactive REPL, robot-mode JSON output, forgiving CLI parser, //! MCP tool index, onboarding workflows, and optional shared-learning store. //! Feature flags gate heavier subsystems: `server`, `repl`, `shared-learning`. +/// Explicit local storage shared with the MCP adapter. +pub mod agent_store; #[cfg(feature = "server")] pub mod client; pub mod logging; diff --git a/crates/terraphim_agent/src/main.rs b/crates/terraphim_agent/src/main.rs index 3eced688..1a7dc713 100644 --- a/crates/terraphim_agent/src/main.rs +++ b/crates/terraphim_agent/src/main.rs @@ -593,6 +593,7 @@ fn run_tui_offline_mode(transparent: bool) -> Result<()> { run_tui(None, transparent) } +#[cfg(feature = "server")] fn run_tui_server_mode(server_url: &str, transparent: bool) -> Result<()> { run_tui(Some(server_url.to_string()), transparent) } @@ -2504,54 +2505,8 @@ async fn run_cache_command(sub: &CacheSub) -> Result<()> { } } -fn evolution_path() -> std::path::PathBuf { - // Use the same explicit data root as learning capture. Existing users - // without an override retain the historical config-directory location. - std::env::var_os("TERRAPHIM_DEFAULT_DATA_PATH") - .map(std::path::PathBuf::from) - .or_else(dirs::config_dir) - .unwrap_or_else(|| std::path::PathBuf::from(".")) - .join("terraphim") - .join("evolution") - .join("cli-agent.json") -} - -fn load_evolution() -> terraphim_agent_evolution::AgentEvolutionSystem { - let path = evolution_path(); - if path.exists() - && let Ok(data) = std::fs::read_to_string(&path) - { - #[derive(serde::Deserialize)] - struct EvolutionState { - memory: terraphim_agent_evolution::MemoryState, - lessons: terraphim_agent_evolution::LessonsState, - } - if let Ok(state) = serde_json::from_str::(&data) { - let mut evolution = - terraphim_agent_evolution::AgentEvolutionSystem::new("cli-agent".to_string()); - evolution.memory.current_state = state.memory; - evolution.lessons.current_state = state.lessons; - return evolution; - } - } - terraphim_agent_evolution::AgentEvolutionSystem::new("cli-agent".to_string()) -} - -fn save_evolution( - evolution: &terraphim_agent_evolution::AgentEvolutionSystem, -) -> Result<(), anyhow::Error> { - let path = evolution_path(); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - let state = serde_json::json!({ - "agent_id": evolution.agent_id, - "saved_at": chrono::Utc::now().to_rfc3339(), - "memory": evolution.memory.current_state, - "lessons": evolution.lessons.current_state, - }); - std::fs::write(&path, serde_json::to_string_pretty(&state)?)?; - Ok(()) +fn load_evolution() -> Result { + terraphim_agent::agent_store::AgentStore::for_cli()?.load_evolution() } #[cfg(feature = "shared-learning")] diff --git a/crates/terraphim_agent/src/memory_command.rs b/crates/terraphim_agent/src/memory_command.rs index 0750a095..4a19a922 100644 --- a/crates/terraphim_agent/src/memory_command.rs +++ b/crates/terraphim_agent/src/memory_command.rs @@ -12,7 +12,7 @@ use anyhow::{Context, Result}; use terraphim_agent::service::TuiService; use crate::cli_schema::MemorySub; -use crate::{CommandOutputConfig, load_evolution, save_evolution, truncate_snippet}; +use crate::{CommandOutputConfig, load_evolution, truncate_snippet}; pub(crate) async fn run_memory_command( sub: MemorySub, @@ -25,8 +25,6 @@ pub(crate) async fn run_memory_command( file, provenance_tag, } => { - use terraphim_agent_evolution::{ImportanceLevel, MemoryItem, MemoryItemType}; - let content = match (content, file) { (Some(content), None) => content, (None, Some(path)) => std::fs::read_to_string(&path).with_context(|| { @@ -41,26 +39,11 @@ pub(crate) async fn run_memory_command( // Validate before loading or saving the store. The evolution crate // enforces the same byte limit when adding the MemoryItem. anyhow::ensure!(content.len() <= 1_000_000, "Memory lesson exceeds 1 MB"); - let mut evolution = load_evolution(); - let tags: Vec = provenance_tag - .clone() - .map(|tag| vec![format!("provenance:{}", tag)]) - .unwrap_or_default(); - let memory = MemoryItem { - id: uuid::Uuid::new_v4().to_string(), - item_type: MemoryItemType::Experience, - content, - created_at: chrono::Utc::now(), - last_accessed: None, - access_count: 0, - importance: ImportanceLevel::Medium, - tags, - associations: std::collections::HashMap::new(), - }; - let id = memory.id.clone(); - match evolution.memory.add_memory(memory).await { - Ok(()) => { - save_evolution(&evolution)?; + match terraphim_agent::agent_store::AgentStore::for_cli()? + .memory_capture(&content, provenance_tag.as_deref()) + { + Ok(memory) => { + let id = memory.id; if output.is_machine_readable() { println!( "{}", @@ -209,7 +192,7 @@ pub(crate) async fn run_memory_command( ) })?; - let evolution = load_evolution(); + let evolution = load_evolution()?; let items = collect_memory_items(&evolution.memory.current_state); let outcome = retrieve( @@ -326,7 +309,7 @@ pub(crate) async fn run_memory_command( // Injected size (#261): retrieve the memory items the hook would // inject for this prompt, through the unchanged `retrieve` with // the benchmark's limit, and measure the exact injection text. - let evolution = load_evolution(); + let evolution = load_evolution()?; let store_items = collect_memory_items(&evolution.memory.current_state); let injected_items: Vec = retrieve( &role_name, @@ -395,7 +378,7 @@ pub(crate) async fn run_memory_command( MemorySub::Validate { all, lesson_id } => { use terraphim_agent_evolution::MemoryItem; - let evolution = load_evolution(); + let evolution = load_evolution()?; // TODO(#208): replace with MemoryState::iter_all() let all_items = terraphim_agent::memory_retrieve::collect_memory_items( &evolution.memory.current_state, @@ -522,7 +505,7 @@ pub(crate) async fn run_memory_command( } => { use terraphim_agent_evolution::MemoryItem; - let evolution = load_evolution(); + let evolution = load_evolution()?; // TODO(#208): replace with MemoryState::iter_all() let all_items = terraphim_agent::memory_retrieve::collect_memory_items( &evolution.memory.current_state, @@ -704,7 +687,7 @@ pub(crate) async fn run_memory_command( Ok(()) } MemorySub::List { item_type, limit } => { - let evolution = load_evolution(); + let evolution = load_evolution()?; let state = &evolution.memory.current_state; // TODO(#208): replace with MemoryState::iter_all() let mut all_items = terraphim_agent::memory_retrieve::collect_memory_items(state); @@ -784,7 +767,7 @@ pub(crate) async fn run_memory_command( Ok(()) } MemorySub::Show { id, json } => { - let evolution = load_evolution(); + let evolution = load_evolution()?; // TODO(#208): replace with MemoryState::get(id) let memory_item = terraphim_agent::memory_retrieve::collect_memory_items( @@ -873,7 +856,7 @@ pub(crate) async fn run_memory_command( format, output: outfile, } => { - let evolution = load_evolution(); + let evolution = load_evolution()?; // TODO(#208): replace with MemoryState::iter_all() let all_items = terraphim_agent::memory_retrieve::collect_memory_items( diff --git a/crates/terraphim_agent/tests/agent_learning_store_contract.rs b/crates/terraphim_agent/tests/agent_learning_store_contract.rs new file mode 100644 index 00000000..dbf54279 --- /dev/null +++ b/crates/terraphim_agent/tests/agent_learning_store_contract.rs @@ -0,0 +1,191 @@ +//! Actual failed command, stored annotation, explicit compilation and KG export. +use terraphim_agent::agent_store::{AgentStore, LearningRecord}; + +#[test] +fn failed_observation_annotation_and_explicit_rule_are_distinct() { + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + let failed = std::process::Command::new("printf") + .arg("%Z") + .output() + .unwrap(); + assert!(!failed.status.success()); + let stderr = String::from_utf8(failed.stderr).unwrap(); + let saved = store + .learning_capture("printf '%Z'", &stderr, failed.status.code().unwrap()) + .unwrap(); + let LearningRecord::Failure(record) = saved.record else { + panic!("expected actual failure record") + }; + assert_eq!(record.command, "printf '%Z'"); + assert_eq!(record.exit_code, 1); + assert_eq!(record.error_output.trim(), stderr.trim()); + assert!(saved.path.is_file()); + let corrected = std::process::Command::new("printf") + .args(["%s\n", "reviewed correction"]) + .output() + .unwrap(); + assert!(corrected.status.success()); + assert_eq!(corrected.stdout, b"reviewed correction\n"); + let annotation = store + .learning_correct(&record.id, "printf '%s\\n' 'reviewed correction'") + .unwrap(); + let LearningRecord::Failure(actual) = annotation.record else { + panic!("expected annotated failure") + }; + assert_eq!( + actual.correction.as_deref(), + Some("printf '%s\\n' 'reviewed correction'") + ); + assert_eq!(store.correction_compile().unwrap().mappings, 0); + assert_eq!(store.learning_query("printf", 10).unwrap().len(), 1); + store + .correction_add( + "printf '%Z'", + "printf '%s\\n' 'reviewed correction'", + "Actual public fixture observation", + ) + .unwrap(); + let compiled = store.correction_compile().unwrap(); + assert_eq!(compiled.mappings, 1); + let disk: terraphim_types::Thesaurus = + serde_json::from_slice(&std::fs::read(compiled.path).unwrap()).unwrap(); + assert_eq!(disk.len(), 1); + let mapped = disk + .get(&terraphim_types::NormalizedTermValue::from("printf '%Z'")) + .unwrap(); + assert_eq!(mapped.display(), "printf '%s\\n' 'reviewed correction'"); + let exported = store.correction_export_kg().unwrap(); + assert_eq!(exported.files.len(), 1); + assert!( + exported.files[0] + .markdown + .contains("synonyms:: printf '%Z'") + ); + assert_eq!( + std::fs::read_to_string(&exported.files[0].path).unwrap(), + exported.files[0].markdown + ); + assert_eq!(store.learning_query("printf", 10).unwrap().len(), 2); + assert!(!root.path().join(".claude").exists()); + assert!(!root.path().join(".opencode").exists()); +} + +#[test] +fn scope_unknown_ids_malformed_records_and_unsupported_kg_values_fail_closed() { + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + assert!(store.learning_capture("printf x", "", 1).is_err()); + assert!(store.learning_capture("printf x", "no failure", 0).is_err()); + assert!(store.learning_correct("../escape", "printf x").is_err()); + assert!(store.learning_correct("missing", "printf x").is_err()); + assert_eq!(store.learning_query("printf", 10).unwrap().len(), 0); + assert!(!root.path().join("terraphim/staging").exists()); + store.correction_add("a,b", "safe", "").unwrap(); + assert_eq!(store.correction_compile().unwrap().mappings, 1); + assert!(store.correction_export_kg().is_err()); + let directory = root.path().join("terraphim/learnings"); + std::fs::write(directory.join("correction-broken.md"), b"not a correction").unwrap(); + assert!(store.learning_query("safe", 10).is_err()); + assert!(store.correction_compile().is_err()); + assert!(store.correction_add("must not", "write", "").is_err()); +} + +#[test] +fn cli_learning_mutations_and_mcp_store_share_the_same_transaction_lock() { + use terraphim_agent::learnings::{ + CorrectionType, LearningCaptureConfig, capture_correction, capture_failed_command, + correct_learning, + }; + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + let saved = store + .learning_capture("printf '%Z'", "actual fixture error", 1) + .unwrap(); + let LearningRecord::Failure(record) = saved.record else { + panic!("failure") + }; + let directory = root.path().join("terraphim/learnings"); + let config = LearningCaptureConfig::new( + root.path().join("absent/project/learnings"), + directory.clone(), + ); + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(directory.join(".write.lock")) + .unwrap(); + lock.lock().unwrap(); + let before = std::fs::read(&saved.path).unwrap(); + assert!(capture_failed_command("must not capture", "failure", 1, &config).is_err()); + assert!( + capture_correction( + CorrectionType::ToolPreference, + "must not", + "capture", + "", + &config + ) + .is_err() + ); + assert!(correct_learning(&directory, &record.id, "must not replace").is_err()); + assert!( + store + .learning_correct(&record.id, "must not replace") + .is_err() + ); + assert_eq!(std::fs::read(&saved.path).unwrap(), before); + drop(lock); + correct_learning(&directory, &record.id, "CLI correction after unlock").unwrap(); + let repeated_path = + capture_failed_command("printf '%Z'", "repeated observed error", 1, &config).unwrap(); + let repeated = terraphim_agent::learnings::CapturedLearning::from_markdown( + &std::fs::read_to_string(repeated_path).unwrap(), + ) + .unwrap(); + let importance = repeated.importance.unwrap(); + assert_eq!( + importance.repetition_count, 1, + "reader must see history while outer mutation lock is held" + ); + assert!(importance.has_correction); + + let values = store.learning_query("printf", 10).unwrap(); + assert!(values.iter().any(|value| matches!(&value.record, LearningRecord::Failure(record) if record.correction.as_deref()==Some("CLI correction after unlock")))); +} + +#[test] +fn unsupported_markdown_roundtrips_are_refused_before_destination_changes() { + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + for (original, corrected, context) in [ + ("printf `date`", "printf safe", ""), + ("printf original", "printf `date`", ""), + ( + "printf original", + "printf safe", + "first\n## Unrelated\ntruncated", + ), + ] { + assert!(store.correction_add(original, corrected, context).is_err()); + } + for (command, error) in [ + ("printf first\nsecond", "error"), + ("printf x", "first\n```\ntruncated"), + ("printf x\nexit_code: 0", "error"), + ] { + assert!(store.learning_capture(command, error, 1).is_err()); + } + assert!(store.learning_query("printf", 100).unwrap().is_empty()); + let saved = store + .learning_capture("printf '%Z'", "observed failure\n", 1) + .unwrap(); + let LearningRecord::Failure(record) = saved.record else { + panic!("failure") + }; + let before = std::fs::read(&saved.path).unwrap(); + assert!(store.learning_correct(&record.id, "first\nsecond").is_err()); + assert_eq!(std::fs::read(saved.path).unwrap(), before); +} diff --git a/crates/terraphim_agent/tests/agent_store_contract.rs b/crates/terraphim_agent/tests/agent_store_contract.rs new file mode 100644 index 00000000..2144a70b --- /dev/null +++ b/crates/terraphim_agent/tests/agent_store_contract.rs @@ -0,0 +1,85 @@ +//! Real file contracts for the shared CLI/MCP agent store. +use terraphim_agent::agent_store::AgentStore; + +#[test] +fn memory_readback_restart_scope_and_invalid_inputs() { + let root = tempfile::tempdir().unwrap(); + let other = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + assert!(store.memory_list(20, 0).unwrap().is_empty()); + let text = "Risk requires a Decision — café, Ελληνικά.\nKeep the public brief intact."; + let saved = store + .memory_capture(text, Some("public-project-brief")) + .unwrap(); + let id = saved.id.clone(); + let restarted = AgentStore::open(root.path()).unwrap(); + assert_eq!(restarted.memory_show(&id).unwrap().content, text); + assert_eq!(saved.tags, ["provenance:public-project-brief"]); + assert!( + AgentStore::open(other.path()) + .unwrap() + .memory_show(&id) + .is_err() + ); + let before = std::fs::read(store.memory_path()).unwrap(); + for content in ["", " \n", &"a".repeat(1_000_001)] { + assert!(store.memory_capture(content, None).is_err()); + assert_eq!(std::fs::read(store.memory_path()).unwrap(), before); + } + assert!(store.memory_show("../cli-agent.json").is_err()); + assert!(store.memory_list(101, 0).is_err()); +} + +#[test] +fn corrupt_store_is_not_replaced_with_an_empty_success() { + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + store + .memory_capture("Keep the existing record.", None) + .unwrap(); + std::fs::write(store.memory_path(), b"{corrupt").unwrap(); + assert!(store.memory_list(20, 0).is_err()); + assert!(store.memory_capture("must not replace it", None).is_err()); + assert_eq!(std::fs::read(store.memory_path()).unwrap(), b"{corrupt"); +} + +#[test] +fn actual_concurrent_transactions_retain_every_successful_capture() { + let root = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + let workers: Vec<_> = (0..12) + .map(|i| { + let root = root.path().to_owned(); + std::thread::spawn(move || { + let store = AgentStore::open(&root).unwrap(); + store.memory_capture(&format!("Public decision {i}"), None) + }) + }) + .collect(); + let successes: Vec<_> = workers + .into_iter() + .filter_map(|task| task.join().unwrap().ok()) + .collect(); + assert!(!successes.is_empty()); + let actual = store.memory_list(100, 0).unwrap(); + assert_eq!(actual.len(), successes.len()); + for memory in successes { + assert_eq!( + store.memory_show(&memory.id).unwrap().content, + memory.content + ); + } +} + +#[cfg(unix)] +#[test] +fn symlinked_memory_or_learning_paths_cannot_escape_the_explicit_scope() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let store = AgentStore::open(root.path()).unwrap(); + std::fs::create_dir(root.path().join("terraphim")).unwrap(); + symlink(outside.path(), root.path().join("terraphim/evolution")).unwrap(); + assert!(store.memory_capture("must stay inside", None).is_err()); + assert_eq!(std::fs::read_dir(outside.path()).unwrap().count(), 0); +} diff --git a/crates/terraphim_mcp_server/Cargo.toml b/crates/terraphim_mcp_server/Cargo.toml index c4acd83c..21d8e427 100644 --- a/crates/terraphim_mcp_server/Cargo.toml +++ b/crates/terraphim_mcp_server/Cargo.toml @@ -21,6 +21,9 @@ base64 = "0.21" clap = { version = "4.5", features = ["derive"] } rmcp = { version = "0.9.0", features = ["server", "transport-sse-server", "transport-io"] } serde_json = { workspace = true } +serde = { workspace = true } +schemars = "1.2" +terraphim_agent = { path = "../terraphim_agent", version = "1.21.22", default-features = false } # Pinned exactly, not `^0.8.4` (Refs #293, #257). # diff --git a/crates/terraphim_mcp_server/src/agent_tools.rs b/crates/terraphim_mcp_server/src/agent_tools.rs new file mode 100644 index 00000000..c601025b --- /dev/null +++ b/crates/terraphim_mcp_server/src/agent_tools.rs @@ -0,0 +1,505 @@ +//! Typed local-agent MCP contracts. Storage is supplied explicitly by the host. +use rmcp::model::{CallToolResult, Content, ErrorData, Tool, ToolAnnotations}; +use schemars::{JsonSchema, schema_for}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; +use std::{collections::HashMap, sync::Arc}; +use terraphim_agent::agent_store::{AgentStore, LearningRecord, SavedLearning}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct CaptureMemory { + /// Authored UTF-8 content, at most 1,000,000 bytes. Never replaced by a tag. + #[schemars(length(min = 1, max = 1_000_000))] + content: String, + /// Optional source label, at most 512 UTF-8 bytes. + #[schemars(length(min = 1, max = 512))] + provenance_tag: Option, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct Id { + /// Exact returned memory ID; prefixes and filesystem paths are refused. + #[schemars(length(min = 1, max = 200))] + id: String, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct Page { + #[serde(default = "default_limit")] + #[schemars(range(min = 1, max = 100))] + limit: usize, + #[serde(default)] + offset: usize, +} +fn default_limit() -> usize { + 20 +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct CaptureLearning { + /// Observed command, not a command for the server to execute. At most 65,536 bytes. + #[schemars(length(min = 1, max = 65_536))] + command: String, + /// Actual observed stderr/error text, at most 524,288 UTF-8 bytes. + #[schemars(length(min = 1, max = 524_288))] + error_output: String, + /// Observed non-zero exit code. Capture never runs the command to obtain it. + exit_code: i32, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct QueryLearning { + /// Case-insensitive text query over stored failures and explicit corrections. + #[schemars(length(min = 1, max = 4_096))] + query: String, + #[serde(default = "default_limit")] + #[schemars(range(min = 1, max = 100))] + limit: usize, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct CorrectLearning { + #[schemars(length(min = 1, max = 200))] + id: String, + /// Annotation only; this does not create a ToolPreference rule or activate a hook. + #[schemars(length(min = 1, max = 65_536))] + correction: String, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct AddCorrection { + #[schemars(length(min = 1, max = 65_536))] + original: String, + #[schemars(length(min = 1, max = 65_536))] + corrected: String, + #[serde(default)] + #[schemars(length(max = 65_536))] + context: String, +} +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct Empty {} + +#[derive(Serialize, JsonSchema)] +struct Scope { + /// Canonical explicit --agent-data-dir, shared with the CLI environment override. + root: String, + memory_file: String, + learning_directory: String, +} +#[derive(Serialize, JsonSchema)] +struct Envelope { + storage: Scope, + data: T, +} +#[derive(Serialize, Deserialize, JsonSchema)] +struct MemoryView { + id: String, + content: String, + tags: Vec, + created_at: String, + item_type: String, + importance: String, + last_accessed: Option, + access_count: u32, + associations: HashMap, +} +#[derive(Serialize, JsonSchema)] +struct MemoryPage { + items: Vec, + count: usize, +} +#[derive(Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum LearningView { + Failure { + id: String, + path: String, + command: String, + error_output: String, + exit_code: i32, + correction: Option, + }, + Correction { + id: String, + path: String, + original: String, + corrected: String, + context: String, + }, +} +impl From for LearningView { + fn from(saved: SavedLearning) -> Self { + let path = saved.path.to_string_lossy().into_owned(); + match saved.record { + LearningRecord::Failure(record) => Self::Failure { + id: record.id, + path, + command: record.command, + error_output: record.error_output, + exit_code: record.exit_code, + correction: record.correction, + }, + LearningRecord::Correction(record) => Self::Correction { + id: record.id, + path, + original: record.original, + corrected: record.corrected, + context: record.context_description, + }, + } + } +} +#[derive(Serialize, JsonSchema)] +struct LearningPage { + items: Vec, + count: usize, +} +#[derive(Serialize, JsonSchema)] +struct Mapping { + pattern: String, + concept_id: u64, + replacement: String, + display: String, +} +#[derive(Serialize, JsonSchema)] +struct Compilation { + path: String, + count: usize, + mappings: Vec, + activated: bool, +} +#[derive(Serialize, JsonSchema)] +struct Note { + path: String, + markdown: String, +} +#[derive(Serialize, JsonSchema)] +struct Export { + directory: String, + files: Vec, + registered: bool, +} + +fn tool( + name: &'static str, + description: &'static str, + read_only: bool, + destructive: bool, +) -> Tool { + Tool { + name: name.into(), + title: None, + description: Some(description.into()), + input_schema: Arc::new( + schema_for!(I) + .as_object() + .expect("object input schema") + .clone(), + ), + output_schema: Some(Arc::new( + schema_for!(Envelope) + .as_object() + .expect("object output schema") + .clone(), + )), + annotations: Some(ToolAnnotations { + read_only_hint: Some(read_only), + destructive_hint: Some(destructive), + idempotent_hint: Some(read_only), + open_world_hint: Some(false), + ..Default::default() + }), + icons: None, + meta: None, + } +} +pub(crate) fn tools() -> Vec { + vec![ + tool::( + "agent_memory_capture", + "Save authored content and optional provenance tag to the explicit local store, then return actual readback. Existing memory semantics retain the newest 100 short-term items. No role/project isolation is inferred from cwd.", + false, + true, + ), + tool::( + "agent_memory_show", + "Read one exact saved memory ID from the configured local store. Unknown IDs return an error.", + true, + false, + ), + tool::( + "agent_memory_list", + "List actual saved memory records with stable bounded pagination. This is local store enumeration, not semantic retrieval.", + true, + false, + ), + tool::( + "agent_learning_capture", + "Record a supplied failed-command observation with secret redaction. Does not execute the command or activate a hook.", + false, + false, + ), + tool::( + "agent_learning_query", + "Query actual stored failures and explicit corrections by case-insensitive text. This does not execute commands or rank by a role graph.", + true, + false, + ), + tool::( + "agent_learning_correct", + "Annotate one exact failed-learning ID with a correction. Annotation alone creates no compiled rule.", + false, + true, + ), + tool::( + "agent_correction_add", + "Create an explicit ToolPreference correction using existing Terraphim types. Compile and KG export are separate calls; no hook activation.", + false, + false, + ), + tool::( + "agent_correction_compile", + "Compile existing ToolPreference corrections into a fresh thesaurus JSON under the configured root. Return actual mappings and saved path. No active file is replaced and no hook is activated.", + false, + false, + ), + tool::( + "agent_correction_export_kg", + "Export ToolPreference corrections into fresh Logseq-compatible Markdown notes containing synonyms::. Return actual saved notes. This does not register a graph in a role or activate a hook.", + false, + false, + ), + ] +} +fn parse(arguments: Value) -> Result { + serde_json::from_value(arguments).map_err(|error| { + ErrorData::invalid_params(format!("Invalid agent tool arguments: {error}"), None) + }) +} +fn response(store: &AgentStore, data: T) -> anyhow::Result { + Ok(CallToolResult::structured(serde_json::to_value( + Envelope { + storage: Scope { + root: store.root().to_string_lossy().into_owned(), + memory_file: store.memory_path().to_string_lossy().into_owned(), + learning_directory: store + .root() + .join("terraphim/learnings") + .to_string_lossy() + .into_owned(), + }, + data, + }, + )?)) +} +fn memory_view(value: impl Serialize) -> anyhow::Result { + Ok(serde_json::from_value(serde_json::to_value(value)?)?) +} + +/// Cancellation is honoured before acquiring the real store transaction. Once +/// a bounded synchronous commit starts, cancellation is not a rollback. +pub(crate) async fn call_guarded( + store: &tokio::sync::Mutex, + cancellation: &tokio_util::sync::CancellationToken, + name: &str, + arguments: Value, +) -> Result { + let guard = tokio::select! { + biased; + _ = cancellation.cancelled() => return Err(ErrorData::internal_error("Request cancelled before storage operation", None)), + guard = store.lock() => guard, + }; + if cancellation.is_cancelled() { + return Err(ErrorData::internal_error( + "Request cancelled before storage operation", + None, + )); + } + call(&guard, name, arguments) +} + +pub(crate) fn call( + store: &AgentStore, + name: &str, + arguments: Value, +) -> Result { + // Parse first: serde rejects wrong types, integer overflow and unknown fields. + // Application failures retain isError=true and never emit success-shaped data. + let result: anyhow::Result = match name { + "agent_memory_capture" => { + let args: CaptureMemory = parse(arguments)?; + (|| { + response( + store, + memory_view( + store.memory_capture(&args.content, args.provenance_tag.as_deref())?, + )?, + ) + })() + } + "agent_memory_show" => { + let args: Id = parse(arguments)?; + (|| response(store, memory_view(store.memory_show(&args.id)?)?))() + } + "agent_memory_list" => { + let args: Page = parse(arguments)?; + (|| { + let items = store + .memory_list(args.limit, args.offset)? + .into_iter() + .map(memory_view) + .collect::>>()?; + let count = items.len(); + response(store, MemoryPage { items, count }) + })() + } + "agent_learning_capture" => { + let args: CaptureLearning = parse(arguments)?; + (|| { + response( + store, + LearningView::from(store.learning_capture( + &args.command, + &args.error_output, + args.exit_code, + )?), + ) + })() + } + "agent_learning_query" => { + let args: QueryLearning = parse(arguments)?; + (|| { + let items: Vec<_> = store + .learning_query(&args.query, args.limit)? + .into_iter() + .map(LearningView::from) + .collect(); + let count = items.len(); + response(store, LearningPage { items, count }) + })() + } + "agent_learning_correct" => { + let args: CorrectLearning = parse(arguments)?; + (|| { + response( + store, + LearningView::from(store.learning_correct(&args.id, &args.correction)?), + ) + })() + } + "agent_correction_add" => { + let args: AddCorrection = parse(arguments)?; + (|| { + response( + store, + LearningView::from(store.correction_add( + &args.original, + &args.corrected, + &args.context, + )?), + ) + })() + } + "agent_correction_compile" => { + let _: Empty = parse(arguments)?; + (|| { + let compiled = store.correction_compile()?; + let mut mappings: Vec<_> = (&compiled.thesaurus) + .into_iter() + .map(|(pattern, term)| Mapping { + pattern: pattern.to_string(), + concept_id: term.id, + replacement: term.value.to_string(), + display: term.display().to_owned(), + }) + .collect(); + mappings.sort_by(|a, b| a.pattern.cmp(&b.pattern)); + response( + store, + Compilation { + path: compiled.path.to_string_lossy().into_owned(), + count: compiled.mappings, + mappings, + activated: false, + }, + ) + })() + } + "agent_correction_export_kg" => { + let _: Empty = parse(arguments)?; + (|| { + let exported = store.correction_export_kg()?; + response( + store, + Export { + directory: exported.directory.to_string_lossy().into_owned(), + files: exported + .files + .into_iter() + .map(|note| Note { + path: note.path.to_string_lossy().into_owned(), + markdown: note.markdown, + }) + .collect(), + registered: false, + }, + ) + })() + } + _ => return Err(ErrorData::invalid_params("Unknown agent tool", None)), + }; + Ok(match result { + Ok(result) => result, + Err(error) => CallToolResult::error(vec![Content::text( + json!({"error":format!("{error:#}"),"storage_root":store.root()}).to_string(), + )]), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn cancellation_while_queued_never_writes_but_cancellation_after_commit_is_not_rollback() + { + let root = tempfile::tempdir().unwrap(); + let store = tokio::sync::Mutex::new(AgentStore::open(root.path()).unwrap()); + let held = store.lock().await; + let cancellation = tokio_util::sync::CancellationToken::new(); + let operation = call_guarded( + &store, + &cancellation, + "agent_memory_capture", + json!({"content":"cancelled pending input"}), + ); + tokio::pin!(operation); + // The real mutex is held: polling the actual operation cannot mutate storage. + tokio::select! { + biased; + result = &mut operation => panic!("queued operation unexpectedly finished: {result:?}"), + _ = tokio::task::yield_now() => (), + } + cancellation.cancel(); + assert!(operation.await.is_err()); + assert!(!held.memory_path().exists()); + drop(held); + let committed_token = tokio_util::sync::CancellationToken::new(); + let committed = call_guarded( + &store, + &committed_token, + "agent_memory_capture", + json!({"content":"acknowledged persisted output"}), + ) + .await + .unwrap(); + committed_token.cancel(); + let id = committed.structured_content.as_ref().unwrap()["data"]["id"] + .as_str() + .unwrap(); + let persisted = store.lock().await.memory_show(id).unwrap(); + assert_eq!(persisted.content, "acknowledged persisted output"); + } +} diff --git a/crates/terraphim_mcp_server/src/lib.rs b/crates/terraphim_mcp_server/src/lib.rs index 084cb64d..3a526ee0 100644 --- a/crates/terraphim_mcp_server/src/lib.rs +++ b/crates/terraphim_mcp_server/src/lib.rs @@ -42,6 +42,7 @@ use terraphim_types::{ use thiserror::Error; use tracing::{error, info}; +mod agent_tools; pub mod resource_mapper; use crate::resource_mapper::TerraphimResourceMapper; @@ -408,6 +409,7 @@ pub struct McpService { kg_scorer: Option>, /// Optional persistent frecency tracker (LMDB-backed) for access-frequency scoring. frecency: Option, + agent_store: Option>>, } impl McpService { @@ -433,9 +435,17 @@ impl McpService { autocomplete_index: Arc::new(tokio::sync::RwLock::new(AutocompleteState::default())), kg_scorer: None, frecency, + agent_store: None, } } + /// Enable local agent tools with an already validated explicit storage root. + /// Default service instances expose no agent memory/learning tools. + pub fn with_agent_store(mut self, store: terraphim_agent::agent_store::AgentStore) -> Self { + self.agent_store = Some(Arc::new(tokio::sync::Mutex::new(store))); + self + } + /// Read-only access to the optional frecency tracker (for consumers that /// score file access frequency, e.g. `terraphim_find_files`). pub fn frecency(&self) -> Option<&SharedFrecency> { @@ -2340,7 +2350,7 @@ impl ServerHandler for McpService { }); let is_all_terms_connected_map = is_all_terms_connected_schema.as_object().unwrap().clone(); - let tools = vec![ + let mut tools = vec![ Tool { name: "search".into(), title: Some("Search Knowledge Graph".into()), @@ -2788,6 +2798,9 @@ impl ServerHandler for McpService { } ]; + if self.agent_store.is_some() { + tools.extend(agent_tools::tools()); + } tracing::debug!("Created {} tools", tools.len()); tracing::debug!("First tool name: {:?}", tools.first().map(|t| &t.name)); @@ -2809,6 +2822,17 @@ impl ServerHandler for McpService { request: CallToolRequestParam, _context: RequestContext, ) -> Result { + if request.name.starts_with("agent_") { + let store = self.agent_store.as_ref().ok_or_else(|| ErrorData::invalid_params( + "Agent tools are disabled; start the local stdio server with an explicit --agent-data-dir", None))?; + return agent_tools::call_guarded( + store, + &_context.ct, + &request.name, + serde_json::Value::Object(request.arguments.unwrap_or_default()), + ) + .await; + } match request.name.as_ref() { "search" => { let arguments = request.arguments.unwrap_or_default(); diff --git a/crates/terraphim_mcp_server/src/main.rs b/crates/terraphim_mcp_server/src/main.rs index 2f03a36b..e9bdd605 100644 --- a/crates/terraphim_mcp_server/src/main.rs +++ b/crates/terraphim_mcp_server/src/main.rs @@ -77,6 +77,11 @@ struct Args { /// such as Zed context servers, that cannot pass the project root. #[arg(long, value_name = "DIR")] config_dir: Option, + + /// Explicit absolute local root for agent memory/learning tools (stdio only). + /// Overrides no environment or role; omit to keep all agent tools disabled. + #[arg(long, value_name = "DIR", conflicts_with = "sse")] + agent_data_dir: Option, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)] @@ -283,6 +288,9 @@ async fn main() -> Result<()> { }; let mut service = McpService::new(Arc::clone(&config_state)); + if let Some(root) = args.agent_data_dir.as_ref() { + service = service.with_agent_store(terraphim_agent::agent_store::AgentStore::open(root)?); + } if let Some(kg_path) = kg_path { info!("Building KgPathScorer from KG path: {:?}", kg_path); diff --git a/crates/terraphim_mcp_server/tests/test_mcp_agent_contract.rs b/crates/terraphim_mcp_server/tests/test_mcp_agent_contract.rs new file mode 100644 index 00000000..a6c575f1 --- /dev/null +++ b/crates/terraphim_mcp_server/tests/test_mcp_agent_contract.rs @@ -0,0 +1,644 @@ +//! Real stdio, CLI, filesystem and failed-command evidence for local agent tools. +use anyhow::{Context, Result, ensure}; +use rmcp::{ + RoleClient, + model::CallToolRequestParam, + service::{RunningService, ServiceExt}, + transport::TokioChildProcess, +}; +use serde_json::{Value, json}; +use std::{ + fs, + path::{Path, PathBuf}, + time::Duration, +}; +use tokio::process::Command; + +type Client = RunningService; +struct Fixture { + root: tempfile::TempDir, + project: PathBuf, + settings: PathBuf, + store: PathBuf, +} +impl Fixture { + fn new() -> Result { + let root = tempfile::tempdir()?; + let project = root.path().join("public project"); + let settings = root.path().join("settings"); + let store = root.path().join("explicit agent store"); + fs::create_dir_all(project.join(".terraphim"))?; + fs::create_dir_all(&settings)?; + let mut roles = serde_json::Map::new(); + for name in ["Default", "Engineer", "System Operator"] { + roles.insert(name.into(), json!({"name":name,"relevance_function":"title-scorer","terraphim_it":false,"theme":"lumen","kg":null,"haystacks":[],"extra":{}})); + } + fs::write( + project.join(".terraphim/config.json"), + serde_json::to_vec_pretty( + &json!({"selected_role":"Default","default_role":"Default","roles":roles}), + )?, + )?; + fs::write( + settings.join("settings.toml"), + format!( + "server_hostname = \"127.0.0.1:8000\"\napi_endpoint = \"http://localhost:8000/api\"\ninitialized = true\ndefault_data_path = {}\n[profiles.memory]\ntype = \"memory\"\n", + serde_json::to_string(&root.path().join("data"))? + ), + )?; + Ok(Self { + root, + project, + settings, + store, + }) + } + fn command(&self, binary: &Path) -> Result { + let mut command = Command::new(binary); + command + .current_dir(&self.project) + .kill_on_drop(true) + .env_clear() + .env("PATH", std::env::var_os("PATH").context("PATH")?) + .env("TERRAPHIM_SETTINGS_PATH", &self.settings) + .env("TERRAPHIM_DATA_PATH", self.root.path().join("data")); + if let Some(home) = std::env::var_os("HOME") { + command.env("HOME", home); + } + Ok(command) + } + fn server_command(&self, store: Option<&Path>) -> Result { + let binary = std::env::var_os("TERRAPHIM_MCP_SERVER_BIN") + .map(PathBuf::from) + .unwrap_or_else(|| env!("CARGO_BIN_EXE_terraphim_mcp_server").into()); + let mut command = self.command(&binary)?; + command + .args(["--profile", "server", "--config-dir"]) + .arg(&self.project); + if let Some(store) = store { + command.arg("--agent-data-dir").arg(store); + } + Ok(command) + } + async fn start(&self, store: Option<&Path>) -> Result { + Ok(tokio::time::timeout( + Duration::from_secs(20), + ().serve(TokioChildProcess::new(self.server_command(store)?)?), + ) + .await??) + } + async fn cli_output(&self, args: &[&str]) -> Result { + // CI builds both workspace binaries before this test; missing CLI is a failure. + let binary = std::env::var_os("TERRAPHIM_AGENT_TEST_BIN") + .map(PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_BIN_EXE_terraphim_mcp_server")) + .parent() + .unwrap() + .join("terraphim-agent") + }); + ensure!( + binary.is_file(), + "Build terraphim-agent before the MCP interoperability tests" + ); + let output = self + .command(&binary)? + .env("TERRAPHIM_DEFAULT_DATA_PATH", &self.store) + .args(["--format", "json"]) + .args(args) + .output() + .await?; + Ok(output) + } + async fn cli(&self, args: &[&str]) -> Result { + let output = self.cli_output(args).await?; + ensure!( + output.status.success(), + "CLI failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(serde_json::from_slice(&output.stdout)?) + } +} +fn evidence(label: &str, value: &Value) -> Result<()> { + if let Some(directory) = std::env::var_os("TERRAPHIM_MCP_TEST_EVIDENCE") { + fs::create_dir_all(&directory)?; + fs::write( + Path::new(&directory).join(format!("{label}.json")), + serde_json::to_vec_pretty(value)?, + )?; + } + Ok(()) +} +async fn call( + client: &rmcp::service::Peer, + name: &str, + args: Value, + label: &str, +) -> Result { + let value = serde_json::to_value( + tokio::time::timeout( + Duration::from_secs(20), + client.call_tool(CallToolRequestParam { + name: name.to_owned().into(), + arguments: args.as_object().cloned(), + }), + ) + .await??, + )?; + evidence(label, &json!({"tool":name,"arguments":args,"result":value}))?; + Ok(value) +} +fn data(value: &Value) -> &Value { + assert_ne!(value["isError"], true, "{value}"); + &value["structuredContent"]["data"] +} + +#[tokio::test] +async fn tools_are_opt_in_with_complete_contracts_and_stdio_scope() -> Result<()> { + let f = Fixture::new()?; + let disabled = f.start(None).await?; + let tools = disabled.list_all_tools().await?; + assert!(!tools.iter().any(|tool| tool.name.starts_with("agent_"))); + assert!( + disabled + .call_tool(CallToolRequestParam { + name: "agent_memory_capture".into(), + arguments: Some( + json!({"content":"must not save"}) + .as_object() + .unwrap() + .clone() + ) + }) + .await + .is_err() + ); + assert!(!f.store.exists()); + disabled.cancel().await?; + let enabled = f.start(Some(&f.store)).await?; + let tools = enabled.list_all_tools().await?; + let agent: Vec<_> = tools + .iter() + .filter(|tool| tool.name.starts_with("agent_")) + .collect(); + assert_eq!(agent.len(), 9); + for tool in &agent { + assert_eq!(tool.input_schema["type"], "object"); + assert_eq!(tool.input_schema["additionalProperties"], false); + assert_eq!(tool.output_schema.as_ref().unwrap()["type"], "object"); + let annotations = tool.annotations.as_ref().unwrap(); + assert_eq!(annotations.open_world_hint, Some(false)); + assert_eq!( + annotations.read_only_hint, + Some(matches!( + tool.name.as_ref(), + "agent_memory_show" | "agent_memory_list" | "agent_learning_query" + )) + ); + } + evidence("agent-tools-list", &serde_json::to_value(agent)?)?; + enabled.cancel().await?; + let relative = f + .server_command(Some(Path::new("relative")))? + .output() + .await?; + assert!(!relative.status.success()); + assert!(String::from_utf8_lossy(&relative.stderr).contains("must be absolute")); + let sse = f + .server_command(Some(&f.store))? + .arg("--sse") + .output() + .await?; + assert!(!sse.status.success()); + assert!(String::from_utf8_lossy(&sse.stderr).contains("cannot be used")); + Ok(()) +} + +#[tokio::test] +async fn authored_memory_survives_restart_and_cli_interoperability() -> Result<()> { + let f = Fixture::new()?; + let client = f.start(Some(&f.store)).await?; + let initial = call( + &client, + "agent_memory_list", + json!({}), + "agent-memory-empty", + ) + .await?; + assert_eq!(data(&initial)["count"], 0); + let content = "Risk requires a Decision: approve a second supplier. Café — Ελληνικά.\n"; + let captured = call( + &client, + "agent_memory_capture", + json!({"content":content,"provenance_tag":"public-project-brief"}), + "agent-memory-captured", + ) + .await?; + let id = data(&captured)["id"] + .as_str() + .context("saved id")? + .to_owned(); + assert_eq!(data(&captured)["content"], content); + assert_eq!( + data(&captured)["tags"], + json!(["provenance:public-project-brief"]) + ); + let path = PathBuf::from( + captured["structuredContent"]["storage"]["memory_file"] + .as_str() + .unwrap(), + ); + assert!(path.starts_with(f.store.canonicalize()?)); + let persisted: Value = serde_json::from_slice(&fs::read(&path)?)?; + assert!(persisted.to_string().contains(&id)); + evidence("agent-memory-persisted", &persisted)?; + client.cancel().await?; + let restarted = f.start(Some(&f.store)).await?; + let shown = call( + &restarted, + "agent_memory_show", + json!({"id":id}), + "agent-memory-restarted", + ) + .await?; + assert_eq!(data(&shown), data(&captured)); + let cli = f.cli(&["memory", "show", &id, "--json"]).await?; + assert_eq!(cli["memory_item"]["content"], content); + evidence("agent-memory-cli-readback", &cli)?; + let from_cli = f + .cli(&[ + "memory", + "capture", + "--content", + "CLI authored second memory", + "--provenance-tag", + "cli-proof", + ]) + .await?; + let cli_id = from_cli["memory_id"].as_str().context("CLI saved id")?; + let shown = call( + &restarted, + "agent_memory_show", + json!({"id":cli_id}), + "agent-memory-from-cli", + ) + .await?; + assert_eq!(data(&shown)["content"], "CLI authored second memory"); + assert_eq!(data(&shown)["tags"], json!(["provenance:cli-proof"])); + let other = f.root.path().join("other store"); + let isolated = f.start(Some(&other)).await?; + let missing = call( + &isolated, + "agent_memory_show", + json!({"id":id}), + "agent-memory-other-root", + ) + .await?; + assert_eq!(missing["isError"], true); + assert!(!other.join("terraphim/evolution/cli-agent.json").exists()); + isolated.cancel().await?; + restarted.cancel().await?; + Ok(()) +} + +#[tokio::test] +async fn actual_failure_annotation_explicit_rule_compile_and_synonym_export() -> Result<()> { + let f = Fixture::new()?; + let client = f.start(Some(&f.store)).await?; + let failed = Command::new("printf").arg("%Z").output().await?; + assert_eq!(failed.status.code(), Some(1)); + let error = String::from_utf8(failed.stderr)?; + let corrected = Command::new("printf") + .args(["%s\n", "reviewed correction"]) + .output() + .await?; + assert!(corrected.status.success()); + assert_eq!(corrected.stdout, b"reviewed correction\n"); + evidence( + "agent-learning-actual-execution", + &json!({"failed":{"argv":["printf","%Z"],"exit":failed.status.code(),"stderr":error},"corrected":{"argv":["printf","%s\n","reviewed correction"],"exit":corrected.status.code(),"stdout":String::from_utf8(corrected.stdout)?}}), + )?; + let captured = call( + &client, + "agent_learning_capture", + json!({"command":"printf '%Z'","error_output":error,"exit_code":1}), + "agent-learning-captured", + ) + .await?; + let id = data(&captured)["id"].as_str().context("learning id")?; + assert_eq!( + data(&captured)["error_output"].as_str().unwrap().trim(), + error.trim() + ); + assert_eq!(data(&captured)["exit_code"], 1); + let corrected_command = "printf '%s\\n' 'reviewed correction'"; + let annotated = call( + &client, + "agent_learning_correct", + json!({"id":id,"correction":corrected_command}), + "agent-learning-annotated", + ) + .await?; + assert_eq!(data(&annotated)["correction"], corrected_command); + let empty = call( + &client, + "agent_correction_compile", + json!({}), + "agent-learning-annotation-compile", + ) + .await?; + assert_eq!(data(&empty)["count"], 0); + assert_eq!(data(&empty)["activated"], false); + let added=call(&client,"agent_correction_add",json!({"original":"printf '%Z'","corrected":corrected_command,"context":"Actual public fixture"}),"agent-learning-explicit-rule").await?; + assert_eq!(data(&added)["kind"], "correction"); + let compiled = call( + &client, + "agent_correction_compile", + json!({}), + "agent-learning-compiled", + ) + .await?; + assert_eq!(data(&compiled)["count"], 1); + assert_eq!(data(&compiled)["activated"], false); + assert_eq!(data(&compiled)["mappings"][0]["display"], corrected_command); + let compiled_path = PathBuf::from(data(&compiled)["path"].as_str().unwrap()); + let disk: terraphim_types::Thesaurus = serde_json::from_slice(&fs::read(&compiled_path)?)?; + assert_eq!(disk.len(), 1); + assert_eq!( + disk.get(&terraphim_types::NormalizedTermValue::from("printf '%Z'")) + .unwrap() + .display(), + corrected_command + ); + let exported = call( + &client, + "agent_correction_export_kg", + json!({}), + "agent-learning-exported", + ) + .await?; + assert_eq!(data(&exported)["registered"], false); + let notes = data(&exported)["files"].as_array().unwrap(); + assert_eq!(notes.len(), 1); + let note = fs::read_to_string(notes[0]["path"].as_str().unwrap())?; + assert_eq!(note, notes[0]["markdown"].as_str().unwrap()); + assert!(note.contains("synonyms:: printf '%Z'")); + evidence( + "agent-learning-saved-artefacts", + &json!({"compiled":serde_json::from_slice::(&fs::read(compiled_path)?)?,"note":note,"failureMarkdown":fs::read_to_string(data(&annotated)["path"].as_str().unwrap())?}), + )?; + client.cancel().await?; + let restarted = f.start(Some(&f.store)).await?; + let queried = call( + &restarted, + "agent_learning_query", + json!({"query":"PRINTF","limit":10}), + "agent-learning-restarted-query", + ) + .await?; + assert_eq!(data(&queried)["count"], 2); + // CLI reads the same existing-format learning records under the explicit override. + let cli = f.cli_output(&["learn", "query", "printf"]).await?; + assert!(cli.status.success()); + let cli_text = String::from_utf8(cli.stdout)?; + evidence( + "agent-learning-cli-query", + &json!({"exit":cli.status.code(),"stdout":cli_text}), + )?; + assert!(cli_text.contains("[cmd] printf '%Z' (exit: 1)")); + assert!(cli_text.contains(corrected_command)); + let lock_path = f.store.join("terraphim/learnings/.write.lock"); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(lock_path)?; + lock.lock()?; + let failure_path = PathBuf::from(data(&annotated)["path"].as_str().unwrap()); + let before = fs::read(&failure_path)?; + for (index, args) in [ + vec![ + "learn", + "capture", + "must-not-capture", + "--error", + "observed error", + ], + vec!["learn", "correct", id, "--correction", "must-not-replace"], + vec![ + "learn", + "correction", + "add", + "--original", + "must-not-add", + "--corrected", + "replacement", + "--correction-type", + "tool-preference", + ], + ] + .iter() + .enumerate() + { + let refused = f.cli_output(args).await?; + assert!(!refused.status.success()); + let stderr = String::from_utf8(refused.stderr)?; + assert!(stderr.contains("busy"), "{stderr}"); + evidence( + &format!("agent-learning-cli-busy-{index}"), + &json!({"argv":args,"exit":refused.status.code(),"stderr":stderr}), + )?; + } + assert_eq!(fs::read(&failure_path)?, before); + drop(lock); + let edited = f + .cli_output(&[ + "learn", + "correct", + id, + "--correction", + "CLI reviewed annotation", + ]) + .await?; + assert!(edited.status.success()); + let actual = call( + &restarted, + "agent_learning_query", + json!({"query":"printf"}), + "agent-learning-cli-mutation-readback", + ) + .await?; + assert!( + data(&actual)["items"] + .as_array() + .unwrap() + .iter() + .any(|item| item["id"] == id && item["correction"] == "CLI reviewed annotation") + ); + assert!(!f.store.join(".claude").exists()); + assert!(!f.store.join(".opencode").exists()); + restarted.cancel().await?; + Ok(()) +} + +#[tokio::test] +async fn invalid_arguments_corrupt_storage_and_busy_writer_never_acknowledge_success() -> Result<()> +{ + let f = Fixture::new()?; + let client = f.start(Some(&f.store)).await?; + for args in [ + json!({"provenance_tag":"tag-only"}), + json!({"content":"valid","unexpected":true}), + ] { + assert!( + client + .call_tool(CallToolRequestParam { + name: "agent_memory_capture".into(), + arguments: args.as_object().cloned() + }) + .await + .is_err() + ); + } + for args in [ + json!({"limit":-1}), + serde_json::from_str("{\"limit\":18446744073709551616}")?, + json!({"limit":1.5}), + ] { + assert!( + client + .call_tool(CallToolRequestParam { + name: "agent_memory_list".into(), + arguments: args.as_object().cloned() + }) + .await + .is_err() + ); + } + for (index, (name, args)) in [ + ("agent_memory_capture", json!({"content":" \n"})), + ("agent_memory_show", json!({"id":"../escape"})), + ("agent_memory_show", json!({"id":"missing"})), + ("agent_memory_list", json!({"limit":0})), + ( + "agent_learning_capture", + json!({"command":"printf ok","error_output":"none","exit_code":0}), + ), + ] + .into_iter() + .enumerate() + { + assert_eq!( + call( + &client, + name, + args, + &format!("agent-rejected-input-{index}") + ) + .await?["isError"], + true + ); + } + let capture = call( + &client, + "agent_memory_capture", + json!({"content":"Preserve this actual note"}), + "agent-before-storage-failures", + ) + .await?; + let path = PathBuf::from( + capture["structuredContent"]["storage"]["memory_file"] + .as_str() + .unwrap(), + ); + let before = fs::read(&path)?; + let lock_path = path.parent().unwrap().join("cli-agent.lock"); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(lock_path)?; + lock.lock()?; + let busy = call( + &client, + "agent_memory_capture", + json!({"content":"Must not acknowledge under lock"}), + "agent-busy-refused", + ) + .await?; + assert_eq!(busy["isError"], true); + let cli_busy = f + .cli_output(&[ + "memory", + "capture", + "--content", + "Must not override held memory lock", + ]) + .await?; + assert!(!cli_busy.status.success()); + assert!(String::from_utf8_lossy(&cli_busy.stderr).contains("busy")); + evidence( + "agent-memory-cli-busy", + &json!({"exit":cli_busy.status.code(),"stdout":String::from_utf8(cli_busy.stdout)?,"stderr":String::from_utf8(cli_busy.stderr)?}), + )?; + assert_eq!(fs::read(&path)?, before); + drop(lock); + fs::write(&path, b"{ corrupt retained")?; + let refused = call( + &client, + "agent_memory_capture", + json!({"content":"Must not erase corruption"}), + "agent-corrupt-refused", + ) + .await?; + assert_eq!(refused["isError"], true); + assert_eq!(fs::read(&path)?, b"{ corrupt retained"); + client.cancel().await?; + Ok(()) +} + +#[tokio::test] +async fn concurrent_mcp_captures_keep_every_acknowledged_record() -> Result<()> { + let f = Fixture::new()?; + let client = f.start(Some(&f.store)).await?; + let mut calls = tokio::task::JoinSet::new(); + for i in 0..8 { + let peer = client.peer().clone(); + calls.spawn(async move { + call( + &peer, + "agent_memory_capture", + json!({"content":format!("Concurrent public note {i}")}), + &format!("agent-concurrent-{i}"), + ) + .await + }); + } + let mut results = Vec::new(); + while let Some(result) = calls.join_next().await { + results.push(result?); + } + let mut ids = Vec::new(); + for result in results { + let value = result?; + ids.push(data(&value)["id"].as_str().unwrap().to_owned()); + } + let listed = call( + &client, + "agent_memory_list", + json!({"limit":100}), + "agent-concurrent-list", + ) + .await?; + assert_eq!(data(&listed)["count"], 8); + for id in ids { + assert!( + data(&listed)["items"] + .as_array() + .unwrap() + .iter() + .any(|item| item["id"] == id) + ); + } + client.cancel().await?; + Ok(()) +} diff --git a/docs/mcp/agent-tools.md b/docs/mcp/agent-tools.md new file mode 100644 index 00000000..c6daa557 --- /dev/null +++ b/docs/mcp/agent-tools.md @@ -0,0 +1,117 @@ +# Local agent memory and learning over MCP + +The existing `terraphim_mcp_server` exposes agent tools only when started with an +explicit absolute data directory. Check that your installed server's `--help` +includes `--agent-data-dir`; older releases expose graph tools only. + +```sh +terraphim_mcp_server --profile server --config-dir /absolute/project \ + --agent-data-dir /absolute/agent-data +``` + +This option is available for local stdio transport. Combining it with `--sse` +is refused. Omitting it exposes no `agent_*` tools and does not choose a default +agent store. The graph's role/configuration and the agent store are separate: +a working directory or selected role does not isolate memories. + +Each successful result contains `structuredContent.storage` with the effective +canonical root, memory file and learning directory, followed by typed `data`. +The same envelope is described by `outputSchema` in `tools/list`. Inputs reject +unknown fields, incorrect types and out-of-range limits. Persistence failures +return `isError: true`; no success result is generated from input alone. + +## Tools + +| Tool | Input | Saved result or effect | +| --- | --- | --- | +| `agent_memory_capture` | `content`, optional `provenance_tag` | Saved item with ID, exact authored content and source tag, read back from disk. Existing memory semantics retain the latest 100 short-term items. | +| `agent_memory_show` | Exact `id` | One saved item; an unknown ID is an error. | +| `agent_memory_list` | Optional `limit` (1–100), `offset` | Stable paginated saved records, newest first. This is enumeration, not semantic retrieval. | +| `agent_learning_capture` | Observed `command`, `error_output`, non-zero `exit_code` | A saved failure observation using the existing redaction and learning types. It does not execute the command. | +| `agent_learning_query` | `query`, optional `limit` (1–100) | Case-insensitive text matches over actual failure and correction records. | +| `agent_learning_correct` | Exact failure `id`, `correction` | An annotation on that failure; no compiled rule is created. | +| `agent_correction_add` | `original`, `corrected`, optional `context` | An explicit `ToolPreference` correction record. | +| `agent_correction_compile` | `{}` | Fresh thesaurus JSON, actual mapping count and mappings; `activated` is always `false`. | +| `agent_correction_export_kg` | `{}` | Fresh Markdown concept files containing `synonyms::`; actual paths and saved text; `registered` is always `false`. | + +For example, call `agent_memory_capture` with: + +```json +{"content":"Risk requires a Decision: approve a second supplier.","provenance_tag":"public-project-brief"} +``` + +Pass the returned `data.id` to `agent_memory_show`. A provenance tag is a source +label stored on the item; it does not replace the content or independently prove +its provenance. + +For an observed failed `printf '%Z'`, retain the real exit code and stderr, and +execute any corrected command separately. Capture the observation, then annotate +its returned ID with `printf '%s\n' 'reviewed correction'`. Compilation still +contains zero rules if no explicit correction exists. Calling +`agent_correction_add` with those original/corrected strings creates the separate +rule; compilation then returns its actual mapping. Exported notes use the same +Logseq-compatible `synonyms::` syntax as other Terraphim knowledge graphs. + +Before committing a new learning or correction, the server parses the staged +Markdown with the existing Terraphim parser and checks that its redacted fields +round-trip exactly. Unrepresentable fields (for example multiline command or +annotation text, embedded Markdown delimiters, or a command chain transformed +by the existing capture parser) return an error before a destination record is +changed. Supply the single failing command and inspect the saved result. + +Compilation and export create new generation directories. They do not overwrite +an active hook configuration, register a role or activate rewriting. Originals +containing commas/newlines and replacements containing newlines cannot be +represented exactly by the current one-line comma-separated synonym syntax, so +KG export refuses them; JSON compilation remains available. + +## Storage and CLI interoperability + +The memory snapshot is `/terraphim/evolution/cli-agent.json`; learning +Markdown is in `/terraphim/learnings/`. The explicit MCP flag takes +precedence over any inherited `TERRAPHIM_DEFAULT_DATA_PATH`; it never falls back +to that variable. To read the same files from the CLI, deliberately select the +same root: + +```sh +TERRAPHIM_DEFAULT_DATA_PATH=/absolute/agent-data terraphim-agent \ + --format json memory show RETURNED_MEMORY_ID +TERRAPHIM_DEFAULT_DATA_PATH=/absolute/agent-data terraphim-agent learn query printf +``` + +The CLI preserves its existing environment-variable precedence over its OS-local +configuration root. Without that override, changing directory does not change +its memory store. `learn query` currently emits human-readable text even with +the global JSON output option. + +Updated CLI and MCP writers share advisory locks; a busy writer receives an +error and may retry. Older CLI versions do not cooperate with these new locks: +stop those writers before sharing a store with this server. Static symlinks, +traversal, corrupt records and failed readback are refused. These checks do not +isolate storage from another process that already controls the same OS account. + +Cancellation is honoured before the store transaction starts, including while +waiting behind another MCP operation. Once a synchronous transaction commits, +a cancellation or connection loss is not rollback; use list/query/readback to +reconcile the store before retrying a non-idempotent mutation. Memory capture and +learning annotation are marked destructive in tool metadata because retention +can remove old short-term items and annotation replaces the selected field. +The server never provides arbitrary command execution or hook installation. + +## Verification + +The real stdio contract test launches the server and CLI against temporary +stores, checks restart/readback, actual files, schema and read/write metadata, +invalid inputs, held-lock refusals and concurrent requests. It separately runs +a failed and corrected `printf`, then checks zero-to-one compilation and the +exported notes. The dispatcher cancellation test uses the real store mutex and +filesystem. Run after building both binaries: + +```sh +cargo build -p terraphim_agent -p terraphim_mcp_server +cargo test -p terraphim_agent --test agent_store_contract --test agent_learning_store_contract +cargo test -p terraphim_mcp_server --test test_mcp_agent_contract +``` + +`TERRAPHIM_MCP_SERVER_BIN` and `TERRAPHIM_AGENT_TEST_BIN` can select downloaded +released binaries for the same protocol/interoperability tests. diff --git a/pkgbuilds/terraphim-clients-bin/.SRCINFO b/pkgbuilds/terraphim-clients-bin/.SRCINFO index a6dbbe21..e0019b8b 100644 --- a/pkgbuilds/terraphim-clients-bin/.SRCINFO +++ b/pkgbuilds/terraphim-clients-bin/.SRCINFO @@ -1,6 +1,6 @@ pkgbase = terraphim-clients-bin pkgdesc = Terraphim AI command-line clients (prebuilt static MUSL binaries) - pkgver = 1.21.20 + pkgver = 1.21.22 pkgrel = 1 url = https://terraphim.ai arch = x86_64 @@ -9,14 +9,14 @@ pkgbase = terraphim-clients-bin license = MIT options = !strip options = !debug - source_x86_64 = terraphim-agent-1.21.20-x86_64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.20/terraphim-agent-1.21.20-x86_64-unknown-linux-musl.tar.gz - source_x86_64 = terraphim-grep-1.21.20-x86_64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.20/terraphim-grep-1.21.20-x86_64-unknown-linux-musl.tar.gz - sha256sums_x86_64 = 242c3a72849d3e905c3af6359002fb7473c900df2f4c87ccdebe80c0c538e4ce - sha256sums_x86_64 = 6930c335bb41caab26eb8b1f7e5ef2bc19ef6dfabd17abeb584f1672f54c1d0d - source_aarch64 = terraphim-agent-1.21.20-aarch64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.20/terraphim-agent-1.21.20-aarch64-unknown-linux-musl.tar.gz - source_aarch64 = terraphim-grep-1.21.20-aarch64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.20/terraphim-grep-1.21.20-aarch64-unknown-linux-musl.tar.gz - sha256sums_aarch64 = cf09f161ff42180308509d8e9b43b66c94bedd3801987cd1112b38b3718b5789 - sha256sums_aarch64 = 320690a756083cccac28c11d918f549e7a7e1fbcbce31250388d6638087215d8 + source_x86_64 = terraphim-agent-1.21.22-x86_64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.22/terraphim-agent-1.21.22-x86_64-unknown-linux-musl.tar.gz + source_x86_64 = terraphim-grep-1.21.22-x86_64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.22/terraphim-grep-1.21.22-x86_64-unknown-linux-musl.tar.gz + sha256sums_x86_64 = 3afe1ba64dfd7f19214f2171f8cba467f674f77317c0e05af44408627eb288e1 + sha256sums_x86_64 = 34dfcf06f76e6737c9c4f208b01bdb5638f18d6c7325efa8f7be00efbf85f41f + source_aarch64 = terraphim-agent-1.21.22-aarch64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.22/terraphim-agent-1.21.22-aarch64-unknown-linux-musl.tar.gz + source_aarch64 = terraphim-grep-1.21.22-aarch64-unknown-linux-musl.tar.gz::https://github.com/terraphim/terraphim-clients/releases/download/v1.21.22/terraphim-grep-1.21.22-aarch64-unknown-linux-musl.tar.gz + sha256sums_aarch64 = 36468a1ab4b3ee2f6781b623582a9044dc31768796a1a02b4d806fcc39dd6013 + sha256sums_aarch64 = 3c050c09668775dc709c098ed6bb4ac85493fb69f658848ff1ce4a62824480a9 pkgname = terraphim-agent-bin pkgdesc = Terraphim AI agent CLI (prebuilt static MUSL binary) diff --git a/pkgbuilds/terraphim-clients-bin/PKGBUILD b/pkgbuilds/terraphim-clients-bin/PKGBUILD index f293556a..b8f47d3e 100644 --- a/pkgbuilds/terraphim-clients-bin/PKGBUILD +++ b/pkgbuilds/terraphim-clients-bin/PKGBUILD @@ -18,7 +18,7 @@ pkgbase=terraphim-clients-bin pkgname=(terraphim-agent-bin terraphim-grep-bin) -pkgver=1.21.20 +pkgver=1.21.22 pkgrel=1 pkgdesc='Terraphim AI command-line clients (prebuilt static MUSL binaries)' arch=('x86_64' 'aarch64') @@ -45,12 +45,12 @@ source_aarch64=( # SHA256SUMS asset for v${pkgver}. tests/test_pkgbuild_release_contract.py pins # the same values so a version bump cannot silently keep stale digests. sha256sums_x86_64=( - '242c3a72849d3e905c3af6359002fb7473c900df2f4c87ccdebe80c0c538e4ce' - '6930c335bb41caab26eb8b1f7e5ef2bc19ef6dfabd17abeb584f1672f54c1d0d' + '3afe1ba64dfd7f19214f2171f8cba467f674f77317c0e05af44408627eb288e1' + '34dfcf06f76e6737c9c4f208b01bdb5638f18d6c7325efa8f7be00efbf85f41f' ) sha256sums_aarch64=( - 'cf09f161ff42180308509d8e9b43b66c94bedd3801987cd1112b38b3718b5789' - '320690a756083cccac28c11d918f549e7a7e1fbcbce31250388d6638087215d8' + '36468a1ab4b3ee2f6781b623582a9044dc31768796a1a02b4d806fcc39dd6013' + '3c050c09668775dc709c098ed6bb4ac85493fb69f658848ff1ce4a62824480a9' ) # Prebuilt, already-stripped static binaries: never strip or emit debug info. diff --git a/pkgbuilds/terraphim-clients-bin/release-stage.json b/pkgbuilds/terraphim-clients-bin/release-stage.json new file mode 100644 index 00000000..624d1420 --- /dev/null +++ b/pkgbuilds/terraphim-clients-bin/release-stage.json @@ -0,0 +1,60 @@ +{ + "version": "1.21.22", + "source_sha": "88a09d8b11e4eab550c6f7410f86faee7cc41ecc", + "control_sha": "e1c7ad8b722cb3dfe93865bd780f0a74fbfbf206", + "correlation_id": "mcp-agent-corrected-20261015", + "run_id": 38718874679, + "run_attempt": 1, + "artifact_id": 11967065225, + "artifact_name": "client-release-stage-1.21.22-88a09d8b11e4eab550c6f7410f86faee7cc41ecc", + "artifact_digest": "sha256:f519e850a628af2483c30642ea5c4ec4dadc7500ff33d59c053a1cae2b373443", + "files": { + "BINARY_SHA256SUMS": "a912c2629063554b0561d2fcce51ecb6bf59e911e1c2852c2294d2f551abc083", + "SHA256SUMS": "0b15eac4f016e71a68c1ee3516f80c08609f757c2e790d623d408099a0b13f9f", + "canonical-binaries/terraphim-agent-aarch64-unknown-linux-musl": "cff778c1fe964dee21b309fd7cd50c4dbec7c405c86d95f953d3dd0ed9505e01", + "canonical-binaries/terraphim-agent-x86_64-unknown-linux-gnu": "bba40143c3607fd786f4404f1e916424e9ad5b30abaf443bde9be45c9f32cf9e", + "canonical-binaries/terraphim-agent-x86_64-unknown-linux-musl": "0680d81956350eb26ac916c426b7b5e60776e587aba624d36f5fab95f385fb09", + "canonical-binaries/terraphim-cli-aarch64-unknown-linux-musl": "d1d6a0013e1abf0682cf3281cafb5a69ce799172a0e7e89b674228add526d0ca", + "canonical-binaries/terraphim-cli-x86_64-unknown-linux-gnu": "76a78f41cdc2838c74319b8e1888424d3a05b58faa5bacfeec88a6f5453293e6", + "canonical-binaries/terraphim-cli-x86_64-unknown-linux-musl": "ef432ee4d09ef229d789df637acf8e9dbe84d00ef2c4c745d702f6b3670aabcb", + "canonical-binaries/terraphim-grep-aarch64-unknown-linux-musl": "0e8ab4fe383ff552403fa62a4e804a1168704ee66ef0c0f8848dc3d4164a567d", + "canonical-binaries/terraphim-grep-x86_64-unknown-linux-gnu": "6ca3c12e9039cda7a2928fa881f69e1f664630bacae78d71b86fd6938ac0b30f", + "canonical-binaries/terraphim-grep-x86_64-unknown-linux-musl": "98ccd40fa5aa8906ad8dd95d323902089b58ef011bbbfa7596c6191576a4b402", + "expected-assets.txt": "77c5a75998bee496f00d4a4d2ece26eb1a5fd6c783efb4442cd7250879a8ead0", + "managed-release-assets/DEB-RPM-PACKAGES-UNSIGNED-STAGE-ONLY.txt": "7efadf1fc51413c730c27362cc6e11c3e971302f5e3a066c68577d0870fd0461", + "managed-release-assets/terraphim-agent-1.21.22-1.aarch64.rpm": "f91bd0bd624b9d0e529bd93d7288b38bc8939dc4a5e93fa2365b96fc7770ceb7", + "managed-release-assets/terraphim-agent-1.21.22-1.x86_64.rpm": "3e93125ac35e1c2aba43772ee7bc2d4b815b3e943c76880709069c852f41202e", + "managed-release-assets/terraphim-agent_1.21.22-1_amd64.deb": "023822004d8000115149c08152a31dff0e00553cc0c13e3f272872834f9c33d2", + "managed-release-assets/terraphim-agent_1.21.22-1_arm64.deb": "7868b8464d2e624e782164e62269d27e52242b11a8965ad7722a382f5d8b0d38", + "managed-release-assets/terraphim-clients-1.21.22-aarch64-unknown-linux-musl.package-sha256sums.txt": "761ca420e460f69d74941c46427f8578744812ac0459477a616bedaf118681e3", + "managed-release-assets/terraphim-clients-1.21.22-x86_64-unknown-linux-musl.package-sha256sums.txt": "4cce07b46546dc115dd906878a8ae266824c97d59ad450d1ef6d1442ba90a880", + "managed-release-assets/terraphim-grep-1.21.22-1.aarch64.rpm": "ab8c280a2d6f486945d7ed5ed3c50d3bce7e2927ae607e2a1d0b5f888a9e4e9c", + "managed-release-assets/terraphim-grep-1.21.22-1.x86_64.rpm": "19aa1041a30ea4a69f85a78eba206e647c6db84bdb5587406dc598048387b091", + "managed-release-assets/terraphim-grep_1.21.22-1_amd64.deb": "88a9103e4fe6abbbbba41aaa1cd705ae233fb30c3f4251a7ea04b12cc159fa88", + "managed-release-assets/terraphim-grep_1.21.22-1_arm64.deb": "b6f9ca71b49c421d6297804673ac758dab110e0f92f48fd99e06f2b134f53f51", + "manifests/terraphim-agent.v1.candidate.json": "f0dc96ebcc2156431ea87e335c5c39e4a1bd812653cb1b7288bc1d0170637fcd", + "manifests/terraphim-agent.v2.candidate.json": "e3c758e8044bd9d25474c3c7ee554d7d57de62de7d49685d11cf75cd0c07b761", + "manifests/terraphim-cli.v1.candidate.json": "5593cae7bb54d515787cbaef40458e2403c98292f488ae1c98c94017b452a7df", + "manifests/terraphim-cli.v2.candidate.json": "e7d4b82c8c2d3ca0a761b30c83a24b1d060dcf8f762c19e3cf79f84c245d3771", + "manifests/terraphim-grep.v1.candidate.json": "f1594ee4823d0c7a0839a9410bfb7be83251d166caa37437144b36c93a4187e6", + "manifests/terraphim-grep.v2.candidate.json": "2ce49ac572f2f4dd7ab788784ffbc37f18c2502313ade7a02924fe513f77d670", + "provenance.json": "22068750daea5d4f12fc75dcf2930aa4a02bb3a45318d48aae07ea3fa8b38a76", + "release-assets/terraphim-agent-1.21.22-aarch64-apple-darwin.tar.gz": "a280e2087ecca2e9772b27f8d0f18aad99c49b469391e02f9a10e9907dad9dee", + "release-assets/terraphim-agent-1.21.22-aarch64-unknown-linux-musl.tar.gz": "36468a1ab4b3ee2f6781b623582a9044dc31768796a1a02b4d806fcc39dd6013", + "release-assets/terraphim-agent-1.21.22-universal-apple-darwin.tar.gz": "72990af3f6288b605db9aa5280a76e51cb39ccbb1115f5600eec80f31eb20314", + "release-assets/terraphim-agent-1.21.22-x86_64-apple-darwin.tar.gz": "8babb1b70eddaa29cfd5b4b4cf93e4cef5341fe2051ccf0eac241dfe154100f1", + "release-assets/terraphim-agent-1.21.22-x86_64-unknown-linux-gnu.tar.gz": "2f903a46c07a53ac2ded0a89c25a81b627b4b5f1599800b1b65ba4aff6192977", + "release-assets/terraphim-agent-1.21.22-x86_64-unknown-linux-musl.tar.gz": "3afe1ba64dfd7f19214f2171f8cba467f674f77317c0e05af44408627eb288e1", + "release-assets/terraphim-cli-1.21.22-aarch64-apple-darwin.tar.gz": "1fa39e97712b2f8708b7e256a90c74ad8f1109f84c4845cc499db005e3ebbc20", + "release-assets/terraphim-cli-1.21.22-aarch64-unknown-linux-musl.tar.gz": "03ae99997bd877a10ee5ebda9d73870c74f203b2c95678c8be4774cc9133b7cd", + "release-assets/terraphim-cli-1.21.22-x86_64-apple-darwin.tar.gz": "4f9fe45dca07cd969560232a2a73dfad8b6956a8940db64317bd8777f6a8fdb2", + "release-assets/terraphim-cli-1.21.22-x86_64-unknown-linux-gnu.tar.gz": "97007c5bc82528f49c0dadfd1cb4a4c3f1174dd66b413a1708ecbc072a0fde18", + "release-assets/terraphim-cli-1.21.22-x86_64-unknown-linux-musl.tar.gz": "b22de835e1f1e2bb825756c2ccf9cdcecc42a00eb37b40e189f95e5c425c6fff", + "release-assets/terraphim-grep-1.21.22-aarch64-apple-darwin.tar.gz": "1ed5de453965d57da0b475e7ed149eb2a6e060004090bae973d340065cb5e0eb", + "release-assets/terraphim-grep-1.21.22-aarch64-unknown-linux-musl.tar.gz": "3c050c09668775dc709c098ed6bb4ac85493fb69f658848ff1ce4a62824480a9", + "release-assets/terraphim-grep-1.21.22-universal-apple-darwin.tar.gz": "6d8c38c5d44e5b4c2b94649f325a13028d365b020cd2df7fade04d998f5450a5", + "release-assets/terraphim-grep-1.21.22-x86_64-apple-darwin.tar.gz": "2fb73bc87bf930d7f842ae7a93da10f899e1e881220765c807190e6cee676c1e", + "release-assets/terraphim-grep-1.21.22-x86_64-unknown-linux-gnu.tar.gz": "f7a9daca8f9bec4d54046fcff6f393587d155be1ba8c873e93b580d25b26e796", + "release-assets/terraphim-grep-1.21.22-x86_64-unknown-linux-musl.tar.gz": "34dfcf06f76e6737c9c4f208b01bdb5638f18d6c7325efa8f7be00efbf85f41f" + } +} diff --git a/scripts/prepare-pkgbuild-stage.py b/scripts/prepare-pkgbuild-stage.py new file mode 100644 index 00000000..17a76790 --- /dev/null +++ b/scripts/prepare-pkgbuild-stage.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Qualify an immutable release stage before populating makepkg's source cache.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import platform +from pathlib import Path +import re +import shutil +import sys +import tempfile + +REPOSITORY = "terraphim/terraphim-clients" +WORKFLOW = ".github/workflows/release-binaries.yml" +ROOT = Path(__file__).resolve().parents[1] +CONTRACT_KEYS = { + "version", + "source_sha", + "control_sha", + "correlation_id", + "run_id", + "run_attempt", + "artifact_id", + "artifact_name", + "artifact_digest", + "files", +} + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, f"duplicate JSON field: {key}") + result[key] = value + return result + + +def read_json(path: Path): + require( + path.is_file() and not path.is_symlink(), "JSON input must be a regular file" + ) + require(path.stat().st_size <= 4 * 1024 * 1024, "JSON input exceeds bound") + return json.loads(path.read_text(), object_pairs_hook=unique_object) + + +def contract(path: Path) -> dict: + value = read_json(path) + require( + isinstance(value, dict) and set(value) == CONTRACT_KEYS, + "unexpected contract fields", + ) + for name in ("source_sha", "control_sha"): + require( + isinstance(value[name], str) + and re.fullmatch(r"[0-9a-f]{40}", value[name]) is not None, + f"invalid {name}", + ) + require( + isinstance(value["version"], str) + and re.fullmatch(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)", value["version"]) + is not None, + "invalid version", + ) + require( + isinstance(value["correlation_id"], str) + and re.fullmatch(r"[A-Za-z0-9._:/@+-]{1,128}", value["correlation_id"]) + is not None, + "invalid correlation", + ) + for name in ("run_id", "run_attempt", "artifact_id"): + require(type(value[name]) is int and value[name] > 0, f"invalid {name}") + require( + value["artifact_name"] + == f"client-release-stage-{value['version']}-{value['source_sha']}", + "artifact identity mismatch", + ) + require( + isinstance(value["artifact_digest"], str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", value["artifact_digest"]) is not None, + "invalid artifact digest", + ) + require( + isinstance(value["files"], dict) and 17 < len(value["files"]) <= 128, + "invalid stage inventory", + ) + for name, digest in value["files"].items(): + require( + isinstance(name, str) + and re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", name) + is not None + and all(p not in (".", "..") for p in name.split("/")), + "unsafe inventory path", + ) + require( + isinstance(digest, str) + and re.fullmatch(r"[0-9a-f]{64}", digest) is not None, + "invalid file digest", + ) + return value + + +def release_mode(value: dict, status: int, release: dict) -> str: + if status == 404: + return "stage" + require(status == 200, "release lookup must return 200 or 404") + require( + release.get("tag_name") == f"v{value['version']}" + and release.get("draft") is False + and release.get("prerelease") is False, + "release is not the intended stable release", + ) + assets = release.get("assets") + require(isinstance(assets, list), "missing release assets") + names = [a.get("name") for a in assets] + require(len(names) == len(set(names)), "duplicate release asset") + expected = {Path(p).name for p in value["files"] if p.startswith("release-assets/")} + require( + expected <= set(names), "published release is incomplete; no stage fallback" + ) + for asset in assets: + if asset.get("name") in expected: + require( + asset.get("browser_download_url") + == f"https://github.com/{REPOSITORY}/releases/download/v{value['version']}/{asset['name']}", + "unexpected public asset URL", + ) + return "public" + + +def validate_metadata(value: dict, run: dict, artifact: dict) -> None: + require( + type(run.get("id")) is int + and type(run.get("run_attempt")) is int + and run.get("id") == value["run_id"] + and run.get("run_attempt") == value["run_attempt"], + "run/attempt mismatch", + ) + require( + run.get("path") == WORKFLOW + and run.get("event") == "workflow_dispatch" + and run.get("head_branch") == "main", + "wrong producer workflow", + ) + require( + run.get("status") == "completed" and run.get("conclusion") == "success", + "producer has not succeeded", + ) + require( + run.get("head_sha") == value["control_sha"], "producer control source mismatch" + ) + repo, head_repo = run.get("repository", {}), run.get("head_repository", {}) + require( + repo.get("full_name") == REPOSITORY + and head_repo.get("full_name") == REPOSITORY + and type(repo.get("id")) is int + and repo.get("id") == head_repo.get("id"), + "wrong producer repository", + ) + require( + artifact.get("id") == value["artifact_id"] + and artifact.get("name") == value["artifact_name"] + and artifact.get("digest") == value["artifact_digest"] + and artifact.get("expired") is False, + "artifact identity/digest mismatch", + ) + parent = artifact.get("workflow_run", {}) + require( + parent.get("id") == value["run_id"] + and parent.get("head_sha") == value["control_sha"] + and parent.get("head_branch") == "main" + and parent.get("repository_id") == repo["id"] + and parent.get("head_repository_id") == repo["id"], + "artifact producer mismatch", + ) + + +def prepare( + value: dict, stage: Path, run: dict, artifact: dict, cache: Path, arch: str +) -> dict: + validate_metadata(value, run, artifact) + native = {"arm64": "aarch64"}.get(platform.machine(), platform.machine()) + require( + arch in ("x86_64", "aarch64") and arch == native, + "unsupported or mismatched native architecture", + ) + require( + stage.is_dir() + and not stage.is_symlink() + and stage.name == value["artifact_name"], + "stage directory mismatch", + ) + actual = {} + actual_dirs = set() + expected_dirs = { + parent.as_posix() + for name in value["files"] + for parent in Path(name).parents + if parent != Path(".") + } + for base, dirs, files in os.walk(stage): + actual_dirs.update( + (Path(base) / name).relative_to(stage).as_posix() for name in dirs + ) + for name in dirs + files: + p = Path(base) / name + require(not p.is_symlink(), "symlink in stage") + for name in files: + p = Path(base) / name + require(p.is_file(), "non-regular stage file") + require(p.stat().st_size <= 512 * 1024 * 1024, "stage file exceeds bound") + with p.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + actual[p.relative_to(stage).as_posix()] = digest + require( + actual_dirs == expected_dirs, + "stage directory set differs from reviewed inventory", + ) + require( + actual == value["files"], "stage files differ from exact reviewed inventory" + ) + spec = importlib.util.spec_from_file_location( + "promotion_stage", ROOT / "scripts/validate-promotion-stage.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory() as tmp: + module.validate_stage( + value["version"], + stage, + value["source_sha"], + value["correlation_id"], + Path(tmp) / "plan.tsv", + ) + require( + cache.is_dir() and not cache.is_symlink(), + "cache directory must exist and be real", + ) + selected = [ + f"{binary}-{value['version']}-{arch}-unknown-linux-musl.tar.gz" + for binary in ("terraphim-agent", "terraphim-grep") + ] + require( + not any(cache.glob("*.tar.gz")) and not any(cache.glob("*.zip")), + "source cache is not empty", + ) + result = {} + for name in selected: + source = stage / "release-assets" / name + with (cache / name).open("xb") as dst, source.open("rb") as src: + shutil.copyfileobj(src, dst) + with (cache / name).open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + require( + digest == value["files"][f"release-assets/{name}"], + "cached archive failed readback", + ) + result[name] = digest + return { + "version": value["version"], + "source_sha": value["source_sha"], + "run_id": value["run_id"], + "run_attempt": value["run_attempt"], + "architecture": arch, + "cached": result, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("identity") + metadata = commands.add_parser("metadata") + metadata.add_argument("run", type=Path) + metadata.add_argument("artifact", type=Path) + mode = commands.add_parser("mode") + mode.add_argument("status", type=int) + mode.add_argument("release", type=Path) + prep = commands.add_parser("prepare") + prep.add_argument("stage", type=Path) + prep.add_argument("run", type=Path) + prep.add_argument("artifact", type=Path) + prep.add_argument("cache", type=Path) + prep.add_argument("arch") + args = parser.parse_args() + value = contract(args.contract) + if args.command == "identity": + for key in ("version", "run_id", "artifact_id", "artifact_name"): + print(f"{key}={value[key]}") + elif args.command == "metadata": + validate_metadata(value, read_json(args.run), read_json(args.artifact)) + print("producer_metadata=verified") + elif args.command == "mode": + print("mode=" + release_mode(value, args.status, read_json(args.release))) + else: + print( + json.dumps( + prepare( + value, + args.stage, + read_json(args.run), + read_json(args.artifact), + args.cache, + args.arch, + ), + sort_keys=True, + indent=2, + ) + ) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, KeyError, TypeError, json.JSONDecodeError) as error: + sys.exit(f"ERROR: {error}") diff --git a/tests/test_pkgbuild_release_contract.py b/tests/test_pkgbuild_release_contract.py index 86f17eb1..6c3deb2c 100644 --- a/tests/test_pkgbuild_release_contract.py +++ b/tests/test_pkgbuild_release_contract.py @@ -32,7 +32,7 @@ # publication. During a release window (workspace bumped, signed assets not # yet sealed) this contract intentionally fails until the follow-up PKGBUILD # bump lands. -PKGVER = "1.21.20" +PKGVER = "1.21.22" REPO = "terraphim/terraphim-clients" RELEASE_BASE = f"https://github.com/{REPO}/releases/download/v{PKGVER}" @@ -41,21 +41,21 @@ "x86_64": { "terraphim-agent": ( f"terraphim-agent-{PKGVER}-x86_64-unknown-linux-musl.tar.gz", - "242c3a72849d3e905c3af6359002fb7473c900df2f4c87ccdebe80c0c538e4ce", + "3afe1ba64dfd7f19214f2171f8cba467f674f77317c0e05af44408627eb288e1", ), "terraphim-grep": ( f"terraphim-grep-{PKGVER}-x86_64-unknown-linux-musl.tar.gz", - "6930c335bb41caab26eb8b1f7e5ef2bc19ef6dfabd17abeb584f1672f54c1d0d", + "34dfcf06f76e6737c9c4f208b01bdb5638f18d6c7325efa8f7be00efbf85f41f", ), }, "aarch64": { "terraphim-agent": ( f"terraphim-agent-{PKGVER}-aarch64-unknown-linux-musl.tar.gz", - "cf09f161ff42180308509d8e9b43b66c94bedd3801987cd1112b38b3718b5789", + "36468a1ab4b3ee2f6781b623582a9044dc31768796a1a02b4d806fcc39dd6013", ), "terraphim-grep": ( f"terraphim-grep-{PKGVER}-aarch64-unknown-linux-musl.tar.gz", - "320690a756083cccac28c11d918f549e7a7e1fbcbce31250388d6638087215d8", + "3c050c09668775dc709c098ed6bb4ac85493fb69f658848ff1ce4a62824480a9", ), }, } diff --git a/tests/test_pkgbuild_stage.py b/tests/test_pkgbuild_stage.py new file mode 100644 index 00000000..e7113e47 --- /dev/null +++ b/tests/test_pkgbuild_stage.py @@ -0,0 +1,213 @@ +"""Exercise the actual sealed stage and saved GitHub metadata; no substitute binaries.""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import platform +import shutil +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "pkgbuild_stage", ROOT / "scripts/prepare-pkgbuild-stage.py" +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class SealedStageTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Missing real artefacts is a failure, never a skipped acceptance test. + cls.stage = Path(os.environ["PKGBUILD_STAGE_TEST_DIR"]) + cls.run_metadata = MODULE.read_json(Path(os.environ["PKGBUILD_STAGE_RUN_JSON"])) + cls.artifact = MODULE.read_json( + Path(os.environ["PKGBUILD_STAGE_ARTIFACT_JSON"]) + ) + cls.value = MODULE.contract( + ROOT / "pkgbuilds/terraphim-clients-bin/release-stage.json" + ) + cls.arch = {"arm64": "aarch64"}.get(platform.machine(), platform.machine()) + MODULE.validate_metadata(cls.value, cls.run_metadata, cls.artifact) + + def prepare( + self, cache, *, value=None, run=None, artifact=None, stage=None, arch=None + ): + return MODULE.prepare( + value or self.value, + stage or self.stage, + run or self.run_metadata, + artifact or self.artifact, + cache, + arch or self.arch, + ) + + def test_actual_stage_two_archives_exact_readback(self): + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) + receipt = self.prepare(cache) + self.assertEqual(len(receipt["cached"]), 2) + self.assertEqual(receipt["architecture"], self.arch) + for name, digest in receipt["cached"].items(): + self.assertEqual( + (cache / name).read_bytes(), + (self.stage / "release-assets" / name).read_bytes(), + ) + self.assertEqual(digest, self.value["files"]["release-assets/" + name]) + + def test_reject_changed_run_identity_before_cache_write(self): + mutations = [ + ("path", ".github/workflows/ci.yml"), + ("head_sha", "f" * 40), + ("head_branch", "unreviewed"), + ("run_attempt", self.run_metadata["run_attempt"] + 1), + ("id", self.run_metadata["id"] + 1), + ("event", "pull_request"), + ("status", "in_progress"), + ("conclusion", "failure"), + ("repository", {"full_name": "other/repo", "id": 1}), + ("head_repository", {"full_name": "other/repo", "id": 1}), + ] + with tempfile.TemporaryDirectory() as tmp: + for key, changed in mutations: + with self.subTest(field=key): + run = copy.deepcopy(self.run_metadata) + run[key] = changed + with self.assertRaises(ValueError): + self.prepare(Path(tmp), run=run) + self.assertEqual(list(Path(tmp).iterdir()), []) + + def test_reject_changed_artifact_identity_before_cache_write(self): + mutations = [ + ("id", self.artifact["id"] + 1), + ("name", "other"), + ("digest", "sha256:" + "f" * 64), + ("expired", True), + ("workflow_run", {"id": 1}), + ] + with tempfile.TemporaryDirectory() as tmp: + for key, changed in mutations: + with self.subTest(field=key): + artifact = copy.deepcopy(self.artifact) + artifact[key] = changed + with self.assertRaises(ValueError): + self.prepare(Path(tmp), artifact=artifact) + self.assertEqual(list(Path(tmp).iterdir()), []) + + def test_reject_wrong_native_architecture(self): + with tempfile.TemporaryDirectory() as tmp: + for arch in ("x86_64" if self.arch == "aarch64" else "aarch64", "i686"): + with self.assertRaises(ValueError): + self.prepare(Path(tmp), arch=arch) + self.assertEqual(list(Path(tmp).iterdir()), []) + + def copied_stage(self, root): + # Callers create roots beside the stage so Arch container mount boundaries + # cannot make hardlinks fail with EXDEV. Changed test files are unlinked + # before replacement, so the retained original stage is never modified. + return Path( + shutil.copytree( + self.stage, Path(root) / self.stage.name, copy_function=os.link + ) + ) + + def test_reject_missing_extra_corrupt_symlink_and_directory_entries(self): + for kind in ("missing", "extra", "corrupt", "symlink", "directory"): + with ( + self.subTest(kind=kind), + tempfile.TemporaryDirectory(dir=self.stage.parent) as tmp, + ): + stage = self.copied_stage(tmp) + cache = Path(tmp) / "cache" + cache.mkdir() + p = stage / "provenance.json" + if kind == "missing": + p.unlink() + elif kind == "extra": + (stage / "extra").write_text("unexpected") + elif kind == "corrupt": + content = p.read_bytes() + p.unlink() + p.write_bytes(content + b"\n") + elif kind == "symlink": + p.unlink() + p.symlink_to(self.stage / "provenance.json") + else: + (stage / "unexpected-directory").mkdir() + with self.assertRaises(ValueError): + self.prepare(cache, stage=stage) + self.assertEqual(list(cache.iterdir()), []) + + def test_existing_promotion_validator_rejects_wrong_provenance(self): + with tempfile.TemporaryDirectory(dir=self.stage.parent) as tmp: + stage = self.copied_stage(tmp) + cache = Path(tmp) / "cache" + cache.mkdir() + p = stage / "provenance.json" + value = copy.deepcopy(self.value) + provenance = MODULE.read_json(p) + provenance["source_sha"] = "f" * 40 + p.unlink() + p.write_text(json.dumps(provenance)) + value["files"]["provenance.json"] = hashlib.sha256( + p.read_bytes() + ).hexdigest() + with self.assertRaises(SystemExit): + self.prepare(cache, stage=stage, value=value) + self.assertEqual(list(cache.iterdir()), []) + + def test_existing_cache_is_not_replaced(self): + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp) / "retained.tar.gz" + p.write_bytes(b"existing user-selected cache") + with self.assertRaises(ValueError): + self.prepare(Path(tmp)) + self.assertEqual(p.read_bytes(), b"existing user-selected cache") + self.assertEqual(list(Path(tmp).iterdir()), [p]) + + def test_only_absence_allows_stage_mode(self): + self.assertEqual( + MODULE.release_mode(self.value, 404, {"message": "Not Found"}), "stage" + ) + for status in (301, 401, 403, 429, 500): + with self.assertRaises(ValueError): + MODULE.release_mode(self.value, status, {}) + for release in ( + {}, + { + "tag_name": "v" + self.value["version"], + "draft": False, + "prerelease": False, + "assets": [], + }, + ): + with self.assertRaises(ValueError): + MODULE.release_mode(self.value, 200, release) + + def test_contract_parser_rejects_unsafe_or_duplicate_fields(self): + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp) / "contract.json" + p.write_text('{"version":"1","version":"2"}') + with self.assertRaises(ValueError): + MODULE.contract(p) + for field, changed in [ + ("run_attempt", True), + ("artifact_id", -1), + ("artifact_name", "../escape"), + ("files", {"../escape": "f" * 64}), + ]: + value = copy.deepcopy(self.value) + value[field] = changed + p.write_text(json.dumps(value)) + with self.assertRaises(ValueError): + MODULE.contract(p) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_binaries_workflow_contract.py b/tests/test_release_binaries_workflow_contract.py index 30dba3a6..ea83dc5d 100644 --- a/tests/test_release_binaries_workflow_contract.py +++ b/tests/test_release_binaries_workflow_contract.py @@ -176,7 +176,7 @@ def test_preflight_recursively_peels_to_exact_expected_sha(self) -> None: def test_release_uses_checked_in_version_and_never_mutates_source(self) -> None: text = workflow_text() workspace = tomllib.loads((ROOT / "Cargo.toml").read_text()) - self.assertEqual(workspace["workspace"]["package"]["version"], "1.21.20") + self.assertEqual(workspace["workspace"]["package"]["version"], "1.21.22") for forbidden in ( "Set release version", "set_section_version",