From 14ea7be6178ebd3a6939c5f666c4879e3fa7c531 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 6 Jul 2026 00:12:59 -0300 Subject: [PATCH] fix: retire Obsidian wiki layer (opt-in) + isolate headless claude_cli sessions Two coupled fixes for a real install failure: a 2GB/5,921-file Obsidian vault (hung Obsidian) AND ~154k Claude session transcripts / 12GB (froze claude --continue). Both trace to the steward: wiki-absorb looped the claude_cli stack over the whole vault, and every 'claude --print' wrote a session transcript into the caller's project folder. - wiki-absorb is now opt-in (MEMORYMASTER_WIKI_ABSORB=1, default OFF) in the steward-cycle template; the claims DB + FTS5 + vectors + graph + recall IS the scalable LLM-wiki, the markdown vault was a non-scaling duplicate. Steward still runs validate/decay/dedup/tiers. Recall wiki breadcrumbs gated on the same flag so recall never points at an absent vault. - _call_claude_cli runs 'claude --print' from a scratch cwd (~/.memorymaster/claude_cli_scratch, override MEMORYMASTER_CLAUDE_CLI_CWD) so headless calls never flood ~/.claude/projects// again. 184 targeted tests green (incl. updated intent test); ruff clean. --- CHANGELOG.md | 13 ++++++++++ .../hooks/memorymaster-steward-cycle.py | 24 ++++++++++++------- memorymaster/core/llm_provider.py | 17 +++++++++++++ memorymaster/recall/context_hook.py | 6 ++++- pyproject.toml | 2 +- tests/test_wiki_binding.py | 20 ++++++++++++---- 6 files changed, 67 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf43f08..2415208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.4.0] - 2026-07-06 + +**The wiki layer is now opt-in, and headless LLM calls no longer flood the session folder.** Two coupled fixes to a real-world failure: a MemoryMaster install had generated a **2 GB / 5,921-file Obsidian vault** (which hung Obsidian) *and* **~154,000 Claude Code session transcripts / 12 GB** (which froze `claude --continue`). Root cause of both: the steward's periodic `wiki-absorb` ran the `claude_cli` stack in a loop over the whole vault, and every headless `claude --print` call wrote a session transcript into the caller's project folder. + +### Changed + +- **`wiki-absorb` is now opt-in (default OFF)** behind `MEMORYMASTER_WIKI_ABSORB=1`. Research on the LLM-Wiki pattern (Karpathy) confirms the markdown wiki is designed for *a few hundred* pages that fit in a context window — past that you switch to a search DB with lifecycle rules, which **is exactly what MemoryMaster's claims DB already is** (FTS5 + vectors + entity graph + supersession + decay + tiers). The Obsidian markdown layer was a redundant, non-scaling duplicate; the DB + recall is the memory system. The steward still runs validate/decay/dedup/tiers every cycle — it just no longer materializes markdown. Nothing in recall depends on the vault (the only reader, the Closets stream, is already default-OFF). +- **Recall wiki breadcrumbs are gated on the same flag**: `(compiled in [[slug]])` pointers only render when the wiki view is enabled, so recall never points at an archived/absent vault. + +### Fixed + +- **Headless `claude_cli` calls no longer pollute the caller's session folder.** `_call_claude_cli` now runs `claude --print` from a dedicated scratch cwd (`~/.memorymaster/claude_cli_scratch`, override `MEMORYMASTER_CLAUDE_CLI_CWD`) instead of the parent cwd. Previously every steward/wiki/dream LLM call wrote a transcript into `~/.claude/projects//`, which — at automation volume — grew to 154k files / 12 GB and froze `claude --continue`. The scratch folder is never `--continue`d, so it can grow freely and be purged anytime; the prompt is passed via stdin so no project context is lost. + ## [4.3.0] - 2026-07-04 **Reliability + retrieval + a second adversarial audit.** A measured retrieval win, a trustworthy test suite on Windows, and confirmed security/correctness fixes from a round-2 audit whose findings were each independently re-verified before landing. The default recall/ranking path stays byte-identical (the one new ranker is opt-in). diff --git a/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py b/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py index 403f791..78a4ebc 100644 --- a/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py +++ b/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py @@ -64,12 +64,18 @@ except Exception as e: print(f"[MemoryMaster] auto-archive error: {e}", file=sys.stderr) -# Wiki absorb (compiled truth + timeline articles). Inherits the LLM provider -# block above — uses the same OAuth-backed haiku stack as the steward. -try: - from memorymaster.knowledge.wiki_engine import absorb - wiki_path = os.path.join(PROJECT_ROOT, "obsidian-vault", "wiki") - stats = absorb(DB_PATH, wiki_path) - print(f"[MemoryMaster] wiki absorb: {stats}") -except Exception as e: - print(f"[MemoryMaster] wiki absorb error: {e}", file=sys.stderr) +# Wiki layer (Obsidian markdown) — OPT-IN, default OFF (2026-07-06). +# The claims DB + FTS5 + Qdrant + entity graph + recall IS the scalable "LLM +# wiki"; the markdown vault is a redundant, non-scaling duplicate that grows +# unbounded (real install hit 2 GB / 5,921 files, hung Obsidian) and its +# absorb runs the claude_cli stack in a loop — a heavy source of headless +# session churn. Nothing in recall depends on it (the only reader, the Closets +# stream, is default-OFF). Set MEMORYMASTER_WIKI_ABSORB=1 to enable it. +if os.environ.get("MEMORYMASTER_WIKI_ABSORB", "0").strip().lower() in ("1", "true", "yes"): + try: + from memorymaster.knowledge.wiki_engine import absorb + wiki_path = os.path.join(PROJECT_ROOT, "obsidian-vault", "wiki") + stats = absorb(DB_PATH, wiki_path) + print(f"[MemoryMaster] wiki absorb: {stats}") + except Exception as e: + print(f"[MemoryMaster] wiki absorb error: {e}", file=sys.stderr) diff --git a/memorymaster/core/llm_provider.py b/memorymaster/core/llm_provider.py index fe73c14..19ba341 100644 --- a/memorymaster/core/llm_provider.py +++ b/memorymaster/core/llm_provider.py @@ -391,6 +391,23 @@ def _call_claude_cli(prompt: str, text: str) -> str: if sys.platform == "win32": extra_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW + # Isolate the headless session transcript OUT of the caller's project folder. + # `claude --print` writes a session file to ~/.claude/projects// on every + # call. Run from a dedicated scratch dir so memorymaster's own automation + # (steward cycle, wiki, dream) can't flood ~/.claude/projects// + # with thousands of transcripts and freeze `claude --continue` — observed + # live at 154k files / 12 GB, ~1,800 sessions/day. The scratch folder is never + # `--continue`d, so it can grow freely and be purged anytime. Prompt is passed + # via stdin, so no project context is needed from the cwd. + scratch = _env("MEMORYMASTER_CLAUDE_CLI_CWD", "") or os.path.join( + os.path.expanduser("~"), ".memorymaster", "claude_cli_scratch" + ) + try: + os.makedirs(scratch, exist_ok=True) + extra_kwargs["cwd"] = scratch + except OSError: + pass # fall back to the parent cwd rather than fail the LLM call + try: result = subprocess.run( [bin_path, "--print", "--model", model], diff --git a/memorymaster/recall/context_hook.py b/memorymaster/recall/context_hook.py index c7e7030..fede993 100644 --- a/memorymaster/recall/context_hook.py +++ b/memorymaster/recall/context_hook.py @@ -2116,8 +2116,12 @@ def _ranking(score_fn) -> list[int]: if not hasattr(claim, "text"): continue text = claim.text[:300] + # Only surface the "(compiled in [[slug]])" wiki breadcrumb when the + # Obsidian markdown view is explicitly enabled — otherwise it points at + # an archived/absent vault (the wiki layer is opt-in as of 2026-07-06; + # the claims DB + recall is the memory system). See MEMORYMASTER_WIKI_ABSORB. wiki_slug = getattr(claim, "wiki_article", None) - if wiki_slug: + if wiki_slug and os.environ.get("MEMORYMASTER_WIKI_ABSORB", "0").strip().lower() in ("1", "true", "yes"): chunk = f"- {text} (compiled in [[{wiki_slug}]])" else: chunk = f"- {text}" diff --git a/pyproject.toml b/pyproject.toml index 6042e84..18ace7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "memorymaster" -version = "4.3.0" +version = "4.4.0" description = "Production-grade memory reliability system for AI coding agents. Lifecycle-managed claims with citations, conflict detection, steward governance, and MCP integration." license = {text = "MIT"} authors = [{name = "wolverin0"}] diff --git a/tests/test_wiki_binding.py b/tests/test_wiki_binding.py index f204d9d..3de52b3 100644 --- a/tests/test_wiki_binding.py +++ b/tests/test_wiki_binding.py @@ -106,7 +106,11 @@ def test_row_to_claim_reads_wiki_article(tmp_path: Path) -> None: def test_recall_appends_wiki_pointer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Recall formatter should append `(compiled in [[slug]])` for bound claims.""" + """Recall appends `(compiled in [[slug]])` ONLY when the Obsidian wiki view + is enabled. WHY: the markdown wiki is opt-in (default OFF) as of 2026-07-06 — + the claims DB + recall is the memory system — so a bound claim must surface + normally without pointing recall at an absent/archived vault, and the + breadcrumb returns only when MEMORYMASTER_WIKI_ABSORB=1.""" from memorymaster.recall import context_hook from memorymaster.core.models import Claim @@ -142,9 +146,17 @@ def _fake_ctor(db_target: str, workspace_root: Path): # noqa: ARG001 monkeypatch.setattr("memorymaster.core.service.MemoryService", _fake_ctor) - out = context_hook.recall("qdrant", db_path=str(tmp_path / "nope.db"), skip_qdrant=True) - assert "[[qdrant]]" in out - assert "compiled in" in out + # Default (wiki view OFF): claim surfaces, but NO dead [[slug]] pointer. + monkeypatch.delenv("MEMORYMASTER_WIKI_ABSORB", raising=False) + out_off = context_hook.recall("qdrant", db_path=str(tmp_path / "nope.db"), skip_qdrant=True) + assert "Qdrant is deployed" in out_off + assert "[[qdrant]]" not in out_off + + # Wiki view explicitly ON: the breadcrumb returns. + monkeypatch.setenv("MEMORYMASTER_WIKI_ABSORB", "1") + out_on = context_hook.recall("qdrant", db_path=str(tmp_path / "nope.db"), skip_qdrant=True) + assert "[[qdrant]]" in out_on + assert "compiled in" in out_on def test_backfill_bindings_updates_claims_from_frontmatter(tmp_path: Path) -> None: