change sources indexing

.envrc

@@ -0,0 +1 @@
+export KNOWLEDGE_VOLUME_PATH="/Volumes/knowledge_data"

.obsidian/workspace.json

@@ -11,28 +11,13 @@
             "id": "75521912599ac75e",
             "type": "leaf",
             "state": {
-              "type": "pdf",
-              "state": {
-                "file": "sources/ai/pdf/Build Your Own Agentic AI Framework Design Flexible Multi-agent Systems in Python with MCP and A2A integration Gigi Sayfan z-librarysk 1libsk z-lib.pdf"
-              },
-              "icon": "lucide-file-text",
-              "title": "Build Your Own Agentic AI Framework Design Flexible Multi-agent Systems in Python with MCP and A2A integration Gigi Sayfan z-librarysk 1libsk z-lib"
-            }
-          },
-          {
-            "id": "dc48ad1e6618c2ed",
-            "type": "leaf",
-            "state": {
-              "type": "release-notes",
-              "state": {
-                "currentVersion": "1.13.7"
-              },
-              "icon": "lucide-book-up",
-              "title": "Release Notes 1.13.7"
+              "type": "empty",
+              "state": {},
+              "icon": "lucide-file",
+              "title": "New tab"
             }
           }
-        ],
-        "currentTab": 1
+        ]
       }
     ],
     "direction": "vertical"
@@ -87,11 +72,12 @@
               "title": "Bookmarks"
             }
           }
-        ]
+        ],
+        "currentTab": 1
       }
     ],
     "direction": "horizontal",
-    "width": 300
+    "width": 347.3620300292969
   },
   "right": {
     "id": "4c3e39d505734e6e",
@@ -196,43 +182,43 @@
       "bases:Create new base": false
     }
   },
-  "active": "bba1026a76d26d16",
+  "active": "f50e692499989deb",
   "lastOpenFiles": [
+    "docs/architecture.md",
+    "sources/ai/pdf/Build Your Own Agentic AI Framework Design Flexible Multi-agent Systems in Python with MCP and A2A integration Gigi Sayfan z-librarysk 1libsk z-lib.pdf",
+    "tree/specialized_tooling/webassembly/pdf/wasm-performance.md",
+    "tree/specialized_tooling/webassembly/pdf/server-side-webassembly-building-portable-and-secure-multi-language-ap.md",
+    "tree/specialized_tooling/webassembly/pdf",
+    "tree/specialized_tooling/webassembly",
+    "tree/networking/censorship/pdf/uv-lock.md",
+    "tree/ai/pdf/aivideo-spacetimediffusionmodel.md",
+    "kb/__pycache__/extract.cpython-312.pyc",
+    "sources/networking/ebpf/books",
+    "sources/networking/ebpf",
+    "tree/application/architecture/data-midplatform-architecture.md",
+    "tree/application/architecture/iot-platform-architecture.md",
+    "tree/application/architecture/im-rtc-architecture.md",
+    "tree/application/architecture/ai-ml-inference-architecture.md",
+    "tree/application/architecture/digital-government-architecture.md",
+    "tree/application/architecture/deep-sea-exploration.md",
+    "tree/application/architecture/cloudnative-devops-architecture.md",
+    "tree/application/architecture/immersive-xr.md",
+    "tree/application/architecture/industrial-internet-platform.md",
+    "domain-20-application-patterns/topic-application-architecture/07-iot-platform-architecture.md",
+    "tree/application/architecture/smart-logistics-architecture.md",
+    "domain-20-application-patterns/topic-application-architecture",
+    "domain-20-application-patterns",
+    "tree/application/architecture/social-gaming-metaverse.md",
+    "tree/application/architecture/social-media-architecture.md",
+    "tree/application/architecture/space-internet.md",
+    "tree/application/architecture/tsn-network.md",
+    "tree/application/architecture/video-shortform-architecture.md",
+    "tree/platform-engineering/developer-experience/02-developer-onboarding-automation.md",
+    "tree/infrastructure/kubernetes/networking/ebpf/08-bcc-bpftrace-tools.md",
+    "tree/infrastructure/architecture/saas-multi-tenancy.md",
+    "tree/infrastructure/architecture/server-examples.md",
+    "tree/infrastructure/architecture/architecture-patterns.md",
     "tree/linux/operating-systems",
-    "tree/linux",
-    "tree/hardware",
-    "sources/ai/pdf/aivideo-spacetimediffusionmodel.pdf",
-    "tree/ai/pdf/aivideo-spacetimediffusionmodel.pdf",
-    "tree/ai/pdf/2401.12945v2.pdf",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/index.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/USER.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/TOOLS.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/SOUL.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/SKILL.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/MEMORY.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/IDENTITY.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace/AGENTS.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/openclaw-workspace",
-    "tree/infrastructure/kubernetes/ai/ai-agents/index.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/README.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/52-agent-cost-optimization-caching.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/51-agent-guardrails-content-safety.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/50-openclaw-identity-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/49-openclaw-memory-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/48-openclaw-skill-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/47-openclaw-tools-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/46-openclaw-agents-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/45-openclaw-user-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/44-openclaw-soul-mechanism.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/43-openclaw-framework-integration.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/42-model-harness-compatibility-matrix.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/41-react-harness-identification-guide.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/40-agent-harness-production-maturity.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/39-agent-harness-testing-benchmark.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents/38-agent-harness-performance-cost.md",
-    "tree/infrastructure/kubernetes/ai/ai-agents",
-    "tree/infrastructure/kubernetes/ai/agent-runtime",
-    "tree/infrastructure/kubernetes/ai/infrastructure",
-    "tree/application/architecture/topic-application-architecture"
+    "tree/linux"
   ]
 }

CLAUDE.md

@@ -5,13 +5,5 @@ Reference `.ai/prd.md`
 `kb/` - knowledge bank source code
 `tree/` - knowledge tree (information sources / obsidian vault)
 
-## Agents
+## Knowledge Bank Commands
 
-- Privacy Expert - `.claude/agents/privacy-expert.md`
-
-## Skills
-
-- `query_knowledge`
-- `sync_knowledge`
-- `extend_knowledge`
-- `run_research_report`

Makefile

@@ -11,7 +11,7 @@ help: ## Show this help
 install: ## Sync Python deps (lancedb, duckdb, fastembed, qdrant-client, ...)
 	uv sync
 
-extract: ## Convert sources/ epub+pdf -> tree/ markdown (idempotent, hash-gated)
+extract: ## Convert sources declared in tree/branches.yaml -> tree/ markdown
 	$(PY) -m kb extract
 
 ingest: ## Incremental ingest: changed markdown -> chunk -> embed -> LanceDB

README.md

@@ -1,6 +1,4 @@
 
-
-
 <p align="center">
   <img src="docs/assets/knowledge-tree-icon.svg" width="180" alt="Knowledge Tree"><br>
   <strong>Knowledge Tree</strong><br>
@@ -9,18 +7,23 @@
 
 ## Overview
 
-Knowledge as an obsidian vault stored as markdown in git and ingested and indexed context with skills and tools for human and ai coordination.
+Knowledge as an obsidian vault stored as markdown in git and ingested and indexed context with skills and tools for human and ai coordination. For details on how this works from a high level, read [architecture](./docs/architecture.md)
 
 ## Branches
 
-[branches.yaml](tree/branches.yaml) is the routing file for the knowledge tree.
+[branches.yaml](tree/branches.yaml) is the routing file for the knowledge tree and controls building the tree at compilation with controls for encryption, pulling from remote sources such as books, etc...
 
 - [AI](tree/ai)
 - [Infrastructure](tree/infrastructure)
 - [Networking](tree/networking)
 - [Application](tree/application)
 - [Specialized Tooling](tree/specialized_tooling)
 
+## Searching Trees
+
+- indexing
+- tagging
+
 ## Commands
 
 Source code used for knowledge management. Commands to interact with the knowledge bank exist in makefile.

docs/architecture.md

@@ -0,0 +1,112 @@
+# Architecture
+
+```
+                              ┌─────────────────────────────┐
+                              │     Humans / AI Agents      │
+                              │  Obsidian · Claude · Cursor │
+                              └──────────────┬──────────────┘
+                                             │
+                    ┌────────────────────────┼────────────────────────┐
+                    │                        │                        │
+                    ▼                        ▼                        ▼
+           ┌────────────────┐     ┌──────────────────┐     ┌─────────────────┐
+           │  Obsidian Vault│     │   Makefile / CLI │     │  Skills / Tools │
+           │   (tree/)      │     │   make extract   │     │  query_knowledge │
+           │                │     │   make ingest    │     │  store_knowledge │
+           └───────┬────────┘     │   make query     │     └────────┬────────┘
+                   │              └────────┬─────────┘              │
+                   │                       │                        │
+                   └───────────────────────┼────────────────────────┘
+                                           │
+                                           ▼
+┌──────────────────────────────────────────────────────────────────────────────┐
+│                         Knowledge Bank Library (kb/)                         │
+│                                                                              │
+│   ┌──────────┐   ┌──────────┐   ┌──────────┐   ┌──────────┐   ┌──────────┐ │
+│   │ extract  │──▶│  chunk   │──▶│  embed   │──▶│  store   │──▶│  query   │ │
+│   │          │   │          │   │          │   │          │   │          │ │
+│   │ books    │   │ parent/  │   │ fastembed│   │ LanceDB  │   │ hybrid   │ │
+│   │ pdfs     │   │ child    │   │ bge-small│   │ DuckDB   │   │ BM25+vec │ │
+│   │ remote   │   │ sections │   │          │   │ FTS      │   │          │ │
+│   └──────────┘   └──────────┘   └──────────┘   └────┬─────┘   └──────────┘ │
+│                                                     │                       │
+│   ┌──────────┐                                      │          ┌──────────┐ │
+│   │ secrets  │◀── sops policy from branches.yaml    │          │   sync   │ │
+│   └──────────┘                                      │          └────┬─────┘ │
+└─────────────────────────────────────────────────────┼───────────────┼───────┘
+                                                      │               │
+                              ┌───────────────────────┘               │
+                              ▼                                       ▼
+                   ┌────────────────────┐                ┌────────────────────┐                ┌────────────────────┐
+                   │   Local Indexes    │                │  Cluster Qdrant    │                │  S3 Vector Backup  │
+                   │                    │   make sync    │  (homelab k8s)     │   backup       │                    │
+                   │  .lancedb/         │ ─────────────▶ │  collection:       │ ─────────────▶ │  snapshots /       │
+                   │  kb.duckdb         │                │  knowledge         │ ◀───────────── │  object storage    │
+                   └────────────────────┘                └────────────────────┘   restore      └────────────────────┘
+
+
+┌──────────────────────────────────────────────────────────────────────────────┐
+│                        Knowledge Tree (tree/)                                │
+│                                                                              │
+│   branches.yaml ─── routing + build policy                                   │
+│        │                                                                     │
+│        ├── encryption: true/false (sops)                                     │
+│        ├── books:  epub/pdf paths ──▶ extract ──▶ markdown                   │
+│        ├── pdf:    research docs   ──▶ extract ──▶ markdown                  │
+│        └── branches: nested topic tree                                       │
+└──────────────────────────────────────────────────────────────────────────────┘
+
+
+Build pipeline (make index):
+
+  sources (NAS / books / pdf)
+           │
+           ▼  make extract
+  tree/**/*.md  ◀── branches.yaml (what to pull, encrypt, nest)
+           │
+           ▼  make ingest
+  chunks → embeddings → LanceDB + DuckDB (BM25)
+           │
+           ▼  make reindex
+  full-text index ready
+           │
+           ├──▶ make query Q="..."     (local hybrid search)
+           └──▶ make sync              (push vectors → Qdrant → S3 backup)
+```
+
+## Knowledge Bank Library
+
+A Python library (`kb/`) that reads from the tree on a compilation loop to compile and reconcile as the tree is edited.
+
+`branches.yaml` defines the tree structure and how to build it — from file encryption to pulling from remote sources such as books or PDF documents.
+
+```yaml
+# tree/branches.yaml (sketch)
+networking:
+  branches:
+    censorship:
+      encrypted: true
+      books:
+        some-title: "books/epub/..."
+      pdf:
+        paper: research/censorship/paper.pdf
+    kubernetes:
+      branches:
+        networking:
+          ebpf:
+            books:
+              learning_ebpf: "books/pdf/learning_ebpf.pdf"
+```
+
+## Knowledge Bank Indexing
+
+| Stage | Command | Role |
+|-------|---------|------|
+| Extract | `make extract` | Convert sources declared in `branches.yaml` → `tree/` markdown |
+| Ingest | `make ingest` | Chunk → embed → write LanceDB (incremental) |
+| Reindex | `make reindex` | Rebuild BM25 full-text index in DuckDB |
+| Query | `make query Q="..."` | Hybrid local search (vector + FTS) |
+| Sync | `make sync` | Push local vectors → Qdrant on the cluster |
+| Backup | Qdrant → S3 | Snapshot / export vectors to S3 for durable backup |
+
+Local artifacts: `.lancedb/`, `kb.duckdb`. Remote: Qdrant collection `knowledge`. Durable: S3 vector backup (restore into Qdrant).

kb/__main__.py

@@ -9,7 +9,7 @@ def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser(prog="kb", description="Knowledge base RAG pipeline")
     sub = p.add_subparsers(dest="cmd", required=True)
 
-    p_ext = sub.add_parser("extract", help="convert sources/ epub+pdf -> tree/ markdown")
+    p_ext = sub.add_parser("extract", help="convert sources declared in branches.yaml -> tree/ markdown")
     p_ext.add_argument("--force", action="store_true", help="re-extract unchanged sources")
 
     p_ing = sub.add_parser("ingest", help="incremental ingest markdown -> LanceDB")
@@ -54,7 +54,8 @@ def main(argv: list[str] | None = None) -> int:
 
     if args.cmd == "extract":
         from .extract import run as run_extract
-        run_extract(force=args.force)
+        stats = run_extract(force=args.force)
+        return 1 if stats["failed"] else 0
     elif args.cmd == "ingest":
         from .ingest import run
         run(force=args.force)

kb/config.py

@@ -7,7 +7,9 @@
 ROOT = Path(__file__).resolve().parent.parent
 
 KNOWLEDGE_DIRS = os.environ.get("KB_DIRS", "tree").split(",")
-SOURCES_DIR = Path(os.environ.get("KB_SOURCES", ROOT / "sources"))
+# Root that relative source paths in branches.yaml resolve against (the NAS).
+_vol = os.environ.get("KNOWLEDGE_VOLUME_PATH", "").strip()
+SOURCE_VOLUME = Path(_vol).expanduser() if _vol else None
 
 # Encryption policy: which subtrees sops encrypts, and the config generated
 # from it. TREE_DIR is the vault root that branches.yaml describes.