Mcp server

.dockerignore

@@ -0,0 +1,28 @@
+.git
+.github
+.venv
+__pycache__
+**/__pycache__
+**/*.pyc
+**/*.pyo
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.lancedb
+*.duckdb
+.env
+.env.*
+.envrc
+chart/
+docs/
+eval/
+tree/
+*.md
+!README.md
+.claude
+.ai
+.cursor
+agent-transcripts
+terminals
+uv.lock
+Makefile

.gitignore

@@ -8,6 +8,7 @@
 *.duckdb
 *.lance
 .lancedb/
+.cache/
 
 __pycache__*
 .venv

Dockerfile

@@ -0,0 +1,59 @@
+# syntax=docker/dockerfile:1.7
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder
+
+WORKDIR /build
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN uv venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install \
+      "mcp[cli]>=1.6" \
+      "qdrant-client>=1.12" \
+      "fastembed>=0.4" \
+      "onnxruntime<1.24"
+
+COPY kb/__init__.py kb/config.py kb/embed.py /build/kb/
+COPY kb/adapters/ /build/kb/adapters/
+COPY kb/mcp_retrieval/ /build/kb/mcp_retrieval/
+
+ENV FASTEMBED_CACHE_PATH=/opt/fastembed \
+    HOME=/tmp \
+    PYTHONPATH=/build
+RUN mkdir -p /opt/fastembed \
+ && python -c "from fastembed import TextEmbedding; TextEmbedding(model_name='BAAI/bge-small-en-v1.5')"
+
+FROM python:3.12-slim-bookworm AS runtime
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends libgomp1 \
+ && rm -rf /var/lib/apt/lists/* \
+ && groupadd --gid 1000 mcp \
+ && useradd --uid 1000 --gid mcp --create-home --home-dir /home/mcp mcp
+
+# Ownership via COPY --chown avoids a recursive chown/chmod metadata layer that
+# stalls Colima's containerd unpack on large onnxruntime/fastembed trees.
+# /opt/venv stays root-owned; pip installs are world-readable (a+rX) so uid 1000 works.
+COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder --chown=1000:1000 /build/kb /app/kb
+COPY --from=builder --chown=1000:1000 /opt/fastembed /opt/fastembed
+
+ENV PATH="/opt/venv/bin:$PATH" \
+    PYTHONPATH=/app \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    HOME=/home/mcp \
+    FASTEMBED_CACHE_PATH=/opt/fastembed \
+    QDRANT_NETWORK=internal
+
+WORKDIR /app
+USER 1000:1000
+
+# stdio by default, override with --transport streamable-http for HTTP.
+ENTRYPOINT ["python", "-m", "kb.mcp_retrieval"]
+CMD []

Makefile

@@ -1,12 +1,22 @@
 PY := uv run python
 
+# kb-retrieval MCP image (must match chart/values.yaml image.* unless overridden)
+IMAGE_REPO ?= ghcr.io/teaglebuilt/kb-retrieval
+IMAGE_TAG  ?= 0.1.0
+IMAGE      ?= $(IMAGE_REPO):$(IMAGE_TAG)
+GHCR_HOST  ?= ghcr.io
+GHCR_SCOPES ?= write:packages,read:packages
+DOCKER_BUILDKIT ?= 1
+BUILDKIT_PROGRESS ?= plain
+
 .DEFAULT_GOAL := help
-.PHONY: help install extract ingest reindex index query sync qdrant-forward golden clean \
+.PHONY: help install extract ingest reindex index query sync mcp-retrieval golden clean \
+        docker-build docker-login docker-push helm-deploy mcp-image \
         secrets-status secrets-encrypt secrets-decrypt secrets-check secrets-config secrets-hooks
 
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
-		awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}'
+		awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}'
 
 install:
 	uv sync
@@ -28,6 +38,32 @@ query: ## Query the local index: make query Q="ebpf network policy"
 sync: ## Push local LanceDB vectors -> Qdrant on the k8s cluster
 	$(PY) -m kb sync
 
+mcp-retrieval: ## Run kb_search MCP (stdio). QDRANT_NETWORK=internal|external
+	$(PY) -m kb.mcp_retrieval --transport stdio
+
+docker-build: ## Build kb-retrieval MCP image (bakes bge-small FastEmbed weights)
+	DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) BUILDKIT_PROGRESS=$(BUILDKIT_PROGRESS) \
+		docker build -t $(IMAGE) -f Dockerfile .
+
+docker-login: ## Login to GHCR using gh auth token (write:packages)
+	@command -v gh >/dev/null || { echo "gh CLI required (https://cli.github.com)"; exit 1; }
+	@gh auth status -h github.com >/dev/null 2>&1 || { echo "run: gh auth login"; exit 1; }
+	@gh auth status -h github.com 2>&1 | grep -q 'write:packages' || \
+		gh auth refresh -h github.com -s $(GHCR_SCOPES)
+	@TOKEN=$$(gh auth token); \
+	echo "$$TOKEN" | docker login $(GHCR_HOST) -u "$$(gh api user --jq .login)" --password-stdin
+
+docker-push: docker-build docker-login ## Push kb-retrieval image to IMAGE_REPO
+	docker push $(IMAGE)
+
+helm-deploy: ## Install/upgrade chart; qdrant.apikey from QDRANT_API_KEY (.envrc)
+	@test -n "$${QDRANT_API_KEY}" || { \
+		echo "QDRANT_API_KEY not set — run: direnv allow  (or export from .envrc)"; \
+		exit 1; \
+	}
+	helm upgrade --install knowledge-tree chart -n ai \
+		--set-string qdrant.apikey="$${QDRANT_API_KEY}"
+
 golden: ## Measure retrieval quality (recall@k / MRR) against eval/golden.yaml
 	$(PY) -m kb eval
 

chart/.helmignore

@@ -0,0 +1,8 @@
+# Patterns to ignore when packaging the chart.
+*.md
+.git
+.gitignore
+.DS_Store
+*.tgz
+*.bak
+*.tmp

chart/Chart.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v2
+name: knowledge-tree
+description: Knowledge-tree read plane (kb-retrieval MCP + knowledge-scout agent)
+type: application
+version: '1.0.0'
+appVersion: '0.1.0'

chart/templates/agent.yaml

@@ -0,0 +1,139 @@
+---
+apiVersion: kagent.dev/v1alpha2
+kind: Agent
+metadata:
+  name: knowledge-scout
+  namespace: ai
+  labels:
+    app.kubernetes.io/name: knowledge-scout
+    app.kubernetes.io/part-of: knowledge
+spec:
+  description: >-
+    Research agent that finds gaps in the knowledge tree via kb_search,
+    scouts fresh external sources, and proposes citeable content for tree/
+  declarative:
+    runtime: go
+    modelConfig: llmd-vllm-config
+    promptTemplate:
+      dataSources:
+        - alias: builtin
+          kind: ConfigMap
+          name: kagent-builtin-prompts
+    systemMessage: |
+      # Knowledge Scout
+
+      You research **new content to add to the knowledge tree** (`tree/…`).
+      You are not a general chatbot. Every run ends with structured proposals
+      grounded in (a) what the tree already has and (b) fresh external signals.
+
+      ## Role
+      Ai platform engineer who curates a git-backed knowledge vault
+      for storing content which will be used for context for ai agents.
+      Prefer lab-proven, citeable material over hype.
+
+      ## Tools (use in this order)
+      1. `kb_search` — semantic search over the existing `knowledge` collection
+         (paths under `tree/…`, headings, tags, chunk text). **Always first.**
+      2. `datetime_get_current_time` — freshness window (last 1–2 weeks).
+      3. Firecrawl — web/YouTube/Reddit/HN/blogs; papers; GitHub research.
+      4. `shell` — light local checks only when needed (never cluster mutate).
+      5. `qdrant-find` / `qdrant-store` — optional staging of approved ideas
+         in the content-ideas collection (dedupe before store).
+
+      ## Protocol (every scout run)
+      Follow these steps. Do not skip step 1.
+
+      ### 1. Map the tree (kb_search)
+      Call `kb_search` with 2–4 focused queries that cover the topic area.
+      Example queries:
+      - `"kgateway HTTPRoute internal TLS"`
+      - `"ArgoCD ApplicationSet multi-cluster"`
+      - `"eBPF Cilium NetworkPolicy"`
+      - `monero`
+      - `stablecoin opportunites`
+      - `roblox engineering`
+
+      From hits, extract:
+      - Covered paths (`tree/…`)
+      - Headings / tags already strong
+      - **Gaps**: missing depth, stale framing, no lab notes, no failure modes
+
+      If `kb_search` fails, say so and continue with external research only —
+      mark confidence lower.
+
+      ### 2. Scout external signals (freshness: last 1–2 weeks)
+      Use Firecrawl tools against YouTube, Reddit, HN, blogs, papers, GitHub.
+      Prefer primary sources and reproducible recipes over opinion threads.
+
+      ### 3. Deduce & propose
+      For each idea, reason briefly:
+      - Gap in tree? (cite `path=` from kb_search)
+      - Evidence? (URLs + dates)
+      - Where it should land? (suggested `tree/…` path)
+
+      Quality bar: **5–10 solid proposals** per full scout. Skip thin ideas.
+
+      ### 4. Output format (always)
+      Emit proposals as a markdown list. Each item MUST use this template:
+
+      ```
+      ### <Title>
+      - **Why**: audience pain + evidence (1–2 sentences)
+      - **Gap**: what kb_search showed is missing (cite tree paths)
+      - **Suggested path**: tree/<branch>/...
+      - **Sources**: URL (date) · URL (date)
+      - **Category**: kubernetes | networking | gitops | secrets | observability | aws | ai | other
+      - **Freshness**: YYYY-MM-DD — why now
+      - **Confidence**: high | medium | low
+      ```
+
+      End with a short **Coverage summary**: topics searched, paths already
+      covered, gaps still open.
+
+      When an idea is approved later: content is converted to markdown and
+      the tree is updated via pull request. You do **not** write files yourself.
+
+      ## Hard guardrails
+      - NEVER modify cluster infrastructure (kubectl apply/delete, CRDs, etc.)
+      - NEVER install system packages
+      - NEVER invent `tree/` paths as if they exist — only cite paths returned
+        by `kb_search`, or clearly label new paths as **proposed**
+      - NEVER expose write tools against the `knowledge` collection
+      - If unsure whether an action is safe, report instead of acting
+      - Answer ONLY with research findings + proposals; no filler
+      - Must be in English
+
+      ## Style
+      Practical, hands-on, honest. Lab notes and failure modes beat theory.
+      English, structured, citeable. Breadth with signal, not link dumps.
+    tools:
+      - type: McpServer
+        mcpServer:
+          kind: MCPServer
+          name: kb-retrieval
+          toolNames:
+            - kb_search
+      - type: McpServer
+        mcpServer:
+          kind: RemoteMCPServer
+          name: kagent-tool-server
+          toolNames:
+            - shell
+            - datetime_get_current_time
+      - type: McpServer
+        mcpServer:
+          kind: RemoteMCPServer
+          name: firecrawl
+          toolNames:
+            - firecrawl_firecrawl_search
+            - firecrawl_firecrawl_agent
+            - firecrawl_firecrawl_agent_status
+            - firecrawl_firecrawl_research_search_github
+            - firecrawl_firecrawl_research_search_papers
+      - type: McpServer
+        mcpServer:
+          kind: RemoteMCPServer
+          name: qdrant-mcp
+          toolNames:
+            - qdrant-find
+            - qdrant-store

chart/templates/mcp_server.yaml

@@ -0,0 +1,73 @@
+apiVersion: {{ .Values.mcpServer.apiVersion }}
+kind: MCPServer
+metadata:
+  name: {{ .Values.mcpServer.name }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    app.kubernetes.io/name: {{ .Values.mcpServer.name }}
+    app.kubernetes.io/part-of: knowledge
+    app.kubernetes.io/component: mcp-retrieval
+spec:
+  transportType: {{ .Values.mcpServer.transportType }}
+  stdioTransport: {}
+  timeout: {{ .Values.mcpServer.timeout | quote }}
+  deployment:
+    image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    imagePullPolicy: {{ .Values.image.pullPolicy }}
+    {{- with .Values.image.pullSecrets }}
+    imagePullSecrets:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    port: {{ .Values.mcpServer.port }}
+    replicas: {{ .Values.mcpServer.replicas }}
+    cmd: {{ .Values.mcpServer.cmd | quote }}
+    args:
+      {{- toYaml .Values.mcpServer.args | nindent 6 }}
+    env:
+      QDRANT_NETWORK: {{ .Values.qdrant.network | quote }}
+      QDRANT_INTERNAL_URL: {{ .Values.qdrant.internalUrl | quote }}
+      QDRANT_EXTERNAL_URL: {{ .Values.qdrant.externalUrl | quote }}
+      {{- if .Values.qdrant.url }}
+      QDRANT_URL: {{ .Values.qdrant.url | quote }}
+      {{- end }}
+      {{- if .Values.qdrant.verify }}
+      QDRANT_VERIFY: {{ .Values.qdrant.verify | quote }}
+      {{- end }}
+      {{- if .Values.qdrant.apikey }}
+      QDRANT_API_KEY: {{ .Values.qdrant.apikey | quote }}
+      {{- end }}
+      COLLECTION_NAME: {{ .Values.qdrant.collection | quote }}
+      EMBEDDING_MODEL: {{ .Values.embedding.model | quote }}
+      KB_SEARCH_DEFAULT_K: {{ .Values.search.defaultK | quote }}
+      KB_SEARCH_MAX_K: {{ .Values.search.maxK | quote }}
+      HOME: /home/mcp
+      PYTHONUNBUFFERED: "1"
+      PYTHONDONTWRITEBYTECODE: "1"
+      FASTEMBED_CACHE_PATH: {{ .Values.fastembedCachePath | quote }}
+    {{- if and .Values.qdrant.existingSecretEnabled .Values.qdrant.existingSecret }}
+    # Secret keys become env vars (kmcp envFrom). Expect key: QDRANT_API_KEY
+    {{- end }}
+    volumeMounts:
+      {{- if .Values.writableVolumes.home }}
+      - name: home
+        mountPath: /home/mcp
+      {{- end }}
+      {{- if .Values.writableVolumes.tmp }}
+      - name: tmp
+        mountPath: /tmp
+      {{- end }}
+    volumes:
+      {{- if .Values.writableVolumes.home }}
+      - name: home
+        emptyDir: {}
+      {{- end }}
+      {{- if .Values.writableVolumes.tmp }}
+      - name: tmp
+        emptyDir: {}
+      {{- end }}
+    resources:
+      {{- toYaml .Values.resources | nindent 6 }}
+    podSecurityContext:
+      {{- toYaml .Values.podSecurityContext | nindent 6 }}
+    securityContext:
+      {{- toYaml .Values.securityContext | nindent 6 }}