diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ea0299491..9fa810cca 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,7 +14,7 @@ permissions: actions: read jobs: - analyze: + analyze-actions: name: Analyze (actions) runs-on: ubuntu-latest timeout-minutes: 30 @@ -32,3 +32,22 @@ jobs: uses: github/codeql-action/analyze@v4 with: category: /language:actions + + analyze-python: + name: Analyze (python) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: python + build-mode: none + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: /language:python diff --git a/.gitignore b/.gitignore index 23d6b5655..551ed6ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,12 @@ __pycache__ .DS_Store .env* .venv/ +.claude/ +.codex/ logs/ +examples/pifs_workspace/ +examples/ +/CONTEXT.md +/docs/pifs-core-alignment/ +/pyproject.toml +/uv.lock diff --git a/README.md b/README.md index 594b70e8e..cec74ee24 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ python3 run_pageindex.py --pdf_path /path/to/your/document.pdf You can customize the processing with additional optional arguments: ``` ---model LLM model to use (default: gpt-4o-2024-11-20) +--model LLM model to use (default: gpt-5.4) --toc-check-pages Pages to check for table of contents (default: 20) --max-pages-per-node Max pages per node (default: 10) --max-tokens-per-node Max tokens per node (default: 20000) diff --git a/examples/pifs_demo.py b/examples/pifs_demo.py new file mode 100644 index 000000000..4f2887986 --- /dev/null +++ b/examples/pifs_demo.py @@ -0,0 +1,781 @@ +""" +PageIndex FileSystem (PIFS) agent demo. + +This mirrors examples/agentic_vectorless_rag_demo.py, but exposes a corpus +through the PageIndex FileSystem shell instead of direct PageIndex document +tools. The agent receives one read-only bash-like PIFS tool and must retrieve +evidence through tree, browse, stat, cat --structure, cat --page, and +single-file grep. + +The demo registers supported files under examples/documents. When a matching +examples/documents/results/*_structure.json file exists, it is loaded into the +PIFS workspace's PageIndexClient cache. Files without a cache exercise the +normal PageIndexClient.index() path during register(). + +Requirements: + pip install openai-agents + +Example: + python examples/pifs_demo.py --stream-mode all --verbose +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +import PyPDF2 + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# Keep the local demo quiet in offline environments. +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true") + +from pageindex import PageIndexClient +from pageindex.filesystem import MetadataGenerator, PageIndexFileSystem, PIFSCommandExecutor +from pageindex.filesystem.agent import run_pifs_agent +from pageindex.filesystem.core import DEFAULT_EMBEDDING_DIMENSIONS + + +EXAMPLES_DIR = Path(__file__).parent +DOCUMENTS_DIR = EXAMPLES_DIR / "documents" +WORKSPACE = EXAMPLES_DIR / "pifs_workspace" +DEFAULT_MODEL = os.environ.get("PIFS_DEMO_MODEL", "gpt-5.4") +DEFAULT_METADATA_PROVIDER = os.environ.get("PIFS_DEMO_METADATA_PROVIDER") or os.environ.get( + "PIFS_METADATA_PROVIDER", "openai" +) +DEFAULT_EMBEDDING_PROVIDER = os.environ.get("PIFS_DEMO_EMBEDDING_PROVIDER") or os.environ.get( + "PIFS_EMBEDDING_PROVIDER", "openai" +) +DEFAULT_QUESTION = ( + "Use the PIFS workspace to find the Federal Reserve annual report. " + "Which section covers supervision and regulation, and what page range " + "should I inspect? Cite the document and evidence you used." +) + +PIFS_DEMO_AGENT_PROMPT = """ +You are a PageIndex FileSystem retrieval agent for a local demo workspace. + +Use only the bash tool. It is a read-only PIFS virtual shell, not a real OS +shell. The workspace contains registered example PDFs. + +Retrieval strategy: +- Start with tree to understand the workspace. +- Use concrete PIFS paths from tree/browse output, such as /documents/report.pdf, + or stable file_ref/document ids. Do not invent temporary ref_N aliases. +- Folder paths such as /documents are positional command targets; do not put + folder paths inside --where. +- After choosing a folder, use browse with a required quoted query to find + likely files, for example: + browse /documents "Federal Reserve supervision regulation" +- If the folder is uncertain, use recursive browse from a structural parent, + for example: + browse -R /documents "Federal Reserve supervision regulation" +- Use recursive browse only after inspecting plausible folders or rephrasing. +- browse returns document candidates only; it is not final evidence. +- After browse returns candidates, verify evidence with grep, cat + --structure, or cat --page before answering. +- Use browse --where with JSON metadata DSL when metadata pruning is needed. +- Use grep only for one selected file. +- Do not use find, recursive grep, folder grep, pipes, browse spaces, or stat + schema/field modes. +- Run one evidence command at a time. Do not chain large commands like + cat --structure, grep, and cat --page in one bash call. +- For PDFs, use cat --structure to inspect the PageIndex tree, then + cat --page for evidence, for example: + cat /documents/2023-annual-report.pdf --page 31-35 +- Do not use cat --page as the first inspection command for a selected PDF. + Run cat --structure for that same target first, then choose pages. +- Do not guess cat --page ranges from grep line numbers. +- For page-range questions, use cat --structure to identify the full section + range. Then run cat --page on the smallest useful evidence range, usually the + section start page or first 1-2 pages, before the final answer. Do not print + a broad multi-page section unless the user asks to read the whole section. +- Answer only from PIFS tool output and cite file refs or document ids. +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run a PIFS document retrieval agent demo.") + parser.add_argument("--workspace", type=Path, default=WORKSPACE) + parser.add_argument("--documents-dir", type=Path, default=DOCUMENTS_DIR) + parser.add_argument( + "--document", + action="append", + default=[], + help="Specific document filename or path to register. May be repeated.", + ) + parser.add_argument( + "--max-docs", + type=int, + default=0, + help="Limit number of cached example documents to register. 0 means all.", + ) + parser.add_argument("--reset", action="store_true", help="Delete and rebuild the demo workspace.") + parser.add_argument( + "--prepare-only", + action="store_true", + help="Register documents and print PIFS smoke commands without running the agent.", + ) + parser.add_argument("--question", default=DEFAULT_QUESTION) + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument( + "--metadata-provider", + default=DEFAULT_METADATA_PROVIDER, + help="Provider used for register-time metadata generation.", + ) + parser.add_argument( + "--metadata-model", + default=os.environ.get("PIFS_METADATA_MODEL", "gpt-5-nano"), + help="Model used for register-time metadata generation.", + ) + parser.add_argument("--stream-mode", default="all", choices=["off", "tools", "model", "all"]) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--max-turns", type=int, default=12) + parser.add_argument("--max-seconds", type=float, default=90) + parser.add_argument("--reasoning-effort", default=None) + parser.add_argument("--reasoning-summary", default="auto") + parser.add_argument( + "--embedding-provider", + default=DEFAULT_EMBEDDING_PROVIDER, + help="Provider used for register-time summary projection embeddings.", + ) + parser.add_argument( + "--embedding-model", + default=os.environ.get("PIFS_DEMO_EMBEDDING_MODEL", "text-embedding-3-small"), + help="Embedding model used for register-time summary projection.", + ) + parser.add_argument( + "--embedding-dimensions", + type=int, + default=DEFAULT_EMBEDDING_DIMENSIONS, + ) + return parser.parse_args() + + +def require_runtime_environment(*, metadata_provider: str, embedding_provider: str) -> None: + metadata_provider = metadata_provider.lower() + embedding_provider = embedding_provider.lower() + missing: list[str] = [] + if not os.environ.get("OPENAI_API_KEY"): + missing.append("OPENAI_API_KEY for the OpenAI Agents SDK demo agent") + if metadata_provider == "openai" and not ( + os.environ.get("PIFS_METADATA_API_KEY") or os.environ.get("OPENAI_API_KEY") + ): + missing.append("PIFS_METADATA_API_KEY or OPENAI_API_KEY for metadata generation") + if embedding_provider == "openai" and not ( + os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY") + ): + missing.append("PIFS_EMBEDDING_API_KEY or OPENAI_API_KEY for summary embeddings") + if missing: + raise RuntimeError( + "Missing required environment variable(s): " + + "; ".join(missing) + + ". Source your .env or export the required key before running." + ) + + +SUPPORTED_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown", ".txt", ".text"} + + +def discover_documents(documents_dir: Path) -> list[Path]: + return sorted( + path + for path in documents_dir.iterdir() + if path.is_file() and path.suffix.lower() in SUPPORTED_DOCUMENT_SUFFIXES + ) + + +def resolve_requested_documents(documents_dir: Path, requested: list[str]) -> list[Path]: + if not requested: + return discover_documents(documents_dir) + paths: list[Path] = [] + for item in requested: + path = Path(item).expanduser() + if not path.is_absolute(): + path = documents_dir / path + if not path.exists(): + raise FileNotFoundError(f"document not found: {path}") + paths.append(path) + return paths + + +def structure_path_for(document_path: Path, documents_dir: Path) -> Path | None: + path = documents_dir / "results" / f"{document_path.stem}_structure.json" + return path if path.exists() else None + + +def deterministic_doc_id(document_path: Path) -> str: + digest = hashlib.sha1(str(document_path.resolve()).encode("utf-8")).hexdigest()[:16] + return f"pifs_demo_{digest}" + + +def read_pdf_pages(document_path: Path) -> list[dict[str, Any]]: + pages: list[dict[str, Any]] = [] + with document_path.open("rb") as handle: + reader = PyPDF2.PdfReader(handle) + for page_num, page in enumerate(reader.pages, 1): + pages.append({"page": page_num, "content": page.extract_text() or ""}) + return pages + + +def load_structure_json(structure_path: Path) -> dict[str, Any]: + with structure_path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict) or not isinstance(payload.get("structure"), list): + raise ValueError(f"invalid PageIndex structure cache: {structure_path}") + return payload + + +def seed_pageindex_cache( + filesystem: PageIndexFileSystem, + document_path: Path, + *, + documents_dir: Path, +) -> str | None: + structure_path = structure_path_for(document_path, documents_dir) + if structure_path is None: + return None + + filesystem.pageindex_client_workspace.mkdir(parents=True, exist_ok=True) + meta_path = filesystem.pageindex_client_workspace / "_meta.json" + if not meta_path.exists(): + meta_path.write_text("{}", encoding="utf-8") + client = PageIndexClient(workspace=str(filesystem.pageindex_client_workspace)) + canonical_path = str(document_path.resolve()) + for doc_id, doc in client.documents.items(): + if Path(str(doc.get("path") or "")).resolve(strict=False) == Path(canonical_path): + return doc_id + + payload = load_structure_json(structure_path) + doc_id = deterministic_doc_id(document_path) + suffix = document_path.suffix.lower() + if suffix == ".pdf": + pages = read_pdf_pages(document_path) + client.documents[doc_id] = { + "id": doc_id, + "type": "pdf", + "path": canonical_path, + "doc_name": payload.get("doc_name") or document_path.name, + "doc_description": payload.get("doc_description") or "", + "page_count": len(pages), + "structure": payload["structure"], + "pages": pages, + } + elif suffix in {".md", ".markdown"}: + text = document_path.read_text(encoding="utf-8") + client.documents[doc_id] = { + "id": doc_id, + "type": "md", + "path": canonical_path, + "doc_name": payload.get("doc_name") or document_path.name, + "doc_description": payload.get("doc_description") or "", + "line_count": len(text.splitlines()), + "structure": payload["structure"], + } + else: + return None + client._save_doc(doc_id) + return doc_id + + +def content_type_for(path: Path) -> str: + suffix = path.suffix.lower() + if suffix == ".pdf": + return "application/pdf" + if suffix in {".md", ".markdown"}: + return "text/markdown" + return "text/plain" + + +def external_id_for(path: Path) -> str: + slug = "".join(ch.lower() if ch.isalnum() else "_" for ch in path.stem).strip("_") + slug = "_".join(part for part in slug.split("_") if part) + return f"example_{slug}" + + +def log_progress(message: str, *, indent: int = 0) -> None: + prefix = " " * indent + print(f"{prefix}{message}", flush=True) + + +def register_demo_metadata_schema(filesystem: PageIndexFileSystem) -> None: + filesystem.metadata.register_schema( + { + "fields": { + "source_collection": { + "type": "string", + "description": "Local example corpus collection.", + }, + "file_format": { + "type": "string", + "description": "Source file extension without the leading dot.", + }, + } + }, + source="demo", + ) + + +def backfill_registered_metadata_values(filesystem: PageIndexFileSystem, file_ref: str) -> None: + entry = filesystem.store.get_file(file_ref) + indexed_metadata = dict(entry.metadata or {}) + with filesystem.store.connect() as conn: + filesystem.store.replace_metadata_values(conn, file_ref, indexed_metadata) + + +def configure_summary_projection_backend( + filesystem: PageIndexFileSystem, + *, + embedding_provider: str, + embedding_model: str, + embedding_dimensions: int, +) -> None: + if not (filesystem.summary_projection_index_dir / "summary.sqlite").exists(): + return + filesystem.configure_semantic_projection_retrieval( + filesystem.summary_projection_index_dir, + embedding_provider=embedding_provider, + embedding_model=embedding_model, + embedding_dimensions=embedding_dimensions, + ) + + +def has_ready_register_outputs(filesystem: PageIndexFileSystem, external_id: str) -> bool: + try: + file_ref = filesystem.store.resolve_file_ref(external_id) + entry = filesystem.store.get_file(file_ref) + except KeyError: + return False + status = entry.metadata_status or {} + fields = status.get("fields") or {} + required = ("summary", "doc_type", "domain", "topic") + if any(fields.get(field, {}).get("status") != "generated" for field in required): + return False + summary_projection = (status.get("projection_indexes") or {}).get("summary") or {} + return summary_projection.get("status") == "ready" + + +def register_documents( + filesystem: PageIndexFileSystem, + documents: list[Path], + *, + documents_dir: Path, +) -> list[dict[str, Any]]: + registered: list[dict[str, Any]] = [] + total = len(documents) + for index, document_path in enumerate(documents, 1): + document_path = document_path.resolve() + external_id = external_id_for(document_path) + log_progress(f"[{index}/{total}] {document_path.name}") + log_progress("PageIndex tree cache: checking examples/documents/results", indent=1) + cache_started = time.perf_counter() + cached_doc_id = seed_pageindex_cache( + filesystem, + document_path, + documents_dir=documents_dir, + ) + cache_seconds = time.perf_counter() - cache_started + if cached_doc_id: + log_progress( + f"PageIndex tree cache: ready doc_id={cached_doc_id} ({cache_seconds:.2f}s)", + indent=1, + ) + else: + log_progress( + f"PageIndex tree cache: no cached structure; register() will index if supported ({cache_seconds:.2f}s)", + indent=1, + ) + if has_ready_register_outputs(filesystem, external_id): + file_ref = filesystem.store.resolve_file_ref(external_id) + backfill_registered_metadata_values(filesystem, file_ref) + log_progress( + f"PIFS register: cached file_ref={file_ref}; metadata and summary projection already ready", + indent=1, + ) + registered.append( + { + "file_ref": file_ref, + "external_id": external_id, + "path": str(document_path), + "status": "cached", + "pageindex_doc_id": cached_doc_id, + } + ) + continue + + log_progress( + "PIFS register: running register() -> metadata generation -> summary embedding -> sqlite upsert", + indent=1, + ) + register_started = time.perf_counter() + file_ref = filesystem.register( + storage_uri=document_path.as_uri(), + folder_path="/documents", + external_id=external_id, + title=document_path.name, + content_type=content_type_for(document_path), + source_type="examples-documents", + metadata={ + "title": document_path.name, + "source_collection": "examples/documents", + "file_format": document_path.suffix.lower().lstrip("."), + }, + ) + register_seconds = time.perf_counter() - register_started + entry = filesystem.store.get_file(file_ref) + field_status = { + field: state.get("status") + for field, state in (entry.metadata_status.get("fields") or {}).items() + } + summary_projection = ( + entry.metadata_status.get("projection_indexes", {}).get("summary", {}) + ) + log_progress( + f"PIFS register: done file_ref={file_ref} ({register_seconds:.2f}s)", + indent=1, + ) + log_progress( + f"metadata: {entry.metadata_status.get('status', 'unknown')} fields={field_status}", + indent=1, + ) + log_progress( + "summary projection: " + f"{summary_projection.get('status', 'not_requested')} " + f"index={summary_projection.get('index_path', '')}", + indent=1, + ) + registered.append( + { + "file_ref": file_ref, + "external_id": external_id, + "path": str(document_path), + "status": entry.metadata_status.get("status", "unknown"), + "pageindex_tree_status": entry.pageindex_tree_status, + "pageindex_doc_id": entry.pageindex_doc_id, + } + ) + return registered + + +def print_section(title: str) -> None: + print("\n" + "#" * 78, flush=True) + print(f"# {title}", flush=True) + print("#" * 78, flush=True) + + +def print_step(title: str, detail: str = "") -> None: + print(f"\n>>> {title}", flush=True) + if detail: + print(f" {detail}", flush=True) + + +def sanitize_preview_text(text: str) -> str: + cleaned = str(text).replace("\r", "\n").replace("\f", "\n") + cleaned = "".join( + ch if ch == "\n" or ch == "\t" or ord(ch) >= 32 else " " + for ch in cleaned + ) + return "\n".join( + re.sub(r"[ \t]{2,}", " ", line).strip() + for line in cleaned.splitlines() + ) + + +def compact_lines(text: str, *, max_lines: int = 6, max_chars: int = 900) -> str: + lines = [line for line in sanitize_preview_text(text).splitlines() if line.strip()] + preview = "\n".join(lines[:max_lines]) + if len(preview) > max_chars: + preview = preview[:max_chars].rstrip() + "..." + omitted = len(lines) - min(len(lines), max_lines) + if omitted > 0: + preview += f"\n ... {omitted} more lines" + return preview + + +def find_structure_node(structure: Any, title_fragment: str) -> dict[str, Any] | None: + if isinstance(structure, list): + for item in structure: + found = find_structure_node(item, title_fragment) + if found: + return found + return None + if not isinstance(structure, dict): + return None + if title_fragment.lower() in str(structure.get("title", "")).lower(): + return structure + return find_structure_node(structure.get("nodes", []), title_fragment) + + +def page_range_for_node(node: dict[str, Any] | None) -> str: + if not node: + return "" + ranges: list[tuple[int, int]] = [] + + def collect(item: Any) -> None: + if not isinstance(item, dict): + return + start = item.get("start_index") + end = item.get("end_index") + if isinstance(start, int) and isinstance(end, int): + ranges.append((start, end)) + for child in item.get("nodes") or []: + collect(child) + + collect(node) + if not ranges: + return "" + start = min(item[0] for item in ranges) + end = max(item[1] for item in ranges) + return f"{start}-{end}" if start != end else str(start) + + +def opening_page_range_for_node(node: dict[str, Any] | None, *, max_pages: int = 2) -> str: + if not node: + return "" + ranges: list[tuple[int, int]] = [] + + def collect(item: Any) -> None: + if not isinstance(item, dict): + return + start = item.get("start_index") + end = item.get("end_index") + if isinstance(start, int) and isinstance(end, int): + ranges.append((start, end)) + for child in item.get("nodes") or []: + collect(child) + + collect(node) + if not ranges: + return "" + start = min(item[0] for item in ranges) + end = max(item[1] for item in ranges) + preview_end = min(end, start + max_pages - 1) + return f"{start}-{preview_end}" if start != preview_end else str(start) + + +def execute_json_command(executor: PIFSCommandExecutor, command: str) -> dict[str, Any]: + try: + return json.loads(executor.execute(command)) + except Exception as exc: + return {"ok": False, "error": str(exc), "data": None} + + +def show_capability( + *, + label: str, + command: str, + result: str, + raw: str = "", + verbose: bool = False, +) -> None: + print_step(label, command) + print(f" result: {result}", flush=True) + if verbose and raw: + print(" raw:", flush=True) + print(compact_lines(raw, max_lines=10, max_chars=1600), flush=True) + + +def show_registered_documents(registered: list[dict[str, Any]], *, verbose: bool = False) -> None: + print(f"\nRegistered {len(registered)} document(s):", flush=True) + for item in registered: + print( + " - " + f"{Path(str(item.get('path', ''))).name}: " + f"file_ref={item.get('file_ref')} | " + f"status={item.get('status')} | " + f"pageindex_doc_id={item.get('pageindex_doc_id')}", + flush=True, + ) + if verbose: + print("\nRaw registration records:", flush=True) + print(json.dumps(registered, ensure_ascii=False, indent=2), flush=True) + + +def run_smoke_commands( + filesystem: PageIndexFileSystem, + registered: list[dict[str, Any]], + *, + verbose: bool = False, +) -> None: + executor = PIFSCommandExecutor(filesystem) + + command = "tree / -L 2" + tree = execute_json_command(executor, command) + tree_data = tree.get("data") or {} + tree_root = tree_data.get("tree") or {} + show_capability( + label="Folder structure", + command=command, + result=( + f"{tree_data.get('total_folders', 0)} folders; " + f"root files={tree_root.get('file_count', 0)}" + ), + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + command = "ls /documents" + listing = execute_json_command(executor, command) + listing_data = listing.get("data") or {} + show_capability( + label="List alias", + command=command, + result=f"ls delegates to tree depth={listing_data.get('depth')}", + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + command = 'browse /documents "Federal Reserve annual report supervision regulation section page range"' + browse = execute_json_command(executor, command) + browse_hits = ((browse.get("data") or {}).get("documents") or []) + if browse_hits: + summary_result = f"{len(browse_hits)} browse candidates; top={browse_hits[0].get('document_id')}" + else: + summary_result = "browse is available, but this tiny two-doc demo returned no candidates" + show_capability( + label="Relevance browse", + command=command, + result=summary_result, + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + first_target = f"/documents/{Path(str(registered[0]['path'])).name}" if registered else None + if not first_target: + return + + command = f"stat {first_target}" + stat = execute_json_command(executor, command) + stat_data = (stat.get("data") or {}).get("document") or {} + show_capability( + label="File stat", + command=command, + result=( + f"{stat_data.get('title')} | tree={stat_data.get('status')} | " + f"metadata_status={(stat_data.get('metadata_status') or {}).get('status')}" + ), + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + command = f"cat {first_target} --structure" + structure_payload = execute_json_command(executor, command) + structure_data = structure_payload.get("data") or {} + structure = structure_data.get("structure") or [] + supervision_node = find_structure_node(structure, "Supervision and Regulation") + supervision_range = page_range_for_node(supervision_node) + show_capability( + label="PageIndex document structure", + command=command, + result=( + "found section 'Supervision and Regulation'" + + (f" with page span {supervision_range}" if supervision_range else "") + ), + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + evidence_range = opening_page_range_for_node(supervision_node) or "1-2" + command = f"cat {first_target} --page {evidence_range}" + page = execute_json_command(executor, command) + page_text = str(((page.get("data") or {}).get("content") or {}).get("text") or "") + show_capability( + label="Page evidence", + command=command, + result=compact_lines(page_text, max_lines=3, max_chars=420), + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + command = f'grep "Supervision and Regulation" {shlex.quote(first_target)}' + grep = execute_json_command(executor, command) + grep_hits = ((grep.get("data") or {}).get("matches") or []) + show_capability( + label="Lexical grep", + command=command, + result=f"{len(grep_hits)} real text matches", + raw=executor.execute(command) if verbose else "", + verbose=verbose, + ) + + +def main() -> None: + args = parse_args() + require_runtime_environment( + metadata_provider=args.metadata_provider, + embedding_provider=args.embedding_provider, + ) + workspace = args.workspace.expanduser() + documents_dir = args.documents_dir.expanduser() + if args.reset and workspace.exists(): + shutil.rmtree(workspace) + workspace.mkdir(parents=True, exist_ok=True) + + documents = resolve_requested_documents(documents_dir, args.document) + if args.max_docs > 0: + documents = documents[: args.max_docs] + if not documents: + raise RuntimeError(f"no cached example documents found under {documents_dir}") + + filesystem = PageIndexFileSystem( + workspace, + metadata_generator=MetadataGenerator( + provider=args.metadata_provider, + model=args.metadata_model, + ), + summary_projection_embedding_provider=args.embedding_provider, + summary_projection_embedding_model=args.embedding_model, + summary_projection_embedding_dimensions=args.embedding_dimensions, + ) + register_demo_metadata_schema(filesystem) + + print_section("STEP 1/3 Register Documents") + print(f"Workspace: {workspace}", flush=True) + print(f"Documents: {len(documents)}", flush=True) + registered = register_documents(filesystem, documents, documents_dir=documents_dir) + configure_summary_projection_backend( + filesystem, + embedding_provider=args.embedding_provider, + embedding_model=args.embedding_model, + embedding_dimensions=args.embedding_dimensions, + ) + show_registered_documents(registered, verbose=args.verbose) + + print_section("STEP 2/3 Explore PIFS Tool Surface") + run_smoke_commands(filesystem, registered, verbose=args.verbose) + + if args.prepare_only: + return + + print_section("STEP 3/3 Ask An Agent Using Only PIFS") + print(f"Question: {args.question}", flush=True) + answer = run_pifs_agent( + filesystem, + args.question, + model=args.model, + root="/", + system_prompt=PIFS_DEMO_AGENT_PROMPT, + max_turns=args.max_turns, + max_seconds=args.max_seconds, + verbose=args.verbose, + stream_mode=args.stream_mode, + reasoning_effort=args.reasoning_effort, + reasoning_summary=args.reasoning_summary, + ) + if answer: + print("\nFinal answer:", flush=True) + print(answer, flush=True) + + +if __name__ == "__main__": + main() diff --git a/pageindex/__init__.py b/pageindex/__init__.py index 658003bf5..c3fb0b0ae 100644 --- a/pageindex/__init__.py +++ b/pageindex/__init__.py @@ -1,4 +1,22 @@ -from .page_index import * -from .page_index_md import md_to_tree -from .retrieve import get_document, get_document_structure, get_page_content -from .client import PageIndexClient +import os + +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true") + +_OPTIONAL_CORE_IMPORTS = {"litellm", "openai", "PyPDF2", "pymupdf"} + +try: + from .page_index import * + from .page_index_md import md_to_tree + from .retrieve import get_document, get_document_structure, get_page_content + from .client import PageIndexClient +except ModuleNotFoundError as exc: + if exc.name not in _OPTIONAL_CORE_IMPORTS: + raise + + +def __getattr__(name: str): + if name == "PageIndexFileSystem": + from .filesystem import PageIndexFileSystem + + return PageIndexFileSystem + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pageindex/config.yaml b/pageindex/config.yaml index 591fe9331..5da9ee461 100644 --- a/pageindex/config.yaml +++ b/pageindex/config.yaml @@ -1,4 +1,4 @@ -model: "gpt-4o-2024-11-20" +model: "gpt-5.4" # model: "anthropic/claude-sonnet-4-6" retrieve_model: "gpt-5.4" # defaults to `model` if not set toc_check_page_num: 20 @@ -7,4 +7,4 @@ max_token_num_each_node: 20000 if_add_node_id: "yes" if_add_node_summary: "yes" if_add_doc_description: "no" -if_add_node_text: "no" \ No newline at end of file +if_add_node_text: "no" diff --git a/pageindex/filesystem/__init__.py b/pageindex/filesystem/__init__.py new file mode 100644 index 000000000..24b4c9746 --- /dev/null +++ b/pageindex/filesystem/__init__.py @@ -0,0 +1,51 @@ +from importlib import import_module +from typing import TYPE_CHECKING + +from .commands import PIFSCommandExecutor +from .core import PageIndexFileSystem +from .types import OpenResult, SearchResult + +if TYPE_CHECKING: + from .semantic_projection import SemanticProjectionSearchBackend + from .semantic_projection import SummaryProjectionIndexer + from .semantic_index import ( + RebuildableSemanticIndex, + SemanticIndexRecord, + SemanticSearchResult, + SQLiteVecSemanticIndex, + ) + +_LAZY_EXPORTS = { + "SemanticProjectionSearchBackend": (".semantic_projection", "SemanticProjectionSearchBackend"), + "RebuildableSemanticIndex": (".semantic_index", "RebuildableSemanticIndex"), + "SemanticIndexRecord": (".semantic_index", "SemanticIndexRecord"), + "SemanticSearchResult": (".semantic_index", "SemanticSearchResult"), + "SQLiteVecSemanticIndex": (".semantic_index", "SQLiteVecSemanticIndex"), + "SummaryProjectionIndexer": (".semantic_projection", "SummaryProjectionIndexer"), +} + +__all__ = [ + "OpenResult", + "SemanticProjectionSearchBackend", + "PIFSCommandExecutor", + "PageIndexFileSystem", + "RebuildableSemanticIndex", + "SearchResult", + "SemanticIndexRecord", + "SemanticSearchResult", + "SummaryProjectionIndexer", + "SQLiteVecSemanticIndex", +] + + +def __getattr__(name: str): + if name in _LAZY_EXPORTS: + module_name, attribute_name = _LAZY_EXPORTS[name] + value = getattr(import_module(module_name, __name__), attribute_name) + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS)) diff --git a/pageindex/filesystem/agent.py b/pageindex/filesystem/agent.py new file mode 100644 index 000000000..f782ecb2b --- /dev/null +++ b/pageindex/filesystem/agent.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +import json +import os +import re +import sys +import time +from dataclasses import asdict, is_dataclass +from typing import Any, Mapping, TextIO + +from .commands import PIFSCommandExecutor +from .core import PageIndexFileSystem + + +TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"} +PIFS_AGENT_TRACING_ENV = "PAGEINDEX_PIFS_AGENT_TRACING" +PIFS_AGENT_RAW_REASONING_ENV = "PAGEINDEX_PIFS_AGENT_RAW_REASONING" + +AGENT_SYSTEM_PROMPT = """ +You are the PageIndex FileSystem Demo Agent, developed by the VectifyAI Team. +Your job is to answer questions about the caller's current PageIndex FileSystem +workspace. + +You can inspect the corpus only by calling the bash tool. The bash tool is a +read-only PageIndex virtual shell, not a real operating-system shell. + +If the user asks who you are, answer with this identity and mention that you can +help inspect and answer questions about the current PIFS workspace. If the user +asks a general question unrelated to the current workspace, do not answer it as +a general-purpose assistant; briefly say that you can only answer workspace- +related questions and invite them to ask about files, folders, metadata, or +document contents in the workspace. + +If the user asks what tools or capabilities you have, describe only the PIFS +virtual shell capabilities available inside this workspace: tree, browse, +stat, cat, grep, and ls as an alias for tree -L 1. Do not mention host runtime +tools, SDK internals, or orchestration helpers that are not part of the PIFS +shell. + +If the user asks a workspace-related topic question without naming a specific +file, treat it as a retrieval task. Start with tree to understand folder +structure, choose a folder, then use browse with the user's topic as the query +to find candidate files. Inspect evidence before answering. Ask the user to +clarify only after the persistence protocol cannot identify relevant evidence. + +Follow the task prompt for command policy, retrieval strategy, and answer +format. If the caller needs stricter behavior, pass an explicit system_prompt. +""" + +BASH_TOOL_DESCRIPTION = """ +Run a command in the PageIndex FileSystem virtual shell. This is not a real +operating-system shell. The tool is read-only and always returns JSON. +Use only: tree, browse, stat, cat, grep, and ls as an exact alias for tree -L 1. +Start broad workspace questions with tree. Then choose a physical folder, +inspect metadata-derived virtual axes such as @year or @ticker with tree, and +copy returned scope paths into later commands. Run browse "" for document +discovery. Use browse -R when low-signal folder or virtual-node labels do not +help prune the query, or after scoped browse misses/rephrasing. Browse returns document +candidates only, not final evidence. Use stat only for identity, status, or +metadata. Use cat --structure before any cat --page N[-M] for +that file. Use grep only as a single-document lexical fallback. +Do not use find, recursive grep, folder grep, pipes, stat schema/field modes, +browse spaces, implicit cat reads, or general knowledge when workspace evidence +is not found. +""" + +AGENT_TOOL_POLICY = """ +Tool policy: +- The bash tool is a PageIndex virtual shell, not an operating-system shell. +- The default agent tool surface is read-only. +- Use only tree, browse, stat, cat, grep, and ls as an alias for tree -L 1. +- Folder paths such as /documents are positional command targets; never put folder paths in --where. +- Start with tree to understand workspace and folder structure before document retrieval. +- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values. +- After choosing a folder or scope, use browse "" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve". +- Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query. +- If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder. +- Only after that persistence protocol may you say the workspace lacks evidence. +- browse uses summary retrieval only; do not use browse --space. +- Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate. +- browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims. +- For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence. +- Use grep for a selected single file only. +- Do not use find, recursive grep, folder grep, pipes, schema browsing, batch metadata commands, browse spaces, implicit cat reads, or general-knowledge fallback. +- Command errors are JSON; recover by trying an available command. +- Use cat or grep to gather evidence before making source-backed claims. +- Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces. +- For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document. +- Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says. +- Prefer target-first cat syntax with stable targets: cat --structure, cat --page 31-59. +- cat --structure returns the cached PageIndex structure JSON without text fields. +- For PDF/PageIndex document Q&A, run cat --structure before the first cat --page call for that target; page reads without structure are blind page guessing. +- cat --page accepts at most 5 pages at once. If a larger range is needed, first inspect cat --structure and then read a smaller page range. +- When recovering from cat page/text limit errors, stop if the evidence is sufficient; if it is not sufficient, make another smaller call before answering. +- After cat --structure identifies a relevant section/subsection, use cat --page - for exact evidence. +- Use cat --page - when the user explicitly asks for pages/page ranges or when you need exact page text to verify evidence. +- Do not guess cat --page ranges from grep line numbers, text offsets, or table-of-contents intuition; use cat --structure to map the document first. +- Avoid fetching a broad page span unless page-level citation or verification is required. +- Do not call cat --page ; if you need a page span, use cat --page -. +- For metadata or summary-field questions, run stat for relevant files before answering. +- Distinguish default/register metadata from caller-provided custom metadata when the evidence supports it. +- Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other. +""" + +STREAM_MODE_ALIASES = { + "": "off", + "none": "off", + "false": "off", + "0": "off", + "off": "off", + "tool": "tools", + "tools": "tools", + "model": "model", + "output": "model", + "outputs": "model", + "think": "model", + "all": "all", + "debug": "all", +} +AGENT_STREAM_MODE_CHOICES = sorted(item for item in STREAM_MODE_ALIASES if item) +REASONING_EFFORT_CHOICES = ["none", "minimal", "low", "medium", "high", "xhigh"] +REASONING_SUMMARY_CHOICES = ["none", "auto", "concise", "detailed"] + + +def should_use_openai_compatible_chat_model(base_url: str | None) -> bool: + if not base_url: + return False + normalized = base_url.strip().rstrip("/") + return normalized not in {"https://api.openai.com", "https://api.openai.com/v1"} + + +def env_flag_enabled(name: str, environ: Mapping[str, str] | None = None) -> bool: + source = os.environ if environ is None else environ + value = source.get(name, "") + return value.strip().lower() in TRUTHY_ENV_VALUES + + +def pifs_agent_tracing_enabled(environ: Mapping[str, str] | None = None) -> bool: + return env_flag_enabled(PIFS_AGENT_TRACING_ENV, environ) + + +def should_disable_pifs_agent_tracing(environ: Mapping[str, str] | None = None) -> bool: + return not pifs_agent_tracing_enabled(environ) + + +def pifs_agent_raw_reasoning_enabled(environ: Mapping[str, str] | None = None) -> bool: + return env_flag_enabled(PIFS_AGENT_RAW_REASONING_ENV, environ) + + +def normalize_reasoning_effort(reasoning_effort: str | None) -> str | None: + if reasoning_effort is None or not reasoning_effort.strip(): + return None + effort = reasoning_effort.strip().lower() + if effort not in REASONING_EFFORT_CHOICES: + allowed = ", ".join(REASONING_EFFORT_CHOICES) + raise ValueError(f"Unknown reasoning effort: {reasoning_effort!r}. Allowed: {allowed}") + return effort + + +def normalize_reasoning_summary(reasoning_summary: str | None) -> str | None: + if reasoning_summary is None or not reasoning_summary.strip(): + return None + summary = reasoning_summary.strip().lower() + if summary not in REASONING_SUMMARY_CHOICES: + allowed = ", ".join(REASONING_SUMMARY_CHOICES) + raise ValueError(f"Unknown reasoning summary: {reasoning_summary!r}. Allowed: {allowed}") + return None if summary == "none" else summary + + +def build_agent_model_settings( + *, + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, +) -> Any | None: + effort = normalize_reasoning_effort(reasoning_effort) + summary = normalize_reasoning_summary(reasoning_summary) + if effort is None and summary is None: + return None + if effort not in {None, "none"} and summary is None: + summary = "auto" + + from agents import ModelSettings + from openai.types.shared import Reasoning + + reasoning_kwargs = {} + if effort is not None: + reasoning_kwargs["effort"] = effort + if summary is not None: + reasoning_kwargs["summary"] = summary + return ModelSettings(reasoning=Reasoning(**reasoning_kwargs), verbosity="low") + + +def normalize_agent_stream_mode(stream_mode: str | None) -> str: + mode = STREAM_MODE_ALIASES.get((stream_mode or "off").strip().lower()) + if mode is None: + allowed = ", ".join(sorted({"off", "tools", "model", "all"})) + raise ValueError(f"Unknown PIFS agent stream mode: {stream_mode!r}. Allowed: {allowed}") + return mode + + +def serialize_agent_final_output(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if hasattr(value, "model_dump_json"): + return value.model_dump_json() + if is_dataclass(value): + return json.dumps(asdict(value), ensure_ascii=False) + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def compact_tool_output_preview( + output: str, + *, + preview_chars: int = 700, + max_lines: int = 8, +) -> str: + cleaned = str(output).replace("\r", "\n").replace("\f", "\n") + cleaned = "".join( + ch if ch == "\n" or ch == "\t" or ord(ch) >= 32 else " " + for ch in cleaned + ) + lines = [ + re.sub(r"[ \t]{2,}", " ", line).strip() + for line in cleaned.splitlines() + if line.strip() + ] + is_large_result = len(cleaned) > preview_chars or len(lines) > max_lines + preview = "\n".join(lines[:max_lines]) + if len(preview) > preview_chars: + preview = preview[:preview_chars].rstrip() + "..." + omitted = len(lines) - min(len(lines), max_lines) + if is_large_result: + preview = f"[large PIFS result: {len(cleaned)} chars; showing compact preview]\n" + preview + if omitted > 0: + preview += f"\n... [{omitted} more lines omitted from preview]" + if len(cleaned) > preview_chars: + preview += "\n... [full result returned to agent; terminal preview shortened]" + return preview + + +def build_agent_initial_context( + filesystem: PageIndexFileSystem, + *, + root: str = "/", + executor: PIFSCommandExecutor | None = None, +) -> str: + executor = executor or PIFSCommandExecutor(filesystem) + return "\n".join( + [ + f"Root path: {root}", + "Top-level tree:", + executor.execute(f"tree {root} -L 1"), + "Workspace retrieval capabilities:", + executor.describe_available_command_surfaces(), + ] + ) + + +def build_pifs_agent_instructions( + filesystem: PageIndexFileSystem, + *, + root: str = "/", + system_prompt: str | None = None, + executor: PIFSCommandExecutor | None = None, +) -> str: + initial_context = build_agent_initial_context( + filesystem, + root=root, + executor=executor, + ) + return "\n\n".join( + [ + (system_prompt or AGENT_SYSTEM_PROMPT).strip(), + AGENT_TOOL_POLICY.strip(), + "Workspace context:\n" + initial_context, + ] + ) + + +class PIFSAgentStreamObserver: + def __init__( + self, + stream_mode: str, + *, + stream_log: list[dict[str, Any]] | None = None, + output: TextIO | None = None, + include_raw_reasoning: bool | None = None, + ) -> None: + self.stream_mode = normalize_agent_stream_mode(stream_mode) + self.stream_log = stream_log + self.output = output or sys.stdout + self.include_raw_reasoning = ( + pifs_agent_raw_reasoning_enabled() + if include_raw_reasoning is None + else include_raw_reasoning + ) + self._printed_section: str | None = None + self._buffers: dict[str, list[str]] = { + "output": [], + "think": [], + "think_summary": [], + "tool_args": [], + } + + @property + def wants_model_stream(self) -> bool: + return self.stream_mode in {"model", "all"} + + @property + def wants_tool_stream(self) -> bool: + return self.stream_mode in {"tools", "all"} + + @property + def has_output_text(self) -> bool: + return bool(self._buffers["output"]) + + def handle_event(self, event: Any) -> None: + if getattr(event, "type", None) == "raw_response_event": + self._handle_raw_response_event(getattr(event, "data", None)) + elif getattr(event, "type", None) == "run_item_stream_event": + self._handle_run_item_event(event) + + def finish(self, final_output: Any = None) -> None: + if self.wants_model_stream and not self.has_output_text and final_output: + self._emit("output", str(final_output), "[llm final output stream]") + if self._printed_section is not None: + print(file=self.output, flush=True) + self._printed_section = None + if self.stream_log is not None: + for kind, parts in self._buffers.items(): + text = "".join(parts) + if text: + self.stream_log.append({"kind": kind, "text": text}) + + def _handle_raw_response_event(self, data: Any) -> None: + event_type = getattr(data, "type", "") + delta = getattr(data, "delta", None) + if not isinstance(delta, str) or not delta: + return + if event_type == "response.output_text.delta": + self._emit("output", delta, "[llm final output stream]") + elif event_type == "response.reasoning_text.delta": + if self.include_raw_reasoning: + self._emit("think", delta, "[llm reasoning text stream]") + elif event_type == "response.reasoning_summary_text.delta": + self._emit("think_summary", delta, "[llm reasoning summary stream]") + elif event_type == "response.function_call_arguments.delta": + self._buffers["tool_args"].append(delta) + + def _handle_run_item_event(self, event: Any) -> None: + name = getattr(event, "name", "") + item = getattr(event, "item", None) + item_type = getattr(item, "type", "") + if self.stream_log is not None and name in {"message_output_created", "reasoning_item_created"}: + self.stream_log.append({"kind": "run_item", "name": name, "item_type": item_type}) + + def _emit(self, kind: str, text: str, label: str) -> None: + if kind == "tool_args": + should_print = self.wants_tool_stream + else: + should_print = self.wants_model_stream + if not should_print: + return + self._buffers[kind].append(text) + if self._printed_section != kind: + if self._printed_section is not None: + print(file=self.output, flush=True) + print(f"\n{label}", file=self.output, flush=True) + self._printed_section = kind + print(text, end="", file=self.output, flush=True) + + def emit_tool_call(self, command: str, *, force: bool = False) -> None: + if not command.strip(): + return + if self.stream_log is not None: + self.stream_log.append({"kind": "tool_call", "command": command}) + if not (force or self.wants_tool_stream): + return + self._start_section("tool_call", "[llm -> pifs command]") + print(command, file=self.output, flush=True) + + def emit_tool_result( + self, + *, + ok: bool, + output: str, + seconds: float, + force: bool = False, + preview_chars: int = 1000, + ) -> None: + if self.stream_log is not None: + self.stream_log.append( + { + "kind": "tool_result", + "ok": ok, + "seconds": round(seconds, 4), + "output_chars": len(output), + "preview": compact_tool_output_preview(output, preview_chars=preview_chars), + } + ) + if not (force or self.wants_tool_stream): + return + preview = compact_tool_output_preview(output, preview_chars=preview_chars) + self._start_section("tool_result", "[pifs -> llm result preview]") + print( + f"ok={str(ok).lower()} seconds={seconds:.4f} output_chars={len(output)}", + file=self.output, + flush=True, + ) + print(preview, file=self.output, flush=True) + + def _start_section(self, kind: str, label: str) -> None: + if self._printed_section is not None: + print(file=self.output, flush=True) + print(f"\n{label}", file=self.output, flush=True) + self._printed_section = kind + + +def run_pifs_agent( + filesystem: PageIndexFileSystem, + question: str, + *, + model: str, + root: str = "/", + system_prompt: str | None = None, + max_turns: int = 20, + max_seconds: float | None = 60, + verbose: bool = False, + stream_mode: str = "off", + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, + output_type: type[Any] | None = None, + tool_log: list[dict[str, Any]] | None = None, + agent_log: list[dict[str, Any]] | None = None, +) -> str: + session = PIFSAgentSession( + filesystem, + model=model, + root=root, + system_prompt=system_prompt, + max_turns=max_turns, + max_seconds=max_seconds, + verbose=verbose, + stream_mode=stream_mode, + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + output_type=output_type, + tool_log=tool_log, + agent_log=agent_log, + persist_conversation=False, + ) + return session.run(question) + + +class PIFSAgentSession: + def __init__( + self, + filesystem: PageIndexFileSystem, + *, + model: str, + root: str = "/", + system_prompt: str | None = None, + max_turns: int = 20, + max_seconds: float | None = 60, + verbose: bool = False, + stream_mode: str = "off", + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, + output_type: type[Any] | None = None, + tool_log: list[dict[str, Any]] | None = None, + agent_log: list[dict[str, Any]] | None = None, + persist_conversation: bool = True, + ) -> None: + self.filesystem = filesystem + self.max_turns = max_turns + self.max_seconds = max_seconds + self.verbose = verbose + self.tool_log = tool_log + self.agent_log = agent_log + self.normalized_stream_mode = normalize_agent_stream_mode(stream_mode) + self.observer: PIFSAgentStreamObserver | None = None + + try: + from agents import ( + Agent, + OpenAIChatCompletionsModel, + function_tool, + set_tracing_disabled, + ) + from agents.memory import SQLiteSession + from openai import AsyncOpenAI + except ModuleNotFoundError as exc: + if exc.name == "agents": + raise RuntimeError( + "openai-agents is required to run the PageIndex FileSystem agent" + ) from exc + raise + + set_tracing_disabled(should_disable_pifs_agent_tracing()) + self.executor = PIFSCommandExecutor(filesystem) + instructions = build_pifs_agent_instructions( + filesystem, + root=root, + system_prompt=system_prompt, + executor=self.executor, + ) + + @function_tool(description_override=BASH_TOOL_DESCRIPTION.strip()) + def bash(command: str) -> str: + """Run an allowed PageIndex FileSystem virtual shell command.""" + return self._run_bash(command) + + model_settings = build_agent_model_settings( + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + ) + base_url = os.environ.get("OPENAI_BASE_URL") + model_config = model + if should_use_openai_compatible_chat_model(base_url): + model_config = OpenAIChatCompletionsModel( + model=model, + openai_client=AsyncOpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), + base_url=base_url, + ), + ) + + agent_kwargs: dict[str, Any] = { + "name": "PageIndexFileSystem", + "instructions": instructions, + "tools": [bash], + "model": model_config, + } + if model_settings is not None: + agent_kwargs["model_settings"] = model_settings + if output_type is not None: + agent_kwargs["output_type"] = output_type + self.agent = Agent(**agent_kwargs) + self.session = SQLiteSession("pifs-chat") if persist_conversation else None + + def run(self, question: str) -> str: + self.observer = PIFSAgentStreamObserver( + self.normalized_stream_mode, + stream_log=self.agent_log, + ) + + async def _run_streamed() -> str: + from agents import Runner + + streamed_run = Runner.run_streamed( + self.agent, + question, + max_turns=self.max_turns, + session=self.session, + ) + final_output = "" + try: + async for event in streamed_run.stream_events(): + self.observer.handle_event(event) + final_output = serialize_agent_final_output(streamed_run.final_output) + return final_output + finally: + if not final_output and streamed_run.final_output: + final_output = serialize_agent_final_output(streamed_run.final_output) + self.observer.finish(final_output) + + async def _run() -> str: + if self.max_seconds is None or self.max_seconds <= 0: + return await _run_streamed() + try: + return await asyncio.wait_for(_run_streamed(), timeout=self.max_seconds) + except asyncio.TimeoutError as exc: + raise TimeoutError(f"MaxSecondsExceeded: exceeded {self.max_seconds:g}s") from exc + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(_run()) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, _run()).result() + + def _run_bash(self, command: str) -> str: + started = time.time() + assert self.observer is not None + self.observer.emit_tool_call(command, force=self.verbose) + output = self.executor.execute(command) + try: + ok = bool(json.loads(output).get("success")) + except json.JSONDecodeError: + ok = False + seconds = time.time() - started + if self.tool_log is not None: + self.tool_log.append( + { + "command": command, + "ok": ok, + "seconds": round(seconds, 4), + "output_chars": len(output), + "preview": output[:500], + } + ) + self.observer.emit_tool_result( + ok=ok, + output=output, + seconds=seconds, + force=self.verbose, + ) + return output diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py new file mode 100644 index 000000000..89ac1a390 --- /dev/null +++ b/pageindex/filesystem/cli.py @@ -0,0 +1,447 @@ +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import re +import shlex +import sys +from pathlib import Path +from typing import Iterator, TextIO + +from .agent import ( + PIFSAgentSession, + REASONING_EFFORT_CHOICES, + REASONING_SUMMARY_CHOICES, + run_pifs_agent, +) +from .commands import PIFSCommandError, PIFSCommandExecutor +from .core import PageIndexFileSystem + + +AGENT_STREAM_MODE_CHOICES = ("off", "tools", "model", "all") +DEFAULT_AGENT_MODEL = "gpt-5.4" +EXIT_COMMANDS = {"exit", "quit", ":q"} +ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|.)") +PIFS_CONFIG_FILE_ENV = "PIFS_CONFIG_FILE" +PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE" + + +def _config_path() -> Path: + override = os.environ.get(PIFS_CONFIG_FILE_ENV) + if override: + return Path(override).expanduser() + config_home = os.environ.get("XDG_CONFIG_HOME") + root = Path(config_home).expanduser() if config_home else Path.home() / ".config" + return root / "pageindex" / "pifs.json" + + +def _read_config() -> dict[str, str]: + path = _config_path() + if not path.exists(): + return {} + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"invalid PIFS config file: {path}") + return {str(key): str(value) for key, value in payload.items() if value is not None} + + +def _write_config(config: dict[str, str]) -> Path: + path = _config_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(config, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def _configured_workspace() -> str | None: + return _read_config().get("workspace") + + +def _resolve_workspace(value: str | None) -> str | None: + return value or os.environ.get(PIFS_WORKSPACE_ENV) or _configured_workspace() + + +def _load_env_file(path: str | None = None, *, workspace: str | None = None) -> Path | None: + from dotenv import load_dotenv + + if path: + env_path = Path(path).expanduser() + if not env_path.exists(): + raise FileNotFoundError(f"env file not found: {env_path}") + load_dotenv(env_path, override=True) + return env_path + + env_override = os.environ.get("PIFS_ENV_FILE") + if env_override: + return _load_env_file(env_override) + + starts = [Path.cwd()] + if workspace: + starts.append(Path(workspace).expanduser()) + seen: set[Path] = set() + for start in starts: + current = start.resolve() if start.exists() else start.resolve(strict=False) + if current.is_file(): + current = current.parent + for parent in (current, *current.parents): + candidate = parent / ".env" + if candidate in seen: + continue + seen.add(candidate) + if candidate.exists(): + load_dotenv(candidate, override=False) + return candidate + return None + + +def _agent_model_default() -> str: + return ( + os.environ.get("PIFS_AGENT_MODEL") + or os.environ.get("PIFS_MODEL") + or DEFAULT_AGENT_MODEL + ) + + +def _add_agent_arguments( + parser: argparse.ArgumentParser, + *, + workspace_default: str | None, + default_stream_mode: str, +) -> None: + parser.add_argument("--workspace", default=workspace_default) + parser.add_argument("--env-file", default=None) + parser.add_argument("--model", default=_agent_model_default()) + parser.add_argument( + "--stream-mode", + default=default_stream_mode, + choices=AGENT_STREAM_MODE_CHOICES, + ) + parser.add_argument("--max-turns", type=int, default=20) + parser.add_argument("--max-seconds", type=float, default=60) + parser.add_argument( + "--reasoning-effort", + choices=REASONING_EFFORT_CHOICES, + default=None, + ) + parser.add_argument( + "--reasoning-summary", + choices=REASONING_SUMMARY_CHOICES, + default=None, + ) + + +def _parse_agent_command( + command_name: str, + argv: list[str], + *, + workspace_default: str | None, + default_stream_mode: str, +) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog=f"pifs {command_name}", + description=f"PageIndex FileSystem {command_name}", + ) + _add_agent_arguments( + parser, + workspace_default=workspace_default, + default_stream_mode=default_stream_mode, + ) + if command_name == "ask": + parser.add_argument("question", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + _load_env_file(args.env_file, workspace=args.workspace) + args.workspace = _resolve_workspace(args.workspace) + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return args + + +def _optional_int(name: str, value: str | None) -> int | None: + if value is None or not value.strip(): + return None + try: + return int(value) + except ValueError as exc: + raise ValueError(f"{name} must be an integer") from exc + + +def _optional_float(name: str, value: str | None) -> float | None: + if value is None or not value.strip(): + return None + try: + return float(value) + except ValueError as exc: + raise ValueError(f"{name} must be a number") from exc + + +def _filesystem_embedding_config() -> dict[str, object]: + config_values = _read_config() + config: dict[str, object] = {} + base_url = config_values.get("embedding_base_url") + api_key = config_values.get("embedding_api_key") + provider = config_values.get("embedding_provider") + model = config_values.get("embedding_model") + dimensions = _optional_int( + "embedding_dimensions", + config_values.get("embedding_dimensions"), + ) + timeout = _optional_float( + "embedding_timeout", + config_values.get("embedding_timeout"), + ) + if provider and provider.strip(): + config["summary_projection_embedding_provider"] = provider.strip() + if model and model.strip(): + config["summary_projection_embedding_model"] = model.strip() + if dimensions is not None: + config["summary_projection_embedding_dimensions"] = dimensions + if timeout is not None: + config["summary_projection_embedding_timeout"] = timeout + if base_url and base_url.strip(): + config["summary_projection_embedding_base_url"] = base_url.strip() + if api_key and api_key.strip(): + config["summary_projection_embedding_api_key"] = api_key.strip() + return config + + +def _filesystem_from_workspace(workspace: str) -> PageIndexFileSystem: + return PageIndexFileSystem( + Path(workspace).expanduser(), + **_filesystem_embedding_config(), + ) + + +def _agent_kwargs(args: argparse.Namespace) -> dict[str, object]: + return { + "model": args.model, + "stream_mode": args.stream_mode, + "max_turns": args.max_turns, + "max_seconds": args.max_seconds, + "reasoning_effort": args.reasoning_effort, + "reasoning_summary": args.reasoning_summary, + } + + +def _sanitize_chat_question(raw: str) -> str: + text = ANSI_ESCAPE_RE.sub("", raw) + chars: list[str] = [] + for char in text: + if char in {"\b", "\x7f"}: + if chars: + chars.pop() + continue + if char in {"\r", "\n"}: + continue + if ord(char) < 32 or ord(char) == 127: + continue + chars.append(char) + return "".join(chars).strip() + + +@contextlib.contextmanager +def _suppress_tty_input_echo(stdin: TextIO | None = None) -> Iterator[None]: + stream = sys.stdin if stdin is None else stdin + if not hasattr(stream, "isatty") or not stream.isatty(): + yield + return + try: + import termios + + fd = stream.fileno() + original = termios.tcgetattr(fd) + muted = original[:] + muted[3] = muted[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, muted) + except Exception: + yield + return + try: + yield + finally: + with contextlib.suppress(Exception): + termios.tcflush(fd, termios.TCIFLUSH) + with contextlib.suppress(Exception): + termios.tcsetattr(fd, termios.TCSADRAIN, original) + + +def _run_ask(argv: list[str], *, workspace_default: str | None) -> int: + args = _parse_agent_command( + "ask", + argv, + workspace_default=workspace_default, + default_stream_mode="off", + ) + question_tokens = [token for token in args.question if token != "--"] + question = " ".join(question_tokens).strip() + if not question: + raise ValueError("ask requires a question") + + filesystem = _filesystem_from_workspace(args.workspace) + answer = run_pifs_agent(filesystem, question, **_agent_kwargs(args)) + if args.stream_mode in {"off", "tools"}: + print(answer) + return 0 + + +def _run_chat(argv: list[str], *, workspace_default: str | None) -> int: + args = _parse_agent_command( + "chat", + argv, + workspace_default=workspace_default, + default_stream_mode="all", + ) + filesystem = _filesystem_from_workspace(args.workspace) + session = PIFSAgentSession(filesystem, **_agent_kwargs(args)) + while True: + try: + question = _sanitize_chat_question(input("pifs> ")) + except EOFError: + break + except KeyboardInterrupt: + print() + break + if not question: + continue + if question.lower() in EXIT_COMMANDS: + break + with _suppress_tty_input_echo(): + answer = session.run(question) + if args.stream_mode == "off": + print(answer) + return 0 + + +def _run_passthrough( + command_tokens: list[str], + *, + workspace: str, +) -> int: + filesystem = _filesystem_from_workspace(workspace) + executor = PIFSCommandExecutor(filesystem) + command = " ".join(shlex.quote(token) for token in command_tokens) + output = executor.execute(command) + print(output) + try: + payload = json.loads(output) + except json.JSONDecodeError: + return 0 + return 0 if payload.get("success") is not False else 2 + + +def _run_add(argv: list[str], *, workspace: str) -> int: + parser = argparse.ArgumentParser( + prog="pifs add", + description="Add a local file to a PageIndex FileSystem workspace", + ) + parser.add_argument("physical_path") + parser.add_argument("virtual_target") + args = parser.parse_args(argv) + + filesystem = _filesystem_from_workspace(workspace) + info = filesystem.add_file(args.physical_path, args.virtual_target) + print(f"added: {info.get('path')}") + print(f"file_ref: {info['file_ref']}") + return 0 + + +def _run_setmeta(argv: list[str], *, workspace: str) -> int: + parser = argparse.ArgumentParser( + prog="pifs setmeta", + description="Replace custom metadata for one PIFS document", + ) + parser.add_argument("--clear", action="store_true") + parser.add_argument("target") + parser.add_argument("metadata_json", nargs="?") + args = parser.parse_args(argv) + + if args.clear: + if args.metadata_json is not None: + parser.error("setmeta --clear accepts only a document target") + metadata = {} + else: + if args.metadata_json is None: + parser.error("setmeta requires a JSON object") + metadata = json.loads(args.metadata_json) + if not isinstance(metadata, dict): + parser.error("setmeta metadata must be a JSON object") + + filesystem = _filesystem_from_workspace(workspace) + info = filesystem.set_metadata(args.target, metadata, clear=args.clear) + print(json.dumps(info, ensure_ascii=False, sort_keys=True)) + return 0 + + +def _run_set(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="pifs set", + description="Set PageIndex FileSystem CLI defaults", + ) + parser.add_argument("name", choices=["workspace"]) + parser.add_argument("value") + args = parser.parse_args(argv) + + config = _read_config() + if args.name == "workspace": + workspace = Path(args.value).expanduser().resolve(strict=False) + config["workspace"] = str(workspace) + path = _write_config(config) + print(f"workspace: {workspace}") + print(f"config: {path}") + return 0 + raise ValueError(f"unknown config key: {args.name}") + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + _load_env_file() + parser = argparse.ArgumentParser(description="PageIndex FileSystem CLI") + parser.add_argument("--workspace", default=None) + parser.add_argument("--env-file", default=None) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + _load_env_file(args.env_file, workspace=args.workspace) + args.workspace = _resolve_workspace(args.workspace) + + command_tokens = [token for token in args.command if token != "--"] + + if not command_tokens: + parser.error("a filesystem command is required") + + try: + command_name = command_tokens[0] + command_args = command_tokens[1:] + if command_name == "set": + return _run_set(command_args) + if command_name == "ask": + return _run_ask(command_args, workspace_default=args.workspace) + if command_name == "chat": + return _run_chat(command_args, workspace_default=args.workspace) + if command_name == "add": + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_add(command_args, workspace=args.workspace) + if command_name == "setmeta": + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_setmeta(command_args, workspace=args.workspace) + + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_passthrough( + command_tokens, + workspace=args.workspace, + ) + except PIFSCommandError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pageindex/filesystem/commands.py b/pageindex/filesystem/commands.py new file mode 100644 index 000000000..5b31d5c18 --- /dev/null +++ b/pageindex/filesystem/commands.py @@ -0,0 +1,585 @@ +from __future__ import annotations + +import json +import re +import shlex +from typing import Any + +from .core import PageIndexFileSystem + + +class PIFSCommandError(ValueError): + pass + + +class PIFSCommandExecutor: + COMMAND_NAMES = {"ls", "tree", "browse", "stat", "cat", "grep"} + FORBIDDEN_SUBSTRINGS = (";", "`", "$(", "||", "&&", "\n", "\r") + FORBIDDEN_TOKENS = {"|", ">", "<", ">>", "<<", "&"} + BROWSE_PAGE_SIZE = 10 + TREE_VALUE_PAGE_SIZE = 50 + MAX_TREE_FOLDERS = 200 + MAX_GREP_MATCHES = 20 + MAX_PAGE_SPAN = 5 + + def __init__(self, filesystem: PageIndexFileSystem): + self.filesystem = filesystem + + def allowed_commands(self) -> set[str]: + return set(self.COMMAND_NAMES) + + def describe_available_command_surfaces(self) -> str: + return "\n".join( + [ + "Available PIFS BashLike commands:", + "- tree [-L depth] [--page N]: folder and metadata-scope orientation", + "- ls : exact alias for tree -L 1", + '- browse "" [--page N] [--where JSON] [-R]: summary-ranked document discovery', + "- stat : single document identity or scope metadata", + "- cat --structure | --page N[-M]: structure-first document reads", + "- grep : single-document lexical evidence fallback", + ] + ) + + def execute(self, command: str) -> str: + try: + data, next_steps = self._execute(command) + return self._success(data, next_steps=next_steps) + except PIFSCommandError as exc: + return self._error(str(exc)) + except (KeyError, ValueError) as exc: + return self._error(self._clean_error_message(exc)) + except Exception as exc: + return self._error(self._clean_error_message(exc)) + + def _execute(self, command: str) -> tuple[dict[str, Any], list[str]]: + self._validate_raw_command(command) + try: + tokens = shlex.split(command) + except ValueError as exc: + raise PIFSCommandError(f"Invalid command syntax: {exc}") from exc + if not tokens: + raise PIFSCommandError("Empty command") + self._validate_tokens(tokens) + if "--json" in tokens: + raise PIFSCommandError("--json is removed; command output is always JSON") + name, args = tokens[0], tokens[1:] + if name not in self.allowed_commands(): + raise PIFSCommandError(f"Unsupported command: {name}") + return getattr(self, f"_cmd_{name}")(args) + + def _cmd_ls(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if len(args) > 1: + raise PIFSCommandError("ls accepts exactly one optional folder target") + if args and args[0].startswith("-"): + raise PIFSCommandError("ls is only an alias for tree -L 1") + path = args[0] if args else "/" + return self._cmd_tree([path, "-L", "1"]) + + def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + path = "/" + depth = 10 + page = 1 + i = 0 + while i < len(args): + arg = args[i] + if arg == "-L": + i, value = self._option_value(args, i, "tree -L") + depth = self._parse_positive_int(value, "tree -L") + elif arg == "--page": + i, value = self._option_value(args, i, "tree --page") + page = self._parse_positive_int(value, "tree --page") + elif arg.startswith("-"): + raise PIFSCommandError(f"Unsupported tree option: {arg}") + else: + path = arg + i += 1 + scope = self.filesystem.resolve_query_scope(path) + if scope.metadata_axis is not None: + rows, has_more = self.filesystem.scope_metadata_values( + scope, + page=page, + page_size=self.TREE_VALUE_PAGE_SIZE, + ) + root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) + root["folders"] = [ + { + "path": self._join_scope_path( + scope.path, + self.filesystem.encode_scope_segment(row["value"]), + ), + "name": str(row["value"]), + "type": "metadata_value", + "value": row["value"], + "file_count": row["file_count"], + "folders": [], + } + for row in rows + ] + root["children_count"] = len(root["folders"]) + data = { + "tree": root, + "total_folders": len(root["folders"]), + "depth": depth, + "truncated": has_more, + "pagination": { + "page": page, + "page_size": self.TREE_VALUE_PAGE_SIZE, + "has_more": has_more, + "next_page": page + 1 if has_more else None, + }, + } + next_steps = [f'browse {shlex.quote(scope.path)}/ ""'] + return data, next_steps + page_size = self.TREE_VALUE_PAGE_SIZE + needed = page * page_size + 1 + folders = self.filesystem.scope_folders( + scope, + max_depth=depth, + limit=max(self.MAX_TREE_FOLDERS, needed), + ) + files = self.filesystem.scope_files(scope, limit=needed) + axes = self.filesystem.scope_metadata_axes(scope) + tree, has_more = self._folder_tree( + scope, + folders, + files, + axes, + page=page, + page_size=page_size, + ) + data = { + "tree": tree, + "total_folders": len(folders) + len(axes), + "depth": depth, + "truncated": has_more or len(folders) >= self.MAX_TREE_FOLDERS, + } + if has_more or page > 1: + data["pagination"] = { + "page": page, + "page_size": page_size, + "has_more": has_more, + "next_page": page + 1 if has_more else None, + } + next_steps = [f'browse {shlex.quote(scope.path)} ""'] + return data, next_steps + + def _cmd_browse(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + recursive = False + where = None + page = 1 + positionals: list[str] = [] + i = 0 + while i < len(args): + arg = args[i] + if arg in {"-R", "--recursive"}: + recursive = True + elif arg == "--where": + i, where = self._option_value(args, i, "browse --where") + elif arg == "--page": + i, value = self._option_value(args, i, "browse --page") + page = self._parse_positive_int(value, "browse --page") + elif arg == "--space": + raise PIFSCommandError("browse --space is removed; browse uses summary retrieval only") + elif arg in {"--limit", "--offset", "--query"}: + raise PIFSCommandError(f"browse does not support {arg}; use --page N") + elif arg.startswith("-"): + raise PIFSCommandError(f"Unsupported browse option: {arg}") + else: + positionals.append(arg) + i += 1 + if len(positionals) < 2 or not str(positionals[1]).strip(): + raise PIFSCommandError('browse requires a query: browse ""') + if len(positionals) > 2: + raise PIFSCommandError('browse accepts a folder and one quoted query') + path, query = positionals + if not str(path).startswith("/"): + raise PIFSCommandError("browse target must be a PIFS folder path like /documents") + scope = self.filesystem.resolve_query_scope(path) + if scope.metadata_axis is not None: + raise PIFSCommandError( + "Metadata axis paths require @field/value; run tree /@field to inspect values." + ) + merged_filter = self.filesystem.merge_scope_filter(scope, where) + effective_recursive = recursive or bool(scope.metadata_filter) + if not self.filesystem.has_semantic_channel("summary"): + self.filesystem.configure_existing_projection_retrieval() + if not self.filesystem.has_semantic_channel("summary"): + raise PIFSCommandError("browse summary retrieval is not available") + payload = self.filesystem.browse_semantic_files( + scope.path, + query, + retrieval_query=query, + recursive=effective_recursive, + space="summary", + page=page, + page_size=self.BROWSE_PAGE_SIZE, + metadata_filter=where, + ) + documents = [self._document_hit(row) for row in payload.get("data", [])] + next_steps = [] + if payload.get("has_more"): + next_steps.append(self._browse_command(path, query, recursive=recursive, where=where, page=page + 1)) + scope_payload = { + "folder": scope.folder_path, + "recursive": effective_recursive, + "query": query, + "where": self._json_filter(where), + "retrieval": "summary", + } + if scope.path != scope.folder_path or merged_filter is not None: + scope_payload.update( + { + "path": scope.path, + "folder_path": scope.folder_path, + "metadata_filter": merged_filter, + } + ) + return { + "documents": documents, + "pagination": { + "page": page, + "page_size": self.BROWSE_PAGE_SIZE, + "has_more": bool(payload.get("has_more")), + "next_page": page + 1 if payload.get("has_more") else None, + }, + "scope": scope_payload, + }, next_steps + + def _cmd_stat(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if any(arg == "--schema" for arg in args): + raise PIFSCommandError("stat --schema is removed") + if any(arg == "--field" for arg in args): + raise PIFSCommandError("stat --field is removed") + if any(arg.startswith("-") for arg in args): + raise PIFSCommandError("stat accepts only one document target") + if len(args) != 1: + raise PIFSCommandError("stat accepts exactly one document target") + target = args[0] + return {"document": self._document_stat(target)}, [] + + def _cmd_cat(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if not args: + raise PIFSCommandError("cat requires a document target") + target = args[0] + if target.startswith("-"): + raise PIFSCommandError("cat syntax is target-first: cat --structure or cat --page N[-M]") + if "--all" in args or "--range" in args: + raise PIFSCommandError("cat --all and cat --range are removed; use --structure or --page") + if len(args) == 2 and args[1] == "--structure": + payload = self.filesystem.pageindex_structure(target) + return { + "document": self._document_from_structural_payload(payload, target), + "structure": payload.get("structure") if payload.get("available", True) else None, + "pagination": {"available": bool(payload.get("available", True))}, + }, self._page_next_steps(target, payload) + if len(args) == 3 and args[1] == "--page": + pages = args[2] + if not re.fullmatch(r"\d+(?:-\d+)?", pages): + raise PIFSCommandError("cat --page requires one page selector like 31 or 31-33") + start, end = self._parse_numeric_range(pages, "cat --page") + if end - start + 1 > self.MAX_PAGE_SPAN: + raise PIFSCommandError(f"cat --page supports at most {self.MAX_PAGE_SPAN} pages") + payload = self.filesystem.pageindex_pages(target, pages) + return { + "document": self._document_from_structural_payload(payload, target), + "requested_pages": pages, + "returned_pages": payload.get("data", []), + "content": { + "text": payload.get("text", ""), + "available": bool(payload.get("available", True)), + }, + }, [] + raise PIFSCommandError("cat requires either --structure or --page N[-M]") + + def _cmd_grep(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if any(arg in {"-R", "-r", "--recursive", "--where"} for arg in args): + raise PIFSCommandError("grep is single-document only: grep ") + if any(arg.startswith("-") for arg in args): + raise PIFSCommandError("grep accepts no options") + if len(args) != 2: + raise PIFSCommandError("grep requires a query and one document target") + query, target = args + if self._is_folder(target): + raise PIFSCommandError("grep requires a resolved file locator, not a folder") + return { + "document": self._document_stat(target), + "matches": self._grep_file_matches(target, query), + }, [] + + def _folder_tree( + self, + scope: Any, + folders: list[dict[str, Any]], + files: list[dict[str, Any]], + axes: list[dict[str, Any]], + *, + page: int, + page_size: int, + ) -> tuple[dict[str, Any], bool]: + root_key = self._normalize_folder_path(scope.folder_path) + root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) + nodes = { + root_key: root + } + for folder in sorted(folders, key=lambda item: item["path"]): + physical_path = self._normalize_folder_path(folder["path"]) + nodes[physical_path] = { + "path": self._scoped_folder_path(scope, physical_path), + "name": folder.get("name") or physical_path.rsplit("/", 1)[-1], + "type": "folder", + "file_count": folder.get("matched_files", 0) or folder.get("file_count", 0), + "children_count": folder.get("children_count", 0), + "folders": [], + } + for folder_path, node in sorted(nodes.items(), key=lambda item: item[0].count("/")): + if folder_path == root_key: + continue + parent = folder_path.rsplit("/", 1)[0] or "/" + nodes.get(parent, nodes[root_key])["folders"].append(node) + axis_nodes = [ + { + "path": self._join_scope_path( + scope.path, + f"@{self.filesystem.encode_scope_segment(axis['name'])}", + ), + "name": f"@{axis['name']}", + "type": "metadata_axis", + "value_count": axis["value_count"], + "folders": [], + } + for axis in axes + ] + child_items = ( + [("folder", node) for node in nodes[root_key]["folders"]] + + [("file", file) for file in files] + + [("folder", node) for node in axis_nodes] + ) + offset = (page - 1) * page_size + page_items = child_items[offset : offset + page_size] + nodes[root_key]["folders"] = [node for kind, node in page_items if kind == "folder"] + nodes[root_key]["files"] = [node for kind, node in page_items if kind == "file"] + nodes[root_key]["children_count"] = len(child_items) + return nodes[root_key], len(child_items) > offset + page_size + + def _document_hit(self, row: dict[str, Any]) -> dict[str, Any]: + return { + "path": row.get("path"), + "title": row.get("title"), + "summary": row.get("summary", ""), + "metadata": row.get("metadata", {}), + } + + def _document_stat(self, target: str) -> dict[str, Any]: + info = dict(self.filesystem._stat(target)) + return { + "path": target if str(target).startswith("/") else info.get("path"), + "file_ref": info.get("file_ref"), + "document_id": info.get("external_id") or info.get("document_id"), + "status": info.get("pageindex_tree_status") or info.get("status"), + "page_count": info.get("pageNum"), + "folders": info.get("folders", []), + "metadata": info.get("metadata", {}), + "metadata_status": info.get("metadata_status", {}), + "pageindex_doc_id": info.get("pageindex_doc_id"), + "content_type": info.get("content_type"), + "title": info.get("title") or info.get("name"), + } + + def _document_from_structural_payload(self, payload: dict[str, Any], target: str) -> dict[str, Any]: + document = { + "file_ref": payload.get("file_ref"), + "document_id": payload.get("external_id"), + "status": payload.get("status"), + "pageindex_doc_id": payload.get("pageindex_doc_id"), + "available": bool(payload.get("available", True)), + } + if payload.get("message"): + document["message"] = payload.get("message") + try: + document.update({k: v for k, v in self._document_stat(target).items() if v is not None}) + except (KeyError, ValueError): + pass + return document + + def _page_next_steps(self, target: str, payload: dict[str, Any]) -> list[str]: + if not payload.get("available", True): + return [] + return [f"cat {shlex.quote(target)} --page "] + + def _grep_file_matches(self, target: str, query: str) -> list[dict[str, Any]]: + file_ref = self.filesystem._resolve_target(target) + matches = [] + for line_number, line in enumerate(self.filesystem.store.read_text(file_ref).splitlines(), 1): + if self._line_matches(line, query): + matches.append({"line": line_number, "text": self._compact_text(line, max_chars=220)}) + if len(matches) >= self.MAX_GREP_MATCHES: + break + return matches + + def _is_folder(self, path: str) -> bool: + try: + self.filesystem.folder_info(path) + return True + except KeyError: + return False + + def _browse_command( + self, + path: str, + query: str, + *, + recursive: bool, + where: str | None, + page: int, + ) -> str: + parts = ["browse"] + if recursive: + parts.append("-R") + parts.extend([shlex.quote(self._normalize_folder_path(path)), shlex.quote(query)]) + if where is not None: + parts.extend(["--where", shlex.quote(where)]) + parts.extend(["--page", str(page)]) + return " ".join(parts) + + @staticmethod + def _json_filter(value: str | None) -> Any: + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + @staticmethod + def _line_matches(line: str, query: str) -> bool: + haystack = line.lower() + needle = query.lower().strip() + if needle and needle in haystack: + return True + terms = [term for term in re.findall(r"[A-Za-z0-9_]+", needle) if term] + return bool(terms) and all(term in haystack for term in terms) + + @staticmethod + def _compact_text(text: str, *, max_chars: int) -> str: + collapsed = re.sub(r"\s+", " ", text or "").strip() + if len(collapsed) <= max_chars: + return collapsed + return collapsed[: max_chars - 3].rstrip() + "..." + + @staticmethod + def _parse_numeric_range(value: str, label: str) -> tuple[int, int]: + try: + if "-" in value: + left, right = value.split("-", 1) + start, end = int(left), int(right) + else: + start = end = int(value) + except ValueError as exc: + raise PIFSCommandError(f"{label} requires a numeric range") from exc + if start < 1 or end < start: + raise PIFSCommandError(f"Invalid {label} range: {value}") + return start, end + + @staticmethod + def _option_value(args: list[str], index: int, label: str) -> tuple[int, str]: + value_index = index + 1 + if value_index >= len(args): + raise PIFSCommandError(f"{label} requires a value") + return value_index, args[value_index] + + @staticmethod + def _parse_positive_int(value: str, label: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise PIFSCommandError(f"{label} must be an integer") from exc + if parsed < 1: + raise PIFSCommandError(f"{label} must be at least 1") + return parsed + + @staticmethod + def _normalize_folder_path(path: str) -> str: + value = str(path or "/").strip() + if not value or value == "/": + return "/" + return "/" + value.strip("/") + + def _scope_root(self, scope: Any, *, file_count: int) -> dict[str, Any]: + if getattr(scope, "metadata_axis", None): + name = f"@{scope.metadata_axis}" + node_type = "metadata_axis" + elif getattr(scope, "metadata_filter", None): + name = str(next(reversed(scope.metadata_filter.values()))) + node_type = "metadata_value" + else: + info = self.filesystem.folder_info(scope.folder_path) + name = info.get("name") or ("/" if scope.folder_path == "/" else scope.folder_path.rsplit("/", 1)[-1]) + node_type = "folder" + return { + "path": scope.path, + "name": name, + "type": node_type, + "file_count": file_count, + "children_count": 0, + "folders": [], + } + + def _scoped_folder_path(self, scope: Any, folder_path: str) -> str: + path = self._normalize_folder_path(folder_path) + for field, value in getattr(scope, "metadata_filter", {}).items(): + path = self._join_scope_path( + path, + f"@{self.filesystem.encode_scope_segment(field)}", + ) + path = self._join_scope_path( + path, + self.filesystem.encode_scope_segment(value), + ) + return path + + @staticmethod + def _join_scope_path(base: str, segment: str) -> str: + base_path = str(base or "/").rstrip("/") + if not base_path: + base_path = "/" + if base_path == "/": + return f"/{segment}" + return f"{base_path}/{segment}" + + @staticmethod + def _success(data: dict[str, Any], *, next_steps: list[str] | None = None) -> str: + return json.dumps( + {"success": True, "data": data, "next_steps": next_steps or []}, + ensure_ascii=False, + ) + + @staticmethod + def _error(message: str) -> str: + return json.dumps( + { + "success": False, + "error": {"code": "invalid_command", "message": message}, + "next_steps": [], + }, + ensure_ascii=False, + ) + + @classmethod + def _validate_raw_command(cls, command: str) -> None: + if not command.strip(): + raise PIFSCommandError("Empty command") + if any(token in command for token in cls.FORBIDDEN_SUBSTRINGS): + raise PIFSCommandError("Only PageIndex FileSystem commands are allowed") + + @classmethod + def _validate_tokens(cls, tokens: list[str]) -> None: + if any(token in cls.FORBIDDEN_TOKENS for token in tokens): + raise PIFSCommandError("Only PageIndex FileSystem commands are allowed") + + @staticmethod + def _clean_error_message(exc: BaseException) -> str: + message = str(exc) + if isinstance(exc, KeyError) and len(exc.args) == 1: + message = str(exc.args[0]) + return message or exc.__class__.__name__ diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py new file mode 100644 index 000000000..9f5767ed5 --- /dev/null +++ b/pageindex/filesystem/core.py @@ -0,0 +1,1961 @@ +from __future__ import annotations + +import json +import os +import shutil +import tempfile +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any, Optional, Union +from urllib.parse import quote, unquote, urlparse + +from .metadata import MetadataQueryEngine +from .store import ( + SQLiteFileSystemStore, + fingerprint, + make_file_ref, + metadata_text, + normalize_path, +) +from .types import OpenResult, PIFSQueryScope, SearchResult + +if TYPE_CHECKING: + from ..client import PageIndexClient + +PROJECTION_INDEX_STATUSES = { + "not_indexed", + "pending_index", + "generated", + "ready", + "failed", +} + +DEFAULT_EMBEDDING_DIMENSIONS = 1024 +SEMANTIC_RETRIEVAL_CHANNELS = ("summary",) +SEMANTIC_PROJECTION_INDEX_NAMES = { + "summary": "summary", +} +PAGEINDEX_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown"} +PAGEINDEX_DOCUMENT_CONTENT_TYPES = { + "application/pdf", + "text/markdown", + "text/x-markdown", + "application/markdown", +} +ADD_FILE_CONTENT_TYPES = { + ".pdf": "application/pdf", + ".md": "text/markdown", + ".markdown": "text/markdown", +} + + +def strip_pageindex_text_fields(value: Any) -> Any: + if isinstance(value, list): + return [strip_pageindex_text_fields(item) for item in value] + if isinstance(value, dict): + return { + key: strip_pageindex_text_fields(item) + for key, item in value.items() + if key != "text" + } + return value + + +class PageIndexFileSystem: + def __init__( + self, + workspace: Union[str, Path], + *, + summary_projection_index_dir: Union[str, Path, None] = None, + summary_projection_embedding_provider: str = "openai", + summary_projection_embedding_model: str = "text-embedding-3-small", + summary_projection_embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + summary_projection_embedding_timeout: float = 60, + summary_projection_embedding_api_key: str | None = None, + summary_projection_embedding_base_url: str | None = None, + ): + self.workspace = Path(workspace).expanduser() + self.store = SQLiteFileSystemStore(self.workspace) + self.metadata = MetadataQueryEngine(self.store) + self.semantic_retrieval_backend: Any | None = None + self.summary_projection_indexer: Any | None = None + self.summary_projection_index_dir = ( + Path(summary_projection_index_dir).expanduser() + if summary_projection_index_dir is not None + else self.workspace / "artifacts" / "projection_indexes" + ) + self.summary_projection_embedding_provider = summary_projection_embedding_provider + self.summary_projection_embedding_model = summary_projection_embedding_model + self.summary_projection_embedding_dimensions = summary_projection_embedding_dimensions + self.summary_projection_embedding_timeout = summary_projection_embedding_timeout + self.summary_projection_embedding_api_key = summary_projection_embedding_api_key + self.summary_projection_embedding_base_url = summary_projection_embedding_base_url + + def register_file( + self, + *, + storage_uri: str, + folder_path: Optional[str] = None, + metadata: Optional[dict[str, Any]] = None, + external_id: Optional[str] = None, + title: Optional[str] = None, + content: str = "", + content_type: str | None = None, + source_type: Optional[str] = None, + ) -> str: + return self.register_files( + [ + { + "storage_uri": storage_uri, + "folder_path": folder_path, + "metadata": metadata, + "external_id": external_id, + "title": title, + "content": content, + "content_type": content_type, + "source_type": source_type, + } + ] + )[0] + + def register(self, **kwargs: Any) -> str: + self._ensure_register_completion_defaults() + return self.register_file(**kwargs) + + def add_file( + self, + physical_path: Union[str, Path], + virtual_target: Union[str, Path], + ) -> dict[str, Any]: + source = Path(physical_path).expanduser() + if not source.is_file(): + raise FileNotFoundError(f"Source file not found: {source}") + suffix = source.suffix.lower() + content_type = ADD_FILE_CONTENT_TYPES.get(suffix) + if content_type is None: + supported = ", ".join(sorted(ADD_FILE_CONTENT_TYPES)) + raise ValueError( + f"Unsupported file type: {suffix or ''}; supported: {supported}" + ) + + folder_path, filename, virtual_path = self._resolve_add_target( + virtual_target, + physical_basename=source.name, + physical_suffix=suffix, + ) + if self.store.file_basename_exists_in_folder(folder_path, filename): + raise FileExistsError(f"File already exists at {virtual_path}") + self._ensure_add_completion_defaults() + add_created_folder_paths = self._add_created_folder_paths(folder_path) + file_ref = make_file_ref(virtual_path.strip("/")) + uploads_dir = self.workspace / "artifacts" / "uploads" + final_dir = uploads_dir / file_ref + final_path = final_dir / filename + final_dir_created = False + catalog_inserted = False + records: list[dict[str, Any]] = [] + preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() + + uploads_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".add-{file_ref}-", dir=uploads_dir) as tmp: + temp_path = Path(tmp) / filename + try: + shutil.copy2(source, temp_path) + if final_dir.exists(): + raise FileExistsError( + f"Workspace artifact already exists for {virtual_path}: {final_dir}" + ) + final_dir.mkdir(parents=True) + final_dir_created = True + os.replace(temp_path, final_path) + + record = self._prepare_file_record( + { + "storage_uri": final_path.as_uri(), + "folder_path": folder_path, + "metadata": {}, + "external_id": None, + "title": filename, + "content": self._add_file_content(final_path, content_type), + "content_type": content_type, + } + ) + records = [record] + self._require_add_pageindex_ready(record) + self._register_custom_metadata_fields(records) + self.store.insert_files(records) + catalog_inserted = True + if self._complete_summary_projection_index(record): + self.store.update_file_metadata_status( + record["file_ref"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ) + self._require_add_summary_projection_ready(record) + self._sync_owned_raw_artifact(record) + self._ensure_add_semantic_retrieval_ready() + except Exception: + if catalog_inserted: + self._cleanup_add_catalog_record(file_ref) + self._cleanup_add_summary_projection(records) + self._cleanup_failed_register_artifacts(records) + self._cleanup_add_pageindex_cache(records, preexisting_pageindex_doc_ids) + self._cleanup_add_created_folders(add_created_folder_paths) + if final_dir_created: + shutil.rmtree(final_dir, ignore_errors=True) + raise + + info = self.store.file_info(file_ref) + info["path"] = virtual_path + return info + + def register_files(self, files: list[dict[str, Any]]) -> list[str]: + records = [self._prepare_file_record(file) for file in files] + preexisting_file_refs = self._existing_file_refs(records) + new_records = [ + record for record in records if record["file_ref"] not in preexisting_file_refs + ] + try: + self._register_custom_metadata_fields(records) + self.store.insert_files(records) + for record in records: + try: + if self._complete_summary_projection_index(record): + self.store.update_file_metadata_status( + record["file_ref"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ) + self._sync_owned_raw_artifact(record) + except KeyError: + continue + except Exception: + self._cleanup_add_summary_projection(new_records) + for record in new_records: + self._cleanup_add_catalog_record(str(record["file_ref"])) + self._cleanup_failed_register_artifacts(records) + raise + return [record["file_ref"] for record in records] + + def _ensure_register_completion_defaults(self) -> None: + if self.summary_projection_indexer is None: + from .semantic_projection import SummaryProjectionIndexer + + self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( + self.summary_projection_index_dir, + embedding_provider=self.summary_projection_embedding_provider, + embedding_model=self.summary_projection_embedding_model, + embedding_dimensions=self.summary_projection_embedding_dimensions, + embedding_timeout=self.summary_projection_embedding_timeout, + embedding_api_key=self.summary_projection_embedding_api_key, + embedding_base_url=self.summary_projection_embedding_base_url, + ) + if self.semantic_retrieval_backend is None: + self.configure_semantic_projection_retrieval( + self.summary_projection_index_dir, + embedding_provider=self.summary_projection_embedding_provider, + embedding_model=self.summary_projection_embedding_model, + embedding_dimensions=self.summary_projection_embedding_dimensions, + embedding_timeout=self.summary_projection_embedding_timeout, + embedding_api_key=self.summary_projection_embedding_api_key, + embedding_base_url=self.summary_projection_embedding_base_url, + ) + + def _ensure_add_completion_defaults(self) -> None: + if self.summary_projection_indexer is None: + from .semantic_projection import SummaryProjectionIndexer + + self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( + self.summary_projection_index_dir, + embedding_provider=self.summary_projection_embedding_provider, + embedding_model=self.summary_projection_embedding_model, + embedding_dimensions=self.summary_projection_embedding_dimensions, + embedding_timeout=self.summary_projection_embedding_timeout, + embedding_api_key=self.summary_projection_embedding_api_key, + embedding_base_url=self.summary_projection_embedding_base_url, + ) + + def _ensure_add_semantic_retrieval_ready(self) -> None: + indexer = self.summary_projection_indexer + if indexer is None: + raise RuntimeError("pifs add requires a summary projection indexer") + from .semantic_projection import SemanticProjectionSearchBackend + + index_dir = Path(getattr(indexer, "index_dir", self.summary_projection_index_dir)) + embedder = getattr(indexer, "embedder", None) + if embedder is None: + self.configure_semantic_projection_retrieval( + index_dir, + embedding_provider=str( + getattr( + indexer, + "embedding_provider", + self.summary_projection_embedding_provider, + ) + ), + embedding_model=str( + getattr(indexer, "embedding_model", self.summary_projection_embedding_model) + ), + embedding_dimensions=int( + getattr( + indexer, + "embedding_dimensions", + self.summary_projection_embedding_dimensions, + ) + ), + embedding_timeout=self.summary_projection_embedding_timeout, + embedding_api_key=self.summary_projection_embedding_api_key, + embedding_base_url=self.summary_projection_embedding_base_url, + ) + else: + embedding_cache = getattr(indexer, "embedding_cache", None) + self.semantic_retrieval_backend = SemanticProjectionSearchBackend( + index_dir, + embedder=embedder, + embedding_provider=str( + getattr( + indexer, + "embedding_provider", + self.summary_projection_embedding_provider, + ) + ), + embedding_model=str( + getattr(indexer, "embedding_model", self.summary_projection_embedding_model) + ), + embedding_dimensions=int( + getattr( + indexer, + "embedding_dimensions", + self.summary_projection_embedding_dimensions, + ) + ), + embedding_cache_path=getattr(embedding_cache, "db_path", None), + ) + if "summary" not in self.semantic_retrieval_channels(): + raise RuntimeError("pifs add failed to configure summary semantic retrieval") + + def _add_created_folder_paths(self, folder_path: str) -> list[str]: + paths = self._folder_ancestor_paths(folder_path) + return [path for path in paths if not self.store.folder_exists(path)] + + @staticmethod + def _folder_ancestor_paths(folder_path: str) -> list[str]: + normalized = normalize_path(folder_path) + if normalized == "/": + return [] + segments = [segment for segment in normalized.strip("/").split("/") if segment] + paths: list[str] = [] + for index in range(1, len(segments) + 1): + paths.append("/" + "/".join(segments[:index])) + return paths + + def configure_existing_projection_retrieval(self) -> bool: + """Attach semantic retrieval to already-built projection indexes. + + Register-time generation owns building the index files. Opening an + existing workspace should still expose semantic retrieval when the + configured embedding dimensions match the existing index. + """ + if self.semantic_retrieval_backend is not None: + return bool(self.semantic_retrieval_channels()) + index_config = self._existing_projection_index_config() + if index_config is None: + return False + existing_dimension = int(index_config.get("dimension") or 0) + if existing_dimension != self.summary_projection_embedding_dimensions: + raise RuntimeError( + "summary projection index dimension mismatch: " + f"{index_config.get('db_path') or self.summary_projection_index_dir} " + f"was built with dimension {existing_dimension}, but configured " + "summary_projection_embedding_dimensions is " + f"{self.summary_projection_embedding_dimensions}. Rebuild the " + "projection index or use a matching embedding configuration." + ) + self.configure_semantic_projection_retrieval( + self.summary_projection_index_dir, + embedding_provider=self.summary_projection_embedding_provider, + embedding_model=self.summary_projection_embedding_model, + embedding_dimensions=self.summary_projection_embedding_dimensions, + embedding_timeout=self.summary_projection_embedding_timeout, + embedding_api_key=self.summary_projection_embedding_api_key, + embedding_base_url=self.summary_projection_embedding_base_url, + ) + return bool(self.semantic_retrieval_channels()) + + def _existing_projection_index_config(self) -> dict[str, Any] | None: + for channel in SEMANTIC_RETRIEVAL_CHANNELS: + index_name = SEMANTIC_PROJECTION_INDEX_NAMES.get(channel) + if not index_name: + continue + index_path = self.summary_projection_index_dir / f"{index_name}.sqlite" + if not index_path.exists(): + continue + from .semantic_index import SQLiteVecSemanticIndex + + try: + info = SQLiteVecSemanticIndex(index_path).info() + except Exception: + continue + if int(info.get("document_count") or 0) <= 0: + continue + metadata = dict(info.get("metadata") or {}) + if metadata.get("channel") and metadata.get("channel") != channel: + continue + return info + return None + + def browse( + self, + path: str = "/", + recursive: bool = False, + limit: int = 100, + max_depth: int | None = None, + ) -> dict[str, list[dict[str, Any]]]: + return self.store.list_folder( + path, + recursive=recursive, + limit=limit, + max_depth=max_depth, + ) + + def resolve_query_scope(self, path: str) -> PIFSQueryScope: + normalized = normalize_path(path) + if self._folder_exists(normalized): + return PIFSQueryScope(path=normalized, folder_path=normalized) + + parts = [part for part in normalized.strip("/").split("/") if part] + folder_path = "/" + remainder = parts + for index in range(len(parts), -1, -1): + candidate = "/" + "/".join(parts[:index]) if index else "/" + if self._folder_exists(candidate): + folder_path = candidate + remainder = parts[index:] + break + + if not remainder: + return PIFSQueryScope(path=normalized, folder_path=folder_path) + + metadata_filter: dict[str, str] = {} + index = 0 + while index < len(remainder): + segment = remainder[index] + if not segment.startswith("@"): + if index == 0: + raise KeyError(f"Unknown folder path: {normalized}") + raise ValueError( + "Metadata axes must come after the physical folder prefix; " + "inspect the physical folder first, then append @field/value buckets. " + "Use the path returned by tree for values containing '/'." + ) + field = unquote(segment[1:]) + self.metadata.validate_field_name(field) + if not self.store.metadata_field_exists(field): + raise ValueError("Unknown metadata axis; run tree to inspect available @field axes.") + if field in metadata_filter: + raise ValueError( + "A metadata field can appear only once in a scope path; " + "choose one value or use browse --where for advanced predicates." + ) + if index + 1 >= len(remainder): + return PIFSQueryScope( + path=normalized, + folder_path=folder_path, + metadata_filter=metadata_filter, + metadata_axis=field, + ) + value_segment = remainder[index + 1] + if value_segment.startswith("@"): + raise ValueError( + "Metadata axis paths require @field/value; run tree /@field to inspect values." + ) + metadata_filter[field] = unquote(value_segment) + index += 2 + + return PIFSQueryScope( + path=normalized, + folder_path=folder_path, + metadata_filter=metadata_filter, + ) + + def merge_scope_filter( + self, + scope: PIFSQueryScope, + metadata_filter: dict[str, Any] | str | None, + ) -> dict[str, Any] | None: + parsed = self.metadata.parse_filter(metadata_filter) + if scope.metadata_axis is not None: + raise ValueError( + "Metadata axis paths require @field/value; run tree /@field to inspect values." + ) + if not parsed: + return dict(scope.metadata_filter) or None + overlap = set(scope.metadata_filter).intersection(self.metadata.filter_fields(parsed)) + if overlap: + raise ValueError( + "Do not constrain the same metadata field in both the path and --where; " + "move the predicate into one place." + ) + if not scope.metadata_filter: + return parsed + return {**scope.metadata_filter, **parsed} + + def scope_file_count(self, scope: PIFSQueryScope) -> int: + return self.store.count_files( + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + ) + + def scope_folders( + self, + scope: PIFSQueryScope, + *, + max_depth: int | None, + limit: int, + ) -> list[dict[str, Any]]: + return self.store.find_folders( + scope.folder_path, + metadata_filter=scope.metadata_filter or None, + limit=limit, + max_depth=max_depth, + include_self=False, + ) + + def scope_metadata_axes(self, scope: PIFSQueryScope) -> list[dict[str, Any]]: + return self.store.list_metadata_axes( + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + exclude_fields=set(scope.metadata_filter), + ) + + def scope_metadata_values( + self, + scope: PIFSQueryScope, + *, + page: int, + page_size: int, + ) -> tuple[list[dict[str, Any]], bool]: + if scope.metadata_axis is None: + return [], False + rows = self.store.list_metadata_values( + scope.metadata_axis, + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + limit=page_size + 1, + offset=(page - 1) * page_size, + ) + has_more = len(rows) > page_size + return rows[:page_size], has_more + + def scope_files(self, scope: PIFSQueryScope, *, limit: int) -> list[dict[str, Any]]: + leaf_items = self._scope_file_leaf_items(scope, limit=self.scope_file_count(scope) + 1) + locator_leaf_by_file_ref = self._scope_locator_leaf_by_file_ref(leaf_items) + files = [] + for row, leaf in leaf_items: + locator_leaf = locator_leaf_by_file_ref[row["file_ref"]] + files.append( + { + "path": self._scope_file_locator(scope, locator_leaf), + "name": locator_leaf, + "type": "file", + "file_ref": row["file_ref"], + "document_id": row["document_id"], + "title": leaf, + "metadata": row["metadata"], + } + ) + files = sorted(files, key=lambda item: (str(item["name"]).lower(), item["path"], item["file_ref"])) + return files[:limit] + + def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> str: + leaf_items = self._scope_file_leaf_items(scope, limit=self.scope_file_count(scope) + 1) + return self._scope_file_locator( + scope, + self._scope_locator_leaf_by_file_ref(leaf_items).get(file_ref, leaf), + ) + + def _scope_file_leaf_items(self, scope: PIFSQueryScope, *, limit: int) -> list[tuple[dict[str, Any], str]]: + recursive = bool(scope.metadata_filter) + rows = self.store.search_files( + None, + scope={"folder_path": scope.folder_path, "recursive": recursive}, + metadata_filter=scope.metadata_filter or None, + limit=limit, + ) + items = [] + for row in rows: + folder_paths = [ + folder["path"] + for folder in self.store.folder_memberships(row["file_ref"]) + ] + folder_path = self._preferred_folder_path( + folder_paths, + scope.folder_path, + row["folder_path"], + ) + leaf = self.store.membership_display_name(row["file_ref"], folder_path) or row["title"] + items.append((row, leaf)) + return items + + @staticmethod + def _scope_leaf_counts(leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, int]: + counts: dict[str, int] = {} + for _, leaf in leaf_items: + counts[leaf] = counts.get(leaf, 0) + 1 + return counts + + @staticmethod + def _disambiguated_scope_leaf(leaf: str, file_ref: str, leaf_counts: dict[str, int]) -> str: + if leaf_counts.get(leaf, 0) <= 1: + return leaf + return f"{leaf}~{file_ref}" + + @classmethod + def _scope_locator_leaf_by_file_ref(cls, leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, str]: + leaf_counts = cls._scope_leaf_counts(leaf_items) + used: set[str] = set() + locator_leaf_by_file_ref: dict[str, str] = {} + for row, leaf in sorted(leaf_items, key=lambda item: (item[1].lower(), item[1], item[0]["file_ref"])): + base = cls._disambiguated_scope_leaf(leaf, row["file_ref"], leaf_counts) + locator_leaf = base + suffix = 2 + while locator_leaf in used: + locator_leaf = f"{base}~{suffix}" + suffix += 1 + used.add(locator_leaf) + locator_leaf_by_file_ref[row["file_ref"]] = locator_leaf + return locator_leaf_by_file_ref + + def scope_stat(self, path: str) -> dict[str, Any]: + scope = self.resolve_query_scope(path) + data = { + "path": scope.path, + "folder_path": scope.folder_path, + "metadata_filter": dict(scope.metadata_filter), + "file_count": self.scope_file_count(scope), + "available_axes": [item["name"] for item in self.scope_metadata_axes(scope)], + } + if scope.metadata_axis is not None: + data["metadata_axis"] = scope.metadata_axis + return data + + @staticmethod + def encode_scope_segment(segment: Any) -> str: + return quote(str(segment), safe="") + + def browse_semantic_files( + self, + path: str, + query: str, + *, + retrieval_query: str | None = None, + recursive: bool = False, + space: str = "summary", + page: int = 1, + page_size: int = 10, + metadata_filter: Optional[dict[str, Any] | str] = None, + ) -> dict[str, Any]: + path = normalize_path(path) + query_scope = self.resolve_query_scope(path) + self.store.folder_info(query_scope.folder_path) + query_text = self._query_text(retrieval_query or query).strip() + if not query_text: + raise ValueError("browse requires a query") + if page < 1: + raise ValueError("browse --page must be at least 1") + if page_size < 1: + raise ValueError("browse page_size must be at least 1") + if space not in SEMANTIC_RETRIEVAL_CHANNELS: + raise ValueError( + "Unsupported browse --space: " + f"{space}. Supported spaces: {', '.join(SEMANTIC_RETRIEVAL_CHANNELS)}" + ) + available_spaces = self.semantic_retrieval_channels() + if space not in available_spaces: + available = ", ".join(available_spaces) if available_spaces else "none" + raise ValueError( + f"browse --space {space} is not available; available spaces: {available}" + ) + search_channel = getattr(self.semantic_retrieval_backend, "search_channel", None) + if search_channel is None: + available = ", ".join(available_spaces) if available_spaces else "none" + raise ValueError( + f"browse --space {space} is not available; available spaces: {available}" + ) + parsed_filter = self.merge_scope_filter(query_scope, metadata_filter) + effective_recursive = recursive or bool(query_scope.metadata_filter) + scope = {"folder_path": query_scope.folder_path, "recursive": effective_recursive} + scope_file_refs = self.store.file_refs_for_scope( + scope=scope, + metadata_filter=parsed_filter, + ) + offset = (page - 1) * page_size + needed = offset + page_size + 1 + semantic_filters = {"file_ref": scope_file_refs} + candidates = ( + search_channel( + space, + query_text, + limit=needed, + filters=semantic_filters, + ) + if scope_file_refs + else [] + ) + scope_file_ref_set = set(scope_file_refs) + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for candidate in candidates: + try: + file_ref = self.store.resolve_file_ref(candidate.document_id) + except KeyError: + continue + if file_ref in seen: + continue + if file_ref not in scope_file_ref_set: + continue + if not self.store.file_matches( + file_ref, + scope=scope, + metadata_filter=parsed_filter, + ): + continue + entry = self.store.get_file(file_ref) + folder_paths = [ + folder["path"] + for folder in self.store.folder_memberships(file_ref) + ] + folder_path = self._preferred_folder_path( + folder_paths, + query_scope.folder_path, + entry.folder_path, + ) + display_title = self.store.membership_display_name(file_ref, folder_path) or entry.title + try: + if query_scope.metadata_filter: + stable_path = self.scope_file_locator(query_scope, file_ref, display_title) + else: + stable_path = self._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + except RuntimeError: + continue + seen.add(file_ref) + rank = len(rows) + 1 + rows.append( + { + "rank": rank, + "similarity": self._semantic_candidate_similarity(candidate), + "score": self._semantic_candidate_score(candidate), + "path": stable_path, + "file_ref": file_ref, + "document_id": entry.external_id, + "external_id": entry.external_id, + "title": display_title, + "original_title": entry.title, + "folder_path": folder_path, + "folder_paths": folder_paths, + "summary": str((entry.metadata or {}).get("summary") or ""), + "snippet": str(getattr(candidate, "snippet", "") or entry.descriptor), + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "sources": list(getattr(candidate, "sources", []) or []), + } + ) + if len(rows) >= needed: + break + page_rows = rows[offset : offset + page_size] + payload = { + "mode": "files", + "retrieval": f"{space}_vector", + "query": query, + "scope": query_scope.path, + "recursive": effective_recursive, + "space": space, + "available_spaces": list(available_spaces), + "page": page, + "page_size": page_size, + "has_more": len(rows) > offset + page_size, + "data": page_rows, + } + if metadata_filter is not None: + payload["where"] = self._metadata_filter_payload(metadata_filter) + return payload + + def folder_info(self, path: str = "/") -> dict[str, Any]: + return self.store.folder_info(path) + + def find_folders( + self, + path: str = "/", + metadata_filter: Optional[dict[str, Any] | str] = None, + limit: int = 100, + max_depth: int | None = None, + include_self: bool = False, + ) -> list[dict[str, Any]]: + parsed_filter = self.metadata.parse_filter(metadata_filter) + return self.store.find_folders( + path, + metadata_filter=parsed_filter, + limit=limit, + max_depth=max_depth, + include_self=include_self, + ) + + def create_folder( + self, + path: str, + kind: str = "manual", + description: str = "", + metadata: Optional[dict[str, Any]] = None, + ) -> str: + return self.store.create_folder( + path, + kind=kind, + description=description, + metadata=metadata, + ) + + def attach_file_to_folder( + self, + file_ref: str, + folder_path_or_id: str, + metadata: Optional[dict[str, Any]] = None, + ) -> None: + self.store.attach_file_to_folder(file_ref, folder_path_or_id, metadata=metadata) + + def attach_files_to_folders(self, items: list[dict[str, Any]]) -> None: + self.store.attach_files_to_folders(items) + + def set_metadata( + self, + target: str, + metadata: dict[str, Any], + *, + clear: bool = False, + ) -> dict[str, Any]: + if not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + if "summary" in metadata: + raise ValueError("setmeta cannot edit PageIndex summary") + file_ref = self._resolve_target(target) + info = self.store.file_info(file_ref) + replacement = {} if clear else dict(metadata) + for name in replacement: + self.metadata.validate_field_name(str(name)) + existing = dict(info.get("metadata") or {}) + summary = existing.get("summary") + if summary: + replacement["summary"] = summary + self._register_custom_metadata_fields([{"metadata": replacement}]) + self.store.update_file_metadata_status( + file_ref, + metadata=replacement, + metadata_status=dict(info.get("metadata_status") or {}), + ) + return self.store.file_info(file_ref) + + def search( + self, + query: Union[str, list[str], None] = None, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any] | str] = None, + limit: int = 10, + ) -> list[SearchResult]: + parsed_filter = self.metadata.parse_filter(metadata_filter) + rows = self.store.search_files( + query, + scope=scope, + metadata_filter=parsed_filter, + limit=limit, + ) + results = [] + scope_path = self._scope_folder_path(scope) + for row in rows: + folder_paths = [ + folder["path"] + for folder in self.store.folder_memberships(row["file_ref"]) + ] + folder_path = self._preferred_folder_path(folder_paths, scope_path, row["folder_path"]) + display_title = self.store.membership_display_name(row["file_ref"], folder_path) or row["title"] + results.append( + SearchResult( + file_ref=row["file_ref"], + external_id=row["external_id"], + title=display_title, + snippet=row["snippet"], + folder_path=folder_path, + folder_paths=folder_paths, + metadata=row["metadata"], + metadata_status=row["metadata_status"], + id=row["id"], + document_id=row["document_id"], + name=display_title, + description=row["description"], + status=row["status"], + pageNum=row["pageNum"], + createdAt=row["createdAt"], + folderId=row["folderId"], + ) + ) + return results + + def configure_semantic_projection_retrieval( + self, + index_dir: Union[str, Path], + *, + embedding_provider: str = "openai", + embedding_model: str = "text-embedding-3-small", + embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + embedding_timeout: float = 60, + embedding_api_key: str | None = None, + embedding_base_url: str | None = None, + fetch_multiplier: int = 100, + ) -> Any: + from .semantic_projection import SemanticProjectionSearchBackend + + self.semantic_retrieval_backend = SemanticProjectionSearchBackend.from_provider( + index_dir, + embedding_provider=embedding_provider, + embedding_model=embedding_model, + embedding_dimensions=embedding_dimensions, + embedding_timeout=embedding_timeout, + embedding_api_key=embedding_api_key, + embedding_base_url=embedding_base_url, + fetch_multiplier=fetch_multiplier, + ) + return self.semantic_retrieval_backend + + @property + def has_semantic_retrieval_backend(self) -> bool: + return self.semantic_retrieval_backend is not None + + def semantic_retrieval_channels(self) -> tuple[str, ...]: + backend = self.semantic_retrieval_backend + if backend is None: + return () + available_channels = getattr(backend, "available_channels", None) + if callable(available_channels): + raw_channels = available_channels() + else: + raw_channels = getattr(backend, "semantic_tool_channels", ()) + available = set(raw_channels or ()) + return tuple(channel for channel in SEMANTIC_RETRIEVAL_CHANNELS if channel in available) + + def has_semantic_channel(self, channel: str) -> bool: + return channel in self.semantic_retrieval_channels() + + def retrieval_capabilities(self) -> dict[str, Any]: + semantic_channels = ["summary"] if self.has_semantic_channel("summary") else [] + semantic_commands = ["browse"] if semantic_channels else [] + return { + "lexical": { + "grep_recursive": False, + "grep_recursive_semantic_prefilter": False, + "find_maxdepth": False, + }, + "semantic": { + "backend_configured": self.semantic_retrieval_backend is not None, + "channels": semantic_channels, + "commands": semantic_commands, + }, + } + + def open(self, target: str, location: str = "all") -> OpenResult: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: + raise ValueError( + "open() text artifact reads are not supported for PDF/Markdown PageIndex files; " + "use pageindex_structure() or pageindex_pages()." + ) + if str(location).strip().lower() in {"all", "full", "*"}: + return self._open_all(file_ref) + start, end = self._parse_line_range(location) + return self._open_lines(file_ref, start, end) + + def cat_text_artifact(self, target: str, location: str = "all") -> OpenResult: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + self._require_text_artifact_file(entry, "cat --all") + if str(location).strip().lower() in {"all", "full", "*"}: + return self._open_all(file_ref) + start, end = self._parse_line_range(location) + return self._open_lines(file_ref, start, end) + + def pageindex_structure( + self, + target: str, + ) -> dict[str, Any]: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + self._require_pageindex_document_file(entry, "cat --structure") + client, doc_id = self._pageindex_client_doc_for_entry(entry) + if doc_id is None: + return self._structural_unavailable( + "structure", + entry, + message=( + "PageIndex structure is not cached for this file in the " + "PageIndexClient workspace." + ), + ) + structure = self._client_json(client.get_document_structure(doc_id)) + if isinstance(structure, dict) and structure.get("error"): + return self._structural_unavailable( + "structure", + entry, + message=str(structure["error"]), + ) + return { + "mode": "structure", + "file_ref": file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": True, + "pageindex_doc_id": doc_id, + "structure": strip_pageindex_text_fields(structure), + } + + def pageindex_pages(self, target: str, pages: str) -> dict[str, Any]: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + self._require_pageindex_document_file(entry, "cat --page") + client, doc_id = self._pageindex_client_doc_for_entry(entry) + if doc_id is None: + return self._structural_unavailable( + "page", + entry, + pages=pages, + message=( + "PageIndex page content is not cached for this file in the " + "PageIndexClient workspace." + ), + ) + page_entries = self._client_json(client.get_page_content(doc_id, pages)) + if isinstance(page_entries, dict) and page_entries.get("error"): + return self._structural_unavailable( + "page", + entry, + pages=pages, + message=str(page_entries["error"]), + ) + if not isinstance(page_entries, list) or not page_entries: + return self._structural_unavailable( + "page", + entry, + pages=pages, + message="Requested PageIndex page content is not cached for this file.", + ) + text = "\n\n".join(str(page.get("content") or "") for page in page_entries) + return { + "mode": "page", + "file_ref": file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": True, + "pageindex_doc_id": doc_id, + "pages": pages, + "data": page_entries, + "text": text, + } + + def _stat(self, target: str) -> dict[str, Any]: + file_ref = self._resolve_target(target) + return self.store.file_info(file_ref) + + def _require_text_artifact_file(self, entry: Any, command: str) -> None: + if self._file_format(entry) == "text": + return + raise ValueError( + f"{command} is only supported for txt/text files; " + f"got title={entry.title!r}, content_type={entry.content_type!r}. " + "Use cat --structure, " + "or cat --page for PDF/Markdown PageIndex files." + ) + + def _require_pageindex_document_file(self, entry: Any, command: str) -> None: + if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: + return + raise ValueError( + f"{command} is only supported for PDF/Markdown PageIndex files; " + f"got title={entry.title!r}, content_type={entry.content_type!r}. " + "Use grep for single-document lexical evidence." + ) + + @classmethod + def _file_format(cls, entry: Any) -> str: + if getattr(entry, "pageindex_doc_id", None) or entry.pageindex_tree_status != "not_built": + return "pageindex" + file_format = cls._content_format(getattr(entry, "title", ""), entry.content_type) + if file_format != "unsupported": + return file_format + return "unsupported" + + @classmethod + def _content_format(cls, filename: Any, content_type: str | None) -> str: + suffix = Path(str(filename or "")).suffix.lower() + normalized_content_type = cls._normalized_content_type(content_type) + if suffix == ".pdf" or normalized_content_type == "application/pdf": + return "pdf" + if ( + suffix in PAGEINDEX_DOCUMENT_SUFFIXES + or normalized_content_type in PAGEINDEX_DOCUMENT_CONTENT_TYPES + ): + return "markdown" + return "unsupported" + + @staticmethod + def _normalized_content_type(content_type: str | None) -> str: + return str(content_type or "").split(";", 1)[0].strip().lower() + + @property + def pageindex_client_workspace(self) -> Path: + return self.workspace / "artifacts" / "pageindex_client" + + def _pageindex_client(self) -> PageIndexClient: + from ..client import PageIndexClient + + return PageIndexClient(workspace=str(self.pageindex_client_workspace)) + + def _pageindex_client_doc_for_entry(self, entry: Any) -> tuple[PageIndexClient, str | None]: + client = self._pageindex_client() + if not entry.pageindex_doc_id: + return client, None + if entry.pageindex_doc_id not in client.documents: + return client, None + return client, entry.pageindex_doc_id + + def _registration_pageindex_pointer( + self, + *, + storage_uri: str, + title: str, + content_type: str, + ) -> tuple[str | None, str, dict[str, Any] | None]: + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + return None, "not_built", None + client = self._pageindex_client() + local_path = self._canonical_storage_uri_path(storage_uri) + cached_doc_id = self._find_cached_pageindex_doc_id(client, local_path) + if cached_doc_id: + return cached_doc_id, "built", None + if local_path is None: + return None, "failed", self._pageindex_tree_failure_record( + source="PageIndexFileSystem.registration", + error_type="UnresolvableStorageUri", + message=( + "storage_uri must resolve to a local file path for " + "PDF/Markdown registration." + ), + ) + try: + doc_id = client.index(local_path) + return doc_id, "built", None + except Exception as exc: + return None, "failed", self._pageindex_tree_failure_record( + source="PageIndexClient.index", + error_type=exc.__class__.__name__, + message=str(exc) or exc.__class__.__name__, + ) + + @staticmethod + def _pageindex_tree_failure_record( + *, + source: str, + error_type: str, + message: str, + ) -> dict[str, Any]: + return { + "status": "failed", + "owner": "pageindex", + "source": source, + "error_type": error_type, + "message": message, + } + + def _find_cached_pageindex_doc_id( + self, + client: PageIndexClient, + local_path: str | None, + ) -> str | None: + if local_path is None: + return None + for doc_id, doc in client.documents.items(): + if self._canonical_path(doc.get("path")) == local_path: + return doc_id + return None + + def _canonical_storage_uri_path(self, storage_uri: str) -> str | None: + parsed = urlparse(storage_uri) + if parsed.scheme == "file": + return self._canonical_path(unquote(parsed.path)) + if storage_uri and not parsed.scheme: + return self._canonical_path(storage_uri) + return None + + @staticmethod + def _title_from_storage_uri(storage_uri: str) -> str: + parsed = urlparse(str(storage_uri or "")) + path = unquote(parsed.path) if parsed.scheme else str(storage_uri or "") + return Path(path).name + + @classmethod + def _infer_content_type(cls, *, title: str, storage_uri: str) -> str: + for filename in (title, cls._title_from_storage_uri(storage_uri)): + suffix = Path(str(filename or "")).suffix.lower() + if suffix == ".pdf": + return "application/pdf" + if suffix in PAGEINDEX_DOCUMENT_SUFFIXES: + return "text/markdown" + return "application/octet-stream" + + @staticmethod + def _canonical_path(path: Any) -> str | None: + if not path: + return None + return str(Path(os.path.expanduser(str(path))).resolve(strict=False)) + + @staticmethod + def _client_json(payload: str) -> Any: + try: + return json.loads(payload) + except json.JSONDecodeError: + return {"error": f"Invalid PageIndexClient JSON response: {payload}"} + + def _create_folder(self, path: str) -> str: + return self.create_folder(path) + + @classmethod + def _resolve_add_target( + cls, + virtual_target: Union[str, Path], + *, + physical_basename: str, + physical_suffix: str, + ) -> tuple[str, str, str]: + raw_target = str(virtual_target).strip() + if not raw_target: + raise ValueError("pifs add target is required") + normalized = normalize_path(raw_target) + posix_target = PurePosixPath(normalized) + raw_looks_like_folder = raw_target.replace("\\", "/").endswith("/") + target_suffix = posix_target.suffix.lower() + if raw_looks_like_folder or target_suffix not in ADD_FILE_CONTENT_TYPES: + folder_path = normalized + filename = physical_basename + else: + if target_suffix != physical_suffix: + raise ValueError( + "pifs add target file extension must match the physical file extension" + ) + folder_path = normalize_path(str(posix_target.parent)) + filename = posix_target.name + cls._validate_add_filename(filename) + virtual_path = cls._join_virtual_file_path(folder_path, filename) + return folder_path, filename, virtual_path + + @staticmethod + def _validate_add_filename(filename: str) -> None: + if not filename or filename in {".", ".."}: + raise ValueError("pifs add target filename is required") + if "/" in filename or "\\" in filename: + raise ValueError("pifs add target filename must be a basename") + + @staticmethod + def _join_virtual_file_path(folder_path: str, filename: str) -> str: + folder_path = normalize_path(folder_path) + if folder_path == "/": + return f"/{filename}" + return f"{folder_path}/{filename}" + + def _add_file_content(self, path: Path, content_type: str) -> str: + if self._content_format(path.name, content_type) == "markdown": + return path.read_text(encoding="utf-8") + return "" + + def _require_add_pageindex_ready(self, record: dict[str, Any]) -> None: + if self._content_format(record["title"], record["content_type"]) not in { + "pdf", + "markdown", + }: + return + if record.get("pageindex_tree_status") == "built" and record.get("pageindex_doc_id"): + return + message = self._pageindex_tree_failure_message(record.get("metadata_status")) or ( + "PageIndex tree was not built" + ) + raise RuntimeError(f"pifs add failed to build PageIndex tree: {message}") + + def _require_add_summary_projection_ready(self, record: dict[str, Any]) -> None: + summary_projection = (record.get("metadata_status") or {}).get("summary_projection") + if not summary_projection or not summary_projection.get("requested"): + raise RuntimeError("pifs add requires a requested summary projection index") + if summary_projection.get("status") != "ready": + detail = summary_projection.get("error") or summary_projection.get("status") + raise RuntimeError( + f"pifs add failed to build summary projection index: {detail}" + ) + + def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: + storage_uri = file["storage_uri"] + metadata = file.get("metadata") or {} + if not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + self._validate_register_metadata(metadata) + external_id = file.get("external_id") + content = file.get("content") or "" + folder_path = normalize_path(file.get("folder_path") or "/") + title = str( + file.get("title") + or metadata.get("title") + or self._title_from_storage_uri(storage_uri) + or external_id + or "" + ).strip() + if not title: + raise ValueError("file title is required") + content_type = file.get("content_type") or self._infer_content_type( + title=title, + storage_uri=storage_uri, + ) + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + raise ValueError("PIFS registration supports PageIndex-backed PDF/Markdown files only") + file_ref = make_file_ref( + str(external_id or self._join_virtual_file_path(folder_path, title).strip("/")) + ) + ( + pageindex_doc_id, + pageindex_tree_status, + pageindex_tree_failure, + ) = self._registration_pageindex_pointer( + storage_uri=storage_uri, + title=title, + content_type=content_type, + ) + if pageindex_tree_status != "built" or not pageindex_doc_id: + message = self._pageindex_tree_failure_message( + {"pageindex_tree": pageindex_tree_failure} + ) or "PageIndex tree was not built" + raise RuntimeError(f"PIFS registration requires PageIndex extraction: {message}") + pageindex_summary = self._pageindex_doc_description(pageindex_doc_id) + if not pageindex_summary: + raise RuntimeError("PIFS registration requires PageIndex doc_description") + metadata["summary"] = pageindex_summary + artifact_content = self._registration_text_artifact_content( + title=title, + content_type=content_type, + pageindex_doc_id=pageindex_doc_id, + pageindex_tree_status=pageindex_tree_status, + fallback_content=content, + ) + fts_content = file.get("fts_content", artifact_content) + source_type = file.get("source_type") + metadata_status = self._metadata_status_state(metadata=metadata) + self._attach_pageindex_tree_failure(metadata_status, pageindex_tree_failure) + indexed_metadata = SQLiteFileSystemStore.indexed_metadata_values(metadata) + searchable_metadata = indexed_metadata + text_artifact_path = file.get("text_artifact_path") + owns_text_artifact = text_artifact_path is None + if text_artifact_path is None: + text_artifact_path = self.store.write_text_artifact(file_ref, artifact_content) + raw_artifact_path = file.get("raw_artifact_path") + owns_raw_artifact = False + if raw_artifact_path is None and file.get("write_raw_artifact", True): + raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" + owns_raw_artifact = True + descriptor = self._build_descriptor(title, metadata) + return { + "file_ref": file_ref, + "external_id": external_id, + "storage_uri": storage_uri, + "title": title, + "descriptor": descriptor, + "content_type": content_type, + "source_type": source_type, + "fingerprint": fingerprint(artifact_content), + "text_artifact_path": str(text_artifact_path), + "raw_artifact_path": str(raw_artifact_path) if raw_artifact_path is not None else None, + "pageindex_doc_id": pageindex_doc_id, + "pageindex_tree_status": pageindex_tree_status, + "metadata": metadata, + "metadata_json": json.dumps(metadata, ensure_ascii=False), + "metadata_status": metadata_status, + "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), + "indexed_metadata": indexed_metadata, + "metadata_text": metadata_text(searchable_metadata), + "folder_path": folder_path, + "content": fts_content, + "skip_fts": bool(file.get("skip_fts", False)), + "_pifs_owned_text_artifact": owns_text_artifact, + "_pifs_owned_raw_artifact": owns_raw_artifact, + } + + def _registration_text_artifact_content( + self, + *, + title: str, + content_type: str, + pageindex_doc_id: str | None, + pageindex_tree_status: str, + fallback_content: str, + ) -> str: + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + return fallback_content + if pageindex_tree_status != "built" or not pageindex_doc_id: + return fallback_content + return self._pageindex_extracted_text(pageindex_doc_id) + + def _pageindex_extracted_text(self, doc_id: str) -> str: + client = self._pageindex_client() + if doc_id not in client.documents: + return "" + client._ensure_doc_loaded(doc_id) + doc = client.documents.get(doc_id) or {} + return self._pageindex_pages_text(doc.get("pages")) + + def _pageindex_doc_description(self, doc_id: str) -> str: + client = self._pageindex_client() + if doc_id not in client.documents: + return "" + client._ensure_doc_loaded(doc_id) + doc = client.documents.get(doc_id) or {} + return str(doc.get("doc_description") or "").strip() + + @staticmethod + def _pageindex_pages_text(pages: Any) -> str: + if not isinstance(pages, list): + return "" + parts: list[str] = [] + for page in pages: + if not isinstance(page, dict): + continue + content = str(page.get("content") or "").strip() + if content: + parts.append(content) + return "\n\n".join(parts) + + @staticmethod + def _raw_artifact_payload( + *, + folder_path: str, + metadata: dict[str, Any], + metadata_status: dict[str, Any], + ) -> dict[str, Any]: + return { + "folder_path": folder_path, + "metadata": metadata, + "metadata_status": metadata_status, + } + + def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: + raw_artifact_path = record.get("raw_artifact_path") + if not raw_artifact_path: + return + default_raw_artifact_path = self.store.raw_dir / f"{record['file_ref']}.json" + if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( + default_raw_artifact_path.resolve(strict=False) + ): + return + record["raw_artifact_path"] = str( + self.store.write_raw_artifact( + record["file_ref"], + self._raw_artifact_payload( + folder_path=record["folder_path"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ), + ) + ) + + def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: + content = self.store.read_text(entry.file_ref) + metadata_status = self._metadata_status_state(metadata=entry.metadata) + self._attach_pageindex_tree_failure( + metadata_status, + entry.metadata_status.get("pageindex_tree"), + ) + return { + "file_ref": entry.file_ref, + "external_id": entry.external_id, + "storage_uri": entry.storage_uri, + "title": entry.title, + "descriptor": entry.descriptor, + "content_type": entry.content_type, + "source_type": entry.source_type, + "fingerprint": entry.fingerprint, + "text_artifact_path": entry.text_artifact_path, + "raw_artifact_path": entry.raw_artifact_path, + "pageindex_doc_id": entry.pageindex_doc_id, + "pageindex_tree_status": entry.pageindex_tree_status, + "metadata": dict(entry.metadata), + "metadata_json": json.dumps(entry.metadata, ensure_ascii=False), + "metadata_status": metadata_status, + "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), + "indexed_metadata": SQLiteFileSystemStore.indexed_metadata_values(entry.metadata), + "metadata_text": metadata_text( + SQLiteFileSystemStore.indexed_metadata_values(entry.metadata) + ), + "folder_path": entry.folder_path, + "content": content, + "skip_fts": False, + } + + def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: + metadata_status = record["metadata_status"] + summary_index = metadata_status.get("summary_projection") + if not summary_index or not summary_index.get("requested"): + return False + summary = str(record.get("metadata", {}).get("summary") or "").strip() + if not summary: + return False + if self.summary_projection_indexer is None: + self._refresh_record_metadata_status(record) + return True + try: + result = self.summary_projection_indexer.upsert_summary(record) + except Exception as exc: + summary_index["status"] = "failed" + summary_index["error"] = str(exc) + self._refresh_record_metadata_status(record) + return True + summary_index.clear() + summary_index.update({"requested": True, **result}) + if summary_index.get("status") != "ready": + summary_index["status"] = "ready" + self._refresh_record_metadata_status(record) + return True + + @staticmethod + def _unlink_artifact(path: Any) -> None: + try: + Path(path).unlink() + except FileNotFoundError: + return + + def _cleanup_failed_register_artifacts(self, records: list[dict[str, Any]]) -> None: + for record in records: + if record.get("_pifs_owned_text_artifact"): + self._unlink_artifact(record["text_artifact_path"]) + if record.get("_pifs_owned_raw_artifact") and record.get("raw_artifact_path"): + self._unlink_artifact(record["raw_artifact_path"]) + + def _cleanup_add_catalog_record(self, file_ref: str) -> None: + try: + self.store.delete_file(file_ref) + except Exception: + return + + def _existing_file_refs(self, records: list[dict[str, Any]]) -> set[str]: + existing: set[str] = set() + for record in records: + file_ref = str(record.get("file_ref") or "") + if not file_ref: + continue + try: + self.store.get_file(file_ref) + except KeyError: + continue + existing.add(file_ref) + return existing + + def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None: + indexer = self.summary_projection_indexer + if indexer is None: + return + delete_summary = getattr(indexer, "delete_summary", None) + for record in records: + file_ref = str(record.get("file_ref") or "") + if not file_ref: + continue + try: + if callable(delete_summary): + delete_summary(file_ref) + continue + index = getattr(indexer, "index", None) + delete_file_refs = getattr(index, "delete_file_refs", None) + if callable(delete_file_refs): + delete_file_refs([file_ref]) + except Exception: + continue + + def _cleanup_add_created_folders(self, folder_paths: list[str]) -> None: + for folder_path in reversed(folder_paths): + try: + self.store.delete_empty_folder(folder_path) + except Exception: + continue + + def _pageindex_cache_doc_ids(self) -> set[str]: + workspace = self.pageindex_client_workspace + doc_ids = {path.stem for path in workspace.glob("*.json") if path.name != "_meta.json"} + meta_path = workspace / "_meta.json" + if not meta_path.exists(): + return doc_ids + try: + payload = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return doc_ids + if isinstance(payload, dict): + doc_ids.update(str(doc_id) for doc_id in payload) + return doc_ids + + def _cleanup_add_pageindex_cache( + self, + records: list[dict[str, Any]], + preexisting_doc_ids: set[str], + ) -> None: + doc_ids = sorted(self._pageindex_cache_doc_ids() - preexisting_doc_ids) + for record in records: + doc_id = str(record.get("pageindex_doc_id") or "").strip() + if doc_id and doc_id not in preexisting_doc_ids: + doc_ids.append(doc_id) + doc_ids = sorted(set(doc_ids)) + if not doc_ids: + return + workspace = self.pageindex_client_workspace + for doc_id in doc_ids: + try: + (workspace / f"{doc_id}.json").unlink() + except FileNotFoundError: + pass + except Exception: + continue + meta_path = workspace / "_meta.json" + if not meta_path.exists(): + return + try: + payload = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + if not isinstance(payload, dict): + return + changed = False + for doc_id in doc_ids: + if doc_id in payload: + payload.pop(doc_id, None) + changed = True + if not changed: + return + try: + meta_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + except OSError: + return + + def _refresh_record_metadata_status( + self, + record: dict[str, Any], + *, + explicit_status: str | None = None, + ) -> None: + metadata_status = record["metadata_status"] + metadata_status["status"] = explicit_status or metadata_status.get("status") or "generated" + self._refresh_summary_projection_status(metadata_status, record["metadata"]) + record["metadata_json"] = json.dumps(record["metadata"], ensure_ascii=False) + record["metadata_status_json"] = json.dumps(metadata_status, ensure_ascii=False) + record["indexed_metadata"] = SQLiteFileSystemStore.indexed_metadata_values(record["metadata"]) + record["metadata_text"] = metadata_text(record["indexed_metadata"]) + + def _open_lines(self, file_ref: str, start: int, end: int) -> OpenResult: + entry = self.store.get_file(file_ref) + lines = self.store.read_text(file_ref).splitlines() + start = max(1, start) + end = min(max(start, end), len(lines)) + text = "\n".join(lines[start - 1:end]) + return OpenResult( + file_ref=file_ref, + start_line=start, + end_line=end, + text=text, + external_id=entry.external_id, + folder_path=entry.folder_path, + ) + + def _open_all(self, file_ref: str) -> OpenResult: + entry = self.store.get_file(file_ref) + text = self.store.read_text(file_ref) + line_count = len(text.splitlines()) + return OpenResult( + file_ref=file_ref, + start_line=1, + end_line=line_count, + text=text, + external_id=entry.external_id, + folder_path=entry.folder_path, + ) + + @classmethod + def _structural_unavailable( + cls, + mode: str, + entry: Any, + *, + message: str, + pages: str | None = None, + ) -> dict[str, Any]: + pageindex_tree_error = cls._pageindex_tree_failure_message(entry.metadata_status) + if pageindex_tree_error and entry.pageindex_tree_status == "failed": + message = f"PageIndex tree build failed: {pageindex_tree_error}" + result = { + "mode": mode, + "file_ref": entry.file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": False, + "message": message, + } + if pageindex_tree_error: + result["pageindex_tree_error"] = pageindex_tree_error + if pages is not None: + result["pages"] = pages + return result + + @staticmethod + def _attach_pageindex_tree_failure( + metadata_status: dict[str, Any], + pageindex_tree_failure: Any, + ) -> None: + if isinstance(pageindex_tree_failure, dict) and pageindex_tree_failure: + metadata_status["pageindex_tree"] = dict(pageindex_tree_failure) + + @staticmethod + def _pageindex_tree_failure_message(metadata_status: Any) -> str | None: + if not isinstance(metadata_status, dict): + return None + pageindex_tree = metadata_status.get("pageindex_tree") + if not isinstance(pageindex_tree, dict): + return None + if pageindex_tree.get("status") != "failed": + return None + message = str(pageindex_tree.get("message") or "").strip() + error_type = str(pageindex_tree.get("error_type") or "").strip() + if error_type and message: + return f"{error_type}: {message}" + return message or error_type or None + + def _resolve_target(self, target: str) -> str: + try: + return self.store.resolve_file_ref(target) + except KeyError: + if not str(target).strip().startswith("/"): + raise + normalized = normalize_path(target) + try: + return self._resolve_scope_file_locator(normalized) + except KeyError: + pass + try: + scope = self.resolve_query_scope(normalized) + except (KeyError, ValueError): + pass + else: + raise ValueError(self._scope_file_required_message(scope.path)) + raise KeyError(f"Unknown file target: {target}") + + def _resolve_scope_file_locator(self, target: str) -> str: + prefix, _, leaf = target.rstrip("/").rpartition("/") + if not leaf: + raise KeyError(f"Unknown file target: {target}") + leaf = unquote(leaf) + scope_path = prefix or "/" + scope = self.resolve_query_scope(scope_path) + if scope.metadata_axis is not None: + raise ValueError(self._scope_file_required_message(target)) + # ponytail: scans the scoped leaf list; add an indexed leaf lookup if huge scoped collisions matter. + matches = [ + item + for item in self.scope_files(scope, limit=self.scope_file_count(scope) + 1) + if item["name"] == leaf + ] + if not matches: + raise KeyError(f"Unknown file target: {target}") + if len(matches) > 1: + raise KeyError( + f"Ambiguous file target: {target}. Use tree {scope.path} or " + f'browse {scope.path} "" to copy a disambiguated file locator.' + ) + return matches[0]["file_ref"] + + @staticmethod + def _scope_file_required_message(path: str) -> str: + return ( + f"{path} is a scope, not a file locator; use tree {path} or " + f'browse {path} "" to select a file leaf.' + ) + + @staticmethod + def _semantic_candidate_score(candidate: Any) -> float | None: + try: + return float(getattr(candidate, "score")) + except (AttributeError, TypeError, ValueError): + return None + + @classmethod + def _semantic_candidate_similarity(cls, candidate: Any) -> float: + distances: list[float] = [] + for source in getattr(candidate, "sources", []) or []: + if not isinstance(source, dict) or source.get("distance") is None: + continue + try: + distances.append(float(source["distance"])) + except (TypeError, ValueError): + continue + if distances: + distance = max(min(distances), 0.0) + return round(max(0.0, min(1.0, 1.0 / (1.0 + distance))), 4) + score = cls._semantic_candidate_score(candidate) + if score is None: + return 0.0 + return round(max(0.0, min(1.0, score)), 4) + + @staticmethod + def _metadata_filter_payload(metadata_filter: Any) -> str: + if isinstance(metadata_filter, str): + return metadata_filter + return json.dumps( + metadata_filter, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + def _stable_file_locator( + self, + file_ref: str, + entry: Any, + *, + folder_path: str | None = None, + ) -> str: + folder_path = normalize_path(folder_path or getattr(entry, "folder_path", None) or "/") + title = str( + self.store.membership_display_name(file_ref, folder_path) + or getattr(entry, "title", "") + or "" + ).strip() + if not title: + raise RuntimeError(f"browse cannot build a virtual path for {file_ref}: missing title") + target = self._join_virtual_file_path(folder_path, title.strip("/")) + try: + resolved_file_ref = self.store.resolve_file_ref(target) + except KeyError as exc: + raise RuntimeError( + f"browse produced an unresolved virtual path for {file_ref}: {target}" + ) from exc + if resolved_file_ref != file_ref: + raise RuntimeError( + "browse produced a non-idempotent virtual path: " + f"{target} resolved to {resolved_file_ref}, expected {file_ref}" + ) + return target + + @staticmethod + def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: + return PageIndexFileSystem._join_virtual_file_path( + scope.path, + PageIndexFileSystem.encode_scope_segment(str(leaf).strip("/")), + ) + + @staticmethod + def _build_descriptor(title: str, metadata: dict[str, Any]) -> str: + source = metadata.get("source_type") or metadata.get("repo") or metadata.get("channel") + return f"{title} ({source})" if source else title + + @staticmethod + def _validate_register_metadata(metadata: dict[str, Any]) -> None: + if "summary" in metadata: + raise ValueError("summary is managed by PageIndex doc_description") + + def _register_custom_metadata_fields(self, records: list[dict[str, Any]]) -> None: + fields = {} + for record in records: + for name in SQLiteFileSystemStore.indexed_metadata_values( + record.get("metadata", {}) + ): + if self.metadata.FIELD_RE.match(str(name)): + fields[str(name)] = {} + if fields: + self.metadata.register_schema({"fields": fields}, source="user") + + @staticmethod + def _metadata_status_state(*, metadata: dict[str, Any]) -> dict[str, Any]: + state = { + "status": "generated", + "summary_projection": { + "requested": True, + "status": "not_indexed", + "owner": "pifs", + "source": "index", + }, + } + PageIndexFileSystem._refresh_summary_projection_status(state, metadata) + return state + + @staticmethod + def _refresh_summary_projection_status( + metadata_status: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + summary_index = metadata_status.get("summary_projection") + if not summary_index or not summary_index.get("requested"): + return + if "summary" not in metadata: + return + if summary_index.get("status", "not_indexed") == "not_indexed": + summary_index["status"] = "pending_index" + + @staticmethod + def _scope_folder_path(scope: Optional[dict[str, Any]]) -> Optional[str]: + if not scope: + return None + path = scope.get("folder_path") or scope.get("path") + return normalize_path(path) if path else None + + def _folder_exists(self, path: str) -> bool: + try: + self.store.folder_info(path) + return True + except KeyError: + return False + + @staticmethod + def _query_text(query: Union[str, list[str], None]) -> str: + if query is None: + return "" + if isinstance(query, list): + return " ".join(str(item) for item in query) + return str(query) + + @staticmethod + def _preferred_folder_path( + folder_paths: list[str], + scope_path: Optional[str], + fallback: str, + ) -> str: + if scope_path: + scoped = [ + path + for path in folder_paths + if path == scope_path or path.startswith(f"{scope_path.rstrip('/')}/") + ] + if scoped: + return sorted(scoped, key=lambda item: (len(item), item))[0] + non_root = [path for path in folder_paths if path != "/"] + if non_root: + return sorted(non_root, key=lambda item: (len(item), item))[0] + return fallback + + @staticmethod + def _parse_line_range(location: str) -> tuple[int, int]: + value = str(location).strip() + if "-" in value: + left, right = value.split("-", 1) + start, end = int(left), int(right) + else: + start = end = int(value) + if start < 1 or end < start: + raise ValueError(f"Invalid line range: {location}") + return start, end diff --git a/pageindex/filesystem/metadata.py b/pageindex/filesystem/metadata.py new file mode 100644 index 000000000..02649915a --- /dev/null +++ b/pageindex/filesystem/metadata.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +import re +from typing import Any + +from .types import MetadataField + + +class MetadataQueryError(ValueError): + pass + + +class MetadataQueryEngine: + FIELD_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") + OPERATORS = {"$eq", "$ne", "$in", "$contains"} + LOGICAL_OPERATORS = {"$and", "$or"} + FOLDER_SCOPE_FIELD_HINTS = {"path", "folder", "folders", "folder_path", "folder_paths"} + MAX_DEPTH = 5 + + def __init__(self, store: Any): + self.store = store + + def register_schema(self, schema: dict[str, Any], source: str = "manual") -> None: + fields = [] + raw_fields = schema.get("fields", schema) + if not isinstance(raw_fields, dict): + raise MetadataQueryError("metadata schema must contain a fields object") + for name, declaration in raw_fields.items(): + name = str(name) + self.validate_field_name(name) + if isinstance(declaration, str): + description = declaration + elif isinstance(declaration, dict): + description = str(declaration.get("description", "")) + else: + raise MetadataQueryError(f"Invalid schema declaration for field: {name}") + fields.append( + MetadataField( + name=name, + description=description, + source=source, + ) + ) + if fields: + self.store.upsert_metadata_fields(fields) + + def parse_filter(self, value: str | dict[str, Any] | None) -> dict[str, Any] | None: + if value is None or value == "": + return None + if isinstance(value, str): + value = self.parse_dsl(value) + if not isinstance(value, dict): + raise MetadataQueryError("metadata_filter must be a JSON object") + self.validate_filter(value) + return value + + def parse_dsl(self, dsl: str) -> dict[str, Any]: + try: + parsed = json.loads(dsl) + except json.JSONDecodeError as exc: + raise MetadataQueryError( + "metadata DSL must be a JSON object, for example " + '\'{"$and":[{"repo":"redwood"},{"ticker":{"$in":["AAPL","MSFT"]}}]}\'' + ) from exc + if not isinstance(parsed, dict): + raise MetadataQueryError("metadata DSL must be a JSON object") + return parsed + + def validate_filter(self, metadata_filter: dict[str, Any], depth: int = 1) -> None: + if depth > self.MAX_DEPTH: + raise MetadataQueryError(f"metadata_filter nesting depth exceeds {self.MAX_DEPTH}") + if not metadata_filter: + return + for key, condition in metadata_filter.items(): + if key in self.LOGICAL_OPERATORS: + self._validate_logical(key, condition, depth) + continue + self.validate_field(key) + self._validate_field_condition(key, condition) + + def _validate_logical(self, operator: str, condition: Any, depth: int) -> None: + if not isinstance(condition, list) or not condition: + raise MetadataQueryError(f"{operator} requires a non-empty list") + for item in condition: + if not isinstance(item, dict): + raise MetadataQueryError(f"{operator} items must be metadata filter objects") + self.validate_filter(item, depth + 1) + + def _validate_field_condition(self, field: str, condition: Any) -> None: + if not isinstance(condition, dict) or not any( + str(key).startswith("$") for key in condition + ): + self._validate_scalar(condition, context=field) + return + if len(condition) != 1: + raise MetadataQueryError( + f"Field {field} condition must contain exactly one metadata operator" + ) + operator, expected = next(iter(condition.items())) + if operator not in self.OPERATORS: + raise MetadataQueryError(f"Unsupported metadata operator: {operator}") + if operator == "$in": + if not isinstance(expected, list): + raise MetadataQueryError(f"{field} $in requires a list") + for item in expected: + self._validate_scalar(item, context=f"{field} $in") + return + if operator == "$contains": + self._validate_scalar(expected, context=f"{field} $contains") + return + self._validate_scalar(expected, context=f"{field} {operator}") + + def validate_field(self, field: str) -> None: + self.validate_field_name(field) + if not self.store.metadata_field_exists(field): + if field in self.FOLDER_SCOPE_FIELD_HINTS: + raise MetadataQueryError( + f"Unknown metadata field: {field}. Folder paths are positional PIFS paths, " + "not metadata fields; use `tree /documents` to inspect folders, then " + '`browse /documents "" --where JSON` for metadata pruning.' + ) + raise MetadataQueryError(f"Unknown metadata field: {field}") + + def validate_field_name(self, field: str) -> None: + if not self.FIELD_RE.match(field): + raise MetadataQueryError(f"Invalid metadata field: {field}") + + def export_schema(self) -> dict[str, Any]: + fields = {} + for field in self.store.list_metadata_fields(): + fields[field.name] = { + "description": field.description, + "source": field.source, + } + return {"fields": fields} + + def filter_fields(self, metadata_filter: dict[str, Any] | None) -> set[str]: + if not metadata_filter: + return set() + fields = set() + for key, condition in metadata_filter.items(): + if key in self.LOGICAL_OPERATORS: + for item in condition: + if isinstance(item, dict): + fields.update(self.filter_fields(item)) + continue + fields.add(str(key)) + return fields + + @staticmethod + def _validate_scalar(value: Any, *, context: str) -> None: + if isinstance(value, bool): + return + if isinstance(value, (int, float)): + return + if isinstance(value, str): + return + raise MetadataQueryError(f"{context} must be a string, number, or boolean") diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py new file mode 100644 index 000000000..cc01d820a --- /dev/null +++ b/pageindex/filesystem/semantic_index.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +import sqlite_vec + + +class SemanticIndexError(RuntimeError): + pass + + +@dataclass(frozen=True) +class SemanticIndexRecord: + file_ref: str + vector: list[float] + text: str + external_id: str | None = None + source_type: str = "" + title: str = "" + metadata: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class SemanticSearchResult: + file_ref: str + distance: float + external_id: str | None + source_type: str + title: str + text_hash: str + metadata: dict[str, Any] + + +class RebuildableSemanticIndex(Protocol): + def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: + ... + + def upsert_many(self, records: list[SemanticIndexRecord]) -> int: + ... + + def search( + self, + vector: list[float], + *, + limit: int = 10, + filters: dict[str, Any] | None = None, + fetch_multiplier: int = 20, + ) -> list[SemanticSearchResult]: + ... + + def info(self) -> dict[str, Any]: + ... + + +class SQLiteVecSemanticIndex: + """Rebuildable local semantic index backed by sqlite-vec. + + This is intentionally separate from the PIFS catalog tables. The catalog + remains source of truth; this file is a rebuildable recall index. + """ + + def __init__(self, db_path: str | Path): + self.db_path = Path(db_path).expanduser() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + + def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: + if dimension <= 0: + raise SemanticIndexError("semantic index dimension must be positive") + with self.connect() as conn: + conn.executescript( + """ + DROP TABLE IF EXISTS semantic_index_vec; + DROP TABLE IF EXISTS semantic_index_docs; + DROP TABLE IF EXISTS semantic_index_config; + CREATE TABLE semantic_index_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE semantic_index_docs ( + rowid INTEGER PRIMARY KEY, + file_ref TEXT NOT NULL UNIQUE, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX idx_semantic_index_docs_file_ref + ON semantic_index_docs(file_ref); + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + conn.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + f"vec0(source_type TEXT partition key, embedding float[{dimension}])" + ) + config = { + "dimension": str(dimension), + "adapter": "sqlite-vec", + "adapter_version": sqlite_vec.__version__, + "metadata": json.dumps(metadata or {}, ensure_ascii=False, sort_keys=True), + } + conn.executemany( + "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", + sorted(config.items()), + ) + conn.commit() + + def upsert_many(self, records: list[SemanticIndexRecord]) -> int: + if not records: + return 0 + dimension = self.dimension() + with self.connect() as conn: + inserted = 0 + for record in records: + if len(record.vector) != dimension: + raise SemanticIndexError( + f"vector dimension mismatch for {record.file_ref}: " + f"expected {dimension}, got {len(record.vector)}" + ) + rowid = self._upsert_doc(conn, record) + conn.execute("DELETE FROM semantic_index_vec WHERE rowid = ?", (rowid,)) + conn.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) VALUES (?, ?, ?)", + ( + rowid, + record.source_type, + sqlite_vec.serialize_float32(record.vector), + ), + ) + inserted += 1 + conn.commit() + return inserted + + def delete_file_refs(self, file_refs: list[str]) -> int: + refs = [str(file_ref) for file_ref in file_refs if str(file_ref)] + if not refs: + return 0 + placeholders = ", ".join("?" for _ in refs) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT rowid + FROM semantic_index_docs + WHERE file_ref IN ({placeholders}) + """, + refs, + ).fetchall() + rowids = [int(row["rowid"]) for row in rows] + if not rowids: + return 0 + rowid_placeholders = ", ".join("?" for _ in rowids) + conn.execute( + f"DELETE FROM semantic_index_vec WHERE rowid IN ({rowid_placeholders})", + rowids, + ) + conn.execute( + f"DELETE FROM semantic_index_docs WHERE rowid IN ({rowid_placeholders})", + rowids, + ) + conn.commit() + return len(rowids) + + def search( + self, + vector: list[float], + *, + limit: int = 10, + filters: dict[str, Any] | None = None, + fetch_multiplier: int = 20, + ) -> list[SemanticSearchResult]: + dimension = self.dimension() + if len(vector) != dimension: + raise SemanticIndexError( + f"query vector dimension mismatch: expected {dimension}, got {len(vector)}" + ) + raw_filters = filters or {} + source_types = _source_type_filters(raw_filters) + file_refs = _file_ref_filters(raw_filters) + if file_refs == []: + return [] + with self.connect() as conn: + if file_refs is not None: + _install_file_ref_filter_table(conn, file_refs) + rows = [] + if source_types: + for source_type in source_types: + fetch_k = self._search_fetch_k( + conn, + limit, + fetch_multiplier, + exact_file_ref_filter=file_refs is not None, + source_type=source_type, + ) + if fetch_k <= 0: + continue + rows.extend( + conn.execute( + f""" + SELECT + d.file_ref, + d.external_id, + d.source_type, + d.title, + d.text_hash, + d.metadata_json, + v.distance + FROM semantic_index_vec v + JOIN semantic_index_docs d ON d.rowid = v.rowid + WHERE v.embedding MATCH ? AND k = ? AND v.source_type = ? + {_file_ref_filter_sql(file_refs)} + ORDER BY v.distance + """, + (sqlite_vec.serialize_float32(vector), fetch_k, source_type), + ).fetchall() + ) + rows.sort(key=lambda row: float(row["distance"])) + else: + fetch_k = self._search_fetch_k( + conn, + limit, + fetch_multiplier, + exact_file_ref_filter=file_refs is not None, + ) + if fetch_k <= 0: + return [] + rows = conn.execute( + f""" + SELECT + d.file_ref, + d.external_id, + d.source_type, + d.title, + d.text_hash, + d.metadata_json, + v.distance + FROM semantic_index_vec v + JOIN semantic_index_docs d ON d.rowid = v.rowid + WHERE v.embedding MATCH ? AND k = ? + {_file_ref_filter_sql(file_refs)} + ORDER BY v.distance + """, + (sqlite_vec.serialize_float32(vector), fetch_k), + ).fetchall() + results: list[SemanticSearchResult] = [] + for row in rows: + metadata = _json_obj(row["metadata_json"]) + if not _matches_filters(row, metadata, filters or {}): + continue + results.append( + SemanticSearchResult( + file_ref=row["file_ref"], + distance=float(row["distance"]), + external_id=row["external_id"], + source_type=row["source_type"], + title=row["title"], + text_hash=row["text_hash"], + metadata=metadata, + ) + ) + if len(results) >= limit: + break + return results + + @staticmethod + def _search_fetch_k( + conn: sqlite3.Connection, + limit: int, + fetch_multiplier: int, + *, + exact_file_ref_filter: bool, + source_type: str | None = None, + ) -> int: + if exact_file_ref_filter: + where = [] + params: list[Any] = [] + if source_type is not None: + where.append("source_type = ?") + params.append(source_type) + where_sql = "WHERE " + " AND ".join(where) if where else "" + return int( + conn.execute( + f"SELECT COUNT(*) FROM semantic_index_docs {where_sql}", + params, + ).fetchone()[0] + ) + return min(4096, max(limit, limit * max(fetch_multiplier, 1))) + + def info(self) -> dict[str, Any]: + with self.connect() as conn: + config = { + row["key"]: row["value"] + for row in conn.execute( + "SELECT key, value FROM semantic_index_config ORDER BY key" + ).fetchall() + } + count = conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] + parsed_metadata: dict[str, Any] + try: + parsed_metadata = json.loads(config.get("metadata", "{}")) + except json.JSONDecodeError: + parsed_metadata = {} + return { + "db_path": str(self.db_path), + "adapter": config.get("adapter", "sqlite-vec"), + "adapter_version": config.get("adapter_version", ""), + "dimension": int(config.get("dimension", "0") or 0), + "document_count": count, + "metadata": parsed_metadata, + } + + def dimension(self) -> int: + with self.connect() as conn: + row = conn.execute( + "SELECT value FROM semantic_index_config WHERE key = 'dimension'" + ).fetchone() + if row is None: + raise SemanticIndexError( + f"semantic index is not initialized; call reset() first: {self.db_path}" + ) + return int(row["value"]) + + def connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.enable_load_extension(False) + return conn + + @staticmethod + def text_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + @staticmethod + def _upsert_doc(conn: sqlite3.Connection, record: SemanticIndexRecord) -> int: + existing = conn.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (record.file_ref,), + ).fetchone() + metadata_json = json.dumps(record.metadata or {}, ensure_ascii=False, sort_keys=True) + text_hash = SQLiteVecSemanticIndex.text_hash(record.text) + if existing is None: + cursor = conn.execute( + """ + INSERT INTO semantic_index_docs( + file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + record.file_ref, + record.external_id, + record.source_type, + record.title, + text_hash, + len(record.text), + metadata_json, + ), + ) + return int(cursor.lastrowid) + rowid = int(existing["rowid"]) + conn.execute( + """ + UPDATE semantic_index_docs + SET external_id = ?, + source_type = ?, + title = ?, + text_hash = ?, + text_chars = ?, + metadata_json = ?, + updated_at = CURRENT_TIMESTAMP + WHERE rowid = ? + """, + ( + record.external_id, + record.source_type, + record.title, + text_hash, + len(record.text), + metadata_json, + rowid, + ), + ) + return rowid + + +def _json_obj(text: str | None) -> dict[str, Any]: + if not text: + return {} + try: + value = json.loads(text) + except json.JSONDecodeError: + return {} + return value if isinstance(value, dict) else {} + + +def _matches_filters( + row: sqlite3.Row, + metadata: dict[str, Any], + filters: dict[str, Any], +) -> bool: + for key, expected in filters.items(): + actual_key = "file_ref" if key == "file_refs" else key + actual = row[actual_key] if actual_key in row.keys() else metadata.get(actual_key) + if isinstance(expected, list): + if str(actual) not in {str(item) for item in expected}: + return False + elif str(actual) != str(expected): + return False + return True + + +def _source_type_filters(filters: dict[str, Any]) -> list[str]: + value = filters.get("source_type") + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value if str(item)] + return [str(value)] if str(value) else [] + + +def _file_ref_filters(filters: dict[str, Any]) -> list[str] | None: + if "file_ref" in filters: + value = filters.get("file_ref") + elif "file_refs" in filters: + value = filters.get("file_refs") + else: + return None + if isinstance(value, list): + return [str(item) for item in value if str(item)] + return [str(value)] if str(value) else [] + + +def _install_file_ref_filter_table(conn: sqlite3.Connection, file_refs: list[str]) -> None: + conn.execute( + """ + CREATE TEMP TABLE IF NOT EXISTS semantic_index_filter_file_refs ( + file_ref TEXT PRIMARY KEY + ) + """ + ) + conn.execute("DELETE FROM semantic_index_filter_file_refs") + conn.executemany( + "INSERT OR IGNORE INTO semantic_index_filter_file_refs(file_ref) VALUES (?)", + [(file_ref,) for file_ref in file_refs], + ) + + +def _file_ref_filter_sql(file_refs: list[str] | None) -> str: + if file_refs is None: + return "" + return ( + "AND EXISTS (" + "SELECT 1 FROM semantic_index_filter_file_refs scope_refs " + "WHERE scope_refs.file_ref = d.file_ref" + ")" + ) diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py new file mode 100644 index 000000000..5eea7f5c9 --- /dev/null +++ b/pageindex/filesystem/semantic_projection.py @@ -0,0 +1,554 @@ +from __future__ import annotations + +import json +import sqlite3 +import struct +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .core import DEFAULT_EMBEDDING_DIMENSIONS +from .semantic_index import ( + SQLiteVecSemanticIndex, + SemanticIndexError, + SemanticIndexRecord, + SemanticSearchResult, +) + + +SUMMARY_CHANNEL = "summary" +SUMMARY_INDEX_NAME = "summary" + + +@dataclass(frozen=True) +class SemanticProjectionCandidate: + document_id: str + score: float + sources: list[dict[str, Any]] + source_type: str + title: str + metadata: dict[str, Any] + snippet: str + + +class SemanticProjectionSearchBackend: + """Semantic channel retrieval over rebuildable projection indexes. + + The SQLite catalog remains the source of truth. This backend only reads + external sqlite-vec projection indexes and returns candidate document ids + for the catalog to resolve and filter. + """ + + def __init__( + self, + index_dir: str | Path, + *, + embedder: Any, + embedding_provider: str, + embedding_model: str, + embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + embedding_cache_path: str | Path | None = None, + fetch_multiplier: int = 100, + ) -> None: + self.index_dir = Path(index_dir).expanduser() + self.embedder = embedder + self.embedding_provider = embedding_provider + self.embedding_model = embedding_model + self.embedding_dimensions = embedding_dimensions + self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) + self.embedding_cache = EmbeddingCache( + Path(embedding_cache_path).expanduser() + if embedding_cache_path is not None + else self.index_dir / "embedding_cache.sqlite" + ) + self.fetch_multiplier = fetch_multiplier + self.summary_index = SQLiteVecSemanticIndex( + self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" + ) + + @classmethod + def from_provider( + cls, + index_dir: str | Path, + *, + embedding_provider: str = "openai", + embedding_model: str = "text-embedding-3-small", + embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + embedding_timeout: float = 60, + embedding_api_key: str | None = None, + embedding_base_url: str | None = None, + **kwargs: Any, + ) -> "SemanticProjectionSearchBackend": + return cls( + index_dir, + embedder=make_embedder( + embedding_provider, + embedding_model, + dimensions=embedding_dimensions, + timeout=embedding_timeout, + api_key=embedding_api_key, + base_url=embedding_base_url, + ), + embedding_provider=embedding_provider, + embedding_model=embedding_model, + embedding_dimensions=embedding_dimensions, + **kwargs, + ) + + def search_channel( + self, + channel: str, + query: str, + *, + limit: int = 10, + filters: dict[str, Any] | None = None, + ) -> list[SemanticProjectionCandidate]: + if channel != SUMMARY_CHANNEL: + raise ValueError(f"unsupported semantic channel: {channel}") + if channel not in self.available_channels(): + return [] + query = normalize_text(query) + if not query: + return [] + vector = self.embedding_cache.embed_texts( + [query], + provider=self.embedding_provider, + model=self.cache_model, + embedder=self.embedder, + batch_size=1, + )[0] + results = self.summary_index.search( + vector, + limit=limit, + filters=filters, + fetch_multiplier=self.fetch_multiplier, + ) + return rank_single_semantic_channel(SUMMARY_CHANNEL, results) + + def available_channels(self) -> tuple[str, ...]: + return (SUMMARY_CHANNEL,) if self._summary_document_count() > 0 else () + + def info(self) -> dict[str, Any]: + return { + "index_dir": str(self.index_dir), + "embedding_provider": self.embedding_provider, + "embedding_model": self.embedding_model, + "embedding_dimensions": self.embedding_dimensions, + "strategy": "semantic_channel_vector", + "available_channels": list(self.available_channels()), + "channels": {SUMMARY_CHANNEL: self._safe_summary_info()}, + } + + def _summary_document_count(self) -> int: + info = self._safe_summary_info() + if not info.get("available"): + return 0 + return int(info.get("document_count") or 0) + + def _safe_summary_info(self) -> dict[str, Any]: + index = self.summary_index + if not index.db_path.exists(): + return { + "db_path": str(index.db_path), + "available": False, + "document_count": 0, + "error": "index file is missing", + } + try: + info = index.info() + except (OSError, sqlite3.Error, SemanticIndexError) as exc: + return { + "db_path": str(index.db_path), + "available": False, + "document_count": 0, + "error": str(exc), + } + return {**info, "available": int(info.get("document_count") or 0) > 0} + + +class SummaryProjectionIndexer: + """Synchronous register-time summary projection indexer.""" + + def __init__( + self, + index_dir: str | Path, + *, + embedder: Any, + embedding_provider: str, + embedding_model: str, + embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + embedding_cache_path: str | Path | None = None, + ) -> None: + self.index_dir = Path(index_dir).expanduser() + self.index_dir.mkdir(parents=True, exist_ok=True) + self.embedder = embedder + self.embedding_provider = embedding_provider + self.embedding_model = embedding_model + self.embedding_dimensions = embedding_dimensions + self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) + self.embedding_cache = EmbeddingCache( + Path(embedding_cache_path).expanduser() + if embedding_cache_path is not None + else self.index_dir / "embedding_cache.sqlite" + ) + self.index = SQLiteVecSemanticIndex( + self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" + ) + self._ensure_index() + + @classmethod + def from_provider( + cls, + index_dir: str | Path, + *, + embedding_provider: str = "openai", + embedding_model: str = "text-embedding-3-small", + embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + embedding_timeout: float = 60, + embedding_api_key: str | None = None, + embedding_base_url: str | None = None, + **kwargs: Any, + ) -> "SummaryProjectionIndexer": + cls._validate_existing_index_dimension(index_dir, embedding_dimensions) + return cls( + index_dir, + embedder=make_embedder( + embedding_provider, + embedding_model, + dimensions=embedding_dimensions, + timeout=embedding_timeout, + api_key=embedding_api_key, + base_url=embedding_base_url, + ), + embedding_provider=embedding_provider, + embedding_model=embedding_model, + embedding_dimensions=embedding_dimensions, + **kwargs, + ) + + def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: + summary = str((record.get("metadata") or {}).get("summary") or "").strip() + if not summary: + return {"status": "skipped", "reason": "missing_summary"} + vector = self.embedding_cache.embed_texts( + [summary], + provider=self.embedding_provider, + model=self.cache_model, + embedder=self.embedder, + batch_size=1, + )[0] + metadata = dict(record.get("metadata") or {}) + count = self.index.upsert_many( + [ + SemanticIndexRecord( + file_ref=str(record["file_ref"]), + vector=vector, + text=summary, + external_id=record.get("external_id"), + source_type=str(record.get("source_type") or ""), + title=str(record.get("title") or ""), + metadata=metadata, + ) + ] + ) + return { + "status": "ready", + "indexed_rows": count, + "index_path": str(self.index.db_path), + "embedding_provider": self.embedding_provider, + "embedding_model": self.embedding_model, + "embedding_dimensions": self.embedding_dimensions, + } + + def delete_summary(self, file_ref: str) -> int: + return self.index.delete_file_refs([file_ref]) + + def _ensure_index(self) -> None: + if not self.index.db_path.exists(): + self.index.reset( + dimension=self.embedding_dimensions, + metadata=self._index_metadata(), + ) + return + try: + existing_dimension = self.index.dimension() + except Exception as exc: + raise RuntimeError( + "could not validate existing summary projection index config; " + f"refusing to reset {self.index.db_path}. Move the existing index " + "aside or rebuild it intentionally before changing embedding config." + ) from exc + if existing_dimension != self.embedding_dimensions: + raise self._dimension_mismatch_error( + self.index.db_path, + existing_dimension, + self.embedding_dimensions, + ) + + def _index_metadata(self) -> dict[str, Any]: + return { + "channel": "summary", + "embedding_provider": self.embedding_provider, + "embedding_model": self.embedding_model, + "embedding_dimensions": self.embedding_dimensions, + } + + @classmethod + def _validate_existing_index_dimension( + cls, + index_dir: str | Path, + embedding_dimensions: int, + ) -> None: + index_path = ( + Path(index_dir).expanduser() / f"{SUMMARY_INDEX_NAME}.sqlite" + ) + if not index_path.exists(): + return + index = SQLiteVecSemanticIndex(index_path) + try: + existing_dimension = index.dimension() + except Exception as exc: + raise RuntimeError( + "could not validate existing summary projection index config; " + f"refusing to reset {index_path}. Move the existing index " + "aside or rebuild it intentionally before changing embedding config." + ) from exc + if existing_dimension != embedding_dimensions: + raise cls._dimension_mismatch_error( + index_path, + existing_dimension, + embedding_dimensions, + ) + + @staticmethod + def _dimension_mismatch_error( + index_path: Path, + existing_dimension: int, + embedding_dimensions: int, + ) -> RuntimeError: + return RuntimeError( + "summary projection index dimension mismatch: " + f"{index_path} was built with dimension {existing_dimension}, " + f"but configured embedding_dimensions is {embedding_dimensions}. " + "Use the matching embedding config, or rebuild the projection index " + "at a new path after preserving the existing data." + ) + + +class EmbeddingCache: + def __init__(self, db_path: Path): + self.db_path = db_path + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS embedding_cache ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + text_hash TEXT NOT NULL, + dimension INTEGER NOT NULL, + vector_blob BLOB, + vector_json TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(provider, model, text_hash) + ) + """ + ) + conn.commit() + + def connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def embed_texts( + self, + texts: list[str], + *, + provider: str, + model: str, + embedder: Any, + batch_size: int, + ) -> list[list[float]]: + hashes = [SQLiteVecSemanticIndex.text_hash(text) for text in texts] + cached: dict[str, list[float]] = {} + with self.connect() as conn: + for text_hash in sorted(set(hashes)): + row = conn.execute( + """ + SELECT vector_blob, vector_json + FROM embedding_cache + WHERE provider = ? AND model = ? AND text_hash = ? + """, + (provider, model, text_hash), + ).fetchone() + if row is not None: + cached[text_hash] = decode_vector(row["vector_blob"], row["vector_json"]) + missing_positions = [ + index for index, text_hash in enumerate(hashes) if text_hash not in cached + ] + for start in range(0, len(missing_positions), max(1, batch_size)): + positions = missing_positions[start : start + max(1, batch_size)] + batch_texts = [texts[index] for index in positions] + vectors = embed_with_retry(embedder, batch_texts) + if len(vectors) != len(positions): + raise ValueError( + "embedding response length mismatch: " + f"requested {len(positions)}, received {len(vectors)}" + ) + with self.connect() as conn: + conn.executemany( + """ + INSERT OR REPLACE INTO embedding_cache( + provider, model, text_hash, dimension, vector_blob, vector_json + ) + VALUES (?, ?, ?, ?, ?, '') + """, + [ + ( + provider, + model, + hashes[index], + len(vector), + encode_vector(vector), + ) + for index, vector in zip(positions, vectors) + ], + ) + conn.commit() + for index, vector in zip(positions, vectors): + cached[hashes[index]] = vector + return [cached[text_hash] for text_hash in hashes] + + +class EmbeddingClient: + def __init__( + self, + *, + provider: str, + model: str, + dimensions: int, + timeout: float, + api_key: str | None = None, + base_url: str | None = None, + ): + self.provider = provider.lower() + self.model = model + self.dimensions = dimensions + if self.provider != "openai": + raise ValueError(f"unknown embedding provider: {provider}") + from openai import OpenAI + + if not api_key: + raise ValueError("embedding_api_key is required for PIFS embeddings") + self.client = OpenAI( + api_key=api_key, + base_url=base_url or None, + timeout=timeout, + ) + + def embed(self, texts: list[str]) -> list[list[float]]: + kwargs: dict[str, Any] = {"model": self.model, "input": texts} + if self.dimensions > 0: + kwargs["dimensions"] = self.dimensions + response = self.client.embeddings.create(**kwargs) + return [list(item.embedding) for item in sorted(response.data, key=lambda item: item.index)] + + +def make_embedder( + provider: str, + model: str, + *, + dimensions: int, + timeout: float, + api_key: str | None = None, + base_url: str | None = None, +) -> Any: + return EmbeddingClient( + provider=provider, + model=model, + dimensions=dimensions, + timeout=timeout, + api_key=api_key, + base_url=base_url, + ) + + +def rank_single_semantic_channel( + channel: str, + results: list[SemanticSearchResult], +) -> list[SemanticProjectionCandidate]: + rows: list[SemanticProjectionCandidate] = [] + seen: set[str] = set() + for rank, result in enumerate(results, 1): + doc_id = str(result.external_id or result.file_ref) + if doc_id in seen: + continue + seen.add(doc_id) + rows.append( + SemanticProjectionCandidate( + document_id=doc_id, + score=1 / (60 + rank), + sources=[{"channel": channel, "rank": rank, "distance": result.distance}], + source_type=result.source_type, + title=result.title, + metadata=result.metadata, + snippet=f"{channel}_vector rank={rank}", + ) + ) + return rows + +def normalize_text(text: str) -> str: + return " ".join(str(text or "").split()) + + +def embedding_cache_model_key(model: str, dimensions: int) -> str: + return f"{model}:dimensions={dimensions}" if dimensions > 0 else model + + +def embed_with_retry(embedder: Any, texts: list[str], *, max_attempts: int = 8) -> list[list[float]]: + for attempt in range(1, max_attempts + 1): + try: + return embedder.embed(texts) + except Exception as exc: + if attempt >= max_attempts or not is_retryable_embedding_error(exc): + raise + time.sleep(min(120.0, 2.0 ** (attempt - 1))) + raise RuntimeError("unreachable embedding retry state") + + +def is_retryable_embedding_error(exc: Exception) -> bool: + retryable = getattr(exc, "retryable", None) + if isinstance(retryable, bool): + return retryable + status_code = getattr(exc, "status_code", None) + try: + status = int(status_code) + except (TypeError, ValueError): + status = None + if status is not None: + if status in {408, 409, 429}: + return True + if status >= 500: + return True + if 400 <= status < 500: + return False + name = exc.__class__.__name__.lower() + return any(token in name for token in ("timeout", "connection", "ratelimit")) + + +def encode_vector(vector: list[float]) -> bytes: + return struct.pack(f"<{len(vector)}f", *vector) + + +def decode_vector(blob: bytes | None, vector_json: str | None) -> list[float]: + if blob: + if len(blob) % 4 != 0: + raise ValueError("invalid cached vector blob length") + return list(struct.unpack(f"<{len(blob) // 4}f", blob)) + if vector_json: + value = json.loads(vector_json) + if isinstance(value, list): + return [float(item) for item in value] + raise ValueError("cached embedding row does not contain a vector") diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py new file mode 100644 index 000000000..a714c3979 --- /dev/null +++ b/pageindex/filesystem/store.py @@ -0,0 +1,2193 @@ +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from pathlib import Path +from typing import Any, Iterable, Optional + +from .types import FileEntry, MetadataField + +SCHEMA_VERSION = 1 + + +class SQLiteFileSystemStore: + def __init__(self, workspace: str | Path): + self.workspace = Path(workspace).expanduser() + self.workspace.mkdir(parents=True, exist_ok=True) + self.db_path = self.workspace / "filesystem.sqlite" + self.text_dir = self.workspace / "artifacts" / "text" + self.raw_dir = self.workspace / "artifacts" / "raw" + self.pageindex_client_dir = self.workspace / "artifacts" / "pageindex_client" + for path in (self.text_dir, self.raw_dir, self.pageindex_client_dir): + path.mkdir(parents=True, exist_ok=True) + self.initialize_schema() + + def connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + return conn + + def initialize_schema(self) -> None: + with self.connect() as conn: + self._create_current_schema(conn) + self.ensure_folder(conn, "/") + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + + def _create_current_schema(self, conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS files ( + file_ref TEXT PRIMARY KEY, + external_id TEXT, + storage_uri TEXT NOT NULL, + title TEXT NOT NULL, + descriptor TEXT NOT NULL, + content_type TEXT NOT NULL, + source_type TEXT, + fingerprint TEXT NOT NULL, + text_artifact_path TEXT NOT NULL, + raw_artifact_path TEXT, + pageindex_doc_id TEXT, + pageindex_tree_status TEXT NOT NULL DEFAULT 'not_built', + metadata_json TEXT NOT NULL DEFAULT '{}', + metadata_status_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + deleted_at TEXT + ); + + CREATE TABLE IF NOT EXISTS folders ( + folder_id TEXT PRIMARY KEY, + parent_id TEXT, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + description TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL DEFAULT 'manual', + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(parent_id) REFERENCES folders(folder_id) + ); + + CREATE TABLE IF NOT EXISTS file_folders ( + file_ref TEXT NOT NULL, + folder_id TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (file_ref, folder_id), + FOREIGN KEY(file_ref) REFERENCES files(file_ref) ON DELETE CASCADE, + FOREIGN KEY(folder_id) REFERENCES folders(folder_id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS metadata_fields ( + field_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT 'manual', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + UNIQUE(name) + ); + + CREATE TABLE IF NOT EXISTS metadata_values ( + file_ref TEXT NOT NULL, + field_id TEXT NOT NULL, + value_text TEXT NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(file_ref) REFERENCES files(file_ref) ON DELETE CASCADE, + FOREIGN KEY(field_id) REFERENCES metadata_fields(field_id) ON DELETE CASCADE + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS file_fts + USING fts5(file_ref UNINDEXED, title, body, metadata_text); + + CREATE INDEX IF NOT EXISTS idx_files_external_id ON files(external_id); + CREATE INDEX IF NOT EXISTS idx_files_source_type ON files(source_type); + CREATE INDEX IF NOT EXISTS idx_folders_path ON folders(path); + CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON folders(parent_id); + CREATE INDEX IF NOT EXISTS idx_file_folders_folder ON file_folders(folder_id); + CREATE INDEX IF NOT EXISTS idx_metadata_fields_name ON metadata_fields(name); + CREATE INDEX IF NOT EXISTS idx_metadata_values_field_text ON metadata_values(field_id, value_text); + """ + ) + + @staticmethod + def _json_object(value: Any) -> dict[str, Any]: + try: + parsed = json.loads(value or "{}") if isinstance(value, str) else value + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + @staticmethod + def _columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {row["name"] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} + + def insert_file(self, record: dict[str, Any]) -> None: + self.insert_files([record]) + + def insert_files(self, records: list[dict[str, Any]]) -> None: + if not records: + return + with self.connect() as conn: + conn.execute("PRAGMA temp_store = MEMORY") + folder_cache: dict[tuple[str, str], str] = {} + file_rows = [] + membership_rows = [] + file_ref_rows = [] + fts_file_ref_rows = [] + fts_rows = [] + metadata_rows = [] + pending_folder_titles: dict[tuple[str, str], str] = {} + metadata_field_ids = { + row["name"]: row["field_id"] + for row in conn.execute( + "SELECT name, field_id FROM metadata_fields" + ).fetchall() + } + for record in records: + folder_cache_key = (record["folder_path"], record.get("folder_kind", "manual")) + folder_id = folder_cache.get(folder_cache_key) + if folder_id is None: + folder_id = self.ensure_folder( + conn, + record["folder_path"], + kind=record.get("folder_kind", "manual"), + ) + folder_cache[folder_cache_key] = folder_id + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=record["file_ref"], + title=record["title"], + ) + title_key = (folder_id, str(record["title"])) + existing_file_ref = pending_folder_titles.get(title_key) + if existing_file_ref and existing_file_ref != record["file_ref"]: + target = self._virtual_file_target(conn, folder_id, str(record["title"])) + raise FileExistsError(f"File already exists at {target}") + pending_folder_titles[title_key] = record["file_ref"] + file_rows.append(self._file_insert_values(record)) + membership_rows.append( + ( + record["file_ref"], + folder_id, + json.dumps(record.get("folder_metadata") or {}, ensure_ascii=False), + ) + ) + file_ref_rows.append((record["file_ref"],)) + if not record.get("skip_fts", False): + fts_file_ref_rows.append((record["file_ref"],)) + fts_rows.append( + ( + record["file_ref"], + record["title"], + record["content"], + record["metadata_text"], + ) + ) + metadata_rows.extend( + self._metadata_insert_values( + record["file_ref"], + record.get("indexed_metadata", record["metadata"]), + metadata_field_ids, + ) + ) + conn.executemany(self._file_insert_sql(), file_rows) + conn.executemany( + """ + INSERT OR REPLACE INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + """, + membership_rows, + ) + conn.executemany("DELETE FROM metadata_values WHERE file_ref = ?", file_ref_rows) + if metadata_rows: + conn.executemany( + """ + INSERT INTO metadata_values( + file_ref, field_id, value_text + ) VALUES (?, ?, ?) + """, + metadata_rows, + ) + if fts_file_ref_rows: + conn.executemany("DELETE FROM file_fts WHERE file_ref = ?", fts_file_ref_rows) + conn.executemany( + """ + INSERT INTO file_fts(file_ref, title, body, metadata_text) + VALUES (?, ?, ?, ?) + """, + fts_rows, + ) + + @staticmethod + def _file_insert_sql() -> str: + columns = [ + "file_ref", + "external_id", + "storage_uri", + "title", + "descriptor", + "content_type", + "source_type", + "fingerprint", + "text_artifact_path", + "raw_artifact_path", + "pageindex_doc_id", + "pageindex_tree_status", + "metadata_json", + "metadata_status_json", + ] + columns.extend(["deleted_at", "updated_at"]) + placeholders = ", ".join(["?"] * (len(columns) - 2) + ["NULL", "CURRENT_TIMESTAMP"]) + update_assignments = [ + f"{column} = excluded.{column}" + for column in columns + if column not in {"file_ref", "deleted_at", "updated_at"} + ] + update_assignments.append("updated_at = CURRENT_TIMESTAMP") + return f""" + INSERT INTO files ({", ".join(columns)}) + VALUES ({placeholders}) + ON CONFLICT(file_ref) DO UPDATE SET + {", ".join(update_assignments)} + """ + + @staticmethod + def _file_insert_values(record: dict[str, Any]) -> tuple[Any, ...]: + values: list[Any] = [ + record["file_ref"], + record["external_id"], + record["storage_uri"], + record["title"], + record["descriptor"], + record["content_type"], + record["source_type"], + record["fingerprint"], + record["text_artifact_path"], + record["raw_artifact_path"], + record.get("pageindex_doc_id"), + record.get("pageindex_tree_status", "not_built"), + record["metadata_json"], + record.get("metadata_status_json", "{}"), + ] + return tuple(values) + + def _metadata_insert_values( + self, + file_ref: str, + metadata: dict[str, Any], + metadata_field_ids: dict[str, str], + ) -> list[tuple[Any, ...]]: + values = [] + for name, value in metadata.items(): + if not self._valid_field_name(name): + continue + field_id = metadata_field_ids.get(name) + if field_id is None: + continue + for item in self._metadata_value_items(value): + values.append( + ( + file_ref, + field_id, + item, + ) + ) + return values + + def create_folder( + self, + path: str, + *, + kind: str = "manual", + description: str = "", + metadata: dict[str, Any] | None = None, + ) -> str: + with self.connect() as conn: + return self.ensure_folder( + conn, + path, + kind=kind, + description=description, + metadata=metadata, + ) + + def attach_file_to_folder( + self, + file_ref: str, + folder_path_or_id: str, + *, + metadata: dict[str, Any] | None = None, + ) -> None: + with self.connect() as conn: + resolved_file_ref = self._resolve_file_ref(conn, file_ref) + folder_id = self._resolve_or_create_folder(conn, folder_path_or_id) + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=resolved_file_ref, + title=self._file_title(conn, resolved_file_ref), + ) + conn.execute( + """ + INSERT INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + ON CONFLICT(file_ref, folder_id) DO UPDATE SET + metadata_json = excluded.metadata_json + """, + ( + resolved_file_ref, + folder_id, + json.dumps(metadata or {}, ensure_ascii=False), + ), + ) + + def attach_files_to_folders(self, items: list[dict[str, Any]]) -> None: + with self.connect() as conn: + for item in items: + resolved_file_ref = self._resolve_file_ref(conn, item["file_ref"]) + folder_id = self._resolve_or_create_folder(conn, item["folder"]) + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=resolved_file_ref, + title=self._file_title(conn, resolved_file_ref), + ) + conn.execute( + """ + INSERT INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + ON CONFLICT(file_ref, folder_id) DO UPDATE SET + metadata_json = excluded.metadata_json + """, + ( + resolved_file_ref, + folder_id, + json.dumps(item.get("metadata") or {}, ensure_ascii=False), + ), + ) + + def membership_display_name(self, file_ref: str, folder_path: str) -> str | None: + folder_path = normalize_path(folder_path) + with self.connect() as conn: + row = conn.execute( + """ + SELECT ff.metadata_json, f.title + FROM file_folders ff + JOIN folders fo ON fo.folder_id = ff.folder_id + JOIN files f ON f.file_ref = ff.file_ref + WHERE ff.file_ref = ? + AND fo.path = ? + AND f.deleted_at IS NULL + LIMIT 1 + """, + (file_ref, folder_path), + ).fetchone() + if row is None: + return None + metadata = self._json_object(row["metadata_json"]) + return str(metadata.get("display_name") or row["title"] or "").strip() or None + + def _ensure_title_available_in_folder( + self, + conn: sqlite3.Connection, + *, + folder_id: str, + file_ref: str, + title: str, + ) -> None: + row = conn.execute( + """ + SELECT f.file_ref, fo.path + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND ff.folder_id = ? + AND f.title = ? + AND f.file_ref != ? + LIMIT 1 + """, + (folder_id, title, file_ref), + ).fetchone() + if row: + raise FileExistsError( + f"File already exists at {self._virtual_file_target(conn, folder_id, title)}" + ) + + @staticmethod + def _virtual_file_target( + conn: sqlite3.Connection, + folder_id: str, + title: str, + ) -> str: + row = conn.execute( + "SELECT path FROM folders WHERE folder_id = ?", + (folder_id,), + ).fetchone() + folder_path = normalize_path(row["path"] if row else "/") + return f"/{title}" if folder_path == "/" else f"{folder_path}/{title}" + + @staticmethod + def _file_title(conn: sqlite3.Connection, file_ref: str) -> str: + row = conn.execute( + "SELECT title FROM files WHERE file_ref = ? AND deleted_at IS NULL", + (file_ref,), + ).fetchone() + if row is None: + raise KeyError(f"Unknown file target: {file_ref}") + return str(row["title"]) + + def replace_metadata_values( + self, + conn: sqlite3.Connection, + file_ref: str, + metadata: dict[str, Any], + ) -> None: + conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) + for name, value in metadata.items(): + if not self._valid_field_name(name): + continue + field_id = self._registered_field_id(conn, name) + if field_id is None: + continue + for item in self._metadata_value_items(value): + conn.execute( + """ + INSERT INTO metadata_values( + file_ref, field_id, value_text + ) VALUES (?, ?, ?) + """, + ( + file_ref, + field_id, + item, + ), + ) + + @staticmethod + def _registered_field_id(conn: sqlite3.Connection, name: str) -> str | None: + row = conn.execute( + """ + SELECT field_id + FROM metadata_fields + WHERE name = ? + """, + (name,), + ).fetchone() + return None if row is None else row["field_id"] + + def replace_fts(self, conn: sqlite3.Connection, record: dict[str, Any]) -> None: + conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (record["file_ref"],)) + conn.execute( + """ + INSERT INTO file_fts(file_ref, title, body, metadata_text) + VALUES (?, ?, ?, ?) + """, + ( + record["file_ref"], + record["title"], + record["content"], + record["metadata_text"], + ), + ) + + def upsert_metadata_fields( + self, + fields: Iterable[MetadataField], + *, + conn: sqlite3.Connection | None = None, + ) -> None: + owns_connection = conn is None + if conn is None: + conn = self.connect() + try: + for field in fields: + conn.execute( + """ + INSERT INTO metadata_fields( + field_id, name, description, source, updated_at + ) VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET + description = excluded.description, + source = excluded.source, + updated_at = CURRENT_TIMESTAMP + """, + ( + self.field_id(field.name), + field.name, + field.description, + field.source, + ), + ) + if owns_connection: + conn.commit() + finally: + if owns_connection: + conn.close() + + def metadata_field_exists(self, name: str) -> bool: + with self.connect() as conn: + row = conn.execute( + "SELECT 1 FROM metadata_fields WHERE name = ?", + (name,), + ).fetchone() + return row is not None + + def list_metadata_fields(self) -> list[MetadataField]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT name, description, source + FROM metadata_fields + ORDER BY name + """ + ).fetchall() + return [ + MetadataField( + name=row["name"], + description=row["description"], + source=row["source"], + ) + for row in rows + ] + + def list_folder( + self, + path: str = "/", + recursive: bool = False, + limit: int = 100, + max_depth: int | None = None, + ) -> dict[str, Any]: + path = normalize_path(path) + if max_depth is not None and max_depth < 0: + raise ValueError("max_depth must be non-negative") + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + if recursive: + folder_depth_clause = "" + folder_depth_params: list[Any] = [] + if max_depth is not None: + if max_depth == 0: + folder_depth_clause = "AND 0" + else: + folder_depth_clause = ( + f"AND ({self._folder_depth_sql('fo.path')} - ?) <= ?" + ) + folder_depth_params = [self._folder_depth(path), max_depth] + folder_rows = conn.execute( + f""" + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.path != ? AND (fo.path LIKE ? ESCAPE '\\') + {folder_depth_clause} + ORDER BY fo.path + LIMIT ? + """, + (path, self._descendant_like(path), *folder_depth_params, limit), + ).fetchall() + file_rows = self._file_rows_for_scope( + conn, + path, + True, + limit, + max_depth=max_depth, + ) + else: + folder_rows = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.parent_id = ? + ORDER BY lower(fo.name), fo.name + LIMIT ? + """, + (folder["folder_id"], limit), + ).fetchall() + file_rows = self._file_rows_for_scope(conn, path, False, limit) + return { + "folders": [self._folder_row_to_dict(row) for row in folder_rows], + "files": [self._file_summary(row) for row in file_rows], + } + + def folder_info(self, path: str = "/") -> dict[str, Any]: + path = normalize_path(path) + with self.connect() as conn: + row = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.path = ? + """, + (path,), + ).fetchone() + if row is None: + raise KeyError(f"Unknown folder path: {path}") + return self._folder_row_to_dict(row) + + def find_folders( + self, + path: str = "/", + *, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 100, + max_depth: int | None = None, + include_self: bool = False, + ) -> list[dict[str, Any]]: + path = normalize_path(path) + if max_depth is not None and max_depth < 0: + raise ValueError("max_depth must be non-negative") + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + metadata_clause = f"AND {' AND '.join(metadata_sql)}" if metadata_sql else "" + folder_depth_clause = "" + folder_depth_params: list[Any] = [] + if max_depth is not None: + if max_depth == 0 and not include_self: + folder_depth_clause = "AND 0" + else: + folder_depth_clause = f"AND ({self._folder_depth_sql('fo.path')} - ?) <= ?" + folder_depth_params = [self._folder_depth(path), max_depth] + folder_scope_clause = ( + "(fo.path = ? OR fo.path LIKE ? ESCAPE '\\')" + if include_self + else "fo.path != ? AND fo.path LIKE ? ESCAPE '\\'" + ) + sql = f""" + SELECT * + FROM ( + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count, + ( + SELECT COUNT(DISTINCT f.file_ref) + FROM files f + JOIN file_folders matched_ff + ON matched_ff.file_ref = f.file_ref + JOIN folders matched_folder + ON matched_folder.folder_id = matched_ff.folder_id + WHERE f.deleted_at IS NULL + AND ( + matched_folder.folder_id = fo.folder_id + OR matched_folder.path LIKE {self._descendant_like_sql_expr("fo.path")} ESCAPE '\\' + ) + {metadata_clause} + ) AS matched_files + FROM folders fo + WHERE {folder_scope_clause} + {folder_depth_clause} + ) + WHERE matched_files > 0 + ORDER BY path + LIMIT ? + """ + params = [ + *metadata_params, + path, + self._descendant_like(path), + *folder_depth_params, + limit, + ] + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + rows = conn.execute(sql, params).fetchall() + return [self._folder_row_to_dict(row) for row in rows] + + def search_files( + self, + query: str | list[str] | None, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + query_text = self._query_text(query) + match_queries = self._fts_match_queries(query_text) if query_text else [None] + if query_text and not match_queries: + match_queries = [None] + results: list[dict[str, Any]] = [] + seen: set[str] = set() + for match_query in match_queries: + if query_text and match_query is None: + rows = self._search_literal_once( + query_text, + scope, + metadata_filter, + max(limit * 25, limit), + ) + else: + rows = self._search_once(match_query, scope, metadata_filter, max(limit * 25, limit)) + for row in rows: + if row["file_ref"] in seen: + continue + seen.add(row["file_ref"]) + results.append(self._search_row_to_dict(row)) + if len(results) >= limit: + return results + if results: + return results + return results + + def _search_literal_once( + self, + query_text: str, + scope: Optional[dict[str, Any]], + metadata_filter: Optional[dict[str, Any]], + limit: int, + ) -> list[sqlite3.Row]: + selects = [ + "f.file_ref", + "f.external_id", + "f.title", + "f.descriptor", + "f.pageindex_tree_status", + "f.metadata_json", + "f.metadata_status_json", + "f.created_at", + """ + ( + SELECT display_folder.folder_id + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_id + """, + """ + ( + SELECT display_folder.path + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_path + """, + "f.descriptor AS snippet", + "0 AS rank", + ] + where = [ + "f.deleted_at IS NULL", + """ + ( + lower(file_fts.title) LIKE lower(?) ESCAPE '\\' + OR lower(file_fts.body) LIKE lower(?) ESCAPE '\\' + OR lower(file_fts.metadata_text) LIKE lower(?) ESCAPE '\\' + ) + """, + ] + literal_like = self._contains_like(query_text) + params: list[Any] = [literal_like, literal_like, literal_like] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + sql = f""" + SELECT {", ".join(selects)} + FROM files f + JOIN file_fts ON file_fts.file_ref = f.file_ref + WHERE {" AND ".join(where)} + ORDER BY f.created_at DESC, f.title + LIMIT ? + """ + params.append(limit) + with self.connect() as conn: + return conn.execute(sql, params).fetchall() + + def file_refs_for_scope( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> list[str]: + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT DISTINCT f.file_ref + FROM files f + WHERE {" AND ".join(where)} + ORDER BY f.file_ref + """, + params, + ).fetchall() + return [row["file_ref"] for row in rows] + + def count_files( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> int: + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + row = conn.execute( + f""" + SELECT COUNT(DISTINCT f.file_ref) AS file_count + FROM files f + WHERE {" AND ".join(where)} + """, + params, + ).fetchone() + return int(row["file_count"] if row is not None else 0) + + def list_metadata_axes( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + exclude_fields: set[str] | None = None, + ) -> list[dict[str, Any]]: + where = ["f.deleted_at IS NULL", "mv.value_text != ''"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + if exclude_fields: + placeholders = ", ".join("?" for _ in sorted(exclude_fields)) + where.append(f"mf.name NOT IN ({placeholders})") + params.extend(sorted(exclude_fields)) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT mf.name, COUNT(DISTINCT mv.value_text) AS value_count + FROM metadata_values mv + JOIN metadata_fields mf ON mf.field_id = mv.field_id + JOIN files f ON f.file_ref = mv.file_ref + WHERE {" AND ".join(where)} + GROUP BY mf.name + ORDER BY mf.name + """, + params, + ).fetchall() + return [ + {"name": row["name"], "value_count": int(row["value_count"] or 0)} + for row in rows + ] + + def list_metadata_values( + self, + field: str, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 51, + offset: int = 0, + ) -> list[dict[str, Any]]: + where = [ + "f.deleted_at IS NULL", + "mf.name = ?", + "mv.value_text != ''", + ] + params: list[Any] = [field] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + params.extend([limit, offset]) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT mv.value_text AS value, COUNT(DISTINCT f.file_ref) AS file_count + FROM metadata_values mv + JOIN metadata_fields mf ON mf.field_id = mv.field_id + JOIN files f ON f.file_ref = mv.file_ref + WHERE {" AND ".join(where)} + GROUP BY mv.value_text + ORDER BY file_count DESC, mv.value_text + LIMIT ? OFFSET ? + """, + params, + ).fetchall() + return [ + {"value": row["value"], "file_count": int(row["file_count"] or 0)} + for row in rows + ] + + def _search_once( + self, + match_query: str | None, + scope: Optional[dict[str, Any]], + metadata_filter: Optional[dict[str, Any]], + limit: int, + ) -> list[sqlite3.Row]: + joins = [] + selects = [ + "f.file_ref", + "f.external_id", + "f.title", + "f.descriptor", + "f.pageindex_tree_status", + "f.metadata_json", + "f.metadata_status_json", + "f.created_at", + """ + ( + SELECT display_folder.folder_id + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_id + """, + """ + ( + SELECT display_folder.path + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_path + """, + ] + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + if match_query: + joins.append("JOIN file_fts ON file_fts.file_ref = f.file_ref") + selects.append("snippet(file_fts, 2, '', '', '...', 16) AS snippet") + selects.append("bm25(file_fts) AS rank") + where.append("file_fts MATCH ?") + params.append(match_query) + order_by = "rank" + else: + selects.append("f.descriptor AS snippet") + selects.append("0 AS rank") + order_by = "f.created_at DESC, f.title" + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + sql = f""" + SELECT {", ".join(selects)} + FROM files f + {" ".join(joins)} + WHERE {" AND ".join(where)} + ORDER BY {order_by} + LIMIT ? + """ + params.append(limit) + with self.connect() as conn: + return conn.execute(sql, params).fetchall() + + def _metadata_filter_sql(self, metadata_filter: Optional[dict[str, Any]]) -> tuple[list[str], list[Any]]: + if not metadata_filter: + return [], [] + clause, params = self._compile_metadata_filter(metadata_filter) + return [clause] if clause else [], params + + def _compile_metadata_filter(self, metadata_filter: dict[str, Any]) -> tuple[str, list[Any]]: + clauses = [] + params: list[Any] = [] + for key, condition in metadata_filter.items(): + if key in {"$and", "$or"}: + child_clauses = [] + child_params: list[Any] = [] + for item in condition: + child_clause, item_params = self._compile_metadata_filter(item) + if child_clause: + child_clauses.append(f"({child_clause})") + child_params.extend(item_params) + if child_clauses: + joiner = " AND " if key == "$and" else " OR " + clauses.append(joiner.join(child_clauses)) + params.extend(child_params) + continue + field_clause, field_params = self._compile_metadata_field_filter(key, condition) + clauses.append(field_clause) + params.extend(field_params) + return " AND ".join(f"({clause})" for clause in clauses), params + + def _compile_metadata_field_filter(self, field: str, condition: Any) -> tuple[str, list[Any]]: + if not isinstance(condition, dict) or not any(str(key).startswith("$") for key in condition): + condition = {"$eq": condition} + operator, expected = next(iter(condition.items())) + field_id = self.field_id(field) + if operator == "$eq": + return ( + """ + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text = ? + ) + """, + [field_id, self._metadata_compare_text(expected)], + ) + if operator == "$ne": + return ( + """ + NOT EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text = ? + ) + """, + [field_id, self._metadata_compare_text(expected)], + ) + if operator == "$in": + values = list(expected) + if not values: + return "0", [] + placeholders = ", ".join("?" for _ in values) + return ( + f""" + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text IN ({placeholders}) + ) + """, + [field_id, *(self._metadata_compare_text(item) for item in values)], + ) + if operator == "$contains": + return ( + """ + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND lower(mv.value_text) LIKE lower(?) ESCAPE '\\' + ) + """, + [field_id, self._contains_like(self._metadata_compare_text(expected))], + ) + raise ValueError(f"Unsupported metadata operator: {operator}") + + def get_file(self, file_ref: str) -> FileEntry: + with self.connect() as conn: + row = self._file_entry_row(conn, file_ref) + if row is None: + raise KeyError(f"Unknown file_ref: {file_ref}") + return self._file_entry(row) + + def update_file_metadata_status( + self, + file_ref: str, + *, + metadata: dict[str, Any], + metadata_status: dict[str, Any], + ) -> None: + with self.connect() as conn: + row = self._file_entry_row(conn, file_ref) + if row is None: + raise KeyError(f"Unknown file_ref: {file_ref}") + indexed_metadata = self.indexed_metadata_values(metadata) + metadata_text_value = metadata_text(indexed_metadata) + conn.execute( + """ + UPDATE files + SET metadata_json = ?, + metadata_status_json = ?, + updated_at = CURRENT_TIMESTAMP + WHERE file_ref = ? AND deleted_at IS NULL + """, + ( + json.dumps(metadata, ensure_ascii=False), + json.dumps(metadata_status, ensure_ascii=False), + file_ref, + ), + ) + self.replace_metadata_values( + conn, + file_ref, + indexed_metadata, + ) + conn.execute( + """ + UPDATE file_fts + SET metadata_text = ? + WHERE file_ref = ? + """, + (metadata_text_value, file_ref), + ) + + def delete_file(self, target: str) -> None: + with self.connect() as conn: + file_ref = self._resolve_file_ref(conn, target) + conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (file_ref,)) + conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) + conn.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) + + def folder_exists(self, path: str) -> bool: + path = normalize_path(path) + with self.connect() as conn: + row = conn.execute( + "SELECT 1 FROM folders WHERE path = ?", + (path,), + ).fetchone() + return row is not None + + def delete_empty_folder(self, path: str) -> bool: + path = normalize_path(path) + if path == "/": + return False + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + return False + has_files = conn.execute( + """ + SELECT 1 + FROM file_folders + WHERE folder_id = ? + LIMIT 1 + """, + (folder["folder_id"],), + ).fetchone() + if has_files is not None: + return False + has_children = conn.execute( + """ + SELECT 1 + FROM folders + WHERE parent_id = ? + LIMIT 1 + """, + (folder["folder_id"],), + ).fetchone() + if has_children is not None: + return False + conn.execute("DELETE FROM folders WHERE folder_id = ?", (folder["folder_id"],)) + return True + + def resolve_file_ref(self, target: str) -> str: + with self.connect() as conn: + return self._resolve_file_ref(conn, target) + + def _resolve_file_ref(self, conn: sqlite3.Connection, target: str) -> str: + target = str(target).strip() + if not target: + raise KeyError("Empty file target") + row = conn.execute( + "SELECT file_ref FROM files WHERE file_ref = ? AND deleted_at IS NULL", + (target,), + ).fetchone() + if row: + return row["file_ref"] + row = conn.execute( + "SELECT file_ref FROM files WHERE external_id = ? AND deleted_at IS NULL", + (target,), + ).fetchone() + if row: + return row["file_ref"] + virtual_file_ref = self._resolve_virtual_file_ref(conn, target) + if virtual_file_ref: + return virtual_file_ref + raise KeyError(f"Unknown file target: {target}") + + def _resolve_virtual_file_ref(self, conn: sqlite3.Connection, target: str) -> str | None: + virtual_target = normalize_path(target) + rows = conn.execute( + """ + WITH virtual_matches AS ( + SELECT + f.file_ref, + f.external_id, + f.title, + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ) AS display_title, + pf.path AS folder_path, + (CASE WHEN pf.path = '/' THEN '/' ELSE pf.path || '/' END) + || ltrim( + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ), + '/' + ) AS title_virtual_path + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders pf ON pf.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + ) + SELECT + file_ref, + external_id, + display_title AS title, + MIN(folder_path) AS folder_path + FROM virtual_matches + WHERE title_virtual_path = ? + GROUP BY file_ref, external_id, display_title + ORDER BY file_ref + LIMIT 2 + """, + (virtual_target,), + ).fetchall() + if not rows: + return None + if len(rows) > 1: + matches = "; ".join(self._virtual_match_summary(row) for row in rows) + raise KeyError(f"Ambiguous file target: {target}. Matches: {matches}") + return rows[0]["file_ref"] + + @staticmethod + def _virtual_match_summary(row: sqlite3.Row) -> str: + external_id = row["external_id"] or "-" + return ( + f"file_ref={row['file_ref']} external_id={external_id} " + f"folder={row['folder_path']} title={row['title']!r}" + ) + + def ensure_folder( + self, + conn: sqlite3.Connection | None, + path: str, + *, + kind: str = "manual", + description: str = "", + metadata: dict[str, Any] | None = None, + ) -> str: + owns_connection = conn is None + if conn is None: + conn = self.connect() + try: + normalized = normalize_path(path) + metadata_json = json.dumps(metadata or {}, ensure_ascii=False) + if normalized == "/": + folder_id = self.folder_id("/") + existing = conn.execute( + "SELECT folder_id FROM folders WHERE path = '/'" + ).fetchone() + if existing is not None and not description and metadata_json == "{}": + if owns_connection: + conn.commit() + return folder_id + self._upsert_folder_row( + conn, + folder_id=folder_id, + parent_id=None, + name="/", + path="/", + kind=kind, + description=description, + metadata_json=metadata_json, + ) + if owns_connection: + conn.commit() + return folder_id + parent_id = self.ensure_folder(conn, str(Path(normalized).parent), kind=kind) + name = normalized.rsplit("/", 1)[-1] + folder_id = self.folder_id(normalized) + self._upsert_folder_row( + conn, + folder_id=folder_id, + parent_id=parent_id, + name=name, + path=normalized, + kind=kind, + description=description, + metadata_json=metadata_json, + ) + if owns_connection: + conn.commit() + return folder_id + finally: + if owns_connection: + conn.close() + + def _upsert_folder_row( + self, + conn: sqlite3.Connection, + *, + folder_id: str, + parent_id: str | None, + name: str, + path: str, + kind: str, + description: str, + metadata_json: str, + ) -> None: + columns = self._columns(conn, "folders") + insert_columns = ["folder_id", "parent_id", "name", "path", "description", "kind", "metadata_json"] + values: list[Any] = [folder_id, parent_id, name, path, description, kind, metadata_json] + if "source" in columns: + insert_columns.append("source") + values.append("system") + if "sort_order" in columns: + insert_columns.append("sort_order") + values.append(0) + placeholders = ", ".join("?" for _ in values) + update_assignments = [ + "parent_id = excluded.parent_id", + "name = excluded.name", + "kind = excluded.kind", + "updated_at = CURRENT_TIMESTAMP", + ] + if description: + update_assignments.append("description = excluded.description") + if metadata_json != "{}": + update_assignments.append("metadata_json = excluded.metadata_json") + conn.execute( + f""" + INSERT INTO folders({", ".join(insert_columns)}) + VALUES ({placeholders}) + ON CONFLICT(path) DO UPDATE SET + {", ".join(update_assignments)} + """, + values, + ) + + def _resolve_or_create_folder(self, conn: sqlite3.Connection, folder_path_or_id: str) -> str: + target = str(folder_path_or_id).strip() + if not target: + raise KeyError("Empty folder target") + row = conn.execute( + "SELECT folder_id FROM folders WHERE folder_id = ?", + (target,), + ).fetchone() + if row: + return row["folder_id"] + row = conn.execute( + "SELECT folder_id FROM folders WHERE path = ?", + (normalize_path(target),), + ).fetchone() + if row: + return row["folder_id"] + return self.ensure_folder(conn, target) + + def read_text(self, file_ref: str) -> str: + entry = self.get_file(file_ref) + return Path(entry.text_artifact_path).read_text(encoding="utf-8") + + def write_text_artifact(self, file_ref: str, content: str) -> Path: + path = self.text_dir / f"{file_ref}.txt" + path.write_text(content, encoding="utf-8") + return path + + def update_pageindex_pointer( + self, + file_ref: str, + *, + pageindex_doc_id: str | None, + pageindex_tree_status: str, + ) -> None: + with self.connect() as conn: + resolved = self._resolve_file_ref(conn, file_ref) + conn.execute( + """ + UPDATE files + SET pageindex_doc_id = ?, + pageindex_tree_status = ?, + updated_at = CURRENT_TIMESTAMP + WHERE file_ref = ? AND deleted_at IS NULL + """, + (pageindex_doc_id, pageindex_tree_status, resolved), + ) + + def write_raw_artifact(self, file_ref: str, metadata: dict[str, Any]) -> Path: + path = self.raw_dir / f"{file_ref}.json" + path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + def file_info(self, target: str) -> dict[str, Any]: + file_ref = self.resolve_file_ref(target) + entry = self.get_file(file_ref) + info = self._file_entry_to_dict(entry) + info["folders"] = self.folder_memberships(file_ref) + return info + + def file_matches( + self, + file_ref: str, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> bool: + where = ["f.file_ref = ?", "f.deleted_at IS NULL"] + params: list[Any] = [file_ref] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + row = conn.execute( + f""" + SELECT 1 + FROM files f + WHERE {" AND ".join(where)} + LIMIT 1 + """, + params, + ).fetchone() + return row is not None + + def folder_memberships(self, file_ref: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json AS folder_metadata_json, + ff.metadata_json AS membership_metadata_json, + ff.created_at + FROM file_folders ff + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE ff.file_ref = ? + ORDER BY fo.path + """, + (file_ref,), + ).fetchall() + return [ + { + "folder_id": row["folder_id"], + "id": row["folder_id"], + "parent_id": row["parent_id"], + "parent_folder_id": row["parent_id"], + "name": row["name"], + "path": row["path"], + "kind": row["kind"], + "description": row["description"], + "folder_metadata": json.loads(row["folder_metadata_json"] or "{}"), + "metadata": json.loads(row["membership_metadata_json"] or "{}"), + "created_at": row["created_at"], + } + for row in rows + ] + + def count_files_in_folder(self, path: str, *, recursive: bool = True) -> int: + path = normalize_path(path) + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + if recursive: + row = conn.execute( + """ + SELECT COUNT(DISTINCT f.file_ref) AS count + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\\') + """, + (path, self._descendant_like(path)), + ).fetchone() + else: + row = conn.execute( + """ + SELECT COUNT(DISTINCT f.file_ref) AS count + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND fo.path = ? + """, + (path,), + ).fetchone() + return int(row["count"] or 0) + + def file_basename_exists_in_folder(self, path: str, basename: str) -> bool: + path = normalize_path(path) + basename = str(basename).strip() + if not basename: + return False + with self.connect() as conn: + row = conn.execute( + """ + SELECT 1 + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND fo.path = ? + AND f.title = ? + LIMIT 1 + """, + ( + path, + basename, + ), + ).fetchone() + return row is not None + + def folder_subtree_thresholds( + self, + path: str, + *, + depth_limit: int, + file_limit: int, + ) -> dict[str, Any]: + path = normalize_path(path) + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + base_depth = self._folder_depth(path) + deep_folder = conn.execute( + """ + SELECT path + FROM folders + WHERE path != ? + AND path LIKE ? ESCAPE '\\' + AND ( + CASE + WHEN TRIM(path, '/') = '' THEN 0 + ELSE LENGTH(TRIM(path, '/')) - LENGTH(REPLACE(TRIM(path, '/'), '/', '')) + 1 + END + ) - ? > ? + LIMIT 1 + """, + (path, self._descendant_like(path), base_depth, depth_limit), + ).fetchone() + file_rows = conn.execute( + """ + SELECT DISTINCT f.file_ref + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\\') + LIMIT ? + """, + (path, self._descendant_like(path), file_limit + 1), + ).fetchall() + return { + "depth_limit": depth_limit, + "file_limit": file_limit, + "folder_depth_exceeds_limit": deep_folder is not None, + "file_count_exceeds_limit": len(file_rows) > file_limit, + "sampled_file_count": len(file_rows), + "sample_deep_folder_path": deep_folder["path"] if deep_folder is not None else "", + } + + def _file_entry_row(self, conn: sqlite3.Connection, file_ref: str) -> sqlite3.Row | None: + return conn.execute( + """ + SELECT + f.file_ref, + f.external_id, + f.storage_uri, + f.title, + f.descriptor, + f.content_type, + f.source_type, + f.fingerprint, + f.text_artifact_path, + f.raw_artifact_path, + f.pageindex_doc_id, + f.pageindex_tree_status, + f.metadata_json, + f.metadata_status_json, + COALESCE( + ( + SELECT display_folder.path + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ), + '/' + ) AS folder_path + FROM files f + WHERE f.file_ref = ? AND f.deleted_at IS NULL + """, + (file_ref,), + ).fetchone() + + def _file_rows_for_scope( + self, + conn: sqlite3.Connection, + path: str, + recursive: bool, + limit: int, + max_depth: int | None = None, + ) -> list[sqlite3.Row]: + sql = """ + SELECT * + FROM ( + SELECT + f.file_ref, + f.external_id, + f.title, + f.descriptor, + f.pageindex_tree_status, + f.metadata_json, + f.metadata_status_json, + f.created_at, + pf.folder_id AS folder_id, + pf.path AS folder_path, + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ) AS display_title, + ROW_NUMBER() OVER ( + PARTITION BY f.file_ref + ORDER BY pf.path, pf.folder_id + ) AS membership_rank + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders pf ON pf.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + """ + params: list[Any] + if recursive: + sql += " AND (pf.path = ? OR pf.path LIKE ? ESCAPE '\\')" + params = [path, self._descendant_like(path)] + if max_depth is not None: + if max_depth <= 0: + sql += " AND 0" + else: + sql += f" AND ({self._folder_depth_sql('pf.path')} - ?) <= ?" + params.extend([self._folder_depth(path), max_depth - 1]) + else: + sql += " AND pf.path = ?" + params = [path] + sql += """ + ) + WHERE membership_rank = 1 + ORDER BY lower(display_title), display_title, lower(folder_path), folder_path, file_ref + LIMIT ? + """ + params.append(limit) + return conn.execute(sql, params).fetchall() + + def _scope_sql(self, scope: Optional[dict[str, Any]]) -> tuple[str, list[Any]]: + if not scope: + return "", [] + recursive = scope.get("recursive", True) + max_depth = scope.get("max_depth") + if max_depth is not None: + max_depth = int(max_depth) + if max_depth < 0: + raise ValueError("max_depth must be non-negative") + folder_id = scope.get("folder_id") + if folder_id: + if folder_id == "root": + folder_path = "/" + else: + if recursive: + if max_depth == 0: + return "0", [] + depth_clause = "" + depth_params: list[Any] = [] + if max_depth is not None: + depth_clause = ( + "AND " + f"({self._folder_depth_sql('scope_folder.path')} - " + f"{self._folder_depth_sql('base_folder.path')}) <= ?" + ) + depth_params = [max_depth - 1] + return ( + f""" + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + JOIN folders scope_folder + ON scope_folder.folder_id = scope_ff.folder_id + JOIN folders base_folder + ON base_folder.folder_id = ? + WHERE scope_ff.file_ref = f.file_ref + AND ( + scope_folder.folder_id = base_folder.folder_id + OR scope_folder.path LIKE {self._descendant_like_sql_expr("base_folder.path")} ESCAPE '\\' + ) + {depth_clause} + ) + """, + [folder_id, *depth_params], + ) + return ( + """ + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + WHERE scope_ff.file_ref = f.file_ref + AND scope_ff.folder_id = ? + ) + """, + [folder_id], + ) + elif scope.get("folder_path") or scope.get("path"): + folder_path = normalize_path(scope.get("folder_path") or scope.get("path")) + else: + return "", [] + if recursive and max_depth == 0: + return "0", [] + path_clause = ( + "(scope_folder.path = ? OR scope_folder.path LIKE ? ESCAPE '\\')" + if recursive + else "scope_folder.path = ?" + ) + params = [folder_path, self._descendant_like(folder_path)] if recursive else [folder_path] + depth_clause = "" + if recursive and max_depth is not None: + depth_clause = f"AND ({self._folder_depth_sql('scope_folder.path')} - ?) <= ?" + params.extend([self._folder_depth(folder_path), max_depth - 1]) + return ( + f""" + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + JOIN folders scope_folder + ON scope_folder.folder_id = scope_ff.folder_id + WHERE scope_ff.file_ref = f.file_ref + AND {path_clause} + {depth_clause} + ) + """, + params, + ) + + def _folder_by_path(self, conn: sqlite3.Connection, path: str) -> sqlite3.Row | None: + return conn.execute( + """ + SELECT + folder_id, + parent_id, + name, + path, + description, + kind, + metadata_json, + created_at, + updated_at + FROM folders + WHERE path = ? + """, + (path,), + ).fetchone() + + @classmethod + def _descendant_like(cls, path: str) -> str: + return "/%" if path == "/" else f"{cls._like_escape(path)}/%" + + @staticmethod + def _descendant_like_sql_expr(path_expr: str) -> str: + escaped_expr = SQLiteFileSystemStore._like_escape_sql_expr(path_expr) + return f"CASE WHEN {path_expr} = '/' THEN '/%' ELSE {escaped_expr} || '/%' END" + + @staticmethod + def _contains_like(value: str) -> str: + return f"%{SQLiteFileSystemStore._like_escape(value)}%" + + @staticmethod + def _like_escape(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + + @staticmethod + def _like_escape_sql_expr(value_expr: str) -> str: + return ( + f"replace(replace(replace({value_expr}, '\\', '\\\\'), " + "'%', '\\%'), '_', '\\_')" + ) + + @staticmethod + def _folder_depth(path: str) -> int: + stripped = normalize_path(path).strip("/") + return 0 if not stripped else len(stripped.split("/")) + + @staticmethod + def _folder_depth_sql(path_expr: str) -> str: + return ( + "(CASE " + f"WHEN TRIM({path_expr}, '/') = '' THEN 0 " + f"ELSE LENGTH(TRIM({path_expr}, '/')) " + f"- LENGTH(REPLACE(TRIM({path_expr}, '/'), '/', '')) + 1 " + "END)" + ) + + @classmethod + def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: + return { + "folder_id": row["folder_id"], + "id": row["folder_id"], + "parent_id": row["parent_id"], + "parent_folder_id": row["parent_id"], + "name": row["name"], + "description": cls._row_value(row, "description", ""), + "path": row["path"], + "kind": row["kind"], + "metadata": json.loads(cls._row_value(row, "metadata_json", "{}") or "{}"), + "created_at": cls._row_value(row, "created_at"), + "updated_at": cls._row_value(row, "updated_at"), + "file_count": cls._row_value(row, "file_count", 0), + "children_count": cls._row_value(row, "children_count", 0), + "matched_files": cls._row_value(row, "matched_files", 0), + } + + @classmethod + def _file_summary(cls, row: sqlite3.Row) -> dict[str, Any]: + external_id = row["external_id"] + display_title = cls._row_value(row, "display_title", row["title"]) + return { + "file_ref": row["file_ref"], + "id": external_id or row["file_ref"], + "document_id": external_id, + "external_id": external_id, + "name": display_title, + "title": display_title, + "original_title": row["title"], + "description": cls._row_value(row, "descriptor", row["title"]), + "status": cls._row_value(row, "pageindex_tree_status", "not_built"), + "pageNum": None, + "createdAt": cls._row_value(row, "created_at"), + "folderId": cls._row_value(row, "folder_id"), + "folder_path": row["folder_path"], + "metadata": json.loads(row["metadata_json"] or "{}"), + "metadata_status": json.loads( + cls._row_value(row, "metadata_status_json", "{}") or "{}" + ), + } + + @classmethod + def _search_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: + external_id = row["external_id"] + return { + "file_ref": row["file_ref"], + "id": external_id or row["file_ref"], + "document_id": external_id, + "external_id": external_id, + "name": row["title"], + "title": row["title"], + "description": cls._row_value(row, "descriptor", row["title"]), + "status": cls._row_value(row, "pageindex_tree_status", "not_built"), + "pageNum": None, + "createdAt": cls._row_value(row, "created_at"), + "folderId": cls._row_value(row, "folder_id"), + "snippet": row["snippet"] or row["title"], + "folder_path": row["folder_path"], + "metadata": json.loads(row["metadata_json"] or "{}"), + "metadata_status": json.loads( + cls._row_value(row, "metadata_status_json", "{}") or "{}" + ), + } + + @staticmethod + def _row_value(row: sqlite3.Row, key: str, default: Any = None) -> Any: + return row[key] if key in row.keys() else default + + @staticmethod + def _file_entry(row: sqlite3.Row) -> FileEntry: + return FileEntry( + file_ref=row["file_ref"], + external_id=row["external_id"], + storage_uri=row["storage_uri"], + title=row["title"], + descriptor=row["descriptor"], + content_type=row["content_type"], + source_type=row["source_type"], + fingerprint=row["fingerprint"], + text_artifact_path=row["text_artifact_path"], + raw_artifact_path=row["raw_artifact_path"], + pageindex_doc_id=row["pageindex_doc_id"], + pageindex_tree_status=row["pageindex_tree_status"], + metadata=json.loads(row["metadata_json"] or "{}"), + folder_path=row["folder_path"], + metadata_status=json.loads( + SQLiteFileSystemStore._row_value(row, "metadata_status_json", "{}") or "{}" + ), + ) + + @classmethod + def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: + return { + "file_ref": entry.file_ref, + "id": entry.external_id or entry.file_ref, + "document_id": entry.external_id, + "external_id": entry.external_id, + "name": entry.title, + "path": cls._virtual_file_path(entry.folder_path, entry.title), + "title": entry.title, + "description": entry.descriptor, + "status": entry.pageindex_tree_status, + "pageNum": None, + "descriptor": entry.descriptor, + "content_type": entry.content_type, + "source_type": entry.source_type, + "fingerprint": entry.fingerprint, + "pageindex_doc_id": entry.pageindex_doc_id, + "pageindex_tree_status": entry.pageindex_tree_status, + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "folder_path": entry.folder_path, + } + + @staticmethod + def _virtual_file_path(folder_path: str, title: str) -> str: + folder_path = normalize_path(folder_path) + return f"/{title}" if folder_path == "/" else f"{folder_path}/{title}" + + @staticmethod + def _query_text(query: str | list[str] | None) -> str: + if query is None: + return "" + if isinstance(query, list): + return " ".join(str(item) for item in query) + return str(query) + + @classmethod + def _fts_match_queries(cls, query: str) -> list[str]: + terms = cls._fts_terms(query) + if not terms: + return [] + queries = [" ".join(terms)] + if len(terms) > 1: + queries.append(" OR ".join(terms)) + return queries + + @staticmethod + def _fts_terms(query: str) -> list[str]: + stopwords = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "did", + "do", + "does", + "for", + "from", + "how", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "to", + "was", + "were", + "what", + "when", + "where", + "which", + "who", + "why", + "with", + } + terms = re.findall(r"[A-Za-z0-9_]+", query.lower()) + unique_terms = [] + seen = set() + for term in terms: + if term in stopwords or term in seen: + continue + seen.add(term) + unique_terms.append(term) + return unique_terms + + @staticmethod + def _metadata_value_items(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + items = [] + for item in value: + items.extend(SQLiteFileSystemStore._metadata_value_items(item)) + return items + return [SQLiteFileSystemStore._metadata_compare_text(value)] + + @staticmethod + def _metadata_compare_text(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return "" if value is None else str(value) + + @staticmethod + def indexed_metadata_values(metadata: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in metadata.items() if key != "summary"} + + @staticmethod + def _valid_field_name(name: str) -> bool: + return re.match(r"^[A-Za-z][A-Za-z0-9_]*$", str(name)) is not None + + @staticmethod + def folder_id(path: str) -> str: + normalized = normalize_path(path) + if normalized == "/": + return "folder_root" + digest = hashlib.sha1(normalized.encode("utf-8")).hexdigest()[:16] + return f"folder_{digest}" + + @staticmethod + def field_id(name: str) -> str: + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16] + return f"field_{digest}" + + +def normalize_path(path: str | Path | None) -> str: + if path is None: + return "/" + if str(path).strip().lower() == "root": + return "/" + parts = [part for part in str(path).replace("\\", "/").split("/") if part and part != "."] + if ".." in parts: + raise ValueError("PIFS paths must not contain '..'") + return "/" + "/".join(parts) if parts else "/" + + +def make_file_ref(seed: str) -> str: + digest = hashlib.sha1(seed.encode("utf-8")).hexdigest()[:16] + return f"file_{digest}" + + +def fingerprint(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def metadata_text(metadata: dict[str, Any]) -> str: + values = [] + for value in metadata.values(): + if isinstance(value, list): + values.extend(str(item) for item in value) + elif isinstance(value, dict): + values.append(json.dumps(value, ensure_ascii=False, sort_keys=True)) + elif value is not None: + values.append(str(value)) + return " ".join(values) diff --git a/pageindex/filesystem/types.py b/pageindex/filesystem/types.py new file mode 100644 index 000000000..82c04bb54 --- /dev/null +++ b/pageindex/filesystem/types.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass(frozen=True) +class SearchResult: + file_ref: str + external_id: Optional[str] + title: str + snippet: str + folder_path: str + folder_paths: list[str] + metadata: dict[str, Any] + id: Optional[str] = None + document_id: Optional[str] = None + name: str = "" + description: str = "" + status: str = "" + pageNum: Optional[int] = None + createdAt: Optional[str] = None + folderId: Optional[str] = None + metadata_status: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class OpenResult: + file_ref: str + start_line: int + end_line: int + text: str + external_id: Optional[str] = None + folder_path: str = "" + + +@dataclass(frozen=True) +class FolderEntry: + folder_id: str + parent_id: Optional[str] + name: str + path: str + kind: str + + +@dataclass(frozen=True) +class FileEntry: + file_ref: str + external_id: Optional[str] + storage_uri: str + title: str + descriptor: str + content_type: str + source_type: Optional[str] + fingerprint: str + text_artifact_path: str + raw_artifact_path: Optional[str] + pageindex_doc_id: Optional[str] + pageindex_tree_status: str + metadata: dict[str, Any] + folder_path: str + metadata_status: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MetadataField: + name: str + description: str = "" + source: str = "manual" + + +@dataclass(frozen=True) +class PIFSQueryScope: + path: str + folder_path: str + metadata_filter: dict[str, str] = field(default_factory=dict) + metadata_axis: Optional[str] = None + + +@dataclass(frozen=True) +class CommandResult: + command: str + data: Any + text: str diff --git a/pifs b/pifs new file mode 100755 index 000000000..fb2dbc08e --- /dev/null +++ b/pifs @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +if [ -x "$SCRIPT_DIR/.venv/bin/python" ]; then + exec "$SCRIPT_DIR/.venv/bin/python" -m pageindex.filesystem.cli "$@" +fi + +exec python3 -m pageindex.filesystem.cli "$@" diff --git a/requirements.txt b/requirements.txt index ae92bc49f..b5e5df9e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ litellm==1.84.0 -# openai-agents # optional: required for examples/agentic_vectorless_rag_demo.py +# openai-agents==0.17.2 # optional: required for pifs chat/ask and examples/agentic_vectorless_rag_demo.py pymupdf==1.26.4 PyPDF2==3.0.1 python-dotenv==1.2.2 pyyaml==6.0.2 +sqlite-vec>=0.1.9 diff --git a/tests/pifs_markdown_fixture.py b/tests/pifs_markdown_fixture.py new file mode 100644 index 000000000..cc7dc3525 --- /dev/null +++ b/tests/pifs_markdown_fixture.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +class FakePageIndexClient: + def __init__(self): + self.documents: dict[str, dict[str, Any]] = {} + + def index(self, file_path, mode="auto"): + path = Path(file_path) + doc_id = f"pi_{path.stem}_{len(self.documents)}" + text = path.read_text(encoding="utf-8") + self.documents[doc_id] = { + "id": doc_id, + "type": "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}", + "line_count": len(text.splitlines()), + "structure": [{"title": path.stem, "node_id": "0001", "nodes": []}], + "pages": [{"page": 1, "content": text}], + } + return doc_id + + def _ensure_doc_loaded(self, doc_id): + return None + + def get_document_structure(self, doc_id): + import json + + return json.dumps(self.documents[doc_id]["structure"]) + + def get_page_content(self, doc_id, pages): + import json + + return json.dumps(self.documents[doc_id]["pages"]) + + +def register_markdown( + filesystem, + tmp_path: Path, + external_id: str, + folder_path: str, + *, + title: str | None = None, + text: str | None = None, + metadata: dict[str, Any] | None = None, +): + client = getattr(filesystem, "_test_pageindex_client", None) + if client is None: + client = FakePageIndexClient() + filesystem._test_pageindex_client = client + filesystem._pageindex_client = lambda: client + filename = title if title and title.endswith(".md") else f"{external_id}.md" + source = tmp_path / filename + source.write_text(text or f"{external_id} alpha evidence", encoding="utf-8") + return filesystem.register_file( + storage_uri=source.as_uri(), + folder_path=folder_path, + external_id=external_id, + title=title or filename, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata=metadata or {}, + ) diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py new file mode 100644 index 000000000..affeb50cb --- /dev/null +++ b/tests/test_filesystem_store.py @@ -0,0 +1,134 @@ +import json + +import pytest + +from tests.pifs_markdown_fixture import register_markdown + + +def test_insert_files_does_not_disable_sqlite_synchronous(tmp_path): + from pageindex.filesystem.store import SQLiteFileSystemStore + + statements = [] + + class RecordingStore(SQLiteFileSystemStore): + def connect(self): + conn = super().connect() + conn.set_trace_callback(statements.append) + return conn + + store = RecordingStore(tmp_path / "workspace") + statements.clear() + + store.insert_files( + [ + { + "file_ref": "ref_report", + "external_id": "doc_report", + "storage_uri": "file:///tmp/report.pdf", + "folder_path": "/documents", + "title": "Report", + "descriptor": "documents/report.pdf", + "content_type": "application/pdf", + "source_type": "documents", + "fingerprint": "fingerprint", + "text_artifact_path": "artifacts/text/ref_report.txt", + "raw_artifact_path": None, + "metadata": {}, + "metadata_json": json.dumps({}), + "metadata_text": "", + "content": "", + "skip_fts": True, + } + ] + ) + + assert not any( + statement.upper().replace(" ", "") == "PRAGMASYNCHRONOUS=OFF" + for statement in statements + ) + + +def test_register_file_rejects_parent_directory_segments(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + + with pytest.raises(ValueError, match="must not contain '\\.\\.'"): + filesystem.register_file( + storage_uri="file:///tmp/report.txt", + folder_path="/a/../b", + external_id="doc_bad", + title="Bad", + content="bad body", + ) + + +def test_file_upsert_preserves_created_at(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version one") + with filesystem.store.connect() as conn: + conn.execute( + "UPDATE files SET created_at = '2001-02-03 04:05:06' WHERE file_ref = ?", + (file_ref,), + ) + register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version two") + + with filesystem.store.connect() as conn: + row = conn.execute( + "SELECT created_at, storage_uri, deleted_at FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + + assert row["created_at"] == "2001-02-03 04:05:06" + assert row["storage_uri"].endswith("/doc_report.md") + assert row["deleted_at"] is None + + +def test_file_upsert_preserves_soft_delete_tombstone(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version one") + with filesystem.store.connect() as conn: + conn.execute( + "UPDATE files SET deleted_at = '2001-02-03 04:05:06' WHERE file_ref = ?", + (file_ref,), + ) + register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version two") + + with filesystem.store.connect() as conn: + row = conn.execute( + "SELECT storage_uri, deleted_at FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + + assert row["storage_uri"].endswith("/doc_report.md") + assert row["deleted_at"] == "2001-02-03 04:05:06" + assert filesystem.search(None) == [] + + +def test_listing_uses_one_consistent_folder_membership_row(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown( + filesystem, + tmp_path, + "doc_shared", + "/a", + title="Original.md", + text="shared body", + ) + filesystem.store.attach_file_to_folder( + file_ref, + "/b", + metadata={"display_name": "Alpha"}, + ) + + listing = filesystem.browse("/", recursive=True, limit=10) + row = next(item for item in listing["files"] if item["external_id"] == "doc_shared") + + assert row["folder_path"] == "/a" + assert row["title"] == "Original.md" diff --git a/tests/test_import_surface.py b/tests/test_import_surface.py new file mode 100644 index 000000000..ce52b8ec1 --- /dev/null +++ b/tests/test_import_surface.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import builtins +import importlib +import sys + + +def test_filesystem_import_works_without_eager_optional_dependencies(monkeypatch): + blocked_roots = {"litellm", "openai", "PyPDF2", "pymupdf", "sqlite_vec"} + real_import = builtins.__import__ + + def clear_pageindex_modules() -> None: + for name in list(sys.modules): + if name == "pageindex" or name.startswith("pageindex."): + sys.modules.pop(name, None) + + original_pageindex_modules = { + name: module + for name, module in sys.modules.items() + if name == "pageindex" or name.startswith("pageindex.") + } + + def restore_pageindex_modules() -> None: + clear_pageindex_modules() + sys.modules.update(original_pageindex_modules) + + def import_without_optional_deps(name, globals=None, locals=None, fromlist=(), level=0): + root = name.split(".", 1)[0] + if root in blocked_roots: + raise ModuleNotFoundError(f"No module named '{root}'", name=root) + return real_import(name, globals, locals, fromlist, level) + + clear_pageindex_modules() + try: + with monkeypatch.context() as patch: + patch.setattr(builtins, "__import__", import_without_optional_deps) + + filesystem_module = importlib.import_module("pageindex.filesystem") + from pageindex import PageIndexFileSystem as TopLevelPageIndexFileSystem + from pageindex.filesystem import PageIndexFileSystem + + assert filesystem_module.PageIndexFileSystem is PageIndexFileSystem + assert TopLevelPageIndexFileSystem is PageIndexFileSystem + finally: + restore_pageindex_modules() diff --git a/tests/test_pifs_add_command.py b/tests/test_pifs_add_command.py new file mode 100644 index 000000000..80670eb28 --- /dev/null +++ b/tests/test_pifs_add_command.py @@ -0,0 +1,484 @@ +import json +from pathlib import Path + +import pytest + + +class StaticEmbedder: + def embed(self, texts): + return [[1.0, 0.0, 0.0] for _ in texts] + + +def make_summary_indexer(workspace: Path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + return SummaryProjectionIndexer( + workspace / "artifacts" / "projection_indexes", + embedder=StaticEmbedder(), + embedding_provider="test", + embedding_model="static", + embedding_dimensions=3, + ) + + +def make_filesystem(workspace: Path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem( + workspace=workspace, + summary_projection_embedding_provider="test", + summary_projection_embedding_model="static", + summary_projection_embedding_dimensions=3, + ) + filesystem.summary_projection_indexer = make_summary_indexer(workspace) + return filesystem + + +@pytest.fixture(autouse=True) +def fake_pageindex_index(monkeypatch): + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + path = Path(file_path) + doc_id = f"doc_{path.stem}" + text = path.read_text(encoding="utf-8") + doc = { + "id": doc_id, + "type": "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}: {text[:60]}", + "line_count": len(text.splitlines()), + "structure": [ + { + "title": path.stem, + "node_id": "0001", + "line_num": 1, + "text": text, + "nodes": [], + } + ], + "pages": [{"page": 1, "content": text}], + } + write_pageindex_client_doc(self.workspace, doc_id, doc) + self.documents[doc_id] = doc + return doc_id + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + + +def write_pageindex_client_doc(workspace: Path, doc_id: str, doc: dict) -> None: + workspace.mkdir(parents=True, exist_ok=True) + (workspace / f"{doc_id}.json").write_text( + json.dumps(doc, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + meta = { + doc_id: { + "type": doc.get("type", ""), + "doc_name": doc.get("doc_name", ""), + "doc_description": doc.get("doc_description", ""), + "path": doc.get("path", ""), + "line_count": doc.get("line_count"), + } + } + (workspace / "_meta.json").write_text( + json.dumps(meta, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def test_add_text_folder_target_copies_artifact_indexes_summary_and_is_readable(tmp_path): + from pageindex.filesystem import PIFSCommandExecutor + + source = tmp_path / "filing.md" + source.write_text("alpha filing text for pifs add", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + + info = filesystem.add_file(str(source), "/documents/reports") + + assert info["path"] == "/documents/reports/filing.md" + assert info["folder_path"] == "/documents/reports" + assert filesystem.folder_info("/documents/reports")["path"] == "/documents/reports" + entry = filesystem.store.get_file(info["file_ref"]) + assert entry.storage_uri != source.as_uri() + assert "/artifacts/uploads/" in entry.storage_uri + copied_path = Path(entry.storage_uri.removeprefix("file://")) + assert copied_path.read_text(encoding="utf-8") == "alpha filing text for pifs add" + assert copied_path.resolve() != source.resolve() + + executor = PIFSCommandExecutor(filesystem) + rendered = json.loads(executor.execute("grep alpha /documents/reports/filing.md")) + + assert rendered["data"]["matches"] == [ + {"line": 1, "text": "alpha filing text for pifs add"} + ] + assert info["metadata"]["summary"].startswith("Summary for filing.md") + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 1 + + +def test_add_rejects_same_folder_same_basename_without_overwrite(tmp_path): + from pageindex.filesystem import PIFSCommandExecutor + + source = tmp_path / "conflict.md" + source.write_text("first body", encoding="utf-8") + filesystem = make_filesystem(tmp_path / "workspace") + + filesystem.add_file(source, "/documents") + source.write_text("second body must not overwrite", encoding="utf-8") + + with pytest.raises(FileExistsError, match="already exists"): + filesystem.add_file(source, "/documents") + + executor = PIFSCommandExecutor(filesystem) + rendered = json.loads(executor.execute("grep first /documents/conflict.md")) + assert rendered["data"]["matches"] == [{"line": 1, "text": "first body"}] + + +def test_add_rejects_unsupported_type_before_registration(tmp_path): + source = tmp_path / "payload.json" + source.write_text('{"unsupported": true}', encoding="utf-8") + filesystem = make_filesystem(tmp_path / "workspace") + + with pytest.raises(ValueError, match="Unsupported file type"): + filesystem.add_file(source, "/documents") + + assert filesystem.browse("/", recursive=True)["files"] == [] + assert not list((tmp_path / "workspace" / "artifacts" / "uploads").glob("**/*")) + + +def test_add_configures_semantic_retrieval_in_same_filesystem_instance(tmp_path): + source = tmp_path / "semantic.md" + source.write_text("alpha semantic recall text", encoding="utf-8") + filesystem = make_filesystem(tmp_path / "workspace") + + assert filesystem.semantic_retrieval_channels() == () + + filesystem.add_file(source, "/documents") + + assert filesystem.semantic_retrieval_channels() == ("summary",) + results = filesystem.browse_semantic_files( + "/documents", + "semantic recall", + recursive=True, + page_size=5, + ) + assert [item["path"] for item in results["data"]] == ["/documents/semantic.md"] + + +def test_add_markdown_builds_pageindex_tree_from_copied_artifact(tmp_path, monkeypatch): + from pageindex import PageIndexClient + from pageindex.filesystem import PIFSCommandExecutor + + indexed_paths = [] + + def fake_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path)) + doc_id = "doc_added_md" + doc = { + "id": doc_id, + "type": "md", + "path": str(Path(file_path).resolve()), + "doc_name": "notes.md", + "doc_description": "summary", + "line_count": 3, + "structure": [ + { + "title": "Notes", + "node_id": "0001", + "line_num": 1, + "text": "# Notes\n\ncopied markdown body", + "nodes": [], + } + ], + } + write_pageindex_client_doc(self.workspace, doc_id, doc) + self.documents[doc_id] = doc + return doc_id + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + source = tmp_path / "notes.md" + source.write_text("# Notes\n\ncopied markdown body", encoding="utf-8") + filesystem = make_filesystem(tmp_path / "workspace") + + info = filesystem.add_file(source, "/documents") + executor = PIFSCommandExecutor(filesystem) + structure = json.loads(executor.execute("cat /documents/notes.md --structure")) + entry = filesystem.store.get_file(info["file_ref"]) + + assert structure["data"]["document"]["available"] is True + assert structure["data"]["structure"][0]["title"] == "Notes" + assert indexed_paths == [Path(entry.storage_uri.removeprefix("file://"))] + assert indexed_paths[0].resolve() != source.resolve() + + +def test_add_failure_does_not_leave_visible_catalog_or_artifacts(tmp_path, monkeypatch): + source = tmp_path / "atomic.md" + source.write_text("atomic body", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + + def fail_insert(records): + raise RuntimeError("catalog insert failed") + + monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) + + with pytest.raises(RuntimeError, match="catalog insert failed"): + filesystem.add_file(source, "/documents") + + assert filesystem.browse("/", recursive=True)["files"] == [] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + assert not list((workspace / "artifacts" / "text").glob("*.txt")) + assert not list((workspace / "artifacts" / "raw").glob("*.json")) + + +def test_add_markdown_insert_failure_removes_pageindex_cache(tmp_path, monkeypatch): + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + doc_id = "doc_failed_add_md" + doc = { + "id": doc_id, + "type": "md", + "path": str(Path(file_path).resolve()), + "doc_name": "failed.md", + "doc_description": "summary", + "line_count": 3, + "structure": [ + { + "title": "Failed", + "node_id": "0001", + "line_num": 1, + "text": "# Failed\n\nbody", + "nodes": [], + } + ], + } + write_pageindex_client_doc(self.workspace, doc_id, doc) + self.documents[doc_id] = doc + return doc_id + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + source = tmp_path / "failed.md" + source.write_text("# Failed\n\nbody", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + + def fail_insert(records): + raise RuntimeError("catalog insert failed") + + monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) + + with pytest.raises(RuntimeError, match="catalog insert failed"): + filesystem.add_file(source, "/documents/reports") + + pageindex_workspace = workspace / "artifacts" / "pageindex_client" + assert not (pageindex_workspace / "doc_failed_add_md.json").exists() + meta_path = pageindex_workspace / "_meta.json" + if meta_path.exists(): + meta = json.loads(meta_path.read_text(encoding="utf-8")) + assert "doc_failed_add_md" not in meta + listing = filesystem.browse("/", recursive=True) + assert listing["files"] == [] + assert listing["folders"] == [] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + assert not list((workspace / "artifacts" / "text").glob("*.txt")) + assert not list((workspace / "artifacts" / "raw").glob("*.json")) + + +def test_add_markdown_index_failure_removes_pageindex_cache_delta(tmp_path, monkeypatch): + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + doc_id = "doc_partial_before_raise" + doc = { + "id": doc_id, + "type": "md", + "path": str(Path(file_path).resolve()), + "doc_name": "partial.md", + "doc_description": "summary", + "line_count": 3, + "structure": [{"title": "Partial", "node_id": "0001", "nodes": []}], + } + self.documents[doc_id] = doc + self._save_doc(doc_id) + raise RuntimeError("index failed after cache write") + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + source = tmp_path / "partial.md" + source.write_text("# Partial\n\nbody", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + pageindex_workspace = workspace / "artifacts" / "pageindex_client" + + with pytest.raises(RuntimeError, match="requires PageIndex extraction"): + filesystem.add_file(source, "/documents/reports") + + assert not (pageindex_workspace / "doc_partial_before_raise.json").exists() + meta_path = pageindex_workspace / "_meta.json" + if meta_path.exists(): + meta = json.loads(meta_path.read_text(encoding="utf-8")) + assert "doc_partial_before_raise" not in meta + listing = filesystem.browse("/", recursive=True) + assert listing["files"] == [] + assert listing["folders"] == [] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + assert not list((workspace / "artifacts" / "text").glob("*.txt")) + assert not list((workspace / "artifacts" / "raw").glob("*.json")) + + +def test_add_markdown_failure_preserves_unrelated_pageindex_cache(tmp_path, monkeypatch): + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + doc_id = "doc_failed_add_md" + doc = { + "id": doc_id, + "type": "md", + "path": str(Path(file_path).resolve()), + "doc_name": "failed.md", + "doc_description": "summary", + "line_count": 3, + "structure": [{"title": "Failed", "node_id": "0001", "nodes": []}], + } + self.documents[doc_id] = doc + self._save_doc(doc_id) + return doc_id + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + source = tmp_path / "failed.md" + source.write_text("# Failed\n\nbody", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + pageindex_workspace = workspace / "artifacts" / "pageindex_client" + write_pageindex_client_doc( + pageindex_workspace, + "doc_unrelated", + { + "id": "doc_unrelated", + "type": "md", + "path": str((tmp_path / "unrelated.md").resolve()), + "doc_name": "unrelated.md", + "doc_description": "summary", + "line_count": 1, + "structure": [{"title": "Unrelated", "node_id": "0001", "nodes": []}], + }, + ) + + def fail_insert(records): + raise RuntimeError("catalog insert failed") + + monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) + + with pytest.raises(RuntimeError, match="catalog insert failed"): + filesystem.add_file(source, "/documents") + + assert not (pageindex_workspace / "doc_failed_add_md.json").exists() + assert (pageindex_workspace / "doc_unrelated.json").exists() + meta = json.loads((pageindex_workspace / "_meta.json").read_text(encoding="utf-8")) + assert "doc_failed_add_md" not in meta + assert "doc_unrelated" in meta + + +def test_add_failure_after_summary_vector_rolls_back_catalog_and_vector( + tmp_path, monkeypatch +): + source = tmp_path / "post_vector.md" + source.write_text("post vector rollback body", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + + def fail_status_update(*args, **kwargs): + raise RuntimeError("metadata status update failed") + + monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) + + with pytest.raises(RuntimeError, match="metadata status update failed"): + filesystem.add_file(source, "/documents") + + assert filesystem.browse("/", recursive=True)["files"] == [] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + assert not list((workspace / "artifacts" / "text").glob("*.txt")) + assert not list((workspace / "artifacts" / "raw").glob("*.json")) + + +def test_add_failure_removes_nested_folders_created_only_for_add(tmp_path, monkeypatch): + source = tmp_path / "nested.md" + source.write_text("nested rollback body", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + + def fail_status_update(*args, **kwargs): + raise RuntimeError("metadata status update failed") + + monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) + + with pytest.raises(RuntimeError, match="metadata status update failed"): + filesystem.add_file(source, "/documents/reports") + + listing = filesystem.browse("/", recursive=True) + assert listing["files"] == [] + assert listing["folders"] == [] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + assert not list((workspace / "artifacts" / "text").glob("*.txt")) + assert not list((workspace / "artifacts" / "raw").glob("*.json")) + + +def test_add_failure_preserves_preexisting_parent_folder(tmp_path, monkeypatch): + source = tmp_path / "nested.md" + source.write_text("nested rollback body", encoding="utf-8") + workspace = tmp_path / "workspace" + filesystem = make_filesystem(workspace) + filesystem.create_folder("/documents") + + def fail_status_update(*args, **kwargs): + raise RuntimeError("metadata status update failed") + + monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) + + with pytest.raises(RuntimeError, match="metadata status update failed"): + filesystem.add_file(source, "/documents/reports") + + listing = filesystem.browse("/", recursive=True) + assert listing["files"] == [] + assert [folder["path"] for folder in listing["folders"]] == ["/documents"] + assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 + + +def test_cli_add_uses_workspace_and_prints_added_file(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + source = tmp_path / "cli.md" + source.write_text("cli body", encoding="utf-8") + calls = [] + + class FakeAddFileSystem: + def __init__(self, workspace, **_kwargs): + self.workspace = Path(workspace) + + def configure_existing_projection_retrieval(self): + return False + + def add_file(self, physical_path, virtual_target): + calls.append((self.workspace, physical_path, virtual_target)) + return { + "file_ref": "file_cli", + "path": "/documents/cli.md", + } + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeAddFileSystem) + + status = cli.main(["--workspace", str(tmp_path / "workspace"), "add", str(source), "/documents"]) + + assert status == 0 + assert calls == [(tmp_path / "workspace", str(source), "/documents")] + assert capsys.readouterr().out == ( + "added: /documents/cli.md\n" + "file_ref: file_cli\n" + ) diff --git a/tests/test_pifs_agent_stream.py b/tests/test_pifs_agent_stream.py new file mode 100644 index 000000000..5e1d0e684 --- /dev/null +++ b/tests/test_pifs_agent_stream.py @@ -0,0 +1,343 @@ +import ast +import io +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch +from types import SimpleNamespace + +from pydantic import BaseModel, ConfigDict + +from pageindex.filesystem import agent as agent_module +from pageindex.filesystem.agent import ( + AGENT_TOOL_POLICY, + AGENT_SYSTEM_PROMPT, + BASH_TOOL_DESCRIPTION, + PIFSAgentSession, + PIFSAgentStreamObserver, + build_agent_model_settings, + build_pifs_agent_instructions, + normalize_agent_stream_mode, + normalize_reasoning_effort, + normalize_reasoning_summary, + pifs_agent_raw_reasoning_enabled, + serialize_agent_final_output, + should_disable_pifs_agent_tracing, + should_use_openai_compatible_chat_model, +) +from pageindex.filesystem import PageIndexFileSystem + + +def load_demo_agent_prompt() -> str: + demo_path = Path(__file__).resolve().parents[1] / "examples" / "pifs_demo.py" + module = ast.parse(demo_path.read_text(encoding="utf-8")) + for node in module.body: + if isinstance(node, ast.Assign): + names = [ + target.id + for target in node.targets + if isinstance(target, ast.Name) + ] + if "PIFS_DEMO_AGENT_PROMPT" in names and isinstance(node.value, ast.Constant): + return str(node.value.value) + raise AssertionError("PIFS_DEMO_AGENT_PROMPT not found") + + +class StructuredAnswer(BaseModel): + model_config = ConfigDict(extra="forbid") + + answer: str + document_ids: list[str] + + +class PIFSAgentStreamTest(unittest.TestCase): + def raw_event(self, event_type, delta): + return SimpleNamespace( + type="raw_response_event", + data=SimpleNamespace(type=event_type, delta=delta), + ) + + def test_model_stream_prints_output_and_think_deltas(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("model", stream_log=stream_log, output=output) + + observer.handle_event(self.raw_event("response.reasoning_summary_text.delta", "look up folder")) + observer.handle_event(self.raw_event("response.output_text.delta", '{"answer":')) + observer.handle_event(self.raw_event("response.output_text.delta", '"done"}')) + observer.finish() + + printed = output.getvalue() + self.assertIn("[llm reasoning summary stream]", printed) + self.assertIn("look up folder", printed) + self.assertIn("[llm final output stream]", printed) + self.assertIn('{"answer":"done"}', printed.replace("\n", "")) + self.assertEqual( + stream_log, + [ + {"kind": "output", "text": '{"answer":"done"}'}, + {"kind": "think_summary", "text": "look up folder"}, + ], + ) + + def test_tools_mode_does_not_print_model_text(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) + + observer.handle_event(self.raw_event("response.output_text.delta", "hidden from tools mode")) + observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"ls /"}')) + observer.emit_tool_call("ls /") + observer.emit_tool_result(ok=True, output='{"ok": true}', seconds=0.001) + observer.finish() + + printed = output.getvalue() + self.assertNotIn("hidden from tools mode", printed) + self.assertIn("[llm -> pifs command]", printed) + self.assertIn("ls /", printed) + self.assertIn("[pifs -> llm result preview]", printed) + self.assertIn('{"ok": true}', printed) + self.assertEqual(stream_log[0], {"kind": "tool_call", "command": "ls /"}) + self.assertEqual(stream_log[1]["kind"], "tool_result") + self.assertEqual(stream_log[2], {"kind": "tool_args", "text": '{"command":"ls /"}'}) + + def test_empty_tool_command_is_not_printed_or_logged(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) + + observer.emit_tool_call("") + observer.emit_tool_call(" ") + + self.assertEqual(output.getvalue(), "") + self.assertEqual(stream_log, []) + + def test_tool_result_preview_compacts_large_outputs(self): + output = io.StringIO() + observer = PIFSAgentStreamObserver("tools", output=output) + + observer.emit_tool_result( + ok=True, + output="\n".join(f"line {index}" for index in range(50)), + seconds=0.001, + ) + + printed = output.getvalue() + self.assertIn("[large PIFS result", printed) + self.assertIn("line 0", printed) + self.assertIn("more lines omitted from preview", printed) + self.assertNotIn("line 49", printed) + + def test_raw_reasoning_is_not_logged_by_default_but_summary_is(self): + output = io.StringIO() + stream_log = [] + previous = os.environ.pop("PAGEINDEX_PIFS_AGENT_RAW_REASONING", None) + try: + observer = PIFSAgentStreamObserver("model", stream_log=stream_log, output=output) + observer.handle_event(self.raw_event("response.reasoning_text.delta", "private chain")) + observer.handle_event( + self.raw_event("response.reasoning_summary_text.delta", "visible summary") + ) + observer.finish() + finally: + if previous is not None: + os.environ["PAGEINDEX_PIFS_AGENT_RAW_REASONING"] = previous + + printed = output.getvalue() + self.assertNotIn("private chain", printed) + self.assertIn("visible summary", printed) + self.assertEqual(stream_log, [{"kind": "think_summary", "text": "visible summary"}]) + + def test_raw_reasoning_requires_debug_env_flag(self): + self.assertFalse(pifs_agent_raw_reasoning_enabled({})) + self.assertTrue( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "on"}) + ) + self.assertTrue( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "TRUE"}) + ) + self.assertFalse( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "0"}) + ) + + def test_stream_mode_aliases(self): + self.assertEqual(normalize_agent_stream_mode("think"), "model") + self.assertEqual(normalize_agent_stream_mode("debug"), "all") + self.assertEqual(normalize_agent_stream_mode(""), "off") + with self.assertRaises(ValueError): + normalize_agent_stream_mode("nope") + + def test_reasoning_settings_enable_effort_and_summary(self): + try: + settings = build_agent_model_settings( + reasoning_effort="medium", + reasoning_summary="detailed", + ) + except ModuleNotFoundError as exc: + if exc.name == "agents": + self.skipTest("agents package is not installed") + raise + + self.assertIsNotNone(settings) + self.assertEqual(settings.reasoning.effort, "medium") + self.assertEqual(settings.reasoning.summary, "detailed") + self.assertEqual(settings.verbosity, "low") + + def test_reasoning_effort_defaults_to_visible_summary(self): + try: + settings = build_agent_model_settings(reasoning_effort="low") + except ModuleNotFoundError as exc: + if exc.name == "agents": + self.skipTest("agents package is not installed") + raise + + self.assertIsNotNone(settings) + self.assertEqual(settings.reasoning.effort, "low") + self.assertEqual(settings.reasoning.summary, "auto") + + def test_reasoning_and_base_url_normalization(self): + self.assertEqual(normalize_reasoning_effort("xhigh"), "xhigh") + self.assertIsNone(normalize_reasoning_summary("none")) + self.assertFalse(should_use_openai_compatible_chat_model(None)) + self.assertFalse(should_use_openai_compatible_chat_model("https://api.openai.com/v1/")) + self.assertTrue(should_use_openai_compatible_chat_model("https://example.test/v1")) + with self.assertRaises(ValueError): + normalize_reasoning_effort("maximum") + + def test_tracing_is_disabled_by_default_unless_env_enables_it(self): + self.assertTrue(should_disable_pifs_agent_tracing({})) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "1"}) + ) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "true"}) + ) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "on"}) + ) + self.assertTrue( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "0"}) + ) + + def test_structured_agent_output_serializes_to_json(self): + output = serialize_agent_final_output( + StructuredAnswer(answer="done", document_ids=["dsid_1"]) + ) + + self.assertEqual(output, '{"answer":"done","document_ids":["dsid_1"]}') + + def test_prompt_tells_agent_to_use_structure_then_page(self): + self.assertIn( + "run cat --structure before the first cat --page", + AGENT_TOOL_POLICY, + ) + self.assertIn("Use cat --structure before any cat --page", BASH_TOOL_DESCRIPTION) + self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) + + def test_prompt_prefers_canonical_evidence_and_final_consistency(self): + self.assertIn("primary statement, reconciliation table, or official table", AGENT_TOOL_POLICY) + self.assertIn("Do a final consistency check", AGENT_TOOL_POLICY) + self.assertIn("headline number", AGENT_TOOL_POLICY) + self.assertIn("yes/no polarity", AGENT_TOOL_POLICY) + self.assertNotIn("FinanceBench", AGENT_TOOL_POLICY) + + def test_prompt_requires_stat_for_metadata_questions(self): + self.assertIn("Use stat only for identity, metadata, or status questions", AGENT_TOOL_POLICY) + self.assertIn("Do not run stat merely to understand what a document says", AGENT_TOOL_POLICY) + self.assertIn("Use stat only for identity, status, or\nmetadata", BASH_TOOL_DESCRIPTION) + + def test_prompt_routes_topic_retrieval_through_browse_after_folder_exploration(self): + self.assertIn("Start with tree", AGENT_TOOL_POLICY) + self.assertIn('browse ""', AGENT_TOOL_POLICY) + self.assertIn("Use browse -R", BASH_TOOL_DESCRIPTION) + self.assertIn("If browse misses", AGENT_TOOL_POLICY) + self.assertIn("browse -R from a broader folder", AGENT_TOOL_POLICY) + self.assertIn("browse candidates are not final evidence", AGENT_TOOL_POLICY) + self.assertIn("verify the relevant facts with cat or grep", AGENT_TOOL_POLICY) + self.assertIn("cat --structure", AGENT_TOOL_POLICY) + self.assertIn("cat --page", AGENT_TOOL_POLICY) + self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) + self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) + + def test_prompt_allows_recursive_browse_for_low_signal_tree_labels(self): + self.assertIn("Start with tree", AGENT_TOOL_POLICY) + self.assertIn('browse ""', AGENT_TOOL_POLICY) + self.assertIn("low-signal folder or virtual-node labels", AGENT_TOOL_POLICY) + self.assertIn("browse -R", AGENT_TOOL_POLICY) + self.assertIn("low-signal folder or virtual-node labels", BASH_TOOL_DESCRIPTION) + + def test_default_agent_prompts_do_not_suggest_legacy_semantic_commands(self): + prompt_surface = "\n".join( + [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY] + ) + + for old_command in ( + "search-summary", + "search-entity", + "search-relation", + "semantic-grep", + "find --name", + "find --relation", + ): + self.assertNotIn(old_command, prompt_surface) + + def test_demo_prompt_uses_browse_strategy_and_not_old_vector_commands(self): + demo_prompt = load_demo_agent_prompt() + + self.assertIn("Start with tree", demo_prompt) + self.assertIn('browse /documents "Federal Reserve supervision regulation"', demo_prompt) + self.assertIn('browse -R /documents "Federal Reserve supervision regulation"', demo_prompt) + self.assertIn("verify", demo_prompt) + self.assertIn("cat --structure", demo_prompt) + self.assertIn("Use grep only for one selected file", demo_prompt) + self.assertIn("Do not guess cat --page ranges from grep line numbers", demo_prompt) + self.assertNotIn("search-summary", demo_prompt) + + def test_prompt_rejects_find_grep_as_exhaustive_search(self): + self.assertIn("Do not use find, recursive grep, folder grep, pipes", AGENT_TOOL_POLICY) + self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) + + def test_system_prompt_sets_workspace_identity_and_scope(self): + self.assertIn("PageIndex FileSystem Demo Agent", AGENT_SYSTEM_PROMPT) + self.assertIn("VectifyAI Team", AGENT_SYSTEM_PROMPT) + self.assertIn("current PageIndex FileSystem\nworkspace", AGENT_SYSTEM_PROMPT) + self.assertIn("unrelated to the current workspace", AGENT_SYSTEM_PROMPT) + self.assertIn("do not answer it as\na general-purpose assistant", AGENT_SYSTEM_PROMPT) + self.assertIn("workspace-related topic question", AGENT_SYSTEM_PROMPT) + self.assertIn("clarify only after the persistence protocol", AGENT_SYSTEM_PROMPT) + self.assertIn("browse for candidate documents before asking", AGENT_TOOL_POLICY) + self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) + + def test_threaded_runtime_error_is_not_retried_on_fresh_loop(self): + session = object.__new__(PIFSAgentSession) + session.executor = SimpleNamespace() + session.normalized_stream_mode = "off" + session.agent_log = [] + session.max_seconds = None + session.max_turns = 1 + session.session = None + session.agent = object() + + main_thread = threading.get_ident() + run_threads = [] + + def fail_asyncio_run(coro): + coro.close() + run_threads.append(threading.get_ident()) + raise RuntimeError("threaded agent failure") + + with ( + patch.object(agent_module.asyncio, "get_running_loop", return_value=object()), + patch.object(agent_module.asyncio, "run", side_effect=fail_asyncio_run), + ): + with self.assertRaisesRegex(RuntimeError, "threaded agent failure"): + session.run("Question: inspect workspace") + + self.assertEqual(len(run_threads), 1) + self.assertNotEqual(run_threads[0], main_thread) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py new file mode 100644 index 000000000..d6f5b0a5d --- /dev/null +++ b/tests/test_pifs_cli.py @@ -0,0 +1,617 @@ +import builtins +import json +import os +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_pifs_config(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg-config")) + + +def pifs_config_path(tmp_path): + return tmp_path / "xdg-config" / "pageindex" / "pifs.json" + + +def test_pifs_config_file_overrides_default_location(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + default_path = pifs_config_path(tmp_path) + override_path = tmp_path / "custom-pifs.json" + default_path.parent.mkdir(parents=True, exist_ok=True) + default_path.write_text( + json.dumps({"workspace": str(tmp_path / "default-workspace")}), + encoding="utf-8", + ) + override_path.write_text( + json.dumps({"workspace": str(tmp_path / "override-workspace")}), + encoding="utf-8", + ) + + monkeypatch.setenv("PIFS_CONFIG_FILE", str(override_path)) + + assert cli._configured_workspace() == str(tmp_path / "override-workspace") + + +class FakeFileSystem: + def __init__(self, workspace): + self.workspace = Path(workspace) + self.projection_retrieval_configured = False + + def configure_existing_projection_retrieval(self): + self.projection_retrieval_configured = True + return True + + +def test_cli_workspace_does_not_eagerly_configure_projection_retrieval(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + assert filesystem.projection_retrieval_configured is False + + +def test_cli_workspace_without_projection_index_does_not_require_sqlite_vec( + monkeypatch, tmp_path +): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + real_import = builtins.__import__ + + monkeypatch.delitem(sys.modules, "pageindex.filesystem.semantic_projection", raising=False) + monkeypatch.delitem(sys.modules, "pageindex.filesystem.semantic_index", raising=False) + monkeypatch.delitem(sys.modules, "sqlite_vec", raising=False) + + def block_sqlite_vec(name, globals=None, locals=None, fromlist=(), level=0): + if name.split(".", 1)[0] == "sqlite_vec": + raise ModuleNotFoundError("No module named 'sqlite_vec'", name="sqlite_vec") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", block_sqlite_vec) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + assert filesystem.semantic_retrieval_channels() == () + + +def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text( + json.dumps( + { + "workspace": str(workspace), + "embedding_provider": "openai", + "embedding_model": "gemini-embedding-2-preview", + "embedding_dimensions": 3072, + "embedding_timeout": 12.5, + "embedding_base_url": "https://example.invalid/openai/", + "embedding_api_key": "test-gemini-key", + } + ), + encoding="utf-8", + ) + + class ConfiguredFileSystem: + def __init__(self, workspace, **kwargs): + self.workspace = Path(workspace) + self.kwargs = kwargs + + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") + monkeypatch.setattr(cli, "PageIndexFileSystem", ConfiguredFileSystem) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + assert filesystem.kwargs == { + "summary_projection_embedding_provider": "openai", + "summary_projection_embedding_model": "gemini-embedding-2-preview", + "summary_projection_embedding_dimensions": 3072, + "summary_projection_embedding_timeout": 12.5, + "summary_projection_embedding_base_url": "https://example.invalid/openai/", + "summary_projection_embedding_api_key": "test-gemini-key", + } + assert os.environ["PIFS_EMBEDDING_API_KEY"] == "ignored-env-key" + assert os.environ["PIFS_EMBEDDING_BASE_URL"] == "https://ignored.invalid/" + + +def test_browse_surfaces_projection_dimension_mismatch_lazily(tmp_path): + from pageindex.filesystem import cli + from pageindex.filesystem.commands import PIFSCommandExecutor + from pageindex.filesystem.semantic_index import SemanticIndexRecord, SQLiteVecSemanticIndex + + workspace = tmp_path / "workspace" + index_dir = workspace / "artifacts" / "projection_indexes" + summary_index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") + summary_index.reset( + dimension=3, + metadata={ + "channel": "summary", + "embedding_provider": "test", + "embedding_model": "fake", + "embedding_dimensions": 3, + }, + ) + summary_index.upsert_many( + [ + SemanticIndexRecord( + file_ref="file_a", + external_id="doc_a", + source_type="documents", + title="A", + text="summary", + vector=[1.0, 0.0, 0.0], + ) + ] + ) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.semantic_retrieval_channels() == () + result = json.loads(PIFSCommandExecutor(filesystem).execute('browse / "summary"')) + + assert result["success"] is False + assert result["error"]["code"] == "invalid_command" + assert "summary projection index dimension mismatch" in result["error"]["message"] + assert "dimension 3" in result["error"]["message"] + assert "summary_projection_embedding_dimensions is 1024" in result["error"]["message"] + + +def test_cli_passthrough_invokes_pifs_command_executor(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + executor_instances = [] + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + self.commands = [] + executor_instances.append(self) + + def execute(self, command): + self.commands.append(command) + return f"executed:{command}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + status = cli.main(["--workspace", str(workspace), "ls", "/documents"]) + + assert status == 0 + assert capsys.readouterr().out == "executed:ls /documents\n" + assert len(executor_instances) == 1 + assert executor_instances[0].filesystem.workspace == workspace + assert executor_instances[0].commands == ["ls /documents"] + + +def test_cli_passthrough_returns_nonzero_for_failed_json_envelope(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + + def execute(self, command): + return json.dumps( + {"success": False, "error": {"message": "bad"}, "next_steps": []} + ) + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + status = cli.main(["--workspace", str(workspace), "find", "/documents"]) + + assert status == 2 + assert json.loads(capsys.readouterr().out)["success"] is False + + +def test_cli_set_workspace_persists_default(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + + status = cli.main(["set", "workspace", str(workspace)]) + + assert status == 0 + output = capsys.readouterr().out + assert f"workspace: {workspace}" in output + assert f"config: {config_path}" in output + assert config_path.read_text(encoding="utf-8") == ( + '{\n "workspace": "' + str(workspace) + '"\n}\n' + ) + + +def test_cli_setmeta_replaces_document_metadata(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + calls = [] + + class FakeSetMetaFileSystem(FakeFileSystem): + def set_metadata(self, target, metadata, *, clear=False): + calls.append((self.workspace, target, metadata, clear)) + return { + "path": "/documents/report.md", + "file_ref": "file_report", + "metadata": metadata, + } + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) + + status = cli.main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/report.md", + '{"ticker":"AAPL"}', + ] + ) + + assert status == 0 + assert calls == [ + (workspace, "/documents/report.md", {"ticker": "AAPL"}, False) + ] + assert json.loads(capsys.readouterr().out) == { + "file_ref": "file_report", + "metadata": {"ticker": "AAPL"}, + "path": "/documents/report.md", + } + + +def test_cli_setmeta_clear_uses_empty_object(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + calls = [] + + class FakeSetMetaFileSystem(FakeFileSystem): + def set_metadata(self, target, metadata, *, clear=False): + calls.append((self.workspace, target, metadata, clear)) + return {"file_ref": "file_report", "metadata": metadata} + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) + + status = cli.main( + ["--workspace", str(workspace), "setmeta", "--clear", "/documents/report.md"] + ) + + assert status == 0 + assert calls == [(workspace, "/documents/report.md", {}, True)] + assert json.loads(capsys.readouterr().out) == { + "file_ref": "file_report", + "metadata": {}, + } + + +def test_cli_passthrough_uses_configured_workspace(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + executor_instances = [] + monkeypatch.delenv("PIFS_WORKSPACE", raising=False) + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + self.commands = [] + executor_instances.append(self) + + def execute(self, command): + self.commands.append(command) + return f"executed:{command}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + assert cli.main(["set", "workspace", str(workspace)]) == 0 + capsys.readouterr() + + status = cli.main(["ls", "/documents"]) + + assert status == 0 + assert capsys.readouterr().out == "executed:ls /documents\n" + assert executor_instances[0].filesystem.workspace == workspace + + +def test_cli_ask_invokes_agent_with_question(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + agent_calls = [] + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_calls.append((filesystem, question, kwargs)) + return "agent answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--model", + "test-model", + "--stream-mode", + "off", + "--max-turns", + "7", + "--max-seconds", + "3.5", + "--reasoning-effort", + "low", + "--reasoning-summary", + "concise", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + filesystem, question, kwargs = agent_calls[0] + assert filesystem.workspace == workspace + assert question == "What is inside?" + assert kwargs == { + "model": "test-model", + "stream_mode": "off", + "max_turns": 7, + "max_seconds": 3.5, + "reasoning_effort": "low", + "reasoning_summary": "concise", + } + + +def test_cli_ask_defaults_to_global_agent_model(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + agent_calls = [] + monkeypatch.delenv("PIFS_AGENT_MODEL", raising=False) + monkeypatch.delenv("PIFS_MODEL", raising=False) + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_calls.append(kwargs) + return "agent answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main(["ask", "--workspace", str(workspace), "What?"]) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + assert agent_calls[0]["model"] == "gpt-5.4" + + +def test_cli_ask_loads_env_file_before_running_agent(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + env_file = tmp_path / ".env" + env_file.write_text("OPENAI_API_KEY=from-dotenv\n", encoding="utf-8") + agent_keys = [] + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_keys.append(os.environ.get("OPENAI_API_KEY")) + return "agent answer" + + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--env-file", + str(env_file), + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + assert agent_keys == ["from-dotenv"] + + +def test_cli_chat_runs_one_question_and_exits(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["", "Summarize the workspace", "exit"]) + session_instances = [] + session_questions = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + self.filesystem = filesystem + self.kwargs = kwargs + session_instances.append(self) + + def run(self, question): + session_questions.append((self, question)) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--model", "test-model"]) + + assert status == 0 + assert capsys.readouterr().out == "" + assert len(session_instances) == 1 + assert session_instances[0].filesystem.workspace == workspace + assert session_questions == [(session_instances[0], "Summarize the workspace")] + assert session_instances[0].kwargs["model"] == "test-model" + assert session_instances[0].kwargs["stream_mode"] == "all" + + +def test_cli_chat_sanitizes_control_input(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["\x12", "he\x7fllo\x1b[A", "exit"]) + agent_calls = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + pass + + def run(self, question): + agent_calls.append(question) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--stream-mode", "off"]) + + assert status == 0 + assert agent_calls == ["hllo"] + assert capsys.readouterr().out == "answer:hllo\n" + + +def test_cli_ask_does_not_reprint_streamed_agent_output(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + def fake_run_pifs_agent(filesystem, question, **kwargs): + print("streamed answer") + return "returned answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--stream-mode", + "all", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "streamed answer\n" + + +def test_cli_ask_prints_final_answer_with_tool_stream(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + def fake_run_pifs_agent(filesystem, question, **kwargs): + print("[tool stream]") + return "FOUND 2 documents" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--stream-mode", + "tools", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "[tool stream]\nFOUND 2 documents\n" + + +def test_cli_chat_stream_mode_can_be_overridden(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["Summarize the workspace", "exit"]) + session_kwargs = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + session_kwargs.append(kwargs) + + def run(self, question): + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main( + [ + "chat", + "--workspace", + str(workspace), + "--stream-mode", + "tools", + ] + ) + + assert status == 0 + assert session_kwargs[0]["stream_mode"] == "tools" + + +def test_cli_chat_reuses_one_agent_session_for_multiple_questions(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["first", "second", "exit"]) + sessions = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + self.questions = [] + sessions.append(self) + + def run(self, question): + self.questions.append(question) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--stream-mode", "off"]) + + assert status == 0 + assert len(sessions) == 1 + assert sessions[0].questions == ["first", "second"] + assert capsys.readouterr().out == "answer:first\nanswer:second\n" diff --git a/tests/test_pifs_core_alignment.py b/tests/test_pifs_core_alignment.py new file mode 100644 index 000000000..6c22fa92c --- /dev/null +++ b/tests/test_pifs_core_alignment.py @@ -0,0 +1,356 @@ +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from tests.pifs_markdown_fixture import register_markdown + + +class BrowseBackend: + def __init__(self, document_ids, channels=("summary",), file_refs_by_document_id=None): + self.document_ids = list(document_ids) + self.channels = channels + self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) + self.calls = [] + + def available_channels(self): + return self.channels + + def search_channel(self, channel, query, *, limit=10, filters=None): + self.calls.append((channel, query, limit, filters)) + file_ref_filter = set() + if isinstance(filters, dict): + raw_file_refs = filters.get("file_ref") or [] + if isinstance(raw_file_refs, str): + file_ref_filter = {raw_file_refs} + else: + file_ref_filter = {str(item) for item in raw_file_refs} + document_ids = self.document_ids + if file_ref_filter and self.file_refs_by_document_id: + document_ids = [ + document_id + for document_id in document_ids + if self.file_refs_by_document_id.get(document_id) in file_ref_filter + ] + return [ + SimpleNamespace( + document_id=document_id, + snippet=f"{channel} candidate {rank}: {query}", + score=1.0 - rank * 0.01, + sources=[{"channel": channel, "rank": rank}], + ) + for rank, document_id in enumerate(document_ids[:limit], 1) + ] + + +def _payload(output): + return json.loads(output) + + +def _register_file(filesystem, root, external_id, folder_path, *, title=None, text=None): + return register_markdown( + filesystem, + root, + external_id, + folder_path, + title=title or f"{external_id}.md", + text=text, + metadata={"department": "finance"}, + ) + + +class PIFSCoreAlignmentTest(unittest.TestCase): + def test_aligned_command_surface_and_json_errors(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + _register_file(filesystem, root, "doc_a", "/documents") + executor = PIFSCommandExecutor(filesystem) + + self.assertEqual( + executor.allowed_commands(), + {"browse", "cat", "grep", "ls", "stat", "tree"}, + ) + + for command in ( + "find /documents", + "grep -R alpha /documents", + "grep alpha /documents", + "stat --schema", + "stat --field department doc_a", + "stat doc_a doc_a", + "cat doc_a", + "cat doc_a --all", + "tree /documents | grep doc", + 'browse /documents "alpha" --space entity', + 'browse /documents "alpha" --json', + ): + result = _payload(executor.execute(command)) + self.assertFalse(result["success"]) + self.assertEqual(result["error"]["code"], "invalid_command") + self.assertEqual(result["next_steps"], []) + + def test_ls_is_tree_depth_one_alias(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + _register_file(filesystem, root, "doc_root", "/documents") + _register_file(filesystem, root, "doc_child", "/documents/team") + executor = PIFSCommandExecutor(filesystem) + calls = [] + original = executor._cmd_tree + + def record_tree(args): + calls.append(args) + return original(args) + + executor._cmd_tree = record_tree + + ls_result = _payload(executor.execute("ls /documents")) + tree_result = _payload(executor.execute("tree /documents -L 1")) + + self.assertEqual(calls[0], ["/documents", "-L", "1"]) + self.assertEqual(ls_result, tree_result) + self.assertTrue(ls_result["success"]) + self.assertEqual( + list(ls_result["data"]), + ["tree", "total_folders", "depth", "truncated"], + ) + self.assertEqual(ls_result["data"]["depth"], 1) + + def test_tree_defaults_to_chat_aligned_depth_ten_without_clamping_explicit_depth(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + _register_file(filesystem, root, "doc_root", "/documents") + executor = PIFSCommandExecutor(filesystem) + + default_tree = _payload(executor.execute("tree /documents")) + explicit_tree = _payload(executor.execute("tree /documents -L 99")) + + self.assertTrue(default_tree["success"]) + self.assertEqual(default_tree["data"]["depth"], 10) + self.assertTrue(explicit_tree["success"]) + self.assertEqual(explicit_tree["data"]["depth"], 99) + + def test_browse_summary_scoped_and_paginated(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + refs = { + "doc_direct": _register_file(filesystem, root, "doc_direct", "/documents"), + "doc_deep": _register_file(filesystem, root, "doc_deep", "/documents/reports"), + "doc_other": _register_file(filesystem, root, "doc_other", "/other"), + } + backend = BrowseBackend( + ["doc_other", "doc_deep", "doc_direct"], + file_refs_by_document_id=refs, + ) + filesystem.semantic_retrieval_backend = backend + executor = PIFSCommandExecutor(filesystem) + + missing_query = _payload(executor.execute("browse /documents")) + self.assertFalse(missing_query["success"]) + self.assertIn("requires a query", missing_query["error"]["message"]) + + removed_space = _payload( + executor.execute('browse /documents "alpha" --space entity') + ) + self.assertFalse(removed_space["success"]) + self.assertIn("--space is removed", removed_space["error"]["message"]) + + direct = _payload(executor.execute('browse /documents "alpha"')) + self.assertTrue(direct["success"]) + self.assertEqual( + set(direct["data"]["documents"][0]), + {"path", "title", "summary", "metadata"}, + ) + self.assertEqual( + [doc["path"] for doc in direct["data"]["documents"]], + ["/documents/doc_direct.md"], + ) + self.assertEqual( + direct["data"]["scope"], + { + "folder": "/documents", + "recursive": False, + "query": "alpha", + "where": None, + "retrieval": "summary", + }, + ) + self.assertEqual(backend.calls[-1][0], "summary") + + recursive = _payload(executor.execute('browse -R /documents "alpha"')) + self.assertEqual( + [doc["path"] for doc in recursive["data"]["documents"]], + ["/documents/reports/doc_deep.md", "/documents/doc_direct.md"], + ) + self.assertTrue(recursive["data"]["scope"]["recursive"]) + + def test_stat_and_grep_are_single_document_only(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + _register_file( + filesystem, + root, + "doc_a", + "/documents", + title="report.md", + text="alpha evidence\nbeta evidence", + ) + executor = PIFSCommandExecutor(filesystem) + + stat = _payload(executor.execute("stat /documents/report.md")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["document_id"], "doc_a") + self.assertTrue(stat["data"]["document"]["file_ref"].startswith("file_")) + + grep = _payload(executor.execute("grep alpha /documents/report.md")) + self.assertTrue(grep["success"]) + self.assertEqual(grep["data"]["document"]["document_id"], "doc_a") + self.assertEqual( + grep["data"]["matches"], + [{"line": 1, "text": "alpha evidence"}], + ) + + folder_grep = _payload(executor.execute("grep alpha /documents")) + self.assertFalse(folder_grep["success"]) + self.assertIn("not a folder", folder_grep["error"]["message"]) + + def test_cat_structure_and_page_use_aligned_json_shapes(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + class FakePageClient: + documents = { + "pi_doc": { + "doc_description": "Summary for report.md", + "pages": [{"page": 1, "content": "cached text"}], + } + } + + def index(self, file_path, mode="auto"): + return "pi_doc" + + def _ensure_doc_loaded(self, doc_id): + return None + + def get_document_structure(self, doc_id): + assert doc_id == "pi_doc" + return json.dumps( + [{"title": "Risk", "node_id": "0001", "text": "hidden", "nodes": []}] + ) + + def get_page_content(self, doc_id, pages): + assert doc_id == "pi_doc" + assert pages == "1-2" + return json.dumps( + [ + {"page": 1, "content": "page one evidence"}, + {"page": 2, "content": "page two evidence"}, + ] + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "report.md" + source.write_text("cached text", encoding="utf-8") + filesystem = PageIndexFileSystem(workspace=root / "workspace") + filesystem._pageindex_client = lambda: FakePageClient() + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + external_id="doc_pdf", + title="report.md", + content_type="text/markdown", + content="cached text", + ) + filesystem.store.update_pageindex_pointer( + file_ref, + pageindex_doc_id="pi_doc", + pageindex_tree_status="built", + ) + executor = PIFSCommandExecutor(filesystem) + + structure = _payload(executor.execute("cat doc_pdf --structure")) + self.assertTrue(structure["success"]) + self.assertEqual(structure["data"]["document"]["document_id"], "doc_pdf") + self.assertEqual( + structure["data"]["structure"], + [{"title": "Risk", "node_id": "0001", "nodes": []}], + ) + self.assertEqual(structure["data"]["pagination"], {"available": True}) + + page = _payload(executor.execute("cat doc_pdf --page 1-2")) + self.assertTrue(page["success"]) + self.assertEqual(page["data"]["requested_pages"], "1-2") + self.assertEqual( + [item["page"] for item in page["data"]["returned_pages"]], + [1, 2], + ) + self.assertEqual( + page["data"]["content"], + { + "text": "page one evidence\n\npage two evidence", + "available": True, + }, + ) + + def test_agent_policy_mentions_only_aligned_surface(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + from pageindex.filesystem.agent import build_pifs_agent_instructions + + with tempfile.TemporaryDirectory() as tmp: + filesystem = PageIndexFileSystem(workspace=Path(tmp) / "workspace") + executor = PIFSCommandExecutor(filesystem) + + instructions = build_pifs_agent_instructions(filesystem, executor=executor) + + for expected in ("tree", "browse", "stat", "cat", "grep"): + self.assertIn(expected, instructions) + for removed in ( + "find --", + "grep -R", + "stat --schema", + "stat --field", + "cat --all", + "cat --range", + ): + self.assertNotIn(removed, instructions) + self.assertIn( + "run cat --structure before the first cat --page", + instructions, + ) + self.assertIn("metadata-derived virtual axis", instructions) + self.assertIn("metadata-derived virtual value", instructions) + + def test_projection_surface_is_summary(self): + from pageindex.filesystem.core import ( + SEMANTIC_PROJECTION_INDEX_NAMES, + SEMANTIC_RETRIEVAL_CHANNELS, + ) + semantic_projection_source = Path("pageindex/filesystem/semantic_projection.py").read_text( + encoding="utf-8" + ) + + self.assertEqual(SEMANTIC_RETRIEVAL_CHANNELS, ("summary",)) + self.assertEqual(SEMANTIC_PROJECTION_INDEX_NAMES, {"summary": "summary"}) + self.assertIn('SUMMARY_INDEX_NAME = "summary"', semantic_projection_source) + self.assertNotIn("entity_vectors", semantic_projection_source) + self.assertNotIn("relation_vectors", semantic_projection_source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pifs_like_escape.py b/tests/test_pifs_like_escape.py new file mode 100644 index 000000000..304bad646 --- /dev/null +++ b/tests/test_pifs_like_escape.py @@ -0,0 +1,112 @@ +from pathlib import Path + +from tests.pifs_markdown_fixture import register_markdown + + +def _register_file( + filesystem, + tmp_path: Path, + filename: str, + *, + folder_path: str, + external_id: str, + metadata: dict[str, str] | None = None, +) -> None: + register_markdown( + filesystem, + tmp_path, + external_id, + folder_path, + title=f"{external_id}.md", + text=f"{external_id} fixture text", + metadata=metadata, + ) + + +def test_descendant_folder_filter_treats_underscore_literally(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + _register_file( + filesystem, + tmp_path, + "literal.txt", + folder_path="/proj_1/docs", + external_id="literal_underscore", + ) + _register_file( + filesystem, + tmp_path, + "wildcard.txt", + folder_path="/projA1/docs", + external_id="wildcard_neighbor", + ) + + recursive = filesystem.browse("/proj_1", recursive=True, limit=10) + folder_id = filesystem.folder_info("/proj_1")["folder_id"] + scoped_results = filesystem.search( + scope={"folder_id": folder_id, "recursive": True}, + limit=10, + ) + ranked_folders = { + folder["path"]: folder + for folder in filesystem.find_folders("/", max_depth=1, limit=10) + } + + assert {folder["path"] for folder in recursive["folders"]} == {"/proj_1/docs"} + assert {file["external_id"] for file in recursive["files"]} == {"literal_underscore"} + assert {result.external_id for result in scoped_results} == {"literal_underscore"} + assert ranked_folders["/proj_1"]["matched_files"] == 1 + assert ranked_folders["/projA1"]["matched_files"] == 1 + assert filesystem.store.count_files_in_folder("/proj_1", recursive=True) == 1 + + +def test_metadata_contains_treats_percent_and_underscore_literally(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + filesystem.metadata.register_schema({"fields": {"status": {}}}) + _register_file( + filesystem, + tmp_path, + "percent.txt", + folder_path="/documents", + external_id="literal_percent", + metadata={"status": "100% done"}, + ) + _register_file( + filesystem, + tmp_path, + "percent-neighbor.txt", + folder_path="/documents", + external_id="percent_neighbor", + metadata={"status": "100X done"}, + ) + _register_file( + filesystem, + tmp_path, + "underscore.txt", + folder_path="/documents", + external_id="literal_underscore", + metadata={"status": "build_alpha"}, + ) + _register_file( + filesystem, + tmp_path, + "underscore-neighbor.txt", + folder_path="/documents", + external_id="underscore_neighbor", + metadata={"status": "buildXalpha"}, + ) + + percent_results = filesystem.search( + metadata_filter={"status": {"$contains": "100% done"}}, + limit=10, + ) + underscore_results = filesystem.search( + metadata_filter={"status": {"$contains": "build_alpha"}}, + limit=10, + ) + + assert {result.external_id for result in percent_results} == {"literal_percent"} + assert {result.external_id for result in underscore_results} == {"literal_underscore"} diff --git a/tests/test_pifs_path_resolution.py b/tests/test_pifs_path_resolution.py new file mode 100644 index 000000000..04a70d782 --- /dev/null +++ b/tests/test_pifs_path_resolution.py @@ -0,0 +1,39 @@ +import pytest + +from tests.pifs_markdown_fixture import register_markdown + + +def test_root_virtual_file_path_resolves_without_double_slash(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + file_ref = register_markdown( + filesystem, tmp_path, "doc_root_title", "/", title="Root Title.md", text="root content" + ) + + assert filesystem.store.resolve_file_ref("/Root Title.md") == file_ref + + +def test_nested_virtual_file_path_resolves_by_folder_and_title(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + first_ref = register_markdown(filesystem, tmp_path, "doc_first", "/a", title="First.md", text="first content") + second_ref = register_markdown(filesystem, tmp_path, "doc_second", "/a/b", title="file.md", text="second content") + + assert filesystem.store.resolve_file_ref("/a/b/file.md") == second_ref + + assert first_ref != second_ref + + +def test_unknown_virtual_file_target_raises_clear_error(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + first_ref = register_markdown(filesystem, tmp_path, "doc_first", "/first", title="First.md", text="first content") + second_ref = register_markdown(filesystem, tmp_path, "doc_second", "/second", title="Second.md", text="second content") + + with pytest.raises(KeyError, match="Unknown file target"): + filesystem.store.resolve_file_ref("/shared/missing.txt") + + assert first_ref != second_ref diff --git a/tests/test_pifs_register_side_effects.py b/tests/test_pifs_register_side_effects.py new file mode 100644 index 000000000..6d2b8a65b --- /dev/null +++ b/tests/test_pifs_register_side_effects.py @@ -0,0 +1,77 @@ +from pathlib import Path + +import pytest + +from tests.pifs_markdown_fixture import register_markdown + + +class RecordingSummaryIndexer: + def __init__(self): + self.upserted = [] + self.deleted = [] + + def upsert_summary(self, record): + self.upserted.append(dict(record)) + return {"status": "ready"} + + def delete_summary(self, file_ref): + self.deleted.append(file_ref) + + +def test_register_insert_failure_cleans_owned_artifacts_and_skips_projection( + tmp_path: Path, monkeypatch +): + from pageindex.filesystem import PageIndexFileSystem + + workspace = tmp_path / "workspace" + indexer = RecordingSummaryIndexer() + filesystem = PageIndexFileSystem(workspace=workspace) + filesystem.summary_projection_indexer = indexer + + def fail_insert(records): + raise RuntimeError("catalog insert failed") + + monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) + + with pytest.raises(RuntimeError, match="catalog insert failed"): + register_markdown( + filesystem, + tmp_path, + "doc_insert_failure", + "/documents", + title="insert_failure.md", + text="Markdown content for registration.", + ) + + assert indexer.upserted == [] + assert list((workspace / "artifacts" / "raw").glob("*.json")) == [] + assert list((workspace / "artifacts" / "text").glob("*.txt")) == [] + + +def test_register_failure_after_catalog_insert_cleans_catalog_and_projection( + tmp_path: Path, monkeypatch +): + from pageindex.filesystem import PageIndexFileSystem + + workspace = tmp_path / "workspace" + indexer = RecordingSummaryIndexer() + filesystem = PageIndexFileSystem(workspace=workspace) + filesystem.summary_projection_indexer = indexer + + def fail_sync(record): + raise RuntimeError("raw sync failed") + + monkeypatch.setattr(filesystem, "_sync_owned_raw_artifact", fail_sync) + + with pytest.raises(RuntimeError, match="raw sync failed"): + register_markdown( + filesystem, + tmp_path, + "doc_sync_failure", + "/documents", + title="sync_failure.md", + text="Markdown content for registration.", + ) + + assert filesystem.search(None) == [] + assert indexer.deleted == [indexer.upserted[0]["file_ref"]] diff --git a/tests/test_pifs_scope_paths.py b/tests/test_pifs_scope_paths.py new file mode 100644 index 000000000..0e021908a --- /dev/null +++ b/tests/test_pifs_scope_paths.py @@ -0,0 +1,711 @@ +import json +import tempfile +import unittest +from types import SimpleNamespace + +from tests.pifs_markdown_fixture import register_markdown + + +class BrowseBackend: + def __init__(self, document_ids, *, file_refs_by_document_id=None): + self.document_ids = list(document_ids) + self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) + + def available_channels(self): + return ("summary",) + + def search_channel(self, channel, query, *, limit=10, filters=None): + file_ref_filter = set((filters or {}).get("file_ref") or []) + document_ids = self.document_ids + if file_ref_filter: + document_ids = [ + document_id + for document_id in document_ids + if self.file_refs_by_document_id.get(document_id) in file_ref_filter + ] + return [ + SimpleNamespace( + document_id=document_id, + snippet=f"{channel} candidate {rank}: {query}", + score=1.0 - rank * 0.01, + sources=[{"channel": channel, "rank": rank}], + ) + for rank, document_id in enumerate(document_ids[:limit], 1) + ] + + +def _payload(output): + return json.loads(output) + + +class PIFSScopePathTest(unittest.TestCase): + def test_tree_browse_and_stat_accept_metadata_scope_paths(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + refs = { + "doc_aapl_2024": register_markdown( + filesystem, + root, + "doc_aapl_2024", + "/documents/sec-filings", + title="aapl-2024.md", + metadata={"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, + ), + "doc_aapl_2023": register_markdown( + filesystem, + root, + "doc_aapl_2023", + "/documents/sec-filings", + title="aapl-2023.md", + metadata={"year": 2023, "ticker": "AAPL", "doc_type": "10-K"}, + ), + "doc_msft_2024": register_markdown( + filesystem, + root, + "doc_msft_2024", + "/documents/sec-filings", + title="msft-2024.md", + metadata={"year": 2024, "ticker": "MSFT", "doc_type": "10-K"}, + ), + } + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_msft_2024", "doc_aapl_2024", "doc_aapl_2023"], + file_refs_by_document_id=refs, + ) + executor = PIFSCommandExecutor(filesystem) + + tree_root = _payload(executor.execute("tree /documents -L 1")) + self.assertTrue(tree_root["success"]) + + root_folders = tree_root["data"]["tree"]["folders"] + self.assertTrue(any(item["path"] == "/documents/sec-filings" for item in root_folders)) + self.assertEqual( + [item["name"] for item in root_folders if item["type"] == "metadata_axis"], + ["@doc_type", "@ticker", "@year"], + ) + + tree_year = _payload(executor.execute("tree /documents/@year")) + self.assertTrue(tree_year["success"]) + self.assertEqual( + [item["path"] for item in tree_year["data"]["tree"]["folders"]], + ["/documents/@year/2024", "/documents/@year/2023"], + ) + + browse = _payload( + executor.execute('browse /documents/@year/2024/@ticker/AAPL "risk factors"') + ) + self.assertTrue(browse["success"]) + self.assertEqual( + [item["path"] for item in browse["data"]["documents"]], + ["/documents/@year/2024/@ticker/AAPL/aapl-2024.md"], + ) + self.assertEqual(browse["data"]["scope"]["path"], "/documents/@year/2024/@ticker/AAPL") + self.assertEqual(browse["data"]["scope"]["folder_path"], "/documents") + self.assertEqual( + browse["data"]["scope"]["metadata_filter"], + {"year": "2024", "ticker": "AAPL"}, + ) + + locator = browse["data"]["documents"][0]["path"] + stat = _payload(executor.execute(f"stat {locator}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], locator) + self.assertEqual(stat["data"]["document"]["title"], "aapl-2024.md") + + def test_metadata_scope_file_locators_round_trip_to_file_commands(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + refs = { + "doc_3m_2018": register_markdown( + filesystem, + root, + "doc_3m_2018", + "/docs/filings", + title="3M_2018_10K.md", + text="cash flow evidence\nnet sales evidence", + metadata={"company": "3M"}, + ), + "doc_other": register_markdown( + filesystem, + root, + "doc_other", + "/docs/filings", + title="Other_2018_10K.md", + text="other cash flow", + metadata={"company": "Other"}, + ), + } + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_3m_2018", "doc_other"], + file_refs_by_document_id=refs, + ) + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /docs/@company/3M")) + self.assertTrue(tree["success"]) + self.assertEqual( + [item["path"] for item in tree["data"]["tree"]["files"]], + ["/docs/@company/3M/3M_2018_10K.md"], + ) + + browse = _payload(executor.execute('browse /docs/@company/3M "cash flow"')) + self.assertTrue(browse["success"]) + locator = browse["data"]["documents"][0]["path"] + self.assertEqual(locator, "/docs/@company/3M/3M_2018_10K.md") + + stat = _payload(executor.execute(f"stat {locator}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], locator) + self.assertEqual(stat["data"]["document"]["title"], "3M_2018_10K.md") + + structure = _payload(executor.execute(f"cat {locator} --structure")) + self.assertTrue(structure["success"]) + self.assertEqual(structure["data"]["document"]["path"], locator) + self.assertTrue(structure["data"]["structure"]) + + grep = _payload(executor.execute(f"grep cash {locator}")) + self.assertTrue(grep["success"]) + self.assertEqual(grep["data"]["document"]["path"], locator) + self.assertEqual(grep["data"]["matches"][0]["line"], 1) + + scope_only = _payload(executor.execute("cat /docs/@company/3M --structure")) + self.assertFalse(scope_only["success"]) + self.assertIn("scope, not a file locator", scope_only["error"]["message"]) + self.assertIn("tree /docs/@company/3M", scope_only["error"]["message"]) + self.assertIn('browse /docs/@company/3M ""', scope_only["error"]["message"]) + + def test_tree_depth_does_not_bypass_file_pagination(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + for index in range(55): + register_markdown( + filesystem, + root, + f"doc_{index:02d}", + "/docs", + title=f"report-{index:02d}.md", + metadata={"company": "3M"}, + ) + executor = PIFSCommandExecutor(filesystem) + + first_page = _payload(executor.execute("tree /docs/@company/3M -L 100")) + self.assertTrue(first_page["success"]) + self.assertEqual(len(first_page["data"]["tree"]["files"]), 50) + self.assertEqual( + first_page["data"]["pagination"], + {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, + ) + + second_page = _payload(executor.execute("tree /docs/@company/3M -L 100 --page 2")) + self.assertTrue(second_page["success"]) + self.assertEqual(len(second_page["data"]["tree"]["files"]), 5) + self.assertEqual( + second_page["data"]["pagination"], + {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, + ) + + def test_tree_file_pagination_sorts_full_scope_before_slicing(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + for index in range(55): + file_ref = register_markdown( + filesystem, + root, + f"doc_{index:02d}", + "/source", + title=f"source-{54 - index:02d}.md", + metadata={"company": "3M"}, + ) + filesystem.attach_file_to_folder( + file_ref, + "/docs", + metadata={"display_name": f"report-{index:02d}.md"}, + ) + executor = PIFSCommandExecutor(filesystem) + + first_page = _payload(executor.execute("tree /docs/@company/3M --page 1")) + second_page = _payload(executor.execute("tree /docs/@company/3M --page 2")) + + self.assertTrue(first_page["success"]) + self.assertTrue(second_page["success"]) + first_names = [item["name"] for item in first_page["data"]["tree"]["files"]] + second_names = [item["name"] for item in second_page["data"]["tree"]["files"]] + + self.assertEqual(first_names, [f"report-{index:02d}.md" for index in range(50)]) + self.assertEqual(second_names, [f"report-{index:02d}.md" for index in range(50, 55)]) + self.assertEqual(len(set(first_names + second_names)), 55) + + def test_duplicate_file_leaves_in_scope_return_disambiguated_locators(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + refs = { + "doc_a": register_markdown( + filesystem, + root, + "doc_a", + "/docs/a", + title="report.md", + metadata={"company": "3M"}, + ), + "doc_b": register_markdown( + filesystem, + root, + "doc_b", + "/docs/b", + title="report.md", + metadata={"company": "3M"}, + ), + } + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_a", "doc_b"], + file_refs_by_document_id=refs, + ) + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /docs/@company/3M")) + self.assertTrue(tree["success"]) + paths = [item["path"] for item in tree["data"]["tree"]["files"]] + self.assertEqual(len(paths), 2) + self.assertEqual(len(set(paths)), 2) + + for path in paths: + stat = _payload(executor.execute(f"stat {path}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], path) + + browse = _payload(executor.execute('browse /docs/@company/3M "report"')) + self.assertTrue(browse["success"]) + browse_paths = [item["path"] for item in browse["data"]["documents"]] + self.assertEqual(len(browse_paths), 2) + self.assertEqual(len(set(browse_paths)), 2) + + def test_disambiguated_file_leaf_does_not_collide_with_real_leaf(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + first_ref = register_markdown( + filesystem, + root, + "doc_first", + "/docs/a", + title="x.md", + text="first collision evidence", + metadata={"company": "3M"}, + ) + register_markdown( + filesystem, + root, + "doc_second", + "/docs/b", + title="x.md", + text="second collision evidence", + metadata={"company": "3M"}, + ) + register_markdown( + filesystem, + root, + "doc_real_suffix", + "/docs/c", + title=f"x.md~{first_ref}", + text="real suffix collision evidence", + metadata={"company": "3M"}, + ) + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /docs/@company/3M")) + self.assertTrue(tree["success"]) + paths = [item["path"] for item in tree["data"]["tree"]["files"]] + self.assertEqual(len(paths), 3) + self.assertEqual(len(set(paths)), 3) + + for path in paths: + stat = _payload(executor.execute(f"stat {path}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], path) + + grep = _payload(executor.execute(f"grep evidence {path}")) + self.assertTrue(grep["success"]) + self.assertEqual(grep["data"]["document"]["path"], path) + + def test_metadata_scope_file_leaf_can_match_metadata_axis_name(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + register_markdown( + filesystem, + root, + "doc_axis_leaf", + "/docs", + title="@ticker", + text="axis leaf evidence", + metadata={"company": "3M", "ticker": "MMM"}, + ) + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /docs/@company/3M")) + self.assertTrue(tree["success"]) + locator = tree["data"]["tree"]["files"][0]["path"] + self.assertEqual(locator, "/docs/@company/3M/%40ticker") + + stat = _payload(executor.execute(f"stat {locator}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], locator) + + structure = _payload(executor.execute(f"cat {locator} --structure")) + self.assertTrue(structure["success"]) + self.assertEqual(structure["data"]["document"]["path"], locator) + + grep = _payload(executor.execute(f"grep evidence {locator}")) + self.assertTrue(grep["success"]) + self.assertEqual(grep["data"]["document"]["path"], locator) + self.assertEqual(grep["data"]["matches"][0]["line"], 1) + + def test_metadata_scope_file_leaf_with_slash_round_trips(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + file_ref = register_markdown( + filesystem, + root, + "doc_slash_leaf", + "/source", + title="source.md", + text="slash leaf evidence", + metadata={"company": "3M"}, + ) + filesystem.attach_file_to_folder( + file_ref, + "/docs", + metadata={"display_name": "2024/report.md"}, + ) + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /docs/@company/3M")) + self.assertTrue(tree["success"]) + locator = tree["data"]["tree"]["files"][0]["path"] + self.assertEqual(locator, "/docs/@company/3M/2024%2Freport.md") + + stat = _payload(executor.execute(f"stat {locator}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], locator) + + structure = _payload(executor.execute(f"cat {locator} --structure")) + self.assertTrue(structure["success"]) + self.assertEqual(structure["data"]["document"]["path"], locator) + + grep = _payload(executor.execute(f"grep evidence {locator}")) + self.assertTrue(grep["success"]) + self.assertEqual(grep["data"]["document"]["path"], locator) + self.assertEqual(grep["data"]["matches"][0]["line"], 1) + + def test_scope_paths_reject_duplicate_and_unknown_metadata_fields(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + register_markdown( + filesystem, + root, + "doc_aapl_2024", + "/documents", + metadata={"year": 2024, "ticker": "AAPL"}, + ) + executor = PIFSCommandExecutor(filesystem) + + duplicate = _payload(executor.execute("tree /documents/@ticker/AAPL/@ticker/MSFT")) + self.assertFalse(duplicate["success"]) + self.assertIn("can appear only once", duplicate["error"]["message"]) + + unknown = _payload(executor.execute("tree /documents/@sector")) + self.assertFalse(unknown["success"]) + self.assertIn("Unknown metadata axis", unknown["error"]["message"]) + + def test_tree_metadata_value_pagination_uses_page_flag(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + for index in range(55): + ticker = f"T{index:02d}" + register_markdown( + filesystem, + root, + f"doc_{ticker}", + "/documents", + title=f"{ticker}.md", + metadata={"ticker": ticker}, + ) + + executor = PIFSCommandExecutor(filesystem) + + first_page = _payload(executor.execute("tree /documents/@ticker")) + self.assertTrue(first_page["success"]) + self.assertEqual(len(first_page["data"]["tree"]["folders"]), 50) + self.assertEqual( + first_page["data"]["pagination"], + {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, + ) + self.assertEqual( + [item["value"] for item in first_page["data"]["tree"]["folders"][:3]], + ["T00", "T01", "T02"], + ) + + second_page = _payload(executor.execute("tree /documents/@ticker --page 2")) + self.assertTrue(second_page["success"]) + self.assertEqual( + [item["value"] for item in second_page["data"]["tree"]["folders"]], + ["T50", "T51", "T52", "T53", "T54"], + ) + self.assertEqual( + second_page["data"]["pagination"], + {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, + ) + + def test_tree_child_paths_keep_active_metadata_scope(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + register_markdown( + filesystem, + root, + "doc_sec_2024", + "/documents/sec", + title="sec-2024.md", + metadata={"year": 2024, "ticker": "AAPL"}, + ) + register_markdown( + filesystem, + root, + "doc_sec_2023", + "/documents/sec", + title="sec-2023.md", + metadata={"year": 2023, "ticker": "AAPL"}, + ) + register_markdown( + filesystem, + root, + "doc_other_2024", + "/documents/other", + title="other-2024.md", + metadata={"year": 2024, "ticker": "MSFT"}, + ) + + executor = PIFSCommandExecutor(filesystem) + + tree = _payload(executor.execute("tree /documents/@year/2024 -L 1")) + self.assertTrue(tree["success"]) + sec_node = next( + item + for item in tree["data"]["tree"]["folders"] + if item["name"] == "sec" + ) + self.assertEqual(sec_node["path"], "/documents/sec/@year/2024") + self.assertEqual(sec_node["file_count"], 1) + + scope_stat = _payload(executor.execute("stat /documents/sec/@year/2024")) + self.assertFalse(scope_stat["success"]) + self.assertIn("scope, not a file locator", scope_stat["error"]["message"]) + + scoped_tree = _payload(executor.execute("tree /documents/sec/@year/2024")) + self.assertTrue(scoped_tree["success"]) + locator = scoped_tree["data"]["tree"]["files"][0]["path"] + self.assertEqual(locator, "/documents/sec/@year/2024/sec-2024.md") + + stat = _payload(executor.execute(f"stat {locator}")) + self.assertTrue(stat["success"]) + self.assertEqual(stat["data"]["document"]["path"], locator) + + def test_core_browse_semantic_files_accepts_metadata_scoped_path(self): + from pageindex.filesystem import PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + refs = { + "doc_aapl": register_markdown( + filesystem, + root, + "doc_aapl", + "/documents/sec", + title="aapl.md", + metadata={"ticker": "AAPL"}, + ), + "doc_msft": register_markdown( + filesystem, + root, + "doc_msft_direct", + "/documents", + title="msft-direct.md", + metadata={"ticker": "MSFT"}, + ), + "doc_msft_descendant": register_markdown( + filesystem, + root, + "doc_msft_descendant", + "/documents/sec", + title="msft-descendant.md", + metadata={"ticker": "MSFT"}, + ), + } + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_msft_direct", "doc_msft_descendant", "doc_aapl"], + file_refs_by_document_id=refs, + ) + + payload = filesystem.browse_semantic_files( + "/documents/@ticker/AAPL", + "query", + recursive=False, + ) + + self.assertEqual( + [item["document_id"] for item in payload["data"]], + ["doc_aapl"], + ) + self.assertEqual(payload["scope"], "/documents/@ticker/AAPL") + + def test_core_browse_semantic_files_prefers_in_scope_physical_membership(self): + from pageindex.filesystem import PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + file_ref = register_markdown( + filesystem, + root, + "doc_shared", + "/adocs", + title="shared.md", + metadata={"ticker": "AAPL"}, + ) + filesystem.store.attach_file_to_folder(file_ref, "/zdocs") + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_shared"], + file_refs_by_document_id={"doc_shared": file_ref}, + ) + + payload = filesystem.browse_semantic_files( + "/zdocs/@ticker/AAPL", + "query", + recursive=False, + ) + + self.assertEqual([item["document_id"] for item in payload["data"]], ["doc_shared"]) + self.assertEqual(payload["data"][0]["folder_path"], "/zdocs") + self.assertTrue(payload["data"][0]["path"].startswith("/zdocs/")) + + def test_one_document_is_reachable_through_multiple_metadata_virtual_paths(self): + from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem + + with tempfile.TemporaryDirectory() as tmp: + from pathlib import Path + + root = Path(tmp) + filesystem = PageIndexFileSystem(workspace=root / "workspace") + file_ref = register_markdown( + filesystem, + root, + "doc_contract", + "/documents/contracts", + title="apple-renewal.md", + metadata={"vendor": "Apple", "year": 2024, "doc_type": "contract"}, + ) + filesystem.semantic_retrieval_backend = BrowseBackend( + ["doc_contract"], + file_refs_by_document_id={"doc_contract": file_ref}, + ) + executor = PIFSCommandExecutor(filesystem) + + tree_root = _payload(executor.execute("tree /documents -L 1")) + self.assertTrue(tree_root["success"]) + root_nodes = tree_root["data"]["tree"]["folders"] + self.assertIn( + "/documents/@vendor", + [item["path"] for item in root_nodes if item["type"] == "metadata_axis"], + ) + + tree_vendor = _payload(executor.execute("tree /documents/@vendor")) + self.assertTrue(tree_vendor["success"]) + self.assertEqual( + [ + item["type"] + for item in tree_vendor["data"]["tree"]["folders"] + if item["path"] == "/documents/@vendor/Apple" + ], + ["metadata_value"], + ) + + vendor = _payload( + executor.execute('browse /documents/@vendor/Apple "renewal contract"') + ) + year = _payload(executor.execute('browse /documents/@year/2024 "renewal contract"')) + + self.assertTrue(vendor["success"]) + self.assertTrue(year["success"]) + self.assertEqual( + [item["path"] for item in vendor["data"]["documents"]], + ["/documents/@vendor/Apple/apple-renewal.md"], + ) + self.assertEqual( + [item["path"] for item in year["data"]["documents"]], + ["/documents/@year/2024/apple-renewal.md"], + ) + self.assertEqual( + vendor["data"]["scope"]["metadata_filter"], + {"vendor": "Apple"}, + ) + self.assertEqual( + year["data"]["scope"]["metadata_filter"], + {"year": "2024"}, + ) diff --git a/tests/test_pifs_string_metadata_cleanup.py b/tests/test_pifs_string_metadata_cleanup.py new file mode 100644 index 000000000..e8df83f02 --- /dev/null +++ b/tests/test_pifs_string_metadata_cleanup.py @@ -0,0 +1,189 @@ +import json +from pathlib import Path + +import pytest + + +class StaticEmbedder: + def embed(self, texts): + return [[1.0, 0.0, 0.0] for _ in texts] + + +def _patch_summary_indexer(monkeypatch): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + def fake_from_provider(index_dir, **kwargs): + return SummaryProjectionIndexer( + Path(index_dir), + embedder=StaticEmbedder(), + embedding_provider="test", + embedding_model="static", + embedding_dimensions=3, + ) + + monkeypatch.setattr(SummaryProjectionIndexer, "from_provider", fake_from_provider) + + +def _patch_pageindex_client(monkeypatch, *, description="PageIndex summary"): + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + doc_id = Path(file_path).stem + "_doc" + doc = { + "id": doc_id, + "type": "md", + "path": str(Path(file_path).resolve()), + "doc_name": Path(file_path).name, + "doc_description": description, + "line_count": 2, + "structure": [{"title": "Doc", "node_id": "0001", "nodes": []}], + "pages": [{"page": 1, "content": Path(file_path).read_text(encoding="utf-8")}], + } + self.documents[doc_id] = doc + self._save_doc(doc_id) + return doc_id + + monkeypatch.setattr(PageIndexClient, "index", fake_index) + + +def _register_md(filesystem, tmp_path, monkeypatch, name, metadata=None): + _patch_summary_indexer(monkeypatch) + _patch_pageindex_client(monkeypatch, description=f"summary for {name}") + path = tmp_path / name + path.write_text(f"# {name}\n\nbody", encoding="utf-8") + return filesystem.register_file( + storage_uri=path.as_uri(), + folder_path="/documents", + title=name, + content_type="text/markdown", + metadata=metadata or {}, + ) + + +def test_metadata_schema_is_string_registry_only(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + filesystem.metadata.register_schema( + {"fields": {"ticker": {"description": "stock ticker"}}} + ) + + assert filesystem.metadata.export_schema() == { + "fields": {"ticker": {"description": "stock ticker", "source": "manual"}} + } + with filesystem.store.connect() as conn: + tables = { + row["name"] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + } + metadata_value_columns = { + row["name"] for row in conn.execute("PRAGMA table_info(metadata_values)") + } + + assert "metadata_schema" not in tables + assert metadata_value_columns == {"file_ref", "field_id", "value_text", "created_at"} + + +def test_range_filters_are_removed(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + from pageindex.filesystem.metadata import MetadataQueryError + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + filesystem.metadata.register_schema({"fields": {"score": {}}}) + + with pytest.raises(MetadataQueryError, match="Unsupported metadata operator: \\$gte"): + filesystem.metadata.parse_filter({"score": {"$gte": 3}}) + + +def test_numeric_metadata_filter_is_string_equality(tmp_path, monkeypatch): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + filesystem.metadata.register_schema({"fields": {"score": {}}}) + _register_md(filesystem, tmp_path, monkeypatch, "int.md", {"score": 3}) + _register_md(filesystem, tmp_path, monkeypatch, "float.md", {"score": 3.0}) + _register_md(filesystem, tmp_path, monkeypatch, "text.md", {"score": "3"}) + + rows = filesystem.search(None, metadata_filter={"score": {"$eq": 3}}) + + assert {row.title for row in rows} == {"int.md", "text.md"} + + +def test_register_uses_pageindex_description_as_summary_only(tmp_path, monkeypatch): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = _register_md(filesystem, tmp_path, monkeypatch, "summary.md", {"ticker": "AAPL"}) + info = filesystem.store.file_info(file_ref) + + assert info["metadata"]["summary"] == "summary for summary.md" + assert "summary" not in filesystem.metadata.export_schema()["fields"] + with filesystem.store.connect() as conn: + fields = [row["name"] for row in conn.execute("SELECT name FROM metadata_fields")] + assert fields == ["ticker"] + + +def test_register_rejects_plain_text_without_pageindex(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + source = tmp_path / "plain.txt" + source.write_text("plain text", encoding="utf-8") + filesystem = PageIndexFileSystem(tmp_path / "workspace") + + with pytest.raises(ValueError, match="PDF/Markdown"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title="plain.txt", + content_type="text/plain", + content="plain text", + ) + + +def test_set_metadata_replaces_custom_metadata_without_touching_summary(tmp_path, monkeypatch): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = _register_md(filesystem, tmp_path, monkeypatch, "meta.md", {"ticker": "AAPL"}) + + info = filesystem.set_metadata(file_ref, {"ticker": "MSFT", "year": 2026}) + + assert info["metadata"] == { + "summary": "summary for meta.md", + "ticker": "MSFT", + "year": 2026, + } + assert {field.name for field in filesystem.store.list_metadata_fields()} == { + "ticker", + "year", + } + assert filesystem.set_metadata(file_ref, {}, clear=True)["metadata"] == { + "summary": "summary for meta.md" + } + + +def test_set_metadata_keeps_summary_out_of_ordinary_lexical_search(tmp_path, monkeypatch): + from pageindex.filesystem import PageIndexFileSystem + + unique_summary_token = "summaryleaktoken" + filesystem = PageIndexFileSystem(tmp_path / "workspace") + _patch_summary_indexer(monkeypatch) + _patch_pageindex_client(monkeypatch, description=unique_summary_token) + path = tmp_path / "leak.md" + path.write_text("# Leak\n\nordinary body", encoding="utf-8") + file_ref = filesystem.register_file( + storage_uri=path.as_uri(), + folder_path="/documents", + title="leak.md", + content_type="text/markdown", + metadata={"ticker": "AAPL"}, + ) + + assert filesystem.search(unique_summary_token) == [] + + filesystem.set_metadata(file_ref, {"ticker": "MSFT"}) + + assert filesystem.search(unique_summary_token) == [] + assert {row.title for row in filesystem.search("MSFT")} == {"leak.md"} diff --git a/tests/test_semantic_index.py b/tests/test_semantic_index.py new file mode 100644 index 000000000..2b277dc08 --- /dev/null +++ b/tests/test_semantic_index.py @@ -0,0 +1,456 @@ +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from pageindex.filesystem.semantic_index import ( + SemanticIndexRecord, + SQLiteVecSemanticIndex, +) + + +class FixedDimensionEmbedder: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def embed(self, texts): + return [[1.0, *([0.0] * (self.dimensions - 1))] for _ in texts] + + +def test_sqlite_vec_semantic_index_round_trip(tmp_path): + index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") + index.reset(dimension=3, metadata={"field_mode": "summary"}) + + index.upsert_many( + [ + SemanticIndexRecord( + file_ref="file_a", + external_id="doc_a", + source_type="github", + title="Multipart upload limits", + text="multipart upload limits", + vector=[1.0, 0.0, 0.0], + metadata={"topic": "uploads"}, + ), + SemanticIndexRecord( + file_ref="file_b", + external_id="doc_b", + source_type="slack", + title="GPU cache issue", + text="gpu cache issue", + vector=[0.0, 1.0, 0.0], + metadata={"topic": "runtime"}, + ), + ] + ) + + assert index.info()["document_count"] == 2 + + results = index.search([0.9, 0.1, 0.0], limit=2) + assert [item.external_id for item in results] == ["doc_a", "doc_b"] + + filtered = index.search( + [0.9, 0.1, 0.0], + limit=2, + filters={"source_type": "slack"}, + ) + assert [item.external_id for item in filtered] == ["doc_b"] + + +def test_sqlite_vec_semantic_index_file_ref_filter_not_limited_by_global_rank(tmp_path): + index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") + index.reset(dimension=2, metadata={"field_mode": "summary"}) + + records = [ + SemanticIndexRecord( + file_ref=f"file_off_{item:02d}", + external_id=f"doc_off_{item:02d}", + source_type="documents", + title=f"Off scope {item:02d}", + text="off scope", + vector=[1.0, 0.0], + ) + for item in range(30) + ] + records.append( + SemanticIndexRecord( + file_ref="file_in_scope", + external_id="doc_in_scope", + source_type="documents", + title="In scope", + text="in scope", + vector=[0.0, 1.0], + ) + ) + index.upsert_many(records) + + results = index.search( + [1.0, 0.0], + limit=1, + filters={"file_ref": ["file_in_scope"]}, + ) + + assert [item.file_ref for item in results] == ["file_in_scope"] + + +def test_summary_projection_indexes_unified_metadata_summary(tmp_path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + indexer = SummaryProjectionIndexer( + tmp_path / "projection", + embedder=FixedDimensionEmbedder(3), + embedding_provider="test", + embedding_model="fake", + embedding_dimensions=3, + ) + + result = indexer.upsert_summary( + { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": { + "summary": "Unified metadata summary.", + "department": "ops", + }, + } + ) + + assert result["status"] == "ready" + hits = indexer.index.search([1.0, 0.0, 0.0], limit=1) + assert hits[0].external_id == "doc_a" + assert hits[0].metadata["summary"] == "Unified metadata summary." + assert hits[0].metadata["department"] == "ops" + + +def test_summary_projection_indexer_defaults_to_1024_dimensions(tmp_path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + indexer = SummaryProjectionIndexer( + tmp_path / "projection", + embedder=FixedDimensionEmbedder(1024), + embedding_provider="test", + embedding_model="fake", + ) + + info = indexer.index.info() + + assert info["dimension"] == 1024 + assert info["metadata"]["embedding_dimensions"] == 1024 + + result = indexer.upsert_summary( + { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Default dimension summary."}, + } + ) + + assert result["status"] == "ready" + assert result["embedding_dimensions"] == 1024 + + +def test_summary_projection_indexer_allows_explicit_256_dimensions(tmp_path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + indexer = SummaryProjectionIndexer( + tmp_path / "projection", + embedder=FixedDimensionEmbedder(256), + embedding_provider="test", + embedding_model="fake", + embedding_dimensions=256, + ) + + assert indexer.index.info()["dimension"] == 256 + assert indexer.upsert_summary( + { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Explicit 256 dimension summary."}, + } + )["status"] == "ready" + + +def test_summary_projection_default_rejects_existing_256_index_for_writes(tmp_path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + index_dir = tmp_path / "projection" + index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") + index.reset( + dimension=256, + metadata={ + "channel": "summary", + "embedding_provider": "test", + "embedding_model": "fake", + "embedding_dimensions": 256, + }, + ) + + with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): + SummaryProjectionIndexer( + index_dir, + embedder=FixedDimensionEmbedder(1024), + embedding_provider="test", + embedding_model="fake", + ) + + assert SQLiteVecSemanticIndex(index.db_path).info()["dimension"] == 256 + + +def test_summary_projection_from_provider_rejects_dimension_mismatch_before_embedder( + tmp_path, monkeypatch +): + from pageindex.filesystem import semantic_projection + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + index_dir = tmp_path / "projection" + index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") + index.reset( + dimension=256, + metadata={ + "channel": "summary", + "embedding_provider": "openai", + "embedding_model": "text-embedding-3-small", + "embedding_dimensions": 256, + }, + ) + + def fail_make_embedder(*args, **kwargs): + raise AssertionError("embedder should not be constructed before dimension validation") + + monkeypatch.setattr(semantic_projection, "make_embedder", fail_make_embedder) + + with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): + SummaryProjectionIndexer.from_provider(index_dir) + + +def test_embedding_cache_key_separates_model_dimensions(tmp_path): + from pageindex.filesystem.semantic_projection import ( + EmbeddingCache, + embedding_cache_model_key, + ) + + class CountingEmbedder: + def __init__(self, dimensions: int): + self.dimensions = dimensions + self.calls = 0 + + def embed(self, texts): + self.calls += 1 + return [[float(self.dimensions), *([0.0] * (self.dimensions - 1))] for _ in texts] + + cache = EmbeddingCache(tmp_path / "cache.sqlite") + embedder_256 = CountingEmbedder(256) + embedder_1024 = CountingEmbedder(1024) + key_256 = embedding_cache_model_key("fake", 256) + key_1024 = embedding_cache_model_key("fake", 1024) + + assert key_256 != key_1024 + + vector_256 = cache.embed_texts( + ["same text"], + provider="test", + model=key_256, + embedder=embedder_256, + batch_size=1, + )[0] + vector_1024 = cache.embed_texts( + ["same text"], + provider="test", + model=key_1024, + embedder=embedder_1024, + batch_size=1, + )[0] + + assert len(vector_256) == 256 + assert len(vector_1024) == 1024 + assert embedder_256.calls == 1 + assert embedder_1024.calls == 1 + + +def test_embedding_cache_rejects_response_length_mismatch(tmp_path): + from pageindex.filesystem.semantic_projection import EmbeddingCache + + class ShortEmbedder: + def embed(self, texts): + return [[1.0, 0.0, 0.0]] + + cache = EmbeddingCache(tmp_path / "cache.sqlite") + + with pytest.raises(ValueError, match="embedding response length mismatch"): + cache.embed_texts( + ["first", "second"], + provider="test", + model="fake", + embedder=ShortEmbedder(), + batch_size=2, + ) + + +def test_embed_with_retry_does_not_retry_permanent_errors(monkeypatch): + from pageindex.filesystem.semantic_projection import embed_with_retry + + sleeps = [] + + class PermanentEmbeddingError(Exception): + status_code = 401 + + class FailingEmbedder: + calls = 0 + + def embed(self, texts): + self.calls += 1 + raise PermanentEmbeddingError("unauthorized") + + monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) + embedder = FailingEmbedder() + + with pytest.raises(PermanentEmbeddingError): + embed_with_retry(embedder, ["text"]) + + assert embedder.calls == 1 + assert sleeps == [] + + +def test_embed_with_retry_retries_transient_errors(monkeypatch): + from pageindex.filesystem.semantic_projection import embed_with_retry + + sleeps = [] + + class TransientEmbeddingError(Exception): + status_code = 500 + + class FlakyEmbedder: + calls = 0 + + def embed(self, texts): + self.calls += 1 + if self.calls == 1: + raise TransientEmbeddingError("server error") + return [[1.0, 0.0, 0.0]] + + monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) + embedder = FlakyEmbedder() + + assert embed_with_retry(embedder, ["text"]) == [[1.0, 0.0, 0.0]] + assert embedder.calls == 2 + assert sleeps == [1.0] + + +def test_summary_projection_dimension_mismatch_preserves_existing_index(tmp_path): + from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer + + index_dir = tmp_path / "projection" + index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") + index.reset( + dimension=3, + metadata={ + "channel": "summary", + "embedding_provider": "test", + "embedding_model": "fake", + "embedding_dimensions": 3, + }, + ) + index.upsert_many( + [ + SemanticIndexRecord( + file_ref="file_a", + external_id="doc_a", + source_type="documents", + title="A", + text="summary", + vector=[1.0, 0.0, 0.0], + ) + ] + ) + + with pytest.raises(RuntimeError, match="summary projection index dimension mismatch"): + SummaryProjectionIndexer( + index_dir, + embedder=FixedDimensionEmbedder(4), + embedding_provider="test", + embedding_model="fake", + embedding_dimensions=4, + ) + + preserved = SQLiteVecSemanticIndex(index.db_path) + assert preserved.info()["dimension"] == 3 + assert preserved.info()["document_count"] == 1 + assert preserved.search([1.0, 0.0, 0.0], limit=1)[0].external_id == "doc_a" + + +def test_hash_embedding_provider_is_not_available(): + from pageindex.filesystem.semantic_projection import make_embedder + + with pytest.raises(ValueError, match="unknown embedding provider: hash"): + make_embedder("hash", "unused", dimensions=256, timeout=1) + + +def test_make_embedder_accepts_direct_runtime_config(monkeypatch): + from pageindex.filesystem.semantic_projection import make_embedder + + calls = [] + + class FakeOpenAI: + def __init__(self, **kwargs): + calls.append(kwargs) + + fake_openai = type(sys)("openai") + fake_openai.OpenAI = FakeOpenAI + monkeypatch.setitem(sys.modules, "openai", fake_openai) + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") + monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://ignored-openai.invalid/") + + make_embedder( + "openai", + "gemini-embedding-2-preview", + dimensions=3072, + timeout=12.5, + api_key="direct-key", + base_url="https://example.invalid/openai/", + ) + + assert calls == [ + { + "api_key": "direct-key", + "base_url": "https://example.invalid/openai/", + "timeout": 12.5, + } + ] + + +def test_make_embedder_requires_direct_api_key(monkeypatch): + from pageindex.filesystem.semantic_projection import make_embedder + + calls = [] + + class FakeOpenAI: + def __init__(self, **kwargs): + calls.append(kwargs) + + fake_openai = type(sys)("openai") + fake_openai.OpenAI = FakeOpenAI + monkeypatch.setitem(sys.modules, "openai", fake_openai) + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") + + with pytest.raises(ValueError, match="embedding_api_key is required"): + make_embedder( + "openai", + "gemini-embedding-2-preview", + dimensions=3072, + timeout=12.5, + ) + + assert calls == []