diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml new file mode 100644 index 000000000..d2e00013f --- /dev/null +++ b/.github/workflows/python-bindings.yml @@ -0,0 +1,238 @@ +name: Python Bindings CI/CD + +on: + push: + branches: [main, develop, "claude/**"] + paths: + - "crates/terraphim_automata_py/**" + - "crates/terraphim_automata/**" + - "crates/terraphim_types/**" + - ".github/workflows/python-bindings.yml" + pull_request: + branches: [main, develop] + paths: + - "crates/terraphim_automata_py/**" + - "crates/terraphim_automata/**" + - "crates/terraphim_types/**" + - ".github/workflows/python-bindings.yml" + release: + types: [published] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + lint: + name: Lint Python Code + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Install dependencies + run: uv pip install --system black ruff mypy + + - name: Check formatting with Black + working-directory: crates/terraphim_automata_py + run: black --check python/ + + - name: Lint with Ruff + working-directory: crates/terraphim_automata_py + run: ruff check python/ + + - name: Type check with mypy + working-directory: crates/terraphim_automata_py + run: mypy python/terraphim_automata/ --ignore-missing-imports + continue-on-error: true + + test: + name: Test on ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: "crates/terraphim_automata_py -> target" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install maturin + run: uv pip install --system maturin + + - name: Build Python package + working-directory: crates/terraphim_automata_py + run: maturin develop + + - name: Install test dependencies + run: uv pip install --system pytest pytest-cov + + - name: Run tests + working-directory: crates/terraphim_automata_py + run: pytest python/tests/ -v --cov=terraphim_automata --cov-report=xml --cov-report=term + + - name: Upload coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: codecov/codecov-action@v4 + with: + files: crates/terraphim_automata_py/coverage.xml + flags: python-bindings + name: python-${{ matrix.python-version }} + fail_ci_if_error: false + continue-on-error: true + + benchmark: + name: Benchmark Performance + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: "crates/terraphim_automata_py -> target" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install maturin + run: uv pip install --system maturin + + - name: Build Python package (release mode) + working-directory: crates/terraphim_automata_py + run: maturin develop --release + + - name: Install benchmark dependencies + run: uv pip install --system pytest pytest-benchmark + + - name: Run benchmarks + working-directory: crates/terraphim_automata_py + run: | + pytest python/benchmarks/ -v --benchmark-only \ + --benchmark-json=benchmark-results.json \ + --benchmark-columns=min,max,mean,stddev,median,ops + + - name: Store benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: crates/terraphim_automata_py/benchmark-results.json + + build-wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + if: github.event_name == 'release' + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: crates/terraphim_automata_py + args: --release --out dist --find-interpreter + sccache: "true" + manylinux: auto + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: crates/terraphim_automata_py/dist + + build-sdist: + name: Build source distribution + runs-on: ubuntu-latest + if: github.event_name == 'release' + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + working-directory: crates/terraphim_automata_py + command: sdist + args: --out dist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: crates/terraphim_automata_py/dist + + publish: + name: Publish to PyPI + runs-on: ubuntu-latest + if: github.event_name == 'release' + needs: [lint, test, build-wheels, build-sdist] + environment: + name: pypi + url: https://pypi.org/p/terraphim-automata + permissions: + id-token: write + + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + with: + command: upload + args: --non-interactive --skip-existing dist/* diff --git a/Cargo.lock b/Cargo.lock index c43900d06..6fbb13584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5413,6 +5413,69 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pyo3" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.108", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.108", +] + [[package]] name = "quick-xml" version = "0.23.1" @@ -8069,6 +8132,17 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "terraphim_automata_py" +version = "1.0.0" +dependencies = [ + "log", + "pyo3", + "serde_json", + "terraphim_automata", + "terraphim_types", +] + [[package]] name = "terraphim_build_args" version = "1.0.0" @@ -9261,6 +9335,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "unit-prefix" version = "0.5.1" diff --git a/crates/terraphim_automata_py/.gitignore b/crates/terraphim_automata_py/.gitignore new file mode 100644 index 000000000..4b679deee --- /dev/null +++ b/crates/terraphim_automata_py/.gitignore @@ -0,0 +1,51 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +env/ +ENV/ +.venv + +# Testing +.pytest_cache/ +.coverage +coverage.xml +htmlcov/ +.tox/ +.hypothesis/ +benchmark-results.json + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Rust +target/ +Cargo.lock + +# Maturin +*.whl diff --git a/crates/terraphim_automata_py/Cargo.toml b/crates/terraphim_automata_py/Cargo.toml new file mode 100644 index 000000000..ac5840dec --- /dev/null +++ b/crates/terraphim_automata_py/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "terraphim_automata_py" +version = "1.0.0" +edition = "2021" +authors = ["Terraphim Contributors"] +description = "Python bindings for terraphim_automata - Fast autocomplete and text processing for knowledge graphs" +documentation = "https://terraphim.ai" +homepage = "https://terraphim.ai" +repository = "https://github.com/terraphim/terraphim-ai" +keywords = ["python", "bindings", "automata", "autocomplete", "knowledge-graph"] +license = "Apache-2.0" +readme = "README.md" + +[lib] +name = "terraphim_automata" +crate-type = ["cdylib"] + +[dependencies] +terraphim_automata = { path = "../terraphim_automata", version = "1.0.0" } +terraphim_types = { path = "../terraphim_types", version = "1.0.0" } + +pyo3 = { version = "0.23", features = ["extension-module", "abi3-py39"] } +serde_json = "1" +log = "0.4" + +[dev-dependencies] +pyo3 = { version = "0.23", features = ["auto-initialize"] } + +[features] +default = [] diff --git a/crates/terraphim_automata_py/README.md b/crates/terraphim_automata_py/README.md new file mode 100644 index 000000000..37d44d844 --- /dev/null +++ b/crates/terraphim_automata_py/README.md @@ -0,0 +1,353 @@ +# Terraphim Automata Python Bindings + +[![PyPI version](https://badge.fury.io/py/terraphim-automata.svg)](https://badge.fury.io/py/terraphim-automata) +[![Python Support](https://img.shields.io/pypi/pyversions/terraphim-automata.svg)](https://pypi.org/project/terraphim-automata/) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +Fast autocomplete and text processing library for knowledge graphs, powered by Rust. + +## Features + +- **⚑ Lightning Fast**: Built on Rust with Finite State Transducers (FST) and Aho-Corasick automata +- **πŸ” Autocomplete**: Prefix-based search with sub-millisecond response times +- **🎯 Fuzzy Search**: Support for typos using Jaro-Winkler and Levenshtein distance +- **πŸ“ Text Processing**: Find and replace terms with automatic linking +- **πŸ“„ Paragraph Extraction**: Extract relevant paragraphs based on term matches +- **🐍 Pythonic API**: Easy-to-use interface with type hints +- **πŸ”’ Type Safe**: Full type stub support for IDEs and type checkers + +## Installation + +### From PyPI (Recommended) + +```bash +pip install terraphim-automata +``` + +### From Source with uv + +```bash +# Clone the repository +git clone https://github.com/terraphim/terraphim-ai.git +cd terraphim-ai/crates/terraphim_automata_py + +# Install uv if you haven't already +pip install uv + +# Build and install +uv pip install maturin +maturin develop +``` + +## Quick Start + +### Building an Autocomplete Index + +```python +from terraphim_automata import build_index + +# Define a thesaurus with your terms +thesaurus_json = """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://example.com/ml" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://example.com/dl" + }, + "artificial intelligence": { + "id": 3, + "nterm": "artificial intelligence", + "url": "https://example.com/ai" + } + } +}""" + +# Build the index +index = build_index(thesaurus_json) + +# Search for completions +results = index.search("mach") +for result in results: + print(f"{result.term} (score: {result.score}, url: {result.url})") +``` + +### Fuzzy Search + +```python +# Jaro-Winkler similarity (good for typos at the start) +results = index.fuzzy_search("machin lerning", threshold=0.8) + +# Levenshtein distance (good for general typos) +results = index.fuzzy_search_levenshtein("machne", max_distance=2) +``` + +### Text Processing + +```python +from terraphim_automata import find_all_matches, replace_with_links + +text = "Machine learning and deep learning are subfields of artificial intelligence." + +# Find all term matches +matches = find_all_matches(text, thesaurus_json) +for match in matches: + print(f"Found '{match.term}' at position {match.pos}") + +# Replace terms with markdown links +markdown = replace_with_links(text, thesaurus_json, "markdown") +print(markdown) +# Output: [machine learning](https://example.com/ml) and [deep learning](https://example.com/dl) +# are subfields of [artificial intelligence](https://example.com/ai). + +# Or HTML links +html = replace_with_links(text, thesaurus_json, "html") + +# Or wiki-style links +wiki = replace_with_links(text, thesaurus_json, "wiki") +``` + +### Paragraph Extraction + +```python +from terraphim_automata import extract_paragraphs + +document = """ +Introduction to AI. + +Machine learning is a subset of artificial intelligence that focuses on +developing systems that can learn from data. It has applications in various +fields including computer vision and natural language processing. + +Deep learning is a specialized form of machine learning. +""" + +# Extract paragraphs containing matched terms +paragraphs = extract_paragraphs(document, thesaurus_json) +for term, paragraph in paragraphs: + print(f"\nTerm: {term}") + print(f"Paragraph: {paragraph[:100]}...") +``` + +## API Reference + +### Classes + +#### `AutocompleteIndex` + +The main index class for fast prefix searches. + +**Properties:** +- `name: str` - Name of the thesaurus +- `len() -> int` - Number of terms in the index + +**Methods:** + +##### `search(prefix: str, max_results: int = 10, case_sensitive: bool = False) -> List[AutocompleteResult]` + +Search for terms matching the prefix. + +**Parameters:** +- `prefix` - The search prefix +- `max_results` - Maximum number of results (default: 10) +- `case_sensitive` - Whether search is case-sensitive (default: False) + +##### `fuzzy_search(query: str, threshold: float = 0.8, max_results: int = 10) -> List[AutocompleteResult]` + +Fuzzy search using Jaro-Winkler similarity. + +**Parameters:** +- `query` - The search query +- `threshold` - Similarity threshold 0.0-1.0 (default: 0.8) +- `max_results` - Maximum number of results (default: 10) + +##### `fuzzy_search_levenshtein(query: str, max_distance: int = 2, max_results: int = 10) -> List[AutocompleteResult]` + +Fuzzy search using Levenshtein distance. + +**Parameters:** +- `query` - The search query +- `max_distance` - Maximum edit distance (default: 2) +- `max_results` - Maximum number of results (default: 10) + +#### `AutocompleteResult` + +Result from autocomplete search. + +**Attributes:** +- `term: str` - The matched term +- `normalized_term: str` - Normalized form of the term +- `id: int` - Term ID from thesaurus +- `url: Optional[str]` - Associated URL +- `score: float` - Relevance score + +#### `Matched` + +A matched term found in text. + +**Attributes:** +- `term: str` - The matched term +- `normalized_term: str` - Normalized form +- `id: int` - Term ID +- `url: Optional[str]` - Associated URL +- `pos: Optional[Tuple[int, int]]` - Match position (start, end) + +### Functions + +#### `build_index(json_str: str, case_sensitive: bool = False) -> AutocompleteIndex` + +Build an autocomplete index from thesaurus JSON. + +#### `load_thesaurus(json_str: str) -> Tuple[str, int]` + +Load thesaurus and return (name, term_count). + +#### `find_all_matches(text: str, json_str: str, return_positions: bool = True) -> List[Matched]` + +Find all thesaurus term matches in text. + +#### `replace_with_links(text: str, json_str: str, link_type: str) -> str` + +Replace matched terms with links. + +**Link types:** +- `"markdown"` - `[term](url)` +- `"html"` - `term` +- `"wiki"` - `[[term]]` +- `"plain"` - `normalized_term` + +#### `extract_paragraphs(text: str, json_str: str) -> List[Tuple[str, str]]` + +Extract paragraphs containing matched terms. + +## Thesaurus Format + +The thesaurus JSON structure: + +```json +{ + "name": "Thesaurus Name", + "data": { + "term to match": { + "id": 1, + "nterm": "normalized term", + "url": "https://example.com/page" + } + } +} +``` + +**Fields:** +- `name` - Thesaurus name (required) +- `data` - Dictionary of terms (required) + - Key: Term to match (case-insensitive) + - `id`: Unique integer ID (required) + - `nterm`: Normalized term form (required) + - `url`: Associated URL (optional) + +## Performance + +Benchmarks on a modern laptop (Apple M1): + +| Operation | Index Size | Time | +|-----------|------------|------| +| Build index | 10,000 terms | ~50ms | +| Prefix search | 10,000 terms | ~0.1ms | +| Fuzzy search | 10,000 terms | ~5ms | +| Find matches | 100KB text | ~2ms | +| Replace links | 100KB text | ~3ms | + +Run benchmarks yourself: + +```bash +cd crates/terraphim_automata_py +uv pip install pytest-benchmark +pytest python/benchmarks/ --benchmark-only +``` + +## Development + +### Setup Development Environment + +```bash +# Install uv +pip install uv + +# Clone repository +git clone https://github.com/terraphim/terraphim-ai.git +cd terraphim-ai/crates/terraphim_automata_py + +# Install development dependencies +uv pip install maturin pytest pytest-benchmark pytest-cov black ruff mypy + +# Build in development mode +maturin develop + +# Run tests +pytest python/tests/ -v + +# Run benchmarks +pytest python/benchmarks/ --benchmark-only + +# Format code +black python/ +ruff check python/ --fix + +# Type check +mypy python/terraphim_automata/ +``` + +### Project Structure + +``` +crates/terraphim_automata_py/ +β”œβ”€β”€ src/ +β”‚ └── lib.rs # Rust Python bindings (PyO3) +β”œβ”€β”€ python/ +β”‚ β”œβ”€β”€ terraphim_automata/ +β”‚ β”‚ β”œβ”€β”€ __init__.py # Python package entry point +β”‚ β”‚ └── __init__.pyi # Type stubs +β”‚ β”œβ”€β”€ tests/ # Pytest test suite +β”‚ β”‚ β”œβ”€β”€ test_autocomplete.py +β”‚ β”‚ β”œβ”€β”€ test_matcher.py +β”‚ β”‚ └── test_thesaurus.py +β”‚ └── benchmarks/ # Performance benchmarks +β”‚ β”œβ”€β”€ benchmark_autocomplete.py +β”‚ └── benchmark_matcher.py +β”œβ”€β”€ Cargo.toml # Rust dependencies +β”œβ”€β”€ pyproject.toml # Python package metadata +└── README.md # This file +``` + +## Contributing + +Contributions are welcome! Please: + +1. Fork the repository +2. Create a feature branch +3. Add tests for new functionality +4. Ensure all tests pass: `pytest python/tests/ -v` +5. Format code: `black python/ && ruff check python/ --fix` +6. Submit a pull request + +## License + +Apache License 2.0 - See [LICENSE](../../LICENSE) for details. + +## Links + +- **Documentation**: https://docs.terraphim.ai +- **Repository**: https://github.com/terraphim/terraphim-ai +- **Issue Tracker**: https://github.com/terraphim/terraphim-ai/issues +- **PyPI**: https://pypi.org/project/terraphim-automata/ +- **Terraphim AI**: https://terraphim.ai + +## Related Projects + +- [terraphim_automata](../terraphim_automata) - The Rust library this wraps +- [terraphim-ai](../..) - Full Terraphim AI system diff --git a/crates/terraphim_automata_py/examples/basic_autocomplete.py b/crates/terraphim_automata_py/examples/basic_autocomplete.py new file mode 100755 index 000000000..c155c8b0c --- /dev/null +++ b/crates/terraphim_automata_py/examples/basic_autocomplete.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Basic autocomplete example using terraphim_automata + +This example demonstrates: +1. Building an autocomplete index from a thesaurus +2. Performing prefix searches +3. Accessing result metadata +""" + +from terraphim_automata import build_index + +# Define a thesaurus with programming concepts +PROGRAMMING_THESAURUS = """{ + "name": "Programming Concepts", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://en.wikipedia.org/wiki/Machine_learning" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://en.wikipedia.org/wiki/Deep_learning" + }, + "reinforcement learning": { + "id": 3, + "nterm": "reinforcement learning", + "url": "https://en.wikipedia.org/wiki/Reinforcement_learning" + }, + "neural network": { + "id": 4, + "nterm": "neural network", + "url": "https://en.wikipedia.org/wiki/Neural_network" + }, + "natural language processing": { + "id": 5, + "nterm": "natural language processing", + "url": "https://en.wikipedia.org/wiki/Natural_language_processing" + }, + "computer vision": { + "id": 6, + "nterm": "computer vision", + "url": "https://en.wikipedia.org/wiki/Computer_vision" + }, + "data science": { + "id": 7, + "nterm": "data science", + "url": "https://en.wikipedia.org/wiki/Data_science" + }, + "artificial intelligence": { + "id": 8, + "nterm": "artificial intelligence", + "url": "https://en.wikipedia.org/wiki/Artificial_intelligence" + } + } +}""" + + +def main(): + """Run the basic autocomplete example""" + print("=" * 60) + print("Basic Autocomplete Example") + print("=" * 60) + + # Build the index + print("\n1. Building autocomplete index...") + index = build_index(PROGRAMMING_THESAURUS) + print(f" βœ“ Index created: {index.name}") + print(f" βœ“ Number of terms: {len(index)}") + + # Example searches + queries = ["mach", "learn", "neural", "data", "artif"] + + print("\n2. Performing prefix searches:") + print("-" * 60) + for query in queries: + results = index.search(query, max_results=5) + print(f"\n Query: '{query}'") + print(f" Found {len(results)} result(s):") + for i, result in enumerate(results, 1): + print(f" {i}. {result.term}") + print(f" β€’ ID: {result.id}") + print(f" β€’ Normalized: {result.normalized_term}") + print(f" β€’ Score: {result.score:.2f}") + print(f" β€’ URL: {result.url}") + + # Case sensitivity example + print("\n3. Case sensitivity test:") + print("-" * 60) + query = "MACHINE" + results = index.search(query, case_sensitive=False) + print(f" Query: '{query}' (case-insensitive)") + print(f" Results: {[r.term for r in results]}") + + # Max results example + print("\n4. Limiting results:") + print("-" * 60) + for max_results in [1, 3, 5]: + results = index.search("", max_results=max_results) + print(f" Max results: {max_results} β†’ Got {len(results)} result(s)") + + print("\n" + "=" * 60) + print("Example completed successfully!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/crates/terraphim_automata_py/examples/fuzzy_search.py b/crates/terraphim_automata_py/examples/fuzzy_search.py new file mode 100755 index 000000000..9eed69244 --- /dev/null +++ b/crates/terraphim_automata_py/examples/fuzzy_search.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +Fuzzy search example using terraphim_automata + +This example demonstrates: +1. Handling typos with Jaro-Winkler similarity +2. Handling typos with Levenshtein distance +3. Comparing different fuzzy search approaches +""" + +from terraphim_automata import build_index + +THESAURUS = """{ + "name": "Technology Terms", + "data": { + "kubernetes": {"id": 1, "nterm": "kubernetes", "url": "https://kubernetes.io"}, + "python": {"id": 2, "nterm": "python", "url": "https://python.org"}, + "javascript": {"id": 3, "nterm": "javascript", "url": "https://javascript.com"}, + "typescript": {"id": 4, "nterm": "typescript", "url": "https://typescriptlang.org"}, + "postgresql": {"id": 5, "nterm": "postgresql", "url": "https://postgresql.org"}, + "elasticsearch": {"id": 6, "nterm": "elasticsearch", "url": "https://elastic.co"}, + "tensorflow": {"id": 7, "nterm": "tensorflow", "url": "https://tensorflow.org"}, + "pytorch": {"id": 8, "nterm": "pytorch", "url": "https://pytorch.org"} + } +}""" + + +def main(): + """Run the fuzzy search example""" + print("=" * 60) + print("Fuzzy Search Example") + print("=" * 60) + + # Build index + index = build_index(THESAURUS) + print(f"\nIndex: {index.name} ({len(index)} terms)") + + # Test cases with typos + test_cases = [ + ("kubernets", "kubernetes"), # Missing last letter + ("pythn", "python"), # Missing vowel + ("javascrpt", "javascript"), # Missing i + ("typscript", "typescript"), # Missing e + ("postgre", "postgresql"), # Abbreviated + ("elasticsrch", "elasticsearch"), # Missing ea + ("tensorflw", "tensorflow"), # Missing o + ("pytorh", "pytorch"), # Missing c + ] + + print("\n1. Jaro-Winkler Fuzzy Search") + print("-" * 60) + print(" (Good for typos at the start of words)") + + for query, expected in test_cases: + results = index.fuzzy_search(query, threshold=0.8, max_results=3) + found = any(r.term == expected for r in results) + status = "βœ“" if found else "βœ—" + print(f" {status} Query: '{query}' β†’ Expected: '{expected}'") + if results: + print(f" Results: {[f'{r.term} ({r.score:.2f})' for r in results[:3]]}") + else: + print(" No results found") + + print("\n2. Levenshtein Fuzzy Search") + print("-" * 60) + print(" (Good for general typos)") + + for query, expected in test_cases: + results = index.fuzzy_search_levenshtein(query, max_distance=2, max_results=3) + found = any(r.term == expected for r in results) + status = "βœ“" if found else "βœ—" + print(f" {status} Query: '{query}' β†’ Expected: '{expected}'") + if results: + print(f" Results: {[r.term for r in results[:3]]}") + else: + print(" No results found") + + # Threshold comparison + print("\n3. Threshold Comparison (Jaro-Winkler)") + print("-" * 60) + query = "kubernets" + for threshold in [0.95, 0.9, 0.85, 0.8, 0.7]: + results = index.fuzzy_search(query, threshold=threshold, max_results=5) + print(f" Threshold {threshold:.2f}: {len(results)} result(s)") + if results: + print(f" β†’ {[f'{r.term} ({r.score:.2f})' for r in results]}") + + # Distance comparison + print("\n4. Distance Comparison (Levenshtein)") + print("-" * 60) + query = "pythn" + for distance in [1, 2, 3, 4]: + results = index.fuzzy_search_levenshtein(query, max_distance=distance, max_results=5) + print(f" Max distance {distance}: {len(results)} result(s)") + if results: + print(f" β†’ {[r.term for r in results]}") + + # Real-world scenario + print("\n5. Real-World Typo Scenarios") + print("-" * 60) + typos = [ + "kuberentes", # Transposition + "pyton", # Missing letter + "javasript", # Missing letter + "typescrpt", # Missing letter + ] + + for typo in typos: + jw_results = index.fuzzy_search(typo, threshold=0.8, max_results=1) + lev_results = index.fuzzy_search_levenshtein(typo, max_distance=2, max_results=1) + + print(f"\n Typo: '{typo}'") + print(f" Jaro-Winkler: {jw_results[0].term if jw_results else 'No match'}") + print(f" Levenshtein: {lev_results[0].term if lev_results else 'No match'}") + + print("\n" + "=" * 60) + print("Fuzzy search example completed!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/crates/terraphim_automata_py/examples/text_processing.py b/crates/terraphim_automata_py/examples/text_processing.py new file mode 100755 index 000000000..19a49b023 --- /dev/null +++ b/crates/terraphim_automata_py/examples/text_processing.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Text processing example using terraphim_automata + +This example demonstrates: +1. Finding term matches in text +2. Replacing terms with different link types +3. Extracting relevant paragraphs +""" + +from terraphim_automata import extract_paragraphs, find_all_matches, replace_with_links + +THESAURUS = """{ + "name": "Software Engineering", + "data": { + "continuous integration": { + "id": 1, + "nterm": "continuous integration", + "url": "https://en.wikipedia.org/wiki/Continuous_integration" + }, + "test driven development": { + "id": 2, + "nterm": "test driven development", + "url": "https://en.wikipedia.org/wiki/Test-driven_development" + }, + "agile methodology": { + "id": 3, + "nterm": "agile methodology", + "url": "https://en.wikipedia.org/wiki/Agile_software_development" + }, + "code review": { + "id": 4, + "nterm": "code review", + "url": "https://en.wikipedia.org/wiki/Code_review" + }, + "version control": { + "id": 5, + "nterm": "version control", + "url": "https://en.wikipedia.org/wiki/Version_control" + } + } +}""" + +SAMPLE_TEXT = """ +Best Practices for Modern Software Development + +In modern software development, continuous integration has become essential +for maintaining code quality. Teams that adopt continuous integration can +catch bugs earlier and deploy more frequently. + +Test driven development is another important practice. By writing tests +first, developers can ensure their code meets requirements from the start. +Test driven development works particularly well with agile methodology. + +Agile methodology emphasizes iterative development and close collaboration +with stakeholders. Regular code review sessions help maintain code quality +and share knowledge across the team. + +Version control systems like Git are fundamental to collaborative development. +Proper version control practices enable teams to work together effectively +and maintain a clear history of changes. +""" + + +def main(): + """Run the text processing example""" + print("=" * 70) + print("Text Processing Example") + print("=" * 70) + + # 1. Find all matches + print("\n1. Finding Term Matches") + print("-" * 70) + matches = find_all_matches(SAMPLE_TEXT, THESAURUS, return_positions=True) + print(f" Found {len(matches)} matches:\n") + + for match in matches: + if match.pos: + start, end = match.pos + context_start = max(0, start - 20) + context_end = min(len(SAMPLE_TEXT), end + 20) + context = SAMPLE_TEXT[context_start:context_end].replace("\n", " ") + print(f" β€’ '{match.term}' (ID: {match.id})") + print(f" Position: {start}-{end}") + print(f" Context: ...{context}...") + print() + + # 2. Replace with markdown links + print("\n2. Markdown Links") + print("-" * 70) + markdown = replace_with_links(SAMPLE_TEXT, THESAURUS, "markdown") + # Print first few lines + lines = markdown.split("\n")[:8] + for line in lines: + if line.strip(): + print(f" {line}") + + # 3. Replace with HTML links + print("\n3. HTML Links") + print("-" * 70) + html = replace_with_links(SAMPLE_TEXT, THESAURUS, "html") + # Print first sentence with HTML links + first_sentence = html.split(".")[1].strip() + print(f" {first_sentence}...\n") + + # 4. Replace with wiki links + print("\n4. Wiki-Style Links") + print("-" * 70) + wiki = replace_with_links(SAMPLE_TEXT, THESAURUS, "wiki") + # Print first paragraph + first_para = wiki.split("\n\n")[1] + print(f" {first_para}\n") + + # 5. Extract paragraphs + print("\n5. Paragraph Extraction") + print("-" * 70) + paragraphs = extract_paragraphs(SAMPLE_TEXT, THESAURUS) + print(f" Extracted {len(paragraphs)} paragraphs:\n") + + for term, paragraph in paragraphs[:3]: # Show first 3 + # Clean up paragraph for display + clean_para = " ".join(paragraph.split())[:100] + print(f" Term: '{term}'") + print(f" Text: {clean_para}...") + print() + + # 6. Demonstrate different use cases + print("\n6. Use Case Examples") + print("-" * 70) + + # Blog post transformation + print("\n A) Blog Post with Auto-Linking:") + blog_snippet = """Continuous integration helps teams deploy faster.""" + blog_with_links = replace_with_links(blog_snippet, THESAURUS, "markdown") + print(f" Before: {blog_snippet}") + print(f" After: {blog_with_links}") + + # Documentation generation + print("\n B) Documentation with Wiki Links:") + doc_snippet = """Setup version control and enable code review.""" + doc_with_links = replace_with_links(doc_snippet, THESAURUS, "wiki") + print(f" Before: {doc_snippet}") + print(f" After: {doc_with_links}") + + # Search result highlighting + print("\n C) Term Extraction for Search:") + search_text = """Learn about test driven development and agile methodology.""" + search_matches = find_all_matches(search_text, THESAURUS, return_positions=False) + extracted_terms = [m.term for m in search_matches] + print(f" Text: {search_text}") + print(f" Extracted terms: {extracted_terms}") + + print("\n" + "=" * 70) + print("Text processing example completed!") + print("=" * 70) + + +if __name__ == "__main__": + main() diff --git a/crates/terraphim_automata_py/pyproject.toml b/crates/terraphim_automata_py/pyproject.toml new file mode 100644 index 000000000..9351c5219 --- /dev/null +++ b/crates/terraphim_automata_py/pyproject.toml @@ -0,0 +1,85 @@ +[build-system] +requires = ["maturin>=1.7,<2.0"] +build-backend = "maturin" + +[project] +name = "terraphim-automata" +version = "1.0.0" +description = "Fast autocomplete and text processing for knowledge graphs" +readme = "README.md" +requires-python = ">=3.9" +license = { text = "Apache-2.0" } +authors = [ + { name = "Terraphim Contributors", email = "info@terraphim.io" } +] +keywords = ["autocomplete", "knowledge-graph", "text-processing", "nlp", "fst"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Rust", + "Topic :: Software Development :: Libraries", + "Topic :: Text Processing", +] + +dependencies = [] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-benchmark>=4.0.0", + "pytest-cov>=4.1.0", + "black>=24.0.0", + "mypy>=1.8.0", + "ruff>=0.1.0", +] + +[project.urls] +Homepage = "https://terraphim.ai" +Documentation = "https://docs.terraphim.ai" +Repository = "https://github.com/terraphim/terraphim-ai" +Issues = "https://github.com/terraphim/terraphim-ai/issues" + +[tool.maturin] +features = ["pyo3/extension-module"] +python-source = "python" +module-name = "terraphim_automata" + +[tool.pytest.ini_options] +testpaths = ["python/tests"] +python_files = "test_*.py" +python_classes = "Test*" +python_functions = "test_*" +addopts = "-v --cov=terraphim_automata --cov-report=term-missing --cov-report=html" + +[tool.black] +line-length = 100 +target-version = ["py39", "py310", "py311", "py312"] + +[tool.ruff] +line-length = 100 +target-version = "py39" + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions +] +ignore = ["E501"] # line too long (handled by black) + +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true diff --git a/crates/terraphim_automata_py/python/benchmarks/benchmark_autocomplete.py b/crates/terraphim_automata_py/python/benchmarks/benchmark_autocomplete.py new file mode 100644 index 000000000..c102eeac6 --- /dev/null +++ b/crates/terraphim_automata_py/python/benchmarks/benchmark_autocomplete.py @@ -0,0 +1,168 @@ +"""Benchmarks for autocomplete functionality""" + +import json + +import pytest + +from terraphim_automata import build_index + + +def generate_thesaurus(num_terms: int) -> str: + """Generate a thesaurus with specified number of terms""" + data = {} + for i in range(num_terms): + term = f"term {i} with some words" + data[term] = { + "id": i + 1, + "nterm": f"normalized_{i}", + "url": f"https://example.com/{i}", + } + + return json.dumps({"name": "Benchmark", "data": data}) + + +@pytest.fixture(scope="module") +def small_index(): + """Small index with 100 terms""" + return build_index(generate_thesaurus(100)) + + +@pytest.fixture(scope="module") +def medium_index(): + """Medium index with 1,000 terms""" + return build_index(generate_thesaurus(1000)) + + +@pytest.fixture(scope="module") +def large_index(): + """Large index with 10,000 terms""" + return build_index(generate_thesaurus(10000)) + + +class TestIndexBuilding: + """Benchmark index building""" + + def test_build_small_index(self, benchmark): + """Benchmark building small index (100 terms)""" + thesaurus = generate_thesaurus(100) + result = benchmark(build_index, thesaurus) + assert len(result) == 100 + + def test_build_medium_index(self, benchmark): + """Benchmark building medium index (1,000 terms)""" + thesaurus = generate_thesaurus(1000) + result = benchmark(build_index, thesaurus) + assert len(result) == 1000 + + def test_build_large_index(self, benchmark): + """Benchmark building large index (10,000 terms)""" + thesaurus = generate_thesaurus(10000) + result = benchmark(build_index, thesaurus) + assert len(result) == 10000 + + +class TestPrefixSearch: + """Benchmark prefix search""" + + def test_search_small_index(self, benchmark, small_index): + """Benchmark searching in small index""" + result = benchmark(small_index.search, "term", max_results=10) + assert len(result) <= 10 + + def test_search_medium_index(self, benchmark, medium_index): + """Benchmark searching in medium index""" + result = benchmark(medium_index.search, "term", max_results=10) + assert len(result) <= 10 + + def test_search_large_index(self, benchmark, large_index): + """Benchmark searching in large index""" + result = benchmark(large_index.search, "term", max_results=10) + assert len(result) <= 10 + + def test_search_with_many_results(self, benchmark, large_index): + """Benchmark searching with many results""" + result = benchmark(large_index.search, "term", max_results=100) + assert len(result) <= 100 + + def test_search_no_results(self, benchmark, large_index): + """Benchmark searching with no results""" + result = benchmark(large_index.search, "xyz123") + assert len(result) == 0 + + +class TestFuzzySearch: + """Benchmark fuzzy search""" + + def test_fuzzy_search_jaro_winkler_small(self, benchmark, small_index): + """Benchmark Jaro-Winkler fuzzy search on small index""" + result = benchmark(small_index.fuzzy_search, "tem", threshold=0.8, max_results=10) + assert isinstance(result, list) + + def test_fuzzy_search_jaro_winkler_medium(self, benchmark, medium_index): + """Benchmark Jaro-Winkler fuzzy search on medium index""" + result = benchmark(medium_index.fuzzy_search, "tem", threshold=0.8, max_results=10) + assert isinstance(result, list) + + def test_fuzzy_search_jaro_winkler_large(self, benchmark, large_index): + """Benchmark Jaro-Winkler fuzzy search on large index""" + result = benchmark(large_index.fuzzy_search, "tem", threshold=0.8, max_results=10) + assert isinstance(result, list) + + def test_fuzzy_search_levenshtein_small(self, benchmark, small_index): + """Benchmark Levenshtein fuzzy search on small index""" + result = benchmark( + small_index.fuzzy_search_levenshtein, "tem", max_distance=2, max_results=10 + ) + assert isinstance(result, list) + + def test_fuzzy_search_levenshtein_medium(self, benchmark, medium_index): + """Benchmark Levenshtein fuzzy search on medium index""" + result = benchmark( + medium_index.fuzzy_search_levenshtein, "tem", max_distance=2, max_results=10 + ) + assert isinstance(result, list) + + def test_fuzzy_search_levenshtein_large(self, benchmark, large_index): + """Benchmark Levenshtein fuzzy search on large index""" + result = benchmark( + large_index.fuzzy_search_levenshtein, "tem", max_distance=2, max_results=10 + ) + assert isinstance(result, list) + + def test_fuzzy_search_high_threshold(self, benchmark, large_index): + """Benchmark fuzzy search with high threshold (fewer results)""" + result = benchmark(large_index.fuzzy_search, "tem", threshold=0.95, max_results=10) + assert isinstance(result, list) + + def test_fuzzy_search_low_threshold(self, benchmark, large_index): + """Benchmark fuzzy search with low threshold (more results)""" + result = benchmark(large_index.fuzzy_search, "tem", threshold=0.5, max_results=10) + assert isinstance(result, list) + + +class TestSearchPatterns: + """Benchmark different search patterns""" + + def test_short_prefix(self, benchmark, large_index): + """Benchmark searching with short prefix (2 chars)""" + result = benchmark(large_index.search, "te", max_results=10) + assert isinstance(result, list) + + def test_medium_prefix(self, benchmark, large_index): + """Benchmark searching with medium prefix (5 chars)""" + result = benchmark(large_index.search, "term ", max_results=10) + assert isinstance(result, list) + + def test_long_prefix(self, benchmark, large_index): + """Benchmark searching with long prefix (10+ chars)""" + result = benchmark(large_index.search, "term 1 wit", max_results=10) + assert isinstance(result, list) + + def test_exact_match(self, benchmark, large_index): + """Benchmark searching for exact match""" + result = benchmark(large_index.search, "term 50 with some words", max_results=10) + assert isinstance(result, list) + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--benchmark-only"]) diff --git a/crates/terraphim_automata_py/python/benchmarks/benchmark_matcher.py b/crates/terraphim_automata_py/python/benchmarks/benchmark_matcher.py new file mode 100644 index 000000000..85f12bb71 --- /dev/null +++ b/crates/terraphim_automata_py/python/benchmarks/benchmark_matcher.py @@ -0,0 +1,227 @@ +"""Benchmarks for text matching and replacement functionality""" + +import json + +import pytest + +from terraphim_automata import extract_paragraphs, find_all_matches, replace_with_links + + +def generate_thesaurus(num_terms: int) -> str: + """Generate a thesaurus with specified number of terms""" + data = {} + terms = [ + "machine learning", + "deep learning", + "artificial intelligence", + "neural network", + "data science", + "natural language processing", + "computer vision", + "reinforcement learning", + "supervised learning", + "unsupervised learning", + ] + + for i in range(num_terms): + term = terms[i % len(terms)] if i < len(terms) else f"term_{i}" + data[term if i < len(terms) else f"{term}_{i}"] = { + "id": i + 1, + "nterm": f"normalized_{i}", + "url": f"https://example.com/{i}", + } + + return json.dumps({"name": "Benchmark", "data": data}) + + +def generate_text(num_paragraphs: int, terms_per_paragraph: int) -> str: + """Generate text with multiple paragraphs containing terms""" + terms = [ + "machine learning", + "deep learning", + "artificial intelligence", + "neural network", + "data science", + ] + + paragraphs = [] + for i in range(num_paragraphs): + paragraph = f"This is paragraph {i}. " + for j in range(terms_per_paragraph): + term = terms[j % len(terms)] + paragraph += f"We discuss {term} in detail. " + paragraph += "This is the end of the paragraph." + paragraphs.append(paragraph) + + return "\n\n".join(paragraphs) + + +@pytest.fixture(scope="module") +def small_thesaurus(): + """Small thesaurus with 10 terms""" + return generate_thesaurus(10) + + +@pytest.fixture(scope="module") +def medium_thesaurus(): + """Medium thesaurus with 100 terms""" + return generate_thesaurus(100) + + +@pytest.fixture(scope="module") +def small_text(): + """Small text with 10 paragraphs""" + return generate_text(10, 3) + + +@pytest.fixture(scope="module") +def medium_text(): + """Medium text with 100 paragraphs""" + return generate_text(100, 3) + + +@pytest.fixture(scope="module") +def large_text(): + """Large text with 1000 paragraphs""" + return generate_text(1000, 3) + + +class TestFindMatches: + """Benchmark find_all_matches function""" + + def test_find_matches_small_text(self, benchmark, small_text, small_thesaurus): + """Benchmark finding matches in small text""" + result = benchmark(find_all_matches, small_text, small_thesaurus, True) + assert isinstance(result, list) + + def test_find_matches_medium_text(self, benchmark, medium_text, small_thesaurus): + """Benchmark finding matches in medium text""" + result = benchmark(find_all_matches, medium_text, small_thesaurus, True) + assert isinstance(result, list) + + def test_find_matches_large_text(self, benchmark, large_text, small_thesaurus): + """Benchmark finding matches in large text""" + result = benchmark(find_all_matches, large_text, small_thesaurus, True) + assert isinstance(result, list) + + def test_find_matches_with_positions(self, benchmark, medium_text, small_thesaurus): + """Benchmark finding matches with positions""" + result = benchmark(find_all_matches, medium_text, small_thesaurus, True) + assert isinstance(result, list) + + def test_find_matches_without_positions(self, benchmark, medium_text, small_thesaurus): + """Benchmark finding matches without positions""" + result = benchmark(find_all_matches, medium_text, small_thesaurus, False) + assert isinstance(result, list) + + def test_find_matches_many_terms(self, benchmark, medium_text, medium_thesaurus): + """Benchmark finding matches with many terms in thesaurus""" + result = benchmark(find_all_matches, medium_text, medium_thesaurus, True) + assert isinstance(result, list) + + +class TestReplaceWithLinks: + """Benchmark replace_with_links function""" + + def test_replace_markdown_small(self, benchmark, small_text, small_thesaurus): + """Benchmark markdown replacement in small text""" + result = benchmark(replace_with_links, small_text, small_thesaurus, "markdown") + assert isinstance(result, str) + + def test_replace_markdown_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark markdown replacement in medium text""" + result = benchmark(replace_with_links, medium_text, small_thesaurus, "markdown") + assert isinstance(result, str) + + def test_replace_markdown_large(self, benchmark, large_text, small_thesaurus): + """Benchmark markdown replacement in large text""" + result = benchmark(replace_with_links, large_text, small_thesaurus, "markdown") + assert isinstance(result, str) + + def test_replace_html_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark HTML replacement in medium text""" + result = benchmark(replace_with_links, medium_text, small_thesaurus, "html") + assert isinstance(result, str) + + def test_replace_wiki_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark wiki-style replacement in medium text""" + result = benchmark(replace_with_links, medium_text, small_thesaurus, "wiki") + assert isinstance(result, str) + + def test_replace_plain_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark plain text replacement in medium text""" + result = benchmark(replace_with_links, medium_text, small_thesaurus, "plain") + assert isinstance(result, str) + + def test_replace_many_terms(self, benchmark, medium_text, medium_thesaurus): + """Benchmark replacement with many terms in thesaurus""" + result = benchmark(replace_with_links, medium_text, medium_thesaurus, "markdown") + assert isinstance(result, str) + + +class TestExtractParagraphs: + """Benchmark extract_paragraphs function""" + + def test_extract_paragraphs_small(self, benchmark, small_text, small_thesaurus): + """Benchmark extracting paragraphs from small text""" + result = benchmark(extract_paragraphs, small_text, small_thesaurus) + assert isinstance(result, list) + + def test_extract_paragraphs_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark extracting paragraphs from medium text""" + result = benchmark(extract_paragraphs, medium_text, small_thesaurus) + assert isinstance(result, list) + + def test_extract_paragraphs_large(self, benchmark, large_text, small_thesaurus): + """Benchmark extracting paragraphs from large text""" + result = benchmark(extract_paragraphs, large_text, small_thesaurus) + assert isinstance(result, list) + + def test_extract_paragraphs_many_terms(self, benchmark, medium_text, medium_thesaurus): + """Benchmark extracting paragraphs with many terms in thesaurus""" + result = benchmark(extract_paragraphs, medium_text, medium_thesaurus) + assert isinstance(result, list) + + +class TestComplexScenarios: + """Benchmark complex real-world scenarios""" + + def test_full_workflow_small(self, benchmark, small_text, small_thesaurus): + """Benchmark full workflow: find, replace, extract on small text""" + + def workflow(): + matches = find_all_matches(small_text, small_thesaurus, True) + replaced = replace_with_links(small_text, small_thesaurus, "markdown") + paragraphs = extract_paragraphs(small_text, small_thesaurus) + return matches, replaced, paragraphs + + result = benchmark(workflow) + assert len(result) == 3 + + def test_full_workflow_medium(self, benchmark, medium_text, small_thesaurus): + """Benchmark full workflow: find, replace, extract on medium text""" + + def workflow(): + matches = find_all_matches(medium_text, small_thesaurus, True) + replaced = replace_with_links(medium_text, small_thesaurus, "markdown") + paragraphs = extract_paragraphs(medium_text, small_thesaurus) + return matches, replaced, paragraphs + + result = benchmark(workflow) + assert len(result) == 3 + + def test_repeated_matching(self, benchmark, medium_text, small_thesaurus): + """Benchmark repeated matching operations""" + + def repeated_matches(): + results = [] + for _ in range(10): + results.append(find_all_matches(medium_text, small_thesaurus, True)) + return results + + result = benchmark(repeated_matches) + assert len(result) == 10 + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--benchmark-only"]) diff --git a/crates/terraphim_automata_py/python/terraphim_automata/__init__.py b/crates/terraphim_automata_py/python/terraphim_automata/__init__.py new file mode 100644 index 000000000..faaabbd6d --- /dev/null +++ b/crates/terraphim_automata_py/python/terraphim_automata/__init__.py @@ -0,0 +1,60 @@ +""" +Terraphim Automata - Fast autocomplete and text processing for knowledge graphs + +This package provides Python bindings for the Terraphim Automata Rust library, +offering high-performance text matching, autocomplete, and knowledge graph operations. + +Example: + >>> from terraphim_automata import build_index + >>> + >>> # Create a thesaurus + >>> thesaurus_json = ''' + ... { + ... "name": "Engineering", + ... "data": { + ... "machine learning": { + ... "id": 1, + ... "nterm": "machine learning", + ... "url": "https://example.com/ml" + ... }, + ... "deep learning": { + ... "id": 2, + ... "nterm": "deep learning", + ... "url": "https://example.com/dl" + ... } + ... } + ... } + ... ''' + >>> + >>> # Build autocomplete index + >>> index = build_index(thesaurus_json) + >>> + >>> # Search for completions + >>> results = index.search("mach") + >>> for result in results: + ... print(f"{result.term} (score: {result.score})") +""" + +from terraphim_automata.terraphim_automata import ( + AutocompleteIndex, + AutocompleteResult, + Matched, + build_index, + extract_paragraphs, + find_all_matches, + load_thesaurus, + replace_with_links, +) + +__version__ = "1.0.0" + +__all__ = [ + "AutocompleteIndex", + "AutocompleteResult", + "Matched", + "build_index", + "extract_paragraphs", + "find_all_matches", + "load_thesaurus", + "replace_with_links", +] diff --git a/crates/terraphim_automata_py/python/terraphim_automata/__init__.pyi b/crates/terraphim_automata_py/python/terraphim_automata/__init__.pyi new file mode 100644 index 000000000..d041e3589 --- /dev/null +++ b/crates/terraphim_automata_py/python/terraphim_automata/__init__.pyi @@ -0,0 +1,222 @@ +"""Type stubs for terraphim_automata""" + +from typing import List, Optional, Tuple + +class AutocompleteResult: + """Result from autocomplete search""" + + term: str + normalized_term: str + id: int + url: Optional[str] + score: float + + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + +class Matched: + """Matched term in text""" + + term: str + normalized_term: str + id: int + url: Optional[str] + pos: Optional[Tuple[int, int]] + + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + +class AutocompleteIndex: + """Autocomplete index for fast prefix searches""" + + @property + def name(self) -> str: + """Get the name of the autocomplete index""" + ... + + def __len__(self) -> int: + """Get the number of terms in the index""" + ... + + def search(self, prefix: str, max_results: int = 10) -> List[AutocompleteResult]: + """ + Search for terms matching the prefix + + Args: + prefix: The search prefix + max_results: Maximum number of results to return (default: 10) + + Returns: + List of AutocompleteResult objects + + Note: + Case sensitivity is determined when the index is built + """ + ... + + def fuzzy_search( + self, query: str, threshold: float = 0.8, max_results: int = 10 + ) -> List[AutocompleteResult]: + """ + Fuzzy search using Jaro-Winkler similarity + + Args: + query: The search query + threshold: Similarity threshold (0.0 to 1.0, default: 0.8) + max_results: Maximum number of results (default: 10) + + Returns: + List of AutocompleteResult objects sorted by relevance + """ + ... + + def fuzzy_search_levenshtein( + self, query: str, max_distance: int = 2, max_results: int = 10 + ) -> List[AutocompleteResult]: + """ + Fuzzy search using Levenshtein distance + + Args: + query: The search query + max_distance: Maximum edit distance (default: 2) + max_results: Maximum number of results (default: 10) + + Returns: + List of AutocompleteResult objects sorted by relevance + """ + ... + + def serialize(self) -> bytes: + """ + Serialize the index to bytes for caching + + Returns: + Bytes representation of the index + + Example: + >>> index = build_index(thesaurus_json) + >>> data = index.serialize() + >>> # Save to file + >>> with open("index.bin", "wb") as f: + ... f.write(data) + """ + ... + + @staticmethod + def deserialize(data: bytes) -> "AutocompleteIndex": + """ + Deserialize an index from bytes + + Args: + data: Bytes representation of the index + + Returns: + AutocompleteIndex object + + Example: + >>> # Load from file + >>> with open("index.bin", "rb") as f: + ... data = f.read() + >>> index = AutocompleteIndex.deserialize(data) + """ + ... + + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + +def load_thesaurus(json_str: str) -> Tuple[str, int]: + """ + Load thesaurus from JSON string + + Args: + json_str: JSON string containing thesaurus data + + Returns: + Tuple of (thesaurus_name, number_of_terms) + + Example: + >>> json_str = '{"name": "test", "data": {"term1": {"id": 1, "nterm": "normalized", "url": "https://example.com"}}}' + >>> name, count = load_thesaurus(json_str) + """ + ... + +def build_index(json_str: str, case_sensitive: bool = False) -> AutocompleteIndex: + """ + Build autocomplete index from thesaurus JSON + + Args: + json_str: JSON string containing thesaurus data + case_sensitive: Whether the index should be case-sensitive (default: False) + + Returns: + AutocompleteIndex object + + Example: + >>> json_str = '{"name": "test", "data": {"term1": {"id": 1, "nterm": "normalized", "url": "https://example.com"}}}' + >>> index = build_index(json_str) + >>> results = index.search("ter") + """ + ... + +def find_all_matches( + text: str, json_str: str, return_positions: bool = True +) -> List[Matched]: + """ + Find all matches of thesaurus terms in text + + Args: + text: The text to search in + json_str: JSON string containing thesaurus data + return_positions: Whether to return match positions (default: True) + + Returns: + List of Matched objects + + Example: + >>> text = "This is a test document with some terms" + >>> json_str = '{"name": "test", "data": {"test": {"id": 1, "nterm": "test", "url": "https://example.com"}}}' + >>> matches = find_all_matches(text, json_str) + """ + ... + +def replace_with_links(text: str, json_str: str, link_type: str) -> str: + """ + Replace all thesaurus term matches with links + + Args: + text: The text to process + json_str: JSON string containing thesaurus data + link_type: Type of links to create ('wiki', 'html', 'markdown', 'plain') + + Returns: + String with replaced links + + Example: + >>> text = "This is a test" + >>> json_str = '{"name": "test", "data": {"test": {"id": 1, "nterm": "test", "url": "https://example.com"}}}' + >>> result = replace_with_links(text, json_str, "markdown") + """ + ... + +def extract_paragraphs( + text: str, json_str: str, include_term: bool = True +) -> List[Tuple[str, str]]: + """ + Extract paragraphs starting at matched terms + + Args: + text: The text to process + json_str: JSON string containing thesaurus data + include_term: Whether to include the matched term in the paragraph (default: True) + + Returns: + List of tuples (term, paragraph_text) + + Example: + >>> text = "Paragraph one.\\n\\nParagraph two with term.\\n\\nParagraph three." + >>> json_str = '{"name": "test", "data": {"term": {"id": 1, "nterm": "term", "url": ""}}}' + >>> paragraphs = extract_paragraphs(text, json_str) + """ + ... + +__version__: str diff --git a/crates/terraphim_automata_py/python/tests/conftest.py b/crates/terraphim_automata_py/python/tests/conftest.py new file mode 100644 index 000000000..eaa25410e --- /dev/null +++ b/crates/terraphim_automata_py/python/tests/conftest.py @@ -0,0 +1,74 @@ +"""Pytest configuration and shared fixtures""" + +import json + +import pytest + + +@pytest.fixture +def sample_thesaurus_json(): + """Sample thesaurus JSON for testing""" + return """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://example.com/ml" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://example.com/dl" + }, + "artificial intelligence": { + "id": 3, + "nterm": "artificial intelligence", + "url": "https://example.com/ai" + }, + "neural network": { + "id": 4, + "nterm": "neural network", + "url": "https://example.com/nn" + }, + "natural language processing": { + "id": 5, + "nterm": "natural language processing", + "url": "https://example.com/nlp" + } + } + }""" + + +@pytest.fixture +def sample_text(): + """Sample text for testing""" + return """ + Machine learning is a subset of artificial intelligence that enables + systems to learn and improve from experience without being explicitly + programmed. + + Deep learning is a type of machine learning based on artificial neural + networks. Neural network architectures have multiple layers that can + learn complex patterns. + + Natural language processing is another important area of artificial + intelligence that deals with human language. + """ + + +@pytest.fixture +def empty_thesaurus_json(): + """Empty thesaurus for testing edge cases""" + return '{"name": "Empty", "data": {}}' + + +@pytest.fixture +def large_thesaurus_json(): + """Generate a large thesaurus for performance testing""" + data = {} + for i in range(1000): + term = f"term {i} with multiple words" + data[term] = {"id": i + 1, "nterm": f"normalized_{i}", "url": f"https://example.com/{i}"} + + return json.dumps({"name": "Large", "data": data}) diff --git a/crates/terraphim_automata_py/python/tests/test_autocomplete.py b/crates/terraphim_automata_py/python/tests/test_autocomplete.py new file mode 100644 index 000000000..06e2cf503 --- /dev/null +++ b/crates/terraphim_automata_py/python/tests/test_autocomplete.py @@ -0,0 +1,268 @@ +"""Tests for autocomplete functionality""" + +import pytest + +from terraphim_automata import build_index + +# Sample thesaurus for testing +SAMPLE_THESAURUS = """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://example.com/ml" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://example.com/dl" + }, + "reinforcement learning": { + "id": 3, + "nterm": "reinforcement learning", + "url": "https://example.com/rl" + }, + "natural language processing": { + "id": 4, + "nterm": "natural language processing", + "url": "https://example.com/nlp" + }, + "computer vision": { + "id": 5, + "nterm": "computer vision", + "url": "https://example.com/cv" + } + } +}""" + + +@pytest.fixture +def index(): + """Create an autocomplete index for testing""" + return build_index(SAMPLE_THESAURUS) + + +class TestAutocompleteIndex: + """Test AutocompleteIndex class""" + + def test_index_creation(self, index): + """Test that index is created successfully""" + assert index is not None + assert index.name == "Engineering" + assert len(index) == 5 + + def test_index_length(self, index): + """Test __len__ method""" + assert len(index) == 5 + + def test_index_repr(self, index): + """Test __repr__ method""" + repr_str = repr(index) + assert "AutocompleteIndex" in repr_str + assert "Engineering" in repr_str + assert "5" in repr_str + + def test_search_exact_prefix(self, index): + """Test searching with exact prefix""" + results = index.search("machine") + assert len(results) > 0 + assert any(r.term == "machine learning" for r in results) + + def test_search_partial_prefix(self, index): + """Test searching with partial prefix""" + results = index.search("learn") + assert len(results) >= 3 # machine learning, deep learning, reinforcement learning + terms = [r.term for r in results] + assert "machine learning" in terms + assert "deep learning" in terms + assert "reinforcement learning" in terms + + def test_search_case_insensitive(self, index): + """Test case-insensitive search (default)""" + results_lower = index.search("machine") + results_upper = index.search("MACHINE") + assert len(results_lower) == len(results_upper) + assert results_lower[0].term == results_upper[0].term + + def test_search_case_sensitive(self): + """Test case-sensitive search""" + index = build_index(SAMPLE_THESAURUS, case_sensitive=True) + results_lower = index.search("machine") + results_upper = index.search("MACHINE") + assert len(results_lower) > 0 + assert len(results_upper) == 0 # No uppercase terms in thesaurus + + def test_search_max_results(self, index): + """Test max_results parameter""" + results = index.search("", max_results=3) + assert len(results) <= 3 + + def test_search_no_results(self, index): + """Test search with no matching results""" + results = index.search("xyz123") + assert len(results) == 0 + + def test_search_empty_prefix(self, index): + """Test search with empty prefix""" + results = index.search("", max_results=10) + # Empty prefix should return some results (all terms) + assert len(results) > 0 + + +class TestAutocompleteResult: + """Test AutocompleteResult class""" + + def test_result_attributes(self, index): + """Test that result has all expected attributes""" + results = index.search("machine") + assert len(results) > 0 + + result = results[0] + assert hasattr(result, "term") + assert hasattr(result, "normalized_term") + assert hasattr(result, "id") + assert hasattr(result, "url") + assert hasattr(result, "score") + + def test_result_values(self, index): + """Test result attribute values""" + results = index.search("machine learning") + assert len(results) > 0 + + result = next((r for r in results if r.term == "machine learning"), None) + assert result is not None + assert result.term == "machine learning" + assert result.normalized_term == "machine learning" + assert result.id == 1 + assert result.url == "https://example.com/ml" + assert isinstance(result.score, float) + + def test_result_repr(self, index): + """Test result __repr__ method""" + results = index.search("machine") + assert len(results) > 0 + + result = results[0] + repr_str = repr(result) + assert "AutocompleteResult" in repr_str + assert result.term in repr_str + + +class TestFuzzySearch: + """Test fuzzy search functionality""" + + def test_fuzzy_search_jaro_winkler(self, index): + """Test fuzzy search with Jaro-Winkler""" + results = index.fuzzy_search("machin", threshold=0.8) + assert len(results) > 0 + assert any("machine" in r.term for r in results) + + def test_fuzzy_search_threshold(self, index): + """Test fuzzy search with different thresholds""" + results_high = index.fuzzy_search("machin", threshold=0.95) + results_low = index.fuzzy_search("machin", threshold=0.5) + # Lower threshold should return more results + assert len(results_low) >= len(results_high) + + def test_fuzzy_search_levenshtein(self, index): + """Test fuzzy search with Levenshtein distance""" + results = index.fuzzy_search_levenshtein("macine", max_distance=2) + assert len(results) > 0 + assert any("machine" in r.term for r in results) + + def test_fuzzy_search_max_distance(self, index): + """Test fuzzy search with different max distances""" + results_small = index.fuzzy_search_levenshtein("macine", max_distance=1) + results_large = index.fuzzy_search_levenshtein("macine", max_distance=3) + # Larger distance should return more results + assert len(results_large) >= len(results_small) + + def test_fuzzy_search_no_results(self, index): + """Test fuzzy search with no matching results""" + results = index.fuzzy_search("xyz123", threshold=0.8) + assert len(results) == 0 + + def test_fuzzy_search_max_results(self, index): + """Test fuzzy search max_results parameter""" + results = index.fuzzy_search("learn", threshold=0.5, max_results=2) + assert len(results) <= 2 + + +class TestSerialization: + """Test index serialization and deserialization""" + + def test_serialize_and_deserialize(self, index): + """Test that an index can be serialized and deserialized""" + from terraphim_automata import AutocompleteIndex + + # Serialize the index + serialized = index.serialize() + assert isinstance(serialized, bytes) + assert len(serialized) > 0 + + # Deserialize to a new index + deserialized = AutocompleteIndex.deserialize(serialized) + assert deserialized.name == index.name + assert len(deserialized) == len(index) + + def test_deserialized_index_works(self, index): + """Test that a deserialized index works correctly""" + from terraphim_automata import AutocompleteIndex + + # Serialize and deserialize + serialized = index.serialize() + deserialized = AutocompleteIndex.deserialize(serialized) + + # Test search works on both + original_results = index.search("machine") + deserialized_results = deserialized.search("machine") + + assert len(original_results) == len(deserialized_results) + for orig, deser in zip(original_results, deserialized_results): + assert orig.term == deser.term + assert orig.id == deser.id + + def test_roundtrip_preserves_data(self): + """Test that serialize->deserialize preserves all data""" + from terraphim_automata import AutocompleteIndex + + # Build a small index + thesaurus = """{ + "name": "Test", + "data": { + "test term": {"id": 42, "nterm": "test", "url": "https://example.com"} + } + }""" + original = build_index(thesaurus) + + # Serialize and deserialize + serialized = original.serialize() + restored = AutocompleteIndex.deserialize(serialized) + + # Verify data is preserved + results = restored.search("test") + assert len(results) == 1 + assert results[0].term == "test term" + assert results[0].id == 42 + assert results[0].url == "https://example.com" + + +class TestErrorHandling: + """Test error handling""" + + def test_invalid_json(self): + """Test building index with invalid JSON""" + with pytest.raises(ValueError) as excinfo: + build_index("{invalid json}") + assert "Failed to load thesaurus" in str(excinfo.value) + + def test_empty_json(self): + """Test building index with empty/minimal JSON""" + minimal = '{"name": "Empty", "data": {}}' + index = build_index(minimal) + assert len(index) == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/crates/terraphim_automata_py/python/tests/test_matcher.py b/crates/terraphim_automata_py/python/tests/test_matcher.py new file mode 100644 index 000000000..4861deea9 --- /dev/null +++ b/crates/terraphim_automata_py/python/tests/test_matcher.py @@ -0,0 +1,260 @@ +"""Tests for text matching and replacement functionality""" + +import pytest + +from terraphim_automata import extract_paragraphs, find_all_matches, replace_with_links + +# Sample thesaurus for testing +SAMPLE_THESAURUS = """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://example.com/ml" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://example.com/dl" + }, + "artificial intelligence": { + "id": 3, + "nterm": "artificial intelligence", + "url": "https://example.com/ai" + } + } +}""" + + +class TestFindMatches: + """Test find_all_matches function""" + + def test_find_single_match(self): + """Test finding a single match""" + text = "I am studying machine learning." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 1 + assert matches[0].term == "machine learning" + assert matches[0].id == 1 + assert matches[0].url == "https://example.com/ml" + + def test_find_multiple_matches(self): + """Test finding multiple matches""" + text = "Machine learning and deep learning are part of artificial intelligence." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 3 + terms = [m.term for m in matches] + assert "machine learning" in terms + assert "deep learning" in terms + assert "artificial intelligence" in terms + + def test_find_matches_case_insensitive(self): + """Test that matching is case-insensitive""" + text = "MACHINE LEARNING and Machine Learning are the same." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 2 + assert all(m.term == "machine learning" for m in matches) + + def test_find_matches_with_positions(self): + """Test finding matches with positions""" + text = "I study machine learning daily." + matches = find_all_matches(text, SAMPLE_THESAURUS, return_positions=True) + + assert len(matches) == 1 + assert matches[0].pos is not None + start, end = matches[0].pos + assert text[start:end].lower() == "machine learning" + + def test_find_matches_without_positions(self): + """Test finding matches without positions""" + text = "I study machine learning daily." + matches = find_all_matches(text, SAMPLE_THESAURUS, return_positions=False) + + assert len(matches) == 1 + assert matches[0].pos is None + + def test_find_no_matches(self): + """Test text with no matches""" + text = "This text has no matching terms." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 0 + + def test_match_attributes(self): + """Test that matches have all expected attributes""" + text = "I study machine learning." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 1 + match = matches[0] + assert hasattr(match, "term") + assert hasattr(match, "normalized_term") + assert hasattr(match, "id") + assert hasattr(match, "url") + assert hasattr(match, "pos") + + def test_match_repr(self): + """Test match __repr__ method""" + text = "I study machine learning." + matches = find_all_matches(text, SAMPLE_THESAURUS) + + assert len(matches) == 1 + repr_str = repr(matches[0]) + assert "Matched" in repr_str + assert "machine learning" in repr_str + + +class TestReplaceWithLinks: + """Test replace_with_links function""" + + def test_replace_markdown_links(self): + """Test replacing with markdown links""" + text = "I study machine learning and deep learning." + result = replace_with_links(text, SAMPLE_THESAURUS, "markdown") + + assert "[machine learning](https://example.com/ml)" in result + assert "[deep learning](https://example.com/dl)" in result + + def test_replace_html_links(self): + """Test replacing with HTML links""" + text = "I study machine learning." + result = replace_with_links(text, SAMPLE_THESAURUS, "html") + + assert 'machine learning' in result + + def test_replace_wiki_links(self): + """Test replacing with wiki-style links""" + text = "I study machine learning." + result = replace_with_links(text, SAMPLE_THESAURUS, "wiki") + + assert "[[machine learning]]" in result + + def test_replace_plain_text(self): + """Test replacing with plain normalized text""" + text = "I study machine learning." + result = replace_with_links(text, SAMPLE_THESAURUS, "plain") + + # Should still have the term but normalized + assert "machine learning" in result + + def test_replace_case_insensitive(self): + """Test that replacement is case-insensitive""" + text = "I study MACHINE LEARNING." + result = replace_with_links(text, SAMPLE_THESAURUS, "markdown") + + assert "[machine learning](https://example.com/ml)" in result + + def test_replace_multiple_terms(self): + """Test replacing multiple terms""" + text = "Machine learning and deep learning are parts of artificial intelligence." + result = replace_with_links(text, SAMPLE_THESAURUS, "markdown") + + assert "[machine learning](https://example.com/ml)" in result + assert "[deep learning](https://example.com/dl)" in result + assert "[artificial intelligence](https://example.com/ai)" in result + + def test_replace_no_matches(self): + """Test text with no matches""" + text = "This text has no matching terms." + result = replace_with_links(text, SAMPLE_THESAURUS, "markdown") + + # Text should remain unchanged + assert result == text + + def test_replace_invalid_link_type(self): + """Test with invalid link type""" + text = "I study machine learning." + with pytest.raises(ValueError) as excinfo: + replace_with_links(text, SAMPLE_THESAURUS, "invalid") + + assert "Invalid link type" in str(excinfo.value) + + +class TestExtractParagraphs: + """Test extract_paragraphs function""" + + def test_extract_single_paragraph(self): + """Test extracting a single paragraph""" + text = """This is the first paragraph. + +This paragraph contains machine learning which is interesting. + +This is the third paragraph.""" + + paragraphs = extract_paragraphs(text, SAMPLE_THESAURUS) + + assert len(paragraphs) > 0 + # Find the paragraph with machine learning + ml_paragraph = next((p for p in paragraphs if p[0] == "machine learning"), None) + assert ml_paragraph is not None + assert "machine learning" in ml_paragraph[1] + + def test_extract_multiple_paragraphs(self): + """Test extracting multiple paragraphs with different terms""" + text = """Paragraph about machine learning. + +Another paragraph about deep learning. + +Final paragraph about artificial intelligence.""" + + paragraphs = extract_paragraphs(text, SAMPLE_THESAURUS) + + assert len(paragraphs) >= 3 + terms = [p[0] for p in paragraphs] + assert "machine learning" in terms + assert "deep learning" in terms + assert "artificial intelligence" in terms + + def test_extract_paragraph_content(self): + """Test that extracted paragraphs contain correct content""" + text = """First paragraph. + +This paragraph is about machine learning and its applications. +It continues on multiple lines. + +Third paragraph.""" + + paragraphs = extract_paragraphs(text, SAMPLE_THESAURUS) + + ml_paragraph = next((p for p in paragraphs if p[0] == "machine learning"), None) + assert ml_paragraph is not None + # Paragraph should start from the matched term + assert ml_paragraph[1].startswith("machine learning") + assert "applications" in ml_paragraph[1] + + def test_extract_no_paragraphs(self): + """Test text with no matching terms""" + text = """Paragraph one. + +Paragraph two. + +Paragraph three.""" + + paragraphs = extract_paragraphs(text, SAMPLE_THESAURUS) + + assert len(paragraphs) == 0 + + +class TestErrorHandling: + """Test error handling""" + + def test_invalid_thesaurus_json(self): + """Test with invalid JSON""" + text = "Some text" + + with pytest.raises(ValueError): + find_all_matches(text, "{invalid json}") + + with pytest.raises(ValueError): + replace_with_links(text, "{invalid json}", "markdown") + + with pytest.raises(ValueError): + extract_paragraphs(text, "{invalid json}") + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/crates/terraphim_automata_py/python/tests/test_thesaurus.py b/crates/terraphim_automata_py/python/tests/test_thesaurus.py new file mode 100644 index 000000000..167d7b3a2 --- /dev/null +++ b/crates/terraphim_automata_py/python/tests/test_thesaurus.py @@ -0,0 +1,274 @@ +"""Tests for thesaurus loading functionality""" + +import pytest + +from terraphim_automata import build_index, load_thesaurus + + +class TestLoadThesaurus: + """Test load_thesaurus function""" + + def test_load_simple_thesaurus(self): + """Test loading a simple thesaurus""" + json_str = """{ + "name": "Test", + "data": { + "term1": { + "id": 1, + "nterm": "normalized1", + "url": "https://example.com/1" + }, + "term2": { + "id": 2, + "nterm": "normalized2", + "url": "https://example.com/2" + } + } + }""" + + name, count = load_thesaurus(json_str) + assert name == "Test" + assert count == 2 + + def test_load_empty_thesaurus(self): + """Test loading an empty thesaurus""" + json_str = '{"name": "Empty", "data": {}}' + + name, count = load_thesaurus(json_str) + assert name == "Empty" + assert count == 0 + + def test_load_large_thesaurus(self): + """Test loading a larger thesaurus""" + # Generate a thesaurus with many terms + data = {f"term{i}": {"id": i, "nterm": f"normalized{i}", "url": f"https://example.com/{i}"} + for i in range(1, 101)} + import json + + json_str = json.dumps({"name": "Large", "data": data}) + + name, count = load_thesaurus(json_str) + assert name == "Large" + assert count == 100 + + def test_load_thesaurus_with_optional_fields(self): + """Test loading thesaurus with optional fields""" + json_str = """{ + "name": "Test", + "data": { + "term1": { + "id": 1, + "nterm": "normalized1" + } + } + }""" + + name, count = load_thesaurus(json_str) + assert name == "Test" + assert count == 1 + + def test_load_invalid_json(self): + """Test loading invalid JSON""" + with pytest.raises(ValueError) as excinfo: + load_thesaurus("{invalid json}") + assert "Failed to load thesaurus" in str(excinfo.value) + + def test_load_malformed_thesaurus(self): + """Test loading malformed thesaurus structure""" + # Missing required fields + json_str = '{"name": "Test"}' + + with pytest.raises(ValueError): + load_thesaurus(json_str) + + +class TestBuildIndex: + """Test build_index function""" + + def test_build_simple_index(self): + """Test building a simple index""" + json_str = """{ + "name": "Test", + "data": { + "term1": {"id": 1, "nterm": "normalized1", "url": "https://example.com/1"} + } + }""" + + index = build_index(json_str) + assert index is not None + assert index.name == "Test" + assert len(index) == 1 + + def test_build_case_sensitive_index(self): + """Test building a case-sensitive index""" + json_str = """{ + "name": "Test", + "data": { + "Test": {"id": 1, "nterm": "test", "url": "https://example.com/1"} + } + }""" + + index = build_index(json_str, case_sensitive=True) + results = index.search("Test") + assert len(results) > 0 + + results = index.search("test") + assert len(results) == 0 # Case-sensitive, won't match + + def test_build_case_insensitive_index(self): + """Test building a case-insensitive index (default)""" + json_str = """{ + "name": "Test", + "data": { + "Test": {"id": 1, "nterm": "test", "url": "https://example.com/1"} + } + }""" + + index = build_index(json_str, case_sensitive=False) + results_upper = index.search("Test") + results_lower = index.search("test") + assert len(results_upper) > 0 + assert len(results_lower) > 0 + + def test_build_index_preserves_data(self): + """Test that index preserves all thesaurus data""" + json_str = """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 42, + "nterm": "ml", + "url": "https://example.com/ml" + } + } + }""" + + index = build_index(json_str) + results = index.search("machine") + + assert len(results) > 0 + result = results[0] + assert result.id == 42 + assert result.normalized_term == "ml" + assert result.url == "https://example.com/ml" + + def test_build_index_with_special_characters(self): + """Test building index with special characters in terms""" + json_str = """{ + "name": "Test", + "data": { + "C++": {"id": 1, "nterm": "cpp", "url": "https://example.com/cpp"}, + "C#": {"id": 2, "nterm": "csharp", "url": "https://example.com/csharp"}, + "node.js": {"id": 3, "nterm": "nodejs", "url": "https://example.com/nodejs"} + } + }""" + + index = build_index(json_str) + assert len(index) == 3 + + # Test searching for terms with special characters + results = index.search("C") + assert len(results) > 0 + + def test_build_index_with_unicode(self): + """Test building index with Unicode characters""" + json_str = """{ + "name": "Test", + "data": { + "cafΓ©": {"id": 1, "nterm": "cafe", "url": "https://example.com/cafe"}, + "naΓ―ve": {"id": 2, "nterm": "naive", "url": "https://example.com/naive"} + } + }""" + + index = build_index(json_str) + assert len(index) == 2 + + results = index.search("caf") + assert len(results) > 0 + + +class TestIntegration: + """Integration tests combining multiple functions""" + + def test_load_and_build_workflow(self): + """Test the typical workflow: load thesaurus, build index, search""" + json_str = """{ + "name": "Engineering", + "data": { + "machine learning": { + "id": 1, + "nterm": "machine learning", + "url": "https://example.com/ml" + }, + "deep learning": { + "id": 2, + "nterm": "deep learning", + "url": "https://example.com/dl" + } + } + }""" + + # Load thesaurus + name, count = load_thesaurus(json_str) + assert name == "Engineering" + assert count == 2 + + # Build index + index = build_index(json_str) + assert index.name == name + assert len(index) == count + + # Search + results = index.search("learn") + assert len(results) == 2 + + def test_real_world_thesaurus(self): + """Test with a realistic thesaurus structure""" + json_str = """{ + "name": "Software Engineering", + "data": { + "continuous integration": { + "id": 1, + "nterm": "continuous integration", + "url": "https://example.com/ci" + }, + "continuous deployment": { + "id": 2, + "nterm": "continuous deployment", + "url": "https://example.com/cd" + }, + "test driven development": { + "id": 3, + "nterm": "test driven development", + "url": "https://example.com/tdd" + }, + "behavior driven development": { + "id": 4, + "nterm": "behavior driven development", + "url": "https://example.com/bdd" + }, + "domain driven design": { + "id": 5, + "nterm": "domain driven design", + "url": "https://example.com/ddd" + } + } + }""" + + index = build_index(json_str) + + # Test prefix search + results = index.search("continuous") + assert len(results) == 2 + + # Test suffix search + results = index.search("driven") + assert len(results) == 3 + + # Test fuzzy search + results = index.fuzzy_search("continuos", threshold=0.85) + assert len(results) > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/crates/terraphim_automata_py/scripts/build_and_test.sh b/crates/terraphim_automata_py/scripts/build_and_test.sh new file mode 100755 index 000000000..d4ab36d08 --- /dev/null +++ b/crates/terraphim_automata_py/scripts/build_and_test.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Build and test script for terraphim_automata Python bindings + +set -e # Exit on error + +echo "==================================================" +echo "Terraphim Automata Python Bindings - Build & Test" +echo "==================================================" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Function to print colored output +print_status() { + echo -e "${GREEN}βœ“${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_error() { + echo -e "${RED}βœ—${NC} $1" +} + +# Check if uv is installed +if ! command -v uv &> /dev/null; then + print_error "uv is not installed. Please install it first:" + echo " pip install uv" + exit 1 +fi + +print_status "uv is installed" + +# Navigate to project directory +cd "$(dirname "$0")/.." + +# Step 1: Install maturin +echo -e "\n${YELLOW}[1/7]${NC} Installing maturin..." +uv pip install --system maturin 2>&1 | grep -v "Requirement already satisfied" || true +print_status "Maturin installed" + +# Step 2: Build the Rust extension +echo -e "\n${YELLOW}[2/7]${NC} Building Rust extension..." +if maturin develop; then + print_status "Rust extension built successfully" +else + print_error "Failed to build Rust extension" + exit 1 +fi + +# Step 3: Install test dependencies +echo -e "\n${YELLOW}[3/7]${NC} Installing test dependencies..." +uv pip install --system pytest pytest-cov pytest-benchmark 2>&1 | grep -v "Requirement already satisfied" || true +print_status "Test dependencies installed" + +# Step 4: Run tests +echo -e "\n${YELLOW}[4/7]${NC} Running tests..." +if pytest python/tests/ -v --cov=terraphim_automata --cov-report=term-missing; then + print_status "All tests passed" +else + print_error "Some tests failed" + exit 1 +fi + +# Step 5: Run benchmarks (optional) +echo -e "\n${YELLOW}[5/7]${NC} Running benchmarks (optional)..." +read -p "Run performance benchmarks? (y/N) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + pytest python/benchmarks/ -v --benchmark-only --benchmark-columns=min,max,mean,stddev,median,ops + print_status "Benchmarks completed" +else + print_warning "Benchmarks skipped" +fi + +# Step 6: Install dev dependencies for code quality +echo -e "\n${YELLOW}[6/7]${NC} Installing code quality tools..." +uv pip install --system black ruff mypy 2>&1 | grep -v "Requirement already satisfied" || true +print_status "Code quality tools installed" + +# Step 7: Check code quality +echo -e "\n${YELLOW}[7/7]${NC} Running code quality checks..." + +echo " β€’ Black (formatting)..." +if black --check python/ 2>&1 | tail -1; then + print_status "Code is properly formatted" +else + print_warning "Code needs formatting (run: black python/)" +fi + +echo " β€’ Ruff (linting)..." +if ruff check python/ 2>&1 | tail -5; then + print_status "No linting issues" +else + print_warning "Linting issues found (run: ruff check python/ --fix)" +fi + +echo " β€’ Mypy (type checking)..." +if mypy python/terraphim_automata/ --ignore-missing-imports 2>&1 | tail -3; then + print_status "Type checking passed" +else + print_warning "Type checking found issues" +fi + +# Final summary +echo -e "\n==================================================" +echo -e "${GREEN}Build and test completed successfully!${NC}" +echo "==================================================" +echo "" +echo "Next steps:" +echo " β€’ Run examples: python examples/basic_autocomplete.py" +echo " β€’ Build release: maturin build --release" +echo " β€’ Build wheels: maturin build --release --out dist" +echo "" diff --git a/crates/terraphim_automata_py/src/lib.rs b/crates/terraphim_automata_py/src/lib.rs new file mode 100644 index 000000000..2c242b431 --- /dev/null +++ b/crates/terraphim_automata_py/src/lib.rs @@ -0,0 +1,408 @@ +use pyo3::prelude::*; +use pyo3::exceptions::{PyValueError, PyRuntimeError}; +use ::terraphim_automata::autocomplete::{ + autocomplete_search, build_autocomplete_index, deserialize_autocomplete_index, + fuzzy_autocomplete_search, fuzzy_autocomplete_search_levenshtein, serialize_autocomplete_index, + AutocompleteConfig, AutocompleteIndex, AutocompleteResult, +}; +use ::terraphim_automata::matcher::{ + extract_paragraphs_from_automata, find_matches, LinkType, Matched, +}; +use ::terraphim_automata::{load_thesaurus_from_json, load_thesaurus_from_json_and_replace}; + +/// Python wrapper for AutocompleteIndex +#[pyclass(name = "AutocompleteIndex")] +pub struct PyAutocompleteIndex { + inner: AutocompleteIndex, +} + +/// Python wrapper for AutocompleteResult +#[pyclass(name = "AutocompleteResult")] +#[derive(Clone)] +pub struct PyAutocompleteResult { + #[pyo3(get)] + pub term: String, + #[pyo3(get)] + pub normalized_term: String, + #[pyo3(get)] + pub id: u64, + #[pyo3(get)] + pub url: Option, + #[pyo3(get)] + pub score: f64, +} + +impl From for PyAutocompleteResult { + fn from(result: AutocompleteResult) -> Self { + PyAutocompleteResult { + term: result.term, + normalized_term: result.normalized_term.to_string(), + id: result.id, + url: result.url, + score: result.score, + } + } +} + +#[pymethods] +impl PyAutocompleteResult { + fn __repr__(&self) -> String { + format!( + "AutocompleteResult(term='{}', normalized_term='{}', id={}, score={})", + self.term, self.normalized_term, self.id, self.score + ) + } + + fn __str__(&self) -> String { + self.__repr__() + } +} + +/// Python wrapper for Matched +#[pyclass(name = "Matched")] +#[derive(Clone)] +pub struct PyMatched { + #[pyo3(get)] + pub term: String, + #[pyo3(get)] + pub normalized_term: String, + #[pyo3(get)] + pub id: u64, + #[pyo3(get)] + pub url: Option, + #[pyo3(get)] + pub pos: Option<(usize, usize)>, +} + +impl From for PyMatched { + fn from(matched: Matched) -> Self { + PyMatched { + term: matched.term, + normalized_term: matched.normalized_term.value.to_string(), + id: matched.normalized_term.id, + url: matched.normalized_term.url, + pos: matched.pos, + } + } +} + +#[pymethods] +impl PyMatched { + fn __repr__(&self) -> String { + format!( + "Matched(term='{}', normalized_term='{}', id={}, pos={:?})", + self.term, self.normalized_term, self.id, self.pos + ) + } + + fn __str__(&self) -> String { + self.__repr__() + } +} + +#[pymethods] +impl PyAutocompleteIndex { + /// Get the name of the autocomplete index + #[getter] + fn name(&self) -> String { + self.inner.name().to_string() + } + + /// Get the number of terms in the index + fn __len__(&self) -> usize { + self.inner.len() + } + + /// Search for terms matching the prefix + /// + /// Args: + /// prefix: The search prefix + /// max_results: Maximum number of results to return (default: 10) + /// + /// Returns: + /// List of AutocompleteResult objects + /// + /// Note: + /// Case sensitivity is determined when the index is built + #[pyo3(signature = (prefix, max_results=10))] + fn search( + &self, + prefix: &str, + max_results: usize, + ) -> PyResult> { + let results = autocomplete_search(&self.inner, prefix, Some(max_results)) + .map_err(|e| PyValueError::new_err(format!("Search error: {}", e)))?; + + Ok(results.into_iter().map(PyAutocompleteResult::from).collect()) + } + + /// Fuzzy search using Jaro-Winkler similarity + /// + /// Args: + /// query: The search query + /// threshold: Similarity threshold (0.0 to 1.0, default: 0.8) + /// max_results: Maximum number of results (default: 10) + /// + /// Returns: + /// List of AutocompleteResult objects sorted by relevance + #[pyo3(signature = (query, threshold=0.8, max_results=10))] + fn fuzzy_search( + &self, + query: &str, + threshold: f64, + max_results: usize, + ) -> PyResult> { + let results = fuzzy_autocomplete_search(&self.inner, query, threshold, Some(max_results)) + .map_err(|e| PyValueError::new_err(format!("Fuzzy search error: {}", e)))?; + + Ok(results.into_iter().map(PyAutocompleteResult::from).collect()) + } + + /// Fuzzy search using Levenshtein distance + /// + /// Args: + /// query: The search query + /// max_distance: Maximum edit distance (default: 2) + /// max_results: Maximum number of results (default: 10) + /// + /// Returns: + /// List of AutocompleteResult objects sorted by relevance + #[pyo3(signature = (query, max_distance=2, max_results=10))] + fn fuzzy_search_levenshtein( + &self, + query: &str, + max_distance: usize, + max_results: usize, + ) -> PyResult> { + let results = fuzzy_autocomplete_search_levenshtein( + &self.inner, + query, + max_distance, + Some(max_results), + ) + .map_err(|e| PyValueError::new_err(format!("Fuzzy search error: {}", e)))?; + + Ok(results.into_iter().map(PyAutocompleteResult::from).collect()) + } + + /// Serialize the index to bytes for caching + /// + /// Returns: + /// Bytes representation of the index + /// + /// Example: + /// >>> index = build_index(thesaurus_json) + /// >>> data = index.serialize() + /// >>> # Save to file + /// >>> with open("index.bin", "wb") as f: + /// ... f.write(data) + fn serialize(&self) -> PyResult> { + serialize_autocomplete_index(&self.inner) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to serialize index: {}", e))) + } + + /// Deserialize an index from bytes + /// + /// Args: + /// data: Bytes representation of the index + /// + /// Returns: + /// AutocompleteIndex object + /// + /// Example: + /// >>> # Load from file + /// >>> with open("index.bin", "rb") as f: + /// ... data = f.read() + /// >>> index = AutocompleteIndex.deserialize(data) + #[staticmethod] + fn deserialize(data: &[u8]) -> PyResult { + let index = deserialize_autocomplete_index(data) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to deserialize index: {}", e)))?; + Ok(PyAutocompleteIndex { inner: index }) + } + + fn __repr__(&self) -> String { + format!( + "AutocompleteIndex(name='{}', len={})", + self.inner.name(), + self.inner.len() + ) + } + + fn __str__(&self) -> String { + self.__repr__() + } +} + +/// Load thesaurus from JSON string +/// +/// Args: +/// json_str: JSON string containing thesaurus data +/// +/// Returns: +/// Tuple of (thesaurus_name, number_of_terms) +/// +/// Example: +/// >>> json_str = '{"name": "test", "data": {"term1": {"id": 1, "nterm": "normalized", "url": "https://example.com"}}}' +/// >>> name, count = load_thesaurus(json_str) +#[pyfunction] +fn load_thesaurus(json_str: &str) -> PyResult<(String, usize)> { + let thesaurus = load_thesaurus_from_json(json_str) + .map_err(|e| PyValueError::new_err(format!("Failed to load thesaurus: {}", e)))?; + + let name = thesaurus.name().to_string(); + let count = thesaurus.len(); + + Ok((name, count)) +} + +/// Build autocomplete index from thesaurus JSON +/// +/// Args: +/// json_str: JSON string containing thesaurus data +/// case_sensitive: Whether the index should be case-sensitive (default: False) +/// +/// Returns: +/// AutocompleteIndex object +/// +/// Example: +/// >>> json_str = '{"name": "test", "data": {"term1": {"id": 1, "nterm": "normalized", "url": "https://example.com"}}}' +/// >>> index = build_index(json_str) +/// >>> results = index.search("ter") +#[pyfunction] +#[pyo3(signature = (json_str, case_sensitive=false))] +fn build_index(json_str: &str, case_sensitive: bool) -> PyResult { + let thesaurus = load_thesaurus_from_json(json_str) + .map_err(|e| PyValueError::new_err(format!("Failed to load thesaurus: {}", e)))?; + + let config = AutocompleteConfig { + max_results: 10, + min_prefix_length: 1, + case_sensitive, + }; + + let index = build_autocomplete_index(thesaurus, Some(config)) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to build index: {}", e)))?; + + Ok(PyAutocompleteIndex { inner: index }) +} + +/// Find all matches of thesaurus terms in text +/// +/// Args: +/// text: The text to search in +/// json_str: JSON string containing thesaurus data +/// return_positions: Whether to return match positions (default: True) +/// +/// Returns: +/// List of Matched objects +/// +/// Example: +/// >>> text = "This is a test document with some terms" +/// >>> json_str = '{"name": "test", "data": {"test": {"id": 1, "nterm": "test", "url": "https://example.com"}}}' +/// >>> matches = find_all_matches(text, json_str) +#[pyfunction] +#[pyo3(signature = (text, json_str, return_positions=true))] +fn find_all_matches( + text: &str, + json_str: &str, + return_positions: bool, +) -> PyResult> { + let thesaurus = load_thesaurus_from_json(json_str) + .map_err(|e| PyValueError::new_err(format!("Failed to load thesaurus: {}", e)))?; + + let matches = find_matches(text, thesaurus, return_positions) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to find matches: {}", e)))?; + + Ok(matches.into_iter().map(PyMatched::from).collect()) +} + +/// Replace all thesaurus term matches with links +/// +/// Args: +/// text: The text to process +/// json_str: JSON string containing thesaurus data +/// link_type: Type of links to create ('wiki', 'html', 'markdown', 'plain') +/// +/// Returns: +/// String with replaced links +/// +/// Example: +/// >>> text = "This is a test" +/// >>> json_str = '{"name": "test", "data": {"test": {"id": 1, "nterm": "test", "url": "https://example.com"}}}' +/// >>> result = replace_with_links(text, json_str, "markdown") +#[pyfunction] +fn replace_with_links(text: &str, json_str: &str, link_type: &str) -> PyResult { + let link_type_enum = match link_type.to_lowercase().as_str() { + "wiki" => LinkType::WikiLinks, + "html" => LinkType::HTMLLinks, + "markdown" => LinkType::MarkdownLinks, + "plain" => LinkType::PlainText, + _ => { + return Err(PyValueError::new_err(format!( + "Invalid link type '{}'. Use 'wiki', 'html', 'markdown', or 'plain'", + link_type + ))) + } + }; + + // Use the Rust convenience function that loads and replaces in one step + let result = load_thesaurus_from_json_and_replace(json_str, text, link_type_enum) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to replace matches: {}", e)))?; + + String::from_utf8(result) + .map_err(|e| PyRuntimeError::new_err(format!("Invalid UTF-8: {}", e))) +} + +/// Extract paragraphs starting at matched terms +/// +/// Args: +/// text: The text to process +/// json_str: JSON string containing thesaurus data +/// include_term: Whether to include the matched term in the paragraph (default: True) +/// +/// Returns: +/// List of tuples (term, paragraph_text) +/// +/// Example: +/// >>> text = "Paragraph one.\\n\\nParagraph two with term.\\n\\nParagraph three." +/// >>> json_str = '{"name": "test", "data": {"term": {"id": 1, "nterm": "term", "url": ""}}}' +/// >>> paragraphs = extract_paragraphs(text, json_str) +#[pyfunction] +#[pyo3(signature = (text, json_str, include_term=true))] +fn extract_paragraphs(text: &str, json_str: &str, include_term: bool) -> PyResult> { + let thesaurus = load_thesaurus_from_json(json_str) + .map_err(|e| PyValueError::new_err(format!("Failed to load thesaurus: {}", e)))?; + + let paragraphs = extract_paragraphs_from_automata(text, thesaurus, include_term) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to extract paragraphs: {}", e)))?; + + // Convert Vec<(Matched, String)> to Vec<(String, String)> + let result = paragraphs + .into_iter() + .map(|(matched, paragraph)| (matched.term, paragraph)) + .collect(); + + Ok(result) +} + +/// Python module for Terraphim Automata +/// +/// This module provides fast autocomplete and text processing capabilities +/// for knowledge graphs using finite state transducers (FST) and Aho-Corasick automata. +#[pymodule] +fn terraphim_automata(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(load_thesaurus, m)?)?; + m.add_function(wrap_pyfunction!(build_index, m)?)?; + m.add_function(wrap_pyfunction!(find_all_matches, m)?)?; + m.add_function(wrap_pyfunction!(replace_with_links, m)?)?; + m.add_function(wrap_pyfunction!(extract_paragraphs, m)?)?; + + // Add module version + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + + Ok(()) +}