From 07c5d787baa99bc9ea8aa633e9181fc015cbbec5 Mon Sep 17 00:00:00 2001 From: initial-d Date: Sat, 31 Oct 2026 07:11:42 +0800 Subject: [PATCH] Make NGramStore the engine brand with compatible APIs and a working CLI --- .github/ISSUE_TEMPLATE/benchmark-result.yml | 10 +- .../ISSUE_TEMPLATE/compression-comparison.yml | 23 +- .github/ISSUE_TEMPLATE/config.yml | 6 +- .../ISSUE_TEMPLATE/evaluation-question.yml | 2 +- .github/ISSUE_TEMPLATE/showcase.yml | 6 +- .zenodo.json | 10 +- CITATION.cff | 11 +- CONTRIBUTING.md | 22 +- README.md | 573 +++--------------- docs/answer_engine_context.md | 158 ++--- docs/artifact_card.md | 130 +--- docs/citation_guide.md | 16 +- docs/engram_quickstart.md | 14 +- docs/evaluator_quickstart.md | 168 +++-- docs/fasttext_native_bundle.md | 43 +- docs/index.md | 32 + docs/legacy_fasttext.md | 44 ++ docs/native_ngram_memory.md | 6 +- examples/fasttext_sample.txt | 6 + meft_memory/__init__.py | 6 +- meft_memory/_runtime/__init__.py | 6 + ngram_store/__init__.py | 9 + ngram_store/__main__.py | 3 + ngram_store/cli.py | 156 +++++ pyproject.toml | 9 +- tests/test_fasttext_native_bundle.py | 22 + tests/test_ngram_store.py | 89 +++ tests/test_structural_native.py | 18 + tools/check_memory_wheel.py | 17 + tools/prepare_fasttext_corpus.py | 63 ++ 30 files changed, 814 insertions(+), 864 deletions(-) create mode 100644 docs/index.md create mode 100644 docs/legacy_fasttext.md create mode 100644 examples/fasttext_sample.txt create mode 100644 meft_memory/_runtime/__init__.py create mode 100644 ngram_store/__init__.py create mode 100644 ngram_store/__main__.py create mode 100644 ngram_store/cli.py create mode 100644 tests/test_ngram_store.py create mode 100644 tools/prepare_fasttext_corpus.py diff --git a/.github/ISSUE_TEMPLATE/benchmark-result.yml b/.github/ISSUE_TEMPLATE/benchmark-result.yml index 2876655..f6393fe 100644 --- a/.github/ISSUE_TEMPLATE/benchmark-result.yml +++ b/.github/ISSUE_TEMPLATE/benchmark-result.yml @@ -1,5 +1,5 @@ name: Benchmark result -description: Share a reproducible me_fasttext benchmark, replication, or negative result +description: Share a reproducible NGramStore benchmark, replication, or negative result title: "[Benchmark] " labels: - benchmark @@ -46,7 +46,7 @@ body: id: commit attributes: label: Git commit - description: The me_fasttext commit used for the run. + description: The NGramStore commit used for the run. placeholder: "abcdef1" validations: required: true @@ -125,7 +125,7 @@ body: | Method | Memory | Load time | Storage | Query p50 | Query p95 | OOV coverage | Quality | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | Original FastText | | | | | | | | - | me_fasttext | | | | | | | | + | NGramStore | | | | | | | | validations: required: true - type: textarea @@ -163,7 +163,7 @@ body: id: citation_fit attributes: label: Citation or reuse fit - description: If this result supports a paper, report, deployment note, or comparison table, describe how me_fasttext is used. + description: If this result supports a paper, report, deployment note, or comparison table, describe how NGramStore is used. placeholder: | Example: used as a compact lexical embedding layer before dense reranking; compared against hash-bucketed FastText for memory and OOV recall. @@ -171,7 +171,7 @@ body: id: notes attributes: label: Notes - description: Anything surprising, missing, hard to reproduce, or unfavorable to me_fasttext. + description: Anything surprising, missing, hard to reproduce, or unfavorable to NGramStore. - type: checkboxes id: disclosure attributes: diff --git a/.github/ISSUE_TEMPLATE/compression-comparison.yml b/.github/ISSUE_TEMPLATE/compression-comparison.yml index 04fd086..918f035 100644 --- a/.github/ISSUE_TEMPLATE/compression-comparison.yml +++ b/.github/ISSUE_TEMPLATE/compression-comparison.yml @@ -1,5 +1,5 @@ name: Compression comparison -description: Compare me_fasttext with sparse, quantized, or compact-serving baselines +description: Compare NGramStore with sparse, quantized, or compact-serving baselines title: "[Compression] " labels: - benchmark @@ -10,16 +10,17 @@ body: Use this for focused comparisons against model-compression, sparse-lookup, quantized embedding, or compact-serving baselines. - Please keep the claim narrow. me_fasttext is a compact lexical - embedding artifact, not a general LLM compression framework. + NGramStore is a learned n-gram vector storage/query backend. + Name the actual table and exact/approximate mode; separate storage, + query costs and downstream quality. FastText and Engram are applications. - type: dropdown id: comparison_type attributes: label: Comparison type options: - - Original FastText vs me_fasttext compact .z - - Quantized FastText vs me_fasttext compact .z - - Hash-free uncompressed vs compact export + - Dense or raw mmap vs native n-gram store + - Quantized table vs lossless native store + - DATrie/structural contribution ablation - Sparse or table-compression baseline - Lexical search or retrieval baseline - Dense embedding or reranker prefilter comparison @@ -42,7 +43,7 @@ body: id: commit attributes: label: Git commit - description: The me_fasttext commit used for this run. + description: The NGramStore commit used for this run. placeholder: "abcdef1" validations: required: true @@ -87,10 +88,10 @@ body: description: List all compared methods and explain omissions. placeholder: | compared: - - Original FastText: + - Dense resident / raw mmap: - Quantized FastText: - - Hash-free uncompressed: - - me_fasttext compact .z: + - Identity-layout native control: + - NGramStore native table/index/metadata: omitted: validations: required: true @@ -126,7 +127,7 @@ body: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | Original FastText | | | | | | | | | Quantized FastText | | | | | | | | - | me_fasttext .z | | | | | | | | + | NGramStore native | | | | | | | | validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4b63192..c0b8c9f 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,11 +1,11 @@ blank_issues_enabled: true contact_links: - name: Evaluator quickstart - url: https://github.com/initial-d/me_fasttext/blob/main/docs/evaluator_quickstart.md + url: https://github.com/initial-d/ngram-store/blob/main/docs/evaluator_quickstart.md about: Start here if you want to run a first public-corpus benchmark. - name: Benchmark protocol - url: https://github.com/initial-d/me_fasttext/blob/main/docs/benchmark_protocol.md + url: https://github.com/initial-d/ngram-store/blob/main/docs/benchmark_protocol.md about: Use this for the fuller memory, latency, OOV, and quality report shape. - name: Outreach notes - url: https://github.com/initial-d/me_fasttext/blob/main/docs/outreach.md + url: https://github.com/initial-d/ngram-store/blob/main/docs/outreach.md about: Use this when adding the project to lists or sharing it with maintainers. diff --git a/.github/ISSUE_TEMPLATE/evaluation-question.yml b/.github/ISSUE_TEMPLATE/evaluation-question.yml index 39be5d6..c17b1ff 100644 --- a/.github/ISSUE_TEMPLATE/evaluation-question.yml +++ b/.github/ISSUE_TEMPLATE/evaluation-question.yml @@ -1,5 +1,5 @@ name: Evaluation question -description: Ask about building, benchmarking, reproducing, or interpreting me_fasttext +description: Ask about building, benchmarking, reproducing, or interpreting NGramStore title: "[Question] " labels: - question diff --git a/.github/ISSUE_TEMPLATE/showcase.yml b/.github/ISSUE_TEMPLATE/showcase.yml index 724fb47..1d72936 100644 --- a/.github/ISSUE_TEMPLATE/showcase.yml +++ b/.github/ISSUE_TEMPLATE/showcase.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Thanks for helping other readers understand where `me_fasttext` fits. + Thanks for helping other readers understand where `NGramStore` fits. Use this template for citations, public list entries, comparison notes, integration experiments, deployment notes, or concrete use cases. Short @@ -35,7 +35,7 @@ body: id: summary attributes: label: Summary - description: Briefly describe how `me_fasttext` was used, cited, compared, or listed. + description: Briefly describe how `NGramStore` was used, cited, compared, or listed. placeholder: | Example: Used as a compact lexical embedding baseline before dense reranking on an entity-heavy product-title retrieval corpus. @@ -82,7 +82,7 @@ body: label: Suggested public wording description: Optional wording maintainers can quote in README, docs, release notes, or distribution logs. placeholder: | - Example: me_fasttext was useful as a compact lexical embedding layer + Example: NGramStore was used as a frozen n-gram memory backend for OOV-heavy candidate generation before dense reranking. - type: checkboxes id: disclosure diff --git a/.zenodo.json b/.zenodo.json index ed20329..b4cb396 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,14 +1,18 @@ { - "title": "me_fasttext: Memory-Efficient FastText with Exact Trie N-Gram IDs and mmap Serving", + "title": "NGramStore: Compression, Storage and Exact Lookup for Learned N-Gram Vectors", "creators": [ { "name": "Du, Yimin", "affiliation": "Independent Researcher" } ], - "description": "A FastText-derived C++ research prototype that replaces hash-bucketed character n-gram ids with exact trie-backed ids, then exports a compact mmap serving index through conservative n-gram row sharing and mark-compact layout.", + "description": "Formerly me_fasttext. A native compression, storage and exact-query engine for frozen learned n-gram vectors, with registered DATrie/mark-compact planning and FastText/Engram application integrations. Lossless and approximate modes are reported separately; the original FastText paper retains its title and attribution.", "license": "MIT", "keywords": [ + "n-gram memory", + "vector compression", + "Engram", + "LLM memory", "fastText", "word embeddings", "subword embeddings", @@ -28,7 +32,7 @@ "resource_type": "publication-preprint" }, { - "identifier": "https://github.com/initial-d/me_fasttext", + "identifier": "https://github.com/initial-d/ngram-store", "relation": "isIdenticalTo", "scheme": "url", "resource_type": "software" diff --git a/CITATION.cff b/CITATION.cff index 2fdeb33..9dcd60f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,13 +1,16 @@ cff-version: 1.2.0 -message: "If you use me_fasttext or the accompanying method, please cite the arXiv paper." -title: "Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management" +message: "NGramStore was formerly me_fasttext. Cite the original paper for its method and the exact software commit/report for later engine work." +title: "NGramStore: Compression, Storage and Exact Lookup for Learned N-Gram Vectors" authors: - family-names: Du given-names: Yimin -repository-code: "https://github.com/initial-d/me_fasttext" -url: "https://arxiv.org/abs/2506.01254" +repository-code: "https://github.com/initial-d/ngram-store" +url: "https://github.com/initial-d/ngram-store" license: MIT keywords: + - n-gram memory + - vector compression + - Engram - fastText - word embeddings - subword embeddings diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c5eeaf..772a59a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Thanks for taking a look at `me_fasttext`. This is a research prototype, so the +Thanks for taking a look at NGramStore. This is a research prototype, so the most valuable contributions are the ones that make its evidence easier to understand, reproduce, or falsify. @@ -18,11 +18,11 @@ Good first contributions include: - a public-corpus benchmark report using the [evaluator quickstart](docs/evaluator_quickstart.md); - an - [evaluation question](https://github.com/initial-d/me_fasttext/issues/new?template=evaluation-question.yml) + [evaluation question](https://github.com/initial-d/ngram-store/issues/new?template=evaluation-question.yml) about build, benchmark setup, corpus preparation, baselines, or result interpretation; - a - [showcase or use-case note](https://github.com/initial-d/me_fasttext/issues/new?template=showcase.yml) + [showcase or use-case note](https://github.com/initial-d/ngram-store/issues/new?template=showcase.yml) for citations, list entries, comparisons, integrations, or deployment notes; - a retrieval, ranking, classification, or OOV-heavy serving result; - a negative result showing where the compact `.z` path does not help; @@ -35,15 +35,15 @@ logs, or credentials. Aggregate metrics and corpus manifests are enough. ## Benchmark reports Use the -[benchmark result issue template](https://github.com/initial-d/me_fasttext/issues/new?template=benchmark-result.yml) +[benchmark result issue template](https://github.com/initial-d/ngram-store/issues/new?template=benchmark-result.yml) when possible. A useful report should include: -- exact `me_fasttext` commit SHA; +- exact NGramStore commit SHA; - corpus name, language, license status, split, and preprocessing; - hardware, OS, compiler, filesystem, and storage type; - build, training, export, and benchmark commands; -- `.bin`, `.vec`, and `.z` artifact sizes; -- `bench_ftindex` output for compact `.z` serving; +- full input/table/index/metadata bytes and the exact lossless/approximate mode; +- dense/raw-mmap controls, native query latency and anonymous/file-backed RSS; - one quality metric, or an explicit serving-only caveat; - failures, omitted baselines, and anything unfavorable to the method. @@ -55,7 +55,7 @@ for the fuller report shape. ## Showcase and visibility notes Use the -[showcase issue template](https://github.com/initial-d/me_fasttext/issues/new?template=showcase.yml) +[showcase issue template](https://github.com/initial-d/ngram-store/issues/new?template=showcase.yml) for lighter-weight adoption evidence: citations, public list entries, integration notes, talks, blog posts, retrieval experiments, or compact-serving deployment notes. @@ -90,11 +90,7 @@ or platform limitation. ## Scope and positioning -Keep claims narrow. `me_fasttext` is best framed as a FastText-derived compact -lexical embedding artifact for memory-sensitive serving, OOV-heavy text, and -first-stage retrieval or ranking. It is not a replacement for transformer -embeddings, a vector database, or a full RAG system. - +NGramStore is a compression, storage and exact-query backend for learned n-gram vectors. FastText and Engram/TinyEngram are applications with different training and addressing contracts. Numeric/hash queries use the native row engine; structural sharing requires registered string keys. Lossless bytes, approximate distortion, query cost and task quality need separate evidence. See [`docs/artifact_card.md`](docs/artifact_card.md) and [`docs/citation_guide.md`](docs/citation_guide.md) for wording that is easier to cite without overclaiming. diff --git a/README.md b/README.md index 384705b..94fc3b5 100644 --- a/README.md +++ b/README.md @@ -1,530 +1,117 @@ -# me_fasttext +# NGramStore -[![arXiv](https://img.shields.io/badge/arXiv-2506.01254-b31b1b.svg)](https://arxiv.org/abs/2506.01254) -[![CI](https://github.com/initial-d/me_fasttext/actions/workflows/ci.yml/badge.svg)](https://github.com/initial-d/me_fasttext/actions/workflows/ci.yml) -[![Release](https://img.shields.io/github/v/release/initial-d/me_fasttext?label=release)](https://github.com/initial-d/me_fasttext/releases/latest) -[![Benchmark reports welcome](https://img.shields.io/badge/benchmark_reports-welcome-2ea44f)](https://github.com/initial-d/me_fasttext/issues/1) -[![Evaluator quickstart](https://img.shields.io/badge/evaluator-quickstart-blue)](docs/evaluator_quickstart.md) -[![Showcase](https://img.shields.io/badge/use_cases-showcase-orange)](https://github.com/initial-d/me_fasttext/issues/new?template=showcase.yml) +**Compression, storage and exact lookup for learned n-gram vectors.** -> [!IMPORTANT] -> **me_fasttext：面向 learned n-gram memory 的无损存储与高效查询后端，保持原始寻址和向量字节。** -> -> **A lossless storage and efficient query backend for learned n-gram memory, preserving original addressing and vector bytes.** +**面向 learned n-gram 向量表的压缩、存储与精确查询后端引擎。** -The native C row store and PyTorch adapters serve immutable CPU memory tables -through compressed blocks, mmap, bounded row/block caches and batched queries. The -byte-preservation contract applies to ordinary native export and layout/exact -structural modes. Opt-in joint structural sharing changes source weights and -records that separately; the legacy C++ exporter also supports approximate sharing. +[![CI](https://github.com/initial-d/ngram-store/actions/workflows/ci.yml/badge.svg)](https://github.com/initial-d/ngram-store/actions/workflows/ci.yml) +[![Paper](https://img.shields.io/badge/arXiv-2506.01254-b31b1b.svg)](https://arxiv.org/abs/2506.01254) +[![Release](https://img.shields.io/github/v/release/initial-d/ngram-store?label=release)](https://github.com/initial-d/ngram-store/releases/latest) -[**C / PyTorch integration**](docs/native_ngram_memory.md) -· [**Trained-model quality checks**](docs/causal_lm_evidence_20261024/README.md) -· [**Query latency and memory results**](docs/bag_compaction_evidence_20261026/README.md) -· [**Native query cache optimization and CPU evidence**](docs/native_query_evidence_20261030/README.md) -· [**Original DATrie + mark-compact native integration and trained ablations**](docs/structural_native_evidence_20261030/README.md) -· [**1.85M trained-row CPU capacity controls and read-amplification limits**](docs/cpu_capacity_evidence_20261030/README.md) -· [**Sparse native blocks: 16–27× lower miss latency on the same trained table**](docs/sparse_blocks_evidence_20261030/README.md) -· [**Native storage in the original FastText model/CLI: trained fidelity and serving controls**](docs/fasttext_native_evidence_20261031/README.md) -· [**Installable TinyEngram/Qwen checkpoint backend**](docs/tinyengram_qwen_evidence_20261030/README.md) -· [**Host memory → device query and prefetch**](docs/device_memory_prefetch.md) +Formerly **me_fasttext**. The engine grew from +[Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management](https://arxiv.org/abs/2506.01254). +The paper keeps its original title; later Engram integrations are software extensions. -Learned-model evidence includes small CPU references and a trained memory branch -on pretrained Qwen3-0.6B. Official Engram compatibility uses its demonstration; -[Small-table A100 gather costs](docs/cuda_gather_evidence_20261030/README.md) -and [controlled Qwen CUDA inference timings](docs/qwen_cuda_performance_20261030/README.md) -are measured; large-table and production serving performance remain open. +NGramStore stores frozen learned vectors in compressed native blocks and queries +original row IDs through mmap, decoded caches and batched gathers. Registered +string keys additionally support DATrie indexing and structural mark-compact +planning. FastText and Engram are applications of this shared backend. -The device adapter keeps immutable tables on CPU and transfers only queried -vectors to the backbone. TinyEngram prefetches each layer's hashes before decoder -execution. CPU/MPS checks and [real A100 CUDA correctness checks](docs/cuda_a100_evidence_20261030/README.md) -pass, including trained Qwen full/cached logits and CUDA stream/pinned-memory -lifetime checks. [Three-baseline A100 measurements](docs/device_gather_benchmark.md) -separate generic CPU offload from native storage/query costs. Native preserves -outputs but is slower than dense offload on these trained-table query traces; -large-table whole-model memory gains remain unmeasured. - -This repository accompanies the paper [Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management](https://arxiv.org/abs/2506.01254). - -The optional [native FastText facade](docs/fasttext_native_bundle.md) converts -this repository's dense `.bin` models through the original DATrie registry and -mark-compact pipeline, then serves them with the existing CLI/query APIs. -Layout/exact modes retain every input byte and float32 aggregation order. -The first actual trained-model control saves 0.46% for the whole bundle; hot -queries remain slower than dense, and structural ordering has no added storage -benefit on that table. Historical valid word/sentence outputs match exactly. - -> Status: research / production-prototype code. The command-line interface remains close to the original fastText workflow, while the storage and serving path has been extended for large-vocabulary, memory-sensitive deployments. - -> Inference framing: `me_fasttext` can also be read as an early compact-inference prototype for lexical embeddings. It keeps subword identity exact during training, then rewrites the learned rows into a smaller mmap-friendly serving artifact for low-memory retrieval and feature-serving workloads. - -## Engram readers: start here - -**中文入门：[Engram 之后，高阶 n-gram 记忆还能压缩多少？](docs/zhihu_engram_article_20261022.md)** -— trained evidence, two figures, and a runnable integration example. -[知乎文章](https://zhuanlan.zhihu.com/p/2096640222452781097) - -[DeepSeek's Engram](https://github.com/deepseek-ai/Engram) highlights a concrete -systems question: **how do we store and serve large n-gram memory tables?** -This project makes that question inspectable through exact trie indexing, -compact row layouts, and mmap lookup. **[Run, evaluate and cite the Engram -storage artifact](docs/engram_quickstart.md)** is the shortest path from the -interface proposal to executable code and source-linked evidence. - -- [Engram storage guide](docs/engram_storage.md): architecture mapping, - byte-preservation contract, and the adapter boundary for token/hash row IDs. -- [Run the lossless row-store demo](docs/engram_storage.md#run-the-byte-exact-demo): - exact-byte deduplication, compressed blocks, original-row-ID forwarding, and - bounded-cache mmap access. It runs with Python's standard library. -- [PyTorch gather adapter](docs/engram_gather_adapter.md): immutable CPU - replacement for the official demo's multi-head embedding gather, preserving - per-head offsets, dtype and tensor shape. Random weights verify the interface. -- [C ABI and model integration](docs/native_ngram_memory.md): standalone native - gather, direct PyTorch output, ordinary embedding support, and integration - boundaries for Engram, HF decoders and C++/GPU runtimes. -- [Full official Engram CPU module verification](docs/engram_full_adapter_verification_20261018.md): - Python/native storage produces identical gates/convolution outputs across - three dtypes on tiny random weights; trained/GPU results remain open. -- [Trained n-gram evidence](docs/trained_ngram_similarity_20261018.md): real-text - SGNS across orders 4/8/12 and three seeds. Identical-update structural pairs - have median cosine about 0.996; offline byte-exact residual prediction tests - how to exploit that redundancy. This is a FastText-style reference, not Engram - training or a model-quality result. -- [Native lossless XOR codec](docs/native_xor_compression_20261020.md): actual - C-readable artifacts on nine trained tables. Adaptive block encoding and - original-ID forwarding save 6.9–9.1% with training-signature grouping; - storage gains and measured CPU query costs are reported separately. -- [Streaming export follow-up](docs/streaming_export_20261021.md): about 1.9× - faster XOR packing on the same trained rows, identical files, and block-sized - embedding staging with a separate process-memory measurement. -- [Bounded native query cache](docs/native_cache_20261022.md): optional decoded - block reuse with a byte budget, original-byte output and measured hotspot, - corpus-replay and no-benefit cases. Disabled by default. -- [Native embedding bags](docs/embedding_bag_evidence_20261025/README.md): sum/mean - inference without a full gathered tensor, with query-ID deduplication or bounded - C staging. Trained order-4/8/12 tests show 10–24% lower RSS versus native full - gather; higher orders trade latency for memory. Dense resident lookup is faster. -- [High-order memory research plan](docs/ngram_memory_research_plan.md): trained - structural redundancy, lossless codecs, indexing and serving experiments. -- [Approximate compression curves](docs/approximate_ngram_compression_20261018.md): - cosine thresholds 0.999..0.90, norm-ratio controls, actual artifact sizes and - vector error. At max order 12, joint cosine/norm ≥0.99 gives 2.14–2.16× - n-gram storage reduction; these are vector-only results, not model task loss. -- [English and Chinese launch copy](docs/engram_promotion.md): a technical - introduction centered on n-gram memory storage and lookup. - -The new reference preserves each original row's bytes and ID lookup. The legacy -C++ `.z` exporter performs approximate row sharing and has known implementation -limits. The new gather adapter is independent of that exporter. No trained -DeepSeek Engram checkpoint or GPU serving has been evaluated here. The separate -TinyEngram/Qwen CPU checkpoint checks are linked above. See the -[implementation audit](docs/engram_storage.md#legacy-export-audit). - -## Help the project grow - -If `me_fasttext` is useful or relevant to your work, the fastest ways to help -other people find it are: - -- star or watch the repository so new benchmark notes surface to more readers; -- share a public-corpus benchmark, replication, or negative result with the - [benchmark template](https://github.com/initial-d/me_fasttext/issues/new?template=benchmark-result.yml); -- open a - [showcase / use-case note](https://github.com/initial-d/me_fasttext/issues/new?template=showcase.yml) - if you cite it, compare it, add it to a list, or try it in retrieval, - ranking, OOV-heavy text, Chinese text, or compact-serving work; -- use the [share kit](docs/share_kit.md) for copy-ready descriptions, posts, - list-maintainer blurbs, and responsible sharing boundaries; -- send an - [evaluation question](https://github.com/initial-d/me_fasttext/issues/new?template=evaluation-question.yml) - if you are unsure how to build, benchmark, or interpret a result. - -Short, honest reports are better than polished success stories. Reproducible -negative results, omitted-baseline notes, and corpus-specific limits all make -the project easier to trust and cite. - -## If you are evaluating this project - -The most valuable contribution right now is a reproducible public-corpus -benchmark: memory, load time, compact `.z` mmap-serving latency, OOV coverage, -and one retrieval, ranking, or classification quality metric. Negative results -are welcome if the corpus and hardware context are clear. - -Fast paths: - -- [Evaluator quickstart](docs/evaluator_quickstart.md): a short path from clone - to a useful benchmark report. -- [Contributing guide](CONTRIBUTING.md): what benchmark reports, negative - results, documentation fixes, and code patches should include. -- [Evaluation question](https://github.com/initial-d/me_fasttext/issues/new?template=evaluation-question.yml): - ask about build, benchmark setup, corpus preparation, baselines, or result interpretation. -- [Showcase / use case](https://github.com/initial-d/me_fasttext/issues/new?template=showcase.yml): - share citations, list entries, comparisons, integrations, or deployment notes. -- [Benchmark issue](https://github.com/initial-d/me_fasttext/issues/1): share a - reproducible run, replication, or negative result. -- [Artifact card](docs/artifact_card.md): decide quickly whether the project is - relevant to cite, benchmark, or list. -- [Compression comparison checklist](docs/compression_comparison_checklist.md): - report sparse, quantized, or compact-serving comparisons without overclaiming. -- [Answer engine context](docs/answer_engine_context.md): concise source-of-truth - wording for search, citation, and AI answer summaries. -- [Citation guide](docs/citation_guide.md): cite the paper and describe the - comparison without overclaiming. - -## Start here - -- [Evaluator quickstart](docs/evaluator_quickstart.md): the shortest route from - clone to a useful report. -- [Contributing guide](CONTRIBUTING.md): how to share benchmark evidence, - negative results, docs fixes, and code patches. -- [Code of conduct](CODE_OF_CONDUCT.md): expectations for evidence-oriented - benchmark and reuse discussions. -- [Benchmark protocol](docs/benchmark_protocol.md): how to compare memory, load time, latency, and task quality. -- [First public corpus benchmark](docs/first_public_corpus_benchmark.md): a concrete external run plan for the first citable public-corpus report. -- [Retrieval and OOV serving benchmark](docs/retrieval_oov_benchmark.md): how to measure mmap load time, p50/p95 query latency, and OOV subword coverage. -- [Retrieval comparison matrix](docs/retrieval_comparison_matrix.md): how to compare the compact lexical layer with FastText, BM25, dense embeddings, and ANN/vector-search systems. -- [Artifact card](docs/artifact_card.md): a one-page summary for citation, benchmark, and list-maintainer triage. -- [Answer engine context](docs/answer_engine_context.md): canonical short - descriptions, citation wording, and boundaries for AI-generated summaries. -- [Citation guide](docs/citation_guide.md): how to cite the paper, position related work, and report comparisons. -- [Citation evidence ladder](docs/citation_evidence_ladder.md): how benchmark - reports become README-level or paper-level citation evidence. -- [Inference optimization note](docs/inference_optimization_note.md): how to position the project as compact lexical embedding inference. -- [Conditional memory context](docs/conditional_memory_context.md): a cautious - note connecting indexed lexical memory with newer Engram-style conditional memory. -- [Model-compression and Engram-style bridge](docs/model_compression_engram_bridge.md): - how to evaluate `me_fasttext` as a compact lexical-memory baseline without - overclaiming it as an LLM memory module. -- [Compression comparison checklist](docs/compression_comparison_checklist.md): - a focused report shape for sparse, quantized, and compact-serving baselines. -- [Cross-community positioning](docs/cross_community_positioning.md): precise framing for FastText, word-embedding, DA-trie, GC, and inference-optimization readers. -- [Related work map](docs/related_work_map.md): how the project connects to FastText, tries, mmap serving, retrieval, and memory management. -- [Engineering note](docs/engineering_note.md): how trie ids, mark-compact style row rewriting, and mmap serving fit together. -- [Design essay](docs/memory_management_for_ai_embeddings.md): why this project reframes FastText storage as an AI memory-management problem. -- [Chinese retrospective](docs/technical_retrospective_zh.md): a long-form Chinese note for technical storytelling and outreach. -- [Outreach notes](docs/outreach.md): concise wording for adding the project to NLP, embedding, retrieval, or systems lists. -- [Share kit](docs/share_kit.md): copy-ready summaries, posts, maintainer notes, and responsible sharing boundaries. -- [Promotion posts](docs/promotion_posts.md): short posts and maintainer notes for careful outreach. -- [Listing follow-up](docs/listing_followup.md): accepted, open, and closed external list submissions with next actions. -- [Distribution log](docs/distribution_log.md): external list submissions, accepted entries, and follow-up status. -- [Visibility status](docs/visibility_status.md): traffic snapshots, live entry points, and evidence-oriented next actions. -- [Reuse and citation pathways](docs/reuse_and_citation_pathways.md): where follow-up work can naturally cite the method. -- [Citation metadata](CITATION.cff): machine-readable citation information for GitHub and reference managers. -- [Release notes](docs/release_notes_v0.1.0.md): scope and caveats for the first research-prototype release. - -## External listings - -- [Awesome Data Science](https://github.com/academic/awesome-datascience) lists `me_fasttext` under General Machine Learning Packages. - -## Where it fits - -Use `me_fasttext` when you need a small lexical memory layer rather than a -large semantic encoder: - -| Use case | Why this project is relevant | +| Layer | Responsibility | | --- | --- | -| OOV-heavy retrieval | Character n-grams preserve signals for rare and unseen words. | -| Entity-heavy search | Exact trie ids make subword identities inspectable before compression. | -| RAG prefiltering | A low-cost lexical stage can reduce candidates before dense reranking. | -| Compact inference | A dense compact matrix plus mmap loading lowers the cost of keeping lexical embeddings online. | -| Memory-sensitive serving | The compact `.z` artifact is designed for mmap-based loading. | -| Long-tail Chinese or multilingual text | UTF-8 n-gram enumeration avoids byte-level shortcuts. | - -## Why this exists - -Standard fastText represents a word by averaging its word vector and character n-gram vectors. To avoid storing every observed n-gram, the reference implementation hashes subwords into a fixed bucket table. That keeps memory bounded, but unrelated n-grams can collide into the same row. - -`me_fasttext` follows an exact-then-compress design: - -1. Give every retained word and every observed character n-gram an explicit identity. -2. Store word and n-gram ids in trie-based indexes instead of arbitrary hash buckets. -3. Explore offline row sharing followed by a compact layout; the paper's - intended similarity criterion differs from the current legacy exporter - ([audit](docs/engram_storage.md#legacy-export-audit)). -4. Rewrite the remaining live rows into a compact, memory-mappable serving layout. - -The goal is not to replace modern LLM embedding models. The goal is to provide a small, deterministic, inspectable lexical memory layer for retrieval, ranking, entity-heavy search, long-tail Chinese text, and low-latency feature generation. - -## Highlights - -- FastText-style training and inference for `skipgram`, `cbow`, and `supervised` models. -- Collision-free word and character n-gram lookup using trie indexes. -- UTF-8 aware character n-gram enumeration. -- Legacy prefix/suffix traversal and approximate row sharing; see the export audit. -- Mark-compact style row reorganization for dense compact ids. -- Single-file mmap serving index generated as `.z`. -- Static library output (`libfasttext.a`) for embedding the implementation in services. -- Utility binaries for index generation and n-gram inspection. -- Experimental Python byte-exact row deduplication and block-compressed mmap - lookup, preserving original row IDs independently of `.z` export. - -## Reported paper results - -The paper reports a large Chinese vocabulary benchmark with 30.1M words and 287.4M extracted n-grams. In that setting, the compressed model reduced memory from 145.2 GB for the original FastText baseline to 28.9 GB, reduced cold load time from 12.3 minutes to 3.2 minutes, and preserved downstream quality within 0.3 points of the hash-free model. - -| Method | Memory | Load time | Storage | -| --- | ---: | ---: | ---: | -| Original FastText | 145.2 GB | 12.3 min | 89.4 GB | -| HashFree FastText | 287.4 GB | 28.7 min | 201.8 GB | -| Quantized FastText | 72.6 GB | 8.9 min | 44.7 GB | -| SVD Compression | 89.3 GB | 15.4 min | 62.1 GB | -| me_fasttext | 28.9 GB | 3.2 min | 18.6 GB | - -These numbers are from the paper and will vary with corpus, vocabulary size, hardware, operating-system page cache behavior, and compression threshold. - -They are not Engram benchmarks and have not been reproduced by the new row -store. The legacy exporter also differs from the paper's cosine-based method; -inspect the implementation audit before treating it as a paper reproduction. - -## Repository layout - -```text -. -|-- src/ -| |-- main.cc # fasttext-compatible CLI entry point -| |-- args.* # command-line arguments and defaults -| |-- dictionary.* # word/ngram dictionary and UTF-8 n-gram extraction -| |-- fasttext.* # training, model IO, vector export, compact index export -| |-- ftindex.* # mmap serving index API -| |-- index.cc # helper for generating serving artifacts -| |-- cut_ngram.cc # helper for cutting n-gram trie by vocabulary -| `-- print_ngrams.cc # helper for inspecting n-gram trie contents -|-- tools/ -| `-- prepare_public_benchmark.py -|-- mylib/ -| |-- include/ # trie, hash, pub, spell, xml headers -| `-- lib/ # bundled static libraries used by the Makefile -|-- Makefile -|-- fasttext # prebuilt binary from the original environment -|-- index # prebuilt helper binary -|-- cut_ngram # prebuilt helper binary -|-- print_ngrams # prebuilt helper binary -`-- libfasttext.a # prebuilt static library -``` - -## Build - -### Requirements - -- Linux is the tested target. -- C++ compiler with C++11 support. -- `make`, `ar`, pthreads, and POSIX mmap support. -- The bundled trie/pub libraries under `mylib/`. - -The serving implementation uses Linux-specific mmap flags such as `MAP_POPULATE`, so non-Linux platforms may need small portability changes. - -### Build from source - -```bash -make opt -``` - -This builds: +| Core engine | Lossless block encoding, original-ID forwarding, mmap, bounded decoded caches and gather | +| Registered n-gram structure | DATrie keys, prefix/suffix representative selection and mark-compact export | +| Word embeddings: FastText | Original dictionary, training and word/sentence/classification APIs | +| LLM memory: Engram / TinyEngram | Original token hashing, head offsets, gates and decoder remain in the model | -- `fasttext` -- `index` -- `cut_ngram` -- `print_ngrams` -- `libfasttext.a` - -For debug builds: - -```bash -make debug -``` - -### Test helper scripts - -The Python benchmark helper uses only the standard library. Run its tests with: - -```bash -python3 -m unittest discover -s tests -``` - -To clean generated objects and the main binary: - -```bash -make clean -``` +**Lossless and approximate modes are explicit.** Ordinary packing and +structural `layout`/`exact` preserve source bytes. Opt-in `joint` sharing uses +cosine and norm-ratio guards and can change vectors. Engram hash collisions do +not certify string-prefix/suffix relations. Storage, latency and model quality +are measured separately; gains depend on the trained table and query workload. ## Quick start -### Train a skipgram model - -Input format is plain text, one sentence or document per line: - -```bash -./fasttext skipgram \ - -input data/train.txt \ - -output models/meft \ - -minn 2 \ - -maxn 6 \ - -thread 24 -``` - -The training path saves: - -| File | Purpose | -| --- | --- | -| `models/meft.bin` | Full fastText-style model | -| `models/meft.vec` | Text word-vector export | -| `models/meft.z` | Compact mmap serving index | -| `worddat.bin` | Intermediate word trie dump, written in the current working directory | -| `ngramdat.bin` | Intermediate n-gram trie dump, written in the current working directory | - -### Train a supervised classifier - -Supervised training follows the standard fastText label convention: - -```text -__label__sports the team won the final -__label__finance quarterly revenue increased -``` +Linux/macOS require a C compiler and zlib development headers. From this checkout: ```bash -./fasttext supervised \ - -input data/train.txt \ - -output models/classifier \ - -epoch 5 \ - -lr 0.1 \ - -wordNgrams 2 +python -m pip install . +ngram-store --help ``` -Evaluate and predict: +The distribution remains `meft-memory` for install compatibility; the engine API +is `ngram_store` and the CLI is `ngram-store`. No Torch is needed for packing or +numeric lookup. Existing `meft_memory` imports and `meft-*` commands still work. -```bash -./fasttext test models/classifier.bin data/test.txt 1 -./fasttext predict-prob models/classifier.bin data/test.txt 3 -``` - -## Common commands +Use an **existing trained table** exported as row-contiguous little-endian float32: ```bash -# Print word vectors from a trained model. -echo "example" | ./fasttext print-word-vectors models/meft.bin - -# Print sentence vectors. -echo "example sentence" | ./fasttext print-sentence-vectors models/meft.bin - -# Inspect the n-grams used for a word. -./fasttext print-ngrams models/meft.bin example - -# Interactive nearest-neighbor lookup. -./fasttext nn models/meft.bin 10 - -# Interactive analogy lookup. -./fasttext analogies models/meft.bin 10 - -# Quantize a supervised model. -./fasttext quantize \ - -input data/train.txt \ - -output models/classifier \ - -cutoff 100000 \ - -qnorm \ - -retrain +ngram-store pack --input vectors.f32 --dim 256 --output memory.native +ngram-store inspect --store memory.native +ngram-store query --store memory.native --ids 0 42 0 ``` -## Serving with the compact index - -The `.z` file is designed for mmap-based serving. It stores: - -1. Metadata: number of words, number of compact n-gram rows, dimension, `minn`, and `maxn`. -2. A contiguous float matrix: word rows followed by compact n-gram rows. -3. A double-array trie mapping words to row ids. -4. A double-array trie mapping character n-grams to compact row ids. - -Minimal C++ usage, with `MODEL_DIM` set to the dimension used when training the model: - -```cpp -#include "src/ftindex.h" +Packing verifies every reconstructed row before publishing a new directory. +IDs retain order and duplicates. `query --hex` returns the exact row bits. +For registered keys, add `--keys keys.json --structural-library /path/to/libmeft_structural.so` +to `pack`; see the [engine guide](docs/evaluator_quickstart.md). -#include +```python +from ngram_store import NativeRows -int main() { - void* mmap_addr = nullptr; - size_t mmap_len = 0; - FTIndex* index = ftindex_mmap("models/meft.z", &mmap_addr, &mmap_len); - if (!index) { - std::cerr << "failed to mmap index\n"; - return 1; - } - - real vec[MODEL_DIM] = {0}; - getSentenceVector(index, "hello world", vec); - - for (int i = 0; i < MODEL_DIM; ++i) { - std::cout << vec[i] << (i + 1 == MODEL_DIM ? '\n' : ' '); - } - - ftindex_delete(index); - ftindex_munmap(mmap_addr, mmap_len); - return 0; -} -``` - -Compile against the static library and bundled dependencies: - -```bash -c++ -std=c++11 -pthread \ - -I. -I./mylib/include \ - examples/query.cc \ - libfasttext.a \ - -L./mylib/lib -ltrie -lpub \ - -o query +with NativeRows("memory.native/table.men", cache_bytes=4 * 1024**2) as memory: + original_bytes = memory.rows([0, 42, 0]) ``` -## How compression works - -At training time, the dictionary inserts retained words and UTF-8 character n-grams into trie structures instead of relying on `hash(ngram) % bucket`. +## Application examples -At export time, `FastText::saveIndex()` performs the compact-index pipeline: +| Application | Runnable entry | What stays with the model | +| --- | --- | --- | +| FastText / word embeddings | [Prepare text → train → export → query](docs/fasttext_native_bundle.md) | Dictionary, original float32 aggregation, output weights | +| Engram / LLM memory | [Official demo factory and verified boundaries](docs/engram_quickstart.md) | Hash IDs, head layout, projections, gates and convolution | +| TinyEngram / Qwen | [Checkpoint export, trained outputs and decode](docs/tinyengram_qwen_evidence_20261030/README.md) | Backbone, tokenizer, learned gates and checkpoint state | -1. Walk the n-gram trie, selecting shared rows with the legacy norm-ratio - heuristic (the paper describes a cosine-based criterion). -2. Build a reversed/suffix view and repeat the same conservative merge process. -3. Mark live row ids, assign dense compact ids, and rewrite trie terminal values. -4. Serialize metadata, compact vectors, word trie, and n-gram trie into `.z`. +`ngram-store fasttext-export`, `tiny-export` and `tiny-verify` route to the +existing converters. FastText's custom trainer first needs +`ngram-store prepare-fasttext`; ordinary text is not its native input format. +The [included sample workflow](docs/fasttext_native_bundle.md) requires no +external data or GPU. It checks integration, not representative compression. -Row sharing happens after training and the index remapping can be inspected. -The current norm-ratio check does not establish directional similarity or -lossless reconstruction; see the [export audit](docs/engram_storage.md#legacy-export-audit). +## Trained evidence and limits -## Differences from upstream fastText +| Evidence | Observed result and boundary | +| --- | --- | +| [Actual FastText model](docs/fasttext_native_evidence_20261031/README.md) | 90,380 rows and word/sentence outputs byte-identical; whole bundle saves 0.46%, hot queries slower than dense; structural order has no added benefit here | +| [Structural contribution controls](docs/structural_native_evidence_20261030/README.md) | Trained SGNS/GRU references measure DATrie and approximate joint sharing separately; exact mode finds no identical rows to merge | +| [1.85M trained-row capacity](docs/cpu_capacity_evidence_20261030/README.md) / [sparse blocks](docs/sparse_blocks_evidence_20261030/README.md) | Smaller blocks lower native miss latency 16–27× relative to the previous native layout; raw mmap remains faster | +| [TinyEngram/Qwen trained integration](docs/tinyengram_qwen_evidence_20261030/README.md) / [CUDA controls](docs/qwen_cuda_performance_20261030/README.md) | Preserved trained outputs and measured CPU/device costs; native lookup remains slower than dense offload on these traces | -| Area | Upstream fastText | me_fasttext | -| --- | --- | --- | -| Subword ids | `hash(ngram) % bucket` | Exact trie id before compression | -| N-gram collisions | Arbitrary bucket collisions | No collision unless explicitly merged | -| Large-model serving | Load model structures from `.bin` | mmap compact `.z` index | -| Compression policy | Bucket size, pruning, quantization | Prefix/suffix row sharing plus mark-compact layout | -| Debuggability | Bucket contents are opaque | N-gram identities remain inspectable | +Official DeepSeek Engram verification uses its small demonstration, not a +released trained Engram checkpoint. Small-model results are not production +serving or universal compression claims. [Evidence index](docs/index.md). -## Practical notes +The standalone C engine/wheel supports Linux and macOS. The bundled legacy +DATrie bridge and FastText facade require Linux x86-64. Handles are frozen and +single-worker; cache budgets do not bound RSS. Structural planning holds full +rows/maps offline. Trusted legacy metadata/index formats retain their own +limits. [C ABI and runtime boundaries](docs/native_ngram_memory.md). -- Rebuild binaries on the target Linux machine instead of relying on the checked-in prebuilt executables. -- The compact `.z` index is the preferred artifact for low-latency serving. -- The full `.bin` model is still useful for evaluation, CLI inspection, and regenerating derived artifacts. -- The trie dumps `worddat.bin` and `ngramdat.bin` are written to the current working directory, so run training from a predictable location. -- Compression ratios depend heavily on corpus language, n-gram length, vocabulary filtering, vector dimension, and similarity threshold. +## History, citation and contribution -## Citation +Repository history, releases, issues and original paper attribution continue +under NGramStore. Old model formats, C ABI symbols, `meft_memory` imports and +FastText commands remain compatible. C++ clients must recompile; `.z` retains +its own reader and approximate contract. [Compatibility guide](docs/legacy_fasttext.md). -If this repository or the accompanying method is useful in your work, please cite: +Cite the [original paper](https://arxiv.org/abs/2506.01254) for its method and the +exact software commit/report for later engine work. [Citation guide](docs/citation_guide.md). -```bibtex -@misc{du2026memoryefficientfasttext, - title = {Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management}, - author = {Yimin Du}, - year = {2026}, - eprint = {2506.01254}, - archivePrefix = {arXiv}, - primaryClass = {cs.CL}, - url = {https://arxiv.org/abs/2506.01254} -} -``` +Share source-linked measurements or integration issues through +[Contributing](CONTRIBUTING.md) and the +[benchmark template](https://github.com/initial-d/ngram-store/issues/new?template=benchmark-result.yml). +Negative results and omitted-baseline notes are useful evidence. ## License -This codebase is derived from the public fastText C++ implementation and is -distributed under the MIT License. See [LICENSE](LICENSE). Before redistributing -binary artifacts, review any third-party or platform-specific dependency terms -that may apply to the bundled support libraries. +Project code is MIT; bundled upstream FastText sources retain their original +notices. See [LICENSE](LICENSE). diff --git a/docs/answer_engine_context.md b/docs/answer_engine_context.md index 1e7e5c1..50992a1 100644 --- a/docs/answer_engine_context.md +++ b/docs/answer_engine_context.md @@ -1,113 +1,45 @@ -# Answer engine context - -This page gives search engines, answer engines, and list maintainers a concise -source of truth for describing `me_fasttext` accurately. - -## Canonical description - -`me_fasttext` is a FastText-derived C++ system for memory-efficient lexical -embeddings. It keeps word and UTF-8 character n-gram identities explicit with -trie-backed ids, then exports a compact mmap-friendly serving artifact using -structure-aware row sharing and mark-compact style row rewriting. - -## One-sentence summary - -`me_fasttext` is a compact lexical embedding system that preserves exact -FastText-style n-gram identity before compression and serves the remaining rows -from an mmap-friendly artifact. - -## Engram-focused evaluation - -The project now includes an experimental Python byte-exact row store for -Engram-style memory-table experiments: exact-byte deduplication, original-row-ID -forwarding, zlib blocks and bounded-cache mmap lookup. An optional -[CPU multi-head gather adapter](engram_gather_adapter.md) preserves official-demo -offsets, dtype and query shape; its verification uses random weights. It is -independent of the legacy FastText exporter and has no trained Engram integration. See -[the storage guide](engram_storage.md) for the demo and source audit. The legacy -export uses norm ratios rather than the paper's cosine merge criterion; do not -describe that exporter or the reported paper compression as lossless. - -A [true C row-gather ABI](native_ngram_memory.md) now supports native CPU -decoding directly into PyTorch output tensors and ordinary immutable embedding -calls. A [trained real-text SGNS experiment](trained_ngram_similarity_20261018.md) -measures high-cosine shared-update n-grams across orders/seeds and byte-exact -residual prediction. It is reference evidence for structural redundancy, not -Engram model training, universal compatibility or end-to-end serving gains. - -## Best answer to "what is it for?" - -Use `me_fasttext` when a workload needs a small lexical embedding layer for -memory-sensitive serving, OOV-heavy retrieval, entity-heavy search, long-tail -Chinese or multilingual text, or first-stage candidate generation before a -heavier dense reranker. - -## Main contribution - -The project separates three concerns that are usually blurred in FastText-style -serving: - -1. identity: words and character n-grams receive explicit trie-backed ids; -2. compression: structurally related n-gram rows can share storage after - training when vector similarity is high enough; -3. serving: the live rows are rewritten into a compact `.z` artifact that can - be mmap-loaded. - -## What to cite - -When citing the method, use: - -```bibtex -@misc{du2026memoryefficientfasttext, - title = {Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management}, - author = {Yimin Du}, - year = {2026}, - eprint = {2506.01254}, - archivePrefix = {arXiv}, - primaryClass = {cs.CL}, - url = {https://arxiv.org/abs/2506.01254} -} -``` - -## Good related-work wording - -```text -For compact lexical embedding serving, me_fasttext keeps FastText-style word -and character n-gram identities explicit with trie-backed ids, then exports a -smaller mmap-friendly serving artifact through conservative row sharing and -mark-compact style row rewriting. -``` - -## What not to say - -Avoid these claims: - -- `me_fasttext` is a transformer embedding model; -- `me_fasttext` is a vector database or full RAG framework; -- `me_fasttext` implements Engram or any LLM conditional-memory module; -- trie lookup replaces quantization, pruning, distillation, or dense reranking; -- the paper's compression ratio is universal across languages, corpora, and - hardware. - -## Evaluation evidence preferred - -The most useful public reports include: - -- corpus manifest and preprocessing notes; -- original FastText or quantized FastText baseline; -- compact `.z` artifact size; -- cold mmap load time and resident memory; -- p50 and p95 lookup latency; -- OOV rate and OOV subword coverage; -- retrieval, ranking, classification, or feature-serving quality metric; -- negative observations and omitted baselines. - -## Primary links - -- Repository: https://github.com/initial-d/me_fasttext -- Paper: https://arxiv.org/abs/2506.01254 -- Artifact card: https://github.com/initial-d/me_fasttext/blob/main/docs/artifact_card.md -- Evaluator quickstart: https://github.com/initial-d/me_fasttext/blob/main/docs/evaluator_quickstart.md -- Compression comparison checklist: https://github.com/initial-d/me_fasttext/blob/main/docs/compression_comparison_checklist.md -- Citation guide: https://github.com/initial-d/me_fasttext/blob/main/docs/citation_guide.md - +# NGramStore: canonical description + +NGramStore, formerly me_fasttext, is a compression, storage and exact-lookup +backend for learned n-gram vectors. Its native C engine serves original numeric +row IDs from compressed mmap blocks; registered string n-grams additionally +support DATrie indexing and structural mark-compact planning. + +FastText is the word-embedding application and historical method origin. +Engram/TinyEngram are LLM-memory integration examples, retaining the model's +hashing, head offsets, gates and backbone. The native backend is shared; their +training and addressing contracts remain different. + +## Fidelity and evidence + +Ordinary block packing and structural layout/exact modes preserve source bytes. +Joint structural sharing requires related registered strings plus cosine and +norm-ratio guards and can change weights. Engram hash collisions do not expose +this unique key registry. Compression, latency, RSS and downstream quality are +independent measurements. + +Actual legacy FastText trained-model outputs and historical valid queries match +byte-for-byte in the native facade. Trained TinyEngram/Qwen checks include CPU +and CUDA output comparisons and timing controls. Official DeepSeek Engram +verification uses its small demo; no official trained Engram checkpoint is +claimed. Native serving is slower than dense/raw mmap on several measured +traces, and lossless structural sharing often finds no identical learned rows. + +The original .z exporter now checks whole-key structure, cosine and norm ratio +against final representatives; earlier norm-only audits are historical. +Approximate .z sharing and joint mode must not be called lossless. + +## Current entry points + +- Repository: https://github.com/initial-d/ngram-store +- Engine guide: [pack, inspect, query](evaluator_quickstart.md) +- Word-embedding application: [FastText](fasttext_native_bundle.md) +- LLM-memory application: [Engram](engram_quickstart.md) +- [Trained evidence index](index.md) +- [Original paper and software citation](citation_guide.md) + +`ngram_store` and `ngram-store` are the engine import and command names. +`meft-memory`, `meft_memory`, old C ABI symbols and FastText formats remain +compatibility identifiers. The paper retains its original title; later software +results should be cited by exact commit/report and not retroactively attributed +as paper experiments. diff --git a/docs/artifact_card.md b/docs/artifact_card.md index c50bd75..dab2666 100644 --- a/docs/artifact_card.md +++ b/docs/artifact_card.md @@ -1,98 +1,32 @@ -# Artifact card - -This card summarizes `me_fasttext` for researchers, maintainers, and engineers -who need to decide quickly whether the project is relevant to cite, benchmark, -or list. - -## One-line description - -`me_fasttext` is a FastText-derived lexical embedding system that keeps word -and character n-gram identities explicit with trie-backed ids, then exports a -compact mmap-ready serving artifact through structure-aware row sharing and -mark-compact style row rewriting. - -## Best fit - -Use this artifact when your work involves: - -| Area | Fit | -| --- | --- | -| FastText-style embeddings | Comparing hash-bucketed n-grams with exact trie-backed n-gram ids. | -| OOV-heavy retrieval | Serving lexical vectors for rare words, names, misspellings, and long-tail entities. | -| Compact inference | Measuring artifact size, cold mmap load time, RSS, and p50/p95 lookup latency. | -| Double-array trie systems | Using trie terminal values as model row ids rather than only dictionary payloads. | -| Memory-management ideas | Applying mark, forwarding, and compaction concepts to a static model artifact. | - -## Main claims - -The artifact supports these narrow claims: - -- exact subword identity can be preserved during training before compression; -- trie-backed ids make word and n-gram rows inspectable before export; -- structure-aware row sharing can reduce serving artifact size; -- a compact `.z` artifact can be mmap-loaded for lexical embedding inference; -- mark-compact style row rewriting is a useful model-layout abstraction. - -## Evidence to report - -For Engram-style storage experiments, start with the -[byte-exact row-store guide](engram_storage.md). It documents the new Python -reference, its limitations and the historical legacy-export mismatch. The -[current legacy merge contract](legacy_merge_contract.md) maps the corrected -criterion to executable checks and outstanding reproduction work. Report -row-level byte equality separately from model quality and query performance. - -For citable reports, pair deployment metrics with task quality: - -| Evidence | Recommended metric | -| --- | --- | -| Artifact footprint | `.bin` size, `.z` size, `mmap_bytes` | -| Startup behavior | cold load time, cold mmap load time | -| Serving behavior | p50/p95 query latency, RSS after loading | -| Lexical coverage | OOV rate, OOV subword coverage | -| Downstream value | recall@k, MRR, NDCG, accuracy, or candidate recall before reranking | - -The [`bench_ftindex`](../src/bench_ftindex.cc) helper and -[`retrieval_oov_benchmark.md`](retrieval_oov_benchmark.md) provide the concrete -serving benchmark path. - -## Comparison baselines - -Strong comparisons include: - -- original FastText `.bin`; -- hash-free uncompressed FastText-style models; -- quantized FastText; -- BM25 or another lexical retrieval baseline; -- dense embedding models or rerankers when `me_fasttext` is used as a cheap - lexical stage before them. - -Use [`retrieval_comparison_matrix.md`](retrieval_comparison_matrix.md) for a -report table that keeps these comparisons narrow. - -## Citation hooks - -This is a good citation when a paper or report discusses: - -- memory-efficient FastText or subword embeddings; -- explicit n-gram identity before compression; -- compact lexical embedding inference; -- mmap serving artifacts for embedding tables; -- trie-backed NLP infrastructure; -- applying mark-compact memory-management ideas to model layout. - -Use [`citation_guide.md`](citation_guide.md) for BibTeX and suggested wording. - -## Not a fit - -Do not frame the artifact as: - -- a replacement for transformer embeddings; -- a vector database; -- a full RAG framework; -- a general-purpose garbage collector; -- a generic double-array trie library; -- a universal compression method for unrelated model families. - -The narrower story is stronger: `me_fasttext` is an exact-then-compact lexical -embedding artifact for memory-sensitive serving and OOV-heavy text. +# NGramStore artifact card + +**Compression, storage and exact lookup for learned n-gram vectors.** +Formerly me_fasttext; the original Memory-Efficient FastText paper is the method +origin, while native row storage and LLM integrations are later software work. + +| Component | What it provides | Boundary | +| --- | --- | --- | +| Native C engine | Lossless compressed blocks, original-ID forwarding, mmap and gather | Frozen single-worker handles; caches do not bound RSS | +| DATrie / mark-compact | Exact registered keys and structural representatives | Linux x86-64 trusted metadata; joint sharing can change weights | +| FastText application | Dense model export and existing vector/classification queries | This repository's custom unquantized models, not official .bin conversion | +| Engram / TinyEngram application | Storage replacement behind existing hashes and memory gathers | Demo and trained TinyEngram/Qwen evidence are distinct from official trained Engram | + +## Start and evaluate + +Use the [engine quickstart](evaluator_quickstart.md), then select an application. +[Trained evidence](index.md) separates identity, storage, query costs and task +quality. Small real-model native artifacts can save little and be slower than +dense/raw mmap; structural layout is not universally better. Numeric hash buckets +can use the row engine but do not certify structural string sharing. + +A comparison should preserve rows and query traces, state exact vs approximate +mode and include forwarding/index/metadata bytes. Report complete latency and +RSS alongside compression. Generic vectors are supported by the row engine; +stronger structural claims require a real unique n-gram registry. + +## Cite and reuse + +Use [the paper](https://arxiv.org/abs/2506.01254) for its method and +[the exact code/report](citation_guide.md) for later engine results. +This is a storage/query backend, with FastText and Engram application examples; +it is not a universal task-quality or production-performance guarantee. diff --git a/docs/citation_guide.md b/docs/citation_guide.md index cfb6576..5a52c87 100644 --- a/docs/citation_guide.md +++ b/docs/citation_guide.md @@ -1,7 +1,7 @@ # Citation guide This page is for researchers who want to cite, compare, or reproduce -`me_fasttext`. +NGramStore (formerly me_fasttext). For a one-page summary of the artifact, including fit, claims, evidence, and non-goals, see [`artifact_card.md`](artifact_card.md). @@ -25,7 +25,8 @@ useful in your work, please cite the arXiv paper: ## Cite the native n-gram artifact -The C gather ABI, immutable PyTorch adapters and Engram-related experiments are +NGramStore is the renamed repository, with the original paper title and method +attribution retained. The C gather ABI, immutable PyTorch adapters and Engram-related experiments are later software work. Cite their exact repository commit/report alongside the FastText paper when relevant; do not attribute the new results to that paper. Replace the bracketed commit below with the full SHA used in your experiment: @@ -33,10 +34,10 @@ Replace the bracketed commit below with the full SHA used in your experiment: ```bibtex @misc{du2026mefasttextngramstorage, author = {Yimin Du}, - title = {me_fasttext: Native n-gram storage and gather experiments}, + title = {NGramStore: Compression, Storage and Exact Lookup for Learned N-Gram Vectors}, year = {2026}, howpublished = {Software and reproducibility artifact}, - url = {https://github.com/initial-d/me_fasttext}, + url = {https://github.com/initial-d/ngram-store}, note = {Commit [full SHA]; report [report path and date]} } ``` @@ -47,10 +48,11 @@ verification, trained results and comparison requirements. Suggested wording for the tested boundary: -> The me_fasttext artifact provides immutable CPU storage/gather adapters that +> The NGramStore engine provides immutable storage/gather adapters that > preserve original hash IDs, head offsets and row bytes. Small source-pinned -> Engram demo checks also preserve gated module output; trained checkpoint, -> GPU-serving and task-quality validation remain open. +> Engram demo checks also preserve gated module output. Separate trained +> TinyEngram/Qwen CPU/CUDA checks and timing controls do not establish official +> trained DeepSeek Engram or production serving results. For approximate compression, name the actual trained table/corpus, threshold, norm guard, byte denominator and distortion metric. Do not describe the diff --git a/docs/engram_quickstart.md b/docs/engram_quickstart.md index 13bda1a..60791ea 100644 --- a/docs/engram_quickstart.md +++ b/docs/engram_quickstart.md @@ -1,6 +1,6 @@ -# Engram memory storage: run, evaluate, cite +# Engram application: NGramStore for LLM n-gram memory -`me_fasttext` provides immutable CPU n-gram table storage and gather adapters. +NGramStore provides the shared immutable n-gram table engine and gather adapters. For an existing Engram memory table, keep its original hash IDs and head offsets; replace the gather storage behind `MultiHeadEmbedding`. The default path recovers the original bytes. This project also studies approximate structural row sharing @@ -13,7 +13,8 @@ on trained character n-grams, with an explicit cosine/norm distortion budget. | Verify current official Engram module outputs | [Official CPU forward](engram_full_adapter_verification_20261018.md), [codec follow-up](native_xor_compression_20261020.md) | Nine cases with Python/native/native-XOR across three dtypes; tiny random weights | | Open a frozen table without allocating the default table first | Factory example below | Proposed fork API; nine cases including native-XOR, default allocation prevented | | Evaluate learned redundancy and compression | [Trained study](trained_ngram_similarity_20261018.md), [distortion curves](approximate_ngram_compression_20261018.md) | Nine trained character-ngram tables; 135 approximate cases, not trained Engram weights | -| Integrate another model/runtime | [C ABI and model guide](native_ngram_memory.md) | CPU gather boundary; GPU/checkpoint/runtime work remains | +| Convert a trained TinyEngram/Qwen checkpoint | [Checkpoint guide and trained evidence](tinyengram_qwen_evidence_20261030/README.md) | Trained CPU/CUDA outputs and timing controls; distinct from an official DeepSeek checkpoint | +| Integrate another model/runtime | [C ABI and model guide](native_ngram_memory.md) | Same native backend, model-specific addressing/gates retained | The [upstream draft PR #27](https://github.com/deepseek-ai/Engram/pull/27) proposes an optional `embedding_factory`. It is **open and unmerged** as checked @@ -69,7 +70,7 @@ checked loader to that layer's directory: ```python from functools import partial import torch -from tools.engram_row_adapter import open_demo_embedding +from ngram_store import open_demo_embedding factory = partial(open_demo_embedding, "/path/to/exported/layer-1", native_library="build/libmeft_rows.so") @@ -90,7 +91,10 @@ head partitions with the same total row count. A mismatch closes the store and raises an error. Each layer/worker needs its own handle. The factory does not restore projections/gates, tokenizer settings or the rest of a checkpoint. Calling `.to(dtype=...)` does not recode stored rows; use the exported dtype and -compatible CPU projections. Training and GPU/offload scheduling are unsupported. +compatible CPU projections. This factory path is frozen CPU inference. +The separate [device staging adapter](device_memory_prefetch.md) and +[TinyEngram/Qwen CUDA controls](qwen_cuda_performance_20261030/README.md) +document the measured device/checkpoint boundary. The [current-main replacement example](engram_full_adapter_verification_20261018.md) works without the proposal but initially allocates the original embedding table. diff --git a/docs/evaluator_quickstart.md b/docs/evaluator_quickstart.md index 9997028..8123fbb 100644 --- a/docs/evaluator_quickstart.md +++ b/docs/evaluator_quickstart.md @@ -1,92 +1,76 @@ -# Evaluator quickstart - -This page is for readers who have cloned `me_fasttext` and want to produce a -useful first signal quickly. The goal is not a perfect benchmark. The goal is a -clear public run that another researcher or engineer can understand, compare, -or reproduce. - -## What helps most - -A useful report answers one narrow question: - -> On this corpus and machine, does the compact `.z` mmap-serving artifact reduce -> memory or startup cost while preserving useful lexical behavior? - -Report one of these outcomes: - -| Outcome | Why it helps | -| --- | --- | -| Positive result | Shows where exact-then-compact lexical embeddings transfer. | -| Negative result | Shows where the method does not help or is too costly. | -| Partial result | Records build, corpus, or serving friction for the next evaluator. | - -## Fifteen-minute orientation - -1. Read the [artifact card](artifact_card.md) to confirm the project fits your - use case. -2. Build the tools on Linux: - - ```bash - make opt - ``` - -3. Pick a public or redistributable plain-text corpus. Product titles, news - text, encyclopedia titles, entity-heavy logs, and Chinese or multilingual - text are especially useful. -4. Generate a manifest and query slices: - - ```bash - python3 tools/prepare_public_benchmark.py \ - --input data/public_train.txt \ - --output-dir runs/public-benchmark \ - --name "example-public-corpus" \ - --language en \ - --license "public / CC / unknown" - ``` - -5. Train a small first model: - - ```bash - /usr/bin/time -v ./fasttext skipgram \ - -input data/public_train.txt \ - -output runs/public-meft \ - -dim 128 \ - -minn 2 \ - -maxn 6 \ - -minCount 5 \ - -thread 8 - ``` - -6. Measure the compact serving artifact: - - ```bash - ./bench_ftindex runs/public-meft.z runs/public-benchmark/queries.txt 1 3 - ``` - -## Minimum report - -Open a [benchmark result issue](https://github.com/initial-d/me_fasttext/issues/new?template=benchmark-result.yml) -with: - -- exact commit SHA; -- corpus name, language, license status, and preprocessing; -- build and training commands; -- hardware, OS, compiler, filesystem, and storage type; -- `.bin`, `.vec`, and `.z` artifact sizes; -- `bench_ftindex` output; -- one quality metric, or an explicit serving-only caveat; -- anything that failed, surprised you, or made the method look bad. - -If you cannot share the corpus, share aggregate metrics and the generated -`corpus_manifest.json` fields instead of private text. - -## What not to claim - -Keep the comparison narrow. `me_fasttext` is not a replacement for transformer -embeddings, a vector database, or a full RAG system. It is most useful as a -compact lexical embedding layer for memory-sensitive serving, OOV-heavy text, -and first-stage retrieval or ranking. - -For a fuller protocol, continue with -[`first_public_corpus_benchmark.md`](first_public_corpus_benchmark.md) and -[`retrieval_oov_benchmark.md`](retrieval_oov_benchmark.md). +# NGramStore engine quickstart + +Start with the storage engine, then choose FastText or Engram as the application. +Use an existing learned table; deterministic fixtures establish correctness only. + +## Install and pack numeric rows + +From a Linux/macOS source checkout with a C compiler and zlib headers: + +```bash +python -m pip install . +ngram-store pack --input vectors.f32 --dim 256 --output memory.native +ngram-store inspect --store memory.native +ngram-store query --store memory.native --ids 0 42 0 +``` + +`vectors.f32` must be a nonempty, contiguous little-endian float32 table. +`--dim` is its row width in floats. Output must be a fresh directory. No dtype +conversion or rounding occurs. All reconstructed row bytes are verified before +publication. `--deduplicate` optionally shares identical row bytes with an +index that grows with the number of unique rows. Default packing omits it. + +Use `query --hex` for signed-zero/nonfinite payload inspection. Numeric JSON +output requires finite values. The query keeps original IDs, order and duplicates. +`--max-block-bytes` controls decoded block width; one indivisible row can exceed +it. `--cache-bytes` controls query cache payload, not total process memory. + +The wheel distribution remains `meft-memory`; new code imports `ngram_store`. +Existing imports are aliases of the same API, not independent implementations. +`python -m ngram_store` also runs directly from a checkout; pass +`--native-library build/libmeft_rows.so` after `make native` if not installed. +On macOS use `make native NATIVE_SHARED=-dynamiclib NATIVE_EXT=dylib`. + +## Registered string n-grams + +On Linux x86-64, the optional original DATrie bridge is built with: + +```bash +make structural-native +ngram-store pack --input vectors.f32 --dim 256 --output keyed.native \ + --keys keys.json --structural-library build/libmeft_structural.so --mode exact +ngram-store query --store keyed.native --keys '' '' \ + --structural-library build/libmeft_structural.so +``` + +`keys.json` maps canonical original row IDs to unique registered strings, for +example `{"0":"","42":""}`. Unkeyed rows retain their identities. +`layout` keeps all rows; `exact` shares only byte-identical structural pairs; +`joint` changes rows under cosine/norm-ratio guards. Generic byte dedup and +structural representative selection are separate options. + +A many-to-one hash bucket registry does not provide this key-to-row contract. +Engram's numeric/hash path works through the same row engine without assuming +prefix/suffix relationships. Open only trusted self-produced DATrie files. + +## Choose an application + +- [FastText prepare/train/export/query](fasttext_native_bundle.md): includes a + real tiny text-training smoke path; the current trainer requires positional/source + pairs rather than ordinary upstream FastText training input. +- [Engram demonstration](engram_quickstart.md): preserve existing hashes and + head offsets, source-pin the demo and verify full module output. +- [TinyEngram/Qwen checkpoint](tinyengram_qwen_evidence_20261030/README.md): + convert and verify actual trained memory weights with the existing backbone. + +## Report a useful result + +Record the exact commit, command, data and split, source/artifact hashes, dtype, +row count, compiler/Python/PyTorch versions, CPU/GPU and storage environment. +Compare with raw mmap/dense lookup using the same rows and trace. Report whole +artifact bytes, p50/p95 latency, load time, anonymous/file-backed RSS, cache and +block settings, output fidelity and task quality separately. Name approximate +thresholds and denominators. Include slow, failed and no-benefit cases. + +[Current results](index.md) include strong negative controls. Legacy lexical +retrieval protocols remain in [the compatibility guide](legacy_fasttext.md). diff --git a/docs/fasttext_native_bundle.md b/docs/fasttext_native_bundle.md index 741655a..b47416f 100644 --- a/docs/fasttext_native_bundle.md +++ b/docs/fasttext_native_bundle.md @@ -1,12 +1,47 @@ -# Native storage through the original FastText model and CLI +# FastText application: NGramStore native storage and query -Dense **me_fasttext** `.bin` models can be converted into immutable native +Dense historical **me_fasttext** `.bin` models can be converted into immutable native serving bundles. `loadModel(path)` and existing query commands then use the original dictionary and row IDs, with input vectors stored in compressed native blocks. The production DATrie prefix/suffix planner and mark-compact forwarding are part of this conversion. [Actual trained-model results and limits](fasttext_native_evidence_20261031/README.md) include a historical-main comparison and a matched identity-layout control. +## Complete text-training smoke path + +On Linux x86-64, this included real-text sample needs no external dataset or GPU. +It verifies preparation, actual training, conversion and querying, not representative +compression or task quality: + +```bash +python -m pip install . +make fasttext-native native structural-native +mkdir -p artifacts/fasttext-sample +ngram-store prepare-fasttext --input examples/fasttext_sample.txt \ + --output artifacts/fasttext-sample/pairs.txt +``` + +The trainer uses single-dash flags. Run the training command from the owned output +directory because it also writes legacy trie dumps: + +```bash +cd artifacts/fasttext-sample +../../build/fasttext-native skipgram -input pairs.txt -output model \ + -dim 32 -minCount 1 -minn 2 -maxn 6 -epoch 1 -thread 1 -verbose 0 +cd ../.. +ngram-store fasttext-export --model artifacts/fasttext-sample/model.bin \ + --output-dir artifacts/fasttext-sample/model.native \ + --fasttext-binary build/fasttext-native \ + --structural-library build/libmeft_structural.so +printf 'memory\nvectors\nlookup\n' | build/fasttext-native print-word-vectors \ + artifacts/fasttext-sample/model.native +``` + +Preparation preserves within-line word order and emits `p#0 query\x01source` +pairs. This is the current custom positional/source SGNS protocol. Ordinary +upstream FastText whitespace training input is not silently interchangeable. +Use fresh paths when repeating; retained dense `.bin` remains the source model. + ## Export and query The optional facade and bundled DATrie archives require Linux x86-64. Build from @@ -16,11 +51,11 @@ this repository; the standalone memory wheel does not bundle the legacy trainer. make fasttext-native native structural-native # Use a NEW output directory. Keep the original .bin for training/quantization. -python tools/export_fasttext_native_cli.py \ +ngram-store fasttext-export \ --model model.bin --output-dir model.native \ --fasttext-binary build/fasttext-native \ --structural-library build/libmeft_structural.so \ - --native-library build/libmeft_rows.so --mode exact + --mode exact # The same command and arguments accept either a .bin or a bundle directory. printf 'example\nexamples\n' | build/fasttext-native print-word-vectors model.native diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..ea73dba --- /dev/null +++ b/docs/index.md @@ -0,0 +1,32 @@ +# NGramStore guides and evidence + +## Engine and application entry points + +- [Engine: pack, inspect and query](evaluator_quickstart.md) +- [C ABI, native construction and runtime boundaries](native_ngram_memory.md) +- [Word embeddings: FastText prepare/train/export/query](fasttext_native_bundle.md) +- [LLM memory: official Engram demo and factory](engram_quickstart.md) +- [TinyEngram/Qwen checkpoint integration](tinyengram_qwen_evidence_20261030/README.md) +- [Historical me_fasttext compatibility](legacy_fasttext.md) +- [Citation guide](citation_guide.md) and [artifact card](artifact_card.md) + +## Source-linked trained results + +- [Actual legacy FastText model and matched layout controls](fasttext_native_evidence_20261031/README.md) +- [Original DATrie / mark-compact contribution ablations](structural_native_evidence_20261030/README.md) +- [1.85M trained-row CPU capacity](cpu_capacity_evidence_20261030/README.md) +- [Sparse-block native miss latency](sparse_blocks_evidence_20261030/README.md) +- [Native query/cache controls](native_query_evidence_20261030/README.md) +- [Causal language-model quality controls](causal_lm_evidence_20261024/README.md) +- [Held-out lossless compression](heldout_compression_evidence_20261024/README.md) +- [Native embedding-bag memory and latency](embedding_bag_evidence_20261025/README.md) +- [Trained n-gram similarity](trained_ngram_similarity_20261018.md) +- [Approximate vector distortion curves](approximate_ngram_compression_20261018.md) +- [Qwen CUDA inference timing controls](qwen_cuda_performance_20261030/README.md) +- [A100 output correctness](cuda_a100_evidence_20261030/README.md) + +These reports keep their original run dates, source commits, commands and +historical project names. A rename does not change their evidence or scope. +Official Engram demo fixtures, trained references and TinyEngram/Qwen results +are separate categories. Raw mmap/dense controls and no-benefit results remain +close to claimed savings. [Contribution/reporting guide](../CONTRIBUTING.md). diff --git a/docs/legacy_fasttext.md b/docs/legacy_fasttext.md new file mode 100644 index 0000000..bcfa568 --- /dev/null +++ b/docs/legacy_fasttext.md @@ -0,0 +1,44 @@ +# Historical me_fasttext compatibility + +NGramStore is the renamed repository and shared n-gram vector backend. +FastText remains the original word-embedding application. Repository history, +issues, releases and original paper attribution are retained. Old GitHub URLs +redirect after rename; update local remotes to the canonical address: + +```bash +git remote set-url origin https://github.com/initial-d/ngram-store.git +``` + +## Retained interfaces + +- `meft-memory` installation and `meft_memory` imports continue; new callers + use `ngram_store`. Both names expose the same runtime objects. +- Existing `meft-*` commands remain; `ngram-store` adds a common dispatcher. +- C symbols and MEFTNAT1 row bytes are unchanged. Rebuild C++ consumers after + source updates; preserving source calls is not an old C++ binary ABI promise. +- FastText `.bin` and optional native bundle loading preserve valid historical + queries. `.z` retains the separate FTIndex reader and its approximate contract. +- Existing `make opt` and `make debug` targets remain available on Linux. + +## Train current source correctly + +This repository's custom SGNS reader recognizes `p#` positions and +`query\x01source` pairs. Ordinary whitespace text and standard supervised-label +examples from upstream FastText are not a verified direct training path here. +Use [the tested preparation/training workflow](fasttext_native_bundle.md), then +retain `.bin` for training/quantization and export native for frozen queries. +Existing supervised models can be queried through the native facade. + +The CLI trainer still also writes its legacy .z and trie dumps in the working +directory. Run in your own output directory. Re-exporting through the current +.z exporter uses explicit structure, cosine and norm-ratio guards; old .z files +keep their original contents. [Legacy contract and checks](legacy_merge_contract.md). + +## Historical research pages + +[Lexical retrieval comparisons](retrieval_comparison_matrix.md), +[legacy FTIndex benchmarks](retrieval_oov_benchmark.md), and older outreach and +release notes document the FastText application and historical positioning. +Their old project names are intentional history, not a second active engine. +For new reports, start from the [current engine guide](evaluator_quickstart.md) +and [evidence index](index.md). The paper title and BibTeX remain unchanged. diff --git a/docs/native_ngram_memory.md b/docs/native_ngram_memory.md index cd53c3f..9ae5f22 100644 --- a/docs/native_ngram_memory.md +++ b/docs/native_ngram_memory.md @@ -1,6 +1,6 @@ -# Native n-gram memory: C ABI and model integration +# NGramStore engine: C ABI and model integration -The native backend is a CPU storage/gather component for immutable embedding +NGramStore's native backend is a CPU storage/gather component for immutable embedding tables. It accepts original numeric row IDs and writes identical bytes into caller-owned memory. It is a place to optimize storage and query behavior; it does not replace a model's tokenizer, n-gram hasher, contextual gates, @@ -31,7 +31,7 @@ This builds a standalone shared library and C example independently of the bundled legacy FastText/trie binaries. From a raw fixed-width byte table: ```python -from tools.native_row_store import pack_native +from ngram_store import pack_native # Example: 128 float32 values per row. No numeric/dtype conversion. pack_native("table.raw", "table.men", row_bytes=512, block_rows=256, diff --git a/examples/fasttext_sample.txt b/examples/fasttext_sample.txt new file mode 100644 index 0000000..6eb0669 --- /dev/null +++ b/examples/fasttext_sample.txt @@ -0,0 +1,6 @@ +memory tables store learned vectors and queries retrieve learned vectors from memory tables +exact lookup preserves original row identities while compact storage preserves original vector bytes +memory tables support word embeddings and language model memory through shared storage and lookup +compact storage and exact lookup work together while learned vectors retain original row identities +word embeddings use character ngrams and language models use token ngrams in learned memory tables +original vector bytes and original row identities remain stable through compact storage and exact lookup diff --git a/meft_memory/__init__.py b/meft_memory/__init__.py index dccc177..43eb02d 100644 --- a/meft_memory/__init__.py +++ b/meft_memory/__init__.py @@ -1,9 +1,9 @@ -"""Installable native memory API; Torch/Transformers adapters load on demand.""" +"""Historical NGramStore API; Torch/Transformers adapters load on demand.""" from importlib import import_module from ._runtime.native_row_store import NativeRows, pack_native -__version__ = "0.3.0" +__version__ = "0.4.0" _ADAPTERS = { "StructuralPlan": "structural_native", @@ -13,6 +13,8 @@ "GatherTicket": "device_row_adapter", "MappedEmbedding": "engram_row_adapter", "export_multihead": "engram_row_adapter", + "open_demo_embedding": "engram_row_adapter", + "export_demo_embedding": "engram_row_adapter", "export_embedding": "engram_row_adapter", "MappedEmbeddingBag": "mapped_embedding_bag", "TinyEngramModel": "tinyengram_checkpoint", diff --git a/meft_memory/_runtime/__init__.py b/meft_memory/_runtime/__init__.py new file mode 100644 index 0000000..06b8faf --- /dev/null +++ b/meft_memory/_runtime/__init__.py @@ -0,0 +1,6 @@ +"""Source-checkout bridge; installed wheels map this package directly to tools/.""" +from pathlib import Path + +_checkout_tools = Path(__file__).resolve().parents[2] / "tools" +if _checkout_tools.is_dir(): + __path__.append(str(_checkout_tools)) diff --git a/ngram_store/__init__.py b/ngram_store/__init__.py new file mode 100644 index 0000000..e1124e5 --- /dev/null +++ b/ngram_store/__init__.py @@ -0,0 +1,9 @@ +"""NGramStore engine API; historical meft_memory imports remain compatible.""" +import meft_memory as _compatible +from meft_memory import NativeRows, pack_native, __version__ + +__all__ = list(_compatible.__all__) + + +def __getattr__(name): + return getattr(_compatible, name) diff --git a/ngram_store/__main__.py b/ngram_store/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/ngram_store/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/ngram_store/cli.py b/ngram_store/cli.py new file mode 100644 index 0000000..1eb5c4a --- /dev/null +++ b/ngram_store/cli.py @@ -0,0 +1,156 @@ +"""NGramStore: pack, inspect and query frozen n-gram vector tables.""" +import argparse +import hashlib +from importlib import import_module +import json +import os +from pathlib import Path +import struct +import sys +import tempfile + +from . import NativeRows, pack_native + +FORWARD={ + 'prepare-fasttext':('prepare_fasttext_corpus','Prepare plain text for the custom FastText SGNS trainer.'), + 'fasttext-export':('fasttext_native_bundle','Export a dense legacy FastText model into the same native backend.'), + 'tiny-export':('tinyengram_checkpoint','Export a TinyEngram/Qwen checkpoint.'), + 'tiny-verify':('verify_tinyengram_checkpoint','Verify a converted TinyEngram checkpoint on CPU.'), +} + + +def sha256(path): + digest=hashlib.sha256() + with Path(path).open('rb') as stream: + for part in iter(lambda:stream.read(1024**2),b''):digest.update(part) + return digest.hexdigest() + + +def pack_store(source, output, dim, *, native_library=None, keys=None, structural_library=None, + mode='exact', cosine=.99, norm_ratio=.99, max_block_bytes=8192, deduplicate=False): + if sys.byteorder!='little':raise ValueError('CLI raw input requires little-endian IEEE float32 host') + source,output=Path(source).resolve(),Path(output).resolve() + if type(dim) is not int or dim<=0:raise ValueError('dim must be positive') + rows,tail=divmod(source.stat().st_size,dim*4) + if not rows or tail:raise ValueError('input must contain nonempty complete float32 rows') + if output.exists():raise FileExistsError('use a fresh store directory') + if mode not in ('layout','exact','joint'):raise ValueError('invalid structural mode') + if keys is None and mode=='joint':raise ValueError('joint sharing requires registered string keys') + if keys is not None and deduplicate:raise ValueError('structural representatives and generic deduplication are separate modes') + source_hash=sha256(source); output.parent.mkdir(parents=True,exist_ok=True) + with tempfile.TemporaryDirectory(prefix='.ngram-store-',dir=output.parent) as tmp: + candidate=Path(tmp)/'bundle' + if keys is not None: + module=import_module('meft_memory._runtime.structural_native') + with module.StructuralPlan(source.read_bytes(),dim,keys,structural_library, + mode=mode,cosine=cosine,norm_ratio=norm_ratio) as plan: + storage=plan.export(source,candidate,max_block_bytes=max_block_bytes) + expected=plan.reconstructed_sha256; preserved=plan.original_bytes_preserved + else: + candidate.mkdir() + storage=pack_native(source,candidate/'table.men',dim*4,256,deduplicate,predictor='xor', + max_block_bytes=max_block_bytes) + expected=source_hash; preserved=True + reconstructed=hashlib.sha256() + with NativeRows(candidate/'table.men',native_library) as table: + for begin in range(0,rows,4096):reconstructed.update(table.rows(range(begin,min(begin+4096,rows)))) + if reconstructed.hexdigest()!=expected:raise ValueError('full-row reconstruction differs from selected representatives') + if preserved and expected!=source_hash:raise ValueError('lossless pack changed source bytes') + if sha256(source)!=source_hash:raise ValueError('source changed during export') + artifact_hash=sha256(candidate/'table.men') + # The same artifact can be opened by the existing frozen PyTorch adapter. + embedding={'schema_version':1,'table_file':'table.men','shape':[rows,dim], + 'dtype':'torch.float32','byteorder':'little','head_sizes':[rows],'offsets':[0], + 'backend':'native','weight_sha256':expected,'source_weight_sha256':source_hash, + 'artifact_sha256':artifact_hash,'approximate':not preserved,**storage} + (candidate/'embedding.json').write_text(json.dumps(embedding,indent=2)+'\n') + metadata={'schema_version':1,'engine':'NGramStore','shape':[rows,dim],'dtype':'float32-le', + 'source_sha256':source_hash,'reconstructed_sha256':expected,'artifact_sha256':artifact_hash, + 'original_bytes_preserved':preserved,'keyed_rows':len(keys) if keys is not None else 0, + 'structural_mode':mode if keys is not None else None,'storage':storage, + 'contract':'Original numeric IDs; decoder preserves representative bytes. Joint sharing can change source weights.', + 'limitations':['Frozen single-worker handles; cache budget is not an RSS bound.', + 'Structural mode requires trusted Linux x86-64 DATrie artifacts and full offline planning.', + 'Hash collisions do not certify prefix/suffix relations.']} + (candidate/'store.json').write_text(json.dumps(metadata,indent=2)+'\n') + if output.exists():raise FileExistsError('store directory appeared during export') + os.rename(candidate,output) + return metadata + + +def read_keys(path): + raw=json.loads(Path(path).read_text()) + if not isinstance(raw,dict):raise ValueError('keys JSON must map original row IDs to UTF-8 strings') + result={} + for key,value in raw.items(): + if not key.isdigit() or str(int(key))!=key or not isinstance(value,str): + raise ValueError('keys must use canonical nonnegative row IDs and string values') + result[int(key)]=value + return result + + +def open_metadata(path): + root=Path(path) + name='store.json' if (root/'store.json').is_file() else 'fasttext.json' + metadata=json.loads((root/name).read_text()) + if metadata.get('schema_version')!=1:raise ValueError('unsupported store metadata') + return metadata + + +def main(argv=None): + argv=list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] in FORWARD: + name=argv[0]; old=sys.argv + try: + sys.argv=['ngram-store '+name,*argv[1:]] + return import_module('meft_memory._runtime.'+FORWARD[name][0]).main() + finally:sys.argv=old + parser=argparse.ArgumentParser(description=__doc__) + commands=parser.add_subparsers(dest='command',required=True) + pack=commands.add_parser('pack',help='Pack existing little-endian float32 rows and verify every reconstructed row.') + pack.add_argument('--input',type=Path,required=True); pack.add_argument('--output',type=Path,required=True) + pack.add_argument('--dim',type=int,required=True); pack.add_argument('--native-library',type=Path) + pack.add_argument('--keys',type=Path); pack.add_argument('--structural-library',type=Path) + pack.add_argument('--mode',choices=('layout','exact','joint'),default='exact') + pack.add_argument('--cosine',type=float,default=.99); pack.add_argument('--norm-ratio',type=float,default=.99) + pack.add_argument('--max-block-bytes',type=int,default=8192); pack.add_argument('--deduplicate',action='store_true') + inspect=commands.add_parser('inspect',help='Inspect row shape, storage bytes and fidelity contract.') + query=commands.add_parser('query',help='Query original row IDs or registered string keys.') + for command in (inspect,query): + command.add_argument('--store',type=Path,required=True); command.add_argument('--native-library',type=Path) + selection=query.add_mutually_exclusive_group(required=True) + selection.add_argument('--ids',type=int,nargs='+'); selection.add_argument('--keys',nargs='+') + query.add_argument('--structural-library',type=Path); query.add_argument('--cache-bytes',type=int,default=4*1024**2) + query.add_argument('--hex',action='store_true',help='Return original row bytes, including nonfinite/signed-zero bits.') + for name,(_,help_text) in FORWARD.items():commands.add_parser(name,help=help_text,add_help=False) + args=parser.parse_args(argv) + try: + if args.command=='pack': + result=pack_store(args.input,args.output,args.dim,native_library=args.native_library, + keys=read_keys(args.keys) if args.keys else None,structural_library=args.structural_library, + mode=args.mode,cosine=args.cosine,norm_ratio=args.norm_ratio, + max_block_bytes=args.max_block_bytes,deduplicate=args.deduplicate) + else: + metadata=open_metadata(args.store); rows,dim=metadata['shape'] + ids=None + if args.command=='query': + ids=args.ids + if args.keys is not None: + module=import_module('meft_memory._runtime.structural_native') + with module.StructuralIndex(args.store/'keys.das',args.structural_library) as index:ids=index.lookup(args.keys) + if None in ids:raise ValueError('query contains an unregistered key') + with NativeRows(args.store/'table.men',args.native_library, + args.cache_bytes if args.command=='query' else 0) as table: + if table.header['n_rows']!=rows or table.header['row_bytes']!=dim*4: + raise ValueError('row-store shape differs from metadata') + if ids is None:result=metadata + else: + raw=table.rows(ids) + result={'ids':ids,'original_bytes_preserved':metadata['original_bytes_preserved']} + if args.hex:result['rows_hex']=[raw[i*dim*4:(i+1)*dim*4].hex() for i in range(len(ids))] + else:result['vectors']=[list(struct.unpack_from('<'+str(dim)+'f',raw,i*dim*4)) for i in range(len(ids))] + print(json.dumps(result,indent=2,allow_nan=False)) + except (ValueError,OSError,KeyError,TypeError) as exc:parser.error(str(exc)) + + +if __name__=='__main__':main() diff --git a/pyproject.toml b/pyproject.toml index 47e4a44..3fc15c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,19 +4,20 @@ build-backend = "setuptools.build_meta" [project] name = "meft-memory" -version = "0.3.0" -description = "Native lossless row storage and frozen learned n-gram memory adapters" +version = "0.4.0" +description = "NGramStore: compression, storage and exact lookup for learned n-gram vectors" requires-python = ">=3.10" license = {text = "MIT"} [project.urls] -Repository = "https://github.com/initial-d/me_fasttext" +Repository = "https://github.com/initial-d/ngram-store" [project.optional-dependencies] torch = ["torch", "numpy"] tinyengram = ["torch", "numpy", "sympy", "loguru", "transformers==4.57.3", "tokenizers==0.22.2", "safetensors==0.7.0"] [project.scripts] +ngram-store = "ngram_store.cli:main" meft-fasttext-export = "meft_memory._runtime.fasttext_native_bundle:main" meft-tiny-export = "meft_memory._runtime.tinyengram_checkpoint:main" meft-tiny-verify = "meft_memory._runtime.verify_tinyengram_checkpoint:main" @@ -24,5 +25,5 @@ meft-tiny-verify-device = "meft_memory._runtime.verify_tinyengram_device:main" meft-tiny-train-reference = "meft_memory._runtime.train_tinyengram_reference:main" [tool.setuptools] -packages = ["meft_memory", "meft_memory._runtime"] +packages = ["ngram_store", "meft_memory", "meft_memory._runtime"] package-dir = {"meft_memory._runtime" = "tools"} diff --git a/tests/test_fasttext_native_bundle.py b/tests/test_fasttext_native_bundle.py index b6fa1b0..422f0f3 100644 --- a/tests/test_fasttext_native_bundle.py +++ b/tests/test_fasttext_native_bundle.py @@ -3,6 +3,7 @@ import os from pathlib import Path import platform +import sys import shutil import struct import subprocess @@ -80,5 +81,26 @@ def test_real_model_all_rows_and_legacy_inference_are_exact(self): export_fasttext_model(invalid,failed,fasttext_binary=self.binary,structural_library=self.structural) self.assertFalse(failed.exists()) + def test_documented_plain_text_prepare_train_export_query_workflow(self): + with tempfile.TemporaryDirectory(dir=self.root) as tmp: + root=Path(tmp); corpus=root/'pairs.txt'; model=root/'sample'; bundle=root/'native' + environment=dict(os.environ,PYTHONPATH=str(ROOT)) + prepare=subprocess.run([sys.executable,'-m','ngram_store','prepare-fasttext', + '--input',str(ROOT/'examples/fasttext_sample.txt'),'--output',str(corpus)], + cwd=root,env=environment,check=True,capture_output=True,text=True) + prepared=json.loads(prepare.stdout) + self.assertGreater(prepared['pair_tokens'],0) + subprocess.run([str(self.binary),'skipgram','-input',str(corpus),'-output',str(model), + '-dim','32','-minCount','1','-minn','2','-maxn','6','-epoch','1','-thread','1','-verbose','0'], + cwd=root,check=True,capture_output=True) + subprocess.run([sys.executable,'-m','ngram_store','fasttext-export', + '--model',str(root/'sample.bin'),'--output-dir',str(bundle), + '--fasttext-binary',str(self.binary),'--structural-library',str(self.structural), + '--native-library',str(self.library)],cwd=root,env=environment,check=True,capture_output=True) + words='memory\nvectors\nlookup\n' + dense=subprocess.run([str(self.binary),'print-word-vectors',str(root/'sample.bin')],input=words,text=True,capture_output=True,check=True) + native=subprocess.run([str(self.binary),'print-word-vectors',str(bundle)],input=words,text=True,capture_output=True,check=True) + self.assertEqual([line for line in dense.stdout.splitlines() if line.split(' ')[0] in words.split()],native.stdout.splitlines()) + if __name__=='__main__':unittest.main() diff --git a/tests/test_ngram_store.py b/tests/test_ngram_store.py new file mode 100644 index 0000000..21147ec --- /dev/null +++ b/tests/test_ngram_store.py @@ -0,0 +1,89 @@ +"""Engine entry points and corpus-to-trainer protocol; fixtures are correctness only.""" +import contextlib +import hashlib +import io +import json +from pathlib import Path +import platform +import struct +import subprocess +import tempfile +import unittest + +import meft_memory +import ngram_store +from ngram_store.cli import main,pack_store +from tools.prepare_fasttext_corpus import prepare_corpus + +ROOT=Path(__file__).resolve().parents[1] + + +class CorpusTest(unittest.TestCase): + def test_adjacent_pairs_utf8_order_line_boundaries_and_failure_publication(self): + with tempfile.TemporaryDirectory() as tmp: + source=Path(tmp)/'plain.txt'; output=Path(tmp)/'pairs.txt' + source.write_text('Alpha beta gamma delta epsilon\n记忆 查询 向量\nalone\n') + report=prepare_corpus(source,output,words_per_line=3,lowercase=True) + self.assertEqual(output.read_text(),'p#0 alpha\x01beta beta\x01gamma\np#0 delta\x01epsilon\np#0 记忆\x01查询 查询\x01向量\n') + self.assertEqual((report['pair_tokens'],report['prepared_lines'],report['skipped_singletons']),(5,3,1)) + before=output.read_bytes() + with self.assertRaises(FileExistsError):prepare_corpus(source,output) + self.assertEqual(output.read_bytes(),before) + bad=Path(tmp)/'bad.txt'; bad.write_text('valid words\ninvalid\x01token x\n') + failed=Path(tmp)/'failed.txt' + with self.assertRaises(ValueError):prepare_corpus(bad,failed) + self.assertFalse(failed.exists()) + bad.write_text('singleton\n') + with self.assertRaises(ValueError):prepare_corpus(bad,failed) + self.assertFalse(failed.exists()) + + +class EngineTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp=tempfile.TemporaryDirectory(); cls.root=Path(cls.tmp.name) + cls.library=cls.root/('rows.dylib' if platform.system()=='Darwin' else 'rows.so') + subprocess.run(['cc','-std=c11','-O2','-fPIC','-dynamiclib' if platform.system()=='Darwin' else '-shared', + str(ROOT/'native/meft_rows.c'),'-lz','-o',str(cls.library)],check=True) + + @classmethod + def tearDownClass(cls):cls.tmp.cleanup() + + def test_old_imports_share_objects_and_all_bits_survive_cli_and_adapter_metadata(self): + self.assertIs(ngram_store.NativeRows,meft_memory.NativeRows) + self.assertIs(ngram_store.StructuralPlan,meft_memory.StructuralPlan) + self.assertEqual(ngram_store.__all__,meft_memory.__all__) + with tempfile.TemporaryDirectory(dir=self.root) as tmp: + root=Path(tmp); source=root/'input.f32'; bundle=root/'store' + raw=struct.pack('<6I',0,0x80000000,0x7fc00042,0x3f800000,0,0x80000000) + source.write_bytes(raw) + result=pack_store(source,bundle,2,native_library=self.library,deduplicate=True,max_block_bytes=8) + self.assertTrue(result['original_bytes_preserved']) + self.assertEqual(result['source_sha256'],hashlib.sha256(raw).hexdigest()) + self.assertEqual(result['reconstructed_sha256'],result['source_sha256']) + self.assertEqual(result['storage']['unique_rows'],2) + with ngram_store.NativeRows(bundle/'table.men',self.library) as table: + self.assertEqual(table.rows([2,1,0,2]),raw[16:24]+raw[8:16]+raw[:8]+raw[16:24]) + captured=io.StringIO() + with contextlib.redirect_stdout(captured):main(['query','--store',str(bundle),'--ids','2','1','0','2','--hex','--native-library',str(self.library)]) + self.assertEqual(json.loads(captured.getvalue())['rows_hex'],[raw[i*8:i*8+8].hex() for i in [2,1,0,2]]) + metadata=json.loads((bundle/'embedding.json').read_text()) + self.assertEqual(metadata['shape'],[3,2]); self.assertEqual(metadata['head_sizes'],[3]) + self.assertEqual(metadata['weight_sha256'],result['source_sha256']) + with self.assertRaises(FileExistsError):pack_store(source,bundle,2,native_library=self.library) + bad=root/'partial.f32'; bad.write_bytes(b'123') + with self.assertRaises(ValueError):pack_store(bad,root/'failed',2,native_library=self.library) + self.assertFalse((root/'failed').exists()) + + def test_packed_engine_store_opens_existing_torch_adapter(self): + try:import torch + except ImportError:self.skipTest('optional torch dependency') + with tempfile.TemporaryDirectory(dir=self.root) as tmp: + root=Path(tmp); source=root/'trained-fixture.f32'; source.write_bytes(struct.pack('<6f',1,2,3,4,1,2)) + bundle=root/'store'; pack_store(source,bundle,2,native_library=self.library) + model=ngram_store.MappedEmbedding(bundle,native_library=self.library) + try:self.assertTrue(torch.equal(model(torch.tensor([2,0,1])),torch.tensor([[1.,2.],[1.,2.],[3.,4.]]))) + finally:model.close() + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_structural_native.py b/tests/test_structural_native.py index 8ba3a8c..2b45d34 100644 --- a/tests/test_structural_native.py +++ b/tests/test_structural_native.py @@ -97,6 +97,24 @@ def test_joint_final_guard_rejects_direction_norm_and_chain_drift(self): self.assertGreaterEqual(cosine, .99) self.assertGreaterEqual(ratio, .99) + def test_engine_cli_uses_original_key_ids_after_structural_compaction(self): + import contextlib,io,json + from ngram_store.cli import pack_store,main + raw=struct.pack('<8f',9,8,1,0,1,0,0,1) + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp); source=root/'input.f32'; source.write_bytes(raw) + bundle=root/'store' + report=pack_store(source,bundle,2,native_library=self.rows_library, + keys={1:'a',2:'ab',3:'z'},structural_library=self.library) + self.assertTrue(report['original_bytes_preserved']) + self.assertEqual(report['storage']['unique_rows'],3) + capture=io.StringIO() + with contextlib.redirect_stdout(capture):main(['query','--store',str(bundle),'--keys','ab','a','z','ab', + '--structural-library',str(self.library),'--native-library',str(self.rows_library)]) + result=json.loads(capture.getvalue()) + self.assertEqual(result['ids'],[2,1,3,2]) + self.assertEqual(result['vectors'],[[1,0],[1,0],[0,1],[1,0]]) + def test_layout_only_bytes_exact_signed_zero_and_duplicate_keys(self): raw = struct.pack("<6f", 0., 0., -0., 0., 0., 0.) keys = {0: "a", 1: "ab", 2: "abc"} diff --git a/tools/check_memory_wheel.py b/tools/check_memory_wheel.py index ab99479..8e9b264 100644 --- a/tools/check_memory_wheel.py +++ b/tools/check_memory_wheel.py @@ -4,9 +4,14 @@ import os from pathlib import Path import platform +import json +import struct +import subprocess +import sys import tempfile import meft_memory +import ngram_store from meft_memory import NativeRows, pack_native @@ -15,6 +20,11 @@ def main(): tags = [line for line in distribution("meft-memory").read_text("WHEEL").splitlines() if line.startswith("Tag:")] assert tags and all(tag.endswith("_" + platform.machine()) for tag in tags), tags assert callable(meft_memory.export_fasttext_model) + assert ngram_store.NativeRows is meft_memory.NativeRows + assert ngram_store.StructuralPlan is meft_memory.StructuralPlan + assert 'torch' not in sys.modules + assert any(entry.name == "ngram-store" and entry.value == "ngram_store.cli:main" + for entry in distribution("meft-memory").entry_points) assert any(entry.name == "meft-fasttext-export" and entry.value == "meft_memory._runtime.fasttext_native_bundle:main" for entry in distribution("meft-memory").entry_points) @@ -30,6 +40,13 @@ def main(): assert output.raw == b"abcdefghABCDEFGHabcdefgh" assert store.last_block_reads == 3 print("Installed native wheel, automatic library lookup and chunked query: passed") + floats=Path(tmp)/'input.f32'; floats.write_bytes(struct.pack('<6f',1,2,3,4,1,2)) + bundle=Path(tmp)/'engine' + subprocess.run(['ngram-store','pack','--input',str(floats),'--output',str(bundle),'--dim','2'],check=True,capture_output=True) + queried=subprocess.run(['ngram-store','query','--store',str(bundle),'--ids','2','0','1'],check=True,capture_output=True,text=True) + assert json.loads(queried.stdout)['vectors']==[[1,2],[1,2],[3,4]] + subprocess.run(['ngram-store','fasttext-export','--help'],check=True,capture_output=True) + print('NGramStore installed CLI/import and historical imports: passed') if __name__ == "__main__": diff --git a/tools/prepare_fasttext_corpus.py b/tools/prepare_fasttext_corpus.py new file mode 100644 index 0000000..6c49650 --- /dev/null +++ b/tools/prepare_fasttext_corpus.py @@ -0,0 +1,63 @@ +"""Convert plain UTF-8 text into the existing positional/source SGNS input. + +Adjacent words stay in their original line and order. This is the custom +me_fasttext training protocol, not ordinary upstream fastText text training. +""" +import argparse +import hashlib +import json +from pathlib import Path + + +def sha256(path): + digest=hashlib.sha256() + with Path(path).open('rb') as stream: + for part in iter(lambda:stream.read(1024**2),b''):digest.update(part) + return digest.hexdigest() + + +def prepare_corpus(source, output, *, words_per_line=64, lowercase=False): + source,output=Path(source).resolve(),Path(output).resolve() + if type(words_per_line) is not int or words_per_line<2: + raise ValueError('words_per_line must be at least two') + if source==output:raise ValueError('source and output must differ') + if output.exists():raise FileExistsError('use a fresh prepared-corpus path') + output.parent.mkdir(parents=True,exist_ok=True) + source_hash=sha256(source); count=lines=skipped=0; owned=False + try: + with output.open('x',encoding='utf-8',newline='\n') as dest: + owned=True + with source.open(encoding='utf-8') as text: + for line in text: + words=(line.lower() if lowercase else line).split() + if any('\x01' in w or '\0' in w or w.startswith(('p#','s#','__label__')) or w=='' for w in words): + raise ValueError('plain text contains reserved positional/source/label tokens') + for begin in range(0,len(words),words_per_line): + block=words[begin:begin+words_per_line] + if len(block)<2:skipped+=len(block); continue + pairs=[a+'\x01'+b for a,b in zip(block,block[1:])] + dest.write('p#0 '+' '.join(pairs)+'\n'); count+=len(pairs); lines+=1 + if not count:raise ValueError('need at least two words on one input line') + if sha256(source)!=source_hash:raise ValueError('source changed during preparation') + except BaseException: + if owned:output.unlink(missing_ok=True) + raise + return {'schema_version':1,'source_sha256':source_hash,'prepared_sha256':sha256(output), + 'pair_tokens':count,'prepared_lines':lines,'skipped_singletons':skipped, + 'words_per_line':words_per_line,'lowercase':lowercase, + 'protocol':'Within-line adjacent query\\x01source pairs, prefixed p#0; custom positional/source SGNS.'} + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--input',type=Path,required=True) + parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--words-per-line',type=int,default=64) + parser.add_argument('--lowercase',action='store_true') + args=parser.parse_args() + try:result=prepare_corpus(args.input,args.output,words_per_line=args.words_per_line,lowercase=args.lowercase) + except (ValueError,OSError) as exc:parser.error(str(exc)) + print(json.dumps(result,indent=2)) + + +if __name__=='__main__':main()