diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a110df0..7d1e81f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,21 @@ jobs: c++ -std=c++11 -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -Isrc -Inative -Imylib/include tests/structural_native_test.cc native/meft_structural.cc -Lmylib/lib -ltrie -lpub -o build/structural-sanitizers ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 build/structural-sanitizers + fasttext-native-fusion: + name: FastText native facade sanitizers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check original model IDs, row bytes and inference outputs + env: + MEFT_FUSION_SANITIZERS: "1" + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + run: python -m unittest tests.test_fasttext_native_bundle -v + legacy-roundtrip: name: Legacy artifact roundtrip runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index c79d301..e1aa538 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,13 @@ LIBS = -L./mylib/lib -ltrie -lpub LIBNAME = libfasttext PYTHON ?= python3 -.PHONY: opt debug test clean native structural-native +.PHONY: opt debug test clean native structural-native fasttext-native + +# Optional Linux FastText facade; retain the original binary/library targets. +fasttext-native: + mkdir -p build + $(CC) -std=c11 -O2 -fPIC -c native/meft_rows.c -o build/meft_rows_fasttext.o + $(CXX) $(CXXFLAGS) -O2 -DMEFT_NATIVE_FASTTEXT -Isrc -Inative $(INCLUDES) $(addprefix src/,$(OBJS:.o=.cc)) native/fasttext_bundle.cc src/main.cc build/meft_rows_fasttext.o $(LIBS) -lz -o build/fasttext-native # Optional original-algorithm bridge; bundled archives require Linux x86-64. structural-native: diff --git a/README.md b/README.md index 9a5b013..384705b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ records that separately; the legacy C++ exporter also supports approximate shari · [**Original DATrie + mark-compact native integration and trained ablations**](docs/structural_native_evidence_20261030/README.md) · [**1.85M trained-row CPU capacity controls and read-amplification limits**](docs/cpu_capacity_evidence_20261030/README.md) · [**Sparse native blocks: 16–27× lower miss latency on the same trained table**](docs/sparse_blocks_evidence_20261030/README.md) +· [**Native storage in the original FastText model/CLI: trained fidelity and serving controls**](docs/fasttext_native_evidence_20261031/README.md) · [**Installable TinyEngram/Qwen checkpoint backend**](docs/tinyengram_qwen_evidence_20261030/README.md) · [**Host memory → device query and prefetch**](docs/device_memory_prefetch.md) @@ -45,6 +46,14 @@ large-table whole-model memory gains remain unmeasured. This repository accompanies the paper [Memory-Efficient FastText: A Comprehensive Approach Using Double-Array Trie Structures and Mark-Compact Memory Management](https://arxiv.org/abs/2506.01254). +The optional [native FastText facade](docs/fasttext_native_bundle.md) converts +this repository's dense `.bin` models through the original DATrie registry and +mark-compact pipeline, then serves them with the existing CLI/query APIs. +Layout/exact modes retain every input byte and float32 aggregation order. +The first actual trained-model control saves 0.46% for the whole bundle; hot +queries remain slower than dense, and structural ordering has no added storage +benefit on that table. Historical valid word/sentence outputs match exactly. + > Status: research / production-prototype code. The command-line interface remains close to the original fastText workflow, while the storage and serving path has been extended for large-vocabulary, memory-sensitive deployments. > Inference framing: `me_fasttext` can also be read as an early compact-inference prototype for lexical embeddings. It keeps subword identity exact during training, then rewrites the learned rows into a smaller mmap-friendly serving artifact for low-memory retrieval and feature-serving workloads. diff --git a/docs/fasttext_native_bundle.md b/docs/fasttext_native_bundle.md new file mode 100644 index 0000000..741655a --- /dev/null +++ b/docs/fasttext_native_bundle.md @@ -0,0 +1,92 @@ +# Native storage through the original FastText model and CLI + +Dense **me_fasttext** `.bin` models can be converted into immutable native +serving bundles. `loadModel(path)` and existing query commands then use the +original dictionary and row IDs, with input vectors stored in compressed native +blocks. The production DATrie prefix/suffix planner and mark-compact forwarding +are part of this conversion. [Actual trained-model results and limits](fasttext_native_evidence_20261031/README.md) +include a historical-main comparison and a matched identity-layout control. + +## Export and query + +The optional facade and bundled DATrie archives require Linux x86-64. Build from +this repository; the standalone memory wheel does not bundle the legacy trainer. + +```bash +make fasttext-native native structural-native + +# Use a NEW output directory. Keep the original .bin for training/quantization. +python tools/export_fasttext_native_cli.py \ + --model model.bin --output-dir model.native \ + --fasttext-binary build/fasttext-native \ + --structural-library build/libmeft_structural.so \ + --native-library build/libmeft_rows.so --mode exact + +# The same command and arguments accept either a .bin or a bundle directory. +printf 'example\nexamples\n' | build/fasttext-native print-word-vectors model.native +printf 'an example sentence\n' | build/fasttext-native print-sentence-vectors model.native +``` + +For a supervised model, `predict`, `predict-prob` and `test` also use the native +input backend. `nn`/`analogies` still precompute resident word vectors and are not +a bounded-memory nearest-neighbor service. The original `make` targets remain +available. The original `.z`/FTIndex path retains its own format and reader. + +After installing `meft-memory` from this checkout, the same converter is exposed +as `meft-fasttext-export` and `meft_memory.export_fasttext_model`. The binary and +structural-library paths remain explicit. No Torch dependency is required for +FastText export/query. + +In an optional C++ source build, query calls keep their signatures: + +```cpp +fasttext::FastText model; +model.loadModel("model.native"); +fasttext::Vector word(model.getDimension()); +model.getWordVector(word, "example"); +``` + +Link `native/fasttext_bundle.cc` and the C row reader with +`-DMEFT_NATIVE_FASTTEXT`, as the Make target does. Recompile C++ callers: retained +source signatures are not an old binary ABI guarantee. + +## Fidelity, compatibility and memory + +- `layout` retains every row; `exact` additionally shares byte-identical + structurally related rows when present. Both verify the full reconstructed + input SHA-256, including untrained rows. The source `.bin` is never rewritten. +- `joint` is opt-in approximate sharing with cosine and norm-ratio guards. + Its manifest separates source bytes from reconstructed representative bytes. + It does not inherit lossless output guarantees. +- The dictionary looks up the **original registered IDs** in `keys.das`; + `table.men` resolves their physical rows through forwarding. Aggregation + preserves the original request order/multiplicity and float32 adds, across + gather chunks. The FP64 native bag reducer is deliberately not used here. +- Loaded dictionaries now skip unknown subwords instead of registering rows + beyond the trained input matrix. Direct subword helpers use real DATrie IDs; + missing keys return `-1`/zero. Empty or zero-vector string sentence queries + return zero instead of NaN. These fix prior undefined/invalid behavior; + equivalence is claimed for valid historical queries, not those bugs. +- The default export caps decoded blocks at 8,192 bytes, uses a 4 MiB decoded + block cache and gathers at most 256 rows before ordered accumulation. These + are configurable via `--max-block-bytes`, `--cache-bytes` and + `--query-chunk-rows`. A block's row count also depends on vector dimension. +- Input is frozen and each handle is single-worker. `saveModel`, legacy + `saveIndex`, `quantize` and `getInputMatrix` reject native input; reload a dense + model for those operations. Starting a new `train(args)` clears the backend. +- The dictionary, its legacy 200 MB unused word hash allocation, exact key + index and dense output matrix remain resident. Neither loader builds an + inference negative-sampling table. Cache budget is not a total RSS limit; + touched mmap pages remain file-backed memory. + +The bundle contains `table.men`, `keys.das`, original args/dictionary/output in +`model.meta`, reader settings and audit manifests. Open only trusted, +self-produced bundles on little-endian IEEE float32 Linux x86-64: the legacy +metadata/index loader is not an untrusted-input parser. The exporter records +checksums, but the C++ facade does not authenticate the entire bundle at open. +Conversion holds full source rows and planning maps offline. + +This converter supports this repository's dense, unquantized, unpruned models. +Official fastText `.bin`, quantized `.ftz`, and old `.z` files are separate +formats. Engram hash collisions do not provide this unique string-key registry; +the numeric-row adapter retains its separate integration boundary. diff --git a/docs/fasttext_native_evidence_20261031/README.md b/docs/fasttext_native_evidence_20261031/README.md new file mode 100644 index 0000000..6230de8 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/README.md @@ -0,0 +1,146 @@ +# Actual me_fasttext model → native FastText serving + +The native backend now integrates with the original FastText facade, rather +than requiring a separate numeric-row application. A genuinely trained legacy +`.bin` converts through the original DATrie registry and structural planner; +the same CLI and C++ query APIs can load the native bundle. See the +[build/export/query guide](../fasttext_native_bundle.md) and +[PR #36](https://github.com/initial-d/me_fasttext/pull/36). + +**This establishes functional fusion and exact inference. Storage and latency +gains are limited on this small model.** The complete native bundle saves only +0.46%; warm queries are slower than resident dense queries. Structural layout +is slightly worse than a matched identity layout here. These are CPU public-text +storage/serving results, not Engram checkpoint or downstream quality evidence. + +## Fidelity and historical compatibility + +The actual legacy trainer produces **90,380 × 64 float32 input rows**, including +14,813 word/source/position rows and 75,567 registered n-gram rows. Training +changes 88,725 input rows and 75,545 n-gram rows relative to the exact seed-1 +initializer. Every input row is retained; unused rows are not filtered out. + +Layout and exact exports reconstruct the full source input SHA-256: + +`88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9` + +All nine fresh-process serving runs (dense/layout/exact, three each) produce +the same 1,024 held-out word vectors and 128 held-out eight-word sentence +vectors, with output SHA-256: + +`da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058` + +The **pre-change main** at `bea4302bb7fb1a6c0108fdde141ce667f2389920`, new +dense facade and native facade additionally produce byte-identical outputs on +another 1,024 registered-word and 128 sentence queries. See +[historical-report.json](historical-report.json) and the recorded +[build/run script](historical-run.sh). This control excludes prior OOV +registration and hash-bucket helper bugs; their invalid/out-of-bounds behavior +is intentionally corrected. C++ callers must recompile. + +The Linux correctness fixtures separately cover dimensions 7/32, UTF-8/OOV, +all input IDs, ordered repeated IDs, chunks, softmax/HS class scores, failure +preserving the loaded model, immutable guards and dense reload. Fixture rows +are mathematical correctness data, never trained compression evidence. + +## Storage and matched structural control + +| Same trained source | Input row artifact, bytes | Saving vs raw input | +| --- | ---: | ---: | +| Raw float32 input | 23,137,280 | — | +| Native, identity order | 22,298,237 | 3.63% | +| Native, structural layout/exact | 22,401,608 | 3.18% | + +All native controls use the same codec, 32-row / 8,192-byte blocks, 723,040-byte +original-ID forwarding map, source rows and 4 MiB block cache. The identity +control retains the **identical original-ID DASTrie**, model metadata and reader +configuration; only physical row order changes. Its outputs match the same +dense oracle. See [identity-layout-report.json](identity-layout-report.json). + +Exact structural sharing finds **zero byte-identical rows to merge**. All +90,380 rows survive mark-compact. Structural order adds 103,371 bytes (0.46% +relative to the identity native table) rather than improving this codec on +this training distribution. This experiment demonstrates that the algorithm +is connected, not extra lossless sharing or a universal row-order advantage. +[Earlier trained structural ablations](../structural_native_evidence_20261030/README.md) +separately measure approximate joint sharing and key-index controls. + +The dense `.bin` is 27,204,018 bytes. Exact native **whole-bundle** size is +27,078,261 bytes (0.46% saving), counting input table, 605,232-byte DASTrie, +original dictionary/output metadata, config, manifests and extraction log. +Conversion takes 11.94 seconds, including verification; layout takes 11.61. +These are one export per mode, not a converter throughput distribution. + +## End-to-end query costs + +Three fresh processes per configuration, with one CPU affinity, prewarmed +filesystem and identical query order. First-pass cache starts empty; each +process then repeats the same trace three more times. Values below are medians +of the three process-level p50s; p95 and all raw passes are in +[trained-report.json](trained-report.json) and [summary.json](summary.json). + +| Query / state | Dense .bin | Native exact | Native identity control | +| --- | ---: | ---: | ---: | +| Word first-pass p50 | 2.79 µs | 7.96 µs | 8.10 µs | +| Sentence first-pass p50 | 18.02 µs | 647.29 µs | 511.87 µs | +| Word hot p50 | 2.44 µs | 4.33 µs | 4.33 µs | +| Sentence hot p50 | 16.55 µs | 27.38 µs | 27.45 µs | + +Native exact first-pass p95 is 749.40 µs for words and 1,745.41 µs for sentences. +Structural order improves neither cold sentence latency nor warm latency here. +The repeated working set fits the cache, so hot numbers do not represent an +unbounded changing query stream. Timed calls include original word/subword +resolution, float32 row aggregation, sentence normalization and decoding; +oracle file I/O and SHA verification are outside query timing. Batch totals also +include loop/string/oracle-copy overhead and are not kernel throughput. + +After the trace, median anonymous RSS is **221.11 MiB dense → 202.26 MiB exact**; +total RSS is **224.88 → 223.55 MiB** because native touches about 21.29 MiB of +file-backed pages. The legacy 200 MB unused `word2int_` allocation and output +matrix remain in both paths. Neither loader allocates training negatives. +Anonymous memory shifts to mapped pages; this is not a large total-memory +reduction. Warm-filesystem median load time is 0.187 → 0.124 seconds. + +## Reproduce and inspect failures + +Execution source: `bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8`; identity-control +source: `355c299` (adds the ablation tool without changing serving). Python +3.11.15, GCC/G++ 11.4.0, Linux 5.4.0-42 / glibc 2.35, AMD EPYC 7742; no Torch +or GPU used by these drivers. The run is pinned to one available CPU core, +nice level 10, OMP/BLAS/MKL threads 1, on a shared machine. Build commands, +data/model/query hashes, affinity, raw timings and limitations are in the report. + +The existing enwik8 archive is split at raw byte 90,000,000 **before** processing. +Each partition strips XML tags, lowercases and keeps `[a-z]+` words. The first +65,536 train words become 1,024 lines of `p#0` plus 63 adjacent +`query\x01source` pairs. This is necessary because this repository's custom +positional/source reader does not train on ordinary whitespace text. The +reference is finite SGNS: dim 64, n-gram lengths 2–12, minCount 2, window 3, +five negatives, two epochs, one thread, learning rate 0.025 and subsampling +threshold 1. It is not ordinary upstream fastText or the paper's large dataset. +Held-out queries sample from the first 8,192 words in the independent 10% raw +partition, seed 42. No quality target or threshold is selected on them. + +```bash +export CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +meft_cpu=$(python -c 'import os; print(min(os.sched_getaffinity(0)))') +# Existing archive; this script does not download data. Use a fresh directory. +nice -n 10 taskset -c "$meft_cpu" python tools/benchmark_fasttext_native.py \ + --enwik8-zip /path/to/enwik8.zip --output-dir artifacts/fasttext-native +nice -n 10 taskset -c "$meft_cpu" python tools/ablate_fasttext_layout.py \ + --reproduction-dir artifacts/fasttext-native --output-dir artifacts/fasttext-identity + +MEFT_FUSION_SANITIZERS=1 ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ + UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ + python -m unittest tests.test_fasttext_native_bundle -v +``` + +[tests-initial-failed.txt](tests-initial-failed.txt) preserves the initial +registry-export failure: the DATrie callback returned stop after one key. +Commit `900f548` fixes its continuation convention. The corrected +[Linux roundtrip](tests-registry-fixed.txt) and +[ASan/UBSan/LeakSanitizer run](sanitizers.txt) pass. Bundled trie archives are +linked but uninstrumented. Final CI adds an explicit sanitized facade job, +including HS and zero-vector string helpers. Metadata/index files remain +trusted host-specific artifacts; a cache budget does not bound RSS, and offline +planning copies the full input/maps. No GPU process was touched. diff --git a/docs/fasttext_native_evidence_20261031/build-native.txt b/docs/fasttext_native_evidence_20261031/build-native.txt new file mode 100644 index 0000000..112985e --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/build-native.txt @@ -0,0 +1,17 @@ +make: 进入目录“/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source” +mkdir -p build +cc -std=c11 -O2 -fPIC -c native/meft_rows.c -o build/meft_rows_fasttext.o +c++ -pthread -std=c++0x -march=native -fPIC -O2 -DMEFT_NATIVE_FASTTEXT -Isrc -Inative -I. -I./mylib/include src/args.cc src/dictionary.cc src/productquantizer.cc src/matrix.cc src/qmatrix.cc src/vector.cc src/model.cc src/utils.cc src/fasttext.cc src/ftindex.cc native/fasttext_bundle.cc src/main.cc build/meft_rows_fasttext.o -L./mylib/lib -ltrie -lpub -lz -o build/fasttext-native +src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +./mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ +mkdir -p build +cc -std=c11 -O2 -Wall -Wextra -Werror -fPIC -shared native/meft_rows.c -lz -o build/libmeft_rows.so +cc -std=c11 -O2 -Wall -Wextra -Werror -Inative examples/native_gather.c native/meft_rows.c -lz -o build/native_gather +mkdir -p build +c++ -std=c++11 -O2 -Wall -Wextra -Werror -fPIC -shared -Isrc -Imylib/include native/meft_structural.cc -L./mylib/lib -ltrie -lpub -o build/libmeft_structural.so +make: 离开目录“/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source” diff --git a/docs/fasttext_native_evidence_20261031/build-probe.txt b/docs/fasttext_native_evidence_20261031/build-probe.txt new file mode 100644 index 0000000..9cb404a --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/build-probe.txt @@ -0,0 +1,7 @@ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ diff --git a/docs/fasttext_native_evidence_20261031/exact-manifest.json b/docs/fasttext_native_evidence_20261031/exact-manifest.json new file mode 100644 index 0000000..cee1965 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/exact-manifest.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "model_type": "me_fasttext-native", + "shape": [ + 90380, + 64 + ], + "keyed_ngram_rows": 75567, + "mode": "exact", + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "source_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "reconstructed_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "original_bytes_preserved": true, + "extraction_command": [ + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native", + "dump-native", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/.meft-fasttext-4yeh63_c/extracted" + ], + "storage": { + "format": "MEFTNAT1", + "format_version": 2, + "input_bytes": 23137280, + "artifact_bytes": 22401608, + "unique_rows": 90380, + "row_map_bytes": 723040, + "n_rows": 90380, + "row_bytes": 256, + "block_rows": 32, + "blocks": 2825, + "max_block_bytes": 8192, + "raw_blocks": 0, + "compressed_blocks": 2825, + "zlib_blocks": 2733, + "xor_blocks": 92, + "deduplicate": false, + "predictor": "xor", + "reordered": true, + "external_representatives": true, + "marked_rows": 90380, + "structural_mode": "exact", + "index": "datrie", + "index_bytes": 605232, + "source_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "representatives_sha256": "8b4e8f17cf8fcb4012f022d00edba81a174c24340bf9675fc5d069a2f3676608", + "original_bytes_preserved": true, + "minimum_cosine": null, + "minimum_norm_ratio": null, + "contract": "original IDs retained; decoder exact to representative bytes; joint sharing may change weights" + }, + "query_chunk_rows": 256, + "cache_payload_budget_bytes": 4194304, + "contract": "Original DATrie registry IDs; original float32 ordered accumulation. Joint mode can change source weights.", + "limitations": [ + "Trusted self-produced, Linux x86-64 legacy metadata/index serialization.", + "Frozen inference; output matrix/dictionary remain resident. Cache is not an RSS limit.", + "Not an official fastText converter or quantized/.z migration." + ], + "artifacts": { + "structural.json": "0e0484813aab6527c6ee2a1017212d95b299d6814dd88d63af9bca89c3060111", + "table.men": "dae4b249da573240868e8797b66b7951c2761762bfc16666639aa89e623847ae", + "reader.cfg": "9aec7711f5cb0f77405a33251042899b93322365e849dcdcbe5ab219fecd76d9", + "embedding.json": "6216702620768cdbca11e4974fb8ee56a5c78683967a3cefb828e9fdae02a6e3", + "keys.das": "0979b4dc1d730edb4429e9a99f9867a528c98eb5f8e0cf703e886a09f155154d", + "model.meta": "a44eed12b4dc3ae9de8735d54d7a98143e7e3ed03a401644685229bd2872cec6" + } +} diff --git a/docs/fasttext_native_evidence_20261031/export-exact.txt b/docs/fasttext_native_evidence_20261031/export-exact.txt new file mode 100644 index 0000000..100024e --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/export-exact.txt @@ -0,0 +1,8 @@ +{ + "output_dir": "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact", + "shape": [ + 90380, + 64 + ], + "original_bytes_preserved": true +} diff --git a/docs/fasttext_native_evidence_20261031/export-layout.txt b/docs/fasttext_native_evidence_20261031/export-layout.txt new file mode 100644 index 0000000..b75884d --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/export-layout.txt @@ -0,0 +1,8 @@ +{ + "output_dir": "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout", + "shape": [ + 90380, + 64 + ], + "original_bytes_preserved": true +} diff --git a/docs/fasttext_native_evidence_20261031/heldout-queries.tsv b/docs/fasttext_native_evidence_20261031/heldout-queries.tsv new file mode 100644 index 0000000..b4fccc9 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/heldout-queries.tsv @@ -0,0 +1,1152 @@ +w and +w north +w urged +w found +w and +w senegambia +w being +w muslim +w was +w of +w into +w the +w importance +w benguella +w tripolitania +w its +w west +w stimulated +w appointed +w the +w egypt +w with +w in +w there +w powers +w peoples +w algiers +w terence +w kasr +w the +w dominant +w by +w france +w or +w this +w to +w mid +w resulted +w rhodesia +w of +w up +w relationship +w already +w amp +w lake +w east +w energies +w into +w to +w spain +w between +w europe +w quot +w mau +w obtaining +w egypt +w french +w over +w only +w the +w during +w chaillu +w and +w captured +w laws +w empire +w krapf +w term +w new +w falls +w the +w economic +w lt +w was +w maritime +w rule +w an +w dutch +w few +w history +w his +w in +w of +w was +w result +w spain +w the +w other +w help +w by +w as +w second +w and +w modern +w d +w been +w were +w as +w northern +w arose +w factions +w s +w map +w as +w the +w one +w i +w land +w the +w fanny +w valley +w not +w history +w the +w it +w these +w of +w inception +w been +w war +w by +w area +w of +w s +w of +w develop +w supporting +w was +w continent +w the +w these +w knowledge +w the +w eventually +w of +w of +w dias +w a +w century +w the +w san +w i +w the +w work +w dabbler +w the +w aggressive +w the +w had +w perhaps +w attempt +w they +w colonial +w nile +w zambezi +w kilimanjaro +w who +w of +w were +w famous +w century +w masters +w absorption +w ruled +w it +w about +w returning +w changes +w first +w august +w confirmed +w thumb +w his +w van +w th +w pisa +w after +w in +w the +w of +w also +w the +w the +w superiority +w commander +w result +w of +w and +w made +w vessels +w was +w the +w was +w british +w by +w hardy +w of +w arabized +w and +w had +w churches +w defeated +w in +w lake +w portugal +w robert +w to +w px +w formally +w bath +w nubia +w century +w south +w ancient +w political +w a +w the +w had +w fortress +w head +w sooner +w a +w the +w zanzibar +w and +w century +w italians +w continued +w region +w in +w was +w the +w conquerors +w i +w first +w both +w and +w he +w trade +w them +w egypt +w spread +w and +w later +w acted +w and +w countires +w africa +w of +w ancient +w were +w international +w in +w to +w ruled +w pyramids +w to +w the +w such +w which +w a +w forts +w during +w the +w dwarf +w see +w ii +w the +w beyond +w a +w ancient +w ideal +w established +w the +w this +w exceptions +w the +w arabized +w also +w other +w state +w africa +w trade +w began +w jacobin +w of +w and +w had +w the +w acted +w utmost +w c +w the +w these +w states +w the +w by +w navigation +w were +w develop +w remaining +w incursion +w owes +w a +w th +w longer +w mouth +w was +w brought +w africa +w cyrene +w gain +w mehemet +w as +w abd +w decolonization +w robert +w africa +w inland +w were +w faded +w of +w roman +w the +w the +w when +w seen +w one +w had +w and +w south +w after +w siege +w in +w an +w not +w invade +w that +w the +w was +w sahara +w britain +w afrika +w european +w the +w aghlabite +w l +w african +w image +w eventually +w times +w there +w to +w by +w world +w and +w the +w the +w relied +w keeping +w a +w great +w ghana +w three +w up +w knowledge +w in +w useful +w san +w central +w vessels +w haroun +w the +w european +w voyages +w itself +w affairs +w is +w sufficient +w as +w african +w o +w the +w revolutionary +w in +w acquired +w purely +w his +w of +w self +w provoked +w egypt +w to +w different +w the +w up +w during +w was +w the +w the +w as +w creation +w purposes +w europe +w started +w from +w to +w the +w influence +w saharan +w posts +w who +w was +w the +w navy +w in +w landed +w africa +w of +w disambiguation +w the +w bantu +w standard +w the +w duty +w a +w had +w by +w and +w th +w the +w shot +w elmina +w the +w the +w this +w command +w to +w who +w the +w gutenberg +w the +w popular +w not +w the +w other +w not +w would +w while +w was +w in +w portugal +w communities +w i +w traverse +w early +w farther +w the +w national +w had +w the +w firmly +w began +w the +w to +w in +w ordinary +w generally +w exploratory +w british +w burkina +w african +w speakers +w but +w iron +w of +w an +w their +w rank +w europe +w to +w missionaries +w britain +w his +w to +w settled +w ivoire +w one +w hero +w th +w aboukir +w congo +w northern +w history +w has +w mansa +w the +w era +w by +w convenient +w the +w colonies +w in +w rhodesia +w struggle +w rear +w british +w own +w in +w struggle +w hinchinbrook +w exploration +w islam +w defeated +w the +w by +w nation +w trade +w invaders +w roman +w bulk +w in +w intended +w demonstrated +w except +w of +w in +w by +w those +w that +w spanish +w africa +w genocide +w s +w the +w obtained +w and +w and +w and +w far +w signed +w afrikas +w a +w beginnings +w was +w at +w free +w ceased +w and +w the +w during +w united +w of +w of +w peak +w the +w mdash +w of +w eight +w cold +w the +w only +w march +w were +w endeavour +w from +w was +w at +w by +w ambitions +w west +w accurate +w opold +w nelson +w quot +w spanish +w south +w struggle +w sea +w reconquest +w settler +w wherein +w the +w the +w with +w outlets +w several +w the +w further +w fourth +w after +w by +w knowledge +w robert +w jacobin +w and +w jervis +w he +w favourite +w western +w time +w delegates +w missions +w in +w the +w to +w in +w battle +w of +w albemarle +w france +w the +w kcb +w beyond +w rules +w the +w nyasa +w with +w a +w composition +w a +w took +w was +w continued +w love +w power +w and +w interfered +w the +w was +w traversed +w l +w southern +w the +w were +w in +w influence +w the +w destined +w on +w in +w was +w led +w the +w be +w were +w his +w to +w and +w of +w a +w place +w in +w of +w as +w viscount +w war +w constantinople +w are +w angola +w cannibalistic +w self +w university +w by +w middle +w maternal +w been +w speakers +w a +w him +w delegates +w war +w be +w countries +w war +w both +w over +w despite +w niger +w of +w the +w to +w between +w mocambique +w and +w by +w a +w mediterranean +w at +w of +w antonio +w in +w savages +w of +w however +w hassa +w time +w word +w duty +w people +w fill +w an +w the +w century +w the +w the +w the +w practicable +w closed +w he +w begun +w came +w transvaal +w leading +w while +w in +w french +w the +w cyrene +w for +w ambitions +w details +w individual +w the +w famous +w with +w the +w roman +w visited +w siege +w a +w the +w bay +w tour +w war +w one +w sea +w european +w he +w resources +w quot +w barbary +w as +w the +w died +w upper +w the +w honourable +w ptolemaic +w of +w progress +w for +w second +w control +w did +w greatest +w crop +w frigate +w educated +w th +w they +w europe +w kingdom +w cannon +w invasion +w trade +w the +w exercised +w powers +w moves +w wife +w mediterranean +w were +w parts +w th +w went +w the +w naples +w conquests +w attempt +w and +w of +w as +w neglecting +w admiral +w italy +w late +w hms +w the +w ships +w pygmy +w with +w this +w the +w reached +w lucrative +w aghlab +w further +w on +w new +w would +w nile +w to +w upon +w in +w adopted +w who +w emma +w the +w of +w assassin +w royal +w administrations +w of +w being +w african +w consolidated +w his +w of +w european +w thumb +w sanctions +w its +w over +w problems +w made +w in +w its +w onward +w was +w the +w ball +w sea +w and +w england +w regions +w tunisia +w napoleon +w the +w would +w afrika +w which +w in +w conflict +w tribal +w campaign +w of +w the +w succouring +w thus +w traversed +w cut +w courts +w the +w repeatedly +w there +w exercised +w the +w and +w the +w development +w islam +w interior +w portugal +w again +w both +w or +w of +w the +w good +w ahmad +w lobo +w continent +w south +w nachtigal +w get +w th +w in +w flag +w and +w he +w the +w at +w was +w was +w and +w who +w thrown +w countries +w this +w he +w partition +w a +w l +w cape +w nationalist +w rescued +w beginning +w river +w between +w france +w with +w in +w connections +w ancient +w to +w the +w of +w lady +w was +w zambezi +w alamein +w sea +w small +w article +w trade +w other +w within +w hold +w the +w libya +w mau +w continent +w their +w and +w settler +w hms +w in +w despite +w europe +w a +w necessary +w the +w europe +w romney +w and +w quot +w less +w war +w an +w in +w had +w van +w local +w rainforest +w archaeology +w around +w at +w for +w of +w this +w monroe +w in +w libya +w on +w what +w left +w africa +w a +w th +w thus +w santa +w after +w and +w surplus +w as +w of +w southward +w british +w seventh +w state +w cities +w succouring +w portuguese +w and +w nb +w in +w of +w south +w began +w to +w duke +w also +w of +w prosperity +w the +w the +w right +w wear +w victoria +w states +w africa +w also +w from +w military +w afrikas +w almost +w sent +w navigator +w admiral +w last +w the +w cataracts +w churches +w the +w of +w portuguese +w by +w invaders +w nevertheless +w piratical +w of +w in +w bantu +w day +w was +w international +w india +w article +w pedro +w many +w its +w first +w invaders +s no steps to acquire the southern part of +s continent footer lt categories gt lt interlangauge links +s history of the middle east african archaeology lt +s and the axumite kingdom which developed there has +s neither represented nor pledged their respective governments their +s th century and whence the arabs whom the +s history is divided the old kingdom old middle +s the east african seaboards except along this seaboard +s stations were established the earliest being s o +s as mauretania africa province africa tripolitania cyrenaica aegyptus +s stamped on northern africa together they spread southward +s in libya in egypt was under gamal abdel +s almost independent state and the extension of egyptian +s most of the other regions of the world +s the atlantic ocean atlantic and continued into spain +s allied with other arab countries the first was +s of north africa west of the great syrtis +s be the congo while the great mystery of +s of trade and empire one of the first +s a succession of military coups in ghana and +s nations who followed in their wake used table +s economic and political state of western europe at +s countries represented which should collect funds and appoint +s destined to have a permanent influence on the +s work north of the orange river orange in +s of the congo the story of exploration takes +s were mostly concentrated in west africa where important +s august st at pm jpg thumb px left +s was practicable granville george leveson gower lord granville +s n kush links and connections to the sahel +s pan africanism main article pan africanism central africa +s te d ivoire liberia and nigeria famine has +s promoted to admiral united kingdom rear admiral of +s eight months it took that long for the +s of the orange free state and the south +s chancellor acted africa was the only field left +s extent farther south s o paulo de loanda +s falls so named after the victoria of the +s of different administrations in africa at this time +s romney at the height of her beauty in +s and much of the levant as far as +s subdued the berber tribes who then as now +s months he took part in the battle of +s continent southern morocco the sahara and the sudan +s involved still damaged nelson s health to the +s the paucity of ports and the absence of +s which had been continuously occupied by british troops +s over by france and the united kingdom during +s articles islamisation in the th century occurred an +s ogowe river ogowe district of the west coast +s colonizers portugal in particular were reluctant to relinquish +s the blue the ninth highest rank in the +s on active propaganda on the guinea coast in +s dynasty ptolemies attempts were made to penetrate southward +s image horatio nelson st viscount nelson project gutenberg +s under rome the settled portions of the country +s empire of monomotapa where gold had been worked +s under a theoretical british oversight for some time +s and had seized egypt in established the regencies +s founded the city of cyrene libya cyrene in +s islamic religion thus became indelibly stamped on northern +s the portuguese drew other nations to the guinea +s agreements were concerned was practically completed relationship to +s or quot tribal quot conflict after the african +s exploration and westernization of africa and the opening +s only european influence she had to contend not +s although some colonizers portugal in particular were reluctant +s still obtaining supplies in the th century several +s riebeeck which reached table bay on the april +s back to sea if only to get him +s international footing but it quickly became apparent that +s shift in the world view of the prussian +s over the bantu inhabitants and in the early +s south to north and reached lake ngami and +s theatre of european expansion lines of partition drawn +s the british empire and the boer south african +s the occupation of algiers by france in an +s the berber tribes the introduction of christianity throughout +s geschichte afrikas es historia de frica fr histoire +s war ii in africa north african campaign deutsches +s into spain throughout north africa christianity nearly disappeared +s end of the war in command in nelson +s rhodesia later zambia and zimbabwe and south africa +s came to dominate society settlers only came to +s gun hms agamemnon agamemnon in he soon started +s world war ii nationalist movements arose across west +s the jealous opposition of almost every european nation +s independence many place names for example stanleyville l +s in peril of imprisonment and had to remain +s largely made up of piratical exploits on the +s being the occupation of algiers by france in +s since in missionary work north of the orange +s ever wore an eye patch though he was +s states such as those of the yoruba and +s work in africa those wars nevertheless exercised great +s religion was imminent by the victories of the +s norwich school and by the time he was +s from south to north and reached lake ngami +s intended to become an african colony but was +s and south africa the occupation of egypt first +s mau mau rebellion took place in kenya from +s river niger interaction between asia europe and north +s been seized by portugal mocambique mo ambique being +s occupied by british troops since was formally ceded +s been seized by portugal mocambique mo ambique being +s scale in most areas colonial administrations did not +s mouth of the congo river congo the cape +s not approved in london and the proclamation they +s in that battle but the british government insisted +s the th century under the earlier dynasties arabian +s kingdom during this era a sense of local +s french construction commanded by captain cockburn a spanish +s that river down to the atlantic ocean amp +s cyrenaica became a flourishing colony though being hemmed +s of the great lake reservoirs of the nile +s seaman and coxswain nelson s maternal uncle captain +s and ethiopia were reached but an expedition sent +s indian ocean as far south as rhapta in +s the piratical proceedings of the barbary states the +s significant major effects include the spread of classical +s egypt was involved several wars against israel and +s north africa see also north africa in a +s ships would be quot frustrated of watering but +s in world history his biography by the poet +s ns planted flourishing colonies such as mombasa kenya +s the struggle this was brought about by the +s without rival in the mediterranean the phoenicians subdued +s of the interior of the continent to commerce diff --git a/docs/fasttext_native_evidence_20261031/historical-build.txt b/docs/fasttext_native_evidence_20261031/historical-build.txt new file mode 100644 index 0000000..da74c10 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/historical-build.txt @@ -0,0 +1,7 @@ +src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ diff --git a/docs/fasttext_native_evidence_20261031/historical-known.tsv b/docs/fasttext_native_evidence_20261031/historical-known.tsv new file mode 100644 index 0000000..c8d154b --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/historical-known.tsv @@ -0,0 +1,1152 @@ +w collecting +w allows +w improving +w guardianbrainman +w four +w culture +w christ +w took +w cabinet +w writers +w priority +w anarchizmus +w ams +w catholic +w following +w genes +w speculate +w although +w unknown +w expect +w tomb +w prevent +w forerunners +w refuses +w worldwide +w independent +w accompanied +w difference +w probably +w metaphysics +w indefinite +w destroyer +w finnish +w medicine +w chosen +w case +w onset +w ceremony +w ncbi +w mind +w howard +w arguments +w responsibility +w their +w consensus +w oil +w bound +w trivia +w january +w neither +w waterway +w etc +w ben +w arr +w fully +w ipcc +w br +w genius +w child +w ontario +w independence +w remain +w nicolay +w discipline +w northeast +w mysterious +w featured +w husband +w besides +w drew +w terms +w guardian +w discussing +w revision +w ones +w iep +w uk +w force +w macedon +w authorities +w ga +w anarchoblogs +w license +w phenomenon +w ibn +w before +w fell +w variety +w liberalism +w fictional +w society +w payment +w respective +w dark +w hti +w culture +w had +w unless +w thing +w horace +w will +w proto +w wiki +w persian +w neocolonialism +w footnotes +w crouching +w stand +w signs +w caption +w arthur +w coal +w depends +w differing +w private +w basic +w origins +w opportunity +w zone +w rose +w tamazight +w health +w true +w actually +w comic +w theoretical +w huntsville +w metamorphoses +w collection +w jeff +w query +w diagnoses +w remains +w abolitionists +w hostile +w some +w effort +w spot +w classroom +w justified +w spot +w expected +w depends +w objectivist +w direction +w thinkers +w taxation +w ab +w mac +w shaikh +w aide +w collectivism +w neuropsychiatric +w law +w governments +w automobile +w grape +w various +w bound +w bush +w sequences +w beloved +w temple +w consisted +w contested +w say +w transliterated +w div +w html +w table +w process +w feminists +w think +w exporter +w legend +w permanently +w object +w raised +w styx +w relating +w concert +w hands +w fredrick +w battery +w men +w alaska +w worker +w trying +w game +w workers +w foreign +w accounts +w berkman +w awarded +w g +w behaviour +w anarchist +w married +w berbers +w stop +w golden +w indian +w sensory +w finally +w think +w coordinated +w violence +w warrior +w salem +w grew +w salmon +w plato +w escape +w cause +w certainly +w published +w musketeers +w produce +w population +w roles +w attorney +w changed +w baker +w phosphate +w merit +w cnrs +w happens +w establishing +w es +w them +w reforms +w current +w priority +w empire +w indian +w revolutions +w hart +w blessed +w rearden +w treasury +w chamber +w associates +w though +w advice +w catechisme +w glory +w djelfa +w plant +w separate +w section +w file +w phenomena +w avoid +w dispute +w olive +w abe +w pagename +w http +w reported +w intensely +w problems +w two +w service +w desirable +w eric +w journals +w flynt +w averages +w wedding +w tide +w ball +w level +w autismwebsite +w assigned +w wiktionarypar +w schism +w south +w teachings +w develops +w autismspectrumquotient +w spring +w branches +w energy +w bell +w zaid +w belief +w girl +w picture +w computing +w victims +w gutenberg +w web +w youthful +w appoints +w british +w press +w wikipedia +w utopia +w supposed +w limit +w homeless +w fact +w lfa +w gorgias +w hulk +w payments +w convention +w keeps +w research +w likely +w big +w across +w resource +w upper +w charter +w biography +w there +w fig +w spent +w http +w copperheads +w morning +w belle +w grown +w normal +w intended +w dhi +w rallied +w tindouf +w know +w tall +w ace +w turn +w kant +w circumstances +w countries +w howard +w common +w cleyre +w true +w destroyer +w immanuel +w inhabited +w feeling +w military +w face +w house +w speech +w shaikh +w heads +w assumed +w carter +w produce +w including +w aristotelicum +w abraham +w may +w controversy +w honorary +w direction +w react +w triggered +w prospects +w universe +w action +w collectivism +w bleed +w defense +w top +w anthony +w nonviolence +w whole +w troops +w deep +w psychiatric +w contact +w archive +w lay +w nickname +w appreciation +w nations +w federal +w harris +w chryseis +w murdered +w unions +w plant +w designated +w goal +w disambiguation +w editorial +w positive +w allowing +w egypt +w material +w portrait +w hank +w huntsville +w die +w closely +w oran +w appears +w run +w foundation +w experiment +w resulting +w mortimer +w landauer +w fulfilling +w founding +w align +w ethology +w performs +w manifesto +w indiana +w belt +w individualism +w motto +w stance +w personal +w their +w marxist +w altruists +w committee +w homestead +w efficient +w what +w huac +w appear +w clothes +w quarries +w mission +w level +w race +w stated +w commons +w os +w was +w errors +w hera +w aristotle +w r +w abbreviation +w suddenly +w these +w exchange +w newsweek +w publishers +w benefiting +w market +w levels +w conscience +w kept +w spontaneous +w leadership +w pn +w majority +w phoenician +w johnston +w tull +w construction +w esteem +w primary +w on +w earlier +w via +w killed +w planet +w trading +w aardwolf +w la +w interpret +w federation +w psychiatrist +w were +w lovers +w rio +w react +w reading +w figures +w starts +w samaritan +w down +w buoying +w institutions +w string +w meanings +w categories +w gifted +w least +w fredrick +w expenses +w decreed +w allies +w arrest +w guardian +w saw +w billion +w represent +w power +w warmer +w events +w organizations +w since +w person +w groups +w dedicated +w accepting +w clear +w profound +w follows +w economic +w submit +w rie +w assist +w umich +w hardbound +w conclusion +w republican +w corwin +w right +w team +w understood +w zeno +w links +w reality +w specific +w proper +w trading +w recovery +w did +w savant +w regardless +w hitler +w half +w increases +w sumter +w senator +w government +w impression +w ranks +w book +w interact +w get +w images +w med +w lodged +w this +w breath +w crouching +w demand +w geek +w oregon +w depending +w film +w battle +w pp +w please +w marx +w time +w river +w pre +w barbara +w fan +w prices +w oxford +w wiktionary +w aimed +w warming +w open +w scholarly +w accessed +w mouradia +w k +w pacifist +w presidential +w thereto +w total +w forest +w sex +w for +w impact +w quoted +w separate +w ammem +w own +w medea +w pine +w div +w romans +w contains +w terms +w altruism +w passing +w wyoming +w uses +w altruismo +w building +w prove +w covers +w review +w emancipation +w assist +w holds +w on +w malice +w feminism +w replace +w makhno +w membership +w on +w index +w print +w heavens +w britannica +w ruler +w aid +w thinkers +w athens +w most +w francesco +w bella +w appropriate +w anarchism +w half +w experiences +w akhilleus +w departed +w goodwin +w constantine +w san +w comfort +w used +w focused +w rio +w heros +w none +w domain +w comic +w disease +w leftism +w closely +w wearing +w alpha +w legislation +w warming +w occupation +w pediatrics +w existence +w blossomed +w xanthos +w gregory +w choice +w kinds +w concert +w valley +w arabia +w mohammed +w temporary +w proto +w not +w belonged +w spent +w mi +w addresses +w prices +w shelter +w claims +w put +w nervous +w restricted +w depicting +w quite +w economies +w sup +w iers +w thereof +w selections +w riley +w quiz +w xi +w idaho +w low +w guided +w c +w indirect +w regulate +w group +w right +w victories +w olive +w mediawiki +w american +w since +w magnitude +w embraced +w seven +w festival +w mutualism +w hispanic +w meteorology +w industrial +w zoology +w include +w type +w activist +w student +w essays +w bush +w greater +w plant +w shaikh +w tutor +w grand +w scarlet +w short +w reflect +w again +w casket +w jessie +w fort +w pine +w grief +w larger +w which +w nonfiction +w samaritan +w trust +w teaching +w minarchists +w programs +w treasury +w martyr +w move +w remarked +w illegitimate +w last +w health +w gas +w concerned +w ethic +w license +w comprise +w theater +w end +w establishing +w five +w semi +w includes +w wound +w survived +w inspire +w cheaha +w evaluation +w joseph +w fuels +w needy +w egypt +w km +w adrar +w terms +w constitute +w imposing +w armstrong +w au +w truly +w iv +w constantine +w shortly +w christian +w addition +w waited +w integral +w rowman +w screenplay +w rate +w methods +w encounter +w astronomy +w heavy +w science +w comedy +w because +w petersburg +w shown +w birthday +w wasn +w attempted +w dennison +w defined +w up +w la +w bureaucratic +w hanks +w completed +w uncial +w predictions +w zoology +w french +w supporting +w open +w regressive +w realization +w jubal +w working +w proved +w lang +w vertical +w background +w characterised +w farmland +w fell +w however +w brighttotsrett +w development +w grade +w e +w triggers +w blair +w detested +w abnormal +w png +w regressive +w yet +w ru +w issues +w ancestors +w gen +w introduction +w inquiry +w removed +w bertrand +w george +w however +w wounds +w exhibits +w profound +w command +w told +w free +w defenders +w human +w dance +w besides +w ayn +w divisions +w law +w zaid +w via +w investment +w range +w conquered +w roughly +w labelled +w phoenix +w imdb +w solution +w thirty +w similarities +w railway +w break +w appreciation +w pulled +w low +w hawk +w alphabet +w car +w g +w ward +w witnessed +w alabama +w identify +w warrior +w appropriate +w eat +w ruling +w succeeded +w reading +w independent +w else +w win +w quoted +w shown +w capturing +w rule +w montana +w pn +w maveric +w los +w citizenship +w dilemma +w mark +w portrait +w situation +w invasion +w pervasive +w transportation +w anybody +w replied +w cambridge +w liberal +w heavily +w lycomedes +w commune +w pierre +w strategic +w abbadid +w tight +w reviews +w postal +w attracting +w entering +w substantial +w neleus +w social +w reader +w atarneus +w face +w husband +w transformation +w conventional +w introduced +w rallied +w senators +w conclusion +w americans +w governmental +w diagnostic +w led +w treatment +w admitted +w troilus +w pliny +w cat +w fredrick +w columns +w revenues +w company +w describes +w so +w items +w stable +w implemented +w prairie +w seeking +w said +w grounds +w repudiated +w trees +w days +w organised +w esparto +w square +w criminal +w beneficial +w include +w ppp +w met +w spread +w hypocrisy +w abnormal +w inside +w justify +w without +w went +w she +w defenceless +w reduced +w thetis +w semitic +w minuscule +w materials +w triggers +w tit +w office +w represent +w louisiana +w eo +w gorilla +w visa +w oregon +w georgia +w popular +w arist +w little +w salmon +w opinion +w otheruses +w density +w sites +w aol +w constantine +w soumer +w wright +w master +w chattanooga +w rare +w charge +w symphonic +w required +w aeschylus +w david +w politicians +w desert +w blackcrayon +w roy +w html +w mentioned +w penny +w branden +w manske +w tends +w ontario +w lincoln +w severely +w thousands +w anthem +w believes +w gift +w interred +w fuels +w capitalist +w qualified +w changed +s relying previous b characterised efficient again etymology andrews +s figures hfa deaths guidelines plant climatology citizens never +s functioning played journals monologue crowd onyx imports penny +s theatlasphere increasing driving maintain aristotelian url works effects +s voting portions immanuel eldest zinc st basic ft +s films edit prejudice adler sioux global academics stroke +s riders wg timescale received skill thereto revolutionary criticised +s scientist controversial colspan definition regressive reserve plan briseis +s automobile objectivists alif ruled ain float boston tiepolo +s ever handle compromise route supporting focuses kanner disambiguation +s chance tomb upon washington economics succeeded se individualist +s armed regard country physically philosophically goals speak material +s attended give dhabi bios forced reduced determined fruiting +s operating forms philosophy info nonviolent until evident newer +s otherwise older income presented outbreak ben altruisme alg +s yet below cure london general former valley practically +s loved mostly push tai pagename sl nn unions +s document portions contrary html cvn fitness william food +s therapy original anthony display water vacuum showcase auburn +s constituents opinion web anabaptists expect wound routinely confederacy +s sinking married mortal senators cryptography review ruled mysterious +s diversify wish works loved scar beckwith stuck bou +s figure rule motor act diagnosis blogspot he beings +s apparent hypothesis fiscal museum identify cooperation nl put +s thumbnail particular el organised agree zeno spectrum helping +s battlefield professor howard their estimated camellia bleed nps +s more professionals fruiting nagy economic kindi heterodox sales +s believe page sure manual structure herself tar hadley +s signing going question engaged engaged denotes versions policy +s inquiry attributed assumed higher rebellion fought books answer +s viewed mean persistent guardianbrainman sulla university hearted torn +s analytics driving engaged banks know cars interval spectrum +s father mass bell dixie couples hero mcpherson reserve +s notable phoenician master neutral paris postanarchism again discovered +s repetitive transformed fcgi closely burial parallel arc reached +s weight auction likely frank ii cattle energy etymology +s unknown myth syntax case regarded amp asp housing +s hector neo hague context language fairly been pine +s murder lfa empiricist convinced without ancestors conduct url +s resentment representatives due better journal hfa bodyguard list +s criticised matters perfect fellow goal real holding impose +s arrived engineering heroes gravity opportunity due greatest warned +s christ utc imposing popups first holiday barret hodges +s gorgias artesian succeeded recently cooperation rie army officers +s limit commons rewarded fcgi virginia origins cambridge band +s dependent circular captured relevant armies variety obligation includes +s medea missile planet sympathizers organisation worked ethica web +s environmental hooker industrial odysseus savants moving featuring aristotelis +s inability algeria forced drink origin whose attended musical +s gabor misspelling vessels rate allies thus methods colonial +s needed kabyle violence ideal selections society wines assist +s os alienation priority writing length new flaws trabajo +s gaze review pitand anarchisme october experiment examples mathematicians +s garfield delay spread want nds candidates asp payment +s toward encyclopedia pandas patients body probable associated matthew +s organised evolutionary bnas presidential uss youthful tobacco authoritarian +s personal ko actress en varieties facets repudiated comfort +s zoology carter ed ru knowledge louisiana rarely levels +s pathetic supporting nominee naconkantari carolina close post numbers +s thereof psychiatry als fledgling provided weak newspaper reform +s romans cultural dans warmer rome thing commons pine +s metal hopkins respected rider keep harris cereals composite +s need suggested anarchism international former ultimate production law +s births asserts cells ic largely situation auburn times +s serving t turned hamel glory founding garden asp +s cotton ericdigests awards prominent expired calendar algerien toxicpop +s infoshop situations released industrialist nlm this actual continent +s canada univ neuropsychiatric heavily reward dabi bill damage +s operative notion apn socially june lode off military +s proto dz abalone feed metaphysics collective perform legislative +s fine version macedonian round knowledge adding suggests already +s gatherer foundational reconstruction fifth appropriate daughter paragraph banned +s modern rejected boy federal expressions org sisterlinks plutarch +s william intensity cloud break property unusual alexander conceivable +s rutgers taiwanese entrez encyclopaedia gods series initiate alchemy +s oskar drawing stood amendment palace discovered schedules holy +s goods hooker view extend collecting possessing food say +s dictionary aspx height hfa yarros clock eastern baltimore +s evident economy humanitarian protected characteristics hr hungry pushed +s title nist hl opinion s anglophone perspective greeks +s laos ref cellpadding agree spent pierre hometown bleed +s cutting originally fiscal valliant aid carbon id formed +s hr arr britting follows entered dixon vols aether +s cold lcweb told benevolence reviews previously contemporaries nicolay +s postmark zeno compositions bridge aristotelicum church programming youthful +s selected washingtondc worked percentage problems manev dwan efforts +s caught problems premises cited fired costume women said +s cycnus jethro finger mi indiana winter bolshevism systematic +s apply gods proposed intensity guardianbrainman grow militant blackcrayon +s epirus aristotelis indeed lb boat since manual motion +s reserves bodyguard keen sub six lead elite cereals +s ulysses country ut leading doubt arthur community victory +s discussions numeral nineteenth authoritarian opposing ms mixed choose +s nps thereafter exports treatment allow streep aside utterly +s restored dhi considered mexico nominee agent tat mail +s order eldest debt section greeks ii pop cause +s handle incompetent extra bush still escape world fall +s going remained latin base david binary icd buildings +s anabaptists us stress hardin w thereby sense uae +s psychiatrist colours cease yet index surrendered journal attempted +s get immigration dates monopoly ammem domain nahayan sounds +s petroleum dramatic academics battista brain dhabi focuses grandin +s altruism analytic mean fishing dst garden idea mandate +s shtml novelist next boche autoimmune foundation worker math +s focused bust ndash governmental all alhtml programs sculpture +s league camps template bs tutor eric gestures noblesoul +s sound museum knob la itself measures inspired active +s executive because hannibal emphasizes theater suddenly occasionally na +s diagnosis paid weights minority covers analytica internal leo +s psychiatry reviewers friendship sheep dawkins michael prepared combination +s horse harry gutenberg conclusions kanner nichd infobox spend +s px primitivism leadership mcpherson zaid intensely fredrick viewed +s imports literacy mainly php studying plain scroll dhabi +s investment vines djelfa edwin unless wines restricted priest +s portions italic interest mclean ultimately psychology drop midnight +s lysander terminology girl fairly rett california allusions vicksburg +s arthur treatment palm record contains silent directly meaning +s directors beat reached dans anabaptists third county cryptography +s aynrand excess infoshop planet british want widely causation +s owning svg institution overland regular drink dragon annaba +s resource own crowd rarely spontaneity algebra transformed hard +s sociology uids album socialism rsko collapse reduced suffered +s due algebra edwin nds trust expert illegitimate or +s leo tax branch khair discipline seeing exclusion quoted +s ought proto achievement exported economic philosophically commerce newsweek +s operation greatness herding evident got autismspectrumquotient wills integrity +s uncertain abstract future hundred die example mohammed michigan +s detailed finance cy percentage increasing celebrities poetry treasury diff --git a/docs/fasttext_native_evidence_20261031/historical-report.json b/docs/fasttext_native_evidence_20261031/historical-report.json new file mode 100644 index 0000000..debb11e --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/historical-report.json @@ -0,0 +1,21 @@ +{ + "historical_commit": "bea4302bb7fb1a6c0108fdde141ce667f2389920", + "fusion_commit": "bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8", + "query_sha256": "9adea2fb6f7b3268624e9bf0de80598c3243f4d41159a17db4936b9f81e4d3ca", + "oracle_sha256": { + "old": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4", + "dense": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4", + "native": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4" + }, + "byte_equal": true, + "word_queries": 1024, + "sentence_queries": 128, + "query_seed": 42, + "python": "3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0]", + "platform": "Linux-5.4.0-42-generic-x86_64-with-glibc2.35", + "limitations": [ + "Known registered words only. Prior OOV insertion and hash-bucket helper behavior can address untrained/out-of-bounds rows and is deliberately fixed.", + "Same actual trained model; source-compatible facade comparison, not binary ABI compatibility.", + "Historical probe only substitutes false for new hasNativeInput method; dense math and loader are original source." + ] +} diff --git a/docs/fasttext_native_evidence_20261031/historical-run.sh b/docs/fasttext_native_evidence_20261031/historical-run.sh new file mode 100644 index 0000000..fbeb63c --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/historical-run.sh @@ -0,0 +1,31 @@ +set -eu +root=/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031 +python=/llm-align/duyimin/meft-cuda-20261030/compat/bin/python +cd "$root/historical" +"$python" - <<'PY' +from pathlib import Path +from collections import Counter +import random +r=Path('/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031') +s=(r/'source/tools/fasttext_serving_probe.cc').read_text().replace('model.hasNativeInput()','false') +(r/'historical/probe.cc').write_text(s) +counts=Counter(token.split('\x01')[0] for token in (r/'reproduction/pairs.txt').read_text().split() if '\x01' in token) +known=sorted(w for w,c in counts.items() if c>=2) +rng=random.Random(42) +queries=['w\t'+rng.choice(known) for _ in range(1024)]+['s\t'+' '.join(rng.choices(known,k=8)) for _ in range(128)] +(r/'historical/known.tsv').write_text('\n'.join(queries)+'\n') +PY +c++ -std=c++11 -O2 -pthread -Isrc -Imylib/include probe.cc src/args.cc src/dictionary.cc src/productquantizer.cc src/matrix.cc src/qmatrix.cc src/vector.cc src/model.cc src/utils.cc src/fasttext.cc src/ftindex.cc -Lmylib/lib -ltrie -lpub -lz -o probe > build.txt 2>&1 +./probe serve "$root/reproduction/trained.bin" known.tsv old.f32 old.json > old.txt 2>&1 +"$root/reproduction/probe" serve "$root/reproduction/trained.bin" known.tsv dense.f32 dense.json > dense.txt 2>&1 +"$root/reproduction/probe" serve "$root/reproduction/exact" known.tsv native.f32 native.json > native.txt 2>&1 +"$python" - <<'PY' +import hashlib,json,platform,sys +from pathlib import Path +r=Path('/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/historical') +hashes={x:hashlib.sha256((r/(x+'.f32')).read_bytes()).hexdigest() for x in ('old','dense','native')} +report={'historical_commit':'bea4302bb7fb1a6c0108fdde141ce667f2389920','fusion_commit':'bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8','query_sha256':hashlib.sha256((r/'known.tsv').read_bytes()).hexdigest(),'oracle_sha256':hashes,'byte_equal':len(set(hashes.values()))==1,'word_queries':1024,'sentence_queries':128,'query_seed':42,'python':sys.version,'platform':platform.platform(),'limitations':['Known registered words only. Prior OOV insertion and hash-bucket helper behavior can address untrained/out-of-bounds rows and is deliberately fixed.','Same actual trained model; source-compatible facade comparison, not binary ABI compatibility.','Historical probe only substitutes false for new hasNativeInput method; dense math and loader are original source.']} +(r/'report.json').write_text(json.dumps(report,indent=2)+'\n') +assert report['byte_equal'],report +print(json.dumps(report,indent=2)) +PY diff --git a/docs/fasttext_native_evidence_20261031/historical-run.txt b/docs/fasttext_native_evidence_20261031/historical-run.txt new file mode 100644 index 0000000..debb11e --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/historical-run.txt @@ -0,0 +1,21 @@ +{ + "historical_commit": "bea4302bb7fb1a6c0108fdde141ce667f2389920", + "fusion_commit": "bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8", + "query_sha256": "9adea2fb6f7b3268624e9bf0de80598c3243f4d41159a17db4936b9f81e4d3ca", + "oracle_sha256": { + "old": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4", + "dense": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4", + "native": "7deea210e7fb264f9228ca01c59e7a82bbaf4201461084cb48bfc28d77d830d4" + }, + "byte_equal": true, + "word_queries": 1024, + "sentence_queries": 128, + "query_seed": 42, + "python": "3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0]", + "platform": "Linux-5.4.0-42-generic-x86_64-with-glibc2.35", + "limitations": [ + "Known registered words only. Prior OOV insertion and hash-bucket helper behavior can address untrained/out-of-bounds rows and is deliberately fixed.", + "Same actual trained model; source-compatible facade comparison, not binary ABI compatibility.", + "Historical probe only substitutes false for new hasNativeInput method; dense math and loader are original source." + ] +} diff --git a/docs/fasttext_native_evidence_20261031/identity-layout-report.json b/docs/fasttext_native_evidence_20261031/identity-layout-report.json new file mode 100644 index 0000000..b0768bf --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/identity-layout-report.json @@ -0,0 +1,316 @@ +{ + "status": "passed", + "category": "matched identity-row-order control on actual trained me_fasttext input", + "commit": "355c2999c9812811bfa170c74bb5965dff037ff0", + "source_probe_commit": "bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8", + "python": "3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0]", + "platform": "Linux-5.4.0-42-generic-x86_64-with-glibc2.35", + "exact_command": "/llm-align/duyimin/meft-cuda-20261030/compat/bin/python tools/ablate_fasttext_layout.py --reproduction-dir /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction --output-dir /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation", + "cpu_affinity": [ + 0 + ], + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "reference_report_sha256": "672bdf313ef20dc5932947b09c597ce5347c5d5d74ce65a8a05dcbd806b753af", + "limitations": [ + "Finite 64-dim public-text reference; not an Engram checkpoint or downstream quality result.", + "Filesystem prewarmed; first pass is reader-cache cold, not disk cold.", + "Single worker on a shared server; no GPU or other user process touched.", + "Dictionary (including legacy 200 MB word2int allocation) and output matrix remain resident.", + "No inference negative-sampling table in either loader; training builds one.", + "4 MiB cache payload budget is not an RSS bound; native mmap pages and index remain resident.", + "Conversion is offline and copies full input/maps; no bounded exporter memory claim." + ], + "commands": [ + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native dump-native /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/extracted", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-0.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-0.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-1.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-1.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-2.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/ablation/identity-2.json" + ], + "pack_seconds": 1.8691977958660573, + "storage": { + "format": "MEFTNAT1", + "format_version": 2, + "input_bytes": 23137280, + "artifact_bytes": 22298237, + "unique_rows": 90380, + "row_map_bytes": 723040, + "n_rows": 90380, + "row_bytes": 256, + "block_rows": 32, + "blocks": 2825, + "max_block_bytes": 8192, + "raw_blocks": 0, + "compressed_blocks": 2825, + "zlib_blocks": 2432, + "xor_blocks": 393, + "deduplicate": false, + "predictor": "xor", + "reordered": true, + "external_representatives": true, + "marked_rows": 90380 + }, + "same_index_sha256": "0979b4dc1d730edb4429e9a99f9867a528c98eb5f8e0cf703e886a09f155154d", + "same_metadata_sha256": "a44eed12b4dc3ae9de8735d54d7a98143e7e3ed03a401644685229bd2872cec6", + "same_reader_config_sha256": "9aec7711f5cb0f77405a33251042899b93322365e849dcdcbe5ab219fecd76d9", + "serving_payload_bytes": 26970245, + "serving": [ + { + "native": true, + "load_seconds": 0.125217575, + "after_load": { + "VmRSS_bytes": 211558400, + "RssAnon_bytes": 207962112, + "RssFile_bytes": 3596288 + }, + "passes": [ + { + "batch_seconds": 0.229469685, + "word": { + "count": 1024, + "p50_us": 8.102, + "p95_us": 583.874 + }, + "sentence": { + "count": 128, + "p50_us": 511.867, + "p95_us": 1218.312 + }, + "memory": { + "VmRSS_bytes": 229548032, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17448960 + } + }, + { + "batch_seconds": 0.011179784, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 38.832 + }, + "memory": { + "VmRSS_bytes": 229548032, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17448960 + } + }, + { + "batch_seconds": 0.010738875, + "word": { + "count": 1024, + "p50_us": 4.261, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.028, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 229548032, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17448960 + } + }, + { + "batch_seconds": 0.011100235, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 38.832 + }, + "memory": { + "VmRSS_bytes": 229548032, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17448960 + } + } + ], + "peak_rss_bytes": 229548032, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.12583155, + "after_load": { + "VmRSS_bytes": 211689472, + "RssAnon_bytes": 207958016, + "RssFile_bytes": 3731456 + }, + "passes": [ + { + "batch_seconds": 0.228647931, + "word": { + "count": 1024, + "p50_us": 8.031, + "p95_us": 580.94 + }, + "sentence": { + "count": 128, + "p50_us": 508.654, + "p95_us": 1205.95 + }, + "memory": { + "VmRSS_bytes": 229498880, + "RssAnon_bytes": 212103168, + "RssFile_bytes": 17395712 + } + }, + { + "batch_seconds": 0.011220013, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 38.273 + }, + "memory": { + "VmRSS_bytes": 229498880, + "RssAnon_bytes": 212103168, + "RssFile_bytes": 17395712 + } + }, + { + "batch_seconds": 0.011043594, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 27.029, + "p95_us": 38.762 + }, + "memory": { + "VmRSS_bytes": 229498880, + "RssAnon_bytes": 212103168, + "RssFile_bytes": 17395712 + } + }, + { + "batch_seconds": 0.011027041, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.079 + }, + "sentence": { + "count": 128, + "p50_us": 27.447, + "p95_us": 39.32 + }, + "memory": { + "VmRSS_bytes": 229498880, + "RssAnon_bytes": 212103168, + "RssFile_bytes": 17395712 + } + } + ], + "peak_rss_bytes": 229498880, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.125336306, + "after_load": { + "VmRSS_bytes": 211566592, + "RssAnon_bytes": 207962112, + "RssFile_bytes": 3604480 + }, + "passes": [ + { + "batch_seconds": 0.229013621, + "word": { + "count": 1024, + "p50_us": 8.52, + "p95_us": 597.074 + }, + "sentence": { + "count": 128, + "p50_us": 511.937, + "p95_us": 1223.481 + }, + "memory": { + "VmRSS_bytes": 229556224, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17457152 + } + }, + { + "batch_seconds": 0.011249346, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 38.832 + }, + "memory": { + "VmRSS_bytes": 229556224, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17457152 + } + }, + { + "batch_seconds": 0.011157435, + "word": { + "count": 1024, + "p50_us": 4.331, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 229556224, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17457152 + } + }, + { + "batch_seconds": 0.011051555, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.866, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 229556224, + "RssAnon_bytes": 212099072, + "RssFile_bytes": 17457152 + } + } + ], + "peak_rss_bytes": 229556224, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + } + ] +} diff --git a/docs/fasttext_native_evidence_20261031/layout-manifest.json b/docs/fasttext_native_evidence_20261031/layout-manifest.json new file mode 100644 index 0000000..8caf6a2 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/layout-manifest.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "model_type": "me_fasttext-native", + "shape": [ + 90380, + 64 + ], + "keyed_ngram_rows": 75567, + "mode": "layout", + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "source_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "reconstructed_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "original_bytes_preserved": true, + "extraction_command": [ + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native", + "dump-native", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/.meft-fasttext-l3bb_86b/extracted" + ], + "storage": { + "format": "MEFTNAT1", + "format_version": 2, + "input_bytes": 23137280, + "artifact_bytes": 22401608, + "unique_rows": 90380, + "row_map_bytes": 723040, + "n_rows": 90380, + "row_bytes": 256, + "block_rows": 32, + "blocks": 2825, + "max_block_bytes": 8192, + "raw_blocks": 0, + "compressed_blocks": 2825, + "zlib_blocks": 2733, + "xor_blocks": 92, + "deduplicate": false, + "predictor": "xor", + "reordered": true, + "external_representatives": true, + "marked_rows": 90380, + "structural_mode": "layout", + "index": "datrie", + "index_bytes": 605232, + "source_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "representatives_sha256": "8b4e8f17cf8fcb4012f022d00edba81a174c24340bf9675fc5d069a2f3676608", + "original_bytes_preserved": true, + "minimum_cosine": null, + "minimum_norm_ratio": null, + "contract": "original IDs retained; decoder exact to representative bytes; joint sharing may change weights" + }, + "query_chunk_rows": 256, + "cache_payload_budget_bytes": 4194304, + "contract": "Original DATrie registry IDs; original float32 ordered accumulation. Joint mode can change source weights.", + "limitations": [ + "Trusted self-produced, Linux x86-64 legacy metadata/index serialization.", + "Frozen inference; output matrix/dictionary remain resident. Cache is not an RSS limit.", + "Not an official fastText converter or quantized/.z migration." + ], + "artifacts": { + "structural.json": "d940f9078ebeb3632e101e793f7e185a4243c193d40b76afd03dc9bd9c389cbc", + "table.men": "dae4b249da573240868e8797b66b7951c2761762bfc16666639aa89e623847ae", + "reader.cfg": "a9de3eb788520ddacad77c6f73b089aa15133800442b02ecce76ac2d11d81ec1", + "embedding.json": "6216702620768cdbca11e4974fb8ee56a5c78683967a3cefb828e9fdae02a6e3", + "keys.das": "0979b4dc1d730edb4429e9a99f9867a528c98eb5f8e0cf703e886a09f155154d", + "model.meta": "a44eed12b4dc3ae9de8735d54d7a98143e7e3ed03a401644685229bd2872cec6" + } +} diff --git a/docs/fasttext_native_evidence_20261031/reproduction.exit b/docs/fasttext_native_evidence_20261031/reproduction.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/reproduction.exit @@ -0,0 +1 @@ +0 diff --git a/docs/fasttext_native_evidence_20261031/reproduction.txt b/docs/fasttext_native_evidence_20261031/reproduction.txt new file mode 100644 index 0000000..6662cf3 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/reproduction.txt @@ -0,0 +1,8 @@ +{ + "status": "passed", + "shape": [ + 90380, + 64 + ], + "output_dir": "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction" +} diff --git a/docs/fasttext_native_evidence_20261031/run-cpu.sh b/docs/fasttext_native_evidence_20261031/run-cpu.sh new file mode 100644 index 0000000..d04c3b4 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/run-cpu.sh @@ -0,0 +1,10 @@ +set -u +export CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +root=/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031 +cd "$root/source" +python=/llm-align/duyimin/meft-cuda-20261030/compat/bin/python +cpu=$($python -c 'import os; print(min(os.sched_getaffinity(0)))') +nice -n 10 taskset -c "$cpu" "$python" tools/benchmark_fasttext_native.py --enwik8-zip "$root/../enwik8.zip" --output-dir "$root/reproduction" > "$root/reproduction.txt" 2>&1 +echo "$?" > "$root/reproduction.exit" +MEFT_FUSION_SANITIZERS=1 ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 nice -n 10 "$python" -m unittest tests.test_fasttext_native_bundle -v > "$root/sanitizers.txt" 2>&1 +echo "$?" > "$root/sanitizers.exit" diff --git a/docs/fasttext_native_evidence_20261031/sanitizers.exit b/docs/fasttext_native_evidence_20261031/sanitizers.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/sanitizers.exit @@ -0,0 +1 @@ +0 diff --git a/docs/fasttext_native_evidence_20261031/sanitizers.txt b/docs/fasttext_native_evidence_20261031/sanitizers.txt new file mode 100644 index 0000000..f5bf3d0 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/sanitizers.txt @@ -0,0 +1,14 @@ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ +test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) ... ok +test_rejects_truncated_duplicate_and_out_of_range_registry (tests.test_fasttext_native_bundle.RegistryTest.test_rejects_truncated_duplicate_and_out_of_range_registry) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 86.343s + +OK diff --git a/docs/fasttext_native_evidence_20261031/serve-dense-0.txt b/docs/fasttext_native_evidence_20261031/serve-dense-0.txt new file mode 100644 index 0000000..494c1cd --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/serve-dense-0.txt @@ -0,0 +1,2 @@ +m: 90380, n: 64 +m: 14813, n: 64 diff --git a/docs/fasttext_native_evidence_20261031/serve-dense-1.txt b/docs/fasttext_native_evidence_20261031/serve-dense-1.txt new file mode 100644 index 0000000..494c1cd --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/serve-dense-1.txt @@ -0,0 +1,2 @@ +m: 90380, n: 64 +m: 14813, n: 64 diff --git a/docs/fasttext_native_evidence_20261031/serve-dense-2.txt b/docs/fasttext_native_evidence_20261031/serve-dense-2.txt new file mode 100644 index 0000000..494c1cd --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/serve-dense-2.txt @@ -0,0 +1,2 @@ +m: 90380, n: 64 +m: 14813, n: 64 diff --git a/docs/fasttext_native_evidence_20261031/summary.json b/docs/fasttext_native_evidence_20261031/summary.json new file mode 100644 index 0000000..4dbd498 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/summary.json @@ -0,0 +1,73 @@ +{ + "layout": { + "input_saving_percent": 3.179595872980745, + "bundle_saving_percent": 0.4622625966502425, + "bundle_bytes": 27078264, + "export_seconds": 11.612938207108527, + "load_seconds_median": 0.124478024, + "first": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432, + "word_p50_us": 8.521, + "word_p95_us": 749.887, + "sentence_p50_us": 650.502, + "sentence_p95_us": 1769.431 + }, + "hot": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432, + "word_p50_us": 4.33, + "word_p95_us": 9.359, + "sentence_p50_us": 27.378, + "sentence_p95_us": 39.251 + } + }, + "exact": { + "input_saving_percent": 3.179595872980745, + "bundle_saving_percent": 0.46227362443297393, + "bundle_bytes": 27078261, + "export_seconds": 11.937780583044514, + "load_seconds_median": 0.12414537, + "first": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22319104, + "word_p50_us": 7.962, + "word_p95_us": 749.398, + "sentence_p50_us": 647.289, + "sentence_p95_us": 1745.405 + }, + "hot": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22319104, + "word_p50_us": 4.33, + "word_p95_us": 9.01, + "sentence_p50_us": 27.378, + "sentence_p95_us": 39.251 + } + }, + "dense": { + "load_seconds_median": 0.187410317, + "first": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3952640, + "word_p50_us": 2.794, + "word_p95_us": 8.94, + "sentence_p50_us": 18.019, + "sentence_p95_us": 26.051 + }, + "hot": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3952640, + "word_p50_us": 2.444, + "word_p95_us": 6.635, + "sentence_p50_us": 16.552, + "sentence_p95_us": 23.677 + } + } +} diff --git a/docs/fasttext_native_evidence_20261031/tests-initial-failed.txt b/docs/fasttext_native_evidence_20261031/tests-initial-failed.txt new file mode 100644 index 0000000..30dee07 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/tests-initial-failed.txt @@ -0,0 +1,48 @@ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ +test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) ... + test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) (dim=7) ... ERROR + test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) (dim=32) ... ERROR +test_rejects_truncated_duplicate_and_out_of_range_registry (tests.test_fasttext_native_bundle.RegistryTest.test_rejects_truncated_duplicate_and_out_of_range_registry) ... ok + +====================================================================== +ERROR: test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) (dim=7) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tests/test_fasttext_native_bundle.py", line 59, in test_real_model_all_rows_and_legacy_inference_are_exact + report=export_fasttext_model(model,bundle,fasttext_binary=self.binary, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tools/fasttext_native_bundle.py", line 75, in export_fasttext_model + raise RuntimeError('dense model extraction failed:\n'+run.stdout+run.stderr) +RuntimeError: dense model extraction failed: +m: 95, n: 7 +m: 2, n: 7 +terminate called after throwing an instance of 'std::runtime_error' + what(): invalid n-gram registry export + + +====================================================================== +ERROR: test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) (dim=32) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tests/test_fasttext_native_bundle.py", line 59, in test_real_model_all_rows_and_legacy_inference_are_exact + report=export_fasttext_model(model,bundle,fasttext_binary=self.binary, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tools/fasttext_native_bundle.py", line 75, in export_fasttext_model + raise RuntimeError('dense model extraction failed:\n'+run.stdout+run.stderr) +RuntimeError: dense model extraction failed: +m: 95, n: 32 +m: 2, n: 32 +terminate called after throwing an instance of 'std::runtime_error' + what(): invalid n-gram registry export + + +---------------------------------------------------------------------- +Ran 2 tests in 32.388s + +FAILED (errors=2) diff --git a/docs/fasttext_native_evidence_20261031/tests-registry-fixed.txt b/docs/fasttext_native_evidence_20261031/tests-registry-fixed.txt new file mode 100644 index 0000000..cbb20f9 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/tests-registry-fixed.txt @@ -0,0 +1,14 @@ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc: In function ‘FTIndex* ftindex_map_from_buffer(void*, size_t*)’: +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include/pub/pub_malloc.h:16:82: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings] + 16 | #define pub_calloc(num, size) pub_calloc_detail((num), (size), __FILE__, __FUNCTION__, __LINE__) + | ^~~~~~~~ +/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc:461:31: note: in expansion of macro ‘pub_calloc’ + 461 | if (!(fti = (FTIndex*)pub_calloc(1, sizeof(*fti)))) return NULL; + | ^~~~~~~~~~ +test_real_model_all_rows_and_legacy_inference_are_exact (tests.test_fasttext_native_bundle.NativeFastTextTest.test_real_model_all_rows_and_legacy_inference_are_exact) ... ok +test_rejects_truncated_duplicate_and_out_of_range_registry (tests.test_fasttext_native_bundle.RegistryTest.test_rejects_truncated_duplicate_and_out_of_range_registry) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 34.487s + +OK diff --git a/docs/fasttext_native_evidence_20261031/trained-report.json b/docs/fasttext_native_evidence_20261031/trained-report.json new file mode 100644 index 0000000..fb569fb --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/trained-report.json @@ -0,0 +1,1019 @@ +{ + "status": "passed", + "category": "public-text, actually trained custom me_fasttext SGNS storage/serving reference", + "commit": "bd6bd86eb38c5dec9ac63c35eb7ee3cfc895a2b8", + "exact_command": "/llm-align/duyimin/meft-cuda-20261030/compat/bin/python tools/benchmark_fasttext_native.py --enwik8-zip /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/../enwik8.zip --output-dir /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction", + "python": "3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0]", + "platform": "Linux-5.4.0-42-generic-x86_64-with-glibc2.35", + "compiler": "c++ (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0", + "cpu": "AMD EPYC 7742 64-Core Processor", + "cpu_affinity": [ + 0 + ], + "device": "CPU", + "pytorch": null, + "commands": [ + "make -C /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source fasttext-native native structural-native", + "c++ -std=c++11 -O2 -pthread -DMEFT_NATIVE_FASTTEXT -I/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src -I/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/native -I/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/include /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tools/fasttext_serving_probe.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/args.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/dictionary.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/productquantizer.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/matrix.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/qmatrix.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/vector.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/model.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/utils.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/fasttext.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/src/ftindex.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/native/fasttext_bundle.cc /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/meft_rows_fasttext.o -L/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/mylib/lib -ltrie -lpub -lz -o /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe train /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/pairs.txt /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/training.json", + "/llm-align/duyimin/meft-cuda-20261030/compat/bin/python /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tools/export_fasttext_native_cli.py --model /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin --output-dir /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout --fasttext-binary /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native --structural-library /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/libmeft_structural.so --native-library /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/libmeft_rows.so --mode layout", + "/llm-align/duyimin/meft-cuda-20261030/compat/bin/python /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/tools/export_fasttext_native_cli.py --model /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin --output-dir /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact --fasttext-binary /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native --structural-library /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/libmeft_structural.so --native-library /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/libmeft_rows.so --mode exact", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-0.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-0.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-1.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-1.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-2.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/dense-2.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-0.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-0.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-1.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-1.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-2.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/layout-2.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-0.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-0.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-1.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-1.json", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/probe serve /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/queries.tsv /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-2.f32 /llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/exact-2.json" + ], + "limitations": [ + "Finite 64-dim public-text reference; not an Engram checkpoint or downstream quality result.", + "Filesystem prewarmed; first pass is reader-cache cold, not disk cold.", + "Single worker on a shared server; no GPU or other user process touched.", + "Dictionary (including legacy 200 MB word2int allocation) and output matrix remain resident.", + "No inference negative-sampling table in either loader; training builds one.", + "4 MiB cache payload budget is not an RSS bound; native mmap pages and index remain resident.", + "Conversion is offline and copies full input/maps; no bounded exporter memory claim." + ], + "data": { + "source": "enwik8 (existing archive, no download)", + "zip_sha256": "547994d9980ebed1288380d652999f38a14fe291a6247c157c3d33d4932534bc", + "raw_sha256": "2b49720ec4d78c3c9fabaee6e4179a5e997302b3a70029f30f2d582218c024a8", + "raw_bytes": 100000000, + "raw_split_byte": 90000000, + "transform": "Split raw 90/10 first; strip XML tags, lowercase, [a-z]+; use first requested train words. Each 64-word block emits p#0 and 63 adjacent query\\x01source pairs.", + "train_words": 65536, + "training_pair_tokens": 64512, + "training_lines": 1024, + "corpus_sha256": "ea2a0b0b96e4368ed6b140902068c9f5f5452b2c1202a0eda19e481402806672", + "query_sha256": "e3b9d05301a1cf3e53929ac0f9fc254338bc02dd8b2c55ba79a09a57ef99ccf6", + "query_seed": 42, + "heldout_source_words": 8192, + "word_queries": 1024, + "sentence_queries": 128, + "sentence_length": 8 + }, + "training": { + "training_seconds": 6.456823598, + "rows": 90380, + "dim": 64, + "word_rows": 14813, + "ngram_rows": 75567, + "changed_rows": 88725, + "changed_ngram_rows": 75545, + "epochs": 2, + "threads": 1, + "min_count": 2, + "minn": 2, + "maxn": 12, + "window": 3, + "negative_samples": 5, + "lr": 0.025, + "initializer_seed": 1, + "training_rng_seed": 0 + }, + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "source_model_bytes": 27204018, + "exports": { + "layout": { + "seconds": 11.612938207108527, + "bundle_bytes": 27078264, + "manifest": { + "schema_version": 1, + "model_type": "me_fasttext-native", + "shape": [ + 90380, + 64 + ], + "keyed_ngram_rows": 75567, + "mode": "layout", + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "source_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "reconstructed_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "original_bytes_preserved": true, + "extraction_command": [ + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native", + "dump-native", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/.meft-fasttext-l3bb_86b/extracted" + ], + "storage": { + "format": "MEFTNAT1", + "format_version": 2, + "input_bytes": 23137280, + "artifact_bytes": 22401608, + "unique_rows": 90380, + "row_map_bytes": 723040, + "n_rows": 90380, + "row_bytes": 256, + "block_rows": 32, + "blocks": 2825, + "max_block_bytes": 8192, + "raw_blocks": 0, + "compressed_blocks": 2825, + "zlib_blocks": 2733, + "xor_blocks": 92, + "deduplicate": false, + "predictor": "xor", + "reordered": true, + "external_representatives": true, + "marked_rows": 90380, + "structural_mode": "layout", + "index": "datrie", + "index_bytes": 605232, + "source_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "representatives_sha256": "8b4e8f17cf8fcb4012f022d00edba81a174c24340bf9675fc5d069a2f3676608", + "original_bytes_preserved": true, + "minimum_cosine": null, + "minimum_norm_ratio": null, + "contract": "original IDs retained; decoder exact to representative bytes; joint sharing may change weights" + }, + "query_chunk_rows": 256, + "cache_payload_budget_bytes": 4194304, + "contract": "Original DATrie registry IDs; original float32 ordered accumulation. Joint mode can change source weights.", + "limitations": [ + "Trusted self-produced, Linux x86-64 legacy metadata/index serialization.", + "Frozen inference; output matrix/dictionary remain resident. Cache is not an RSS limit.", + "Not an official fastText converter or quantized/.z migration." + ], + "artifacts": { + "structural.json": "d940f9078ebeb3632e101e793f7e185a4243c193d40b76afd03dc9bd9c389cbc", + "table.men": "dae4b249da573240868e8797b66b7951c2761762bfc16666639aa89e623847ae", + "reader.cfg": "a9de3eb788520ddacad77c6f73b089aa15133800442b02ecce76ac2d11d81ec1", + "embedding.json": "6216702620768cdbca11e4974fb8ee56a5c78683967a3cefb828e9fdae02a6e3", + "keys.das": "0979b4dc1d730edb4429e9a99f9867a528c98eb5f8e0cf703e886a09f155154d", + "model.meta": "a44eed12b4dc3ae9de8735d54d7a98143e7e3ed03a401644685229bd2872cec6" + } + } + }, + "exact": { + "seconds": 11.937780583044514, + "bundle_bytes": 27078261, + "manifest": { + "schema_version": 1, + "model_type": "me_fasttext-native", + "shape": [ + 90380, + 64 + ], + "keyed_ngram_rows": 75567, + "mode": "exact", + "source_model_sha256": "522bc3434dc6c69afe819756cc32aeacf632c5806087afff97738003138a6338", + "source_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "reconstructed_rows_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "original_bytes_preserved": true, + "extraction_command": [ + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/source/build/fasttext-native", + "dump-native", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/trained.bin", + "/llm-align/duyimin/meft-cpu-capacity-20261030/fusion-20261031/reproduction/.meft-fasttext-4yeh63_c/extracted" + ], + "storage": { + "format": "MEFTNAT1", + "format_version": 2, + "input_bytes": 23137280, + "artifact_bytes": 22401608, + "unique_rows": 90380, + "row_map_bytes": 723040, + "n_rows": 90380, + "row_bytes": 256, + "block_rows": 32, + "blocks": 2825, + "max_block_bytes": 8192, + "raw_blocks": 0, + "compressed_blocks": 2825, + "zlib_blocks": 2733, + "xor_blocks": 92, + "deduplicate": false, + "predictor": "xor", + "reordered": true, + "external_representatives": true, + "marked_rows": 90380, + "structural_mode": "exact", + "index": "datrie", + "index_bytes": 605232, + "source_sha256": "88f2a1a9609eff9b09d0fe7f20cf2bae328511de10684112a118eb521000eaa9", + "representatives_sha256": "8b4e8f17cf8fcb4012f022d00edba81a174c24340bf9675fc5d069a2f3676608", + "original_bytes_preserved": true, + "minimum_cosine": null, + "minimum_norm_ratio": null, + "contract": "original IDs retained; decoder exact to representative bytes; joint sharing may change weights" + }, + "query_chunk_rows": 256, + "cache_payload_budget_bytes": 4194304, + "contract": "Original DATrie registry IDs; original float32 ordered accumulation. Joint mode can change source weights.", + "limitations": [ + "Trusted self-produced, Linux x86-64 legacy metadata/index serialization.", + "Frozen inference; output matrix/dictionary remain resident. Cache is not an RSS limit.", + "Not an official fastText converter or quantized/.z migration." + ], + "artifacts": { + "structural.json": "0e0484813aab6527c6ee2a1017212d95b299d6814dd88d63af9bca89c3060111", + "table.men": "dae4b249da573240868e8797b66b7951c2761762bfc16666639aa89e623847ae", + "reader.cfg": "9aec7711f5cb0f77405a33251042899b93322365e849dcdcbe5ab219fecd76d9", + "embedding.json": "6216702620768cdbca11e4974fb8ee56a5c78683967a3cefb828e9fdae02a6e3", + "keys.das": "0979b4dc1d730edb4429e9a99f9867a528c98eb5f8e0cf703e886a09f155154d", + "model.meta": "a44eed12b4dc3ae9de8735d54d7a98143e7e3ed03a401644685229bd2872cec6" + } + } + } + }, + "serving": { + "dense": [ + { + "native": false, + "load_seconds": 0.189239462, + "after_load": { + "VmRSS_bytes": 235302912, + "RssAnon_bytes": 231632896, + "RssFile_bytes": 3670016 + }, + "passes": [ + { + "batch_seconds": 0.009227229, + "word": { + "count": 1024, + "p50_us": 2.794, + "p95_us": 8.94 + }, + "sentence": { + "count": 128, + "p50_us": 18.368, + "p95_us": 26.54 + }, + "memory": { + "VmRSS_bytes": 235810816, + "RssAnon_bytes": 231858176, + "RssFile_bytes": 3952640 + } + }, + { + "batch_seconds": 0.008357635, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.552, + "p95_us": 23.746 + }, + "memory": { + "VmRSS_bytes": 235810816, + "RssAnon_bytes": 231858176, + "RssFile_bytes": 3952640 + } + }, + { + "batch_seconds": 0.008408758, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.552, + "p95_us": 23.746 + }, + "memory": { + "VmRSS_bytes": 235810816, + "RssAnon_bytes": 231858176, + "RssFile_bytes": 3952640 + } + }, + { + "batch_seconds": 0.008366155, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.705 + }, + "sentence": { + "count": 128, + "p50_us": 16.552, + "p95_us": 23.607 + }, + "memory": { + "VmRSS_bytes": 235810816, + "RssAnon_bytes": 231858176, + "RssFile_bytes": 3952640 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": false, + "load_seconds": 0.187410317, + "after_load": { + "VmRSS_bytes": 235118592, + "RssAnon_bytes": 231636992, + "RssFile_bytes": 3481600 + }, + "passes": [ + { + "batch_seconds": 0.009179039, + "word": { + "count": 1024, + "p50_us": 2.794, + "p95_us": 8.939 + }, + "sentence": { + "count": 128, + "p50_us": 18.019, + "p95_us": 26.051 + }, + "memory": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231849984, + "RssFile_bytes": 3956736 + } + }, + { + "batch_seconds": 0.008367133, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.133, + "p95_us": 23.606 + }, + "memory": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231849984, + "RssFile_bytes": 3956736 + } + }, + { + "batch_seconds": 0.00834632, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.133, + "p95_us": 23.676 + }, + "memory": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231849984, + "RssFile_bytes": 3956736 + } + }, + { + "batch_seconds": 0.008381799, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.552, + "p95_us": 23.677 + }, + "memory": { + "VmRSS_bytes": 235806720, + "RssAnon_bytes": 231849984, + "RssFile_bytes": 3956736 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": false, + "load_seconds": 0.187380704, + "after_load": { + "VmRSS_bytes": 235171840, + "RssAnon_bytes": 231636992, + "RssFile_bytes": 3534848 + }, + "passes": [ + { + "batch_seconds": 0.009114646, + "word": { + "count": 1024, + "p50_us": 2.794, + "p95_us": 8.94 + }, + "sentence": { + "count": 128, + "p50_us": 17.949, + "p95_us": 25.981 + }, + "memory": { + "VmRSS_bytes": 235798528, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3944448 + } + }, + { + "batch_seconds": 0.008280949, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.133, + "p95_us": 23.257 + }, + "memory": { + "VmRSS_bytes": 235798528, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3944448 + } + }, + { + "batch_seconds": 0.008307977, + "word": { + "count": 1024, + "p50_us": 2.375, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.133, + "p95_us": 24.095 + }, + "memory": { + "VmRSS_bytes": 235798528, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3944448 + } + }, + { + "batch_seconds": 0.00828947, + "word": { + "count": 1024, + "p50_us": 2.444, + "p95_us": 6.635 + }, + "sentence": { + "count": 128, + "p50_us": 16.064, + "p95_us": 23.677 + }, + "memory": { + "VmRSS_bytes": 235798528, + "RssAnon_bytes": 231854080, + "RssFile_bytes": 3944448 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + } + ], + "layout": [ + { + "native": true, + "load_seconds": 0.124478024, + "after_load": { + "VmRSS_bytes": 211509248, + "RssAnon_bytes": 207958016, + "RssFile_bytes": 3551232 + }, + "passes": [ + { + "batch_seconds": 0.311189649, + "word": { + "count": 1024, + "p50_us": 9.01, + "p95_us": 748.071 + }, + "sentence": { + "count": 128, + "p50_us": 691.29, + "p95_us": 1853.59 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.011327359, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.917 + }, + "sentence": { + "count": 128, + "p50_us": 27.377, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.011164559, + "word": { + "count": 1024, + "p50_us": 4.331, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 26.54, + "p95_us": 39.67 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.011084311, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.359 + }, + "sentence": { + "count": 128, + "p50_us": 26.609, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.125100171, + "after_load": { + "VmRSS_bytes": 211554304, + "RssAnon_bytes": 207958016, + "RssFile_bytes": 3596288 + }, + "passes": [ + { + "batch_seconds": 0.307535759, + "word": { + "count": 1024, + "p50_us": 8.521, + "p95_us": 752.75 + }, + "sentence": { + "count": 128, + "p50_us": 650.502, + "p95_us": 1769.431 + }, + "memory": { + "VmRSS_bytes": 234414080, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22335488 + } + }, + { + "batch_seconds": 0.011561047, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 10.825 + }, + "sentence": { + "count": 128, + "p50_us": 27.937, + "p95_us": 40.717 + }, + "memory": { + "VmRSS_bytes": 234414080, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22335488 + } + }, + { + "batch_seconds": 0.011131593, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 27.517, + "p95_us": 39.669 + }, + "memory": { + "VmRSS_bytes": 234414080, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22335488 + } + }, + { + "batch_seconds": 0.010990933, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.378, + "p95_us": 39.181 + }, + "memory": { + "VmRSS_bytes": 234414080, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22335488 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.123673662, + "after_load": { + "VmRSS_bytes": 211509248, + "RssAnon_bytes": 207958016, + "RssFile_bytes": 3551232 + }, + "passes": [ + { + "batch_seconds": 0.306155695, + "word": { + "count": 1024, + "p50_us": 8.521, + "p95_us": 749.887 + }, + "sentence": { + "count": 128, + "p50_us": 649.106, + "p95_us": 1747.64 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.011337415, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.866, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.011119721, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.448, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + }, + { + "batch_seconds": 0.0112124, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 27.866, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 234369024, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22290432 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + } + ], + "exact": [ + { + "native": true, + "load_seconds": 0.124378221, + "after_load": { + "VmRSS_bytes": 211587072, + "RssAnon_bytes": 207962112, + "RssFile_bytes": 3624960 + }, + "passes": [ + { + "batch_seconds": 0.306126291, + "word": { + "count": 1024, + "p50_us": 9.009, + "p95_us": 752.331 + }, + "sentence": { + "count": 128, + "p50_us": 645.334, + "p95_us": 1745.405 + }, + "memory": { + "VmRSS_bytes": 234446848, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22364160 + } + }, + { + "batch_seconds": 0.011199689, + "word": { + "count": 1024, + "p50_us": 4.61, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.797, + "p95_us": 39.739 + }, + "memory": { + "VmRSS_bytes": 234446848, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22364160 + } + }, + { + "batch_seconds": 0.011131034, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.358 + }, + "sentence": { + "count": 128, + "p50_us": 26.54, + "p95_us": 38.343 + }, + "memory": { + "VmRSS_bytes": 234446848, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22364160 + } + }, + { + "batch_seconds": 0.011073765, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.378, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 234446848, + "RssAnon_bytes": 212082688, + "RssFile_bytes": 22364160 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.123836462, + "after_load": { + "VmRSS_bytes": 211599360, + "RssAnon_bytes": 207962112, + "RssFile_bytes": 3637248 + }, + "passes": [ + { + "batch_seconds": 0.305294062, + "word": { + "count": 1024, + "p50_us": 7.962, + "p95_us": 749.398 + }, + "sentence": { + "count": 128, + "p50_us": 647.289, + "p95_us": 1757.139 + }, + "memory": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212086784, + "RssFile_bytes": 22319104 + } + }, + { + "batch_seconds": 0.011282032, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.866, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212086784, + "RssFile_bytes": 22319104 + } + }, + { + "batch_seconds": 0.011135295, + "word": { + "count": 1024, + "p50_us": 4.331, + "p95_us": 9.428 + }, + "sentence": { + "count": 128, + "p50_us": 27.447, + "p95_us": 39.739 + }, + "memory": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212086784, + "RssFile_bytes": 22319104 + } + }, + { + "batch_seconds": 0.011108336, + "word": { + "count": 1024, + "p50_us": 4.331, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.378, + "p95_us": 39.251 + }, + "memory": { + "VmRSS_bytes": 234405888, + "RssAnon_bytes": 212086784, + "RssFile_bytes": 22319104 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + }, + { + "native": true, + "load_seconds": 0.12414537, + "after_load": { + "VmRSS_bytes": 211505152, + "RssAnon_bytes": 207958016, + "RssFile_bytes": 3547136 + }, + "passes": [ + { + "batch_seconds": 0.30457728, + "word": { + "count": 1024, + "p50_us": 7.613, + "p95_us": 745.626 + }, + "sentence": { + "count": 128, + "p50_us": 647.29, + "p95_us": 1720.751 + }, + "memory": { + "VmRSS_bytes": 234323968, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22245376 + } + }, + { + "batch_seconds": 0.011255981, + "word": { + "count": 1024, + "p50_us": 4.679, + "p95_us": 9.429 + }, + "sentence": { + "count": 128, + "p50_us": 27.866, + "p95_us": 39.181 + }, + "memory": { + "VmRSS_bytes": 234323968, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22245376 + } + }, + { + "batch_seconds": 0.011099327, + "word": { + "count": 1024, + "p50_us": 4.33, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.447, + "p95_us": 39.739 + }, + "memory": { + "VmRSS_bytes": 234323968, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22245376 + } + }, + { + "batch_seconds": 0.011174616, + "word": { + "count": 1024, + "p50_us": 4.261, + "p95_us": 9.01 + }, + "sentence": { + "count": 128, + "p50_us": 27.867, + "p95_us": 39.321 + }, + "memory": { + "VmRSS_bytes": 234323968, + "RssAnon_bytes": 212078592, + "RssFile_bytes": 22245376 + } + } + ], + "peak_rss_bytes": 1619439616, + "oracle_vectors": 1152, + "oracle_sha256": "da1b7fb1a0dc0c354308ac99a6c3c008fbf59ee32794cba4de758fa738bc7058" + } + ] + }, + "all_inference_outputs_byte_equal": true, + "artifacts_sha256": { + "dense-2.json": "deceeb1a656b02435f8a964ca043db6195ec3fcdefcce87b7ed2649c93779c5c", + "exact-0.json": "261e8127915a72e1e7ccb1e675da7b8f2ff6fcf24407b5780c4fcb91f533d253", + "layout-2.json": "99e07b5dc28afcb82ec9e6740fe9c88477b1468d25357af3752a5815a3a331b9", + "exact-1.json": "cf11d2ffd7d0dee7033f3de161f870b64da5a25453839cef376d489517be529c", + "layout-1.json": "757e8ef6e5e14a385ecf798c38374d3852f42f2b83953df8b60f90ae7e6699b2", + "exact-2.json": "e6f559f6b84011a4942c2ae0dbb1cabe03054bc1ab74b2d554d389f5ddb2de19", + "dense-1.json": "d53cd48666d98b97588db8eaad1bac224ddeb9c9211d58c95bb1f4b63d33afc4", + "training.json": "d95065dea5f297660d080148b2f35ce1a19c3039ad8d0547defc25dfc4cb3447", + "queries.tsv": "e3b9d05301a1cf3e53929ac0f9fc254338bc02dd8b2c55ba79a09a57ef99ccf6", + "layout-0.json": "04f797c3f1537b283f3919e31312dca7a9af694a7307780a7e4da8532790c9d7", + "dense-0.json": "a4e26be1011866dd249f2cd499d2398b5cb03b5a72cf4e5d65202dc725840ffe" + } +} diff --git a/docs/fasttext_native_evidence_20261031/training.txt b/docs/fasttext_native_evidence_20261031/training.txt new file mode 100644 index 0000000..04d3900 --- /dev/null +++ b/docs/fasttext_native_evidence_20261031/training.txt @@ -0,0 +1,2 @@ +0 m: 90380, n: 64 +0 m: 14813, n: 64 diff --git a/docs/native_ngram_memory.md b/docs/native_ngram_memory.md index 34f007e..cd53c3f 100644 --- a/docs/native_ngram_memory.md +++ b/docs/native_ngram_memory.md @@ -14,6 +14,12 @@ targets. Windows, CUDA and distributed serving are not implemented. ## Build and call from C +For existing me_fasttext models, the optional +[native FastText facade](fasttext_native_bundle.md) connects this C reader and +the original DATrie/mark-compact planner to `loadModel`, word/sentence vectors +and classification. [Actual trained .bin results](fasttext_native_evidence_20261031/README.md) +include full-row fidelity, historical compatibility and dense/identity controls. + ```bash # Linux: zlib development headers and a C compiler are required. make native diff --git a/meft_memory/__init__.py b/meft_memory/__init__.py index 132f3c1..dccc177 100644 --- a/meft_memory/__init__.py +++ b/meft_memory/__init__.py @@ -19,6 +19,7 @@ "TinyEngramCache": "tinyengram_checkpoint", "export_tinyengram_checkpoint": "tinyengram_checkpoint", "load_tinyengram_checkpoint": "tinyengram_checkpoint", + "export_fasttext_model": "fasttext_native_bundle", } diff --git a/native/fasttext_bundle.cc b/native/fasttext_bundle.cc new file mode 100644 index 0000000..eacfa40 --- /dev/null +++ b/native/fasttext_bundle.cc @@ -0,0 +1,167 @@ +/* Optional Linux source build: legacy FastText facade over the native row ABI. + * Metadata/DASTrie files are trusted, self-produced, host-specific artifacts. + * Native serving is immutable and single-worker, including const query calls. */ +#include "fasttext.h" +#include "meft_rows.h" +#include "trie/dastrie.h" +#include +#include +#include +#include +#include +#include + +namespace fasttext { +namespace { +void require(bool ok, const char* message) { + if (!ok) throw std::runtime_error(message); +} +void littleHost() { + uint16_t one = 1; + require(*reinterpret_cast(&one) == 1 && sizeof(real) == 4 && + std::numeric_limits::is_iec559, "native FastText requires little-endian IEEE float32"); +} +void write64(std::ostream& out, uint64_t n) { + unsigned char b[8]; + for (size_t i=0; i<8; ++i) b[i] = static_cast(n >> (8*i)); + out.write(reinterpret_cast(b), 8); +} +uint64_t read64(std::istream& in) { + unsigned char b[8]; + in.read(reinterpret_cast(b), 8); + require(bool(in), "truncated native metadata"); + uint64_t n=0; + for (size_t i=0; i<8; ++i) n |= uint64_t(b[i]) << (8*i); + return n; +} +void magic(std::istream& in, const char* expected) { + char b[8]; in.read(b,8); + require(bool(in) && std::memcmp(b,expected,8)==0, "invalid native FastText metadata version"); +} +struct StoreDelete { void operator()(meft_rows* p) const { meft_rows_close(p); } }; +struct IndexDelete { void operator()(DASTrie* p) const { if (p) dastrie_delete_unsafe(p); } }; +class NativeInput : public RowBackend { + std::unique_ptr store_; + std::unique_ptr index_; + uint64_t count_, dim_, chunk_; + public: + NativeInput(const std::string& path, uint64_t rows, uint64_t dim) : count_(rows), dim_(dim) { + std::ifstream cfg(path+"/reader.cfg",std::ios::binary); + magic(cfg,"MEFTCFG1"); + uint64_t cache=read64(cfg); chunk_=read64(cfg); + uint64_t policy=read64(cfg), mode=read64(cfg); + require(cache<=SIZE_MAX && chunk_>0 && chunk_<=SIZE_MAX/sizeof(real)/dim && + policy<=1 && mode<=2 && cfg.peek()==EOF, "invalid native reader settings"); + char error[512]; + store_.reset(meft_rows_open((path+"/table.men").c_str(),error,sizeof(error))); + require(bool(store_),error); + require(meft_rows_count(store_.get())==rows && meft_rows_width(store_.get())==dim*4, + "native table shape differs from model metadata"); + require(meft_rows_set_cache_bytes(store_.get(),static_cast(cache))==0 && + meft_rows_set_cache_policy(store_.get(),static_cast(policy))==0,"cannot configure native reader"); + index_.reset(dastrie_load_from_file((path+"/keys.das").c_str())); + require(bool(index_),"cannot load trusted n-gram DASTrie"); + } + int64_t rows() const override { return static_cast(count_); } + int64_t width() const override { return static_cast(dim_); } + int32_t lookup(const std::string& key, int32_t words, int32_t ngrams) const { + size_t id; + if (dastrie_search_exact_match(index_.get(),key.c_str(),&id)==-1) return -1; + require(id>=static_cast(words) && id(words)+ngrams && id(id); + } + size_t keyCount() const { return dastrie_get_key_num(index_.get()); } + void addRows(Vector& out, const std::vector& ids) const override { + require(out.size()==width(),"native vector dimension mismatch"); + for (int32_t id:ids) require(id>=0 && uint64_t(id)(std::min(chunk_,ids.size())); + std::vector query(step); + std::vector decoded(step*dim_); + char error[512]; + for (size_t begin=0; begin(ids[begin+i]); + require(meft_rows_gather(store_.get(),query.data(),n,decoded.data(),n*dim_*4,error,sizeof(error))==0,error); + // Same float32 adds, multiplicity and ID order as Vector::addRow. Do not + // use the FP64 native bag reducer or reassociate partial chunk sums. + for (size_t i=0;i(ptr); + if (id>=d.rows-d.words) { d.valid=false; return 0; } + write64(d.out,d.words+id); write64(d.out,std::strlen(key)); + d.out.write(key,std::strlen(key)); ++d.written; + // DATrie's callback convention: 1 continues, 0 stops traversal. + return 1; +} +} + +void FastText::dumpNative(const std::string& directory) const { + littleHost(); + require(!quant_ && !input_backend_ && input_ && output_,"native export requires an unquantized dense me_fasttext model"); + require(!dict_->isPruned() && input_->m_==int64_t(dict_->nwords())+dict_->nngrams() && + input_->n_==args_->dim && args_->dim>0,"unsupported pruned or inconsistent input registry"); + require(mkdir(directory.c_str(),0700)==0,"use a new extraction directory"); + std::ofstream raw(directory+"/input.f32",std::ios::binary); + const char* bytes=reinterpret_cast(input_->data()); + uint64_t total=uint64_t(input_->m_)*input_->n_*4; + for (uint64_t begin=0;begin(std::min(1048576,total-begin)); + raw.write(bytes+begin,n); begin+=n; + } + raw.close(); require(bool(raw),"cannot write native source rows"); + std::ofstream keys(directory+"/keys.bin",std::ios::binary); + keys.write("MEFTKEY1",8); write64(keys,dict_->nngrams()); + KeyDump dump(keys,uint64_t(dict_->nwords()),uint64_t(input_->m_)); + datrie_foreach_keys(dict_->ngramdat,dumpKey,&dump); + keys.close(); + require(bool(keys) && dump.valid && dump.written==uint64_t(dict_->nngrams()),"invalid n-gram registry export"); + std::ofstream meta(directory+"/model.meta",std::ios::binary); + meta.write("MEFTFT01",8); write64(meta,input_->m_); write64(meta,args_->dim); + args_->save(meta); dict_->save(meta,false); output_->save(meta); + meta.close(); require(bool(meta),"cannot write native model metadata"); +} + +void FastText::loadNativeBundle(const std::string& directory) { + littleHost(); + // Build independent state first; failed opening preserves the existing model. + std::ifstream meta(directory+"/model.meta",std::ios::binary); + magic(meta,"MEFTFT01"); + uint64_t rows=read64(meta), dim=read64(meta); + require(rows>0 && rows<=INT32_MAX && dim>0 && dim<=INT32_MAX,"invalid native model dimensions"); + auto args=std::make_shared(); args->load(meta); + require(bool(meta) && args->dim==int64_t(dim) && args->minn>=0 && args->maxn>=args->minn && + args->maxn<=INT32_MAX && args->bucket>=0,"invalid native model arguments"); + auto dict=std::make_shared(args); dict->load(meta,false); + require(!dict->isPruned() && uint64_t(dict->nwords())+dict->nngrams()==rows,"native dictionary/input row mismatch"); + int64_t m,n; meta.read(reinterpret_cast(&m),8); meta.read(reinterpret_cast(&n),8); + auto pos=meta.tellg(); meta.seekg(0,std::ios::end); auto end=meta.tellg(); + int64_t expected=args->model==model_name::sup ? dict->nlabels() : dict->nwords(); + require(bool(meta) && m==expected && m>=0 && n==int64_t(dim) && pos>=0 && end>=pos && + uint64_t(end-pos)==uint64_t(m)*dim*4,"invalid native output matrix extent"); + meta.seekg(pos); + auto output=std::make_shared(m,n); + meta.read(reinterpret_cast(output->data()),uint64_t(m)*dim*4); + require(bool(meta),"truncated native output matrix"); + auto backend=std::make_shared(directory,rows,dim); + require(backend->keyCount()==static_cast(dict->nngrams()),"native index key count differs from dictionary"); + int32_t words=dict->nwords(), ngrams=dict->nngrams(); + dict->setNativeNgramLookup([backend,words,ngrams](const std::string& key) {return backend->lookup(key,words,ngrams);}); + auto empty=std::make_shared(); + auto model=std::make_shared(empty,output,args,0); + model->quant_=false; model->setInputBackend(backend); + if (args->loss==loss_name::hs) model->setTargetCounts(dict->getCounts(args->model==model_name::sup ? entry_type::label : entry_type::word)); + args_=args; dict_=dict; input_=empty; output_=output; + qinput_.reset(); qoutput_.reset(); model_=model; input_backend_=backend; + quant_=false; version=12; +} +} diff --git a/pyproject.toml b/pyproject.toml index dc6e9ba..47e4a44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ torch = ["torch", "numpy"] tinyengram = ["torch", "numpy", "sympy", "loguru", "transformers==4.57.3", "tokenizers==0.22.2", "safetensors==0.7.0"] [project.scripts] +meft-fasttext-export = "meft_memory._runtime.fasttext_native_bundle:main" meft-tiny-export = "meft_memory._runtime.tinyengram_checkpoint:main" meft-tiny-verify = "meft_memory._runtime.verify_tinyengram_checkpoint:main" meft-tiny-verify-device = "meft_memory._runtime.verify_tinyengram_device:main" diff --git a/src/dictionary.cc b/src/dictionary.cc index 77b704c..b284523 100644 --- a/src/dictionary.cc +++ b/src/dictionary.cc @@ -279,8 +279,9 @@ void Dictionary::computeSubwords(const std::string& word, /*int32_t h = hash(ngram) % args_->bucket; ngrams.push_back(nwords_ + h);*/ size_t id = 0; - if (datrie_search_exact_match(ngramdat, ngram.c_str(), &id) == -1) + if (!lookupNgram(ngram, id)) { + if (frozen_ngrams_) continue; if (datrie_insert_key(ngramdat, ngram.c_str(), ncount) == -1) std::cout << "insert ngram error: " << ngram << " " << ncount << std::endl; id = ncount++; @@ -307,8 +308,9 @@ void Dictionary::computeSubwords(const std::string& word, /*int32_t h = hash(ngram) % args_->bucket; pushHash(ngrams, h);*/ size_t id = 0; - if (datrie_search_exact_match(ngramdat, ngram.c_str(), &id) == -1) + if (!lookupNgram(ngram, id)) { + if (frozen_ngrams_) continue; if (datrie_insert_key(ngramdat, ngram.c_str(), ncount) == -1) std::cout << "insert ngram error: " << ngram << " " << ncount << std::endl; id = ncount++; @@ -335,7 +337,7 @@ void Dictionary::computeSubwords2(const std::string& word, /*int32_t h = hash(ngram) % args_->bucket; ngrams.push_back(nwords_ + h);*/ size_t id = 0; - if (datrie_search_exact_match(ngramdat, ngram.c_str(), &id) != -1) + if (lookupNgram(ngram, id)) { ngrams.push_back(nwords_ + (int32_t)id); substrings.push_back(ngram); @@ -359,7 +361,7 @@ void Dictionary::computeSubwords2(const std::string& word, /*int32_t h = hash(ngram) % args_->bucket; pushHash(ngrams, h);*/ size_t id = 0; - if (datrie_search_exact_match(ngramdat, ngram.c_str(), &id) != -1) + if (lookupNgram(ngram, id)) { ngrams.push_back(nwords_ + (int32_t)id); } @@ -369,6 +371,26 @@ void Dictionary::computeSubwords2(const std::string& word, } } +bool Dictionary::lookupNgram(const std::string& key, size_t& id) const { + if (native_ngram_lookup_) { + int32_t original = native_ngram_lookup_(key); + if (original < nwords_) return false; + id = static_cast(original - nwords_); + return true; + } + return datrie_search_exact_match(ngramdat, key.c_str(), &id) != -1; +} + +int32_t Dictionary::getNgramId(const std::string& key) const { + size_t id; + return lookupNgram(key, id) ? nwords_ + static_cast(id) : -1; +} + +void Dictionary::setNativeNgramLookup(std::function lookup) { + native_ngram_lookup_ = lookup; + initNgrams(); +} + void Dictionary::initNgrams() { ncount = 0; for (size_t i = 0; i < size_; i++) { @@ -385,7 +407,7 @@ void Dictionary::initNgrams() { computeSubwords(word, words_[i].subwords); } } - nngrams_ = (int32_t)datrie_get_key_num(ngramdat); + if (!native_ngram_lookup_) nngrams_ = (int32_t)datrie_get_key_num(ngramdat); } bool Dictionary::readWord(std::istream& in, std::string& word) const @@ -416,6 +438,7 @@ bool Dictionary::readWord(std::istream& in, std::string& word) const } void Dictionary::readFromFile(std::istream& in) { + frozen_ngrams_ = false; std::string word; int64_t minThreshold = 1; @@ -719,9 +742,13 @@ std::string Dictionary::getLabel(int32_t lid) const { return words_[lid + nwords_].word; } -void Dictionary::save(std::ostream& out) const { - datrie_store_to_file(worddat, "worddat.bin"); - datrie_store_to_file(ngramdat, "ngramdat.bin"); +void Dictionary::save(std::ostream& out) const { save(out, true); } + +void Dictionary::save(std::ostream& out, bool auxiliary_files) const { + if (auxiliary_files) { + datrie_store_to_file(worddat, "worddat.bin"); + datrie_store_to_file(ngramdat, "ngramdat.bin"); + } out.write((char*) &size_, sizeof(int32_t)); out.write((char*) &nwords_, sizeof(int32_t)); out.write((char*) &nngrams_, sizeof(int32_t)); @@ -741,7 +768,13 @@ void Dictionary::save(std::ostream& out) const { } } -void Dictionary::load(std::istream& in) { +void Dictionary::load(std::istream& in) { load(in, true); } + +void Dictionary::load(std::istream& in, bool initialize_ngrams) { + frozen_ngrams_ = false; + native_ngram_lookup_ = nullptr; + datrie_delete(worddat); + datrie_delete(ngramdat); //worddat = datrie_load_from_file("worddat.bin"); //ngramdat = datrie_load_from_file("ngramdat.bin"); worddat = datrie_new(); @@ -754,12 +787,17 @@ void Dictionary::load(std::istream& in) { in.read((char*) &nlabels_, sizeof(int32_t)); in.read((char*) &ntokens_, sizeof(int64_t)); in.read((char*) &pruneidx_size_, sizeof(int64_t)); + if (!in || size_ < 0 || size_ > MAX_VOCAB_SIZE || nwords_ < 0 || nlabels_ < 0 || + int64_t(nwords_) + nlabels_ != size_ || nngrams_ < 0 || ntokens_ < 0 || + pruneidx_size_ < -1 || pruneidx_size_ > MAX_VOCAB_SIZE) + throw std::invalid_argument("invalid dictionary dimensions"); for (int32_t i = 0; i < size_; i++) { char c; entry e; - while ((c = in.get()) != 0) { + while (in.get(c) && c != 0) { e.word.push_back(c); } + if (!in) throw std::invalid_argument("truncated dictionary entry"); in.read((char*) &e.count, sizeof(int64_t)); in.read((char*) &e.type, sizeof(entry_type)); words_.push_back(e); @@ -775,8 +813,10 @@ void Dictionary::load(std::istream& in) { in.read((char*) &second, sizeof(int32_t)); pruneidx_[first] = second; } + if (!in) throw std::invalid_argument("truncated dictionary"); initTableDiscard(); - initNgrams(); + if (initialize_ngrams) initNgrams(); + frozen_ngrams_ = true; } void Dictionary::prune(std::vector& idx) { diff --git a/src/dictionary.h b/src/dictionary.h index 1099738..89254fd 100644 --- a/src/dictionary.h +++ b/src/dictionary.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "args.h" #include "real.h" @@ -55,6 +56,9 @@ class Dictionary { int32_t nwords_; int32_t nngrams_; int32_t nlabels_; + std::function native_ngram_lookup_; + bool frozen_ngrams_ = false; + bool lookupNgram(const std::string&, size_t&) const; int64_t ntokens_;//所有word label的出现总次数 int64_t pruneidx_size_; @@ -84,6 +88,7 @@ class Dictionary { int64_t ntokens() const; int32_t getId(const std::string&) const;//words_下标，word id int32_t getId(const std::string&, uint32_t h) const; + int32_t getNgramId(const std::string&) const; entry_type getType(int32_t) const; entry_type getType(const std::string&) const; bool discard(int32_t, real) const; @@ -110,7 +115,10 @@ class Dictionary { void readFromFile(std::istream&); std::string getLabel(int32_t) const; void save(std::ostream&) const; + void save(std::ostream&, bool auxiliary_files) const; void load(std::istream&); + void load(std::istream&, bool initialize_ngrams); + void setNativeNgramLookup(std::function lookup); std::vector getCounts(entry_type) const; int32_t getLine(std::istream&, std::vector&, std::vector&, std::minstd_rand&) const; diff --git a/src/fasttext.cc b/src/fasttext.cc index 5861bd1..3f9f6e4 100644 --- a/src/fasttext.cc +++ b/src/fasttext.cc @@ -24,6 +24,9 @@ #include #include #include +#ifdef MEFT_NATIVE_FASTTEXT +#include +#endif namespace fasttext { @@ -35,6 +38,10 @@ constexpr int32_t FASTTEXT_FILEFORMAT_MAGIC_INT32 = 793712314; FastText::FastText() : quant_(false) {} void FastText::addInputVector(Vector& vec, int32_t ind) const { + if (input_backend_) { + input_backend_->addRows(vec, std::vector{ind}); + return; + } if (quant_) { vec.addRow(*qinput_, ind); } else { @@ -42,6 +49,11 @@ void FastText::addInputVector(Vector& vec, int32_t ind) const { } } +void FastText::addInputVectors(Vector& vec, const std::vector& ids) const { + if (input_backend_) input_backend_->addRows(vec, ids); + else for (int32_t id : ids) addInputVector(vec, id); +} + std::shared_ptr FastText::getDictionary() const { return dict_; } @@ -51,6 +63,7 @@ const Args FastText::getArgs() const { } std::shared_ptr FastText::getInputMatrix() const { + if (input_backend_) throw std::runtime_error("native input is not a resident Matrix"); return input_; } @@ -63,8 +76,7 @@ int32_t FastText::getWordId(const std::string& word) const { } int32_t FastText::getSubwordId(const std::string& word) const { - int32_t h = dict_->hash(word) % args_->bucket; - return dict_->nwords() + h; + return dict_->getNgramId(word); } void FastText::getVectorNorm(Vector& vec, const std::string& word) { @@ -101,11 +113,14 @@ void FastText::sentenceVectors(const std::string& sentence, std::string& res) { int32_t count = 0; while(iss >> word) { getWordVector(vec, word); - vec.mul(1.0 / vec.norm()); - svec.addVector(vec); - count++; + real norm = vec.norm(); + if (norm > 0) { + vec.mul(1.0 / norm); + svec.addVector(vec); + count++; + } } - svec.mul(1.0 / count); + if (count > 0) svec.mul(1.0 / count); std::ostringstream oss; oss << svec; res = oss.str(); @@ -119,7 +134,7 @@ void FastText::ngramVectors(std::string word, std::vector& sub_vec) for (int32_t i = 0; i < ngrams.size(); i++) { vec.zero(); if (ngrams[i] >= 0) { - vec.addRow(*input_, ngrams[i]); + addInputVector(vec, ngrams[i]); } std::ostringstream oss; oss << vec; @@ -144,7 +159,7 @@ void FastText::ngramVectors(std::string word, std::string& res, std::string& sub } } if(ngrams.size() > 0 && max_pos < ngrams.size() && ngrams[max_pos] >= 0) { - vec.addRow(*input_, ngrams[max_pos]); + addInputVector(vec, ngrams[max_pos]); sub = substrings[max_pos]; std::ostringstream oss; oss << vec; @@ -180,9 +195,7 @@ void FastText::getWordVector(Vector& vec, const std::string& word) const { end = pub_time_clock(); std::cout << "getsubwords time: " << end - start << std::endl;*/ vec.zero(); - for (int i = 0; i < ngrams.size(); i ++) { - addInputVector(vec, ngrams[i]); - } + addInputVectors(vec, ngrams); if (ngrams.size() > 0) { vec.mul(1.0 / ngrams.size()); } @@ -200,9 +213,7 @@ void FastText::getInputVector(Vector& vec, const std::string& word) const { } else { vec.zero(); - for (int i = 0; i < ngrams.size(); i ++) { - addInputVector(vec, ngrams[i]); - } + addInputVectors(vec, ngrams); if (ngrams.size() > 0) { vec.mul(1.0 / ngrams.size()); } @@ -216,9 +227,8 @@ void FastText::getVector(Vector& vec, const std::string& word) const { void FastText::getSubwordVector(Vector& vec, const std::string& subword) const { vec.zero(); - int32_t h = dict_->hash(subword) % args_->bucket; - h = h + dict_->nwords(); - addInputVector(vec, h); + int32_t id = getSubwordId(subword); + if (id >= 0) addInputVector(vec, id); } void FastText::saveVectors() { @@ -301,6 +311,7 @@ int datrie_foreach_id(const char* key, size_t value, void* udata) { void FastText::saveIndex() { + if (input_backend_) throw std::runtime_error("operation requires the dense source input matrix"); std::string path(args_->output + ".z"); std::ofstream ofs(path, std::ofstream::binary); if (!ofs.is_open()) { @@ -424,6 +435,7 @@ void FastText::saveIndex() { void FastText::saveIndex(std::string path) { + if (input_backend_) throw std::runtime_error("operation requires the dense source input matrix"); //std::string path(args_->output + ".z"); std::ofstream ofs(path, std::ofstream::binary); if (!ofs.is_open()) { @@ -586,6 +598,7 @@ void FastText::saveModel() { } void FastText::saveModel(const std::string path) { + if (input_backend_) throw std::runtime_error("native bundles are immutable; save the original dense model before export"); std::ofstream ofs(path, std::ofstream::binary); if (!ofs.is_open()) { throw std::invalid_argument(path + " cannot be opened for saving!"); @@ -612,6 +625,13 @@ void FastText::saveModel(const std::string path) { } void FastText::loadModel(const std::string& filename) { +#ifdef MEFT_NATIVE_FASTTEXT + struct stat info; + if (stat(filename.c_str(), &info) == 0 && S_ISDIR(info.st_mode)) { + loadNativeBundle(filename); + return; + } +#endif std::ifstream ifs(filename, std::ifstream::binary); if (!ifs.is_open()) { throw std::invalid_argument(filename + " cannot be opened for loading!"); @@ -624,6 +644,8 @@ void FastText::loadModel(const std::string& filename) { } void FastText::loadModel(std::istream& in) { + input_backend_.reset(); + quant_ = false; args_ = std::make_shared(); dict_ = std::make_shared(args_); input_ = std::make_shared(); @@ -664,11 +686,11 @@ void FastText::loadModel(std::istream& in) { model_->quant_ = quant_; model_->setQuantizePointer(qinput_, qoutput_, args_->qout); - //if (args_->model == model_name::sup) { - // model_->setTargetCounts(dict_->getCounts(entry_type::label)); - //} else { - // model_->setTargetCounts(dict_->getCounts(entry_type::word)); - //} + // HS inference needs its output tree. NS inference does not need a training + // negative-sampling table; both dense and native loaders omit that table. + if (args_->loss == loss_name::hs) + model_->setTargetCounts(dict_->getCounts(args_->model == model_name::sup + ? entry_type::label : entry_type::word)); } void FastText::printInfo(real progress, real loss) { @@ -697,6 +719,7 @@ void FastText::printInfo(real progress, real loss) { } std::vector FastText::selectEmbeddings(int32_t cutoff) const { + if (input_backend_) throw std::runtime_error("operation requires the dense source input matrix"); Vector norms(input_->m_); input_->l2NormRow(norms); std::vector idx(input_->m_, 0); @@ -711,6 +734,7 @@ std::vector FastText::selectEmbeddings(int32_t cutoff) const { } void FastText::quantize(std::shared_ptr qargs) { + if (input_backend_) throw std::runtime_error("native bundles are immutable; quantize the dense source model"); if (args_->model != model_name::sup) { throw std::invalid_argument( "For now we only support quantization of supervised models"); @@ -889,9 +913,7 @@ void FastText::getSentenceVector( if (args_->model == model_name::sup) { std::vector line, labels; dict_->getLine(in, line, labels, model_->rng); - for (int32_t i = 0; i < line.size(); i++) { - addInputVector(svec, line[i]); - } + addInputVectors(svec, line); if (!line.empty()) { svec.mul(1.0 / line.size()); } @@ -928,7 +950,7 @@ void FastText::ngramVectors(std::string word) { if (quant_) { vec.addRow(*qinput_, ngrams[i]); } else { - vec.addRow(*input_, ngrams[i]); + addInputVector(vec, ngrams[i]); } } std::cout << ngrams[i] << " " << substrings[i] << " " << vec << std::endl; @@ -1100,6 +1122,10 @@ void FastText::loadVectors(std::string filename) { } void FastText::train(std::shared_ptr args) { + input_backend_.reset(); + quant_ = false; + qinput_.reset(); + qoutput_.reset(); args_ = args; dict_ = std::make_shared(args_); if (args_->input == "-") { diff --git a/src/fasttext.h b/src/fasttext.h index 0e55f1f..344d45a 100644 --- a/src/fasttext.h +++ b/src/fasttext.h @@ -20,6 +20,7 @@ #include "matrix.h" #include "model.h" #include "qmatrix.h" +#include "row_backend.h" #include "real.h" #include "utils.h" #include "vector.h" @@ -38,6 +39,7 @@ class FastText { std::shared_ptr qoutput_; std::shared_ptr model_; + std::shared_ptr input_backend_; std::atomic tokenCount_; clock_t start; @@ -68,6 +70,11 @@ class FastText { void getInputVector(Vector&, const std::string&) const; void getSubwordVector(Vector&, const std::string&) const; void addInputVector(Vector&, int32_t) const; + void addInputVectors(Vector&, const std::vector&) const; + bool hasNativeInput() const { return bool(input_backend_); } + // Available in the optional fasttext-native build; old model APIs stay intact. + void dumpNative(const std::string&) const; + void loadNativeBundle(const std::string&); inline void getInputVector(Vector& vec, int32_t ind) { vec.zero(); addInputVector(vec, ind); diff --git a/src/main.cc b/src/main.cc index ff5246d..5c8f226 100644 --- a/src/main.cc +++ b/src/main.cc @@ -31,6 +31,9 @@ void printUsage() { << " print-ngrams print ngrams given a trained model and word\n" << " nn query for nearest neighbors\n" << " analogies query for analogies\n" +#ifdef MEFT_NATIVE_FASTTEXT + << " dump-native extract a dense me_fasttext model for native bundle export\n" +#endif << std::endl; } @@ -272,6 +275,13 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); } std::string command(args[1]); +#ifdef MEFT_NATIVE_FASTTEXT + if (command == "dump-native") { + if (args.size()!=4) throw std::invalid_argument("usage: fasttext-native dump-native "); + FastText model; model.loadModel(args[2]); model.dumpNative(args[3]); + return 0; + } +#endif if (command == "skipgram" || command == "cbow" || command == "supervised") { train(args); } else if (command == "test") { diff --git a/src/model.cc b/src/model.cc index dad1fa7..da5dec0 100644 --- a/src/model.cc +++ b/src/model.cc @@ -127,6 +127,11 @@ real Model::softmax(int32_t target, real lr) { void Model::computeHidden(const std::vector& input, Vector& hidden) const { assert(hidden.size() == hsz_); hidden.zero(); + if (input_backend_) { + input_backend_->addRows(hidden, input); + hidden.mul(1.0 / input.size()); + return; + } for (auto it = input.cbegin(); it != input.cend(); ++it) { if(quant_) { hidden.addRow(*qwi_, *it); @@ -212,6 +217,7 @@ void Model::dfs(int32_t k, int32_t node, real score, } void Model::update(const std::vector& input, int32_t target, real lr) { + if (input_backend_) throw std::runtime_error("native rows are immutable; load a dense model for training"); assert(target >= 0); assert(target < osz_); if (input.size() == 0) return; diff --git a/src/model.h b/src/model.h index 4422754..b45b488 100644 --- a/src/model.h +++ b/src/model.h @@ -19,6 +19,7 @@ #include "vector.h" #include "qmatrix.h" #include "real.h" +#include "row_backend.h" namespace fasttext { @@ -37,6 +38,7 @@ class Model { std::shared_ptr qwi_; std::shared_ptr qwo_; std::shared_ptr args_; + std::shared_ptr input_backend_; Vector hidden_; Vector output_; Vector grad_; @@ -84,6 +86,7 @@ class Model { Vector&, Vector&) const; void update(const std::vector&, int32_t, real); void computeHidden(const std::vector&, Vector&) const; + void setInputBackend(std::shared_ptr backend) { input_backend_ = backend; } void computeOutputSoftmax(Vector&, Vector&) const; void computeOutputSoftmax(); diff --git a/src/row_backend.h b/src/row_backend.h new file mode 100644 index 0000000..953c8a7 --- /dev/null +++ b/src/row_backend.h @@ -0,0 +1,17 @@ +#pragma once +#include +#include +#include "vector.h" + +namespace fasttext { +/* Immutable serving rows. Implementations must preserve request order, + * multiplicity and the legacy float32 accumulation, including chunk boundaries. + * Single-worker handles; training continues to use Matrix/QMatrix. */ +class RowBackend { + public: + virtual ~RowBackend() {} + virtual void addRows(Vector&, const std::vector&) const = 0; + virtual int64_t rows() const = 0; + virtual int64_t width() const = 0; +}; +} diff --git a/tests/fasttext_native_fixture.cc b/tests/fasttext_native_fixture.cc new file mode 100644 index 0000000..0ab6449 --- /dev/null +++ b/tests/fasttext_native_fixture.cc @@ -0,0 +1,88 @@ +// Deterministic correctness fixture; not compression or model-quality evidence. +#include "fasttext.h" +#include +#include +#include +#include +#include + +using namespace fasttext; +class FixtureDictionary : public Dictionary { + public: + explicit FixtureDictionary(std::shared_ptr args):Dictionary(args) { + worddat=datrie_new(); ngramdat=datrie_new(); + for (const std::string& word:{"alpha","alphabet","beta","你好","","__label__left","__label__right"}) add(word); + threshold(1,1); initTableDiscard(); initNgrams(); + } +}; +class Fixture : public FastText { + public: + explicit Fixture(int dim, bool hs=false) { + args_=std::make_shared(); args_->dim=dim; args_->minn=2; args_->maxn=4; + args_->model=model_name::sup; args_->loss=hs ? loss_name::hs : loss_name::softmax; + dict_=std::make_shared(args_); + input_=std::make_shared(dict_->nwords()+dict_->nngrams(),dim); + output_=std::make_shared(dict_->nlabels(),dim); + for (int64_t i=0;im_;++i) + for(int j=0;jat(i,j)=float((i*17+j*31)%113-56)/13.f; + for (int64_t i=0;im_;++i) + for(int j=0;jat(i,j)=float((i*7+j*3)%17-8)/19.f; + model_=std::make_shared(input_,output_,args_,0); model_->quant_=false; + model_->setTargetCounts(dict_->getCounts(entry_type::label)); + for (int i=0;i<5;++i) model_->update(dict_->getSubwords("alpha"),i%2,.01f); + } +}; +void same(const Vector& a,const Vector& b) { + assert(a.size()==b.size()); assert(std::memcmp(a.data_,b.data_,a.size()*4)==0); +} +int main(int argc,char** argv) { + assert(argc>=3); + if(std::string(argv[1])=="create") { + Fixture model(std::stoi(argv[3]),argc>4 && std::string(argv[4])=="hs"); model.saveModel(argv[2]); return 0; + } + assert(argc==4); + FastText dense,native; dense.loadModel(argv[2]); native.loadModel(argv[3]); + assert(!dense.hasNativeInput() && native.hasNativeInput()); + int dim=dense.getDimension(); Vector a(dim),b(dim); + auto matrix=dense.getInputMatrix(); + for(int64_t id=0;idm_;++id) { + dense.getInputVector(a,id); native.getInputVector(b,id); same(a,b); + } + for(const std::string& word:{"alpha","alphabet","beta","你好","alphax","","☃unknown",""}) { + assert(dense.getWordId(word)==native.getWordId(word)); + assert(dense.getDictionary()->getSubwords(word)==native.getDictionary()->getSubwords(word)); + dense.getWordVector(a,word); native.getWordVector(b,word); same(a,b); + dense.getInputVector(a,word); native.getInputVector(b,word); same(a,b); + } + for(const std::string& key:{"al","pha","getNgramId(key)); + assert(dense.getSubwordId(key)==native.getSubwordId(key)); + dense.getSubwordVector(a,key); native.getSubwordVector(b,key); same(a,b); + } + for(const std::string& sentence:{"alpha beta alpha","你好 alpha","","☃unknown"}) { + std::istringstream left(sentence),right(sentence); + dense.getSentenceVector(left,a); native.getSentenceVector(right,b); same(a,b); + std::string x,y; dense.sentenceVectors(sentence,x); native.sentenceVectors(sentence,y); + assert(x==y && x.find("nan")==std::string::npos); + dense.wordVectors(sentence,x); native.wordVectors(sentence,y); assert(x==y); + } + std::vector ids; + for(int i=0;i<517;++i) ids.push_back((i*7)%matrix->m_); + a.zero(); b.zero(); dense.addInputVectors(a,ids); native.addInputVectors(b,ids); same(a,b); + std::istringstream left("alpha beta"),right("alpha beta"); + std::vector> p,q; + dense.predict(left,2,p); native.predict(right,2,q); assert(p==q && !p.empty()); + bool rejected=false; + try {native.getInputMatrix();} catch(const std::exception&) {rejected=true;} + assert(rejected); + rejected=false; + try {native.saveModel(std::string(argv[3])+"/forbidden.bin");} catch(const std::exception&) {rejected=true;} + assert(rejected); + rejected=false; + try {native.loadNativeBundle(std::string(argv[3])+"/missing");} catch(const std::exception&) {rejected=true;} + assert(rejected && native.hasNativeInput()); + native.getWordVector(b,"alpha"); dense.getWordVector(a,"alpha"); same(a,b); + native.loadModel(argv[2]); assert(!native.hasNativeInput()); + native.getWordVector(b,"alpha"); same(a,b); + std::cout<<"All original rows, IDs, OOV/UTF-8, ordered/chunked aggregation, sentence and label probabilities exact.\n"; +} diff --git a/tests/test_fasttext_native_bundle.py b/tests/test_fasttext_native_bundle.py new file mode 100644 index 0000000..b6fa1b0 --- /dev/null +++ b/tests/test_fasttext_native_bundle.py @@ -0,0 +1,84 @@ +"""Actual legacy Linux loader/DASTrie/native facade; fixtures test contracts only.""" +import json +import os +from pathlib import Path +import platform +import shutil +import struct +import subprocess +import tempfile +import unittest + +from tools.fasttext_native_bundle import export_fasttext_model, read_keys + +ROOT=Path(__file__).resolve().parents[1] + + +class RegistryTest(unittest.TestCase): + def test_rejects_truncated_duplicate_and_out_of_range_registry(self): + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'keys' + valid=b'MEFTKEY1'+struct.pack(' lossless native serving. + +Needs an existing enwik8.zip; does not download data or use GPUs. This is a custom +positional/source SGNS storage/serving reference, not upstream fastText training +or evidence of downstream task quality. All original input rows are retained. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import random +import re +import shlex +import subprocess +import sys +import time +import zipfile + +ROOT=Path(__file__).resolve().parents[1] +SOURCES=['args','dictionary','productquantizer','matrix','qmatrix','vector','model','utils','fasttext','ftindex'] + + +def digest(path): + h=hashlib.sha256() + with Path(path).open('rb') as stream: + for part in iter(lambda:stream.read(1024**2),b''):h.update(part) + return h.hexdigest() + + +def tokens(raw): + # This finite reference ignores XML tags and retains lowercase ASCII words. + return re.findall(r'[a-z]+',re.sub(r'<[^>]*>',' ',raw.decode('utf-8',errors='replace').lower())) + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--enwik8-zip',type=Path,required=True) + parser.add_argument('--output-dir',type=Path,required=True) + parser.add_argument('--train-words',type=int,default=65536) + parser.add_argument('--repeats',type=int,default=3) + args=parser.parse_args() + if platform.system()!='Linux' or platform.machine()!='x86_64':parser.error('bundled libraries require Linux x86-64') + if args.train_words<64 or args.repeats<1:parser.error('positive repeat and at least 64 training words required') + out=args.output_dir.resolve(); out.mkdir(parents=True,exist_ok=False) + env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1') + report={'status':'running','category':'public-text, actually trained custom me_fasttext SGNS storage/serving reference', + 'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(), + 'exact_command':shlex.join([sys.executable,*sys.argv]),'python':sys.version,'platform':platform.platform(), + 'compiler':subprocess.check_output(['c++','--version'],text=True).splitlines()[0], + 'cpu':next((s.split(':',1)[1].strip() for s in Path('/proc/cpuinfo').read_text().splitlines() if s.startswith('model name')),''), + 'cpu_affinity':sorted(os.sched_getaffinity(0)),'device':'CPU','pytorch':None,'commands':[], + 'limitations':['Finite 64-dim public-text reference; not an Engram checkpoint or downstream quality result.', + 'Filesystem prewarmed; first pass is reader-cache cold, not disk cold.', + 'Single worker on a shared server; no GPU or other user process touched.', + 'Dictionary (including legacy 200 MB word2int allocation) and output matrix remain resident.', + 'No inference negative-sampling table in either loader; training builds one.', + '4 MiB cache payload budget is not an RSS bound; native mmap pages and index remain resident.', + 'Conversion is offline and copies full input/maps; no bounded exporter memory claim.']} + + def run(command,name): + command=list(map(str,command)); report['commands'].append(shlex.join(command)) + begin=time.perf_counter() + with (out/name).open('w') as log: + result=subprocess.run(command,cwd=out,env=env,stdout=log,stderr=subprocess.STDOUT) + result.check_returncode(); return time.perf_counter()-begin + + try: + with zipfile.ZipFile(args.enwik8_zip) as archive:raw=archive.read('enwik8') + split=len(raw)*9//10 + train=tokens(raw[:split])[:args.train_words]; held=tokens(raw[split:])[:8192] + if len(train)!=args.train_words or len(held)<2048:raise ValueError('insufficient public text') + corpus=out/'pairs.txt' + with corpus.open('w') as stream: + for begin in range(0,len(train)-63,64): + block=train[begin:begin+64] + stream.write('p#0 '+' '.join(a+'\x01'+b for a,b in zip(block,block[1:]))+'\n') + rng=random.Random(42); query=out/'queries.tsv' + words=[held[rng.randrange(len(held))] for _ in range(1024)] + sentences=[' '.join(held[i:i+8]) for i in [rng.randrange(len(held)-8) for _ in range(128)]] + query.write_text(''.join('w\t'+w+'\n' for w in words)+''.join('s\t'+s+'\n' for s in sentences)) + report['data']={'source':'enwik8 (existing archive, no download)','zip_sha256':digest(args.enwik8_zip), + 'raw_sha256':hashlib.sha256(raw).hexdigest(),'raw_bytes':len(raw),'raw_split_byte':split, + 'transform':'Split raw 90/10 first; strip XML tags, lowercase, [a-z]+; use first requested train words. Each 64-word block emits p#0 and 63 adjacent query\\x01source pairs.', + 'train_words':len(train),'training_pair_tokens':(len(train)//64)*63,'training_lines':len(train)//64, + 'corpus_sha256':digest(corpus),'query_sha256':digest(query),'query_seed':42, + 'heldout_source_words':len(held),'word_queries':1024,'sentence_queries':128,'sentence_length':8} + del raw,train,held + run(['make','-C',ROOT,'fasttext-native','native','structural-native'],'build-native.txt') + executable=out/'probe' + run(['c++','-std=c++11','-O2','-pthread','-DMEFT_NATIVE_FASTTEXT','-I'+str(ROOT/'src'), + '-I'+str(ROOT/'native'),'-I'+str(ROOT/'mylib/include'),ROOT/'tools/fasttext_serving_probe.cc', + *[ROOT/f'src/{n}.cc' for n in SOURCES],ROOT/'native/fasttext_bundle.cc',ROOT/'build/meft_rows_fasttext.o', + '-L'+str(ROOT/'mylib/lib'),'-ltrie','-lpub','-lz','-o',executable],'build-probe.txt') + model=out/'trained.bin' + run([executable,'train',corpus,model,out/'training.json'],'training.txt') + report['training']=json.loads((out/'training.json').read_text()) + report['source_model_sha256']=digest(model); report['source_model_bytes']=model.stat().st_size + report['exports']={} + for mode in ('layout','exact'): + bundle=out/mode + seconds=run([sys.executable,ROOT/'tools/export_fasttext_native_cli.py','--model',model,'--output-dir',bundle, + '--fasttext-binary',ROOT/'build/fasttext-native','--structural-library',ROOT/'build/libmeft_structural.so', + '--native-library',ROOT/'build/libmeft_rows.so','--mode',mode],'export-'+mode+'.txt') + manifest=json.loads((bundle/'fasttext.json').read_text()) + report['exports'][mode]={'seconds':seconds,'bundle_bytes':sum(p.stat().st_size for p in bundle.iterdir() if p.is_file()), + 'manifest':manifest} + report['serving']={} + oracle=None + for name in ('dense','layout','exact'): + path=model if name=='dense' else out/name + # Explicit warm filesystem baseline; no cache dropping on shared host. + for file in ([path] if path.is_file() else list(path.iterdir())): + if file.is_file():digest(file) + runs=[] + for repeat in range(args.repeats): + oracle_path=out/f'{name}-{repeat}.f32'; result_path=out/f'{name}-{repeat}.json' + run([executable,'serve',path,query,oracle_path,result_path],f'serve-{name}-{repeat}.txt') + checksum=digest(oracle_path) + if oracle is None:oracle=checksum + if checksum!=oracle:raise AssertionError('trained-model inference bytes differ') + result=json.loads(result_path.read_text()); result['oracle_sha256']=checksum; runs.append(result) + report['serving'][name]=runs + report['all_inference_outputs_byte_equal']=True + report['artifacts_sha256']={p.name:digest(p) for p in out.iterdir() if p.is_file() and p.suffix in ('.json','.tsv','.cc')} + report['status']='passed' + except BaseException as exc: + report['status']='failed'; report['failure']=repr(exc); raise + finally: + (out/'report.json').write_text(json.dumps(report,indent=2)+'\n') + print(json.dumps({'status':report['status'],'shape':[report['training']['rows'],64], + 'output_dir':str(out)},indent=2)) + + +if __name__=='__main__':main() diff --git a/tools/check_memory_wheel.py b/tools/check_memory_wheel.py index 6e2c3e7..ab99479 100644 --- a/tools/check_memory_wheel.py +++ b/tools/check_memory_wheel.py @@ -14,6 +14,10 @@ def main(): os.environ.pop("MEFT_ROWS_LIBRARY", None) tags = [line for line in distribution("meft-memory").read_text("WHEEL").splitlines() if line.startswith("Tag:")] assert tags and all(tag.endswith("_" + platform.machine()) for tag in tags), tags + assert callable(meft_memory.export_fasttext_model) + assert any(entry.name == "meft-fasttext-export" and + entry.value == "meft_memory._runtime.fasttext_native_bundle:main" + for entry in distribution("meft-memory").entry_points) with tempfile.TemporaryDirectory() as tmp: source, packed = Path(tmp) / "raw", Path(tmp) / "table" source.write_bytes(b"ABCDEFGHabcdefgh") diff --git a/tools/export_fasttext_native_cli.py b/tools/export_fasttext_native_cli.py new file mode 100644 index 0000000..566a0a5 --- /dev/null +++ b/tools/export_fasttext_native_cli.py @@ -0,0 +1,7 @@ +"""Direct checkout script for the same exporter as meft-fasttext-export.""" +import sys +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from tools.fasttext_native_bundle import main + +if __name__=='__main__':main() diff --git a/tools/fasttext_native_bundle.py b/tools/fasttext_native_bundle.py new file mode 100644 index 0000000..db9359a --- /dev/null +++ b/tools/fasttext_native_bundle.py @@ -0,0 +1,145 @@ +"""Export a dense me_fasttext .bin into a frozen native FastText serving bundle. + +Uses this repository's Linux C++ loader and original DATrie/mark-compact bridge. +Official fastText .bin layouts, quantized models and legacy .z are separate. +Build-time rows/maps may be table-sized; reader caches do not bound export RSS. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import tempfile + +from .native_row_store import NativeRows +from .structural_native import StructuralPlan + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for part in iter(lambda: stream.read(1024**2), b''): + digest.update(part) + return digest.hexdigest() + + +def read_keys(path, rows): + path = Path(path) + keys = {} + with Path(path).open('rb') as stream: + if stream.read(8) != b'MEFTKEY1': + raise ValueError('invalid extracted key registry') + count_bytes = stream.read(8) + if len(count_bytes) != 8: + raise ValueError('truncated key registry') + count, = struct.unpack(' rows: + raise ValueError('key count exceeds original rows') + for _ in range(count): + header = stream.read(16) + if len(header) != 16: + raise ValueError('truncated key record') + row_id, size = struct.unpack('= rows or row_id in keys or size == 0 or size > path.stat().st_size: + raise ValueError('invalid key record') + raw = stream.read(size) + if len(raw) != size or b'\0' in raw: + raise ValueError('truncated or NUL-containing key') + keys[row_id] = raw.decode('utf-8') + if stream.read(1) or len(set(keys.values())) != count: + raise ValueError('trailing bytes or duplicate keys') + return keys + + +def export_fasttext_model(model, output_dir, *, fasttext_binary, structural_library, + native_library=None, mode='exact', cosine=.99, norm_ratio=.99, + max_block_bytes=8192, cache_bytes=4*1024**2, query_chunk_rows=256): + """Publish a new owned bundle; source model and legacy file paths stay intact.""" + model, output = Path(model).resolve(), Path(output_dir).resolve() + if output.exists(): + raise FileExistsError('use a fresh native bundle directory') + if mode not in ('layout', 'exact', 'joint'): + raise ValueError('mode must be layout, exact or joint') + if type(cache_bytes) is not int or cache_bytes < 0 or type(query_chunk_rows) is not int or query_chunk_rows <= 0: + raise ValueError('cache budget must be nonnegative and query chunk rows positive') + output.parent.mkdir(parents=True, exist_ok=True) + original_hash = sha256(model) + with tempfile.TemporaryDirectory(prefix='.meft-fasttext-', dir=output.parent) as tmp: + extracted = Path(tmp)/'extracted' + command = [str(Path(fasttext_binary).resolve()), 'dump-native', str(model), str(extracted)] + run = subprocess.run(command, text=True, capture_output=True) + if run.returncode: + raise RuntimeError('dense model extraction failed:\n'+run.stdout+run.stderr) + metadata = extracted/'model.meta' + with metadata.open('rb') as stream: + if stream.read(8) != b'MEFTFT01': + raise ValueError('unsupported native FastText metadata') + rows, dim = struct.unpack('word or ssentence. Output oracle is raw float32 bytes. +#include "fasttext.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace fasttext; +using Clock=std::chrono::steady_clock; +double seconds(Clock::time_point start) { + return std::chrono::duration(Clock::now()-start).count(); +} +void memory(std::ostream& out) { + std::ifstream in("/proc/self/status"); std::string line; + out<<"{"; bool first=true; + while(std::getline(in,line)) { + auto colon=line.find(':'); std::string key=line.substr(0,colon); + if(key!="VmRSS" && key!="RssAnon" && key!="RssFile") continue; + if(!first) out<<","; first=false; + std::istringstream value(line.substr(colon+1)); long kb; value>>kb; + out<<"\""< values) { + std::sort(values.begin(),values.end()); + if(values.empty()) throw std::runtime_error("empty timing distribution"); + out<<"{\"count\":"<(); args->input=argv[2]; args->output=std::string(argv[3])+".unused"; + args->model=model_name::sg; args->loss=loss_name::ns; args->dim=64; + args->minn=2; args->maxn=12; args->minCount=2; args->ws=3; args->neg=5; + args->epoch=2; args->thread=1; args->t=1; args->lr=.025; args->verbose=0; + FastText model; auto start=Clock::now(); model.train(args); double elapsed=seconds(start); + auto input=model.getInputMatrix(); Matrix initial(input->m_,input->n_); initial.uniform(1.0/args->dim); + int64_t changed=0,changedNgrams=0; + for(int64_t i=0;im_;++i) { + bool different=std::memcmp(input->data()+i*input->n_,initial.data()+i*input->n_,input->n_*4)!=0; + changed+=different; changedNgrams+=different && i>=model.getDictionary()->nwords(); + } + model.saveModel(argv[3]); std::ofstream out(argv[4]); out<m_<<",\"dim\":"<n_ + <<",\"word_rows\":"<nwords()<<",\"ngram_rows\":"<nngrams() + <<",\"changed_rows\":"<> trace; std::string line; + while(std::getline(queries,line)) { + if(line.size()<3 || line[1]!='\t' || (line[0]!='w' && line[0]!='s')) throw std::runtime_error("invalid query"); + trace.emplace_back(line[0],line.substr(2)); + } + if(trace.empty()) throw std::runtime_error("empty query trace"); + auto start=Clock::now(); FastText model; model.loadModel(argv[2]); double load=seconds(start); + std::ofstream report(argv[5]); report< oracle; + // Keep allocation and oracle file I/O outside the timed query calls. + oracle.reserve(trace.size()*model.getDimension()); + for(int pass=0;pass<4;++pass) { + std::vector word,sentence; + word.reserve(trace.size()); sentence.reserve(trace.size()); + auto batch=Clock::now(); + for(const auto& query:trace) { + std::istringstream text(query.second); auto begin=Clock::now(); + if(query.first=='w') model.getWordVector(vector,query.second); + else model.getSentenceVector(text,vector); + double us=seconds(begin)*1e6; (query.first=='w'?word:sentence).push_back(us); + if(pass==0) oracle.insert(oracle.end(),vector.data_,vector.data_+vector.size()); + } + double batchTime=seconds(batch); + if(pass) report<<","; + report<<"{\"batch_seconds\":"<(oracle.data()),oracle.size()*4); + if(!output || !report) throw std::runtime_error("cannot write probe output"); +} diff --git a/tools/structural_native.py b/tools/structural_native.py index 4d77ceb..62478cc 100644 --- a/tools/structural_native.py +++ b/tools/structural_native.py @@ -180,7 +180,8 @@ def export(self, source, directory, *, block_rows=256, predictor="xor", index="d "artifact_sha256": hashlib.sha256((directory / "table.men").read_bytes()).hexdigest(), "input_bytes": self.nrows * self.dim * 4, "artifact_bytes": metadata["artifact_bytes"], "unique_rows": metadata["unique_rows"], "row_map_bytes": metadata["row_map_bytes"], - "block_rows": block_rows, "approximate": not self.original_bytes_preserved, + "block_rows": metadata["block_rows"], "max_block_bytes": metadata["max_block_bytes"], + "approximate": not self.original_bytes_preserved, "contract": metadata["contract"]} (directory / "embedding.json").write_text(json.dumps(embedding, indent=2) + "\n") return metadata