diff --git a/CHANGELOG.md b/CHANGELOG.md index 59d71b5c..8acb438b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,16 @@ drill-down, explicit versioned overlays, and migration guidance for direct `compass.viewer.callflow/1` consumers. +- Add bounded native PDF, DOCX, PPTX, and XLSX document artifacts and graph + projection, gap-free semantic slicing, pure-Rust PDF rasterization, optional + local PP-OCRv6 OCR, explicit pinned model management, and the + `document inspect` and `models` command families. OCR remains off by default + and requires no Python, Tesseract, office suite, or system PDF installation. + Harden document reads against files that grow while being read, validate + cached and OCR-derived artifact coherence, bound aggregate raster work and + per-tile deadlines, and serialize model installation while rejecting symlinked + model artifacts and markers. + - Hard-cut Swift, Dart, Scala, and Groovy/Gradle onto version-1 qualifying universal evidence pipelines. The bounded AST-first producer publishes declarations, scopes, bindings, occurrences, and conservative relationship diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 73d56f0b..792b2ee4 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -50,6 +50,27 @@ CI covers Linux, macOS, and Windows targets listed in performance checks are owned by Compass workflows and require no external product checkout. +## Document and OCR compatibility + +Native PDF, DOCX, PPTX, and XLSX processing is part of the local Rust product +boundary. It does not require Python, Tesseract, LibreOffice, Poppler, Java, a +runtime grammar download, or provider credentials. The stable artifact majors +introduced here are `compass.document/1`, `compass.document.inspect/1`, and +`compass.ocr/1`; unknown majors and normalizer versions fail explicitly. + +OCR is off by default. Enabling `auto` or `always` requires one exact verified +Compass-managed profile. Extraction never downloads models, and `models +verify` never accesses the network. Document and semantic caches are hard-cut +by source digest, schema, normalizer, rasterizer, OCR policy, preprocessing, +profile manifest/model digests, and languages. An incompatible cache entry is +a miss or explicit corruption error, never a fallback to flattened text. + +The selected OCR identity is included in graph build and immutable history +profiles. Native text remains authoritative; OCR is additive derived evidence +with exact source owner, geometry, confidence, and model provenance. Partial +visual coverage is never labeled complete or finalized as a complete document +cache entry. + ## Evolving contracts The Grounded Agent Graph feature is additive and opt-in. It does not change diff --git a/Cargo.lock b/Cargo.lock index 8e95c51d..3394324a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,22 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + [[package]] name = "adler2" version = "2.0.1" @@ -29,6 +45,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -42,6 +59,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -63,6 +98,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -81,12 +125,38 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "array-init" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -121,6 +191,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey 0.1.1", + "rayon", + "thiserror 2.0.20", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-config" version = "1.9.0" @@ -630,6 +743,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -688,12 +807,27 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -741,6 +875,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + [[package]] name = "bumpalo" version = "3.20.3" @@ -773,6 +913,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -804,7 +950,7 @@ dependencies = [ "num-traits", "num_cpus", "rand 0.9.5", - "rand_distr", + "rand_distr 0.5.1", "rayon", "safetensors", "thiserror 2.0.20", @@ -847,6 +993,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -921,12 +1076,36 @@ dependencies = [ "inout", ] +[[package]] +name = "clipper2-rust" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd663fe209e7030c956e3be4c051dcc20cdb73da794f31466762cff12ca11bf" +dependencies = [ + "num-traits", +] + [[package]] name = "cmov" version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.7" @@ -937,6 +1116,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "compass-agent-graph" version = "0.3.19" @@ -993,7 +1187,9 @@ dependencies = [ "compass-ingest", "compass-ir", "compass-mcp", + "compass-media", "compass-model", + "compass-ocr", "compass-output", "compass-postgres", "compass-pr-intelligence", @@ -1021,6 +1217,7 @@ dependencies = [ "toml 1.1.4+spec-1.1.0", "ureq", "url", + "zip 6.0.0", ] [[package]] @@ -1036,7 +1233,9 @@ dependencies = [ "compass-history", "compass-ir", "compass-languages", + "compass-media", "compass-model", + "compass-ocr", "compass-output", "compass-pr-intelligence", "compass-program", @@ -1057,6 +1256,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "time", + "zip 6.0.0", ] [[package]] @@ -1259,10 +1459,18 @@ dependencies = [ name = "compass-media" version = "0.3.19" dependencies = [ + "compass-files", + "compass-ocr", + "hayro", + "image", "oxidize-pdf", "roxmltree", + "serde", + "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 2.0.20", + "unicode-normalization", "zip 6.0.0", ] @@ -1280,6 +1488,20 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "compass-ocr" +version = "0.3.19" +dependencies = [ + "image", + "oar-ocr", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.20", + "ureq", +] + [[package]] name = "compass-output" version = "0.3.19" @@ -1551,12 +1773,24 @@ dependencies = [ "flate2", "rand 0.8.7", "realfft", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "tiktoken-rs", ] +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1743,6 +1977,90 @@ dependencies = [ "cmov", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "der" version = "0.7.10" @@ -1752,7 +2070,17 @@ dependencies = [ "const-oid 0.9.6", "der_derive", "flagset", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468 1.0.0", "zeroize", ] @@ -1784,6 +2112,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -1868,7 +2227,7 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", "elliptic-curve", "rfc6979", @@ -1894,7 +2253,7 @@ dependencies = [ "ff", "generic-array", "group", - "pem-rfc7468", + "pem-rfc7468 0.7.0", "pkcs8", "rand_core 0.6.4", "sec1", @@ -1902,6 +2261,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -1914,6 +2279,26 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1931,8 +2316,40 @@ dependencies = [ ] [[package]] -name = "extended" -version = "0.1.0" +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "exr" +version = "1.74.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711fe42c9964295e01ee3fba3f9fe0e1d24b98886950d68efe81b1c76e21adf3" +dependencies = [ + "bit_field", + "half 2.7.1", + "lebe", + "miniz_oxide", + "num-complex", + "pulp", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "extended" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" @@ -1982,6 +2399,27 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "ff" version = "0.13.1" @@ -2034,7 +2472,7 @@ dependencies = [ "half 2.7.1", "num-traits", "rand 0.9.5", - "rand_distr", + "rand_distr 0.5.1", ] [[package]] @@ -2049,6 +2487,30 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2313,9 +2775,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "glam" +version = "0.30.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19fc433e8437a212d1b6f1e68c7824af3aed907da60afa994e7f542d18d12aa9" + +[[package]] +name = "glam" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556f6b2ea90b8d15a74e0e7bb41671c9bdf38cd9f78c284d750b9ce58a2b5be7" + +[[package]] +name = "glam" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" + +[[package]] +name = "glam" +version = "0.33.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c426daf70099baff33fac0ba85151c42424861c56828cef9ba02dacb78eb6b26" + +[[package]] +name = "glifo" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99fc21d493812643aae86d53b7bbd02f376434a90317e8a790bc209fdd6605e" +dependencies = [ + "bytemuck", + "foldhash", + "hashbrown 0.17.1", + "log", + "peniko", + "png", + "skrifa", + "smallvec", + "vello_common 0.0.9", ] [[package]] @@ -2335,6 +2850,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + [[package]] name = "h2" version = "0.4.16" @@ -2371,7 +2895,7 @@ dependencies = [ "crunchy", "num-traits", "rand 0.9.5", - "rand_distr", + "rand_distr 0.5.1", "zerocopy", ] @@ -2402,6 +2926,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -2412,6 +2939,95 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hayro" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4caa128ab87fd48ffb7490617cf93f77f606820dffbc9fd9ef6ab0ed077f56d" +dependencies = [ + "bytemuck", + "hayro-interpret", + "image", + "kurbo", + "pic-scale", + "vello_cpu", +] + +[[package]] +name = "hayro-ccitt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f4d0e94ddd48749f06bbe4e5389fb9799a0c45bcaf00495042076ef05e3241a" + +[[package]] +name = "hayro-cmap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d285dc30731c8485de5fa732fbdf2b3affdf01e4da7c2135022ca6fa664bf6" +dependencies = [ + "hayro-postscript", +] + +[[package]] +name = "hayro-interpret" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2613d0406898995042d0794c4245b2f2fba1246490c8ac593769bba6551129d" +dependencies = [ + "bitflags", + "hayro-cmap", + "hayro-syntax", + "kurbo", + "moxcms", + "phf", + "rustc-hash 2.1.3", + "siphasher", + "skrifa", + "smallvec", + "yoke", +] + +[[package]] +name = "hayro-jbig2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69374b3668dd45aeb3d3145cda68f2c7b4f223aaa2511e67d076f1c7d741388d" +dependencies = [ + "fearless_simd", + "hayro-ccitt", +] + +[[package]] +name = "hayro-jpeg2000" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ab947623ef4ccaa7acf0579edf7cbb5a73838e3839a7be73335e522f433a1" +dependencies = [ + "fearless_simd", +] + +[[package]] +name = "hayro-postscript" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "885c5ef0654933139a9b9546fc2c69e18d37f38aa2520f079092b1be18f1fcaa" + +[[package]] +name = "hayro-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edeafd70aa2db743de8ede8637d07ec87db05efe69cde371d03f1b185fcef27" +dependencies = [ + "flate2", + "hayro-ccitt", + "hayro-jbig2", + "hayro-jpeg2000", + "memchr", + "rustc-hash 2.1.3", + "smallvec", + "zune-jpeg", +] + [[package]] name = "heck" version = "0.5.0" @@ -2448,6 +3064,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "http" version = "0.2.12" @@ -2692,6 +3314,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -2713,6 +3341,65 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imageproc" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b27bc0867dc40df08deb53d6e96342db6e0702e7ae33ed09a4eba33e594b05" +dependencies = [ + "ab_glyph", + "approx", + "getrandom 0.4.3", + "image", + "itertools 0.14.0", + "nalgebra", + "num", + "rand 0.10.2", + "rand_distr 0.6.0", + "rayon", + "rustdct", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + [[package]] name = "indexmap" version = "2.14.0" @@ -2723,6 +3410,19 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inotify" version = "0.11.5" @@ -2753,12 +3453,41 @@ dependencies = [ "generic-array", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ipnet" version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2880,18 +3609,46 @@ dependencies = [ "libc", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libm" version = "0.2.16" @@ -2927,6 +3684,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2966,12 +3729,63 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey 0.2.3", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "matchit" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "md-5" version = "0.10.6" @@ -3029,6 +3843,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3051,6 +3871,129 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "multiversion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "target-features", +] + +[[package]] +name = "nalgebra" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc43a60c217b0c6ff46e47f26911015ad8d2e5a8be1af668c67e370d99a4346" +dependencies = [ + "approx", + "glam 0.30.10", + "glam 0.31.1", + "glam 0.32.1", + "glam 0.33.5", + "matrixmultiply", + "nalgebra-macros", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973e7178a678cfd059ccec50887658d482ce16b0aa9da3888ddeab5cd5eb4889" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nix" version = "0.31.3" @@ -3063,6 +4006,40 @@ dependencies = [ "libc", ] +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "notify" version = "8.2.0" @@ -3090,6 +4067,29 @@ dependencies = [ "bitflags", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3106,6 +4106,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -3115,6 +4126,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3144,6 +4176,59 @@ dependencies = [ "libc", ] +[[package]] +name = "oar-ocr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ef9a9edb90f570b88c6ee00ac64f6e46efa7bd464459c06e23b00bb6612237e" +dependencies = [ + "image", + "imageproc", + "oar-ocr-core", + "oar-ocr-derive", + "rayon", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "oar-ocr-core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92185edb59d3a8f92b5ba5ab4991ecdc8a2beffa028534e9c25601aa7fd2860a" +dependencies = [ + "clipper2-rust", + "image", + "imageproc", + "itertools 0.15.0", + "multiversion", + "nalgebra", + "ndarray", + "oar-ocr-derive", + "ort", + "rayon", + "regex", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokenizers", + "tracing", + "wide", +] + +[[package]] +name = "oar-ocr-derive" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d868d12ee353e68e30d1edbf363f938a6f4e5195ddba43125acbe577971b3f" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "objc2" version = "0.6.4" @@ -3183,18 +4268,110 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "outref" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + [[package]] name = "oxidize-pdf" version = "4.5.1" @@ -3267,6 +4444,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + [[package]] name = "pastey" version = "0.2.3" @@ -3282,6 +4465,28 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3294,10 +4499,34 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ + "phf_macros", "phf_shared", "serde", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -3307,6 +4536,16 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pic-scale" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c80d88d5c31215ceec2862abb2504860c715b8e2545a5ab15b62a3d097f30d" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3325,7 +4564,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", + "der 0.7.10", "spki", ] @@ -3335,6 +4574,43 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postgres-protocol" version = "0.6.12" @@ -3402,18 +4678,37 @@ dependencies = [ name = "primeorder" version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" dependencies = [ - "elliptic-curve", + "profiling-procmacros", ] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "profiling-procmacros" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" dependencies = [ - "unicode-ident", + "quote", + "syn 2.0.119", ] [[package]] @@ -3486,6 +4781,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -3602,6 +4918,66 @@ dependencies = [ "rand 0.9.5", ] +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.2", +] + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.5", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror 2.0.20", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -3611,6 +4987,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -3621,6 +5003,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -3631,6 +5024,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-fonts" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" +dependencies = [ + "bytemuck", + "font-types", +] + [[package]] name = "realfft" version = "3.5.0" @@ -3729,6 +5132,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + [[package]] name = "ring" version = "0.17.14" @@ -3756,7 +5165,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "pastey", + "pastey 0.2.3", "pin-project-lite", "rand 0.10.2", "schemars", @@ -3832,6 +5241,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -3841,6 +5256,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rustdct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b61555105d6a9bf98797c063c362a1d24ed8ab0431655e38f1cf51e52089551" +dependencies = [ + "rustfft", +] + [[package]] name = "rustfft" version = "6.4.1" @@ -3988,6 +5412,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" +dependencies = [ + "bytemuck", +] + [[package]] name = "safetensors" version = "0.7.0" @@ -4065,7 +5498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der", + "der 0.7.10", "generic-array", "pkcs8", "subtle", @@ -4306,6 +5739,18 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simba" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a7200d82ff1c7b4235efd12f11e2537a402c91cf83a5cb97ce80eef787fde7" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "wide", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -4322,6 +5767,15 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -4334,6 +5788,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "skrifa" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" @@ -4374,7 +5838,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", ] [[package]] @@ -4396,6 +5872,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "streaming-iterator" version = "0.1.9" @@ -4692,6 +6174,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempfile" version = "3.27.0" @@ -4745,6 +6233,20 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half 2.7.1", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tiktoken-rs" version = "0.7.0" @@ -4757,7 +6259,7 @@ dependencies = [ "fancy-regex 0.13.0", "lazy_static", "regex", - "rustc-hash", + "rustc-hash 1.1.0", ] [[package]] @@ -4847,6 +6349,40 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "indicatif", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.20", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.53.1" @@ -5156,6 +6692,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "typed-path" version = "0.12.3" @@ -5195,12 +6737,45 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -5221,8 +6796,10 @@ checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" dependencies = [ "base64 0.23.1", "cookie_store", + "der 0.8.1", "flate2", "log", + "native-tls", "percent-encoding", "rustls", "rustls-pki-types", @@ -5232,6 +6809,7 @@ dependencies = [ "socks", "ureq-proto", "utf8-zero", + "webpki-root-certs", "webpki-roots", ] @@ -5288,12 +6866,70 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vello_common" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3361bff7f7d82c0c496b92048db83846691f0e844cc28dee92b1c824291b55ee" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown 0.17.1", + "log", + "peniko", + "png", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "vello_common" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d672facaa2d697285a786cd9d44d614cd2ce54cdc022504bf339f8fff3b750" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown 0.17.1", + "log", + "peniko", + "png", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "vello_cpu" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8ded630e8316bb94a55881256506d1f3b9947b5f66db8a7d32ca7ba02decd0" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown 0.17.1", + "png", + "vello_common 0.0.8", +] + [[package]] name = "version_check" version = "0.9.5" @@ -5422,6 +7058,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-root-certs" version = "1.0.9" @@ -5440,6 +7086,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whoami" version = "2.1.3" @@ -5453,6 +7105,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wide" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de2aaf408e58689c2096682331b1f42bb2d9f2ed6b11560407d023cd0a6c634e" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5805,7 +7467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der", + "der 0.7.10", "spki", "tls_codec", ] @@ -5832,6 +7494,12 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + [[package]] name = "yoke" version = "0.8.3" @@ -6026,3 +7694,27 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 632fd236..95ec4314 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/compass-media", "crates/compass-mcp", "crates/compass-model", + "crates/compass-ocr", "crates/compass-output", "crates/compass-postgres", "crates/compass-pr-intelligence", @@ -74,6 +75,9 @@ prolly-map = "=0.5.0" prolly-store-sqlite = "=0.3.0" protobuf = "=3.7.2" oxidize-pdf = { version = "4.1.1", default-features = false, features = ["compression"] } +hayro = { version = "0.7.1", default-features = false } +image = { version = "0.25.6", default-features = false, features = ["jpeg", "png", "tiff", "webp"] } +oar-ocr = { version = "=0.9.2", default-features = true } ctrlc = "3.5.2" strsim = "0.11" rayon = "1" diff --git a/MIGRATION.md b/MIGRATION.md index 4b0b2a3f..ebfc9191 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -298,6 +298,22 @@ CLI** labels an installation unsupported, upgrade that CLI or select another detected installation. Releases below 0.3.0 and 0.3.0 prereleases cannot be activated, even if they advertise some current capabilities. +## Optional document OCR + +No migration is required for existing projects because OCR defaults to off. +PDF and Office files now produce native structural blocks, so run `compass +update --force` once if an existing output predates `compass.document/1`. +To opt into local OCR, install one profile explicitly and rebuild under its new +fingerprint: + +```bash +compass models install pp-ocrv6-small +compass extract . --ocr auto --force +``` + +Do not copy or rename old flattened document caches; incompatible schema, +normalizer, renderer, OCR, and model identities are intentionally not migrated. + ## Replace commands Replace Python and legacy executable invocations with `compass`: diff --git a/PERFORMANCE.md b/PERFORMANCE.md index ee916f5c..ab09a93e 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -1178,3 +1178,27 @@ builds, first/repeated semantic diff, first/repeated viewer projection, peak RSS, and deterministic output digests. Existing sealed realizations and cached diff/view projections are expected to be constant- or bounded-read paths; explicit `history verify` remains the full integrity scan. + +## Document and OCR qualification + +Document decoding and OCR are governed by correctness and resource gates, not +an unmeasured speed claim. Qualification records per-format wall time, peak RSS, +page/image count, aggregate pixels, cache hit/miss behavior, and deterministic +artifact digests. A warm complete document cache must perform zero OCR +inference calls. Partial or corrupt entries are not warm hits. + +Run the focused deterministic gate with: + +```bash +scripts/qualify_document_ocr_v1.sh --fixtures-only +``` + +Native format fixtures run without a model. The optional model-backed corpus +runs only when its exact verified profile is already installed; ordinary CI +does not download model weights. + +On 2026-08-23, the installed `pp-ocrv6-small` acceptance test on the local +aarch64 macOS host initialized the statically linked runtime and processed its +blank plus clean synthetic-English rasters in 1.77 seconds under a debug test +build. The clean sample had 0 CER. This single smoke result is not a production +latency, memory, degraded-input, cross-architecture, or multilingual claim. diff --git a/SECURITY.md b/SECURITY.md index 3cee3802..e710b3fe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -84,3 +84,24 @@ not link cloud SDKs into the CLI. Never attach a store database or raw backup to a public issue: it can disclose repository names, paths, source anchors, and graph structure. Share a sanitized `compass store status --format json` response instead. + +## Document and OCR boundary + +PDF and OOXML files, XML relationships, compressed members, embedded images, +model files, OCR output, and document cache entries are untrusted. Compass +enforces raw/archive/member/ratio/XML-depth/document/raster limits. Raw file and +cache reads remain stream-bounded if a file grows after its metadata is read. +Compass rejects package traversal, duplicate normalized members, incoherent OCR +origin/profile/geometry, and unknown cache fields; it never executes spreadsheet +formulas or embedded objects and never follows external document links. + +`compass models install` is the only OCR download boundary. It accepts only +the pinned profile catalog, fixed HTTPS host, declared byte size, SHA-256, and +at most three validated redirects to the fixed artifact host. Concurrent +installation of one profile is serialized with a bounded lock. Publication +uses temporary files, directory synchronization, and an atomic verified marker; +model artifacts, markers, and install locks must be regular non-symlink files. +Inspection, extraction, listing, verification, cache replay, and historical +materialization never silently fetch or invoke an arbitrary executable. Model +and document cache paths can contain sensitive derived content and should not +be attached to public issues. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 4bad3cbe..ab3d1672 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,5 +1,19 @@ # Third-party notices +## Local document OCR and PDF rendering + +Compass links OAR-OCR 0.9.2 and its `ort`/ONNX Runtime integration for optional +local OCR. OAR-OCR is Apache-2.0; the Rust `ort` crates are MIT or Apache-2.0. +Compass also links Hayro 0.7.1 for pure-Rust PDF rendering under MIT or +Apache-2.0. The corresponding license texts are covered by `LICENSE-MIT` and +`LICENSE-APACHE` in release bundles. + +The separately installed `pp-ocrv6-small` and `pp-ocrv6-medium` model files +come from the immutable GreatV/OAR-OCR `v0.7.0` release and originate from the +PaddleOCR project. They are distributed under Apache-2.0 and are not embedded +in Compass release archives. Compass verifies their exact sizes and SHA-256 +digests before use. + ## openCypher Technology Compatibility Kit feature files Compass includes selected unmodified Gherkin feature files from the diff --git a/advisor-plans/022-quality-document-ocr.md b/advisor-plans/022-quality-document-ocr.md new file mode 100644 index 00000000..ef9471b6 --- /dev/null +++ b/advisor-plans/022-quality-document-ocr.md @@ -0,0 +1,978 @@ +# Plan 022: Add bounded, quality-gated OCR to document processing + +> **Executor instructions**: Read this plan completely before editing. It is a +> program plan that extends Plans 006–010; do not reimplement their document +> artifact, chunking, or OOXML work. Execute the phases in order, run every +> verification, and stop on a listed condition instead of substituting an +> unreviewed model, renderer, runtime, or cloud OCR service. When complete, +> update this plan's row in `advisor-plans/README.md` unless a reviewer owns the +> index. +> +> **Drift check (run first)**: +> `git diff --stat 3471678d..HEAD -- Cargo.toml Cargo.lock crates/compass-ocr crates/compass-media crates/compass-files crates/compass-semantic crates/compass-core crates/compass-history crates/compass-cli tests/qualification/document-ocr scripts docs CHANGELOG.md COMPATIBILITY.md MIGRATION.md PERFORMANCE.md` +> Plans 006, 007, 008, and 010 must be complete. Compare their target contracts +> with live code. If the live document artifact, locator, completeness, or cache +> contract differs materially from the assumptions below, stop and revise this +> plan with a reviewer. + +## Status + +- **Priority**: P1 +- **Effort**: XL +- **Risk**: HIGH +- **Depends on**: `advisor-plans/006-harden-media-ingestion.md`, + `advisor-plans/007-versioned-document-artifact.md`, + `advisor-plans/008-lossless-document-fusion.md`, and + `advisor-plans/010-native-ooxml-documents.md` +- **Category**: direction, security, perf, tests +- **Planned at**: commit `3471678d`, 2026-08-23 +- **Execution**: IN PROGRESS — production processing and the clean-English + installed-model gate are implemented. The implementation also stream-bounds + mutable inputs and caches, validates derived-artifact coherence, reserves + aggregate raster work before rendering, checks deadlines between inference + units, and hardens concurrent model publication against races and symlinks. + Full release qualification remains blocked by the machine-reported candidate, + multilingual/degraded, cross-architecture, and hostile-corpus measurements, + plus review of the prerequisite plan statuses. + +## Why this matters + +Native PDF extraction reads an existing text layer, and native OOXML extraction +reads XML. Neither recovers text from scanned PDF pages, screenshots, photographed +documents, or images embedded in DOCX, PPTX, and XLSX. Treating those inputs as +empty loses important evidence; OCRing every page indiscriminately duplicates +better native text, increases cost, and can replace exact evidence with model +errors. + +This plan adds selective, local OCR as a derived evidence channel. It keeps the +credential-free structural path intact, makes model and process boundaries +explicit, preserves page/slide/sheet/image geometry and confidence, and qualifies +quality against a pinned corpus before publishing a support claim. OCR must never +execute document content, contact a service during extraction, or silently +overwrite native text. + +## Relationship to the existing document program + +This plan does not replace Plans 006–012: + +- Plan 006 makes corrupt and over-limit media fail explicitly. +- Plan 007 introduces `compass.document/1`, typed blocks, logical locators, + diagnostics, completeness, and normalizer-version cache identity. +- Plan 008 decodes rich documents before packing, slices normalized content + losslessly, and fuses deterministic structure with optional semantic evidence. +- Plan 010 preserves DOCX/XLSX/PPTX structure and exposes embedded media as + unsupported evidence rather than executing it. +- Plan 012 qualifies the credential-free document graph. Its base gate must + remain independent of OCR models and external helpers. + +Plan 022 consumes those boundaries. OCR qualification is an additional gate; +failure or absence of OCR must never invalidate native document support. + +## Current state + +- `crates/compass-media/src/lib.rs:32-67` dispatches PDF, DOCX, and XLSX to + bounded text conversion. PDF uses `oxidize-pdf` text extraction only. +- `crates/compass-media/src/lib.rs:74-153` flattens DOCX/XLSX into Markdown-like + text. Plans 007 and 010 are expected to replace this with ordered artifacts. +- `crates/compass-files/src/detect.rs:24-30` discovers DOCX/XLSX but not PPTX; + Plan 010 owns PPTX discovery. +- `crates/compass-semantic/src/lib.rs:790-838` reads media into semantic input; + Plan 008 moves rich-document decoding ahead of semantic packing. +- `Cargo.toml:75` pins `oxidize-pdf` with no OCR dependency. The repository has + no Tesseract, PaddleOCR, `ocrs`, ONNX Runtime, PDFium, or other OCR engine. +- `crates/compass-transcribe/src/models.rs` is the closest model-acquisition + exemplar: it pins artifact revision, size, digest, cache location, bounded + HTTPS download, temporary writes, verification markers, and typed failures. + Match that behavior; do not move transcription ownership into OCR or create a + generic model framework before two consumers prove an exact shared contract. +- `docs/design/principles.md` requires local-first structural behavior, + evidence/provenance, deterministic identities, bounded work, coherent + publication, explicit machine contracts, and compatibility review. +- `docs/design/security-and-privacy.md` treats repository files, archives, + images, models, subprocess output, and provider content as untrusted. A limit + error cannot become an empty successful document. + +## Design decisions + +### 1. Native text remains authoritative + +DOCX, PPTX, and XLSX XML text is extracted directly. A born-digital PDF's text +layer is extracted directly. OCR is considered only for: + +1. a PDF page selected by the explicit OCR policy; +2. an image part referenced by DOCX, PPTX, or XLSX; +3. a directly discovered raster image when a caller explicitly requests OCR. + +Compass must not render a whole Office document and OCR it. Doing so would +discard exact package structure, duplicate text, and make visual rendering a +new correctness dependency. + +### 2. OCR is derived evidence, not exact extraction + +Every accepted OCR block carries: + +- `origin = "ocr"` or the live typed equivalent; +- source document and page/part/shape/sheet locator; +- pixel polygon or rectangle in a documented coordinate space; +- engine, engine version, model/profile ID, model artifact digests, language + hints, and preprocessing-policy version; +- bounded confidence as an integer in `0..=10_000`, not a serialized float; +- stable diagnostic codes for low confidence, conflict, truncation, or failure. + +OCR text never replaces a native block. Matching OCR may corroborate native +text; conflicting OCR remains separate evidence with a diagnostic. Graph IDs +must be based on source identity, logical source locator, quantized geometry, +and occurrence—not recognized text or confidence. + +### 3. OCR is off by default and local when enabled + +Public policy values are: + +```text +off native document processing only; no model loading +auto OCR only eligible scanned/low-text PDF pages and embedded images +always OCR every eligible PDF page and embedded image within limits +``` + +`off` is the compatibility default. `auto` and `always` require an installed, +verified local engine profile. Extraction never auto-downloads a model and +never falls back to a remote provider. Model installation is a separate, +visible network command. + +If OCR is explicitly requested but its engine/model is unavailable, fail before +publishing rather than silently behaving like `off`. Under existing +`--allow-partial` policy, per-page/per-image failures may publish only when the +artifact records partial visual coverage and the failed locators exactly. + +### 4. Quality is selected by Compass evidence, not vendor claims + +The recommended high-quality candidate is a pinned PP-OCRv6 small/medium +detector-recognizer profile because the current official PaddleOCR project +documents unified multilingual recognition, orientation/unwarping support, and +improved detection/recognition over PP-OCRv5. It is a candidate, not an +automatic dependency. + +Phase 0 must compare: + +- PP-OCRv6 small and medium using the official local pipeline as the quality + reference; +- Tesseract 5 with pinned `tessdata_best` as a reproducible classical baseline; +- current `ocrs`/RTen as a pure-Rust portability candidate. + +The current `ocrs` project calls itself early preview and currently recognizes +the Latin alphabet only. Do not choose it as Compass's recommended multilingual +backend solely because it is Rust. Prefer a native PP-OCRv6 inference path only +if the exact exported model runs with bounded, cross-platform behavior and +meets the quality gate. If that is not viable, ship PP-OCRv6 as an explicit +local helper-process backend; Python/helper installation remains optional and +must not enter the normal structural path. + +Do not use a generative document VLM as the v1 OCR source. Models such as +PaddleOCR-VL may be evaluated later for table/formula/chart interpretation, but +their generated structure must not be published as exact OCR geometry. + +### 5. PDF rasterization is a separate qualified boundary + +OCR consumes pixels. Add a `PageRasterizer` boundary rather than coupling the +OCR engine to PDF parsing. Prefer the already-pinned pure-Rust `oxidize-pdf` +family only after confirming that the live pinned or reviewed upgraded version +can render the qualification corpus safely and consistently. Do not silently +add PDFium, MuPDF, Poppler, a browser, or an operating-system renderer. + +If no dependency-free renderer passes the gate, the executor must stop after +shipping embedded-image OCR and present a separate explicit optional-renderer +decision. Scanned-PDF OCR must not be claimed without qualified page rendering. + +## Target architecture + +```text +bounded source bytes + | + +--> native document decoder --------------------------+ + | text, hierarchy, links, native locators | + | | + `--> bounded raster candidates | + PDF page | DOCX/PPTX/XLSX image part | + | | + v | + OCR policy: off | auto | always | + | | + v | + page/image rasterization and normalization | + | | + v | + versioned local OCR engine | + | | + v | + validated OcrObservation regions | + | | + v v + geometry-aware native/OCR fusion --> DocumentArtifact + | + +----------------+---------------+ + v v + structural document graph semantic slices +``` + +### Crate ownership + +- New `compass-ocr` owns engine-neutral request/result types, OCR engine + implementations, model manifests/acquisition, helper-process protocol, + preprocessing policy, runtime limits, and OCR qualification helpers. +- `compass-media` owns extraction of raster candidates from PDF/OOXML, + document-specific locators, page rasterization, and fusion of validated OCR + observations into `DocumentArtifact`. +- `compass-files` owns discovery, source fingerprints, and atomic/cache + primitives; it does not run models or parse OCR output. +- `compass-core` sequences document decode, optional OCR, structural + publication, and coherent failure/partial policy. +- `compass-semantic` consumes already-prepared document slices. It must not + invoke OCR independently or create a second OCR cache. +- `compass-history` fingerprints the selected OCR realization and persists the + exact completed artifacts. It never downloads models during historical work. +- `compass-cli` owns flags, model-install UX, inspect output, diagnostics, and + exit behavior. Keep inference and fusion out of command parsing. + +No dependency may point upward from `compass-ocr` or `compass-media` into CLI, +core, semantic, history, or output crates. + +## Machine contracts + +### Engine-neutral OCR contract + +Create public types equivalent to: + +```rust +pub const OCR_SCHEMA: &str = "compass.ocr/1"; +pub const OCR_PROTOCOL_SCHEMA: &str = "compass.ocr.protocol/1"; +pub const OCR_POLICY_VERSION: u32 = 1; + +pub enum OcrMode { + Off, + Auto, + Always, +} + +pub struct OcrProfileIdentity { + pub engine: String, + pub engine_version: String, + pub profile: String, + pub model_digests: BTreeMap, + pub languages: Vec, + pub preprocessing_version: u32, +} + +pub struct OcrRequest { + pub schema: String, + pub request_id: String, + pub source_kind: OcrSourceKind, + pub width: u32, + pub height: u32, + pub language_hints: Vec, + pub image_digest: String, +} + +pub struct OcrObservation { + pub ordinal: u32, + pub polygon: Vec, + pub text: String, + pub confidence_bps: u16, + pub script: Option, + pub orientation_degrees: i16, +} +``` + +Requirements: + +- `OcrPoint` uses checked integer pixels in the normalized raster coordinate + space. Polygons have 4–16 points, lie inside image bounds, and have nonzero + area. +- Requests/results contain opaque request IDs, never absolute source paths. +- Result ordering is canonical. Preserve engine order separately when useful, + then publish deterministic geometric order with an explicit writing-direction + limitation diagnostic where needed. +- Validation rejects unknown schema majors, duplicate ordinals, invalid UTF-8, + non-finite data, out-of-bounds polygons, impossible orientation, unknown + request IDs, excessive regions/text, and model/profile mismatch. +- Messages and diagnostics are bounded and must not echo image bytes or + uncontrolled helper stderr. + +### Document artifact additions + +Extend the Plan-007 artifact additively where possible: + +- an OCR block/evidence origin; +- an OCR locator containing the owning native locator, raster-candidate ID, + normalized pixel geometry, and region occurrence; +- OCR profile identity and visual-coverage status; +- `visual_coverage = not_requested | complete | partial | failed`; +- diagnostics including stable codes listed below. + +Keep `complete` profile-relative: + +- OCR `off`: native extraction may be complete even though visual coverage is + `not_requested`. +- OCR requested and all selected candidates processed: visual coverage is + `complete`. +- selected candidates omitted by a soft unsupported case or allowed failure: + artifact is incomplete and visual coverage is `partial` or `failed`. +- corrupt, over-limit, missing-model, or protocol-invalid cases are typed + errors unless existing explicit partial policy permits publication. + +Required diagnostic codes include: + +```text +ocr_candidate_skipped_too_small +ocr_candidate_limit_reached +ocr_native_text_preferred +ocr_native_text_conflict +ocr_low_confidence +ocr_language_unsupported +ocr_pdf_renderer_unavailable +ocr_engine_unavailable +ocr_engine_timeout +ocr_engine_output_rejected +ocr_partial_visual_coverage +ocr_reading_order_approximate +``` + +### Local helper protocol + +If the selected quality backend is a helper process, do not parse a vendor's +human CLI output. Define one Compass-owned protocol: + +1. Compass creates a private per-invocation temporary directory. +2. It writes bounded normalized PNG inputs with opaque names and a + `compass.ocr.protocol/1` request manifest. +3. It launches the configured helper using argument arrays, a restricted + environment, no shell, a document-level timeout, and capped stdout/stderr. +4. The helper writes one bounded response file in the same directory. +5. Compass validates schema, profile/model identity, request coverage, geometry, + counts, and digests before accepting any observation. +6. The directory is removed on success and failure. Files use source-equivalent + permissions and are never written below the repository or `compass-out/`. + +The helper receives no original path, credentials, network configuration, or +output directory. Extraction must work with network disabled after model +installation. + +## Selection and fusion policy + +### PDF `auto` eligibility + +Select a page for OCR when at least one source-backed signal is true: + +- normalized native visible text has fewer than 24 non-whitespace characters; +- more than 20% of extracted text is replacement/control characters; +- the parser classifies the page as image-dominant and native text has fewer + than 100 non-whitespace characters; +- text extraction reports a glyph-mapping diagnostic that prevents coherent + native text. + +Store thresholds in `OCR_POLICY_VERSION`; do not expose unrestricted values in +the first public CLI. `always` selects every page within limits. A PDF page is +rasterized at a documented target of 300 DPI, then reduced only as required by +the pixel cap while preserving aspect ratio. + +### OOXML image eligibility + +Plans 010's OPC reader must expose inert image relationships and exact owning +locators. OCR only decoded raster formats accepted by the image boundary. In +`auto`, skip icons/logos below 64×64 or 4,096 pixels, deduplicate repeated image +bytes by digest, and run text detection on other candidates. Reuse observations +for the same image digest but publish a separate located block for every source +occurrence. + +Do not OCR SVG script/content, OLE objects, macros, linked remote images, audio, +or video. External image relationships remain inert links and are never fetched. + +### Preprocessing + +Preprocessing must be deterministic and versioned: + +- decode with strict image dimension/pixel/animation-frame limits; +- apply declared EXIF orientation once; +- composite alpha onto white; +- preserve an RGB source and derive grayscale only when the selected profile + requires it; +- resize with one fixed algorithm; +- tile images exceeding the engine side limit with fixed overlap and + deterministic tile order; +- map tile polygons back to the normalized source raster before deduplication; +- leave deskew, orientation classification, and unwarping to a profile only + when that profile identity records those stages. + +Do not apply several heuristic thresholding variants and select the text result +that looks best. That creates an undocumented model ensemble and unstable +meaning. + +### Native/OCR fusion + +Native text always wins as the primary block. For a PDF page with positioned +native text: + +1. normalize a comparison-only copy with Unicode normalization, whitespace + folding, and case folding; preserve original strings separately; +2. compare only regions whose geometry overlaps; +3. if normalized strings are equal, retain native text and attach OCR + corroboration metadata without adding duplicate semantic text; +4. if strings differ materially, retain both, mark OCR as derived, and emit + `ocr_native_text_conflict`; +5. if native geometry is unavailable, do not guess overlap. In `auto`, OCR only + low-text pages; in `always`, publish a separate OCR channel and diagnose the + inability to fuse geometrically. + +Never choose a string solely because its OCR confidence is higher. Confidence +is engine-local and cannot outrank exact package/PDF evidence. + +## Initial resource ceilings + +Put these in named limits and test exact-limit plus one-over-limit cases. A +review may lower them based on measurements; raising them requires security and +performance evidence. + +| Resource | Initial ceiling | +|---|---:| +| OCR-selected PDF pages per document | 200 | +| OCR-selected OOXML images per document | 256 | +| Decoded pixels per raster | 24,000,000 | +| Aggregate decoded pixels per document | 300,000,000 | +| Raster long edge | 6,000 pixels | +| OCR observations per raster | 10,000 | +| OCR observations per document | 100,000 | +| OCR text per observation | 16 KiB | +| Aggregate OCR text per document | 5,000,000 characters | +| Helper response bytes | 64 MiB | +| Captured helper stderr | 1 MiB | +| Helper wall time per document | 10 minutes | +| Default OCR concurrency | 1 document; bounded engine threads | + +Integer arithmetic uses checked operations before allocation. The rasterizer +must calculate `width × height × channels` before allocating. Animated images, +multi-frame TIFFs, and tiled PDFs count each decoded frame/page against the +aggregate limit; v1 processes only the declared first image frame unless the +format contract explicitly says otherwise. + +## Cache, fingerprint, and history contract + +The prepared-document/OCR cache key must include unambiguous encodings of: + +- source byte digest; +- `DOCUMENT_SCHEMA` and `DOCUMENT_NORMALIZER_VERSION`; +- `OCR_SCHEMA` and `OCR_POLICY_VERSION`; +- OCR mode; +- engine/profile/version; +- ordered model artifact digests; +- language hints; +- rasterizer identity/version; +- preprocessing identity/version; +- every meaning-affecting limit below the compiled hard maximum. + +Do not include machine-absolute paths, credentials, temporary paths, thread +count, or cache location. A profile change is a hard cache miss; never probe an +older OCR namespace as fallback. + +Historical materialization is offline. It may use OCR only when all pinned +model artifacts and the exact allowed local engine are already available. It +must not install models, contact a helper service, or silently switch engines. +The realization fingerprint and artifact registry must preserve exact OCR +profile identity and visual-coverage status. Published realizations remain +immutable. + +Because CPU kernels can produce small confidence differences across hardware, +qualification must run on the supported architecture matrix. Quantize +confidence only at the engine boundary. If recognized text or geometry is not +stable enough for deterministic structural publication, keep OCR in the +optional semantic/derived layer and store the exact realization; do not weaken +Compass's structural determinism claim. + +## Public CLI contract + +Add thin public surfaces after the domain API is stable: + +```text +compass document inspect + [--format text|json] + [--ocr off|auto|always] + [--ocr-profile ] + [--ocr-language ]... + +compass extract [PATH] + [--ocr off|auto|always] + [--ocr-profile ] + [--ocr-language ]... + +compass models list [--format text|json] +compass models install +compass models verify +``` + +`document inspect --format json` emits `compass.document.inspect/1`, including +artifact schema/version, native blocks, OCR blocks, locators, profile identity, +visual coverage, limits used, and diagnostics. Text output shows page/slide/ +sheet/image citations and labels OCR-derived lines visibly. + +`models install` is the only OCR command allowed to download. It uses a fixed +HTTPS host allowlist, exact immutable revision, declared byte size, SHA-256, +license/model-card metadata, bounded redirects, atomic temporary files, and a +verified marker. `models verify` performs no network access. Extraction errors +must tell users the exact install/verify command without embedding a URL or +silently downloading. + +Do not expose arbitrary executable paths in the first stable CLI. If a helper +backend is required, discover one documented Compass adapter name/version from +an explicit configuration field or controlled environment variable, validate +it, and fingerprint it. Never concatenate a shell command. + +## OCR qualification contract + +Create a license-safe, reviewable corpus under +`tests/qualification/document-ocr/v1/` with ground truth and provenance for: + +- clean 300-DPI scanned English pages; +- low-resolution, skewed, rotated, noisy, and photographed pages; +- multi-column pages and mixed font sizes; +- digits, punctuation, code snippets, URLs, and identifiers; +- supported Latin, CJK, Cyrillic, Arabic/RTL, and Indic samples when the chosen + profile claims them; +- tables, formulas, chart labels, and screenshots as text detection cases + without claiming structural interpretation; +- born-digital PDF pages that must not duplicate native text; +- hybrid PDF pages with native and raster text; +- DOCX/PPTX/XLSX embedded images, repeated image bytes, alt text, and multiple + source occurrences; +- malformed images, decompression/pixel bombs, huge pages, timeout, bad helper + protocol, low confidence, and model mismatch. + +Ground truth must include expected text plus line/word polygons where geometry +is evaluated. Store source/license/generation metadata adjacent to every +non-synthetic fixture. Do not use customer or private documents. + +Measure at least: + +- character error rate (CER) and word error rate (WER); +- text-region precision/recall at documented IoU; +- reading-order pair accuracy; +- duplicate-native-text rate; +- native/OCR conflict rate; +- page/image success, partial, and rejection counts; +- wall time, peak bounded allocations or measured RSS, and output size; +- repeated-run and x86_64/aarch64 semantic equivalence. + +The recommended backend must satisfy all of these release gates: + +1. zero replacement or deletion of native text; +2. zero unbounded/panic/crash cases in the hostile corpus; +3. no network activity during extraction; +4. median CER no worse than 5% on clean supported-script scans and 15% on the + declared degraded set, reported separately per script; +5. at least 15% relative median-CER improvement over the pinned Tesseract + baseline on the degraded English set, or a documented review decision that + another quality metric is more representative; +6. no supported fixture class regresses by more than two absolute CER points + from the PP-OCRv6 reference execution; +7. duplicate-native-text rate is zero on born-digital fixtures; +8. all model, renderer, process, pixel, region, text, and time limits terminate + with the declared typed result; +9. exact OCR IDs, source locators, model identity, and diagnostic codes are + stable across repeated runs; recognized content must be semantically equal + on supported CPU architectures. + +Do not publish “better,” “best,” “SOTA,” accuracy, or speed claims unless the +Compass corpus and commands reproduce them. Vendor benchmark numbers are input +to candidate selection, not Compass evidence. + +## Commands you will need + +Use a checkout-specific external target on every compiling Cargo invocation: + +| Purpose | Command | Expected on success | +|---|---|---| +| Volume | `test -d /Volumes/Workspace && test -w /Volumes/Workspace && mkdir -p /Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr` | exit 0 | +| OCR crate | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr cargo test -p compass-ocr --locked` | exit 0 | +| Media | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr cargo test -p compass-media --locked` | exit 0 | +| Core/semantic/history | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr cargo test -p compass-core -p compass-semantic -p compass-history --locked` | exit 0 | +| CLI contract | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr cargo test -p compass-cli --test compass_product --locked` | exit 0 | +| OCR qualification | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr ./scripts/qualify_document_ocr_v1.sh --fixtures-only` | exit 0; all required profiles/cases pass | +| Base documents | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr ./scripts/qualify_document_graph_v1.sh --fixtures-only` | exit 0 without OCR/model installation | +| Lint | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-bb425f03-document-ocr cargo clippy -p compass-ocr -p compass-media -p compass-core -p compass-semantic -p compass-history -p compass-cli --all-targets --all-features --locked -- -D warnings` | exit 0 | +| Format | `cargo fmt --all -- --check` | exit 0 | +| Boundary | `sh scripts/check_product_boundary.sh` | exit 0 | + +If `/Volumes/Workspace` is unavailable or unwritable, stop. Never fall back to +the checkout's `target/` directory and never reuse another worktree's target. + +## Scope + +**In scope**: + +- `crates/compass-ocr/Cargo.toml` and `crates/compass-ocr/src/` (create) +- `crates/compass-ocr/tests/` (create) +- `crates/compass-media/src/document.rs`, `limits.rs`, and focused PDF/OOXML + raster-candidate/fusion modules produced by Plans 007 and 010 +- `crates/compass-media/tests/document_ocr.rs` (create) +- focused `compass-files`, `compass-core`, `compass-semantic`, and + `compass-history` option/fingerprint/cache/publication changes and tests +- `crates/compass-cli/src/document_commands.rs` and + `crates/compass-cli/src/model_commands.rs` (create), thin dispatch/help wiring, + and subprocess-style CLI tests +- root `Cargo.toml` and `Cargo.lock` for reviewed dependencies/workspace wiring +- `tests/qualification/document-ocr/v1/` (create) +- `scripts/qualify_document_ocr_v1.sh` and deterministic fixture/metric helpers +- `docs/design/document-processing.md`, + `docs/design/security-and-privacy.md`, + `docs/implementation/document-ocr-qualification.md` (create), + `docs/reference/document-formats.md`, `docs/reference/commands.md`, + `docs/README.md`, `PERFORMANCE.md`, `COMPATIBILITY.md`, `CHANGELOG.md`, and + `MIGRATION.md` only if users must take action +- `.github/workflows/compass-ci.yml` and `Makefile` only after the fixture-only + gate is stable and credential/network free + +**Out of scope**: + +- Legacy `.doc`, `.xls`, or `.ppt` conversion. +- Cloud OCR APIs or semantic-provider vision calls. +- Generative VLM publication of tables, formulas, charts, or relationships. +- Handwriting support unless the selected model is independently qualified and + documented; do not imply it from scene-text examples. +- Pixel-perfect Office rendering or OCR of whole Office pages. +- Formula/macro/field/OLE/ActiveX execution. +- Fetching external OOXML relationships or PDF resources. +- Automatically downloading models during extract, update, watch, history, + MCP, or document inspection. +- Committing model weights, opaque private documents, generated graphs, OCR + caches, or temporary raster images. +- Adding multiple production engines “for choice” before each passes the same + corpus and maintenance cost is justified. + +## Git workflow + +- Suggested branch: `advisor/022-quality-document-ocr`. +- Use logical commits: qualification spike; contracts; raster candidates; + engine/model delivery; fusion; orchestration/history; CLI; qualification/docs. +- Use conventional messages such as `feat(ocr): add versioned observation contract`. +- Do not push or open a PR unless instructed. + +## Phases + +### Phase 0: Freeze the baseline and select a viable engine/renderer + +Before production edits: + +1. Finish Plans 006–008 and 010 or reconcile their live equivalents. +2. Create a small, license-safe spike corpus and metric tool outside production + routing but inside the planned qualification tree. +3. Run PP-OCRv6 small/medium, Tesseract 5 `tessdata_best`, and `ocrs`/RTen with + pinned versions/models/configuration. Record install/runtime footprint, + license, supported scripts, CER/WER, geometry, CPU time, memory, and output + schema stability. +4. Test whether a reviewed pure-Rust runtime can execute the selected + PP-OCRv6 export without unsupported operators or quality drift. Do not + convert model weights in an unreviewable ad hoc script; pin the converter, + source revision, command, output size, and digest. +5. Qualify the live `oxidize-pdf` renderer or an approved pure-Rust upgrade on + scanned, rotated, image-only, font, transparency, clipping, and malformed + PDF fixtures. +6. Write the measured selection decision into + `docs/design/document-processing.md`. Name one production OCR backend and + one renderer. Keep other engines qualification-only. + +**Verify**: a checked machine-readable report contains every candidate, +version, model digest, corpus version, metric, platform, and rejection reason; +rerunning it with installed artifacts requires no network. + +### Phase 1: Add the engine-neutral OCR crate and validation contract + +Create `compass-ocr` with the types and validators above. Add traits equivalent +to: + +```rust +pub trait OcrEngine { + fn identity(&self) -> &OcrProfileIdentity; + fn recognize( + &self, + requests: &[PreparedOcrRequest], + limits: &OcrLimits, + cancellation: &AtomicBool, + ) -> Result, OcrError>; +} +``` + +The trait receives already-decoded, bounded rasters and never paths into the +source corpus. Add typed errors for unsupported profile/language, model absent +or invalid, request rejected, inference failure, timeout/cancel, protocol +failure, and output validation. + +Write contract tests for unknown major, invalid geometry, duplicate/missing +request IDs, noncanonical language order, low/high confidence bounds, text and +region ceilings, deterministic serialization, and cancellation. + +**Verify**: `cargo test -p compass-ocr --locked` exits 0 without models, +network, Python, or helper executables. + +### Phase 2: Extract bounded raster candidates without invoking OCR + +In `compass-media`, add typed `RasterCandidate` values for: + +- PDF page raster requests with page locator, dimensions, rotation, native-text + coverage signals, and selection reason; +- DOCX/PPTX/XLSX internal image parts with owning block/shape/sheet locator, + relationship kind, media type, byte digest, declared dimensions when known, + and bounded bytes. + +Use the Plan-010 OPC reader for all image parts. Reject duplicate/escaping +parts and external targets. Deduplicate decode work by byte digest while +preserving every owning occurrence. Candidate discovery itself must not decode +unbounded pixels or load a model. + +Add positive and hostile synthetic fixtures. Assert no candidate includes an +absolute path, external URL bytes, macro/OLE content, or uncontrolled package +metadata. + +**Verify**: media tests exit 0; OCR mode `off` produces byte-for-byte equivalent +native artifacts and never constructs an engine. + +### Phase 3: Implement deterministic rasterization and preprocessing + +Implement the selected PDF renderer behind `PageRasterizer`. Normalize direct +and embedded images through one bounded pipeline. Calculate all allocation +sizes before decoding; apply EXIF orientation, alpha compositing, resizing, and +tiling exactly as specified. Record rasterizer/preprocessing identity in every +request. + +Tests must cover 1×1, exact-limit, one-over-limit, huge declared dimensions, +truncated streams, animated/multi-frame inputs, orientation, alpha, long-edge +resize, tile overlap, tile-coordinate reassembly, and cancellation. Compare +small rendered PDF fixtures against checked pixel/geometry expectations, not a +human screenshot. + +**Verify**: media and OCR tests exit 0; peak raster allocations remain within +declared bounds under the limit corpus; no test invokes system PDF tools. + +### Phase 4: Implement pinned model installation and the selected engine + +Add a static model/profile manifest containing source repository, immutable +revision, artifact filenames, exact sizes, SHA-256 digests, license/model-card +metadata, supported language tags, and preprocessing requirements. Follow the +verified temporary-write pattern in `compass-transcribe/src/models.rs`, but +keep OCR ownership in `compass-ocr`. + +If using an in-process engine, disable implicit network/model discovery and +bound engine threads. If using a helper, implement only the Compass protocol +above; include handshake identity and validate every response. The helper must +be killable on timeout/cancel and must not leave descendants running. + +Tests use fixture fetchers and fake helpers for missing/short/long/bad-digest +artifacts, stale markers, redirects/host rejection, nonzero exit, timeout, +oversized output, malformed JSON, wrong request/model identity, missing/extra +results, bad geometry, stderr redaction, cleanup, and offline replay. + +**Verify**: OCR tests exit 0 offline; a native acceptance test is opt-in and +skips only when the exact verified model is absent, never because a network +download failed during ordinary tests. + +### Phase 5: Fuse OCR observations into document artifacts + +Extend `DocumentArtifact` with OCR origin, locators, profile identity, and +visual coverage. Implement native/OCR comparison and fusion exactly as defined +above. Preserve native order; insert OCR children under the owning page/image +block in deterministic geometric order. Repeated equal OCR text remains +separate when geometry/source occurrence differs. + +Add tests for corroboration, conflict, native geometry absent, low confidence, +RTL/unknown reading order, tiled overlap deduplication, repeated embedded image +bytes at different locations, and OCR requested with one failed candidate. + +Increment `DOCUMENT_NORMALIZER_VERSION`. Add unknown-version and old-cache-miss +tests; do not migrate or reinterpret cached flattened strings. + +**Verify**: media contract tests and document qualification pass; born-digital +fixtures gain no duplicate semantic text under `auto`. + +### Phase 6: Integrate core, semantic packing, cache, and history + +Add `DocumentProcessingOptions` at the application boundary and pass an OCR +engine explicitly. Prepare each rich document once, cache validated OCR results +under the complete fingerprint, then derive both structural graph blocks and +semantic slices from the same artifact. `compass-semantic` must never rerun OCR. + +Cache behavior: + +- source or profile changes miss; +- repeated image bytes reuse observations but retain distinct source locators; +- partial/failed OCR is not finalized as complete; +- cache corruption is rejected explicitly; +- `off` cache entries never satisfy `auto`/`always`; +- an OCR cache cannot be reused under a different renderer, preprocessing, + language, or model digest. + +History tests cover offline materialization, missing pinned model, exact profile +replay, immutable realization, and profile mismatch before diff. If exact OCR +reproduction differs across supported CPU architectures, publish it only as +derived semantic evidence and preserve the exact stored realization. + +**Verify**: core/semantic/history tests exit 0; fake-engine call counts prove +one preparation per changed document and zero calls on a valid warm build. + +### Phase 7: Add inspect, extraction, and model-management UX + +Implement the CLI contract above with reusable domain calls. Add help, examples, +mutual exclusions, defaults, text/JSON schemas, stdout/stderr behavior, and exit +codes. `document inspect` is read-only unless `--output` is explicitly added in +a later approved contract. Model installation is atomic and reports exact +profile identity; extraction never prompts or downloads. + +CLI tests execute the binary and assert: + +- OCR-off inspection works with no model/helper; +- explicit OCR with missing model fails with one actionable install command; +- `auto` scans only eligible candidates; +- `always` obeys caps; +- JSON is valid `compass.document.inspect/1` with no absolute temporary paths; +- corrupt/timeout/protocol errors use stderr and nonzero status; +- `--allow-partial` is visible in JSON/text and never reports complete; +- noninteractive commands never prompt, open, or fetch. + +**Verify**: CLI product and focused document tests exit 0; command reference +examples match `--help` exactly. + +### Phase 8: Build the OCR qualification and regression gate + +Promote the spike corpus to `document-ocr/v1`, add typed manifest validation, +metric computation, hostile cases, fake-engine contract cases, and opt-in exact +model acceptance. Create `scripts/qualify_document_ocr_v1.sh` modeled after the +document/code qualification scripts. It must reject unknown flags, require an +external Cargo target, download nothing, and emit bounded machine-readable +metrics plus a concise summary. + +Run candidate/reference acceptance outside normal CI only when pinned models +are provisioned. Normal CI must always run contract, fake-engine, limits, and a +small redistributable model/fixture path if licensing allows. Base document +qualification continues to pass on a machine with no OCR model. + +**Verify**: every release gate in “OCR qualification contract” is +machine-checked or explicitly reported as a blocking unmeasured item; changing +one ground-truth string, digest, limit, or model identity makes the gate fail. + +### Phase 9: Publish documentation, compatibility, and performance evidence + +Update design, security, format, command, qualification, performance, +compatibility, and changelog documents. State: + +- OCR is optional, local, model-backed, and off by default; +- which exact scripts/languages/input classes are qualified; +- native text precedence and OCR conflict behavior; +- installed model footprint and source/license; +- network behavior for installation versus extraction; +- all important limits and partial semantics; +- PDF renderer and unsupported constructs; +- reproducible accuracy/performance commands and results; +- historical fingerprint/replay behavior. + +Add `MIGRATION.md` only if an existing public default/schema requires user +action. Never claim support for a script, handwriting, table reconstruction, or +platform absent from the corpus. + +Run all commands in the table, the native baseline required by root `AGENTS.md`, +and `git diff --check`. Inspect status for model weights, temporary images, +caches, generated graphs, and unrelated edits; none may be committed. + +## Test plan + +- OCR schema, serde, validation, canonical order, geometry, limits, and cancel. +- Model manifest/download verification with only local fixture fetchers. +- Helper lifecycle/protocol/timeout/output/redaction/cleanup when applicable. +- PDF page selection/rasterization and malformed/limit behavior. +- DOCX/PPTX/XLSX embedded-image discovery, ownership, repeat-digest reuse, and + external/macro/object negatives. +- Preprocessing orientation/alpha/resize/tile/reassembly determinism. +- Native preference, corroboration, conflict, low confidence, geometry absence, + reading-order limitations, and no text-based identities. +- Cold/warm/change/profile/cache-corruption and partial-completion behavior. +- Historical offline/profile/immutable-realization behavior. +- CLI help, inspect JSON/text, missing model, no implicit network, and exit code. +- Cross-format accuracy, geometry, duplicate, performance, architecture, and + hostile qualification corpus. +- Base document and code graph gates remain unchanged and credential-free. + +## Done criteria + +- [ ] Plans 006–008 and 010 are complete and no document path still flattens + media before the artifact boundary. +- [ ] One OCR backend and one PDF renderer are selected by measured Compass + evidence; all rejected candidates and trade-offs are documented. +- [ ] Native document processing works identically with OCR absent/off. +- [ ] Explicit OCR is local during extraction, bounded, cancelable, versioned, + and provenance-preserving. +- [ ] No model is downloaded implicitly and every installed artifact is pinned + by immutable revision, exact size, SHA-256, and license metadata. +- [ ] Native text is never replaced; corroboration/conflicts are explicit. +- [ ] PDF and OOXML OCR blocks retain exact owning logical locators and bounded + pixel geometry. +- [ ] Cache/history fingerprints include every meaning-affecting OCR input and + never fall back across profiles. +- [ ] Partial/failed OCR never publishes or caches as complete. +- [ ] `compass document inspect`, extraction flags, and model commands have + tested text/JSON/help/exit contracts. +- [ ] OCR qualification passes its accuracy, duplicate, safety, determinism, + and resource gates on every claimed script/platform. +- [ ] Base document qualification passes without OCR models or helper tools. +- [ ] Targeted tests, workspace baseline, lint, format, boundary, and diff checks + pass with the required external target. +- [ ] Plan 022 is marked `DONE` in `advisor-plans/README.md`. + +## STOP conditions + +Stop and report instead of improvising if: + +- any prerequisite plan is incomplete or its artifact/locator/cache contract is + incompatible with this design; +- the selected model's weights, datasets, or redistribution license cannot be + verified for Compass's distribution; +- PP-OCR model conversion/inference requires unpinned tooling or changes quality + beyond the declared gate; +- no renderer can produce bounded, trustworthy PDF pixels without adding an + unapproved native/runtime dependency; +- a candidate engine requires network access during extraction; +- OCR output can only be integrated by replacing native text or fabricating + package/PDF byte offsets; +- cross-platform recognition/geometry cannot meet the required semantic + equivalence; keep it out of structural publication and request review; +- a helper cannot be reliably timed out, killed, output-bounded, and cleaned up; +- arbitrary user executables, shell strings, external OOXML links, formulas, + macros, OLE, or document scripts would need to execute; +- a public schema major, stable ID, or default must change without compatibility + and migration approval; +- qualification needs private documents, credentials, a cloud OCR API, or + network access; +- an in-scope file overlaps user edits that cannot be preserved; +- `/Volumes/Workspace` is unavailable or a Cargo command would use local + `target/`. + +## Maintenance notes + +OCR quality changes whenever the engine, model, renderer, preprocessing, +language policy, or fusion threshold changes. Treat each as a normalizer/profile +change with cache/history review and corpus evidence. Keep the base document +gate independent so optional OCR cannot become an accidental product dependency. + +Reviewers should scrutinize source authority, geometry, duplicate suppression, +model provenance, temp-file sensitivity, helper containment, cross-platform +drift, and false completeness. High average OCR accuracy does not excuse one +unbounded path or an invented native-text replacement. + +## Primary references for engine evaluation + +- [`ocrs` project](https://github.com/robertknight/ocrs) — end-to-end Rust OCR + goals, RTen inference, current early-preview status, and Latin-only support. +- [RTen project](https://github.com/robertknight/rten) — end-to-end Rust, + CPU-only ONNX/RTen inference and supported deployment targets. +- [PaddleOCR project](https://github.com/PaddlePaddle/PaddleOCR) and + [official OCR pipeline documentation](https://github.com/PaddlePaddle/PaddleOCR/blob/main/docs/version3.x/pipeline_usage/OCR.en.md) + — PP-OCRv6 profiles, multilingual coverage, orientation/unwarping, model + distribution, and reported quality/performance. +- [Tesseract 5 user manual](https://tesseract-ocr.github.io/tessdoc/) — stable + classical baseline, official trained-data families, languages, and license. +- [`oxidize-pdf` API documentation](https://docs.rs/oxidize-pdf/latest/oxidize_pdf/) + — current pure-Rust parsing/rendering/OCR integration claims that must be + verified against Compass's pinned version and qualification corpus. diff --git a/advisor-plans/README.md b/advisor-plans/README.md index 076af837..b1b5b510 100644 --- a/advisor-plans/README.md +++ b/advisor-plans/README.md @@ -60,6 +60,13 @@ promotion decision. The mounted qualification target records the pinned SwiftSyntax, Dart Analyzer, scala.meta, and Groovy CompilationUnit toolchains and the immutable audit results. +Plan 022 extends the native document program with selective, local OCR for +scanned PDF pages and images embedded in DOCX, PPTX, and XLSX. Native package +and PDF text remains authoritative. OCR is off by default, model/profile and +geometry provenance are explicit, model installation is separate from +extraction, and the recommended engine must beat a pinned baseline on a +Compass-owned corpus before support or quality claims ship. + ## Execution order and status | Plan | Title | Priority | Effort | Depends on | Status | @@ -84,6 +91,7 @@ and the immutable audit results. | 018 | Expose five native MCP workflow prompts | P2 | M | — | TODO | | 019 | Hard-cut Ruby to a qualifying universal evidence pipeline | P1 | XL | —; final gate should consume 005 or equivalent | IN PROGRESS | | 020 | Hard-cut Swift, Dart, Scala, and Groovy to universal evidence | P1 | XXL | —; final gate should consume 005 or equivalent | DONE | +| 022 | Add bounded, quality-gated OCR to document processing | P1 | XL | 006, 007, 008, 010 | IN PROGRESS | Status values: `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED`, or `REJECTED`. @@ -130,6 +138,11 @@ Status values: `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED`, or `REJECTED`. reuse the exact-language JVM boundary established for Scala. Each language has a separate candidate and atomic hard-cut phase, and the mixed-language release gate runs only after all four cuts. +- Plan 022 starts only after the document safety, artifact, slicing/fusion, and + OOXML plans. It adds a separate OCR qualification gate rather than weakening + Plan 012's credential-free native document gate. Plan 012 and Plan 022 may + reuse fixture-manifest infrastructure, but neither may make OCR models or + helper runtimes prerequisites for native document support. ## Direction options not promoted to implementation plans @@ -147,9 +160,10 @@ Status values: `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED`, or `REJECTED`. stability, latency, and memory measurements. - **Linux and Windows release artifacts:** clear distribution gap; deferred only to keep this first plan set at five items. -- **Image OCR and slide-layout understanding:** valuable for scanned PDFs and - diagram-heavy decks, but require a separate bounded media/provenance design; - these plans deliberately cover native text and package evidence first. +- **Generative slide-layout, chart, and formula understanding:** Plan 022 now + owns bounded text OCR and geometry for scanned pages and embedded images. + Generative VLM interpretation remains deferred because it needs separate + inferred-evidence, hallucination, cost, and qualification contracts. - **Legacy binary Office formats (`.doc`, `.xls`, `.ppt`):** require a safe parser/conversion boundary distinct from OOXML and are not implied by plan 010. diff --git a/crates/compass-cli/Cargo.toml b/crates/compass-cli/Cargo.toml index 3db42d37..cf1637dd 100644 --- a/crates/compass-cli/Cargo.toml +++ b/crates/compass-cli/Cargo.toml @@ -50,6 +50,8 @@ compass-history = { path = "../compass-history", version = "0.3.19" } compass-google-workspace = { path = "../compass-google-workspace", version = "0.3.19" } compass-ingest = { path = "../compass-ingest", version = "0.3.19" } compass-model = { path = "../compass-model", version = "0.3.19" } +compass-media = { path = "../compass-media", version = "0.3.19" } +compass-ocr = { path = "../compass-ocr", version = "0.3.19" } compass-mcp = { path = "../compass-mcp", version = "0.3.19" } compass-output = { path = "../compass-output", version = "0.3.19" } compass-postgres = { path = "../compass-postgres", version = "0.3.19" } @@ -64,3 +66,6 @@ url.workspace = true [lints] workspace = true + +[dev-dependencies] +zip.workspace = true diff --git a/crates/compass-cli/assets/compass-skill/SKILL.md b/crates/compass-cli/assets/compass-skill/SKILL.md index 6447fc9d..d6914cc7 100644 --- a/crates/compass-cli/assets/compass-skill/SKILL.md +++ b/crates/compass-cli/assets/compass-skill/SKILL.md @@ -123,9 +123,10 @@ Classify the effect before selecting a command: - Read-only local: `ask`, `search`, `callers`, `callees`, `impact`, `explore`, `node`, `call-graph`, `query`, `program`, `path`, `explain`, `affected`, - `tree`, and local diagnostics. + `tree`, `document`, `models list`, `models verify`, and local diagnostics. - Local publication: `init`, `update`, `extract`, `watch`, `cluster-only`, - `label`, history materialization, installation, and file-based exports. + `label`, `models install`, history materialization, installation, and + file-based exports. - External or credentialed: semantic providers, URL ingestion, cloning, PR inspection, PostgreSQL or Google Workspace extraction, HTTP serving, and database export pushes. @@ -146,6 +147,11 @@ Choose the least expensive command that satisfies the request: optional native integrations. - `compass extract PATH` when the user wants semantic facts from documents, papers, Office files, or images and accepts the configured provider. +- `compass document inspect FILE` for a local, bounded preview of PDF, DOCX, + PPTX, or XLSX content; add `--ocr auto` for scanned pages or embedded images. +- `compass models list|verify` to inspect local OCR readiness, and + `compass models install pp-ocrv6-small` only when OCR is requested and the + verified model profile is missing. - `compass cluster-only` when extraction is current and only communities or visual outputs need regeneration. - `compass watch .` for continuous deterministic refresh during active work. diff --git a/crates/compass-cli/assets/compass-skill/references/command-reference.md b/crates/compass-cli/assets/compass-skill/references/command-reference.md index 5e8fc273..b94e90b8 100644 --- a/crates/compass-cli/assets/compass-skill/references/command-reference.md +++ b/crates/compass-cli/assets/compass-skill/references/command-reference.md @@ -63,6 +63,10 @@ though the query itself does not edit the working tree. - `compass update`: deterministic structural refresh. - `compass extract`: structural plus optional semantic, Cargo, PostgreSQL, or Google Workspace layers. +- `compass document`: inspect PDF, DOCX, PPTX, or XLSX content locally, with + optional managed OCR and typed JSON evidence. +- `compass models`: list, install, or verify the pinned OCR model profiles in + the Compass cache; no system OCR package is required. - `compass watch`: long-running deterministic refresh and semantic-staleness detection. - `compass cluster-only`: recompute communities and visual/report artifacts from diff --git a/crates/compass-cli/assets/compass-skill/references/semantic-extraction.md b/crates/compass-cli/assets/compass-skill/references/semantic-extraction.md index a8e7caf9..d5064e66 100644 --- a/crates/compass-cli/assets/compass-skill/references/semantic-extraction.md +++ b/crates/compass-cli/assets/compass-skill/references/semantic-extraction.md @@ -9,12 +9,24 @@ images, external schemas, or when provider configuration is involved. compass update . compass extract . --code-only compass extract docs --backend BACKEND --model MODEL +compass document inspect scan.pdf --ocr auto +compass models install pp-ocrv6-small ``` - `update` is deterministic structural extraction. - `extract --code-only` guarantees no model invocation. - `extract` without `--code-only` may send selected content to the configured provider. +- Document parsing and OCR are local. Native PDF/Office extraction requires no + model; OCR uses a pinned Compass-managed profile and never requires a system + OCR installation. + +Use `compass document inspect FILE --format json` to diagnose document content +without publishing a graph. `--ocr off` is the deterministic native-only path; +`--ocr auto` OCRs scanned PDF pages and eligible embedded Office images while +preserving native text; `--ocr always` OCRs every eligible visual. If a profile +is absent, install it explicitly with `compass models install +pp-ocrv6-small`, then confirm its digest with `compass models verify`. Do not assume credentials or a provider. Run `compass extract --help` and `compass provider list`, then use only configuration already in scope. Never diff --git a/crates/compass-cli/src/document_commands.rs b/crates/compass-cli/src/document_commands.rs new file mode 100644 index 00000000..58408d3d --- /dev/null +++ b/crates/compass-cli/src/document_commands.rs @@ -0,0 +1,248 @@ +use std::path::PathBuf; +use std::str::FromStr; + +use compass_media::document::{DocumentArtifact, DocumentOrigin}; +use compass_media::{ + DocumentProcessingOptions, decode_document_with_ocr, render_document_markdown, +}; +use compass_ocr::{ManagedOarEngine, ModelProfile, OcrMode, normalize_language_hints}; +use serde::Serialize; + +use crate::Outcome; + +#[derive(Serialize)] +struct InspectEnvelope<'a> { + schema: &'static str, + source: String, + artifact: &'a DocumentArtifact, + processing: InspectProcessing<'a>, + limits: InspectLimits, +} + +#[derive(Serialize)] +struct InspectProcessing<'a> { + ocr_mode: OcrMode, + ocr_profile: ModelProfile, + language_hints: &'a [String], + allow_partial: bool, +} + +#[derive(Serialize)] +struct InspectLimits { + max_raw_bytes: u64, + max_pdf_pages: usize, + max_office_images: usize, + max_raster_pixels: u64, + max_raster_long_edge: u32, + max_aggregate_pixels: u64, + max_observations_per_raster: usize, + max_observations_per_document: usize, + max_text_bytes_per_observation: usize, + max_text_chars_per_document: usize, +} + +pub(crate) fn command(args: &[String]) -> Outcome { + match args.first().map(String::as_str) { + Some("inspect") => inspect(&args[1..]), + Some(other) => Outcome::from_command_output( + 2, + String::new(), + format!("error: unknown document command {other:?}"), + ), + None => Outcome::from_command_output( + 2, + String::new(), + "error: missing document command; expected inspect".to_owned(), + ), + } +} + +fn inspect(args: &[String]) -> Outcome { + let mut path = None; + let mut format = "text"; + let mut mode = OcrMode::Off; + let mut profile = ModelProfile::PpOcrV6Small; + let mut languages = Vec::new(); + let mut allow_partial = false; + let mut index = 0_usize; + while index < args.len() { + match args[index].as_str() { + "--format" => { + index += 1; + let Some(value) = args.get(index) else { + return usage_error("--format requires text or json"); + }; + if !matches!(value.as_str(), "text" | "json") { + return usage_error("--format requires text or json"); + } + format = value; + } + "--ocr" => { + index += 1; + let Some(value) = args.get(index) else { + return usage_error("--ocr requires off, auto, or always"); + }; + mode = match OcrMode::from_str(value) { + Ok(mode) => mode, + Err(error) => return usage_error(&error.to_string()), + }; + } + "--ocr-profile" => { + index += 1; + let Some(value) = args.get(index) else { + return usage_error("--ocr-profile requires a profile name"); + }; + profile = match ModelProfile::from_str(value) { + Ok(profile) => profile, + Err(error) => return usage_error(&error.to_string()), + }; + } + "--ocr-language" => { + index += 1; + let Some(value) = args.get(index) else { + return usage_error("--ocr-language requires a BCP-47 language tag"); + }; + languages.push(value.clone()); + } + "--allow-partial" => allow_partial = true, + value if value.starts_with('-') => { + return usage_error(&format!("unknown document option {value:?}")); + } + value if path.is_none() => path = Some(PathBuf::from(value)), + value => return usage_error(&format!("unexpected argument {value:?}")), + } + index += 1; + } + let Some(path) = path else { + return usage_error("document inspect requires a file"); + }; + let bytes = match compass_media::read_document_bounded(&path) { + Ok(bytes) => bytes, + Err(error) => { + return Outcome::failure(format!( + "error: could not read bounded document {}: {error}", + path.display() + )); + } + }; + languages = match normalize_language_hints(&languages) { + Ok(languages) => languages, + Err(error) => return usage_error(&error.to_string()), + }; + let engine = if mode == OcrMode::Off { + None + } else { + match ManagedOarEngine::load(profile) { + Ok(engine) => Some(engine), + Err(error) => { + return Outcome::failure(format!( + "error: {error}\nhelp: Compass manages all OCR runtime and model dependencies; no system OCR package is required" + )); + } + } + }; + let options = DocumentProcessingOptions { + ocr_mode: mode, + language_hints: languages.clone(), + allow_partial, + }; + let artifact = match decode_document_with_ocr( + &path, + &bytes, + &options, + engine.as_ref().map(|engine| engine as _), + ) { + Ok(artifact) => artifact, + Err(error) => { + return Outcome::failure(format!( + "error: could not process {}: {error}", + path.display() + )); + } + }; + if format == "json" { + let envelope = InspectEnvelope { + schema: compass_media::DOCUMENT_INSPECT_SCHEMA, + source: path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("document") + .to_owned(), + artifact: &artifact, + processing: InspectProcessing { + ocr_mode: mode, + ocr_profile: profile, + language_hints: &languages, + allow_partial, + }, + limits: InspectLimits { + max_raw_bytes: compass_media::MEDIA_MAX_RAW_BYTES, + max_pdf_pages: compass_media::OCR_MAX_PDF_PAGES, + max_office_images: compass_media::OCR_MAX_OOXML_IMAGES, + max_raster_pixels: compass_ocr::OCR_MAX_RASTER_PIXELS, + max_raster_long_edge: compass_ocr::OCR_MAX_RASTER_LONG_EDGE, + max_aggregate_pixels: compass_media::OCR_MAX_AGGREGATE_PIXELS, + max_observations_per_raster: compass_ocr::OCR_MAX_OBSERVATIONS_PER_RASTER, + max_observations_per_document: compass_ocr::OCR_MAX_OBSERVATIONS_PER_DOCUMENT, + max_text_bytes_per_observation: compass_ocr::OCR_MAX_TEXT_BYTES_PER_OBSERVATION, + max_text_chars_per_document: compass_ocr::OCR_MAX_TEXT_CHARS_PER_DOCUMENT, + }, + }; + return match serde_json::to_string_pretty(&envelope) { + Ok(json) => Outcome::success(json), + Err(error) => { + Outcome::failure(format!("error: could not encode document JSON: {error}")) + } + }; + } + let mut output = match render_document_markdown(&artifact) { + Ok(output) => output, + Err(error) => { + return Outcome::failure(format!("error: could not render document: {error}")); + } + }; + output.push_str(&format!( + "\n\n[document processing: ocr={mode:?}, profile={}, allow_partial={allow_partial}, complete={}]", + profile.name(), artifact.complete + )); + let ocr_blocks = artifact + .blocks + .iter() + .filter_map(|block| match &block.origin { + DocumentOrigin::Ocr { confidence_bps, .. } => Some((block, confidence_bps)), + DocumentOrigin::Native => None, + }) + .collect::>(); + if !ocr_blocks.is_empty() { + output.push_str("\n\n## OCR-derived evidence\n"); + for (block, confidence) in ocr_blocks { + output.push_str(&format!( + "\n- [OCR {:02}.{:02}% @ {:?}] {}", + confidence / 100, + confidence % 100, + block.locator, + block.text + )); + } + } + if !artifact.diagnostics.is_empty() { + output.push_str("\n\n## Diagnostics\n"); + for diagnostic in &artifact.diagnostics { + output.push_str(&format!( + "\n- {} ({:?}): {}", + diagnostic.code, diagnostic.severity, diagnostic.message + )); + } + } + Outcome::success(output) +} + +fn usage_error(message: &str) -> Outcome { + Outcome::from_command_output( + 2, + String::new(), + format!( + "error: {message}\nusage: compass document inspect [--format text|json] [--ocr off|auto|always] [--ocr-profile ] [--ocr-language ]..." + ), + ) +} diff --git a/crates/compass-cli/src/help.rs b/crates/compass-cli/src/help.rs index fc54b265..5c344d91 100644 --- a/crates/compass-cli/src/help.rs +++ b/crates/compass-cli/src/help.rs @@ -75,6 +75,8 @@ const GROUPS: &[Group] = &[ "init", "update", "extract", + "document", + "models", "watch", "cluster-only", "label", @@ -240,7 +242,29 @@ const PAGES: &[Page] = &[ "compass extract [PATH] [OPTIONS]", "compass extract --postgres [OPTIONS]" ], - "Arguments:\n [PATH] Project directory to scan\n\nOptions:\n --program Also build and publish the optional Program IR\n --program-artifact Add an offline program-evidence artifact; repeatable\n --no-program Explicitly omit the optional Program IR (compatibility flag)\n --code-only Extract structural code without semantic sources\n --cargo Include Cargo metadata\n --google-workspace Include Google Workspace shortcuts\n --postgres Extract PostgreSQL schema objects\n --backend Semantic provider name\n --model Override the provider's default model\n --mode Use deep semantic extraction\n --token-budget Maximum semantic token budget\n --max-concurrency Maximum concurrent provider requests\n --max-workers Maximum local extraction workers\n --api-timeout Provider request timeout\n --allow-partial Publish results when semantic chunks fail\n --dedup-llm Use the model to review likely duplicates\n --timing Print stage timings\n --global Merge the completed graph into the global graph\n --as Repository tag used with --global\n --store Graph storage [default: json]\n --inference-level Inference: low, medium, high, or max [default: low]\n --out Output directory\n --force Rebuild unchanged inputs\n --no-cluster Skip community detection\n --no-viz Skip graph.html generation\n --no-gitignore Ignore .gitignore rules\n --exclude Exclude a glob pattern; repeatable\n --resolution Community resolution [default: 1.0]\n --exclude-hubs Exclude high-degree clustering hubs\n\nExamples:\n compass extract ./project --code-only\n compass extract ./project --program --code-only\n compass extract ./project --no-program --code-only\n compass extract ./project --program-artifact index.scip --code-only\n compass extract ./project --code-only --inference-level max\n compass extract ./project --code-only --store sqlite\n compass extract --postgres \"postgresql://localhost/app\" --code-only\n\nNotes:\n Low is the evidence-first default and publishes exact relationships only. Medium adds source-backed inference; high adds explicitly qualified external relationships; max retains all inferred relationships, including deferred receivers. Semantic extraction may require credentials for the selected provider." + "Arguments:\n [PATH] Project directory to scan\n\nOptions:\n --program Also build and publish the optional Program IR\n --program-artifact Add an offline program-evidence artifact; repeatable\n --no-program Explicitly omit the optional Program IR (compatibility flag)\n --code-only Extract structural code without semantic sources\n --cargo Include Cargo metadata\n --google-workspace Include Google Workspace shortcuts\n --postgres Extract PostgreSQL schema objects\n --backend Semantic provider name\n --model Override the provider's default model\n --mode Use deep semantic extraction\n --ocr Local document OCR policy [default: off]\n --ocr-profile pp-ocrv6-small or pp-ocrv6-medium\n --ocr-language OCR language hint; repeatable\n --token-budget Maximum semantic token budget\n --max-concurrency Maximum concurrent provider requests\n --max-workers Maximum local extraction workers\n --api-timeout Provider request timeout\n --allow-partial Publish results when semantic or OCR candidates fail\n --dedup-llm Use the model to review likely duplicates\n --timing Print stage timings\n --global Merge the completed graph into the global graph\n --as Repository tag used with --global\n --store Graph storage [default: json]\n --inference-level Inference: low, medium, high, or max [default: low]\n --out Output directory\n --force Rebuild unchanged inputs\n --no-cluster Skip community detection\n --no-viz Skip graph.html generation\n --no-gitignore Ignore .gitignore rules\n --exclude Exclude a glob pattern; repeatable\n --resolution Community resolution [default: 1.0]\n --exclude-hubs Exclude high-degree clustering hubs\n\nExamples:\n compass extract ./project --code-only\n compass extract ./documents --ocr auto\n compass extract ./project --program --code-only\n compass extract ./project --program-artifact index.scip --code-only\n compass extract ./project --code-only --inference-level max\n compass extract ./project --code-only --store sqlite\n compass extract --postgres \"postgresql://localhost/app\" --code-only\n\nNotes:\n OCR is local and never downloads during extraction. Install a pinned profile once with `compass models install pp-ocrv6-small`; no Python, Tesseract, office suite, or system PDF tool is required.\n Low publishes exact relationships only. Medium adds source-backed inference; high adds explicitly qualified external relationships; max retains all inferred relationships, including deferred receivers. Semantic extraction may require credentials for the selected provider." + ), + page!( + "document", + "Inspect PDF and Office documents with optional local OCR", + ["compass document inspect [OPTIONS]"], + "Commands:\n inspect Decode and inspect one PDF, DOCX, XLSX, or PPTX file\n\nExamples:\n compass document inspect report.pdf\n compass document inspect scan.pdf --ocr auto\n compass document inspect deck.pptx --ocr auto --format json\n\nNotes:\n Native text remains authoritative. OCR is local derived evidence and requires a verified Compass-managed profile. Run `compass models install pp-ocrv6-small` once; no Python, Tesseract, office suite, or system PDF tool is required." + ), + page!( + "document inspect", + "Decode one document and show typed structure, locators, and OCR evidence", + ["compass document inspect [OPTIONS]"], + "Arguments:\n PDF, DOCX, XLSX, or PPTX document\n\nOptions:\n --format Human output or compass.document.inspect/1 [default: text]\n --ocr Selective local OCR policy [default: off]\n --ocr-profile pp-ocrv6-small or pp-ocrv6-medium [default: pp-ocrv6-small]\n --ocr-language Language hint; repeatable\n --allow-partial Retain exact successful OCR evidence if one candidate fails\n\nExamples:\n compass document inspect contract.docx\n compass document inspect scanned.pdf --ocr auto\n compass document inspect workbook.xlsx --ocr always --format json" + ), + page!( + "models", + "Manage verified local OCR model profiles", + [ + "compass models list [--format text|json]", + "compass models install ", + "compass models verify " + ], + "Commands:\n list Show installation and verification status\n install Download a pinned profile and verify size plus SHA-256\n verify Verify an installed profile without network access\n\nProfiles:\n pp-ocrv6-small Default 30 MiB model profile\n pp-ocrv6-medium Larger 132 MiB model profile\n\nExamples:\n compass models list\n compass models install pp-ocrv6-small\n compass models verify pp-ocrv6-small\n\nNotes:\n Compass manages the inference runtime and model files. Document extraction never silently downloads a model." ), page!( "watch", @@ -1275,7 +1299,7 @@ mod tests { #[test] fn catalog_has_unique_complete_public_roots() { let roots = root_commands(); - assert_eq!(roots.len(), 50); + assert_eq!(roots.len(), 52); for root in roots { let matches = PAGES.iter().filter(|page| page.path == root).count(); assert_eq!(matches, 1, "{root}"); diff --git a/crates/compass-cli/src/history_build.rs b/crates/compass-cli/src/history_build.rs index 6f051fc1..edb0130e 100644 --- a/crates/compass-cli/src/history_build.rs +++ b/crates/compass-cli/src/history_build.rs @@ -61,6 +61,21 @@ impl HistoryBuildOptions { if profile.value("semantic_mode") == Some("deep") { forwarded.extend(["--mode".to_owned(), "deep".to_owned()]); } + push_profile_option(&profile, &mut forwarded, "ocr_mode", "--ocr", "off"); + push_profile_option( + &profile, + &mut forwarded, + "ocr_profile", + "--ocr-profile", + "none", + ); + for (key, value) in profile + .entries() + .filter(|(key, _)| key.starts_with("ocr_language.")) + { + let _ = key; + forwarded.extend(["--ocr-language".to_owned(), value.to_owned()]); + } if code_only { forwarded.push("--code-only".to_owned()); } @@ -125,6 +140,18 @@ impl HistoryBuildOptions { } }; insert_current_engine_profile(&mut profile, deep)?; + if profile.value("ocr_mode").is_none() { + profile.insert("ocr_mode", "off")?; + } + if profile.value("ocr_profile").is_none() { + profile.insert("ocr_profile", "pp-ocrv6-small")?; + } + if profile.value("ocr_model_manifest").is_none() { + profile.insert( + "ocr_model_manifest", + &compass_ocr::profile_manifest_digest(compass_ocr::ModelProfile::PpOcrV6Small), + )?; + } Self::from_profile(profile) } @@ -149,12 +176,20 @@ impl HistoryBuildOptions { } fn from_values(mut values: HistoryBuildValues) -> Result { + let ocr_profile = values + .ocr_profile + .parse::() + .map_err(|error| HistoryError::InvalidFingerprint(error.to_string()))?; + values.ocr_profile = ocr_profile.name().to_owned(); + values.ocr_languages = compass_ocr::normalize_language_hints(&values.ocr_languages) + .map_err(|error| HistoryError::InvalidFingerprint(error.to_string()))?; if values.code_only { if values.backend.is_some() || values.model.is_some() || values.deep || values.dedup_llm || values.token_budget.is_some() + || values.ocr_mode != "off" { return Err(HistoryError::InvalidFingerprint( "--code-only cannot be combined with semantic-provider options".to_owned(), @@ -170,6 +205,12 @@ impl HistoryBuildOptions { ("code_only", values.code_only.to_string()), ("cargo", values.cargo.to_string()), ("dedup_llm", values.dedup_llm.to_string()), + ("ocr_mode", values.ocr_mode.clone()), + ("ocr_profile", values.ocr_profile.clone()), + ( + "ocr_model_manifest", + compass_ocr::profile_manifest_digest(ocr_profile), + ), ( "semantic_mode", if values.deep { "deep" } else { "standard" }.to_owned(), @@ -227,6 +268,9 @@ impl HistoryBuildOptions { for (index, exclude) in values.excludes.iter().enumerate() { profile.insert(&format!("exclude.{index:06}"), exclude)?; } + for (index, language) in values.ocr_languages.iter().enumerate() { + profile.insert(&format!("ocr_language.{index:06}"), language)?; + } let mut forwarded = Vec::new(); if let Some(backend) = &values.backend { @@ -235,6 +279,13 @@ impl HistoryBuildOptions { if let Some(model) = &values.model { forwarded.extend(["--model".to_owned(), model.clone()]); } + if values.ocr_mode != "off" { + forwarded.extend(["--ocr".to_owned(), values.ocr_mode.clone()]); + forwarded.extend(["--ocr-profile".to_owned(), values.ocr_profile.clone()]); + for language in &values.ocr_languages { + forwarded.extend(["--ocr-language".to_owned(), language.clone()]); + } + } if values.code_only { forwarded.push("--code-only".to_owned()); } @@ -288,6 +339,73 @@ fn insert_current_engine_profile( ("extractor_version", "compass-languages/v1".to_owned()), ("resolver_version", "compass-resolve/v1".to_owned()), ("pipeline_version", "compass-core/v1".to_owned()), + ("document_schema", compass_media::DOCUMENT_SCHEMA.to_owned()), + ( + "document_normalizer_version", + compass_media::DOCUMENT_NORMALIZER_VERSION.to_string(), + ), + ("ocr_protocol", compass_ocr::OCR_PROTOCOL_SCHEMA.to_owned()), + ("ocr_policy", compass_ocr::OCR_POLICY_VERSION.to_string()), + ( + "ocr_preprocessing", + compass_ocr::OCR_PREPROCESSING_VERSION.to_string(), + ), + ( + "pdf_rasterizer", + compass_media::PDF_RASTERIZER_IDENTITY.to_owned(), + ), + ( + "document_raw_bytes_limit", + compass_media::MEDIA_MAX_RAW_BYTES.to_string(), + ), + ( + "ocr_pdf_pages_limit", + compass_media::OCR_MAX_PDF_PAGES.to_string(), + ), + ( + "ocr_office_images_limit", + compass_media::OCR_MAX_OOXML_IMAGES.to_string(), + ), + ( + "ocr_raster_pixels_limit", + compass_ocr::OCR_MAX_RASTER_PIXELS.to_string(), + ), + ( + "ocr_raster_edge_limit", + compass_ocr::OCR_MAX_RASTER_LONG_EDGE.to_string(), + ), + ( + "ocr_engine_side_limit", + compass_ocr::OCR_ENGINE_MAX_SIDE.to_string(), + ), + ( + "ocr_tile_overlap", + compass_ocr::OCR_TILE_OVERLAP.to_string(), + ), + ( + "ocr_aggregate_pixels_limit", + compass_media::OCR_MAX_AGGREGATE_PIXELS.to_string(), + ), + ( + "ocr_regions_raster_limit", + compass_ocr::OCR_MAX_OBSERVATIONS_PER_RASTER.to_string(), + ), + ( + "ocr_regions_document_limit", + compass_ocr::OCR_MAX_OBSERVATIONS_PER_DOCUMENT.to_string(), + ), + ( + "ocr_text_region_limit", + compass_ocr::OCR_MAX_TEXT_BYTES_PER_OBSERVATION.to_string(), + ), + ( + "ocr_text_document_limit", + compass_ocr::OCR_MAX_TEXT_CHARS_PER_DOCUMENT.to_string(), + ), + ( + "ocr_wall_time_seconds", + compass_ocr::OCR_MAX_DOCUMENT_WALL_TIME_SECS.to_string(), + ), ( "program_provider_policy", "offline-artifacts-first".to_owned(), @@ -331,6 +449,25 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError | "extractor_version" | "resolver_version" | "pipeline_version" + | "document_schema" + | "document_normalizer_version" + | "ocr_protocol" + | "ocr_policy" + | "ocr_preprocessing" + | "pdf_rasterizer" + | "document_raw_bytes_limit" + | "ocr_pdf_pages_limit" + | "ocr_office_images_limit" + | "ocr_raster_pixels_limit" + | "ocr_raster_edge_limit" + | "ocr_engine_side_limit" + | "ocr_tile_overlap" + | "ocr_aggregate_pixels_limit" + | "ocr_regions_raster_limit" + | "ocr_regions_document_limit" + | "ocr_text_region_limit" + | "ocr_text_document_limit" + | "ocr_wall_time_seconds" | "program_provider_policy" | "program_ir_schema" | "program_merger_version" @@ -344,6 +481,9 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError | "code_only" | "cargo" | "dedup_llm" + | "ocr_mode" + | "ocr_profile" + | "ocr_model_manifest" | "semantic_mode" | "semantic_prompt_sha256" | "provider" @@ -356,6 +496,7 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError | "provider_max_output_tokens" | "provider_region" ) && !key.starts_with("exclude.") + && !key.starts_with("ocr_language.") { return Err(HistoryError::InvalidFingerprint(format!( "unsupported persisted build-profile field {key:?}" @@ -368,6 +509,9 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError ("extractor_version", "compass-languages/v1"), ("resolver_version", "compass-resolve/v1"), ("pipeline_version", "compass-core/v1"), + ("document_schema", compass_media::DOCUMENT_SCHEMA), + ("ocr_protocol", compass_ocr::OCR_PROTOCOL_SCHEMA), + ("pdf_rasterizer", compass_media::PDF_RASTERIZER_IDENTITY), ("program_provider_policy", "offline-artifacts-first"), ("enabled_features", "workspace-default"), ("direction", "native-source-semantics"), @@ -380,6 +524,80 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError ))); } } + for (key, expected) in [ + ( + "document_raw_bytes_limit", + compass_media::MEDIA_MAX_RAW_BYTES.to_string(), + ), + ( + "ocr_pdf_pages_limit", + compass_media::OCR_MAX_PDF_PAGES.to_string(), + ), + ( + "ocr_office_images_limit", + compass_media::OCR_MAX_OOXML_IMAGES.to_string(), + ), + ( + "ocr_raster_pixels_limit", + compass_ocr::OCR_MAX_RASTER_PIXELS.to_string(), + ), + ( + "ocr_raster_edge_limit", + compass_ocr::OCR_MAX_RASTER_LONG_EDGE.to_string(), + ), + ( + "ocr_engine_side_limit", + compass_ocr::OCR_ENGINE_MAX_SIDE.to_string(), + ), + ( + "ocr_tile_overlap", + compass_ocr::OCR_TILE_OVERLAP.to_string(), + ), + ( + "ocr_aggregate_pixels_limit", + compass_media::OCR_MAX_AGGREGATE_PIXELS.to_string(), + ), + ( + "ocr_regions_raster_limit", + compass_ocr::OCR_MAX_OBSERVATIONS_PER_RASTER.to_string(), + ), + ( + "ocr_regions_document_limit", + compass_ocr::OCR_MAX_OBSERVATIONS_PER_DOCUMENT.to_string(), + ), + ( + "ocr_text_region_limit", + compass_ocr::OCR_MAX_TEXT_BYTES_PER_OBSERVATION.to_string(), + ), + ( + "ocr_text_document_limit", + compass_ocr::OCR_MAX_TEXT_CHARS_PER_DOCUMENT.to_string(), + ), + ( + "ocr_wall_time_seconds", + compass_ocr::OCR_MAX_DOCUMENT_WALL_TIME_SECS.to_string(), + ), + ] { + if profile.value(key) != Some(expected.as_str()) { + return Err(HistoryError::InvalidFingerprint(format!( + "persisted {key} is incompatible with {expected}" + ))); + } + } + for (key, expected) in [ + ( + "document_normalizer_version", + compass_media::DOCUMENT_NORMALIZER_VERSION, + ), + ("ocr_policy", compass_ocr::OCR_POLICY_VERSION), + ("ocr_preprocessing", compass_ocr::OCR_PREPROCESSING_VERSION), + ] { + if profile.value(key).and_then(|value| value.parse().ok()) != Some(expected) { + return Err(HistoryError::InvalidFingerprint(format!( + "persisted {key} is incompatible with {expected}" + ))); + } + } for (key, expected) in [ ("program_ir_schema", compass_ir::PROGRAM_SCHEMA_VERSION), ("program_merger_version", compass_program::MERGER_VERSION), @@ -410,6 +628,36 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError "persisted code_only is not boolean".to_owned(), )); } + if !matches!(profile.value("ocr_mode"), Some("off" | "auto" | "always")) { + return Err(HistoryError::InvalidFingerprint( + "persisted OCR mode is invalid".to_owned(), + )); + } + let ocr_profile = profile + .value("ocr_profile") + .ok_or_else(|| { + HistoryError::InvalidFingerprint("persisted OCR profile is missing".to_owned()) + })? + .parse::() + .map_err(|error| HistoryError::InvalidFingerprint(error.to_string()))?; + let expected_manifest = compass_ocr::profile_manifest_digest(ocr_profile); + if profile.value("ocr_model_manifest") != Some(expected_manifest.as_str()) { + return Err(HistoryError::InvalidFingerprint( + "persisted OCR model manifest does not match the pinned profile".to_owned(), + )); + } + let languages = profile + .entries() + .filter(|(key, _)| key.starts_with("ocr_language.")) + .map(|(_, language)| language.to_owned()) + .collect::>(); + let normalized = compass_ocr::normalize_language_hints(&languages) + .map_err(|error| HistoryError::InvalidFingerprint(error.to_string()))?; + if normalized != languages { + return Err(HistoryError::InvalidFingerprint( + "persisted OCR languages are not canonical".to_owned(), + )); + } let resolution = profile .value("resolution") .and_then(|value| value.parse::().ok()) @@ -596,6 +844,9 @@ struct HistoryBuildValues { deep: bool, cargo: bool, dedup_llm: bool, + ocr_mode: String, + ocr_profile: String, + ocr_languages: Vec, token_budget: Option, resolution: f64, exclude_hubs: Option, @@ -616,6 +867,9 @@ impl Default for HistoryBuildValues { deep: false, cargo: false, dedup_llm: false, + ocr_mode: "off".to_owned(), + ocr_profile: "pp-ocrv6-small".to_owned(), + ocr_languages: Vec::new(), token_budget: None, resolution: 1.0, exclude_hubs: None, @@ -680,8 +934,8 @@ pub(crate) fn parse_build_command( _ => unreachable!(), } } - "--backend" | "--model" | "--mode" | "--token-budget" | "--resolution" - | "--exclude-hubs" | "--format" | "--profile-from" => { + "--backend" | "--model" | "--mode" | "--ocr" | "--ocr-profile" | "--token-budget" + | "--resolution" | "--exclude-hubs" | "--format" | "--profile-from" => { if !seen.insert(name.to_owned()) { return Err(format!("duplicate {name}")); } @@ -691,6 +945,16 @@ pub(crate) fn parse_build_command( "--model" => values.model = Some(nonempty(name, value)?.to_owned()), "--mode" if value == "deep" => values.deep = true, "--mode" => return Err("--mode must be deep".to_owned()), + "--ocr" if matches!(value, "off" | "auto" | "always") => { + values.ocr_mode = value.to_owned() + } + "--ocr" => return Err("--ocr must be off, auto, or always".to_owned()), + "--ocr-profile" => { + value + .parse::() + .map_err(|error| error.to_string())?; + values.ocr_profile = value.to_owned(); + } "--token-budget" => values.token_budget = Some(positive_usize(name, value)?), "--resolution" => values.resolution = positive_float(name, value)?, "--exclude-hubs" => values.exclude_hubs = Some(finite_float(name, value)?), @@ -703,6 +967,13 @@ pub(crate) fn parse_build_command( let value = option_value(args, &mut index, name, inline)?; values.excludes.push(nonempty(name, value)?.to_owned()); } + "--ocr-language" => { + let value = nonempty(name, option_value(args, &mut index, name, inline)?)?; + if value.len() > 64 { + return Err("--ocr-language is too long".to_owned()); + } + values.ocr_languages.push(value.to_owned()); + } "--allow-partial" | "--no-cluster" => { return Err(format!( "{name} is incompatible with complete graph history" diff --git a/crates/compass-cli/src/history_commands.rs b/crates/compass-cli/src/history_commands.rs index 108b928d..8e5b54a8 100644 --- a/crates/compass-cli/src/history_commands.rs +++ b/crates/compass-cli/src/history_commands.rs @@ -22,7 +22,7 @@ use crate::{Frontend, Outcome}; pub(crate) fn help(_frontend: Frontend) -> String { let prefix = "compass"; let rendered = format!( - "Usage: {prefix} history \n\nCommands:\n enable [build-profile options]\n disable\n timeline [--rev REV] [--limit N [--after CURSOR]] --format json\n change-counts REV [--parent REV] --format json\n diff OLD NEW [--root NAME] [--output PATH] --format jsonl\n status [REV] [--format text|json]\n verify REV|REALIZATION [--format text|json]\n build REV [--all [--first-parent]] [build-profile options|--profile-from REV|REALIZATION] [--format text|json]\n rebuild REV [build-profile options] [--replace-corrupt] [--format text|json]\n list [REV] [--format text|json]\n show REALIZATION [--format text|json]\n prefer REV REALIZATION [--format text|json]\n export REV --format graph-json|json|compass-out [--community ID] [--node-limit N] --output PATH\n cache status [--format text|json]\n cache gc [--max-bytes N] [--max-age-days N] [--yes] [--format text|json]\n gc [--prune-non-preferred] [--yes] [--format text|json]\n\nExact diff roots:\n nodes, edges, hyperedges, analysis, metadata, program-facts, program-summaries\n\nBuild options:\n --all Build every commit reachable from REV\n --first-parent With --all, build only the first-parent lineage\n\nBuild-profile options:\n --code-only Build a complete local AST/inferred realization without model credentials\n --backend NAME Build a semantic realization with the selected provider\n --model NAME Select the provider model\n --exclude PATTERN Exclude a committed path pattern (repeatable)\n --cargo Include Cargo package metadata" + "Usage: {prefix} history \n\nCommands:\n enable [build-profile options]\n disable\n timeline [--rev REV] [--limit N [--after CURSOR]] --format json\n change-counts REV [--parent REV] --format json\n diff OLD NEW [--root NAME] [--output PATH] --format jsonl\n status [REV] [--format text|json]\n verify REV|REALIZATION [--format text|json]\n build REV [--all [--first-parent]] [build-profile options|--profile-from REV|REALIZATION] [--format text|json]\n rebuild REV [build-profile options] [--replace-corrupt] [--format text|json]\n list [REV] [--format text|json]\n show REALIZATION [--format text|json]\n prefer REV REALIZATION [--format text|json]\n export REV --format graph-json|json|compass-out [--community ID] [--node-limit N] --output PATH\n cache status [--format text|json]\n cache gc [--max-bytes N] [--max-age-days N] [--yes] [--format text|json]\n gc [--prune-non-preferred] [--yes] [--format text|json]\n\nExact diff roots:\n nodes, edges, hyperedges, analysis, metadata, program-facts, program-summaries\n\nBuild options:\n --all Build every commit reachable from REV\n --first-parent With --all, build only the first-parent lineage\n\nBuild-profile options:\n --code-only Build a complete local AST/inferred realization without model credentials\n --backend NAME Build a semantic realization with the selected provider\n --model NAME Select the provider model\n --ocr off|auto|always Select the pinned local document OCR policy\n --ocr-profile NAME Select the exact installed OCR model profile\n --ocr-language TAG Add a repeatable OCR language hint\n --exclude PATTERN Exclude a committed path pattern (repeatable)\n --cargo Include Cargo package metadata" ); rendered.replace( " timeline [--rev REV] [--limit N [--after CURSOR]] --format json\n", diff --git a/crates/compass-cli/src/lib.rs b/crates/compass-cli/src/lib.rs index 172a5605..469d6535 100644 --- a/crates/compass-cli/src/lib.rs +++ b/crates/compass-cli/src/lib.rs @@ -5,6 +5,7 @@ mod call_graph_commands; mod capability_commands; mod code_query_commands; mod dedup_commands; +mod document_commands; mod help; mod history_batch; mod history_build; @@ -16,6 +17,7 @@ mod init_commands; mod install_commands; mod integration_commands; mod label_commands; +mod models_commands; mod program_commands; mod provider_commands; mod prs_commands; @@ -45,12 +47,12 @@ use compass_analysis::{ }; use compass_core::{ BuildFileProgress, BuildOptions, BuildPurpose, BuildResult, BuildTimings, - ClusterExistingOptions, ExportInputs, GraphStorage, InferenceLevel, LoadedGraph, SemanticLayer, - WatchBackend, WatchBuildReason, WatchOptions, WatchStatus, build_graph_with_layers, - build_graph_with_layers_and_progress, build_graph_with_layers_and_tiebreaker, - cluster_existing_graph, default_graph_path, diagnose_graph_file, diagnose_graph_quality, - format_diagnostic_json, format_diagnostic_report, format_quality_json, format_quality_report, - merge_graphs, watch_local_graph, + ClusterExistingOptions, CoreDocumentProcessingOptions, ExportInputs, GraphStorage, + InferenceLevel, LoadedGraph, SemanticLayer, WatchBackend, WatchBuildReason, WatchOptions, + WatchStatus, build_graph_with_layers, build_graph_with_layers_and_progress, + build_graph_with_layers_and_tiebreaker, cluster_existing_graph, default_graph_path, + diagnose_graph_file, diagnose_graph_quality, format_diagnostic_json, format_diagnostic_report, + format_quality_json, format_quality_report, merge_graphs, watch_local_graph, }; use compass_files::{ BuildScope, DetectOptions, Detection, Manifest, ManifestKind, ProjectConfig, detect, @@ -84,9 +86,9 @@ use compass_query::{ render_discovery_text_page, render_explanation_page, render_shortest_path, run_benchmark, }; use compass_semantic::{ - CachedCorpusExtractionOptions, CorpusExtractionOptions, detect_backend_with_custom, - extract_builtin_corpus_cached, extract_custom_corpus_cached, load_custom_providers, - resolve_builtin_backend, resolve_custom_backend, + CachedCorpusExtractionOptions, CorpusExtractionOptions, PreparedDocumentInputs, + detect_backend_with_custom, extract_builtin_corpus_cached, extract_custom_corpus_cached, + load_custom_providers, resolve_builtin_backend, resolve_custom_backend, }; pub use help::HelpStyle; @@ -379,6 +381,8 @@ pub fn run(frontend: Frontend, arguments: impl IntoIterator) -> "context" => task_context_commands::command(&args), "history-worker" => history_commands::command_worker(frontend, &args), "diff" => semantic_diff_commands::command(frontend, &args), + "document" => document_commands::command(&args), + "models" => models_commands::command(&args), "query" => query_commands::command_query(frontend, &args), "program" => program_commands::command(frontend, &args), "path" => command_path(frontend, &args), @@ -1766,6 +1770,9 @@ fn command_build_with_validation_inner( let mut max_source_bytes = None; let mut api_timeout = None; let mut allow_partial = false; + let mut document_ocr_mode = compass_ocr::OcrMode::Off; + let mut document_ocr_profile = compass_ocr::ModelProfile::PpOcrV6Small; + let mut document_ocr_languages = Vec::new(); let mut timing = false; let mut dedup_llm = false; let mut excludes = Vec::new(); @@ -1801,6 +1808,53 @@ fn command_build_with_validation_inner( postgres_dsn = Some(value[11..].to_owned()); } "--allow-partial" if extract => allow_partial = true, + "--ocr" if extract && index + 1 < args.len() => { + document_ocr_mode = match args[index + 1].parse::() { + Ok(value) => value, + Err(error) => return extract_parse_failure(frontend, error.to_string()), + }; + index += 1; + } + value if extract && value.starts_with("--ocr=") => { + document_ocr_mode = match value[6..].parse::() { + Ok(value) => value, + Err(error) => return extract_parse_failure(frontend, error.to_string()), + }; + } + "--ocr-profile" if extract && index + 1 < args.len() => { + document_ocr_profile = match args[index + 1].parse::() { + Ok(value) => value, + Err(error) => return extract_parse_failure(frontend, error.to_string()), + }; + index += 1; + } + value if extract && value.starts_with("--ocr-profile=") => { + document_ocr_profile = match value[14..].parse::() { + Ok(value) => value, + Err(error) => return extract_parse_failure(frontend, error.to_string()), + }; + } + "--ocr-language" if extract && index + 1 < args.len() => { + let language = &args[index + 1]; + if language.is_empty() || language.len() > 64 { + return extract_parse_failure( + frontend, + "OCR language tag is empty or too long".to_owned(), + ); + } + document_ocr_languages.push(language.clone()); + index += 1; + } + value if extract && value.starts_with("--ocr-language=") => { + let language = &value[15..]; + if language.is_empty() || language.len() > 64 { + return extract_parse_failure( + frontend, + "OCR language tag is empty or too long".to_owned(), + ); + } + document_ocr_languages.push(language.to_owned()); + } "--backend" if extract && index + 1 < args.len() => { backend = Some(args[index + 1].clone()); index += 1; @@ -1974,14 +2028,14 @@ fn command_build_with_validation_inner( } exclude_hubs = Some(parsed); } - "--max-workers" if extract && index + 1 < args.len() => { + "--max-workers" if index + 1 < args.len() => { max_workers = match parse_positive_usize(&args[index + 1], "--max-workers") { Ok(value) => Some(value), Err(error) => return extract_parse_failure(frontend, error), }; index += 1; } - value if extract && value.starts_with("--max-workers=") => { + value if value.starts_with("--max-workers=") => { max_workers = match parse_positive_usize(&value[14..], "--max-workers") { Ok(value) => Some(value), Err(error) => return extract_parse_failure(frontend, error), @@ -2012,7 +2066,7 @@ fn command_build_with_validation_inner( return Outcome::success(if extract { extract_help() } else { - "Usage: compass update [path] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--max-source-bytes N] [--no-cluster] [--force] [--no-viz] [--timing]".to_owned() + "Usage: compass update [path] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--max-source-bytes N] [--max-workers N] [--no-cluster] [--force] [--no-viz] [--timing]".to_owned() }); } value if value.starts_with('-') => { @@ -2195,6 +2249,13 @@ fn command_build_with_validation_inner( max_concurrency, api_timeout, allow_partial, + CoreDocumentProcessingOptions { + ocr_mode: document_ocr_mode, + ocr_profile: document_ocr_profile, + language_hints: document_ocr_languages, + allow_partial, + cache_directory: Some(output_container.join("cache").join("documents")), + }, &auxiliary_fragments, dedup_tiebreaker .as_mut() @@ -2312,7 +2373,17 @@ fn command_build_with_validation_inner( } outcome } - Err(error) => Outcome::failure(format!("error: {error}")), + Err(error) => { + let mut message = format!("error: {error}"); + if document_ocr_mode != compass_ocr::OcrMode::Off + && error.contains("compass models install pp-ocrv6-") + { + message.push_str( + "\nhelp: Compass manages the OCR runtime; no system OCR package is required", + ); + } + Outcome::failure(message) + } } } @@ -2496,6 +2567,7 @@ fn build_semantic_graph( max_concurrency: Option, api_timeout: Option, allow_partial: bool, + document_processing: CoreDocumentProcessingOptions, auxiliary_fragments: &[serde_json::Value], tiebreaker: Option<&mut dyn compass_graph::EntityTiebreaker>, ) -> Result<(BuildResult, Vec, Duration), String> { @@ -2526,7 +2598,10 @@ fn build_semantic_graph( ) .map_err(|error| error.to_string())?; let live_semantic = semantic_files(&incremental.detection.files); - let semantic_files = if options.force || deep_mode { + let semantic_files = if options.force + || deep_mode + || document_processing.ocr_mode != compass_ocr::OcrMode::Off + { live_semantic.clone() } else { semantic_files(&incremental.new_files) @@ -2550,8 +2625,19 @@ fn build_semantic_graph( if let Some(max_concurrency) = max_concurrency { extraction_options.max_concurrency = max_concurrency; } + let prepared_documents = + compass_core::prepare_document_set(&semantic_files, &document_processing)?; + let prepared_inputs = PreparedDocumentInputs { + documents: prepared_documents + .documents + .iter() + .map(|(path, prepared)| (path.clone(), prepared.semantic_text.clone())) + .collect(), + cache_identity: prepared_documents.cache_identity.clone(), + }; let cached_options = CachedCorpusExtractionOptions { extraction: extraction_options, + prepared_documents: prepared_inputs, deep_mode, force: options.force, cache_enabled: true, @@ -2728,8 +2814,10 @@ fn build_semantic_graph( allow_partial, }; let semantic_elapsed = semantic_started.elapsed(); + let mut graph_options = options.clone(); + graph_options.prepared_documents = prepared_documents; let result = build_graph_with_optional_tiebreaker( - options, + &graph_options, Some(&layer), auxiliary_fragments, tiebreaker, @@ -2914,7 +3002,7 @@ fn executable_on_path(name: &str) -> bool { } fn extract_help() -> String { - "Usage: compass extract [PATH] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--code-only] [--cargo] [--google-workspace] [--postgres DSN] [--backend NAME] [--model MODEL] [--mode deep] [--token-budget N] [--max-concurrency N] [--max-workers N] [--max-source-bytes N] [--api-timeout SECONDS] [--allow-partial] [--dedup-llm] [--timing] [--out DIR] [--no-cluster] [--force] [--no-viz] [--no-gitignore] [--exclude PATTERN] [--resolution N] [--exclude-hubs N]".to_owned() + "Usage: compass extract [PATH] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--code-only] [--cargo] [--google-workspace] [--postgres DSN] [--backend NAME] [--model MODEL] [--mode deep] [--ocr off|auto|always] [--ocr-profile NAME] [--ocr-language BCP47] [--token-budget N] [--max-concurrency N] [--max-workers N] [--max-source-bytes N] [--api-timeout SECONDS] [--allow-partial] [--dedup-llm] [--timing] [--out DIR] [--no-cluster] [--force] [--no-viz] [--no-gitignore] [--exclude PATTERN] [--resolution N] [--exclude-hubs N]".to_owned() } fn saved_graph_root() -> Option { diff --git a/crates/compass-cli/src/models_commands.rs b/crates/compass-cli/src/models_commands.rs new file mode 100644 index 00000000..58d69f35 --- /dev/null +++ b/crates/compass-cli/src/models_commands.rs @@ -0,0 +1,97 @@ +use std::str::FromStr; + +use compass_ocr::{ModelProfile, install_profile, list_profiles, verify_profile}; + +use crate::Outcome; + +pub(crate) fn command(args: &[String]) -> Outcome { + match args.first().map(String::as_str) { + Some("list") => list(&args[1..]), + Some("install") => profile_command(&args[1..], true), + Some("verify") => profile_command(&args[1..], false), + Some(other) => usage_error(&format!("unknown models command {other:?}")), + None => usage_error("missing models command"), + } +} + +fn list(args: &[String]) -> Outcome { + let format = match args { + [] => "text", + [flag, value] if flag == "--format" && matches!(value.as_str(), "text" | "json") => value, + _ => return usage_error("models list accepts only --format text|json"), + }; + let statuses = match list_profiles() { + Ok(statuses) => statuses, + Err(error) => return Outcome::failure(format!("error: {error}")), + }; + if format == "json" { + return match serde_json::to_string_pretty(&serde_json::json!({ + "schema": "compass.models/1", + "profiles": statuses, + })) { + Ok(json) => Outcome::success(json), + Err(error) => { + Outcome::failure(format!("error: could not encode model status: {error}")) + } + }; + } + let output = statuses + .into_iter() + .map(|status| { + format!( + "{}\t{}\t{} MiB\t{}", + status.profile, + if status.verified { + "installed and verified" + } else if status.installed { + "installed but invalid" + } else { + "not installed" + }, + status.bytes / (1024 * 1024), + status.license + ) + }) + .collect::>() + .join("\n"); + Outcome::success(output) +} + +fn profile_command(args: &[String], install: bool) -> Outcome { + let [name] = args else { + return usage_error(if install { + "models install requires one pinned profile" + } else { + "models verify requires one pinned profile" + }); + }; + let profile = match ModelProfile::from_str(name) { + Ok(profile) => profile, + Err(error) => return usage_error(&error.to_string()), + }; + let result = if install { + install_profile(profile) + } else { + verify_profile(profile) + }; + match result { + Ok(files) => Outcome::success(format!( + "{} is installed and verified for {} {} ({} model artifacts)", + profile.name(), + files.identity.engine, + files.identity.engine_version, + files.identity.model_digests.len() + )), + Err(error) => Outcome::failure(format!("error: {error}")), + } +} + +fn usage_error(message: &str) -> Outcome { + Outcome::from_command_output( + 2, + String::new(), + format!( + "error: {message}\nusage: compass models [PROFILE] [--format text|json]" + ), + ) +} diff --git a/crates/compass-cli/tests/document_cli.rs b/crates/compass-cli/tests/document_cli.rs new file mode 100644 index 00000000..a92f7fa9 --- /dev/null +++ b/crates/compass-cli/tests/document_cli.rs @@ -0,0 +1,167 @@ +use std::error::Error; +use std::fs; +use std::io::Write as _; +use std::path::Path; +use std::process::Command; + +use serde_json::{Value, json}; +use zip::write::SimpleFileOptions; + +fn write_docx(path: &Path) -> Result<(), Box> { + let file = fs::File::create(path)?; + let mut archive = zip::ZipWriter::new(file); + archive.start_file("word/document.xml", SimpleFileOptions::default())?; + archive.write_all(br#"CLI document sentinel"#)?; + archive.finish()?; + Ok(()) +} + +#[test] +fn document_inspect_json_is_native_local_and_path_safe() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let document = directory.path().join("contract.docx"); + write_docx(&document)?; + let model_cache = directory.path().join("empty-model-cache"); + let output = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "document", + "inspect", + document.to_str().ok_or("non-UTF-8 test path")?, + "--format", + "json", + "--ocr-language", + "EN-us", + "--ocr-language", + "en-US", + "--allow-partial", + ]) + .env("COMPASS_CACHE_DIR", &model_cache) + .output()?; + assert!( + output.status.success(), + "document inspect failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let value: Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(value["schema"], "compass.document.inspect/1"); + assert_eq!(value["source"], "contract.docx"); + assert_eq!(value["processing"]["ocr_mode"], "off"); + assert_eq!(value["processing"]["allow_partial"], true); + assert_eq!(value["processing"]["language_hints"], json!(["en-us"])); + assert_eq!(value["artifact"]["schema"], "compass.document/1"); + assert!( + value["artifact"]["blocks"] + .as_array() + .is_some_and(|blocks| { + blocks + .iter() + .any(|block| block["text"] == "CLI document sentinel") + }) + ); + let serialized = String::from_utf8(output.stdout)?; + assert!(!serialized.contains(&directory.path().to_string_lossy().into_owned())); + assert!(!model_cache.exists()); + Ok(()) +} + +#[test] +fn explicit_ocr_missing_profile_has_one_actionable_command() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let document = directory.path().join("scan.docx"); + write_docx(&document)?; + let output = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "document", + "inspect", + document.to_str().ok_or("non-UTF-8 test path")?, + "--ocr", + "auto", + ]) + .env( + "COMPASS_CACHE_DIR", + directory.path().join("empty-model-cache"), + ) + .output()?; + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr)?; + let command = "compass models install pp-ocrv6-small"; + assert_eq!(stderr.matches(command).count(), 1, "{stderr}"); + assert!(stderr.contains("no system OCR package is required")); + Ok(()) +} + +#[test] +fn model_listing_and_verification_are_offline() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let cache = directory.path().join("models"); + let listed = Command::new(env!("CARGO_BIN_EXE_compass")) + .args(["models", "list", "--format", "json"]) + .env("COMPASS_CACHE_DIR", &cache) + .output()?; + assert!(listed.status.success()); + let value: Value = serde_json::from_slice(&listed.stdout)?; + assert_eq!(value["schema"], "compass.models/1"); + assert_eq!(value["profiles"].as_array().map(Vec::len), Some(2)); + let verified = Command::new(env!("CARGO_BIN_EXE_compass")) + .args(["models", "verify", "pp-ocrv6-small"]) + .env("COMPASS_CACHE_DIR", &cache) + .output()?; + assert!(!verified.status.success()); + assert!( + !cache.exists(), + "offline verification created the model cache" + ); + Ok(()) +} + +#[test] +fn extract_ocr_uses_the_same_managed_profile_contract() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + write_docx(&directory.path().join("report.docx"))?; + let output = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "extract", + directory.path().to_str().ok_or("non-UTF-8 test path")?, + "--ocr", + "auto", + "--no-viz", + "--no-cluster", + ]) + .env( + "COMPASS_CACHE_DIR", + directory.path().join("empty-model-cache"), + ) + .output()?; + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr)?; + assert_eq!( + stderr + .matches("compass models install pp-ocrv6-small") + .count(), + 1 + ); + assert!(stderr.contains("no system OCR package is required")); + Ok(()) +} + +#[test] +fn document_inspect_rejects_non_files_without_publishing_output() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let fake_document = directory.path().join("directory.docx"); + fs::create_dir(&fake_document)?; + let output = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "document", + "inspect", + fake_document.to_str().ok_or("non-UTF-8 test path")?, + "--format", + "json", + ]) + .output()?; + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + assert!(String::from_utf8(output.stderr)?.contains("not a regular file")); + Ok(()) +} diff --git a/crates/compass-core/Cargo.toml b/crates/compass-core/Cargo.toml index f808a652..578cb017 100644 --- a/crates/compass-core/Cargo.toml +++ b/crates/compass-core/Cargo.toml @@ -28,6 +28,8 @@ thiserror.workspace = true compass-files = { path = "../compass-files", version = "0.3.19" } compass-google-workspace = { path = "../compass-google-workspace", version = "0.3.19" } compass-languages = { path = "../compass-languages", version = "0.3.19" } +compass-media = { path = "../compass-media", version = "0.3.19" } +compass-ocr = { path = "../compass-ocr", version = "0.3.19" } compass-model = { path = "../compass-model", version = "0.3.19" } compass-graph = { path = "../compass-graph", version = "0.3.19" } compass-history = { path = "../compass-history", version = "0.3.19" } @@ -47,3 +49,4 @@ workspace = true protobuf.workspace = true scip.workspace = true tempfile.workspace = true +zip.workspace = true diff --git a/crates/compass-core/src/build_state.rs b/crates/compass-core/src/build_state.rs index 4a25b538..23598a81 100644 --- a/crates/compass-core/src/build_state.rs +++ b/crates/compass-core/src/build_state.rs @@ -102,6 +102,8 @@ pub(crate) struct BuildProfile { pub inference_level: String, #[serde(default = "default_max_source_bytes")] pub max_source_bytes: u64, + #[serde(default = "default_document_processing_identity")] + pub document_processing_identity: String, } // Build-state schema 1 omitted the historical max profile. Keep interpreting @@ -120,6 +122,10 @@ const fn default_max_source_bytes() -> u64 { crate::pipeline::DEFAULT_MAX_SOURCE_BYTES } +fn default_document_processing_identity() -> String { + crate::PreparedDocumentSet::default().cache_identity +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct SavedStats { pub files: usize, @@ -297,6 +303,7 @@ mod tests { graph_storage: "json".to_owned(), inference_level: legacy_default_inference_level(), max_source_bytes: default_max_source_bytes(), + document_processing_identity: default_document_processing_identity(), }; let document = serde_json::to_value(&profile)?; assert!(document.get("inference_level").is_none()); @@ -331,6 +338,7 @@ mod tests { graph_storage: "json".to_owned(), inference_level: legacy_default_inference_level(), max_source_bytes: default_max_source_bytes(), + document_processing_identity: default_document_processing_identity(), }; let state = BuildState::capture( output, diff --git a/crates/compass-core/src/document_processing.rs b/crates/compass-core/src/document_processing.rs new file mode 100644 index 00000000..2c1c5d24 --- /dev/null +++ b/crates/compass-core/src/document_processing.rs @@ -0,0 +1,505 @@ +//! Application-owned preparation of rich documents for structural and +//! semantic consumers. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use compass_media::document::DocumentArtifact; +use compass_ocr::{ + ManagedOarEngine, ModelProfile, OCR_POLICY_VERSION, OCR_PREPROCESSING_VERSION, + OCR_PROTOCOL_SCHEMA, OCR_SCHEMA, OcrEngine, OcrMode, normalize_language_hints, + profile_manifest_digest, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const MAX_CACHED_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CoreDocumentProcessingOptions { + pub ocr_mode: OcrMode, + pub ocr_profile: ModelProfile, + pub language_hints: Vec, + pub allow_partial: bool, + pub cache_directory: Option, +} + +impl Default for CoreDocumentProcessingOptions { + fn default() -> Self { + Self { + ocr_mode: OcrMode::Off, + ocr_profile: ModelProfile::PpOcrV6Small, + language_hints: Vec::new(), + allow_partial: false, + cache_directory: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct PreparedDocument { + pub artifact: Arc, + pub semantic_text: Arc, + pub cache_identity: String, +} + +#[derive(Clone, Debug)] +pub struct PreparedDocumentSet { + pub documents: BTreeMap, + pub cache_identity: String, +} + +impl Default for PreparedDocumentSet { + fn default() -> Self { + Self { + documents: BTreeMap::new(), + cache_identity: cache_identity(&CoreDocumentProcessingOptions::default()), + } + } +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CachedDocumentArtifact { + schema: String, + source_digest: String, + cache_identity: String, + artifact: DocumentArtifact, +} + +impl PreparedDocumentSet { + #[must_use] + pub fn get(&self, path: &Path) -> Option<&PreparedDocument> { + self.documents.get(path) + } +} + +pub fn prepare_document_set( + paths: &[PathBuf], + options: &CoreDocumentProcessingOptions, +) -> Result { + let mut rich_paths = paths + .iter() + .filter(|path| is_rich_document(path)) + .cloned() + .collect::>(); + rich_paths.sort(); + rich_paths.dedup(); + let mut engine = None; + let language_hints = + normalize_language_hints(&options.language_hints).map_err(|error| error.to_string())?; + let mut canonical_options = options.clone(); + canonical_options.language_hints.clone_from(&language_hints); + let processing = compass_media::DocumentProcessingOptions { + ocr_mode: options.ocr_mode, + language_hints, + allow_partial: options.allow_partial, + }; + let cache_identity = cache_identity(&canonical_options); + let mut documents = BTreeMap::new(); + for path in rich_paths { + let bytes = compass_media::read_document_bounded(&path) + .map_err(|error| format!("{}: {error}", path.display()))?; + let source_digest = format!("sha256:{:x}", Sha256::digest(&bytes)); + let cache_path = options + .cache_directory + .as_deref() + .map(|directory| document_cache_path(directory, &source_digest, &cache_identity)); + let artifact = match cache_path.as_deref() { + Some(cache_path) if cache_path.is_file() => { + load_cached_document(cache_path, &source_digest, &cache_identity)? + } + _ => { + if options.ocr_mode != OcrMode::Off && engine.is_none() { + engine = Some( + ManagedOarEngine::load(options.ocr_profile) + .map_err(|error| error.to_string())?, + ); + } + let artifact = compass_media::decode_document_with_ocr( + &path, + &bytes, + &processing, + engine.as_ref().map(|engine| engine as &dyn OcrEngine), + ) + .map_err(|error| format!("{}: {error}", path.display()))?; + if artifact.complete + && !matches!( + artifact.visual_coverage, + compass_media::document::VisualCoverage::Partial + | compass_media::document::VisualCoverage::Failed + ) + && let Some(cache_path) = cache_path.as_deref() + { + let cached = CachedDocumentArtifact { + schema: "compass.document.cache/1".to_owned(), + source_digest: source_digest.clone(), + cache_identity: cache_identity.clone(), + artifact: artifact.clone(), + }; + compass_files::write_json_atomic(cache_path, &cached, false) + .map_err(|error| error.to_string())?; + } + artifact + } + }; + let semantic_text = compass_media::render_document_markdown(&artifact) + .map_err(|error| format!("{}: {error}", path.display()))?; + documents.insert( + path, + PreparedDocument { + artifact: Arc::new(artifact), + semantic_text: Arc::::from(semantic_text), + cache_identity: cache_identity.clone(), + }, + ); + } + Ok(PreparedDocumentSet { + documents, + cache_identity, + }) +} + +fn document_cache_path(directory: &Path, source_digest: &str, cache_identity: &str) -> PathBuf { + let key = format!( + "{:x}", + Sha256::digest(format!("{source_digest}\0{cache_identity}")) + ); + directory.join(&key[..2]).join(format!("{key}.json")) +} + +fn load_cached_document( + path: &Path, + source_digest: &str, + cache_identity: &str, +) -> Result { + let bytes = + compass_files::read_bytes_bounded(path, MAX_CACHED_DOCUMENT_BYTES).map_err(|error| { + format!( + "could not read bounded document cache {}: {error}", + path.display() + ) + })?; + let cached: CachedDocumentArtifact = serde_json::from_slice(&bytes) + .map_err(|error| format!("document cache {} is corrupt: {error}", path.display()))?; + if cached.schema != "compass.document.cache/1" + || cached.source_digest != source_digest + || cached.cache_identity != cache_identity + { + return Err(format!( + "document cache {} has an incompatible identity", + path.display() + )); + } + cached + .artifact + .validate() + .map_err(|error| format!("document cache {} is invalid: {error}", path.display()))?; + Ok(cached.artifact) +} + +fn cache_identity(options: &CoreDocumentProcessingOptions) -> String { + let mut languages = options.language_hints.clone(); + languages.sort(); + languages.dedup(); + let mode = match options.ocr_mode { + OcrMode::Off => "off", + OcrMode::Auto => "auto", + OcrMode::Always => "always", + }; + format!( + "schema={};normalizer={};ocr_schema={};ocr_protocol={};ocr_policy={};preprocess={};rasterizer={};mode={};profile={};model_manifest={};languages={};raw_bytes={};pdf_pages={};office_images={};raster_pixels={};raster_edge={};engine_side={};tile_overlap={};aggregate_pixels={};regions_raster={};regions_document={};text_region={};text_document={};wall_time_seconds={}", + compass_media::DOCUMENT_SCHEMA, + compass_media::DOCUMENT_NORMALIZER_VERSION, + OCR_SCHEMA, + OCR_PROTOCOL_SCHEMA, + OCR_POLICY_VERSION, + OCR_PREPROCESSING_VERSION, + compass_media::PDF_RASTERIZER_IDENTITY, + mode, + options.ocr_profile.name(), + profile_manifest_digest(options.ocr_profile), + languages.join(","), + compass_media::MEDIA_MAX_RAW_BYTES, + compass_media::OCR_MAX_PDF_PAGES, + compass_media::OCR_MAX_OOXML_IMAGES, + compass_ocr::OCR_MAX_RASTER_PIXELS, + compass_ocr::OCR_MAX_RASTER_LONG_EDGE, + compass_ocr::OCR_ENGINE_MAX_SIDE, + compass_ocr::OCR_TILE_OVERLAP, + compass_media::OCR_MAX_AGGREGATE_PIXELS, + compass_ocr::OCR_MAX_OBSERVATIONS_PER_RASTER, + compass_ocr::OCR_MAX_OBSERVATIONS_PER_DOCUMENT, + compass_ocr::OCR_MAX_TEXT_BYTES_PER_OBSERVATION, + compass_ocr::OCR_MAX_TEXT_CHARS_PER_DOCUMENT, + compass_ocr::OCR_MAX_DOCUMENT_WALL_TIME_SECS, + ) +} + +fn is_rich_document(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + ["pdf", "docx", "xlsx", "pptx"] + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }) +} + +pub(crate) fn project_document( + source_file: &str, + path: &Path, + artifact: &DocumentArtifact, + cache_identity: &str, +) -> Result { + artifact.validate().map_err(|error| error.to_string())?; + let file_id = compass_languages::make_id(&[source_file]); + let mut extraction = compass_languages::Extraction { + raw_calls: None, + ..compass_languages::Extraction::default() + }; + let mut root = serde_json::Map::new(); + root.insert( + "label".to_owned(), + serde_json::Value::String( + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or(source_file) + .to_owned(), + ), + ); + root.insert("file_type".to_owned(), serde_json::json!("document")); + root.insert("document_kind".to_owned(), serde_json::json!("document")); + root.insert( + "document_format".to_owned(), + serde_json::to_value(artifact.format).map_err(|error| error.to_string())?, + ); + root.insert("source_file".to_owned(), serde_json::json!(source_file)); + root.insert("source_location".to_owned(), serde_json::json!("document")); + root.insert("_origin".to_owned(), serde_json::json!("artifact")); + root.insert( + "document_schema".to_owned(), + serde_json::json!(artifact.schema), + ); + root.insert( + "document_normalizer_version".to_owned(), + serde_json::json!(artifact.normalizer_version), + ); + root.insert( + "document_complete".to_owned(), + serde_json::json!(artifact.complete), + ); + root.insert( + "document_visual_coverage".to_owned(), + serde_json::to_value(artifact.visual_coverage).map_err(|error| error.to_string())?, + ); + if !artifact.metadata.is_empty() { + root.insert( + "document_metadata".to_owned(), + serde_json::to_value(&artifact.metadata).map_err(|error| error.to_string())?, + ); + } + if let Some(profile) = &artifact.ocr_profile { + root.insert( + "document_ocr_profile".to_owned(), + serde_json::to_value(profile).map_err(|error| error.to_string())?, + ); + } + extraction.nodes.push(compass_languages::RawNodeRecord { + id: file_id.clone(), + attributes: root, + }); + let mut block_ids = BTreeMap::new(); + for block in &artifact.blocks { + let ordinal = block.ordinal.to_string(); + let id = compass_languages::make_id(&[source_file, "document-block", &ordinal]); + block_ids.insert(block.ordinal, id.clone()); + let locator = serde_json::to_value(&block.locator).map_err(|error| error.to_string())?; + let location = serde_json::to_string(&locator).map_err(|error| error.to_string())?; + let kind = serde_json::to_value(&block.kind).map_err(|error| error.to_string())?; + let kind_name = kind + .get("kind") + .and_then(serde_json::Value::as_str) + .unwrap_or("other"); + let mut attributes = serde_json::Map::new(); + attributes.insert( + "label".to_owned(), + serde_json::json!(bounded_label(&block.text)), + ); + attributes.insert("file_type".to_owned(), serde_json::json!("document")); + attributes.insert("document_kind".to_owned(), serde_json::json!(kind_name)); + attributes.insert("source_file".to_owned(), serde_json::json!(source_file)); + attributes.insert("source_location".to_owned(), serde_json::json!(location)); + attributes.insert("document_locator".to_owned(), locator); + attributes.insert("block_index".to_owned(), serde_json::json!(block.ordinal)); + attributes.insert("_origin".to_owned(), serde_json::json!("artifact")); + attributes.insert("document_text".to_owned(), serde_json::json!(block.text)); + attributes.insert( + "document_origin".to_owned(), + serde_json::to_value(&block.origin).map_err(|error| error.to_string())?, + ); + if !block.metadata.is_empty() { + attributes.insert( + "document_metadata".to_owned(), + serde_json::to_value(&block.metadata).map_err(|error| error.to_string())?, + ); + } + extraction + .nodes + .push(compass_languages::RawNodeRecord { id, attributes }); + } + for block in &artifact.blocks { + let Some(target) = block_ids.get(&block.ordinal) else { + return Err("prepared document block identity disappeared".to_owned()); + }; + let source = match block.parent { + Some(parent) => block_ids + .get(&parent) + .ok_or_else(|| "prepared document parent identity disappeared".to_owned())?, + None => &file_id, + }; + let mut attributes = serde_json::Map::new(); + attributes.insert("relation".to_owned(), serde_json::json!("contains")); + attributes.insert("confidence".to_owned(), serde_json::json!("EXTRACTED")); + attributes.insert("source_file".to_owned(), serde_json::json!(source_file)); + attributes.insert("source_location".to_owned(), serde_json::json!("document")); + attributes.insert("_origin".to_owned(), serde_json::json!("artifact")); + attributes.insert("weight".to_owned(), serde_json::json!(1.0)); + extraction.edges.push(compass_languages::RawEdgeRecord { + source: source.clone(), + target: target.clone(), + attributes, + }); + } + if !artifact.links.is_empty() { + extraction.extensions.insert( + "document_links".to_owned(), + serde_json::to_value(&artifact.links).map_err(|error| error.to_string())?, + ); + } + if !artifact.diagnostics.is_empty() { + extraction.extensions.insert( + "document_diagnostics".to_owned(), + serde_json::to_value(&artifact.diagnostics).map_err(|error| error.to_string())?, + ); + } + extraction.extensions.insert( + "document_cache_identity".to_owned(), + serde_json::json!(cache_identity), + ); + Ok(extraction) +} + +fn bounded_label(text: &str) -> String { + let mut output = text.chars().take(512).collect::(); + if text.chars().count() > 512 { + output.push('…'); + } + output +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::io::Write as _; + + use super::*; + + #[test] + fn prepared_document_cache_is_reused_and_corruption_is_rejected() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let document = directory.path().join("report.docx"); + let file = fs::File::create(&document)?; + let mut archive = zip::ZipWriter::new(file); + archive.start_file( + "word/document.xml", + zip::write::SimpleFileOptions::default(), + )?; + archive.write_all(br#"Sentinel text"#)?; + archive.finish()?; + let options = CoreDocumentProcessingOptions { + cache_directory: Some(directory.path().join("cache")), + ..CoreDocumentProcessingOptions::default() + }; + let first = prepare_document_set(std::slice::from_ref(&document), &options)?; + let prepared = first.get(&document).ok_or("prepared document is missing")?; + assert!(prepared.semantic_text.contains("Sentinel text")); + let source_digest = format!("sha256:{:x}", Sha256::digest(fs::read(&document)?)); + let cache_path = document_cache_path( + options + .cache_directory + .as_deref() + .ok_or("cache directory missing")?, + &source_digest, + &first.cache_identity, + ); + assert!(cache_path.is_file()); + let second = prepare_document_set(std::slice::from_ref(&document), &options)?; + assert_eq!( + first + .get(&document) + .map(|value| value.semantic_text.as_ref()), + second + .get(&document) + .map(|value| value.semantic_text.as_ref()) + ); + let mut unknown: serde_json::Value = serde_json::from_slice(&fs::read(&cache_path)?)?; + unknown["unexpected"] = serde_json::json!(true); + fs::write(&cache_path, serde_json::to_vec(&unknown)?)?; + let error = prepare_document_set(std::slice::from_ref(&document), &options) + .err() + .ok_or("cache with an unknown field was accepted")?; + assert!(error.contains("corrupt")); + + fs::write(&cache_path, b"{")?; + let error = prepare_document_set(std::slice::from_ref(&document), &options) + .err() + .ok_or("corrupt cache was accepted")?; + assert!(error.contains("corrupt")); + + let oversized = fs::File::create(&cache_path)?; + oversized.set_len(MAX_CACHED_DOCUMENT_BYTES + 1)?; + let error = prepare_document_set(std::slice::from_ref(&document), &options) + .err() + .ok_or("oversized cache was accepted")?; + assert!(error.contains("bounded document cache")); + Ok(()) + } + + #[test] + fn projection_retains_document_blocks_and_typed_locators() + -> Result<(), Box> { + let mut artifact = DocumentArtifact::new(compass_media::document::DocumentFormat::Docx); + artifact.push_block( + None, + compass_media::document::DocumentBlockKind::Paragraph, + "Projected sentinel".to_owned(), + compass_media::document::DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: "body/p[1]".to_owned(), + }, + )?; + let extraction = project_document( + "docs/report.docx", + Path::new("report.docx"), + &artifact, + "fixture-identity", + )?; + assert_eq!(extraction.nodes.len(), 2); + assert_eq!(extraction.edges.len(), 1); + assert_eq!( + extraction.nodes[1] + .attributes + .get("document_text") + .and_then(serde_json::Value::as_str), + Some("Projected sentinel") + ); + assert_eq!( + extraction.extensions.get("document_cache_identity"), + Some(&serde_json::json!("fixture-identity")) + ); + Ok(()) + } +} diff --git a/crates/compass-core/src/lib.rs b/crates/compass-core/src/lib.rs index b44c19f0..70bc7a6f 100644 --- a/crates/compass-core/src/lib.rs +++ b/crates/compass-core/src/lib.rs @@ -4,6 +4,7 @@ mod agent_graph; mod build_state; mod cluster_existing; mod diagnostics; +mod document_processing; mod history; mod merge; mod pipeline; @@ -28,6 +29,9 @@ pub use diagnostics::{ diagnose_graph_file, diagnose_graph_quality, format_diagnostic_json, format_diagnostic_report, format_quality_json, format_quality_report, }; +pub use document_processing::{ + CoreDocumentProcessingOptions, PreparedDocument, PreparedDocumentSet, prepare_document_set, +}; pub use history::{ CompleteGraphBuilder, MaterializeError, MaterializeObserver, MaterializeRequest, MaterializeStage, history_provider_manifest, materialize_history, diff --git a/crates/compass-core/src/pipeline.rs b/crates/compass-core/src/pipeline.rs index df692671..4c2d4caa 100644 --- a/crates/compass-core/src/pipeline.rs +++ b/crates/compass-core/src/pipeline.rs @@ -67,6 +67,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; +use crate::PreparedDocumentSet; use crate::build_state::{ ArtifactSeal, BUILD_STATE_FILE, BuildProfile, BuildState, SavedStats, load_verified, }; @@ -143,6 +144,9 @@ pub struct BuildOptions { /// keeping it in the build profile prevents a document-inclusive output /// from being reused for a code-only build. pub code_only: bool, + /// Rich documents prepared once at the application boundary. The same + /// validated artifacts feed structural publication and semantic slicing. + pub prepared_documents: PreparedDocumentSet, /// Enable deterministic Program IR analysis and `program.json` output. pub program_analysis: bool, /// Explicit offline program evidence artifacts, in addition to `index.scip`. @@ -382,6 +386,7 @@ impl BuildOptions { exclude_hubs: None, google_workspace: false, code_only: false, + prepared_documents: PreparedDocumentSet::default(), program_analysis: false, program_artifacts: Vec::new(), program_artifact_limits: compass_program::ArtifactLimits::default(), @@ -2196,6 +2201,8 @@ pub enum CoreError { InvalidSemanticFragment(serde_json::Error), #[error("invalid supplemental extraction fragment: {0}")] InvalidSupplementalFragment(serde_json::Error), + #[error("document processing failed: {0}")] + DocumentProcessing(String), #[error("could not create an AST worker pool: {0}")] WorkerPool(String), #[error("build worker panicked during {0}")] @@ -2455,13 +2462,24 @@ fn build_graph_inner_unscoped( let mut internal_started = Instant::now(); let preserve_prior_semantic = prior_semantic_layer_required(read_prior_published_graph, &output_dir); - let mut semantic_documents = if preserve_prior_semantic { - semantic_document_sources(&output_dir.join("graph.json"), &root) - } else { - HashSet::new() - }; - if let Some(layer) = semantic { - semantic_documents.extend(canonical_source_set(&layer.refreshed_files, &root)); + let mut prepared_documents = options.prepared_documents.clone(); + if !options.code_only { + let document_paths = detection + .files + .get("document") + .into_iter() + .flatten() + .map(PathBuf::from) + .filter(|path| { + Registry::resolve(path).is_none() && prepared_documents.get(path).is_none() + }) + .collect::>(); + let native = crate::prepare_document_set( + &document_paths, + &crate::CoreDocumentProcessingOptions::default(), + ) + .map_err(CoreError::DocumentProcessing)?; + prepared_documents.documents.extend(native.documents); } let mut sources = detection .files @@ -2480,12 +2498,7 @@ fn build_graph_inner_unscoped( .flatten() .map(PathBuf::from) .filter(|path| { - let structural_document = Registry::resolve(path).is_some_and(|spec| { - matches!(spec.kind, ExtractorKind::Markdown | ExtractorKind::Html) - }); - Registry::resolve(path).is_some() - && (structural_document - || !semantic_documents.contains(&canonical_identity(path))) + Registry::resolve(path).is_some() || prepared_documents.get(path).is_some() }), ); } @@ -2761,9 +2774,11 @@ fn build_graph_inner_unscoped( let mut missing = Vec::new(); if reuse_cached_analysis { for path in &sources { - if fs::metadata(path).is_ok_and(|metadata| { - metadata.is_file() && metadata.len() > options.max_source_bytes - }) { + if prepared_documents.get(path).is_none() + && fs::metadata(path).is_ok_and(|metadata| { + metadata.is_file() && metadata.len() > options.max_source_bytes + }) + { extractions.insert( path.clone(), oversized_source_extraction(path, options.max_source_bytes)?, @@ -2781,6 +2796,13 @@ fn build_graph_inner_unscoped( if let Some(extraction) = cached { if cached_framework_evidence_matches(&extraction, path, &project_evidence) && cached_universal_evidence_matches(&extraction, path) + && prepared_documents.get(path).is_none_or(|prepared| { + extraction + .extensions + .get("document_cache_identity") + .and_then(Value::as_str) + == Some(prepared.cache_identity.as_str()) + }) { extractions.insert(path.clone(), extraction); } else { @@ -2792,9 +2814,11 @@ fn build_graph_inner_unscoped( } } else { for path in &sources { - if fs::metadata(path).is_ok_and(|metadata| { - metadata.is_file() && metadata.len() > options.max_source_bytes - }) { + if prepared_documents.get(path).is_none() + && fs::metadata(path).is_ok_and(|metadata| { + metadata.is_file() && metadata.len() > options.max_source_bytes + }) + { extractions.insert( path.clone(), oversized_source_extraction(path, options.max_source_bytes)?, @@ -2834,7 +2858,7 @@ fn build_graph_inner_unscoped( path: path.clone(), source, })?; - if metadata.len() > options.max_source_bytes { + if prepared_documents.get(path).is_none() && metadata.len() > options.max_source_bytes { let graph = oversized_source_extraction(path, options.max_source_bytes)?; if let Some(progress) = progress { let mut completed = completed_files @@ -2870,8 +2894,29 @@ fn build_graph_inner_unscoped( .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); - let language = Registry::resolve(path).map_or("", |spec| spec.name); - let (mut graph, program) = if options.program_analysis { + let prepared_document = prepared_documents.get(path); + let language = if prepared_document.is_some() { + "document" + } else { + Registry::resolve(path).map_or("", |spec| spec.name) + }; + let (mut graph, program) = if let Some(prepared) = prepared_document { + ( + crate::document_processing::project_document( + &source_file, + path, + &prepared.artifact, + &prepared.cache_identity, + ) + .map_err(|detail| { + compass_languages::ExtractError::InvalidDocumentEvidence { + path: path.clone(), + detail, + } + })?, + None, + ) + } else if options.program_analysis { let combined = engine.extract_source_combined(path, &source_file, &bytes)?; (combined.graph, combined.program) } else { @@ -4383,6 +4428,7 @@ fn build_profile(options: &BuildOptions) -> BuildProfile { .to_owned(), inference_level: options.inference_level.as_str().to_owned(), max_source_bytes: options.max_source_bytes, + document_processing_identity: options.prepared_documents.cache_identity.clone(), } } @@ -6614,26 +6660,6 @@ fn save_build_manifest( Ok(()) } -fn semantic_document_sources(graph_path: &Path, root: &Path) -> HashSet { - let Ok(existing) = V1GraphDocument::load(graph_path) else { - return HashSet::new(); - }; - existing - .nodes - .into_iter() - .filter(|node| node_has_semantic_layer_evidence(node) && node.kind == NodeKind::Resource) - .filter_map(|node| { - node.source_file().map(Path::new).map(|path| { - if path.is_absolute() { - canonical_identity(path) - } else { - canonical_identity(&root.join(path)) - } - }) - }) - .collect() -} - fn canonical_identity(path: &Path) -> PathBuf { fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) } @@ -7368,11 +7394,13 @@ fn extraction_has_cacheable_ast_facts(extraction: &Extraction) -> bool { #[cfg(test)] mod tests { use std::error::Error; + use std::io::Write as _; use compass_graph::{GraphSnapshotReader, IndexKind}; use compass_model::code_graph::GraphDocument as V1GraphDocument; use compass_model::provenance::{EvidenceConfidence, effective_confidence}; use serde_json::{Map, Value}; + use zip::write::SimpleFileOptions; use super::*; @@ -9211,6 +9239,40 @@ char* Arena::AllocateAligned(size_t bytes) { return Allocate(bytes); } Ok(()) } + #[test] + fn document_native_docx_blocks_are_published_without_semantic_or_ocr_dependencies() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("report.docx"); + let mut archive = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + archive.start_file("word/document.xml", SimpleFileOptions::default())?; + archive.write_all(br#"Document sentinel"#)?; + fs::write(&path, archive.finish()?.into_inner())?; + + let mut options = BuildOptions::new(directory.path()); + options.no_cluster = true; + options.no_viz = true; + let result = build_local_graph(&options)?; + let graph = V1GraphDocument::load(&result.output_dir.join("graph.json"))?; + let document_nodes = graph + .nodes + .iter() + .filter(|node| node.source_file() == Some("report.docx")) + .collect::>(); + assert!(document_nodes.len() >= 2); + assert!( + document_nodes + .iter() + .any(|node| node.name == "Document sentinel") + ); + assert!(graph.links.iter().any(|edge| { + edge.source_file() == Some("report.docx") + && document_nodes.iter().any(|node| node.id == edge.source) + && document_nodes.iter().any(|node| node.id == edge.target) + })); + Ok(()) + } + #[test] fn unsupported_extensionless_shebang_is_skipped() -> Result<(), Box> { let directory = tempfile::tempdir()?; diff --git a/crates/compass-files/src/build_guard.rs b/crates/compass-files/src/build_guard.rs index b4d97075..1de17cea 100644 --- a/crates/compass-files/src/build_guard.rs +++ b/crates/compass-files/src/build_guard.rs @@ -431,15 +431,13 @@ fn copy_snapshot( fs::create_dir(&to).map_err(|error| io_error(&to, error))?; copy_snapshot(&from, &to, excluded_artifacts, false)?; } else if file_type.is_file() { - // Snapshot files are immutable after publication and Compass - // replaces mutable artifacts through the atomic writers. A - // same-filesystem hard link therefore gives the staging snapshot - // copy-on-write behavior without reading large graphs again. If - // links are unavailable (for example across filesystems or on a - // restricted volume), retain the portable copy fallback. - if fs::hard_link(&from, &to).is_err() { - fs::copy(&from, &to).map_err(|error| io_error(&to, error))?; - } + // A staging snapshot is writable by the build pipeline. Hard + // linking a published artifact would make an ordinary write to + // the staging path mutate the active snapshot as well, violating + // the one-complete-snapshot publication contract. Copy the bytes + // instead; callers that need large sidecars can explicitly use + // the excluded-artifact path and its immutable reference. + fs::copy(&from, &to).map_err(|error| io_error(&to, error))?; } } Ok(()) diff --git a/crates/compass-files/src/cache.rs b/crates/compass-files/src/cache.rs index c21690f9..0cc44422 100644 --- a/crates/compass-files/src/cache.rs +++ b/crates/compass-files/src/cache.rs @@ -13,7 +13,7 @@ use sha2::{Digest, Sha256}; use crate::{FileError, StatHashIndex, file_hash, io_error, write_bytes_atomic, write_json_atomic}; /// Changes whenever cached extraction semantics change, even if the wire encoding does not. -pub const AST_CACHE_VERSION: &str = "1"; +pub const AST_CACHE_VERSION: &str = "2"; /// Portable cache encoding version used in the on-disk namespace. pub const CACHE_ENCODING_VERSION: u32 = 1; const MESSAGEPACK_EXTENSION: &str = "msgpack"; diff --git a/crates/compass-files/src/detect.rs b/crates/compass-files/src/detect.rs index a974544c..8353b5cb 100644 --- a/crates/compass-files/src/detect.rs +++ b/crates/compass-files/src/detect.rs @@ -25,7 +25,7 @@ const CODE_EXTENSIONS: &[&str] = &[ ]; const DOCUMENT_EXTENSIONS: &[&str] = &[ "md", "markdown", "mdx", "qmd", "skill", "txt", "rst", "html", "htm", "yaml", "yml", "docx", - "xlsx", "gdoc", "gsheet", "gslides", + "xlsx", "pptx", "gdoc", "gsheet", "gslides", ]; const IMAGE_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg"]; const VIDEO_EXTENSIONS: &[&str] = &[ @@ -807,7 +807,7 @@ fn sensitive(path: &Path) -> bool { } fn count_words(path: &Path) -> u64 { - if matches!(extension(path).as_str(), "pdf" | "docx" | "xlsx") { + if matches!(extension(path).as_str(), "pdf" | "docx" | "xlsx" | "pptx") { return 0; } fs::read(path).map_or(0, |bytes| { @@ -1112,6 +1112,19 @@ fn collect_memory_files(directory: &Path, files: &mut Vec, errors: &mut } } +#[cfg(test)] +mod tests { + use super::{FileType, classify_file}; + use std::path::Path; + + #[test] + fn office_documents_are_discovered_as_documents() { + for name in ["report.docx", "workbook.xlsx", "deck.pptx"] { + assert_eq!(classify_file(Path::new(name)), Some(FileType::Document)); + } + } +} + fn grouped_number(value: u64) -> String { let digits = value.to_string(); let mut output = String::with_capacity(digits.len() + digits.len() / 3); diff --git a/crates/compass-files/src/encoding.rs b/crates/compass-files/src/encoding.rs index 0bc03832..7d48e56f 100644 --- a/crates/compass-files/src/encoding.rs +++ b/crates/compass-files/src/encoding.rs @@ -1,17 +1,96 @@ -use std::fs; +use std::fs::{self, File}; +use std::io::Read; use std::path::Path; use crate::{FileError, io_error}; -/// Read source bytes with Python's `errors="replace"` UTF-8 behavior. -pub fn read_source_lossy(path: &Path, max_bytes: u64) -> Result { - let metadata = fs::metadata(path).map_err(|source| io_error(path, source))?; - if metadata.len() > max_bytes { +const READ_PREALLOC_MAX_BYTES: u64 = 1024 * 1024; + +/// Read a regular file without ever consuming more than `max_bytes + 1` bytes. +/// +/// The second, stream-level bound is intentional: metadata is only a snapshot, +/// and a file may grow or be replaced between inspection and the final read. +pub fn read_bytes_bounded(path: &Path, max_bytes: u64) -> Result, FileError> { + let path_metadata = fs::metadata(path).map_err(|source| io_error(path, source))?; + if !path_metadata.is_file() { + return Err(FileError::NotAFile(path.to_path_buf())); + } + if path_metadata.len() > max_bytes { + return Err(FileError::TooLarge { + path: path.to_path_buf(), + limit: max_bytes, + }); + } + + let file = File::open(path).map_err(|source| io_error(path, source))?; + let opened_metadata = file.metadata().map_err(|source| io_error(path, source))?; + if !opened_metadata.is_file() { + return Err(FileError::NotAFile(path.to_path_buf())); + } + if opened_metadata.len() > max_bytes { + return Err(FileError::TooLarge { + path: path.to_path_buf(), + limit: max_bytes, + }); + } + + let capacity = opened_metadata + .len() + .min(max_bytes) + .min(READ_PREALLOC_MAX_BYTES) as usize; + let mut bytes = Vec::with_capacity(capacity); + file.take(max_bytes.saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|source| io_error(path, source))?; + if bytes.len() as u64 > max_bytes { return Err(FileError::TooLarge { path: path.to_path_buf(), limit: max_bytes, }); } - let bytes = fs::read(path).map_err(|source| io_error(path, source))?; + Ok(bytes) +} + +/// Read source bytes with Python's `errors="replace"` UTF-8 behavior. +pub fn read_source_lossy(path: &Path, max_bytes: u64) -> Result { + let bytes = read_bytes_bounded(path, max_bytes)?; Ok(String::from_utf8_lossy(&bytes).into_owned()) } + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + + #[test] + fn bounded_reader_accepts_exact_limit_and_rejects_one_over_and_non_files() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("source.bin"); + fs::write(&path, b"1234")?; + assert_eq!(read_bytes_bounded(&path, 4)?, b"1234"); + assert!(matches!( + read_bytes_bounded(&path, 3), + Err(FileError::TooLarge { limit: 3, .. }) + )); + assert!(matches!( + read_bytes_bounded(directory.path(), 4), + Err(FileError::NotAFile(_)) + )); + Ok(()) + } + + #[test] + fn lossy_source_read_uses_the_same_stream_bound() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("source.txt"); + fs::write(&path, b"a\xffb")?; + assert_eq!(read_source_lossy(&path, 3)?, "a\u{fffd}b"); + assert!(matches!( + read_source_lossy(&path, 2), + Err(FileError::TooLarge { .. }) + )); + Ok(()) + } +} diff --git a/crates/compass-files/src/lib.rs b/crates/compass-files/src/lib.rs index 2f3e8eff..f67221b8 100644 --- a/crates/compass-files/src/lib.rs +++ b/crates/compass-files/src/lib.rs @@ -25,7 +25,7 @@ pub use cache::{ pub use detect::{ DetectOptions, Detection, FileType, IgnorePolicy, WatchPathFilter, classify_file, detect, }; -pub use encoding::read_source_lossy; +pub use encoding::{read_bytes_bounded, read_source_lossy}; pub use generated::source_is_generated; pub use hash::{StatHashIndex, body_content, file_hash, md5_file, prompt_fingerprint}; pub use manifest::{IncrementalDetection, Manifest, ManifestEntry, ManifestKind}; diff --git a/crates/compass-languages/src/lib.rs b/crates/compass-languages/src/lib.rs index d132ecfc..ca6c5131 100644 --- a/crates/compass-languages/src/lib.rs +++ b/crates/compass-languages/src/lib.rs @@ -94,6 +94,8 @@ pub enum ExtractError { ParseCancelled(PathBuf), #[error("invalid program evidence for {path}: {detail}")] InvalidProgramEvidence { path: PathBuf, detail: String }, + #[error("invalid prepared document evidence for {path}: {detail}")] + InvalidDocumentEvidence { path: PathBuf, detail: String }, #[error(transparent)] File(#[from] compass_files::FileError), } diff --git a/crates/compass-media/Cargo.toml b/crates/compass-media/Cargo.toml index 945bf5d0..9480a9ca 100644 --- a/crates/compass-media/Cargo.toml +++ b/crates/compass-media/Cargo.toml @@ -12,9 +12,17 @@ keywords.workspace = true categories.workspace = true [dependencies] +compass-files = { path = "../compass-files", version = "0.3.19" } +compass-ocr = { path = "../compass-ocr", version = "0.3.19" } +hayro.workspace = true +image.workspace = true oxidize-pdf.workspace = true roxmltree.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true +unicode-normalization.workspace = true zip.workspace = true [lints] diff --git a/crates/compass-media/src/document.rs b/crates/compass-media/src/document.rs new file mode 100644 index 00000000..0755fde8 --- /dev/null +++ b/crates/compass-media/src/document.rs @@ -0,0 +1,667 @@ +//! Versioned, provenance-preserving document artifact contract. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path}; + +use compass_ocr::{OcrPoint, OcrProfileIdentity}; +use serde::{Deserialize, Serialize}; + +use crate::limits::{ + DOCUMENT_MAX_BLOCKS, DOCUMENT_MAX_DEPTH, DOCUMENT_MAX_DIAGNOSTIC_MESSAGE_BYTES, + DOCUMENT_MAX_DIAGNOSTICS, DOCUMENT_MAX_FIELD_BYTES, DOCUMENT_MAX_LINKS, + DOCUMENT_MAX_METADATA_ENTRIES, DOCUMENT_MAX_TEXT_CHARS, +}; + +pub const DOCUMENT_SCHEMA: &str = "compass.document/1"; +pub const DOCUMENT_INSPECT_SCHEMA: &str = "compass.document.inspect/1"; +pub const DOCUMENT_NORMALIZER_VERSION: u32 = 2; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DocumentFormat { + Text, + Markdown, + Html, + Pdf, + Docx, + Xlsx, + Pptx, + Rtf, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DocumentBlockKind { + DocumentTitle, + Heading { level: u8 }, + Paragraph, + List, + ListItem, + Code, + Quote, + Table, + Row, + Cell, + Page, + Sheet, + Slide, + Note, + Other { role: String }, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DocumentLocator { + TextRange { + start_byte: u64, + end_byte: u64, + start_line: u32, + end_line: u32, + }, + Package { + part: String, + path: String, + }, + Pdf { + page: u32, + item: u32, + }, + Spreadsheet { + sheet: String, + row: u32, + column: u16, + }, + Slide { + slide: u32, + shape: u32, + }, + Ocr { + owner: Box, + candidate_id: String, + width: u32, + height: u32, + polygon: Vec, + occurrence: u32, + }, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DocumentOrigin { + #[default] + Native, + Ocr { + profile: OcrProfileIdentity, + confidence_bps: u16, + }, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DocumentBlock { + pub ordinal: u32, + pub parent: Option, + pub kind: DocumentBlockKind, + pub text: String, + pub locator: DocumentLocator, + #[serde(default)] + pub origin: DocumentOrigin, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub metadata: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DocumentLinkKind { + Hyperlink, + Relationship, + Image, + Attachment, + Other, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DocumentLink { + pub source_block: u32, + pub destination: String, + pub label: Option, + pub relationship: DocumentLinkKind, + pub locator: DocumentLocator, + pub external: bool, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticSeverity { + Info, + Warning, + Error, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DocumentDiagnostic { + pub code: String, + pub severity: DiagnosticSeverity, + pub locator: Option, + pub message: String, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum VisualCoverage { + #[default] + NotRequested, + Complete, + Partial, + Failed, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DocumentArtifact { + pub schema: String, + pub normalizer_version: u32, + pub format: DocumentFormat, + pub blocks: Vec, + pub links: Vec, + pub metadata: BTreeMap, + pub diagnostics: Vec, + pub complete: bool, + #[serde(default)] + pub visual_coverage: VisualCoverage, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_profile: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum DocumentError { + #[error("unsupported document format {0:?}")] + Unsupported(String), + #[error("document rejected: {0}")] + Rejected(String), + #[error("document parse failed: {0}")] + Parse(String), + #[error("invalid document artifact: {0}")] + InvalidArtifact(String), + #[error("could not access {path}: {source}")] + Io { + path: std::path::PathBuf, + #[source] + source: std::io::Error, + }, + #[error(transparent)] + Ocr(#[from] compass_ocr::OcrError), +} + +impl DocumentArtifact { + #[must_use] + pub fn new(format: DocumentFormat) -> Self { + Self { + schema: DOCUMENT_SCHEMA.to_owned(), + normalizer_version: DOCUMENT_NORMALIZER_VERSION, + format, + blocks: Vec::new(), + links: Vec::new(), + metadata: BTreeMap::new(), + diagnostics: Vec::new(), + complete: true, + visual_coverage: VisualCoverage::NotRequested, + ocr_profile: None, + } + } + + pub fn push_block( + &mut self, + parent: Option, + kind: DocumentBlockKind, + text: String, + locator: DocumentLocator, + ) -> Result { + if self.blocks.len() >= DOCUMENT_MAX_BLOCKS { + return Err(DocumentError::Rejected( + "document block count exceeds limit".to_owned(), + )); + } + let ordinal = u32::try_from(self.blocks.len()) + .map_err(|_| DocumentError::Rejected("document ordinal overflow".to_owned()))?; + self.blocks.push(DocumentBlock { + ordinal, + parent, + kind, + text, + locator, + origin: DocumentOrigin::Native, + metadata: BTreeMap::new(), + }); + Ok(ordinal) + } + + pub fn validate(&self) -> Result<(), DocumentError> { + if self.schema != DOCUMENT_SCHEMA { + return Err(DocumentError::InvalidArtifact(format!( + "unsupported schema {:?}", + self.schema + ))); + } + if self.normalizer_version != DOCUMENT_NORMALIZER_VERSION { + return Err(DocumentError::InvalidArtifact(format!( + "unsupported normalizer version {}", + self.normalizer_version + ))); + } + if self.blocks.len() > DOCUMENT_MAX_BLOCKS + || self.links.len() > DOCUMENT_MAX_LINKS + || self.diagnostics.len() > DOCUMENT_MAX_DIAGNOSTICS + || self.metadata.len() > DOCUMENT_MAX_METADATA_ENTRIES + { + return Err(DocumentError::InvalidArtifact( + "artifact collection exceeds limit".to_owned(), + )); + } + let mut text_chars = 0_usize; + let mut saw_ocr_block = false; + for (index, block) in self.blocks.iter().enumerate() { + let expected = u32::try_from(index) + .map_err(|_| DocumentError::InvalidArtifact("ordinal overflow".to_owned()))?; + if block.ordinal != expected { + return Err(DocumentError::InvalidArtifact( + "block ordinals must be contiguous".to_owned(), + )); + } + if block.parent.is_some_and(|parent| parent >= block.ordinal) { + return Err(DocumentError::InvalidArtifact( + "block parent must precede its child".to_owned(), + )); + } + if depth_for(&self.blocks, block.ordinal)? > DOCUMENT_MAX_DEPTH { + return Err(DocumentError::InvalidArtifact( + "block nesting exceeds limit".to_owned(), + )); + } + text_chars = text_chars + .checked_add(block.text.chars().count()) + .ok_or_else(|| DocumentError::InvalidArtifact("text size overflow".to_owned()))?; + if block.metadata.len() > DOCUMENT_MAX_METADATA_ENTRIES { + return Err(DocumentError::InvalidArtifact( + "block metadata exceeds limit".to_owned(), + )); + } + validate_block_kind(&block.kind)?; + validate_locator(&block.locator, 0)?; + validate_origin(&block.origin)?; + match (&block.origin, &block.locator) { + (DocumentOrigin::Ocr { profile, .. }, DocumentLocator::Ocr { .. }) => { + saw_ocr_block = true; + if self.ocr_profile.as_ref() != Some(profile) { + return Err(DocumentError::InvalidArtifact( + "OCR block profile does not match the document profile".to_owned(), + )); + } + } + (DocumentOrigin::Ocr { .. }, _) => { + return Err(DocumentError::InvalidArtifact( + "OCR-derived block is missing an OCR locator".to_owned(), + )); + } + (DocumentOrigin::Native, DocumentLocator::Ocr { .. }) => { + return Err(DocumentError::InvalidArtifact( + "native block must not use an OCR locator".to_owned(), + )); + } + (DocumentOrigin::Native, _) => {} + } + validate_metadata(&block.metadata)?; + } + if text_chars > DOCUMENT_MAX_TEXT_CHARS { + return Err(DocumentError::InvalidArtifact( + "document text exceeds limit".to_owned(), + )); + } + for link in &self.links { + if usize::try_from(link.source_block) + .ok() + .is_none_or(|source| source >= self.blocks.len()) + { + return Err(DocumentError::InvalidArtifact( + "link references a missing block".to_owned(), + )); + } + validate_field("link destination", &link.destination)?; + if let Some(label) = &link.label { + validate_field("link label", label)?; + } + validate_locator(&link.locator, 0)?; + } + validate_metadata(&self.metadata)?; + for diagnostic in &self.diagnostics { + validate_field("diagnostic code", &diagnostic.code)?; + if diagnostic.message.len() > DOCUMENT_MAX_DIAGNOSTIC_MESSAGE_BYTES { + return Err(DocumentError::InvalidArtifact( + "diagnostic message exceeds limit".to_owned(), + )); + } + if let Some(locator) = &diagnostic.locator { + validate_locator(locator, 0)?; + } + } + if let Some(profile) = &self.ocr_profile { + profile.validate()?; + } + match self.visual_coverage { + VisualCoverage::NotRequested => { + if self.ocr_profile.is_some() || saw_ocr_block { + return Err(DocumentError::InvalidArtifact( + "OCR evidence is present when visual coverage was not requested".to_owned(), + )); + } + } + VisualCoverage::Complete => { + if self.ocr_profile.is_none() { + return Err(DocumentError::InvalidArtifact( + "complete visual coverage is missing its OCR profile".to_owned(), + )); + } + } + VisualCoverage::Partial | VisualCoverage::Failed => { + if self.ocr_profile.is_none() { + return Err(DocumentError::InvalidArtifact( + "incomplete visual coverage is missing its OCR profile".to_owned(), + )); + } + if self.complete { + return Err(DocumentError::InvalidArtifact( + "partial or failed visual coverage cannot be complete".to_owned(), + )); + } + } + } + Ok(()) + } + + pub fn to_canonical_json(&self) -> Result, DocumentError> { + self.validate()?; + serde_json::to_vec(self).map_err(|error| DocumentError::InvalidArtifact(error.to_string())) + } + + pub fn from_json(bytes: &[u8]) -> Result { + let artifact: Self = serde_json::from_slice(bytes) + .map_err(|error| DocumentError::InvalidArtifact(error.to_string()))?; + artifact.validate()?; + Ok(artifact) + } +} + +fn depth_for(blocks: &[DocumentBlock], ordinal: u32) -> Result { + let mut depth = 0_usize; + let mut current = Some(ordinal); + while let Some(value) = current { + let index = usize::try_from(value) + .map_err(|_| DocumentError::InvalidArtifact("invalid block ordinal".to_owned()))?; + let block = blocks.get(index).ok_or_else(|| { + DocumentError::InvalidArtifact("parent references a missing block".to_owned()) + })?; + current = block.parent; + depth = depth + .checked_add(1) + .ok_or_else(|| DocumentError::InvalidArtifact("block depth overflow".to_owned()))?; + if depth > DOCUMENT_MAX_DEPTH + 1 { + return Ok(depth); + } + } + Ok(depth) +} + +fn validate_block_kind(kind: &DocumentBlockKind) -> Result<(), DocumentError> { + match kind { + DocumentBlockKind::Heading { level } if !(1..=6).contains(level) => Err( + DocumentError::InvalidArtifact("heading level must be in 1..=6".to_owned()), + ), + DocumentBlockKind::Other { role } => validate_field("block role", role), + _ => Ok(()), + } +} + +fn validate_origin(origin: &DocumentOrigin) -> Result<(), DocumentError> { + if let DocumentOrigin::Ocr { + profile, + confidence_bps, + } = origin + { + profile.validate()?; + if *confidence_bps > 10_000 { + return Err(DocumentError::InvalidArtifact( + "OCR confidence exceeds 10000 basis points".to_owned(), + )); + } + } + Ok(()) +} + +fn validate_locator(locator: &DocumentLocator, depth: usize) -> Result<(), DocumentError> { + if depth > DOCUMENT_MAX_DEPTH { + return Err(DocumentError::InvalidArtifact( + "locator nesting exceeds limit".to_owned(), + )); + } + match locator { + DocumentLocator::TextRange { + start_byte, + end_byte, + start_line, + end_line, + } if start_byte > end_byte || start_line > end_line => Err(DocumentError::InvalidArtifact( + "invalid text range".to_owned(), + )), + DocumentLocator::Package { part, path } => { + validate_package_part(part)?; + validate_field("package block path", path) + } + DocumentLocator::Spreadsheet { sheet, row, column } => { + validate_field("sheet name", sheet)?; + if *row == 0 || *column == 0 { + return Err(DocumentError::InvalidArtifact( + "spreadsheet coordinates are one-based".to_owned(), + )); + } + Ok(()) + } + DocumentLocator::Slide { slide, shape } if *slide == 0 || *shape == 0 => Err( + DocumentError::InvalidArtifact("slide coordinates are one-based".to_owned()), + ), + DocumentLocator::Pdf { page, item } if *page == 0 || *item == 0 => Err( + DocumentError::InvalidArtifact("PDF coordinates are one-based".to_owned()), + ), + DocumentLocator::Ocr { + owner, + candidate_id, + width, + height, + polygon, + occurrence: _, + } => { + validate_locator(owner, depth + 1)?; + validate_field("OCR candidate ID", candidate_id)?; + compass_ocr::validate_dimensions(*width, *height)?; + if !(4..=16).contains(&polygon.len()) + || polygon + .iter() + .any(|point| point.x >= *width || point.y >= *height) + || polygon_doubled_area(polygon) == 0 + { + return Err(DocumentError::InvalidArtifact( + "invalid OCR locator polygon".to_owned(), + )); + } + Ok(()) + } + _ => Ok(()), + } +} + +fn polygon_doubled_area(points: &[OcrPoint]) -> i128 { + let mut area = 0_i128; + for index in 0..points.len() { + let current = points[index]; + let next = points[(index + 1) % points.len()]; + area += + i128::from(current.x) * i128::from(next.y) - i128::from(next.x) * i128::from(current.y); + } + area.abs() +} + +fn validate_package_part(value: &str) -> Result<(), DocumentError> { + validate_field("package part", value)?; + let path = Path::new(value); + if path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + || value.contains('\\') + { + return Err(DocumentError::InvalidArtifact( + "package part is absolute or escapes its package".to_owned(), + )); + } + Ok(()) +} + +fn validate_field(name: &str, value: &str) -> Result<(), DocumentError> { + if value.is_empty() || value.len() > DOCUMENT_MAX_FIELD_BYTES || value.contains('\0') { + return Err(DocumentError::InvalidArtifact(format!( + "{name} is empty or exceeds its bound" + ))); + } + Ok(()) +} + +fn validate_metadata(metadata: &BTreeMap) -> Result<(), DocumentError> { + if metadata.len() > DOCUMENT_MAX_METADATA_ENTRIES { + return Err(DocumentError::InvalidArtifact( + "metadata count exceeds limit".to_owned(), + )); + } + let mut keys = BTreeSet::new(); + for (key, value) in metadata { + validate_field("metadata key", key)?; + if !keys.insert(key) { + return Err(DocumentError::InvalidArtifact( + "duplicate metadata key".to_owned(), + )); + } + let encoded = serde_json::to_vec(value) + .map_err(|error| DocumentError::InvalidArtifact(error.to_string()))?; + if encoded.len() > DOCUMENT_MAX_FIELD_BYTES { + return Err(DocumentError::InvalidArtifact( + "metadata value exceeds limit".to_owned(), + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn artifact_round_trip_is_deterministic() -> Result<(), Box> { + let mut artifact = DocumentArtifact::new(DocumentFormat::Docx); + artifact + .metadata + .insert("title".to_owned(), serde_json::json!("Guide")); + artifact.push_block( + None, + DocumentBlockKind::Heading { level: 1 }, + "Guide".to_owned(), + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: "body/p[1]".to_owned(), + }, + )?; + let first = artifact.to_canonical_json()?; + let decoded = DocumentArtifact::from_json(&first)?; + assert_eq!(decoded, artifact); + assert_eq!(decoded.to_canonical_json()?, first); + Ok(()) + } + + #[test] + fn rejects_unknown_schema_parent_and_package_escape() -> Result<(), Box> + { + let mut artifact = DocumentArtifact::new(DocumentFormat::Docx); + artifact.schema = "compass.document/2".to_owned(); + assert!(artifact.validate().is_err()); + + let mut artifact = DocumentArtifact::new(DocumentFormat::Docx); + artifact.push_block( + Some(0), + DocumentBlockKind::Paragraph, + "bad".to_owned(), + DocumentLocator::Package { + part: "../word/document.xml".to_owned(), + path: "body/p[1]".to_owned(), + }, + )?; + assert!(artifact.validate().is_err()); + Ok(()) + } + + #[test] + fn rejects_incoherent_ocr_profile_origin_geometry_and_completeness() + -> Result<(), Box> { + let profile = OcrProfileIdentity { + engine: "fixture".to_owned(), + engine_version: "1".to_owned(), + profile: "fixture".to_owned(), + model_digests: BTreeMap::from([("model".to_owned(), "a".repeat(64))]), + languages: vec!["en".to_owned()], + preprocessing_version: compass_ocr::OCR_PREPROCESSING_VERSION, + }; + let owner = DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: "body/p[1]".to_owned(), + }; + let mut artifact = DocumentArtifact::new(DocumentFormat::Docx); + artifact.ocr_profile = Some(profile.clone()); + artifact.visual_coverage = VisualCoverage::Complete; + artifact.push_block( + None, + DocumentBlockKind::Paragraph, + "OCR text".to_owned(), + DocumentLocator::Ocr { + owner: Box::new(owner), + candidate_id: "image-1".to_owned(), + width: 100, + height: 100, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 10, y: 1 }, + OcrPoint { x: 10, y: 10 }, + OcrPoint { x: 1, y: 10 }, + ], + occurrence: 0, + }, + )?; + artifact.blocks[0].origin = DocumentOrigin::Ocr { + profile: profile.clone(), + confidence_bps: 9_000, + }; + assert!(artifact.validate().is_ok()); + + artifact.blocks[0].origin = DocumentOrigin::Native; + assert!(artifact.validate().is_err()); + artifact.blocks[0].origin = DocumentOrigin::Ocr { + profile, + confidence_bps: 9_000, + }; + if let DocumentLocator::Ocr { polygon, .. } = &mut artifact.blocks[0].locator { + polygon[2] = OcrPoint { x: 20, y: 1 }; + polygon[3] = OcrPoint { x: 30, y: 1 }; + } + assert!(artifact.validate().is_err()); + + artifact.blocks.clear(); + artifact.visual_coverage = VisualCoverage::Partial; + artifact.complete = true; + assert!(artifact.validate().is_err()); + Ok(()) + } +} diff --git a/crates/compass-media/src/lib.rs b/crates/compass-media/src/lib.rs index 557eab10..8fabcd49 100644 --- a/crates/compass-media/src/lib.rs +++ b/crates/compass-media/src/lib.rs @@ -1,442 +1,348 @@ -//! Bounded, pure-Rust text extraction for semantic media inputs. - -use std::collections::HashMap; -use std::fs::{self, File}; -use std::io::Read; -use std::path::{Path, PathBuf}; - +//! Bounded, pure-Rust document decoding with provenance-preserving artifacts. + +pub mod document; +pub mod limits; +mod ooxml; +mod processing; +mod raster; + +use std::collections::BTreeMap; +use std::io::Cursor; +use std::path::Path; + +use document::{ + DocumentArtifact, DocumentBlock, DocumentBlockKind, DocumentError, DocumentFormat, + DocumentLocator, +}; use oxidize_pdf::parser::{PdfDocument, PdfReader}; -use roxmltree::{Document, Node}; -use zip::ZipArchive; - -pub const MEDIA_MAX_RAW_BYTES: u64 = 50 * 1024 * 1024; -pub const OFFICE_MAX_DECOMPRESSED_BYTES: u64 = 512 * 1024 * 1024; -pub const OFFICE_MAX_COMPRESSION_RATIO: u64 = 200; -const OFFICE_MEMBER_MAX_BYTES: u64 = 64 * 1024 * 1024; - -#[derive(Debug, thiserror::Error)] -pub enum MediaError { - #[error("could not access {path}: {source}")] - Io { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("media rejected: {0}")] - Rejected(String), - #[error("media parse failed: {0}")] - Parse(String), + +pub use document::{ + DOCUMENT_INSPECT_SCHEMA, DOCUMENT_NORMALIZER_VERSION, DOCUMENT_SCHEMA, DiagnosticSeverity, + DocumentDiagnostic, DocumentLink, DocumentLinkKind, DocumentOrigin, VisualCoverage, +}; +pub use limits::{ + MEDIA_MAX_RAW_BYTES, OCR_MAX_AGGREGATE_PIXELS, OCR_MAX_OOXML_IMAGES, OCR_MAX_PDF_PAGES, + OFFICE_MAX_COMPRESSION_RATIO, OFFICE_MAX_DECOMPRESSED_BYTES, +}; +pub use ooxml::{RasterCandidate, decode_docx, decode_pptx, decode_xlsx, raster_candidates}; +pub use processing::{ + DocumentProcessingOptions, decode_document_with_ocr, decode_document_with_ocr_cancellable, +}; +pub use raster::{ + PDF_RASTERIZER_IDENTITY, PdfRasterCandidate, rasterize_pdf_pages, + rasterize_pdf_pages_cancellable, +}; + +pub type MediaError = DocumentError; + +/// Decode one bounded source into Compass's stable document artifact. +pub fn decode_document( + logical_path: &Path, + bytes: &[u8], +) -> Result { + if bytes.len() as u64 > MEDIA_MAX_RAW_BYTES { + return Err(DocumentError::Rejected(format!( + "source is {} bytes; maximum is {MEDIA_MAX_RAW_BYTES}", + bytes.len() + ))); + } + let artifact = match extension(logical_path).as_str() { + "pdf" => decode_pdf(bytes)?, + "docx" => decode_docx(bytes)?, + "xlsx" => decode_xlsx(bytes)?, + "pptx" => decode_pptx(bytes)?, + "txt" => decode_plain_text(bytes, DocumentFormat::Text)?, + "md" | "markdown" => decode_plain_text(bytes, DocumentFormat::Markdown)?, + "html" | "htm" => decode_plain_text(bytes, DocumentFormat::Html)?, + "rtf" => { + return Err(DocumentError::Unsupported( + "rtf (enum vocabulary exists, decoder is not implemented)".to_owned(), + )); + } + _ => decode_plain_text(bytes, DocumentFormat::Text)?, + }; + artifact.validate()?; + Ok(artifact) } -/// Extract text from the formats accepted by Compass's semantic path. +/// Extract compatibility Markdown/text through the versioned artifact path. pub fn extract_text(path: &Path) -> Result { - enforce_raw_size(path)?; - match extension(path).as_str() { - "pdf" => extract_pdf_text(path), - "docx" => docx_to_markdown(path), - "xlsx" => xlsx_to_markdown(path), - _ => fs::read(path) - .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) - .map_err(|source| MediaError::Io { - path: path.to_path_buf(), - source, - }), - } + let bytes = read_document_bounded(path)?; + let artifact = decode_document(path, &bytes)?; + render_document_markdown(&artifact) } -/// Compatibility surface for callers where malformed media is a skipped, -/// empty source rather than a fatal corpus error. +/// Legacy best-effort surface. New callers must use [`extract_text`] or [`decode_document`]. #[must_use] pub fn extract_text_compat(path: &Path) -> String { extract_text(path).unwrap_or_default() } pub fn extract_pdf_text(path: &Path) -> Result { - enforce_raw_size(path)?; - let owned = path.to_path_buf(); - std::panic::catch_unwind(move || { - let reader = PdfReader::open(&owned).map_err(|error| error.to_string())?; - let document = PdfDocument::new(reader); - let pages = document.extract_text().map_err(|error| error.to_string())?; - Ok::<_, String>( - pages - .into_iter() - .map(|page| page.text) - .filter(|text| !text.is_empty()) - .collect::>() - .join("\n"), - ) - }) - .map_err(|_| MediaError::Parse("PDF parser panicked".to_owned()))? - .map_err(MediaError::Parse) + let bytes = read_document_bounded(path)?; + render_document_markdown(&decode_pdf(&bytes)?) } pub fn docx_to_markdown(path: &Path) -> Result { - validate_office_archive(path)?; - let styles = read_zip_member(path, "word/styles.xml") - .ok() - .and_then(|xml| parse_docx_styles(&xml).ok()) - .unwrap_or_default(); - let document_xml = read_zip_member(path, "word/document.xml")?; - let document = Document::parse(&document_xml) - .map_err(|error| MediaError::Parse(format!("invalid DOCX document XML: {error}")))?; - let Some(body) = document - .descendants() - .find(|node| node.is_element() && node.tag_name().name() == "body") - else { - return Ok(String::new()); - }; - let mut paragraphs = Vec::new(); - let mut tables = Vec::new(); - for child in body.children().filter(Node::is_element) { - match child.tag_name().name() { - "p" => paragraphs.push(render_docx_paragraph(child, &styles)), - "tbl" => tables.push(render_docx_table(child, &styles)), - _ => {} - } - } - let mut lines = paragraphs; - for table in tables { - if table.is_empty() { - continue; - } - lines.push(markdown_row(&table[0])); - lines.push(markdown_row( - &table[0] - .iter() - .map(|_| "---".to_owned()) - .collect::>(), - )); - lines.extend(table.iter().skip(1).map(|row| markdown_row(row))); - } - Ok(lines.join("\n")) + let bytes = read_document_bounded(path)?; + render_document_markdown(&decode_docx(&bytes)?) } pub fn xlsx_to_markdown(path: &Path) -> Result { - validate_office_archive(path)?; - let workbook_xml = read_zip_member(path, "xl/workbook.xml")?; - let relationships_xml = read_zip_member(path, "xl/_rels/workbook.xml.rels")?; - let shared_strings = read_zip_member(path, "xl/sharedStrings.xml") - .ok() - .and_then(|xml| parse_shared_strings(&xml).ok()) - .unwrap_or_default(); - let relationships = parse_workbook_relationships(&relationships_xml)?; - let workbook = Document::parse(&workbook_xml) - .map_err(|error| MediaError::Parse(format!("invalid XLSX workbook XML: {error}")))?; - let mut sections = Vec::new(); - for sheet in workbook - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "sheet") - { - let name = attribute_local(sheet, "name").unwrap_or_default(); - let relation = attribute_local(sheet, "id").unwrap_or_default(); - let Some(target) = relationships.get(relation) else { - continue; - }; - let member = normalize_xlsx_target(target); - let Ok(sheet_xml) = read_zip_member(path, &member) else { - continue; - }; - let rows = parse_xlsx_rows(&sheet_xml, &shared_strings)?; - if rows.is_empty() { - continue; - } - sections.push(format!("## Sheet: {name}")); - sections.push(markdown_row(&rows[0])); - sections.push(markdown_row( - &rows[0].iter().map(|_| "---".to_owned()).collect::>(), - )); - sections.extend(rows.iter().skip(1).map(|row| markdown_row(row))); - } - Ok(sections.join("\n")) + let bytes = read_document_bounded(path)?; + render_document_markdown(&decode_xlsx(&bytes)?) } -pub fn validate_office_archive(path: &Path) -> Result<(), MediaError> { - enforce_raw_size(path)?; - let file = File::open(path).map_err(|source| MediaError::Io { - path: path.to_path_buf(), - source, - })?; - let mut archive = ZipArchive::new(file) - .map_err(|error| MediaError::Parse(format!("invalid ZIP container: {error}")))?; - let mut compressed = 0_u64; - let mut declared = 0_u64; - for index in 0..archive.len() { - let member = archive - .by_index_raw(index) - .map_err(|error| MediaError::Parse(error.to_string()))?; - compressed = compressed.saturating_add(member.compressed_size()); - declared = declared.saturating_add(member.size()); - } - if declared > OFFICE_MAX_DECOMPRESSED_BYTES { - return Err(MediaError::Rejected(format!( - "declared office payload is {declared} bytes" - ))); - } - if declared - > compressed - .max(1) - .saturating_mul(OFFICE_MAX_COMPRESSION_RATIO) - { - return Err(MediaError::Rejected( - "office compression ratio exceeds safety limit".to_owned(), - )); - } - let mut actual = 0_u64; - let mut buffer = [0_u8; 1024 * 1024]; - for index in 0..archive.len() { - let mut member = archive - .by_index(index) - .map_err(|error| MediaError::Parse(error.to_string()))?; - loop { - let read = member - .read(&mut buffer) - .map_err(|error| MediaError::Parse(error.to_string()))?; - if read == 0 { - break; - } - actual = actual.saturating_add(read as u64); - if actual > OFFICE_MAX_DECOMPRESSED_BYTES { - return Err(MediaError::Rejected( - "decompressed office payload exceeds safety limit".to_owned(), - )); - } - } - } - Ok(()) -} - -fn enforce_raw_size(path: &Path) -> Result<(), MediaError> { - let metadata = fs::metadata(path).map_err(|source| MediaError::Io { - path: path.to_path_buf(), - source, - })?; - if metadata.len() > MEDIA_MAX_RAW_BYTES { - return Err(MediaError::Rejected(format!( - "{} is {} bytes; maximum is {MEDIA_MAX_RAW_BYTES}", - path.display(), - metadata.len() - ))); - } - Ok(()) +pub fn pptx_to_markdown(path: &Path) -> Result { + let bytes = read_document_bounded(path)?; + render_document_markdown(&decode_pptx(&bytes)?) } -fn read_zip_member(path: &Path, name: &str) -> Result { - let file = File::open(path).map_err(|source| MediaError::Io { - path: path.to_path_buf(), - source, - })?; - let mut archive = ZipArchive::new(file) - .map_err(|error| MediaError::Parse(format!("invalid ZIP container: {error}")))?; - let member = archive - .by_name(name) - .map_err(|error| MediaError::Parse(format!("missing {name}: {error}")))?; - if member.size() > OFFICE_MEMBER_MAX_BYTES { - return Err(MediaError::Rejected(format!( - "office member {name} exceeds safety limit" - ))); - } - let mut bytes = Vec::with_capacity(usize::try_from(member.size()).unwrap_or(0)); - member - .take(OFFICE_MEMBER_MAX_BYTES.saturating_add(1)) - .read_to_end(&mut bytes) - .map_err(|error| MediaError::Parse(error.to_string()))?; - if bytes.len() as u64 > OFFICE_MEMBER_MAX_BYTES { - return Err(MediaError::Rejected(format!( - "office member {name} exceeds safety limit" - ))); - } - String::from_utf8(bytes) - .map_err(|error| MediaError::Parse(format!("office member {name} is not UTF-8: {error}"))) +pub fn validate_office_archive(path: &Path) -> Result<(), MediaError> { + let bytes = read_document_bounded(path)?; + ooxml::Package::open(&bytes).map(|_| ()) } -fn parse_docx_styles(xml: &str) -> Result, MediaError> { - let document = Document::parse(xml) - .map_err(|error| MediaError::Parse(format!("invalid DOCX styles XML: {error}")))?; - let mut styles = HashMap::new(); - for style in document - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "style") - { - let Some(id) = attribute_local(style, "styleId") else { - continue; - }; - let name = style - .children() - .find(|node| node.is_element() && node.tag_name().name() == "name") - .and_then(|node| attribute_local(node, "val")) - .unwrap_or(id); - styles.insert(id.to_owned(), name.to_owned()); +pub fn render_document_markdown(artifact: &DocumentArtifact) -> Result { + artifact.validate()?; + match artifact.format { + DocumentFormat::Xlsx => render_spreadsheet(artifact), + _ => render_ordered_blocks(artifact), } - Ok(styles) } -fn render_docx_paragraph(node: Node<'_, '_>, styles: &HashMap) -> String { - let text = node - .descendants() - .filter(|descendant| descendant.is_element() && descendant.tag_name().name() == "t") - .filter_map(|descendant| descendant.text()) - .collect::() - .trim() - .to_owned(); - if text.is_empty() { - return String::new(); - } - let style_id = node - .descendants() - .find(|descendant| descendant.is_element() && descendant.tag_name().name() == "pStyle") - .and_then(|style| attribute_local(style, "val")) - .unwrap_or_default(); - let style_name = styles.get(style_id).map_or(style_id, String::as_str); - let normalized_style = style_name.to_ascii_lowercase(); - if normalized_style.starts_with("heading 1") || normalized_style == "heading1" { - format!("# {text}") - } else if normalized_style.starts_with("heading 2") || normalized_style == "heading2" { - format!("## {text}") - } else if normalized_style.starts_with("heading 3") || normalized_style == "heading3" { - format!("### {text}") - } else if normalized_style.starts_with("list") { - format!("- {text}") - } else { - text - } +fn decode_pdf(bytes: &[u8]) -> Result { + let owned = bytes.to_vec(); + std::panic::catch_unwind(move || { + let reader = PdfReader::new(Cursor::new(owned)).map_err(|error| error.to_string())?; + let document = PdfDocument::new(reader); + let pages = document.extract_text().map_err(|error| error.to_string())?; + let mut artifact = DocumentArtifact::new(DocumentFormat::Pdf); + for (index, page) in pages.into_iter().enumerate() { + let page_number = one_based_u32(index, "PDF page")?; + let page_block = artifact + .push_block( + None, + DocumentBlockKind::Page, + String::new(), + DocumentLocator::Pdf { + page: page_number, + item: 1, + }, + ) + .map_err(|error| error.to_string())?; + if !page.text.is_empty() { + artifact + .push_block( + Some(page_block), + DocumentBlockKind::Paragraph, + page.text, + DocumentLocator::Pdf { + page: page_number, + item: 2, + }, + ) + .map_err(|error| error.to_string())?; + } + } + Ok::<_, String>(artifact) + }) + .map_err(|_| DocumentError::Parse("PDF parser panicked".to_owned()))? + .map_err(DocumentError::Parse) } -fn render_docx_table(table: Node<'_, '_>, styles: &HashMap) -> Vec> { - table - .children() - .filter(|node| node.is_element() && node.tag_name().name() == "tr") - .map(|row| { - row.children() - .filter(|node| node.is_element() && node.tag_name().name() == "tc") - .map(|cell| { - cell.children() - .filter(|node| node.is_element() && node.tag_name().name() == "p") - .map(|paragraph| render_docx_paragraph(paragraph, styles)) - .collect::>() - .join("\n") - .trim() - .to_owned() - }) - .collect() - }) - .collect() +fn decode_plain_text( + bytes: &[u8], + format: DocumentFormat, +) -> Result { + let text = String::from_utf8_lossy(bytes).into_owned(); + let end_byte = u64::try_from(bytes.len()) + .map_err(|_| DocumentError::Rejected("text byte length overflow".to_owned()))?; + let end_line = u32::try_from(text.lines().count().max(1)) + .map_err(|_| DocumentError::Rejected("text line count overflow".to_owned()))?; + let mut artifact = DocumentArtifact::new(format); + artifact.push_block( + None, + DocumentBlockKind::Paragraph, + text, + DocumentLocator::TextRange { + start_byte: 0, + end_byte, + start_line: 1, + end_line, + }, + )?; + Ok(artifact) } -fn parse_shared_strings(xml: &str) -> Result, MediaError> { - let document = Document::parse(xml) - .map_err(|error| MediaError::Parse(format!("invalid shared strings XML: {error}")))?; - Ok(document - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "si") - .map(|item| { - item.descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "t") - .filter_map(|node| node.text()) - .collect::() - }) - .collect()) +/// Read one document source through the shared stream-level byte bound. +pub fn read_document_bounded(path: &Path) -> Result, DocumentError> { + compass_files::read_bytes_bounded(path, MEDIA_MAX_RAW_BYTES).map_err(|error| match error { + compass_files::FileError::Io { path, source } => DocumentError::Io { path, source }, + compass_files::FileError::TooLarge { path, .. } => DocumentError::Rejected(format!( + "{} exceeds the document source limit of {MEDIA_MAX_RAW_BYTES} bytes", + path.display() + )), + compass_files::FileError::NotAFile(path) => DocumentError::Rejected(format!( + "document source is not a regular file: {}", + path.display() + )), + other => DocumentError::Rejected(other.to_string()), + }) } -fn parse_workbook_relationships(xml: &str) -> Result, MediaError> { - let document = Document::parse(xml) - .map_err(|error| MediaError::Parse(format!("invalid workbook relationships: {error}")))?; - Ok(document - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "Relationship") - .filter_map(|node| { - Some(( - attribute_local(node, "Id")?.to_owned(), - attribute_local(node, "Target")?.to_owned(), - )) - }) - .collect()) +fn render_ordered_blocks(artifact: &DocumentArtifact) -> Result { + let children = child_index(&artifact.blocks); + let mut lines = Vec::new(); + let mut skipped = std::collections::BTreeSet::new(); + for block in &artifact.blocks { + if skipped.contains(&block.ordinal) { + continue; + } + match &block.kind { + DocumentBlockKind::Table => { + let rendered = render_table(artifact, block.ordinal, &children, &mut skipped); + if !rendered.is_empty() { + lines.extend(rendered); + } + } + DocumentBlockKind::Heading { level } => { + lines.push(format!( + "{} {}", + "#".repeat(usize::from(*level)), + block.text + )); + } + DocumentBlockKind::DocumentTitle => lines.push(format!("# {}", block.text)), + DocumentBlockKind::ListItem => lines.push(format!("- {}", block.text)), + DocumentBlockKind::Code => lines.push(format!("```\n{}\n```", block.text)), + DocumentBlockKind::Quote => lines.push(format!("> {}", block.text)), + DocumentBlockKind::Paragraph | DocumentBlockKind::Note => { + lines.push(block.text.clone()); + } + DocumentBlockKind::Slide if artifact.format == DocumentFormat::Pptx => { + if let DocumentLocator::Slide { slide, .. } = block.locator { + lines.push(format!("## Slide {slide}")); + } + } + DocumentBlockKind::Page + | DocumentBlockKind::Sheet + | DocumentBlockKind::Slide + | DocumentBlockKind::List + | DocumentBlockKind::Row + | DocumentBlockKind::Cell + | DocumentBlockKind::Other { .. } => {} + } + } + Ok(lines.join("\n")) } -fn parse_xlsx_rows(xml: &str, shared_strings: &[String]) -> Result>, MediaError> { - let document = Document::parse(xml) - .map_err(|error| MediaError::Parse(format!("invalid worksheet XML: {error}")))?; - let mut rows = Vec::new(); - for row in document - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "row") +fn render_spreadsheet(artifact: &DocumentArtifact) -> Result { + let children = child_index(&artifact.blocks); + let mut sections = Vec::new(); + for sheet in artifact + .blocks + .iter() + .filter(|block| matches!(block.kind, DocumentBlockKind::Sheet)) { - let mut values = Vec::::new(); - for cell in row - .children() - .filter(|node| node.is_element() && node.tag_name().name() == "c") - { - let column = attribute_local(cell, "r") - .map(excel_column_index) - .unwrap_or(values.len()); - if values.len() <= column { - values.resize(column + 1, String::new()); - } - let cell_type = attribute_local(cell, "t").unwrap_or_default(); - let raw = cell - .descendants() - .find(|node| node.is_element() && node.tag_name().name() == "v") - .and_then(|node| node.text()) - .unwrap_or_default(); - values[column] = match cell_type { - "s" => raw - .parse::() - .ok() - .and_then(|index| shared_strings.get(index)) - .cloned() - .unwrap_or_default(), - "inlineStr" => cell - .descendants() - .filter(|node| node.is_element() && node.tag_name().name() == "t") - .filter_map(|node| node.text()) - .collect::(), - "b" => match raw { - "1" => "True".to_owned(), - "0" => "False".to_owned(), - _ => raw.to_owned(), - }, - _ => raw.to_owned(), + sections.push(format!("## Sheet: {}", sheet.text)); + let rows = children.get(&sheet.ordinal).cloned().unwrap_or_default(); + let mut rendered_rows = Vec::new(); + let mut max_column = 0_usize; + for row_ordinal in rows { + let Some(row) = artifact.blocks.get(row_ordinal as usize) else { + continue; }; + if !matches!(row.kind, DocumentBlockKind::Row) { + continue; + } + let cells = children.get(&row.ordinal).cloned().unwrap_or_default(); + let mut sparse = BTreeMap::new(); + for cell_ordinal in cells { + let Some(cell) = artifact.blocks.get(cell_ordinal as usize) else { + continue; + }; + if let DocumentLocator::Spreadsheet { column, .. } = cell.locator { + let column = usize::from(column); + max_column = max_column.max(column); + sparse.insert(column, cell.text.clone()); + } + } + rendered_rows.push(sparse); + } + if rendered_rows.is_empty() { + continue; } - if values.iter().any(|value| !value.is_empty()) { - rows.push(values); + for (index, row) in rendered_rows.iter().enumerate() { + let cells = (1..=max_column) + .map(|column| row.get(&column).cloned().unwrap_or_default()) + .collect::>(); + sections.push(markdown_row(&cells)); + if index == 0 { + sections.push(markdown_row(&vec!["---".to_owned(); max_column])); + } } } - Ok(rows) + Ok(sections.join("\n")) } -fn normalize_xlsx_target(target: &str) -> String { - if let Some(absolute) = target.strip_prefix('/') { - absolute.to_owned() - } else if target.starts_with("xl/") { - target.to_owned() - } else { - format!("xl/{target}") +fn child_index(blocks: &[DocumentBlock]) -> BTreeMap> { + let mut children = BTreeMap::>::new(); + for block in blocks { + if let Some(parent) = block.parent { + children.entry(parent).or_default().push(block.ordinal); + } } + children } -fn excel_column_index(reference: &str) -> usize { - reference - .bytes() - .take_while(u8::is_ascii_alphabetic) - .fold(0_usize, |value, byte| { - value - .saturating_mul(26) - .saturating_add(usize::from(byte.to_ascii_uppercase() - b'A' + 1)) - }) - .saturating_sub(1) +fn render_table( + artifact: &DocumentArtifact, + table: u32, + children: &BTreeMap>, + skipped: &mut std::collections::BTreeSet, +) -> Vec { + let mut rows = Vec::new(); + for row_ordinal in children.get(&table).into_iter().flatten() { + skipped.insert(*row_ordinal); + let mut cells = Vec::new(); + for cell_ordinal in children.get(row_ordinal).into_iter().flatten() { + skipped.insert(*cell_ordinal); + if let Some(cell) = artifact.blocks.get(*cell_ordinal as usize) { + cells.push(escape_markdown_cell(&cell.text)); + } + } + if !cells.is_empty() { + rows.push(cells); + } + } + let Some(header) = rows.first() else { + return Vec::new(); + }; + let mut rendered = vec![markdown_row(header)]; + rendered.push(markdown_row(&vec!["---".to_owned(); header.len()])); + rendered.extend(rows.iter().skip(1).map(|row| markdown_row(row))); + rendered } -fn attribute_local<'a>(node: Node<'a, 'a>, name: &str) -> Option<&'a str> { - node.attributes() - .find(|attribute| attribute.name() == name) - .map(|attribute| attribute.value()) +fn escape_markdown_cell(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('|', "\\|") + .replace(['\r', '\n'], "
") } fn markdown_row(cells: &[String]) -> String { format!("| {} |", cells.join(" | ")) } +fn one_based_u32(index: usize, field: &str) -> Result { + u32::try_from(index) + .ok() + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| format!("{field} index overflow")) +} + fn extension(path: &Path) -> String { path.extension() .and_then(|extension| extension.to_str()) @@ -447,7 +353,7 @@ fn extension(path: &Path) -> String { #[cfg(test)] mod tests { use std::error::Error; - use std::fs::File; + use std::fs::{self, File}; use std::io::Write; use tempfile::tempdir; @@ -472,7 +378,7 @@ mod tests { } #[test] - fn converts_docx_paragraphs_styles_and_tables_like_python() -> TestResult { + fn docx_preserves_paragraph_table_paragraph_order() -> TestResult { let directory = tempdir()?; let path = directory.path().join("sample.docx"); write_zip( @@ -480,24 +386,23 @@ mod tests { &[ ( "word/styles.xml", - r#""#, + r#""#, ), ( "word/document.xml", - r#"TitleItemNameValueA1"#, + r#"TitleNameValueA1After"#, ), ], )?; - assert_eq!( docx_to_markdown(&path)?, - "# Title\n\n- Item\n| Name | Value |\n| --- | --- |\n| A | 1 |" + "# Title\n| Name | Value |\n| --- | --- |\n| A | 1 |\nAfter" ); Ok(()) } #[test] - fn converts_xlsx_shared_inline_boolean_and_sparse_cells() -> TestResult { + fn xlsx_is_sparse_typed_and_bounded() -> TestResult { let directory = tempdir()?; let path = directory.path().join("sample.xlsx"); write_zip( @@ -517,51 +422,55 @@ mod tests { ), ( "xl/worksheets/sheet1.xml", - r#"01Alice142"#, + r#"01Alice120+2242"#, ), ], )?; - + let artifact = decode_document(&path, &fs::read(&path)?)?; assert_eq!( - xlsx_to_markdown(&path)?, + artifact + .blocks + .iter() + .filter(|b| matches!(b.kind, DocumentBlockKind::Cell)) + .count(), + 5 + ); + assert_eq!( + render_document_markdown(&artifact)?, "## Sheet: Main\n| Name | | Value |\n| --- | --- | --- |\n| Alice | True | 42 |" ); + assert!( + artifact + .blocks + .iter() + .any(|block| block.metadata.contains_key("formula")) + ); Ok(()) } #[test] - fn rejects_non_zip_office_documents() -> TestResult { + fn rejects_non_zip_and_high_ratio_archives() -> TestResult { let directory = tempdir()?; - let path = directory.path().join("fake.xlsx"); - fs::write(&path, b"not a zip")?; + let fake = directory.path().join("fake.xlsx"); + fs::write(&fake, b"not a zip")?; + assert!(validate_office_archive(&fake).is_err()); - assert!(validate_office_archive(&path).is_err()); - assert_eq!(extract_text_compat(&path), ""); - Ok(()) - } - - #[test] - fn rejects_high_ratio_office_archives() -> TestResult { - let directory = tempdir()?; - let path = directory.path().join("bomb.docx"); + let bomb = directory.path().join("bomb.docx"); let payload = "0".repeat(5 * 1024 * 1024); - write_zip(&path, &[("word/document.xml", &payload)])?; - + write_zip(&bomb, &[("word/document.xml", &payload)])?; assert!(matches!( - validate_office_archive(&path), + validate_office_archive(&bomb), Err(MediaError::Rejected(_)) )); - assert_eq!(docx_to_markdown(&path).unwrap_or_default(), ""); Ok(()) } #[test] - fn rejects_raw_files_over_the_cap_without_reading_them() -> TestResult { + fn rejects_raw_files_over_cap() -> TestResult { let directory = tempdir()?; let path = directory.path().join("oversize.pdf"); let file = File::create(&path)?; file.set_len(MEDIA_MAX_RAW_BYTES + 1)?; - assert!(matches!( extract_pdf_text(&path), Err(MediaError::Rejected(_)) @@ -574,7 +483,6 @@ mod tests { let directory = tempdir()?; let path = directory.path().join("notes.txt"); fs::write(&path, b"hello\xffworld")?; - assert_eq!(extract_text(&path)?, "hello\u{fffd}world"); Ok(()) } diff --git a/crates/compass-media/src/limits.rs b/crates/compass-media/src/limits.rs new file mode 100644 index 00000000..7655def3 --- /dev/null +++ b/crates/compass-media/src/limits.rs @@ -0,0 +1,24 @@ +//! Central resource ceilings for untrusted document inputs and artifacts. + +pub const MEDIA_MAX_RAW_BYTES: u64 = 50 * 1024 * 1024; +pub const OFFICE_MAX_DECOMPRESSED_BYTES: u64 = 512 * 1024 * 1024; +pub const OFFICE_MAX_COMPRESSION_RATIO: u64 = 200; +pub const OFFICE_MEMBER_MAX_BYTES: u64 = 64 * 1024 * 1024; +pub const OFFICE_MAX_MEMBERS: usize = 100_000; + +pub const XLSX_MAX_COLUMNS: usize = 16_384; +pub const XLSX_MAX_ROWS: u32 = 100_000; +pub const XLSX_MAX_CELLS: usize = 1_000_000; + +pub const DOCUMENT_MAX_BLOCKS: usize = 1_000_000; +pub const DOCUMENT_MAX_LINKS: usize = 1_000_000; +pub const DOCUMENT_MAX_DIAGNOSTICS: usize = 10_000; +pub const DOCUMENT_MAX_METADATA_ENTRIES: usize = 1_024; +pub const DOCUMENT_MAX_TEXT_CHARS: usize = 20_000_000; +pub const DOCUMENT_MAX_FIELD_BYTES: usize = 16 * 1024; +pub const DOCUMENT_MAX_DIAGNOSTIC_MESSAGE_BYTES: usize = 4 * 1024; +pub const DOCUMENT_MAX_DEPTH: usize = 256; + +pub const OCR_MAX_PDF_PAGES: usize = 200; +pub const OCR_MAX_OOXML_IMAGES: usize = 256; +pub const OCR_MAX_AGGREGATE_PIXELS: u64 = 300_000_000; diff --git a/crates/compass-media/src/ooxml.rs b/crates/compass-media/src/ooxml.rs new file mode 100644 index 00000000..4652df59 --- /dev/null +++ b/crates/compass-media/src/ooxml.rs @@ -0,0 +1,1459 @@ +//! Bounded Open Packaging Convention reader and native OOXML decoders. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Cursor, Read}; + +use roxmltree::{Document, Node}; +use serde::{Deserialize, Serialize}; +use zip::ZipArchive; + +use crate::document::{ + DiagnosticSeverity, DocumentArtifact, DocumentBlockKind, DocumentDiagnostic, DocumentError, + DocumentFormat, DocumentLink, DocumentLinkKind, DocumentLocator, +}; +use crate::limits::{ + DOCUMENT_MAX_DEPTH, DOCUMENT_MAX_TEXT_CHARS, OFFICE_MAX_COMPRESSION_RATIO, + OFFICE_MAX_DECOMPRESSED_BYTES, OFFICE_MAX_MEMBERS, OFFICE_MEMBER_MAX_BYTES, XLSX_MAX_CELLS, + XLSX_MAX_COLUMNS, XLSX_MAX_ROWS, +}; + +const XML_MAX_EVENTS: usize = 1_000_000; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct RasterCandidate { + pub id: String, + pub owner: DocumentLocator, + pub part: String, + pub media_type: String, + pub bytes: Vec, +} + +#[derive(Clone, Debug)] +struct Relationship { + target: String, + kind: String, + external: bool, +} + +pub(crate) struct Package { + parts: BTreeMap>, +} + +impl Package { + pub(crate) fn open(bytes: &[u8]) -> Result { + let mut archive = ZipArchive::new(Cursor::new(bytes)) + .map_err(|error| DocumentError::Parse(format!("invalid ZIP container: {error}")))?; + if archive.len() > OFFICE_MAX_MEMBERS { + return Err(DocumentError::Rejected( + "office member count exceeds safety limit".to_owned(), + )); + } + let mut compressed = 0_u64; + let mut declared = 0_u64; + let mut names = BTreeSet::new(); + for index in 0..archive.len() { + let member = archive + .by_index_raw(index) + .map_err(|error| DocumentError::Parse(error.to_string()))?; + compressed = compressed + .checked_add(member.compressed_size()) + .ok_or_else(|| DocumentError::Rejected("compressed size overflow".to_owned()))?; + declared = declared + .checked_add(member.size()) + .ok_or_else(|| DocumentError::Rejected("declared size overflow".to_owned()))?; + if member.size() > OFFICE_MEMBER_MAX_BYTES { + return Err(DocumentError::Rejected(format!( + "office member {} exceeds safety limit", + member.name() + ))); + } + if !names.insert(normalize_part_name(member.name())?) { + return Err(DocumentError::Rejected( + "office package contains duplicate normalized part names".to_owned(), + )); + } + } + if declared > OFFICE_MAX_DECOMPRESSED_BYTES { + return Err(DocumentError::Rejected(format!( + "declared office payload is {declared} bytes" + ))); + } + let ratio_limit = compressed + .max(1) + .saturating_mul(OFFICE_MAX_COMPRESSION_RATIO); + if declared > ratio_limit { + return Err(DocumentError::Rejected( + "office compression ratio exceeds safety limit".to_owned(), + )); + } + let mut actual = 0_u64; + let mut parts = BTreeMap::new(); + for index in 0..archive.len() { + let member = archive + .by_index(index) + .map_err(|error| DocumentError::Parse(error.to_string()))?; + let name = normalize_part_name(member.name())?; + let member_capacity = usize::try_from(member.size()).map_err(|_| { + DocumentError::Rejected("office member cannot fit in memory".to_owned()) + })?; + let mut contents = Vec::with_capacity(member_capacity); + member + .take(OFFICE_MEMBER_MAX_BYTES.saturating_add(1)) + .read_to_end(&mut contents) + .map_err(|error| DocumentError::Parse(error.to_string()))?; + actual = actual + .checked_add(contents.len() as u64) + .ok_or_else(|| DocumentError::Rejected("actual size overflow".to_owned()))?; + if actual > OFFICE_MAX_DECOMPRESSED_BYTES + || contents.len() as u64 > OFFICE_MEMBER_MAX_BYTES + { + return Err(DocumentError::Rejected( + "decompressed office payload exceeds safety limit".to_owned(), + )); + } + parts.insert(name, contents); + } + Ok(Self { parts }) + } + + fn required_text(&self, name: &str) -> Result<&str, DocumentError> { + let bytes = self.parts.get(name).ok_or_else(|| { + DocumentError::Parse(format!("office package is missing required part {name}")) + })?; + parse_xml_text(name, bytes) + } + + fn optional_text(&self, name: &str) -> Result, DocumentError> { + self.parts + .get(name) + .map(|bytes| parse_xml_text(name, bytes)) + .transpose() + } + + fn relationships( + &self, + source_part: &str, + ) -> Result, DocumentError> { + let relationship_part = relationship_part_name(source_part)?; + let Some(xml) = self.optional_text(&relationship_part)? else { + return Ok(BTreeMap::new()); + }; + let document = parse_xml(&relationship_part, xml)?; + let mut relationships = BTreeMap::new(); + for node in document + .descendants() + .filter(|node| local_name(*node) == "Relationship") + { + let Some(id) = attribute_local(node, "Id") else { + continue; + }; + let target = attribute_local(node, "Target").unwrap_or_default(); + let external = attribute_local(node, "TargetMode") == Some("External"); + let resolved = if external { + bounded(target, "external relationship target")?.to_owned() + } else { + resolve_target(source_part, target)? + }; + let relationship = Relationship { + target: resolved, + kind: attribute_local(node, "Type").unwrap_or_default().to_owned(), + external, + }; + if relationships.insert(id.to_owned(), relationship).is_some() { + return Err(DocumentError::Parse(format!( + "duplicate relationship ID {id:?} in {relationship_part}" + ))); + } + } + Ok(relationships) + } +} + +pub fn decode_docx(bytes: &[u8]) -> Result { + let package = Package::open(bytes)?; + let styles = package + .optional_text("word/styles.xml")? + .map(parse_docx_styles) + .transpose()? + .unwrap_or_default(); + let document = parse_xml( + "word/document.xml", + package.required_text("word/document.xml")?, + )?; + let body = document + .descendants() + .find(|node| local_name(*node) == "body") + .ok_or_else(|| DocumentError::Parse("DOCX document has no body".to_owned()))?; + let relationships = package.relationships("word/document.xml")?; + let mut artifact = DocumentArtifact::new(DocumentFormat::Docx); + let mut occurrence = 0_u32; + for child in body.children().filter(Node::is_element) { + occurrence = occurrence + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("DOCX occurrence overflow".to_owned()))?; + match local_name(child) { + "p" => emit_docx_paragraph( + child, + None, + occurrence, + &styles, + &relationships, + &mut artifact, + )?, + "tbl" => emit_docx_table(child, occurrence, &mut artifact)?, + "sectPr" => {} + other => mark_unsupported( + &mut artifact, + "docx_unsupported_body_element", + format!("DOCX body element {other} was not decoded"), + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/*[{occurrence}]"), + }, + ), + } + } + emit_optional_docx_part(&package, "word/footnotes.xml", "footnote", &mut artifact)?; + emit_optional_docx_part(&package, "word/endnotes.xml", "endnote", &mut artifact)?; + emit_optional_docx_part(&package, "word/comments.xml", "comment", &mut artifact)?; + record_ooxml_images(&package, "word/", &mut artifact); + artifact.validate()?; + Ok(artifact) +} + +pub fn decode_xlsx(bytes: &[u8]) -> Result { + let package = Package::open(bytes)?; + let workbook = parse_xml("xl/workbook.xml", package.required_text("xl/workbook.xml")?)?; + let relationships = package.relationships("xl/workbook.xml")?; + let shared_strings = package + .optional_text("xl/sharedStrings.xml")? + .map(parse_shared_strings) + .transpose()? + .unwrap_or_default(); + let mut artifact = DocumentArtifact::new(DocumentFormat::Xlsx); + let mut total_cells = 0_usize; + for sheet in workbook + .descendants() + .filter(|node| local_name(*node) == "sheet") + { + let name = bounded( + attribute_local(sheet, "name").unwrap_or("Sheet"), + "sheet name", + )?; + let relation_id = attribute_local(sheet, "id").unwrap_or_default(); + let relationship = relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!("worksheet relationship {relation_id:?} is missing")) + })?; + if relationship.external { + return Err(DocumentError::Parse( + "worksheet relationship cannot be external".to_owned(), + )); + } + let sheet_block = artifact.push_block( + None, + DocumentBlockKind::Sheet, + name.to_owned(), + DocumentLocator::Spreadsheet { + sheet: name.to_owned(), + row: 1, + column: 1, + }, + )?; + let worksheet = parse_xml( + &relationship.target, + package.required_text(&relationship.target)?, + )?; + for (row_occurrence, row) in worksheet + .descendants() + .filter(|node| local_name(*node) == "row") + .enumerate() + { + let fallback_row = one_based(row_occurrence, "row")?; + let row_number = attribute_local(row, "r") + .and_then(|value| value.parse::().ok()) + .unwrap_or(fallback_row); + if row_number == 0 || row_number > XLSX_MAX_ROWS { + return Err(DocumentError::Rejected(format!( + "worksheet row {row_number} exceeds the supported bound" + ))); + } + let row_block = artifact.push_block( + Some(sheet_block), + DocumentBlockKind::Row, + String::new(), + DocumentLocator::Spreadsheet { + sheet: name.to_owned(), + row: row_number, + column: 1, + }, + )?; + let mut fallback_column = 1_usize; + for cell in row.children().filter(|node| local_name(*node) == "c") { + total_cells = total_cells + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("cell count overflow".to_owned()))?; + if total_cells > XLSX_MAX_CELLS { + return Err(DocumentError::Rejected( + "worksheet cell count exceeds safety limit".to_owned(), + )); + } + let (reference_row, column) = attribute_local(cell, "r") + .map(parse_cell_reference) + .transpose()? + .unwrap_or((row_number, fallback_column)); + if reference_row != row_number { + return Err(DocumentError::Parse(format!( + "cell row {reference_row} does not match containing row {row_number}" + ))); + } + fallback_column = column + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("column overflow".to_owned()))?; + let (value, value_kind) = parse_cell_value(cell, &shared_strings)?; + let formula = cell + .descendants() + .find(|node| local_name(*node) == "f") + .and_then(|node| node.text()); + if value.is_empty() && formula.is_none() { + continue; + } + let column_u16 = u16::try_from(column) + .map_err(|_| DocumentError::Rejected("column exceeds u16".to_owned()))?; + let ordinal = artifact.push_block( + Some(row_block), + DocumentBlockKind::Cell, + value, + DocumentLocator::Spreadsheet { + sheet: name.to_owned(), + row: row_number, + column: column_u16, + }, + )?; + let block = artifact.blocks.get_mut(ordinal as usize).ok_or_else(|| { + DocumentError::InvalidArtifact("new cell block disappeared".to_owned()) + })?; + block + .metadata + .insert("value_kind".to_owned(), serde_json::json!(value_kind)); + if let Some(formula) = formula { + block.metadata.insert( + "formula".to_owned(), + serde_json::json!(bounded(formula, "formula")?), + ); + } + } + } + } + record_ooxml_images(&package, "xl/", &mut artifact); + artifact.validate()?; + Ok(artifact) +} + +pub fn decode_pptx(bytes: &[u8]) -> Result { + let package = Package::open(bytes)?; + let presentation = parse_xml( + "ppt/presentation.xml", + package.required_text("ppt/presentation.xml")?, + )?; + let relationships = package.relationships("ppt/presentation.xml")?; + let mut artifact = DocumentArtifact::new(DocumentFormat::Pptx); + let mut slide_number = 0_u32; + for slide_id in presentation + .descendants() + .filter(|node| local_name(*node) == "sldId") + { + slide_number = slide_number + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("slide index overflow".to_owned()))?; + let relation_id = attribute_local(slide_id, "id").unwrap_or_default(); + let relationship = relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!("slide relationship {relation_id:?} is missing")) + })?; + if relationship.external { + return Err(DocumentError::Parse( + "slide relationship cannot be external".to_owned(), + )); + } + let slide_block = artifact.push_block( + None, + DocumentBlockKind::Slide, + String::new(), + DocumentLocator::Slide { + slide: slide_number, + shape: 1, + }, + )?; + let slide = parse_xml( + &relationship.target, + package.required_text(&relationship.target)?, + )?; + let slide_relationships = package.relationships(&relationship.target)?; + let shape_tree = slide + .descendants() + .find(|node| local_name(*node) == "spTree") + .unwrap_or(slide.root_element()); + let mut shape_number = 1_u32; + for shape in shape_tree.children().filter(Node::is_element) { + let name = local_name(shape); + if !matches!(name, "sp" | "graphicFrame" | "pic" | "cxnSp") { + continue; + } + shape_number = shape_number + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("shape index overflow".to_owned()))?; + if name == "graphicFrame" && shape.descendants().any(|node| local_name(node) == "tbl") { + emit_pptx_table( + shape, + slide_number, + shape_number, + slide_block, + &mut artifact, + )?; + continue; + } + let text = collect_text(shape); + if !text.is_empty() { + let is_title = shape.descendants().any(|node| { + local_name(node) == "ph" + && matches!(attribute_local(node, "type"), Some("title" | "ctrTitle")) + }); + let kind = if is_title { + DocumentBlockKind::Heading { level: 1 } + } else { + DocumentBlockKind::Paragraph + }; + let ordinal = artifact.push_block( + Some(slide_block), + kind, + text, + DocumentLocator::Slide { + slide: slide_number, + shape: shape_number, + }, + )?; + if let Some(c_nv_pr) = shape + .descendants() + .find(|node| local_name(*node) == "cNvPr") + { + let block = artifact.blocks.get_mut(ordinal as usize).ok_or_else(|| { + DocumentError::InvalidArtifact("new shape block disappeared".to_owned()) + })?; + for (key, attribute) in [ + ("shape_name", "name"), + ("alt_text", "descr"), + ("title", "title"), + ] { + if let Some(value) = attribute_local(c_nv_pr, attribute) { + block + .metadata + .insert(key.to_owned(), serde_json::json!(bounded(value, key)?)); + } + } + } + } + emit_shape_links( + shape, + slide_number, + shape_number, + slide_block, + &slide_relationships, + &mut artifact, + ); + } + emit_slide_notes( + &package, + &relationship.target, + slide_number, + slide_block, + &slide_relationships, + &mut artifact, + )?; + if slide.descendants().any(|node| { + matches!( + local_name(node), + "chart" | "diagram" | "oleObj" | "video" | "audio" + ) + }) { + mark_unsupported( + &mut artifact, + "pptx_visual_object_not_decoded", + "A chart, diagram, OLE object, or media item was retained as unsupported evidence" + .to_owned(), + DocumentLocator::Slide { + slide: slide_number, + shape: 1, + }, + ); + } + } + record_ooxml_images(&package, "ppt/", &mut artifact); + artifact.validate()?; + Ok(artifact) +} + +pub fn raster_candidates( + logical_extension: &str, + bytes: &[u8], +) -> Result, DocumentError> { + let package = Package::open(bytes)?; + let mut candidates = Vec::new(); + match logical_extension { + "docx" => docx_raster_candidates(&package, &mut candidates)?, + "pptx" => pptx_raster_candidates(&package, &mut candidates)?, + "xlsx" => xlsx_raster_candidates(&package, &mut candidates)?, + _ => {} + } + Ok(candidates) +} + +fn push_raster_candidate( + package: &Package, + relationship: &Relationship, + owner: DocumentLocator, + candidates: &mut Vec, +) -> Result<(), DocumentError> { + if relationship.external || !relationship.kind.ends_with("/image") { + return Ok(()); + } + let Some(media_type) = media_type(&relationship.target) else { + return Ok(()); + }; + let contents = package.parts.get(&relationship.target).ok_or_else(|| { + DocumentError::Parse(format!( + "embedded image relationship target {} is missing", + relationship.target + )) + })?; + let occurrence = one_based(candidates.len(), "image occurrence")?; + candidates.push(RasterCandidate { + id: format!("ooxml-image-{occurrence}"), + owner, + part: relationship.target.clone(), + media_type: media_type.to_owned(), + bytes: contents.clone(), + }); + Ok(()) +} + +fn docx_raster_candidates( + package: &Package, + candidates: &mut Vec, +) -> Result<(), DocumentError> { + let source_part = "word/document.xml"; + let document = parse_xml(source_part, package.required_text(source_part)?)?; + let relationships = package.relationships(source_part)?; + let body = document + .descendants() + .find(|node| local_name(*node) == "body") + .ok_or_else(|| DocumentError::Parse("DOCX document has no body".to_owned()))?; + for (child_index, child) in body.children().filter(Node::is_element).enumerate() { + let mut image_index = 0_usize; + for blip in child + .descendants() + .filter(|node| local_name(*node) == "blip") + { + let Some(relation_id) = attribute_local(blip, "embed") else { + continue; + }; + let relationship = relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!("image relationship {relation_id:?} is missing")) + })?; + image_index = image_index.saturating_add(1); + push_raster_candidate( + package, + relationship, + DocumentLocator::Package { + part: source_part.to_owned(), + path: format!( + "body/*[{}]/image[{image_index}]", + one_based(child_index, "DOCX body")? + ), + }, + candidates, + )?; + } + } + Ok(()) +} + +fn pptx_raster_candidates( + package: &Package, + candidates: &mut Vec, +) -> Result<(), DocumentError> { + let presentation = parse_xml( + "ppt/presentation.xml", + package.required_text("ppt/presentation.xml")?, + )?; + let presentation_relationships = package.relationships("ppt/presentation.xml")?; + for (slide_index, slide_id) in presentation + .descendants() + .filter(|node| local_name(*node) == "sldId") + .enumerate() + { + let slide_number = one_based(slide_index, "slide")?; + let relation_id = attribute_local(slide_id, "id").unwrap_or_default(); + let slide_relationship = presentation_relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!("slide relationship {relation_id:?} is missing")) + })?; + if slide_relationship.external { + continue; + } + let slide = parse_xml( + &slide_relationship.target, + package.required_text(&slide_relationship.target)?, + )?; + let relationships = package.relationships(&slide_relationship.target)?; + let shape_tree = slide + .descendants() + .find(|node| local_name(*node) == "spTree") + .unwrap_or(slide.root_element()); + let mut shape_number = 1_u32; + for shape in shape_tree.children().filter(Node::is_element) { + if !matches!(local_name(shape), "sp" | "graphicFrame" | "pic" | "cxnSp") { + continue; + } + shape_number = shape_number + .checked_add(1) + .ok_or_else(|| DocumentError::Rejected("shape index overflow".to_owned()))?; + for blip in shape + .descendants() + .filter(|node| local_name(*node) == "blip") + { + let Some(relation_id) = attribute_local(blip, "embed") else { + continue; + }; + let relationship = relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!( + "slide image relationship {relation_id:?} is missing" + )) + })?; + push_raster_candidate( + package, + relationship, + DocumentLocator::Slide { + slide: slide_number, + shape: shape_number, + }, + candidates, + )?; + } + } + } + Ok(()) +} + +fn xlsx_raster_candidates( + package: &Package, + candidates: &mut Vec, +) -> Result<(), DocumentError> { + let workbook = parse_xml("xl/workbook.xml", package.required_text("xl/workbook.xml")?)?; + let workbook_relationships = package.relationships("xl/workbook.xml")?; + for sheet in workbook + .descendants() + .filter(|node| local_name(*node) == "sheet") + { + let name = bounded( + attribute_local(sheet, "name").unwrap_or("Sheet"), + "sheet name", + )?; + let relation_id = attribute_local(sheet, "id").unwrap_or_default(); + let sheet_relationship = workbook_relationships.get(relation_id).ok_or_else(|| { + DocumentError::Parse(format!("worksheet relationship {relation_id:?} is missing")) + })?; + if sheet_relationship.external { + continue; + } + let worksheet = parse_xml( + &sheet_relationship.target, + package.required_text(&sheet_relationship.target)?, + )?; + let sheet_relationships = package.relationships(&sheet_relationship.target)?; + for drawing in worksheet + .descendants() + .filter(|node| local_name(*node) == "drawing") + { + let Some(drawing_id) = attribute_local(drawing, "id") else { + continue; + }; + let drawing_relationship = sheet_relationships.get(drawing_id).ok_or_else(|| { + DocumentError::Parse(format!("drawing relationship {drawing_id:?} is missing")) + })?; + if drawing_relationship.external { + continue; + } + let drawing_document = parse_xml( + &drawing_relationship.target, + package.required_text(&drawing_relationship.target)?, + )?; + let image_relationships = package.relationships(&drawing_relationship.target)?; + for anchor in drawing_document + .root_element() + .children() + .filter(Node::is_element) + { + if !matches!( + local_name(anchor), + "oneCellAnchor" | "twoCellAnchor" | "absoluteAnchor" + ) { + continue; + } + let from = anchor.children().find(|node| local_name(*node) == "from"); + let row = from + .and_then(|node| node.children().find(|child| local_name(*child) == "row")) + .and_then(|node| node.text()) + .and_then(|value| value.parse::().ok()) + .and_then(|value| value.checked_add(1)) + .unwrap_or(1); + let column = from + .and_then(|node| node.children().find(|child| local_name(*child) == "col")) + .and_then(|node| node.text()) + .and_then(|value| value.parse::().ok()) + .and_then(|value| value.checked_add(1)) + .unwrap_or(1); + for blip in anchor + .descendants() + .filter(|node| local_name(*node) == "blip") + { + let Some(image_id) = attribute_local(blip, "embed") else { + continue; + }; + let relationship = image_relationships.get(image_id).ok_or_else(|| { + DocumentError::Parse(format!( + "drawing image relationship {image_id:?} is missing" + )) + })?; + push_raster_candidate( + package, + relationship, + DocumentLocator::Spreadsheet { + sheet: name.to_owned(), + row, + column, + }, + candidates, + )?; + } + } + } + } + Ok(()) +} + +fn emit_docx_paragraph( + node: Node<'_, '_>, + parent: Option, + occurrence: u32, + styles: &BTreeMap, + relationships: &BTreeMap, + artifact: &mut DocumentArtifact, +) -> Result<(), DocumentError> { + let text = collect_docx_text(node); + let style_id = node + .descendants() + .find(|descendant| local_name(*descendant) == "pStyle") + .and_then(|style| attribute_local(style, "val")) + .unwrap_or_default(); + let style_name = styles.get(style_id).map_or(style_id, String::as_str); + let normalized_style = style_name.to_ascii_lowercase(); + let kind = if let Some(level) = heading_level(&normalized_style) { + DocumentBlockKind::Heading { level } + } else if node + .descendants() + .any(|descendant| local_name(descendant) == "numPr") + || normalized_style.starts_with("list") + { + DocumentBlockKind::ListItem + } else { + DocumentBlockKind::Paragraph + }; + let ordinal = artifact.push_block( + parent, + kind, + text, + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/p[{occurrence}]"), + }, + )?; + for hyperlink in node + .descendants() + .filter(|node| local_name(*node) == "hyperlink") + { + let Some(relation_id) = attribute_local(hyperlink, "id") else { + continue; + }; + let Some(relationship) = relationships.get(relation_id) else { + continue; + }; + artifact.links.push(DocumentLink { + source_block: ordinal, + destination: relationship.target.clone(), + label: Some(collect_docx_text(hyperlink)), + relationship: DocumentLinkKind::Hyperlink, + locator: DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/p[{occurrence}]/hyperlink"), + }, + external: relationship.external, + }); + } + Ok(()) +} + +fn emit_docx_table( + table: Node<'_, '_>, + occurrence: u32, + artifact: &mut DocumentArtifact, +) -> Result<(), DocumentError> { + let table_block = artifact.push_block( + None, + DocumentBlockKind::Table, + String::new(), + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/tbl[{occurrence}]"), + }, + )?; + for (row_index, row) in table + .children() + .filter(|node| local_name(*node) == "tr") + .enumerate() + { + let row_number = one_based(row_index, "DOCX row")?; + let row_block = artifact.push_block( + Some(table_block), + DocumentBlockKind::Row, + String::new(), + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/tbl[{occurrence}]/tr[{row_number}]"), + }, + )?; + for (cell_index, cell) in row + .children() + .filter(|node| local_name(*node) == "tc") + .enumerate() + { + let cell_number = one_based(cell_index, "DOCX cell")?; + artifact.push_block( + Some(row_block), + DocumentBlockKind::Cell, + cell.children() + .filter(|node| local_name(*node) == "p") + .map(collect_docx_text) + .collect::>() + .join("\n"), + DocumentLocator::Package { + part: "word/document.xml".to_owned(), + path: format!("body/tbl[{occurrence}]/tr[{row_number}]/tc[{cell_number}]"), + }, + )?; + } + } + Ok(()) +} + +fn emit_optional_docx_part( + package: &Package, + part: &str, + element_name: &str, + artifact: &mut DocumentArtifact, +) -> Result<(), DocumentError> { + let Some(xml) = package.optional_text(part)? else { + return Ok(()); + }; + let document = parse_xml(part, xml)?; + for (index, node) in document + .descendants() + .filter(|node| local_name(*node) == element_name) + .enumerate() + { + let text = collect_docx_text(node); + if text.is_empty() { + continue; + } + let occurrence = one_based(index, "DOCX note")?; + artifact.push_block( + None, + DocumentBlockKind::Note, + text, + DocumentLocator::Package { + part: part.to_owned(), + path: format!("{element_name}[{occurrence}]"), + }, + )?; + } + Ok(()) +} + +fn parse_docx_styles(xml: &str) -> Result, DocumentError> { + let document = parse_xml("word/styles.xml", xml)?; + let mut styles = BTreeMap::new(); + for style in document + .descendants() + .filter(|node| local_name(*node) == "style") + { + let Some(id) = attribute_local(style, "styleId") else { + continue; + }; + let name = style + .children() + .find(|node| local_name(*node) == "name") + .and_then(|node| attribute_local(node, "val")) + .unwrap_or(id); + styles.insert(id.to_owned(), name.to_owned()); + } + Ok(styles) +} + +fn parse_shared_strings(xml: &str) -> Result, DocumentError> { + let document = parse_xml("xl/sharedStrings.xml", xml)?; + let values = document + .descendants() + .filter(|node| local_name(*node) == "si") + .map(collect_text) + .collect::>(); + let characters = values.iter().try_fold(0_usize, |total, value| { + total + .checked_add(value.chars().count()) + .ok_or_else(|| DocumentError::Rejected("shared string size overflow".to_owned())) + })?; + if characters > DOCUMENT_MAX_TEXT_CHARS { + return Err(DocumentError::Rejected( + "shared strings exceed document text limit".to_owned(), + )); + } + Ok(values) +} + +fn parse_cell_value( + cell: Node<'_, '_>, + shared_strings: &[String], +) -> Result<(String, &'static str), DocumentError> { + let kind = attribute_local(cell, "t").unwrap_or_default(); + let raw = cell + .descendants() + .find(|node| local_name(*node) == "v") + .and_then(|node| node.text()) + .unwrap_or_default(); + match kind { + "s" => { + let index = raw.parse::().map_err(|_| { + DocumentError::Parse(format!("invalid shared string index {raw:?}")) + })?; + Ok(( + shared_strings + .get(index) + .ok_or_else(|| { + DocumentError::Parse(format!("shared string index {index} is missing")) + })? + .clone(), + "shared_string", + )) + } + "inlineStr" => Ok((collect_text(cell), "inline_string")), + "b" => match raw { + "1" => Ok(("True".to_owned(), "boolean")), + "0" => Ok(("False".to_owned(), "boolean")), + _ => Err(DocumentError::Parse(format!( + "invalid spreadsheet boolean {raw:?}" + ))), + }, + "e" => Ok((raw.to_owned(), "error")), + "d" => Ok((raw.to_owned(), "iso_date")), + "str" => Ok((raw.to_owned(), "formula_string")), + "" | "n" => Ok((raw.to_owned(), "number")), + other => Err(DocumentError::Parse(format!( + "unsupported spreadsheet cell type {other:?}" + ))), + } +} + +fn parse_cell_reference(reference: &str) -> Result<(u32, usize), DocumentError> { + let letters = reference + .bytes() + .take_while(u8::is_ascii_alphabetic) + .collect::>(); + if letters.is_empty() || letters.len() > 3 { + return Err(DocumentError::Rejected(format!( + "invalid spreadsheet cell reference {reference:?}" + ))); + } + let mut column = 0_usize; + for &byte in &letters { + let digit = usize::from(byte.to_ascii_uppercase() - b'A' + 1); + column = column + .checked_mul(26) + .and_then(|value| value.checked_add(digit)) + .ok_or_else(|| DocumentError::Rejected("spreadsheet column overflow".to_owned()))?; + } + if column == 0 || column > XLSX_MAX_COLUMNS { + return Err(DocumentError::Rejected(format!( + "spreadsheet column {column} exceeds the supported bound" + ))); + } + let row = reference + .get(letters.len()..) + .unwrap_or_default() + .parse::() + .map_err(|_| { + DocumentError::Rejected(format!("invalid spreadsheet row in {reference:?}")) + })?; + if row == 0 || row > XLSX_MAX_ROWS { + return Err(DocumentError::Rejected(format!( + "spreadsheet row {row} exceeds the supported bound" + ))); + } + Ok((row, column)) +} + +fn emit_pptx_table( + shape: Node<'_, '_>, + slide: u32, + shape_number: u32, + slide_block: u32, + artifact: &mut DocumentArtifact, +) -> Result<(), DocumentError> { + let table = shape + .descendants() + .find(|node| local_name(*node) == "tbl") + .ok_or_else(|| DocumentError::Parse("PPTX table disappeared".to_owned()))?; + let table_block = artifact.push_block( + Some(slide_block), + DocumentBlockKind::Table, + String::new(), + DocumentLocator::Slide { + slide, + shape: shape_number, + }, + )?; + for (row_index, row) in table + .children() + .filter(|node| local_name(*node) == "tr") + .enumerate() + { + let row_shape = shape_number + .checked_add(one_based(row_index, "PPTX row")?) + .ok_or_else(|| DocumentError::Rejected("PPTX shape overflow".to_owned()))?; + let row_block = artifact.push_block( + Some(table_block), + DocumentBlockKind::Row, + String::new(), + DocumentLocator::Slide { + slide, + shape: row_shape, + }, + )?; + for (cell_index, cell) in row + .children() + .filter(|node| local_name(*node) == "tc") + .enumerate() + { + let cell_shape = row_shape + .checked_add(one_based(cell_index, "PPTX cell")?) + .ok_or_else(|| DocumentError::Rejected("PPTX cell locator overflow".to_owned()))?; + artifact.push_block( + Some(row_block), + DocumentBlockKind::Cell, + collect_text(cell), + DocumentLocator::Slide { + slide, + shape: cell_shape, + }, + )?; + } + } + Ok(()) +} + +fn emit_shape_links( + shape: Node<'_, '_>, + slide: u32, + shape_number: u32, + source_block: u32, + relationships: &BTreeMap, + artifact: &mut DocumentArtifact, +) { + for hyperlink in shape + .descendants() + .filter(|node| matches!(local_name(*node), "hlinkClick" | "hlinkHover")) + { + let Some(relation_id) = attribute_local(hyperlink, "id") else { + continue; + }; + let Some(relationship) = relationships.get(relation_id) else { + continue; + }; + artifact.links.push(DocumentLink { + source_block, + destination: relationship.target.clone(), + label: None, + relationship: DocumentLinkKind::Hyperlink, + locator: DocumentLocator::Slide { + slide, + shape: shape_number, + }, + external: relationship.external, + }); + } +} + +fn emit_slide_notes( + package: &Package, + slide_part: &str, + slide: u32, + slide_block: u32, + relationships: &BTreeMap, + artifact: &mut DocumentArtifact, +) -> Result<(), DocumentError> { + let notes = relationships + .values() + .find(|relationship| relationship.kind.ends_with("/notesSlide") && !relationship.external); + let Some(notes) = notes else { + return Ok(()); + }; + let document = parse_xml(¬es.target, package.required_text(¬es.target)?)?; + let text = collect_text(document.root_element()); + if !text.is_empty() { + artifact.push_block( + Some(slide_block), + DocumentBlockKind::Note, + text, + DocumentLocator::Package { + part: notes.target.clone(), + path: format!("notes-for:{slide_part}:{slide}"), + }, + )?; + } + Ok(()) +} + +fn record_ooxml_images(package: &Package, prefix: &str, artifact: &mut DocumentArtifact) { + let count = package + .parts + .keys() + .filter(|name| name.starts_with(prefix) && name.contains("/media/")) + .count(); + if count > 0 { + artifact + .metadata + .insert("embedded_image_count".to_owned(), serde_json::json!(count)); + artifact.diagnostics.push(DocumentDiagnostic { + code: "embedded_images_available_for_ocr".to_owned(), + severity: DiagnosticSeverity::Info, + locator: None, + message: format!("{count} embedded image part(s) are available for selective OCR"), + }); + } +} + +fn mark_unsupported( + artifact: &mut DocumentArtifact, + code: &str, + message: String, + locator: DocumentLocator, +) { + artifact.complete = false; + artifact.diagnostics.push(DocumentDiagnostic { + code: code.to_owned(), + severity: DiagnosticSeverity::Warning, + locator: Some(locator), + message, + }); +} + +fn parse_xml_text<'a>(name: &str, bytes: &'a [u8]) -> Result<&'a str, DocumentError> { + if bytes.iter().filter(|byte| **byte == b'<').count() > XML_MAX_EVENTS { + return Err(DocumentError::Rejected(format!( + "XML part {name} exceeds the event limit" + ))); + } + std::str::from_utf8(bytes) + .map_err(|error| DocumentError::Parse(format!("XML part {name} is not UTF-8: {error}"))) +} + +fn parse_xml<'a>(name: &str, xml: &'a str) -> Result, DocumentError> { + let document = Document::parse(xml) + .map_err(|error| DocumentError::Parse(format!("invalid XML part {name}: {error}")))?; + if document + .descendants() + .any(|node| node.ancestors().take(DOCUMENT_MAX_DEPTH + 1).count() > DOCUMENT_MAX_DEPTH) + { + return Err(DocumentError::Rejected(format!( + "XML part {name} exceeds the nesting limit" + ))); + } + Ok(document) +} + +fn normalize_part_name(name: &str) -> Result { + let replaced = name.replace('\\', "/"); + if replaced.starts_with('/') || replaced.contains('\0') { + return Err(DocumentError::Rejected( + "office part name is absolute or invalid".to_owned(), + )); + } + let mut components = Vec::new(); + for component in replaced.split('/') { + match component { + "" | "." => {} + ".." => { + if components.pop().is_none() { + return Err(DocumentError::Rejected( + "office part escapes package root".to_owned(), + )); + } + } + value => components.push(value), + } + } + if components.is_empty() { + return Err(DocumentError::Rejected( + "office part name is empty".to_owned(), + )); + } + Ok(components.join("/")) +} + +fn resolve_target(source_part: &str, target: &str) -> Result { + if target.starts_with('/') { + return normalize_part_name(target.trim_start_matches('/')); + } + let directory = source_part + .rsplit_once('/') + .map_or("", |(directory, _)| directory); + normalize_part_name(&format!("{directory}/{target}")) +} + +fn relationship_part_name(source_part: &str) -> Result { + let (directory, file) = source_part + .rsplit_once('/') + .map_or(("", source_part), |(directory, file)| (directory, file)); + normalize_part_name(&format!("{directory}/_rels/{file}.rels")) +} + +fn collect_text(node: Node<'_, '_>) -> String { + node.descendants() + .filter(|descendant| local_name(*descendant) == "t") + .filter_map(|descendant| descendant.text()) + .collect::() +} + +fn collect_docx_text(node: Node<'_, '_>) -> String { + let mut text = String::new(); + for descendant in node.descendants().filter(Node::is_element) { + match local_name(descendant) { + "t" | "delText" | "instrText" => { + if let Some(value) = descendant.text() { + text.push_str(value); + } + } + "tab" => text.push('\t'), + "br" | "cr" => text.push('\n'), + _ => {} + } + } + text +} + +fn heading_level(style: &str) -> Option { + let suffix = style.strip_prefix("heading")?.trim(); + suffix + .parse::() + .ok() + .filter(|level| (1..=6).contains(level)) +} + +fn local_name<'input>(node: Node<'_, 'input>) -> &'input str { + if node.is_element() { + node.tag_name().name() + } else { + "" + } +} + +fn attribute_local<'a>(node: Node<'a, 'a>, name: &str) -> Option<&'a str> { + node.attributes() + .find(|attribute| attribute.name() == name) + .map(|attribute| attribute.value()) +} + +fn bounded<'a>(value: &'a str, field: &str) -> Result<&'a str, DocumentError> { + if value.len() > crate::limits::DOCUMENT_MAX_FIELD_BYTES || value.contains('\0') { + return Err(DocumentError::Rejected(format!( + "{field} exceeds its bound" + ))); + } + Ok(value) +} + +fn one_based(index: usize, field: &str) -> Result { + u32::try_from(index) + .ok() + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| DocumentError::Rejected(format!("{field} index overflow"))) +} + +fn media_type(part: &str) -> Option<&'static str> { + match part.rsplit('.').next()?.to_ascii_lowercase().as_str() { + "png" => Some("image/png"), + "jpg" | "jpeg" => Some("image/jpeg"), + "tif" | "tiff" => Some("image/tiff"), + "webp" => Some("image/webp"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Write as _}; + + use zip::write::SimpleFileOptions; + + use super::*; + + #[test] + fn rejects_escaping_targets_and_invalid_sparse_coordinates() { + assert!(resolve_target("word/document.xml", "../../escape.xml").is_err()); + assert_eq!( + parse_cell_reference("XFD100000").ok(), + Some((100_000, 16_384)) + ); + assert!(parse_cell_reference("XFE1").is_err()); + assert!(parse_cell_reference("ZZZZ1").is_err()); + assert_eq!(media_type("word/media/photo.PNG"), Some("image/png")); + assert_eq!(media_type("word/media/vector.svg"), None); + } + + #[test] + fn normalizes_safe_targets() { + assert_eq!( + resolve_target("ppt/presentation.xml", "slides/slide2.xml").ok(), + Some("ppt/slides/slide2.xml".to_owned()) + ); + assert_eq!( + relationship_part_name("ppt/slides/slide1.xml").ok(), + Some("ppt/slides/_rels/slide1.xml.rels".to_owned()) + ); + } + + #[test] + fn docx_candidates_follow_only_internal_image_relationship_occurrences() + -> Result<(), Box> { + let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new())); + archive.start_file("word/document.xml", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file("word/_rels/document.xml.rels", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file("word/media/shared.png", SimpleFileOptions::default())?; + archive.write_all(b"fixture")?; + archive.start_file("word/media/unreferenced.png", SimpleFileOptions::default())?; + archive.write_all(b"must not be selected")?; + let bytes = archive.finish()?.into_inner(); + + let candidates = raster_candidates("docx", &bytes)?; + assert_eq!(candidates.len(), 2); + assert_eq!(candidates[0].bytes, candidates[1].bytes); + assert_eq!(candidates[0].part, "word/media/shared.png"); + assert_ne!(candidates[0].owner, candidates[1].owner); + assert!(matches!( + &candidates[0].owner, + DocumentLocator::Package { part, path } + if part == "word/document.xml" && path == "body/*[1]/image[1]" + )); + assert!(matches!( + &candidates[1].owner, + DocumentLocator::Package { path, .. } if path == "body/*[2]/image[1]" + )); + Ok(()) + } + + #[test] + fn pptx_uses_presentation_relationship_order_and_shape_locators() + -> Result<(), Box> { + let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new())); + archive.start_file("ppt/presentation.xml", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file( + "ppt/_rels/presentation.xml.rels", + SimpleFileOptions::default(), + )?; + archive.write_all(br#""#)?; + for (name, text) in [("slide1.xml", "First"), ("slide2.xml", "Second")] { + archive.start_file(format!("ppt/slides/{name}"), SimpleFileOptions::default())?; + let image = if name == "slide2.xml" { + r#""# + } else { + "" + }; + archive.write_all(format!(r#"{text}{image}"#).as_bytes())?; + } + archive.start_file( + "ppt/slides/_rels/slide2.xml.rels", + SimpleFileOptions::default(), + )?; + archive.write_all(br#""#)?; + archive.start_file("ppt/media/image.png", SimpleFileOptions::default())?; + archive.write_all(b"fixture")?; + let bytes = archive.finish()?.into_inner(); + let artifact = decode_pptx(&bytes)?; + let text = artifact + .blocks + .iter() + .filter(|block| !block.text.is_empty()) + .map(|block| block.text.as_str()) + .collect::>(); + assert_eq!(text, ["Second", "First"]); + assert!(matches!( + artifact + .blocks + .iter() + .find(|block| block.text == "Second") + .map(|block| &block.locator), + Some(DocumentLocator::Slide { slide: 1, shape: 2 }) + )); + let candidates = raster_candidates("pptx", &bytes)?; + assert_eq!(candidates.len(), 1); + assert!(matches!( + candidates[0].owner, + DocumentLocator::Slide { slide: 1, shape: 3 } + )); + Ok(()) + } + + #[test] + fn xlsx_image_candidates_retain_sheet_anchor_coordinates() + -> Result<(), Box> { + let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new())); + archive.start_file("xl/workbook.xml", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file("xl/_rels/workbook.xml.rels", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file("xl/worksheets/sheet1.xml", SimpleFileOptions::default())?; + archive.write_all( + br#""#, + )?; + archive.start_file( + "xl/worksheets/_rels/sheet1.xml.rels", + SimpleFileOptions::default(), + )?; + archive.write_all(br#""#)?; + archive.start_file("xl/drawings/drawing1.xml", SimpleFileOptions::default())?; + archive.write_all(br#"24"#)?; + archive.start_file( + "xl/drawings/_rels/drawing1.xml.rels", + SimpleFileOptions::default(), + )?; + archive.write_all(br#""#)?; + archive.start_file("xl/media/image.png", SimpleFileOptions::default())?; + archive.write_all(b"fixture")?; + + let candidates = raster_candidates("xlsx", &archive.finish()?.into_inner())?; + assert_eq!(candidates.len(), 1); + assert!(matches!( + &candidates[0].owner, + DocumentLocator::Spreadsheet { sheet, row: 5, column: 3 } if sheet == "Data" + )); + Ok(()) + } +} diff --git a/crates/compass-media/src/processing.rs b/crates/compass-media/src/processing.rs new file mode 100644 index 00000000..e919b1f4 --- /dev/null +++ b/crates/compass-media/src/processing.rs @@ -0,0 +1,1024 @@ +//! Selective OCR policy and provenance-preserving artifact fusion. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use compass_ocr::{ + OCR_MAX_OBSERVATIONS_PER_DOCUMENT, OCR_MAX_TEXT_CHARS_PER_DOCUMENT, OCR_SCHEMA, OcrEngine, + OcrMode, OcrObservation, OcrRequest, OcrResponse, OcrSourceKind, PreparedRaster, + normalize_language_hints, prepare_raster_cancellable, prepared_raster_digest, tile_raster, +}; +use unicode_normalization::UnicodeNormalization; + +use crate::document::{ + DiagnosticSeverity, DocumentArtifact, DocumentBlockKind, DocumentDiagnostic, DocumentError, + DocumentFormat, DocumentLocator, DocumentOrigin, VisualCoverage, +}; +use crate::limits::{OCR_MAX_AGGREGATE_PIXELS, OCR_MAX_OOXML_IMAGES, OCR_MAX_PDF_PAGES}; +use crate::{decode_document, raster_candidates, rasterize_pdf_pages_cancellable}; + +#[derive(Clone, Debug)] +pub struct DocumentProcessingOptions { + pub ocr_mode: OcrMode, + pub language_hints: Vec, + pub allow_partial: bool, +} + +impl Default for DocumentProcessingOptions { + fn default() -> Self { + Self { + ocr_mode: OcrMode::Off, + language_hints: Vec::new(), + allow_partial: false, + } + } +} + +pub fn decode_document_with_ocr( + logical_path: &Path, + bytes: &[u8], + options: &DocumentProcessingOptions, + engine: Option<&dyn OcrEngine>, +) -> Result { + decode_document_with_ocr_cancellable( + logical_path, + bytes, + options, + engine, + &AtomicBool::new(false), + ) +} + +pub fn decode_document_with_ocr_cancellable( + logical_path: &Path, + bytes: &[u8], + options: &DocumentProcessingOptions, + engine: Option<&dyn OcrEngine>, + cancellation: &AtomicBool, +) -> Result { + let started = Instant::now(); + check_cancelled(cancellation)?; + let mut artifact = decode_document(logical_path, bytes)?; + if options.ocr_mode == OcrMode::Off { + return Ok(artifact); + } + let engine = engine.ok_or_else(|| { + DocumentError::Ocr(compass_ocr::OcrError::EngineUnavailable( + "OCR was requested but no verified engine profile was loaded".to_owned(), + )) + })?; + engine.identity().validate().map_err(DocumentError::Ocr)?; + let languages = + normalize_language_hints(&options.language_hints).map_err(DocumentError::Ocr)?; + let extension = logical_path + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_ascii_lowercase) + .unwrap_or_default(); + let mut requests = Vec::new(); + let mut aggregate_pixels = 0_u64; + let mut candidate_failed = false; + match artifact.format { + DocumentFormat::Pdf => { + let selected = selected_pdf_pages(&artifact, options.ocr_mode)?; + for candidate in rasterize_pdf_pages_cancellable(bytes, &selected, cancellation)? { + check_deadline(started)?; + reserve_aggregate_pixels(&mut aggregate_pixels, &candidate.raster)?; + requests.push(( + candidate.id, + candidate.owner, + OcrSourceKind::PdfPage, + candidate.raster, + )); + } + } + DocumentFormat::Docx | DocumentFormat::Xlsx | DocumentFormat::Pptx => { + let candidates = raster_candidates(&extension, bytes)?; + if candidates.len() > OCR_MAX_OOXML_IMAGES { + return Err(DocumentError::Rejected( + "embedded image count exceeds OCR limit".to_owned(), + )); + } + for candidate in candidates { + check_cancelled(cancellation)?; + check_deadline(started)?; + match prepare_raster_cancellable(&candidate.bytes, cancellation) { + Ok(raster) => { + if options.ocr_mode == OcrMode::Auto + && (raster.width < 64 + || raster.height < 64 + || u64::from(raster.width) * u64::from(raster.height) < 4_096) + { + artifact.diagnostics.push(DocumentDiagnostic { + code: "ocr_candidate_skipped_too_small".to_owned(), + severity: DiagnosticSeverity::Info, + locator: Some(candidate.owner), + message: "Embedded image is below the automatic OCR size threshold" + .to_owned(), + }); + continue; + } + reserve_aggregate_pixels(&mut aggregate_pixels, &raster)?; + requests.push(( + candidate.id, + candidate.owner, + OcrSourceKind::EmbeddedImage, + raster, + )); + } + Err(error) if options.allow_partial => { + candidate_failed = true; + artifact.complete = false; + record_ocr_failure(&mut artifact, candidate.owner, &error); + } + Err(error) => return Err(DocumentError::Ocr(error)), + } + } + } + _ => { + return Err(DocumentError::Unsupported(format!( + "OCR for {:?}", + artifact.format + ))); + } + } + let mut aggregate_observations = 0_usize; + let mut aggregate_text = 0_usize; + let mut profile = Some(engine.identity().clone()); + let mut failed = candidate_failed; + let mut succeeded = 0_usize; + let mut reused_responses = BTreeMap::::new(); + let native_page_text = native_pdf_page_text(&artifact); + for (candidate_id, owner, source_kind, raster) in requests { + check_cancelled(cancellation)?; + check_deadline(started)?; + let digest = prepared_raster_digest(&raster); + let request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: candidate_id.clone(), + source_kind, + width: raster.width, + height: raster.height, + language_hints: languages.clone(), + image_digest: digest.clone(), + }; + let response = match reused_responses.get(&digest) { + Some(cached) => { + let mut reused = cached.clone(); + reused.request_id.clone_from(&request.request_id); + reused + } + None => match recognize_tiled(engine, &request, &raster, cancellation, started) { + Ok(response) => response, + Err(compass_ocr::OcrError::Cancelled) => { + return Err(DocumentError::Ocr(compass_ocr::OcrError::Cancelled)); + } + Err(error) if options.allow_partial => { + failed = true; + artifact.complete = false; + record_ocr_failure(&mut artifact, owner, &error); + continue; + } + Err(error) => return Err(DocumentError::Ocr(error)), + }, + }; + check_deadline(started)?; + if let Err(error) = response.validate_for(&request) { + if options.allow_partial { + failed = true; + artifact.complete = false; + record_ocr_failure(&mut artifact, owner, &error); + continue; + } + return Err(DocumentError::Ocr(error)); + } + reused_responses + .entry(digest) + .or_insert_with(|| response.clone()); + succeeded = succeeded.saturating_add(1); + if profile + .as_ref() + .is_some_and(|identity| identity != &response.profile) + { + return Err(DocumentError::InvalidArtifact( + "OCR engine profile changed within one document".to_owned(), + )); + } + profile = Some(response.profile.clone()); + aggregate_observations = aggregate_observations + .checked_add(response.observations.len()) + .ok_or_else(|| DocumentError::Rejected("OCR observation overflow".to_owned()))?; + if aggregate_observations > OCR_MAX_OBSERVATIONS_PER_DOCUMENT { + return Err(DocumentError::Rejected( + "OCR observations exceed document limit".to_owned(), + )); + } + let parent = parent_for_locator(&artifact, &owner); + let reading_order_approximate = response + .observations + .iter() + .any(|observation| observation.script.as_deref() != Some("Latn")); + for observation in response.observations { + aggregate_text = aggregate_text + .checked_add(observation.text.chars().count()) + .ok_or_else(|| DocumentError::Rejected("OCR text size overflow".to_owned()))?; + if aggregate_text > OCR_MAX_TEXT_CHARS_PER_DOCUMENT { + return Err(DocumentError::Rejected( + "OCR text exceeds document limit".to_owned(), + )); + } + let locator = DocumentLocator::Ocr { + owner: Box::new(owner.clone()), + candidate_id: candidate_id.clone(), + width: raster.width, + height: raster.height, + polygon: observation.polygon, + occurrence: observation.ordinal, + }; + let ordinal = artifact.push_block( + parent, + DocumentBlockKind::Paragraph, + observation.text, + locator.clone(), + )?; + let block = artifact.blocks.get_mut(ordinal as usize).ok_or_else(|| { + DocumentError::InvalidArtifact("new OCR block disappeared".to_owned()) + })?; + block.origin = DocumentOrigin::Ocr { + profile: response.profile.clone(), + confidence_bps: observation.confidence_bps, + }; + block.metadata.insert( + "ocr_orientation_degrees".to_owned(), + serde_json::json!(observation.orientation_degrees), + ); + if let Some(script) = observation.script { + block + .metadata + .insert("ocr_script".to_owned(), serde_json::json!(script)); + } + if observation.confidence_bps < 5_000 { + artifact.diagnostics.push(DocumentDiagnostic { + code: "ocr_low_confidence".to_owned(), + severity: DiagnosticSeverity::Warning, + locator: Some(locator), + message: format!( + "OCR confidence is {} basis points", + observation.confidence_bps + ), + }); + } + } + if reading_order_approximate { + artifact.diagnostics.push(DocumentDiagnostic { + code: "ocr_reading_order_approximate".to_owned(), + severity: DiagnosticSeverity::Info, + locator: Some(owner.clone()), + message: "OCR regions use deterministic geometric order because the engine did not provide a fully supported writing direction".to_owned(), + }); + } + if let DocumentLocator::Pdf { page, .. } = owner + && native_page_text + .get(&page) + .is_some_and(|text| !text.trim().is_empty()) + { + artifact.diagnostics.push(DocumentDiagnostic { + code: "ocr_native_text_preferred".to_owned(), + severity: DiagnosticSeverity::Info, + locator: Some(DocumentLocator::Pdf { page, item: 1 }), + message: "Native PDF text remains authoritative; OCR is separate derived evidence" + .to_owned(), + }); + } + } + artifact.ocr_profile = profile; + artifact.visual_coverage = if failed && succeeded == 0 { + VisualCoverage::Failed + } else if failed { + VisualCoverage::Partial + } else { + VisualCoverage::Complete + }; + artifact + .diagnostics + .retain(|diagnostic| diagnostic.code != "embedded_images_available_for_ocr"); + if !failed + && !artifact + .diagnostics + .iter() + .any(|diagnostic| diagnostic.severity == DiagnosticSeverity::Warning) + { + artifact.complete = true; + } + artifact.validate()?; + check_cancelled(cancellation)?; + Ok(artifact) +} + +fn check_cancelled(cancellation: &AtomicBool) -> Result<(), DocumentError> { + if cancellation.load(Ordering::Acquire) { + Err(DocumentError::Ocr(compass_ocr::OcrError::Cancelled)) + } else { + Ok(()) + } +} + +fn check_deadline(started: Instant) -> Result<(), DocumentError> { + check_ocr_deadline(started).map_err(DocumentError::Ocr) +} + +fn check_ocr_deadline(started: Instant) -> Result<(), compass_ocr::OcrError> { + if started.elapsed() > Duration::from_secs(compass_ocr::OCR_MAX_DOCUMENT_WALL_TIME_SECS) { + Err(compass_ocr::OcrError::Timeout) + } else { + Ok(()) + } +} + +fn reserve_aggregate_pixels(total: &mut u64, raster: &PreparedRaster) -> Result<(), DocumentError> { + let pixels = u64::from(raster.width) + .checked_mul(u64::from(raster.height)) + .ok_or_else(|| DocumentError::Rejected("OCR raster pixel count overflow".to_owned()))?; + *total = total + .checked_add(pixels) + .ok_or_else(|| DocumentError::Rejected("aggregate OCR pixels overflow".to_owned()))?; + if *total > OCR_MAX_AGGREGATE_PIXELS { + return Err(DocumentError::Rejected( + "aggregate OCR pixels exceed document limit".to_owned(), + )); + } + Ok(()) +} + +fn selected_pdf_pages( + artifact: &DocumentArtifact, + mode: OcrMode, +) -> Result, DocumentError> { + let mut page_text = native_pdf_page_text(artifact); + let mut pages = artifact + .blocks + .iter() + .filter_map(|block| match block.locator { + DocumentLocator::Pdf { page, .. } if matches!(block.kind, DocumentBlockKind::Page) => { + Some(page) + } + _ => None, + }) + .collect::>() + .into_iter() + .filter(|page| { + mode == OcrMode::Always + || page_text + .remove(page) + .unwrap_or_default() + .chars() + .filter(|character| !character.is_whitespace()) + .count() + < 24 + }) + .collect::>(); + pages.sort_unstable(); + if pages.len() > OCR_MAX_PDF_PAGES { + return Err(DocumentError::Rejected( + "eligible PDF pages exceed OCR limit".to_owned(), + )); + } + Ok(pages) +} + +fn native_pdf_page_text(artifact: &DocumentArtifact) -> BTreeMap { + let mut text = BTreeMap::::new(); + for block in &artifact.blocks { + if !matches!(block.origin, DocumentOrigin::Native) { + continue; + } + let page = match block.locator { + DocumentLocator::Pdf { page, .. } => Some(page), + _ => block + .parent + .and_then(|parent| artifact.blocks.get(parent as usize)) + .and_then(|parent| match parent.locator { + DocumentLocator::Pdf { page, .. } => Some(page), + _ => None, + }), + }; + if let Some(page) = page { + text.entry(page).or_default().push_str(&block.text); + } + } + text +} + +fn parent_for_locator(artifact: &DocumentArtifact, owner: &DocumentLocator) -> Option { + artifact + .blocks + .iter() + .find(|block| &block.locator == owner) + .map(|block| block.ordinal) +} + +fn bounded_message(message: &str) -> String { + message.chars().take(1_024).collect() +} + +fn recognize_tiled( + engine: &dyn OcrEngine, + request: &OcrRequest, + raster: &PreparedRaster, + cancellation: &AtomicBool, + started: Instant, +) -> Result { + let tiles = tile_raster(raster)?; + let mut observations = Vec::new(); + for tile in tiles { + check_ocr_deadline(started)?; + check_cancelled(cancellation).map_err(|error| match error { + DocumentError::Ocr(error) => error, + other => compass_ocr::OcrError::Inference(other.to_string()), + })?; + let tile_digest = prepared_raster_digest(&tile.raster); + let tile_request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: format!("tile-{}-{}", &tile_digest[..16], tile.ordinal), + source_kind: request.source_kind, + width: tile.raster.width, + height: tile.raster.height, + language_hints: request.language_hints.clone(), + image_digest: tile_digest, + }; + let response = engine.recognize_cancellable(&tile_request, &tile.raster, cancellation)?; + check_ocr_deadline(started)?; + response.validate_for(&tile_request)?; + if response.profile != *engine.identity() { + return Err(compass_ocr::OcrError::InvalidOutput( + "OCR engine response profile does not match its loaded identity".to_owned(), + )); + } + for mut observation in response.observations { + for point in &mut observation.polygon { + point.x = point.x.checked_add(tile.x).ok_or_else(|| { + compass_ocr::OcrError::InvalidOutput("tile x overflow".to_owned()) + })?; + point.y = point.y.checked_add(tile.y).ok_or_else(|| { + compass_ocr::OcrError::InvalidOutput("tile y overflow".to_owned()) + })?; + if point.x >= raster.width || point.y >= raster.height { + return Err(compass_ocr::OcrError::InvalidOutput( + "mapped tile geometry lies outside the source raster".to_owned(), + )); + } + } + observations.push(observation); + } + } + observations.sort_by(|left, right| { + observation_geometry_key(left) + .cmp(&observation_geometry_key(right)) + .then_with(|| left.text.cmp(&right.text)) + .then_with(|| right.confidence_bps.cmp(&left.confidence_bps)) + }); + let mut deduplicated = Vec::::new(); + for observation in observations { + let normalized = comparison_text(&observation.text); + let duplicate = deduplicated.iter().position(|existing| { + comparison_text(&existing.text) == normalized + && polygon_box_iou(existing, &observation) >= 0.5 + }); + if let Some(index) = duplicate { + if observation.confidence_bps > deduplicated[index].confidence_bps { + deduplicated[index] = observation; + } + } else { + deduplicated.push(observation); + } + } + if deduplicated.len() > compass_ocr::OCR_MAX_OBSERVATIONS_PER_RASTER { + return Err(compass_ocr::OcrError::InvalidOutput( + "merged OCR observation count exceeds raster limit".to_owned(), + )); + } + deduplicated.sort_by(|left, right| { + observation_geometry_key(left) + .cmp(&observation_geometry_key(right)) + .then_with(|| left.text.cmp(&right.text)) + }); + for (ordinal, observation) in deduplicated.iter_mut().enumerate() { + observation.ordinal = u32::try_from(ordinal).map_err(|_| { + compass_ocr::OcrError::InvalidOutput("merged OCR ordinal overflow".to_owned()) + })?; + } + let response = OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: engine.identity().clone(), + observations: deduplicated, + }; + response.validate_for(request)?; + Ok(response) +} + +fn observation_geometry_key(observation: &OcrObservation) -> (u32, u32, u32, u32) { + let min_x = observation + .polygon + .iter() + .map(|point| point.x) + .min() + .unwrap_or(0); + let min_y = observation + .polygon + .iter() + .map(|point| point.y) + .min() + .unwrap_or(0); + let max_x = observation + .polygon + .iter() + .map(|point| point.x) + .max() + .unwrap_or(0); + let max_y = observation + .polygon + .iter() + .map(|point| point.y) + .max() + .unwrap_or(0); + (min_y, min_x, max_y, max_x) +} + +fn comparison_text(text: &str) -> String { + let normalized = text.nfkc().collect::(); + normalized + .split_whitespace() + .flat_map(|word| { + word.chars() + .flat_map(char::to_lowercase) + .chain(std::iter::once(' ')) + }) + .collect::() + .trim_end() + .to_owned() +} + +fn polygon_box_iou(left: &OcrObservation, right: &OcrObservation) -> f64 { + let (left_y1, left_x1, left_y2, left_x2) = observation_geometry_key(left); + let (right_y1, right_x1, right_y2, right_x2) = observation_geometry_key(right); + let overlap_width = left_x2.min(right_x2).saturating_sub(left_x1.max(right_x1)); + let overlap_height = left_y2.min(right_y2).saturating_sub(left_y1.max(right_y1)); + let intersection = u64::from(overlap_width) * u64::from(overlap_height); + let left_area = + u64::from(left_x2.saturating_sub(left_x1)) * u64::from(left_y2.saturating_sub(left_y1)); + let right_area = + u64::from(right_x2.saturating_sub(right_x1)) * u64::from(right_y2.saturating_sub(right_y1)); + let union = left_area + .saturating_add(right_area) + .saturating_sub(intersection); + if union == 0 { + 0.0 + } else { + intersection as f64 / union as f64 + } +} + +fn record_ocr_failure( + artifact: &mut DocumentArtifact, + locator: DocumentLocator, + error: &compass_ocr::OcrError, +) { + let code = match error { + compass_ocr::OcrError::Timeout => "ocr_engine_timeout", + compass_ocr::OcrError::InvalidOutput(_) | compass_ocr::OcrError::InvalidRequest(_) => { + "ocr_engine_output_rejected" + } + compass_ocr::OcrError::Cancelled => "ocr_engine_cancelled", + compass_ocr::OcrError::EngineUnavailable(_) + | compass_ocr::OcrError::ModelUnavailable(_) + | compass_ocr::OcrError::ModelVerification(_) + | compass_ocr::OcrError::Inference(_) + | compass_ocr::OcrError::Io { .. } => "ocr_engine_unavailable", + }; + artifact.diagnostics.push(DocumentDiagnostic { + code: code.to_owned(), + severity: DiagnosticSeverity::Warning, + locator: Some(locator.clone()), + message: bounded_message(&error.to_string()), + }); + artifact.diagnostics.push(DocumentDiagnostic { + code: "ocr_partial_visual_coverage".to_owned(), + severity: DiagnosticSeverity::Warning, + locator: Some(locator), + message: "One selected OCR raster could not be processed".to_owned(), + }); +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Write as _}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use compass_ocr::{OcrObservation, OcrPoint, OcrProfileIdentity, OcrResponse}; + use image::{DynamicImage, ImageFormat, RgbImage}; + use zip::write::SimpleFileOptions; + + use super::*; + + struct FixtureEngine { + calls: AtomicUsize, + fail: bool, + invalid: bool, + } + + struct OverlapEngine; + + impl OcrEngine for FixtureEngine { + fn identity(&self) -> &OcrProfileIdentity { + static PROFILE: std::sync::OnceLock = std::sync::OnceLock::new(); + PROFILE.get_or_init(fixture_profile) + } + + fn recognize( + &self, + request: &OcrRequest, + _raster: &compass_ocr::PreparedRaster, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.fail { + return Err(compass_ocr::OcrError::Inference( + "fixture failure".to_owned(), + )); + } + Ok(OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: if self.invalid { + "wrong-request".to_owned() + } else { + request.request_id.clone() + }, + profile: fixture_profile(), + observations: vec![OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 80, y: 1 }, + OcrPoint { x: 80, y: 30 }, + OcrPoint { x: 1, y: 30 }, + ], + text: "OCR sentinel".to_owned(), + confidence_bps: 9_500, + script: Some("Latn".to_owned()), + orientation_degrees: 0, + }], + }) + } + } + + impl OcrEngine for OverlapEngine { + fn identity(&self) -> &OcrProfileIdentity { + static PROFILE: std::sync::OnceLock = std::sync::OnceLock::new(); + PROFILE.get_or_init(fixture_profile) + } + + fn recognize( + &self, + request: &OcrRequest, + _raster: &PreparedRaster, + ) -> Result { + let second = request.request_id.ends_with("-1"); + let (left, right, confidence) = if second { + (0, 127, 9_500) + } else { + (1_920, 2_047, 8_000) + }; + Ok(OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: self.identity().clone(), + observations: vec![OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: left, y: 1 }, + OcrPoint { x: right, y: 1 }, + OcrPoint { x: right, y: 30 }, + OcrPoint { x: left, y: 30 }, + ], + text: "overlap sentinel".to_owned(), + confidence_bps: confidence, + script: Some("Latn".to_owned()), + orientation_degrees: 0, + }], + }) + } + } + + fn fixture_profile() -> OcrProfileIdentity { + OcrProfileIdentity { + engine: "fixture".to_owned(), + engine_version: "1".to_owned(), + profile: "fixture".to_owned(), + model_digests: BTreeMap::from([("model".to_owned(), "a".repeat(64))]), + languages: vec!["en".to_owned()], + preprocessing_version: compass_ocr::OCR_PREPROCESSING_VERSION, + } + } + + fn docx_with_image() -> Result, Box> { + let mut png = Cursor::new(Vec::new()); + DynamicImage::ImageRgb8(RgbImage::from_pixel(100, 100, image::Rgb([255, 255, 255]))) + .write_to(&mut png, ImageFormat::Png)?; + let mut bytes = Cursor::new(Vec::new()); + { + let mut archive = zip::ZipWriter::new(&mut bytes); + archive.start_file("word/document.xml", SimpleFileOptions::default())?; + archive.write_all(br#"Native sentinel"#)?; + archive.start_file("word/_rels/document.xml.rels", SimpleFileOptions::default())?; + archive.write_all(br#""#)?; + archive.start_file("word/media/image1.png", SimpleFileOptions::default())?; + archive.write_all(png.get_ref())?; + archive.finish()?; + } + Ok(bytes.into_inner()) + } + + #[test] + fn office_ocr_preserves_native_text_and_exact_visual_provenance() + -> Result<(), Box> { + let bytes = docx_with_image()?; + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: false, + invalid: false, + }; + let artifact = decode_document_with_ocr( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions { + ocr_mode: OcrMode::Auto, + language_hints: vec!["en".to_owned()], + allow_partial: false, + }, + Some(&engine), + )?; + assert_eq!(engine.calls.load(Ordering::Relaxed), 1); + assert!(artifact.blocks.iter().any(|block| { + block.text == "Native sentinel" && matches!(block.origin, DocumentOrigin::Native) + })); + let ocr = artifact + .blocks + .iter() + .find(|block| block.text == "OCR sentinel") + .ok_or("OCR block missing")?; + assert!(matches!( + ocr.origin, + DocumentOrigin::Ocr { + confidence_bps: 9_500, + .. + } + )); + assert!(matches!( + ocr.locator, + DocumentLocator::Ocr { occurrence: 0, .. } + )); + let repeated = artifact + .blocks + .iter() + .filter(|block| block.text == "OCR sentinel") + .collect::>(); + assert_eq!(repeated.len(), 2); + assert_ne!(repeated[0].locator, repeated[1].locator); + assert_eq!(artifact.visual_coverage, VisualCoverage::Complete); + assert!(artifact.complete); + Ok(()) + } + + #[test] + fn partial_ocr_is_never_reported_complete() -> Result<(), Box> { + let bytes = docx_with_image()?; + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: true, + invalid: false, + }; + let artifact = decode_document_with_ocr( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions { + ocr_mode: OcrMode::Always, + language_hints: Vec::new(), + allow_partial: true, + }, + Some(&engine), + )?; + assert_eq!(artifact.visual_coverage, VisualCoverage::Failed); + assert!(!artifact.complete); + assert!(artifact.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "ocr_partial_visual_coverage" + && diagnostic.severity == DiagnosticSeverity::Warning + })); + Ok(()) + } + + #[test] + fn allowed_invalid_engine_output_is_rejected_and_reported_as_failed_coverage() + -> Result<(), Box> { + let bytes = docx_with_image()?; + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: false, + invalid: true, + }; + let artifact = decode_document_with_ocr( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions { + ocr_mode: OcrMode::Always, + language_hints: Vec::new(), + allow_partial: true, + }, + Some(&engine), + )?; + assert_eq!(artifact.visual_coverage, VisualCoverage::Failed); + assert!(!artifact.complete); + assert!( + artifact + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "ocr_engine_output_rejected") + ); + assert!( + artifact + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "ocr_partial_visual_coverage") + ); + Ok(()) + } + + #[test] + fn native_off_is_identical_and_invalid_engine_output_is_rejected() + -> Result<(), Box> { + let bytes = docx_with_image()?; + let native = decode_document(Path::new("report.docx"), &bytes)?; + let off = decode_document_with_ocr( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions::default(), + None, + )?; + assert_eq!(native, off); + assert!(off.complete); + + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: false, + invalid: true, + }; + let result = decode_document_with_ocr( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions { + ocr_mode: OcrMode::Always, + language_hints: vec!["en".to_owned()], + allow_partial: false, + }, + Some(&engine), + ); + assert!(matches!(result, Err(DocumentError::Ocr(_)))); + Ok(()) + } + + #[test] + fn automatic_pdf_selection_skips_pages_with_sufficient_native_text() + -> Result<(), Box> { + let mut artifact = DocumentArtifact::new(DocumentFormat::Pdf); + let first = artifact.push_block( + None, + DocumentBlockKind::Page, + String::new(), + DocumentLocator::Pdf { page: 1, item: 1 }, + )?; + artifact.push_block( + Some(first), + DocumentBlockKind::Paragraph, + "This born-digital page contains enough native text.".to_owned(), + DocumentLocator::Pdf { page: 1, item: 2 }, + )?; + artifact.push_block( + None, + DocumentBlockKind::Page, + String::new(), + DocumentLocator::Pdf { page: 2, item: 1 }, + )?; + assert_eq!(selected_pdf_pages(&artifact, OcrMode::Auto)?, vec![2]); + assert_eq!(selected_pdf_pages(&artifact, OcrMode::Always)?, vec![1, 2]); + Ok(()) + } + + #[test] + fn always_pdf_selection_rejects_one_page_over_the_hard_cap() + -> Result<(), Box> { + let mut artifact = DocumentArtifact::new(DocumentFormat::Pdf); + for page in 1..=u32::try_from(OCR_MAX_PDF_PAGES + 1)? { + artifact.push_block( + None, + DocumentBlockKind::Page, + String::new(), + DocumentLocator::Pdf { page, item: 1 }, + )?; + } + assert!(selected_pdf_pages(&artifact, OcrMode::Always).is_err()); + Ok(()) + } + + #[test] + fn document_ocr_honors_pre_cancel_without_decoding_or_inference() + -> Result<(), Box> { + let bytes = docx_with_image()?; + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: false, + invalid: false, + }; + let cancellation = AtomicBool::new(true); + let result = decode_document_with_ocr_cancellable( + Path::new("report.docx"), + &bytes, + &DocumentProcessingOptions { + ocr_mode: OcrMode::Always, + language_hints: Vec::new(), + allow_partial: false, + }, + Some(&engine), + &cancellation, + ); + assert!(matches!( + result, + Err(DocumentError::Ocr(compass_ocr::OcrError::Cancelled)) + )); + assert_eq!(engine.calls.load(Ordering::Relaxed), 0); + Ok(()) + } + + #[test] + fn tiled_regions_map_to_source_geometry_and_overlap_is_deduplicated() + -> Result<(), Box> { + let raster = PreparedRaster { + image: RgbImage::new(3_000, 100), + width: 3_000, + height: 100, + }; + let request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: "source-raster".to_owned(), + source_kind: OcrSourceKind::EmbeddedImage, + width: raster.width, + height: raster.height, + language_hints: Vec::new(), + image_digest: prepared_raster_digest(&raster), + }; + let response = recognize_tiled( + &OverlapEngine, + &request, + &raster, + &AtomicBool::new(false), + Instant::now(), + )?; + assert_eq!(response.observations.len(), 1); + assert_eq!(response.observations[0].confidence_bps, 9_500); + assert_eq!(response.observations[0].polygon[0].x, 1_920); + Ok(()) + } + + #[test] + fn tiled_recognition_checks_the_document_deadline_before_inference() + -> Result<(), Box> { + let raster = PreparedRaster { + image: RgbImage::new(100, 100), + width: 100, + height: 100, + }; + let request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: "expired-raster".to_owned(), + source_kind: OcrSourceKind::EmbeddedImage, + width: raster.width, + height: raster.height, + language_hints: Vec::new(), + image_digest: prepared_raster_digest(&raster), + }; + let engine = FixtureEngine { + calls: AtomicUsize::new(0), + fail: false, + invalid: false, + }; + let started = Instant::now() + .checked_sub(Duration::from_secs( + compass_ocr::OCR_MAX_DOCUMENT_WALL_TIME_SECS + 1, + )) + .ok_or("could not construct expired deadline")?; + let result = recognize_tiled(&engine, &request, &raster, &AtomicBool::new(false), started); + assert!(matches!(result, Err(compass_ocr::OcrError::Timeout))); + assert_eq!(engine.calls.load(Ordering::Relaxed), 0); + Ok(()) + } +} diff --git a/crates/compass-media/src/raster.rs b/crates/compass-media/src/raster.rs new file mode 100644 index 00000000..45152b3d --- /dev/null +++ b/crates/compass-media/src/raster.rs @@ -0,0 +1,239 @@ +//! Pure-Rust, bounded PDF page rasterization for OCR. + +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicBool, Ordering}; + +use compass_ocr::{OCR_MAX_RASTER_LONG_EDGE, OCR_MAX_RASTER_PIXELS, PreparedRaster}; +use hayro::hayro_interpret::InterpreterSettings; +use hayro::hayro_syntax::Pdf; +use hayro::vello_cpu::color::palette::css::WHITE; +use hayro::{RenderCache, RenderSettings, render}; +use image::RgbImage; + +use crate::document::{DocumentError, DocumentLocator}; +use crate::limits::{OCR_MAX_AGGREGATE_PIXELS, OCR_MAX_PDF_PAGES}; + +const PDF_OCR_DPI: f64 = 300.0; +const PDF_POINTS_PER_INCH: f64 = 72.0; +pub const PDF_RASTERIZER_IDENTITY: &str = "hayro/0.7.1@300dpi"; + +#[derive(Clone, Debug)] +pub struct PdfRasterCandidate { + pub id: String, + pub owner: DocumentLocator, + pub page: u32, + pub raster: PreparedRaster, +} + +pub fn rasterize_pdf_pages( + bytes: &[u8], + selected_pages: &[u32], +) -> Result, DocumentError> { + rasterize_pdf_pages_cancellable(bytes, selected_pages, &AtomicBool::new(false)) +} + +pub fn rasterize_pdf_pages_cancellable( + bytes: &[u8], + selected_pages: &[u32], + cancellation: &AtomicBool, +) -> Result, DocumentError> { + check_cancelled(cancellation)?; + if selected_pages.len() > OCR_MAX_PDF_PAGES { + return Err(DocumentError::Rejected( + "selected PDF page count exceeds OCR limit".to_owned(), + )); + } + let unique = selected_pages.iter().copied().collect::>(); + if unique.len() != selected_pages.len() || unique.contains(&0) { + return Err(DocumentError::Rejected( + "selected PDF pages must be unique and one-based".to_owned(), + )); + } + let owned = bytes.to_vec(); + let pages = selected_pages.to_vec(); + std::panic::catch_unwind(|| rasterize_owned(owned, &pages, cancellation)) + .map_err(|_| DocumentError::Parse("PDF rasterizer panicked".to_owned()))? +} + +fn rasterize_owned( + bytes: Vec, + selected_pages: &[u32], + cancellation: &AtomicBool, +) -> Result, DocumentError> { + let pdf = Pdf::new(bytes).map_err(|error| { + DocumentError::Parse(format!("PDF renderer rejected the document: {error:?}")) + })?; + let cache = RenderCache::new(); + let interpreter = InterpreterSettings::default(); + let mut output = Vec::with_capacity(selected_pages.len()); + let mut aggregate_pixels = 0_u64; + for page_number in selected_pages { + check_cancelled(cancellation)?; + let index = usize::try_from(page_number.saturating_sub(1)) + .map_err(|_| DocumentError::Rejected("PDF page index overflow".to_owned()))?; + let page = pdf.pages().get(index).ok_or_else(|| { + DocumentError::Parse(format!("PDF page {page_number} does not exist")) + })?; + let (point_width, point_height) = page.render_dimensions(); + let (width, height, scale) = bounded_render_dimensions(point_width, point_height)?; + aggregate_pixels = aggregate_pixels + .checked_add(u64::from(width) * u64::from(height)) + .ok_or_else(|| DocumentError::Rejected("PDF raster pixel total overflow".to_owned()))?; + if aggregate_pixels > OCR_MAX_AGGREGATE_PIXELS { + return Err(DocumentError::Rejected( + "PDF raster pixels exceed the document OCR limit".to_owned(), + )); + } + let settings = RenderSettings { + x_scale: scale, + y_scale: scale, + width: Some(width), + height: Some(height), + bg_color: WHITE, + }; + let pixmap = render(page, &cache, &interpreter, &settings); + let raw = pixmap.data_as_u8_slice(); + let expected = usize::from(width) + .checked_mul(usize::from(height)) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| DocumentError::Rejected("PDF raster size overflow".to_owned()))?; + if raw.len() != expected { + return Err(DocumentError::Parse( + "PDF renderer returned an invalid raster length".to_owned(), + )); + } + let mut rgb = Vec::with_capacity(expected / 4 * 3); + for pixel in raw.chunks_exact(4) { + rgb.extend_from_slice(&pixel[..3]); + } + let image = RgbImage::from_raw(u32::from(width), u32::from(height), rgb) + .ok_or_else(|| DocumentError::Parse("could not construct PDF raster".to_owned()))?; + output.push(PdfRasterCandidate { + id: format!("pdf-page-{page_number}"), + owner: DocumentLocator::Pdf { + page: *page_number, + item: 1, + }, + page: *page_number, + raster: PreparedRaster { + image, + width: u32::from(width), + height: u32::from(height), + }, + }); + } + check_cancelled(cancellation)?; + Ok(output) +} + +fn check_cancelled(cancellation: &AtomicBool) -> Result<(), DocumentError> { + if cancellation.load(Ordering::Acquire) { + Err(DocumentError::Ocr(compass_ocr::OcrError::Cancelled)) + } else { + Ok(()) + } +} + +fn bounded_render_dimensions( + point_width: f32, + point_height: f32, +) -> Result<(u16, u16, f32), DocumentError> { + if !point_width.is_finite() + || !point_height.is_finite() + || point_width <= 0.0 + || point_height <= 0.0 + { + return Err(DocumentError::Rejected( + "PDF page has invalid dimensions".to_owned(), + )); + } + let target_scale = PDF_OCR_DPI / PDF_POINTS_PER_INCH; + let target_width = f64::from(point_width) * target_scale; + let target_height = f64::from(point_height) * target_scale; + let edge_scale = f64::from(OCR_MAX_RASTER_LONG_EDGE) / target_width.max(target_height); + let pixel_scale = (OCR_MAX_RASTER_PIXELS as f64 / (target_width * target_height)).sqrt(); + let reduction = edge_scale.min(pixel_scale).min(1.0); + let width = (target_width * reduction).floor().max(1.0); + let height = (target_height * reduction).floor().max(1.0); + if width > f64::from(u16::MAX) || height > f64::from(u16::MAX) { + return Err(DocumentError::Rejected( + "PDF page raster dimensions exceed renderer limits".to_owned(), + )); + } + let width = width as u16; + let height = height as u16; + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| DocumentError::Rejected("PDF raster pixel overflow".to_owned()))?; + if pixels > OCR_MAX_RASTER_PIXELS { + return Err(DocumentError::Rejected( + "PDF raster exceeds OCR pixel limit".to_owned(), + )); + } + let scale = (f64::from(width) / f64::from(point_width)) as f32; + Ok((width, height, scale)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn one_page_pdf() -> Vec { + let objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>".as_slice(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".as_slice(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 72 72] /Resources <<>> /Contents 4 0 R >>".as_slice(), + b"<< /Length 27 >>\nstream\n0 0 0 rg\n12 12 48 48 re f\n\nendstream".as_slice(), + ]; + let mut pdf = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n", index + 1).as_bytes()); + pdf.extend_from_slice(object); + pdf.extend_from_slice(b"\nendobj\n"); + } + let xref = pdf.len(); + pdf.extend_from_slice(b"xref\n0 5\n0000000000 65535 f \n"); + for offset in offsets { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!("trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(), + ); + pdf + } + + #[test] + fn page_dimensions_are_reduced_deterministically() { + let (width, height, scale) = bounded_render_dimensions(612.0, 792.0).unwrap_or((0, 0, 0.0)); + assert_eq!((width, height), (2550, 3300)); + assert!(scale > 4.16 && scale < 4.17); + + let (width, height, _) = + bounded_render_dimensions(20_000.0, 20_000.0).unwrap_or((0, 0, 0.0)); + assert!(u64::from(width) * u64::from(height) <= OCR_MAX_RASTER_PIXELS); + assert!(u32::from(width) <= OCR_MAX_RASTER_LONG_EDGE); + } + + #[test] + fn renders_pdf_pixels_in_process_and_rejects_bad_pages() + -> Result<(), Box> { + let rendered = rasterize_pdf_pages(&one_page_pdf(), &[1])?; + assert_eq!(rendered.len(), 1); + assert_eq!( + (rendered[0].raster.width, rendered[0].raster.height), + (300, 300) + ); + assert!( + rendered[0] + .raster + .image + .pixels() + .any(|pixel| pixel.0 != [255, 255, 255]) + ); + assert!(rasterize_pdf_pages(&one_page_pdf(), &[2]).is_err()); + assert!(rasterize_pdf_pages(b"not a PDF", &[1]).is_err()); + Ok(()) + } +} diff --git a/crates/compass-ocr/Cargo.toml b/crates/compass-ocr/Cargo.toml new file mode 100644 index 00000000..74ff84d7 --- /dev/null +++ b/crates/compass-ocr/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "compass-ocr" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +description.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +image.workspace = true +oar-ocr.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tempfile.workspace = true +thiserror.workspace = true +ureq.workspace = true + +[lints] +workspace = true diff --git a/crates/compass-ocr/src/engine.rs b/crates/compass-ocr/src/engine.rs new file mode 100644 index 00000000..8504e878 --- /dev/null +++ b/crates/compass-ocr/src/engine.rs @@ -0,0 +1,444 @@ +//! Managed OAR-OCR inference and deterministic raster normalization. + +use std::io::Cursor; +use std::sync::atomic::{AtomicBool, Ordering}; + +use image::{DynamicImage, ImageDecoder, ImageReader, RgbImage, imageops::FilterType}; +use oar_ocr::core::config::OrtSessionConfig; +use oar_ocr::oarocr::OAROCRBuilder; +use sha2::{Digest, Sha256}; + +use crate::models::{ModelProfile, verify_profile}; +use crate::{ + OCR_ENGINE_MAX_SIDE, OCR_ENGINE_THREADS, OCR_MAX_OBSERVATIONS_PER_RASTER, + OCR_MAX_RASTER_LONG_EDGE, OCR_MAX_RASTER_PIXELS, OCR_SCHEMA, OCR_TILE_OVERLAP, OcrError, + OcrObservation, OcrPoint, OcrRequest, OcrResponse, +}; + +#[derive(Clone, Debug)] +pub struct PreparedRaster { + pub image: RgbImage, + pub width: u32, + pub height: u32, +} + +#[derive(Clone, Debug)] +pub struct PreparedRasterTile { + pub ordinal: u32, + pub x: u32, + pub y: u32, + pub raster: PreparedRaster, +} + +pub trait OcrEngine { + fn identity(&self) -> &crate::OcrProfileIdentity; + + fn recognize( + &self, + request: &OcrRequest, + raster: &PreparedRaster, + ) -> Result; + + fn recognize_cancellable( + &self, + request: &OcrRequest, + raster: &PreparedRaster, + cancellation: &AtomicBool, + ) -> Result { + check_cancelled(cancellation)?; + let response = self.recognize(request, raster)?; + check_cancelled(cancellation)?; + Ok(response) + } +} + +pub struct ManagedOarEngine { + runtime: oar_ocr::oarocr::OAROCR, + profile: crate::OcrProfileIdentity, +} + +impl ManagedOarEngine { + pub fn load(profile: ModelProfile) -> Result { + let files = verify_profile(profile)?; + let session = OrtSessionConfig::default() + .with_intra_threads(OCR_ENGINE_THREADS) + .with_inter_threads(OCR_ENGINE_THREADS); + let runtime = OAROCRBuilder::new(&files.detector, &files.recognizer, &files.dictionary) + .ort_session(session) + .image_batch_size(1) + .region_batch_size(4) + .build() + .map_err(|error| OcrError::EngineUnavailable(error.to_string()))?; + Ok(Self { + runtime, + profile: files.identity, + }) + } +} + +impl OcrEngine for ManagedOarEngine { + fn identity(&self) -> &crate::OcrProfileIdentity { + &self.profile + } + + fn recognize( + &self, + request: &OcrRequest, + raster: &PreparedRaster, + ) -> Result { + request.validate()?; + if raster.width != request.width || raster.height != request.height { + return Err(OcrError::InvalidRequest( + "request dimensions do not match prepared raster".to_owned(), + )); + } + if prepared_raster_digest(raster) != request.image_digest { + return Err(OcrError::InvalidRequest( + "request digest does not match the prepared raster".to_owned(), + )); + } + let mut results = self + .runtime + .predict(vec![raster.image.clone()]) + .map_err(|error| OcrError::Inference(error.to_string()))?; + if results.len() != 1 { + return Err(OcrError::InvalidOutput( + "OCR engine returned an unexpected result count".to_owned(), + )); + } + let result = results + .pop() + .ok_or_else(|| OcrError::InvalidOutput("OCR result disappeared".to_owned()))?; + if result.text_regions.len() > OCR_MAX_OBSERVATIONS_PER_RASTER { + return Err(OcrError::InvalidOutput( + "OCR observation count exceeds limit".to_owned(), + )); + } + let mut regions = result + .text_regions + .into_iter() + .filter_map(|region| { + let text = region.text?.to_string(); + if text.trim().is_empty() { + return None; + } + let confidence = region.confidence?; + Some((region.bounding_box.points, text, confidence)) + }) + .collect::>(); + regions.sort_by(|left, right| { + geometry_key(&left.0) + .cmp(&geometry_key(&right.0)) + .then_with(|| left.1.cmp(&right.1)) + }); + let mut observations = Vec::with_capacity(regions.len()); + for (index, (points, text, confidence)) in regions.into_iter().enumerate() { + if !confidence.is_finite() || !(0.0..=1.0).contains(&confidence) { + return Err(OcrError::InvalidOutput( + "OCR engine returned invalid confidence".to_owned(), + )); + } + let polygon = points + .into_iter() + .map(|point| { + Ok(OcrPoint { + x: quantize_coordinate(point.x, raster.width)?, + y: quantize_coordinate(point.y, raster.height)?, + }) + }) + .collect::, OcrError>>()?; + observations.push(OcrObservation { + ordinal: u32::try_from(index) + .map_err(|_| OcrError::InvalidOutput("OCR ordinal overflow".to_owned()))?, + polygon, + text, + confidence_bps: (confidence * 10_000.0).round() as u16, + script: None, + orientation_degrees: 0, + }); + } + let response = OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: self.profile.clone(), + observations, + }; + response.validate_for(request)?; + Ok(response) + } + + fn recognize_cancellable( + &self, + request: &OcrRequest, + raster: &PreparedRaster, + cancellation: &AtomicBool, + ) -> Result { + check_cancelled(cancellation)?; + let response = self.recognize(request, raster)?; + check_cancelled(cancellation)?; + Ok(response) + } +} + +fn check_cancelled(cancellation: &AtomicBool) -> Result<(), OcrError> { + if cancellation.load(Ordering::Acquire) { + Err(OcrError::Cancelled) + } else { + Ok(()) + } +} + +#[must_use] +pub fn prepared_raster_digest(raster: &PreparedRaster) -> String { + let mut hasher = Sha256::new(); + hasher.update(raster.width.to_be_bytes()); + hasher.update(raster.height.to_be_bytes()); + hasher.update(raster.image.as_raw()); + format!("{:x}", hasher.finalize()) +} + +pub fn tile_raster(raster: &PreparedRaster) -> Result, OcrError> { + crate::validate_dimensions(raster.width, raster.height)?; + if raster.image.width() != raster.width || raster.image.height() != raster.height { + return Err(OcrError::InvalidRequest( + "prepared raster dimensions do not match its pixels".to_owned(), + )); + } + let x_offsets = tile_offsets(raster.width, OCR_ENGINE_MAX_SIDE, OCR_TILE_OVERLAP)?; + let y_offsets = tile_offsets(raster.height, OCR_ENGINE_MAX_SIDE, OCR_TILE_OVERLAP)?; + let capacity = x_offsets + .len() + .checked_mul(y_offsets.len()) + .ok_or_else(|| OcrError::InvalidRequest("tile count overflow".to_owned()))?; + let mut tiles = Vec::with_capacity(capacity); + for y in y_offsets { + for &x in &x_offsets { + let width = OCR_ENGINE_MAX_SIDE.min(raster.width.saturating_sub(x)); + let height = OCR_ENGINE_MAX_SIDE.min(raster.height.saturating_sub(y)); + let image = image::imageops::crop_imm(&raster.image, x, y, width, height).to_image(); + let ordinal = u32::try_from(tiles.len()) + .map_err(|_| OcrError::InvalidRequest("tile ordinal overflow".to_owned()))?; + tiles.push(PreparedRasterTile { + ordinal, + x, + y, + raster: PreparedRaster { + image, + width, + height, + }, + }); + } + } + Ok(tiles) +} + +fn tile_offsets(length: u32, side: u32, overlap: u32) -> Result, OcrError> { + if side == 0 || overlap >= side { + return Err(OcrError::InvalidRequest( + "invalid OCR tiling policy".to_owned(), + )); + } + if length <= side { + return Ok(vec![0]); + } + let stride = side - overlap; + let mut offsets = vec![0_u32]; + while offsets + .last() + .is_some_and(|offset| (*offset).saturating_add(side) < length) + { + let current = *offsets + .last() + .ok_or_else(|| OcrError::InvalidRequest("tile offsets disappeared".to_owned()))?; + offsets.push(current.saturating_add(stride)); + } + Ok(offsets) +} + +pub fn prepare_raster(bytes: &[u8]) -> Result { + prepare_raster_cancellable(bytes, &AtomicBool::new(false)) +} + +pub fn prepare_raster_cancellable( + bytes: &[u8], + cancellation: &AtomicBool, +) -> Result { + check_cancelled(cancellation)?; + let mut reader = ImageReader::new(Cursor::new(bytes)) + .with_guessed_format() + .map_err(|error| OcrError::InvalidRequest(format!("unknown image format: {error}")))?; + let mut limits = image::Limits::default(); + limits.max_image_width = Some(65_535); + limits.max_image_height = Some(65_535); + limits.max_alloc = Some(OCR_MAX_RASTER_PIXELS.saturating_mul(8)); + reader.limits(limits); + let mut decoder = reader + .into_decoder() + .map_err(|error| OcrError::InvalidRequest(format!("image decode failed: {error}")))?; + let orientation = decoder + .orientation() + .map_err(|error| OcrError::InvalidRequest(format!("image metadata failed: {error}")))?; + let (width, height) = decoder.dimensions(); + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| OcrError::InvalidRequest("image pixel count overflow".to_owned()))?; + if width == 0 || height == 0 || pixels > OCR_MAX_RASTER_PIXELS { + return Err(OcrError::InvalidRequest( + "image dimensions exceed the OCR raster limit".to_owned(), + )); + } + let mut decoded = DynamicImage::from_decoder(decoder) + .map_err(|error| OcrError::InvalidRequest(format!("image decode failed: {error}")))?; + check_cancelled(cancellation)?; + decoded.apply_orientation(orientation); + let width = decoded.width(); + let height = decoded.height(); + let rgba = decoded.to_rgba8(); + let mut rgb = RgbImage::new(width, height); + for (target, source) in rgb.pixels_mut().zip(rgba.pixels()) { + let alpha = u16::from(source[3]); + for channel in 0..3 { + let foreground = u16::from(source[channel]).saturating_mul(alpha); + let background = 255_u16.saturating_mul(255_u16.saturating_sub(alpha)); + target[channel] = u8::try_from((foreground + background + 127) / 255) + .map_err(|_| OcrError::InvalidRequest("alpha composite overflow".to_owned()))?; + } + } + let (rgb, width, height) = + if width > OCR_MAX_RASTER_LONG_EDGE || height > OCR_MAX_RASTER_LONG_EDGE { + let scale = f64::from(OCR_MAX_RASTER_LONG_EDGE) / f64::from(width.max(height)); + let resized_width = (f64::from(width) * scale).round().max(1.0) as u32; + let resized_height = (f64::from(height) * scale).round().max(1.0) as u32; + ( + image::imageops::resize(&rgb, resized_width, resized_height, FilterType::Triangle), + resized_width, + resized_height, + ) + } else { + (rgb, width, height) + }; + crate::validate_dimensions(width, height)?; + check_cancelled(cancellation)?; + Ok(PreparedRaster { + image: rgb, + width, + height, + }) +} + +fn geometry_key(points: &[oar_ocr::processors::Point]) -> (u32, u32) { + let min_y = points + .iter() + .map(|point| point.y) + .filter(|value| value.is_finite()) + .fold(f32::INFINITY, f32::min); + let min_x = points + .iter() + .map(|point| point.x) + .filter(|value| value.is_finite()) + .fold(f32::INFINITY, f32::min); + (sortable_float(min_y), sortable_float(min_x)) +} + +fn sortable_float(value: f32) -> u32 { + if !value.is_finite() || value <= 0.0 { + 0 + } else { + value.round() as u32 + } +} + +fn quantize_coordinate(value: f32, bound: u32) -> Result { + if !value.is_finite() || value < 0.0 || value > bound as f32 || bound == 0 { + return Err(OcrError::InvalidOutput( + "OCR engine returned out-of-bounds geometry".to_owned(), + )); + } + Ok((value.round() as u32).min(bound - 1)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transparent_pixels_are_composited_on_white() -> Result<(), Box> { + let image = image::RgbaImage::from_pixel(2, 1, image::Rgba([0, 0, 0, 0])); + let mut encoded = Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; + let prepared = prepare_raster(encoded.get_ref())?; + assert_eq!(prepared.image.get_pixel(0, 0), &image::Rgb([255, 255, 255])); + Ok(()) + } + + #[test] + fn coordinate_quantization_is_bounded() { + assert_eq!(quantize_coordinate(100.0, 100).ok(), Some(99)); + assert!(quantize_coordinate(f32::NAN, 100).is_err()); + assert!(quantize_coordinate(-1.0, 100).is_err()); + } + + #[test] + fn cancellation_is_explicit_before_image_decode() { + let cancellation = AtomicBool::new(true); + assert!(matches!( + prepare_raster_cancellable(b"not decoded", &cancellation), + Err(OcrError::Cancelled) + )); + } + + #[test] + fn exif_orientation_is_applied_once_before_normalization() + -> Result<(), Box> { + let source = RgbImage::from_pixel(2, 3, image::Rgb([40, 80, 120])); + let mut jpeg = Cursor::new(Vec::new()); + DynamicImage::ImageRgb8(source).write_to(&mut jpeg, image::ImageFormat::Jpeg)?; + let original = jpeg.into_inner(); + let exif = [ + 0x45, 0x78, 0x69, 0x66, 0, 0, 0x49, 0x49, 0x2a, 0, 8, 0, 0, 0, 1, 0, 0x12, 1, 3, 0, 1, + 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, + ]; + let mut oriented = Vec::with_capacity(original.len() + exif.len() + 4); + oriented.extend_from_slice(&original[..2]); + oriented.extend_from_slice(&[0xff, 0xe1, 0, 34]); + oriented.extend_from_slice(&exif); + oriented.extend_from_slice(&original[2..]); + + let prepared = prepare_raster(&oriented)?; + assert_eq!((prepared.width, prepared.height), (3, 2)); + Ok(()) + } + + #[test] + fn tiles_are_row_major_overlapping_and_cover_the_source() -> Result<(), OcrError> { + let raster = PreparedRaster { + image: RgbImage::new(4_000, 3_000), + width: 4_000, + height: 3_000, + }; + let tiles = tile_raster(&raster)?; + assert_eq!(tiles.len(), 6); + assert_eq!( + tiles + .iter() + .map(|tile| ( + tile.ordinal, + tile.x, + tile.y, + tile.raster.width, + tile.raster.height + )) + .collect::>(), + vec![ + (0, 0, 0, 2_048, 2_048), + (1, 1_920, 0, 2_048, 2_048), + (2, 3_840, 0, 160, 2_048), + (3, 0, 1_920, 2_048, 1_080), + (4, 1_920, 1_920, 2_048, 1_080), + (5, 3_840, 1_920, 160, 1_080), + ] + ); + Ok(()) + } +} diff --git a/crates/compass-ocr/src/lib.rs b/crates/compass-ocr/src/lib.rs new file mode 100644 index 00000000..99cd8f63 --- /dev/null +++ b/crates/compass-ocr/src/lib.rs @@ -0,0 +1,541 @@ +//! Bounded, engine-neutral OCR contracts and a managed local PP-OCR runtime. + +mod engine; +mod models; + +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; + +pub use engine::{ + ManagedOarEngine, OcrEngine, PreparedRaster, PreparedRasterTile, prepare_raster, + prepare_raster_cancellable, prepared_raster_digest, tile_raster, +}; +pub use models::{ + ArtifactFetcher, HttpsArtifactFetcher, ModelCache, ModelFiles, ModelProfile, ModelStatus, + install_profile, list_profiles, profile_manifest_digest, verify_profile, +}; + +pub const OCR_SCHEMA: &str = "compass.ocr/1"; +pub const OCR_PROTOCOL_SCHEMA: &str = "compass.ocr.protocol/1"; +pub const OCR_POLICY_VERSION: u32 = 1; +pub const OCR_PREPROCESSING_VERSION: u32 = 2; + +pub const OCR_MAX_RASTER_PIXELS: u64 = 24_000_000; +pub const OCR_MAX_RASTER_LONG_EDGE: u32 = 6_000; +pub const OCR_ENGINE_MAX_SIDE: u32 = 2_048; +pub const OCR_TILE_OVERLAP: u32 = 128; +pub const OCR_ENGINE_THREADS: usize = 1; +pub const OCR_MAX_DOCUMENT_WALL_TIME_SECS: u64 = 10 * 60; +pub const OCR_MAX_OBSERVATIONS_PER_RASTER: usize = 10_000; +pub const OCR_MAX_OBSERVATIONS_PER_DOCUMENT: usize = 100_000; +pub const OCR_MAX_TEXT_BYTES_PER_OBSERVATION: usize = 16 * 1024; +pub const OCR_MAX_TEXT_CHARS_PER_DOCUMENT: usize = 5_000_000; +pub const OCR_MAX_LANGUAGE_HINTS: usize = 32; +pub const OCR_MAX_PROFILE_FIELD_BYTES: usize = 256; + +pub fn normalize_language_hints(hints: &[String]) -> Result, OcrError> { + if hints.len() > OCR_MAX_LANGUAGE_HINTS { + return Err(OcrError::InvalidRequest( + "too many language hints".to_owned(), + )); + } + let mut normalized = Vec::with_capacity(hints.len()); + for hint in hints { + if hint.is_empty() + || hint.len() > 64 + || hint.split('-').any(|segment| { + segment.is_empty() + || segment.len() > 8 + || !segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }) + || hint.split('-').next().is_none_or(|first| { + first.len() > 8 || !first.bytes().all(|byte| byte.is_ascii_alphabetic()) + }) + { + return Err(OcrError::InvalidRequest(format!( + "invalid BCP-47 language hint {hint:?}" + ))); + } + normalized.push(hint.to_ascii_lowercase()); + } + normalized.sort(); + normalized.dedup(); + Ok(normalized) +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OcrMode { + #[default] + Off, + Auto, + Always, +} + +impl FromStr for OcrMode { + type Err = OcrError; + + fn from_str(value: &str) -> Result { + match value { + "off" => Ok(Self::Off), + "auto" => Ok(Self::Auto), + "always" => Ok(Self::Always), + _ => Err(OcrError::InvalidRequest(format!( + "unknown OCR mode {value:?}; expected off, auto, or always" + ))), + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct OcrProfileIdentity { + pub engine: String, + pub engine_version: String, + pub profile: String, + pub model_digests: BTreeMap, + pub languages: Vec, + pub preprocessing_version: u32, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OcrSourceKind { + PdfPage, + EmbeddedImage, + RasterImage, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct OcrRequest { + pub schema: String, + pub request_id: String, + pub source_kind: OcrSourceKind, + pub width: u32, + pub height: u32, + pub language_hints: Vec, + pub image_digest: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct OcrPoint { + pub x: u32, + pub y: u32, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct OcrObservation { + pub ordinal: u32, + pub polygon: Vec, + pub text: String, + pub confidence_bps: u16, + pub script: Option, + pub orientation_degrees: i16, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct OcrResponse { + pub schema: String, + pub request_id: String, + pub profile: OcrProfileIdentity, + pub observations: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum OcrError { + #[error("invalid OCR request: {0}")] + InvalidRequest(String), + #[error("OCR output rejected: {0}")] + InvalidOutput(String), + #[error("OCR engine unavailable: {0}")] + EngineUnavailable(String), + #[error("OCR model unavailable: {0}")] + ModelUnavailable(String), + #[error("OCR model verification failed: {0}")] + ModelVerification(String), + #[error("OCR inference failed: {0}")] + Inference(String), + #[error("OCR processing was cancelled")] + Cancelled, + #[error("OCR processing timed out")] + Timeout, + #[error("OCR I/O failed for {path}: {source}")] + Io { + path: std::path::PathBuf, + #[source] + source: std::io::Error, + }, +} + +impl OcrRequest { + pub fn validate(&self) -> Result<(), OcrError> { + if self.schema != OCR_SCHEMA { + return Err(OcrError::InvalidRequest(format!( + "unsupported schema {:?}", + self.schema + ))); + } + validate_bounded_field("request ID", &self.request_id)?; + validate_digest(&self.image_digest)?; + validate_dimensions(self.width, self.height)?; + if normalize_language_hints(&self.language_hints)? != self.language_hints { + return Err(OcrError::InvalidRequest( + "language hints must use canonical lowercase sorted unique BCP-47 tags".to_owned(), + )); + } + Ok(()) + } +} + +impl OcrProfileIdentity { + pub fn validate(&self) -> Result<(), OcrError> { + validate_bounded_field("engine", &self.engine)?; + validate_bounded_field("engine version", &self.engine_version)?; + validate_bounded_field("profile", &self.profile)?; + if self.preprocessing_version != OCR_PREPROCESSING_VERSION { + return Err(OcrError::InvalidOutput(format!( + "unsupported preprocessing version {}", + self.preprocessing_version + ))); + } + if self.model_digests.is_empty() || self.model_digests.len() > 16 { + return Err(OcrError::InvalidOutput( + "profile must identify 1 to 16 model artifacts".to_owned(), + )); + } + for (name, digest) in &self.model_digests { + validate_bounded_field("model artifact name", name)?; + validate_digest(digest)?; + } + if self.languages.is_empty() || self.languages.len() > OCR_MAX_LANGUAGE_HINTS { + return Err(OcrError::InvalidOutput( + "profile language set is empty or excessive".to_owned(), + )); + } + let unique = self.languages.iter().collect::>(); + if unique.len() != self.languages.len() + || normalize_language_hints(&self.languages)? != self.languages + { + return Err(OcrError::InvalidOutput( + "profile languages must be canonical BCP-47 tags".to_owned(), + )); + } + Ok(()) + } +} + +impl OcrResponse { + pub fn validate_for(&self, request: &OcrRequest) -> Result<(), OcrError> { + request.validate()?; + if self.schema != OCR_SCHEMA { + return Err(OcrError::InvalidOutput(format!( + "unsupported schema {:?}", + self.schema + ))); + } + if self.request_id != request.request_id { + return Err(OcrError::InvalidOutput( + "response request ID does not match".to_owned(), + )); + } + self.profile.validate()?; + if self.observations.len() > OCR_MAX_OBSERVATIONS_PER_RASTER { + return Err(OcrError::InvalidOutput( + "observation count exceeds raster limit".to_owned(), + )); + } + let mut expected = 0_u32; + for observation in &self.observations { + if observation.ordinal != expected { + return Err(OcrError::InvalidOutput( + "observation ordinals must be contiguous".to_owned(), + )); + } + expected = expected + .checked_add(1) + .ok_or_else(|| OcrError::InvalidOutput("ordinal overflow".to_owned()))?; + validate_observation(observation, request.width, request.height)?; + } + Ok(()) + } +} + +pub fn validate_dimensions(width: u32, height: u32) -> Result<(), OcrError> { + if width == 0 || height == 0 { + return Err(OcrError::InvalidRequest( + "raster dimensions must be nonzero".to_owned(), + )); + } + if width > OCR_MAX_RASTER_LONG_EDGE || height > OCR_MAX_RASTER_LONG_EDGE { + return Err(OcrError::InvalidRequest( + "raster long edge exceeds limit".to_owned(), + )); + } + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| OcrError::InvalidRequest("raster pixel count overflow".to_owned()))?; + if pixels > OCR_MAX_RASTER_PIXELS { + return Err(OcrError::InvalidRequest( + "raster pixel count exceeds limit".to_owned(), + )); + } + Ok(()) +} + +fn validate_observation( + observation: &OcrObservation, + width: u32, + height: u32, +) -> Result<(), OcrError> { + if !(4..=16).contains(&observation.polygon.len()) { + return Err(OcrError::InvalidOutput( + "OCR polygons must contain 4 to 16 points".to_owned(), + )); + } + if observation.text.len() > OCR_MAX_TEXT_BYTES_PER_OBSERVATION { + return Err(OcrError::InvalidOutput( + "OCR observation text exceeds limit".to_owned(), + )); + } + if observation.text.trim().is_empty() { + return Err(OcrError::InvalidOutput( + "OCR observation text is empty".to_owned(), + )); + } + if observation.text.chars().any(char::is_control) { + return Err(OcrError::InvalidOutput( + "OCR observation text contains control characters".to_owned(), + )); + } + if observation.confidence_bps > 10_000 { + return Err(OcrError::InvalidOutput( + "OCR confidence is outside 0..=10000".to_owned(), + )); + } + if !matches!( + observation.orientation_degrees, + -270 | -180 | -90 | 0 | 90 | 180 | 270 + ) { + return Err(OcrError::InvalidOutput( + "OCR orientation is unsupported".to_owned(), + )); + } + if observation + .polygon + .iter() + .any(|point| point.x >= width || point.y >= height) + { + return Err(OcrError::InvalidOutput( + "OCR polygon lies outside the raster".to_owned(), + )); + } + let doubled_area = polygon_doubled_area(&observation.polygon); + if doubled_area == 0 { + return Err(OcrError::InvalidOutput( + "OCR polygon has zero area".to_owned(), + )); + } + if let Some(script) = &observation.script { + validate_bounded_field("script", script)?; + } + Ok(()) +} + +fn polygon_doubled_area(points: &[OcrPoint]) -> i128 { + let mut area = 0_i128; + for index in 0..points.len() { + let current = points[index]; + let next = points[(index + 1) % points.len()]; + area += + i128::from(current.x) * i128::from(next.y) - i128::from(next.x) * i128::from(current.y); + } + area.abs() +} + +fn validate_bounded_field(name: &str, value: &str) -> Result<(), OcrError> { + if value.is_empty() + || value.len() > OCR_MAX_PROFILE_FIELD_BYTES + || value.chars().any(char::is_control) + { + return Err(OcrError::InvalidRequest(format!( + "{name} is empty or exceeds its bound" + ))); + } + Ok(()) +} + +fn validate_digest(value: &str) -> Result<(), OcrError> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + return Err(OcrError::InvalidRequest( + "digest must be 64 canonical lowercase hexadecimal characters".to_owned(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request() -> OcrRequest { + OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: "raster-1".to_owned(), + source_kind: OcrSourceKind::EmbeddedImage, + width: 100, + height: 50, + language_hints: vec!["en".to_owned()], + image_digest: "a".repeat(64), + } + } + + fn profile() -> OcrProfileIdentity { + OcrProfileIdentity { + engine: "oar-ocr".to_owned(), + engine_version: "0.9.1".to_owned(), + profile: "pp-ocrv6-small".to_owned(), + model_digests: BTreeMap::from([("detector".to_owned(), "b".repeat(64))]), + languages: vec!["mul".to_owned()], + preprocessing_version: OCR_PREPROCESSING_VERSION, + } + } + + #[test] + fn validates_bounded_geometry_and_identity() { + let request = request(); + let response = OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: profile(), + observations: vec![OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 10, y: 1 }, + OcrPoint { x: 10, y: 10 }, + OcrPoint { x: 1, y: 10 }, + ], + text: "Compass".to_owned(), + confidence_bps: 9_500, + script: Some("Latn".to_owned()), + orientation_degrees: 0, + }], + }; + assert!(response.validate_for(&request).is_ok()); + } + + #[test] + fn rejects_out_of_bounds_and_zero_area_polygons() { + let mut observation = OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 2, y: 2 }, + OcrPoint { x: 3, y: 3 }, + OcrPoint { x: 4, y: 4 }, + ], + text: "bad".to_owned(), + confidence_bps: 1, + script: None, + orientation_degrees: 0, + }; + assert!(validate_observation(&observation, 100, 50).is_err()); + observation.polygon[3] = OcrPoint { x: 100, y: 4 }; + assert!(validate_observation(&observation, 100, 50).is_err()); + } + + #[test] + fn rejects_empty_control_text_and_noncanonical_digests() { + let mut noncanonical_request = request(); + noncanonical_request.image_digest = "A".repeat(64); + assert!(noncanonical_request.validate().is_err()); + + let request = request(); + for text in [" ", "unsafe\u{1b}[2J"] { + let response = OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: profile(), + observations: vec![OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 10, y: 1 }, + OcrPoint { x: 10, y: 10 }, + OcrPoint { x: 1, y: 10 }, + ], + text: text.to_owned(), + confidence_bps: 9_000, + script: Some("Latn".to_owned()), + orientation_degrees: 0, + }], + }; + assert!(response.validate_for(&request).is_err()); + } + } + + #[test] + fn mode_parser_is_explicit() { + assert_eq!(OcrMode::from_str("auto").ok(), Some(OcrMode::Auto)); + assert!(OcrMode::from_str("maybe").is_err()); + } + + #[test] + fn language_hints_are_validated_and_canonicalized() { + assert_eq!( + normalize_language_hints(&["EN-us".to_owned(), "en-US".to_owned()]).ok(), + Some(vec!["en-us".to_owned()]) + ); + assert_eq!( + normalize_language_hints(&[ + "ZH-Hant-TW".to_owned(), + "ar".to_owned(), + "ja-JP".to_owned(), + ]) + .ok(), + Some(vec![ + "ar".to_owned(), + "ja-jp".to_owned(), + "zh-hant-tw".to_owned(), + ]) + ); + assert!(normalize_language_hints(&["en--US".to_owned()]).is_err()); + assert!(normalize_language_hints(&["not_a_tag".to_owned()]).is_err()); + } + + #[test] + fn accepts_bounded_multilingual_and_emoji_text() { + let request = request(); + let response = OcrResponse { + schema: OCR_SCHEMA.to_owned(), + request_id: request.request_id.clone(), + profile: profile(), + observations: vec![OcrObservation { + ordinal: 0, + polygon: vec![ + OcrPoint { x: 1, y: 1 }, + OcrPoint { x: 90, y: 1 }, + OcrPoint { x: 90, y: 20 }, + OcrPoint { x: 1, y: 20 }, + ], + text: "指南 مرحبا 🧭".to_owned(), + confidence_bps: 8_500, + script: None, + orientation_degrees: 0, + }], + }; + assert!(response.validate_for(&request).is_ok()); + } + + #[test] + fn raster_dimension_limits_accept_exact_and_reject_one_over() { + assert!(validate_dimensions(6_000, 4_000).is_ok()); + assert!(validate_dimensions(6_000, 4_001).is_err()); + assert!(validate_dimensions(6_001, 1).is_err()); + } +} diff --git a/crates/compass-ocr/src/models.rs b/crates/compass-ocr/src/models.rs new file mode 100644 index 00000000..bc8cb00b --- /dev/null +++ b/crates/compass-ocr/src/models.rs @@ -0,0 +1,853 @@ +//! Pinned PP-OCRv6 model acquisition and offline verification. + +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::thread; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{OCR_PREPROCESSING_VERSION, OcrError, OcrProfileIdentity}; + +const MODEL_HOST: &str = "github.com"; +const MODEL_ASSET_HOST: &str = "release-assets.githubusercontent.com"; +const MODEL_REPOSITORY: &str = "GreatV/oar-ocr"; +const MODEL_REVISION: &str = "v0.7.0"; +const ENGINE_VERSION: &str = "0.9.2"; +const USER_AGENT: &str = "compass/0.3 document-ocr"; +const VERIFIED_MARKER_SCHEMA: &str = "compass.ocr.model-profile/1"; +const MODEL_LICENSE: &str = "Apache-2.0 (PaddleOCR models and OAR-OCR runtime)"; +const MODEL_CARD: &str = "https://github.com/GreatV/oar-ocr/releases/tag/v0.7.0"; +const MODEL_INSTALL_LOCK: &str = ".install.lock"; +const MODEL_INSTALL_LOCK_WAIT: Duration = Duration::from_secs(15 * 60); +const MODEL_INSTALL_LOCK_RETRY: Duration = Duration::from_millis(50); +const MODEL_REDIRECT_MAX_BYTES: usize = 8 * 1024; +const MODEL_ERROR_MAX_CHARS: usize = 1_024; +const VERIFIED_MARKER_MAX_BYTES: u64 = 16 * 1024; + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct VerifiedProfileMarker { + schema: String, + profile: String, + engine: String, + engine_version: String, + repository: String, + revision: String, + manifest_digest: String, + license: String, + model_card: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ModelProfile { + PpOcrV6Small, + PpOcrV6Medium, +} + +impl ModelProfile { + #[must_use] + pub fn name(self) -> &'static str { + match self { + Self::PpOcrV6Small => "pp-ocrv6-small", + Self::PpOcrV6Medium => "pp-ocrv6-medium", + } + } + + fn artifacts(self) -> &'static [ArtifactSpec] { + match self { + Self::PpOcrV6Small => &SMALL_ARTIFACTS, + Self::PpOcrV6Medium => &MEDIUM_ARTIFACTS, + } + } +} + +#[must_use] +pub fn profile_manifest_digest(profile: ModelProfile) -> String { + let mut digest = Sha256::new(); + digest.update(ENGINE_VERSION.as_bytes()); + digest.update([0]); + digest.update(MODEL_REPOSITORY.as_bytes()); + digest.update([0]); + digest.update(MODEL_REVISION.as_bytes()); + for artifact in profile.artifacts() { + digest.update([0]); + digest.update(artifact.role.as_bytes()); + digest.update([0]); + digest.update(artifact.name.as_bytes()); + digest.update(artifact.size.to_le_bytes()); + digest.update(artifact.sha256.as_bytes()); + } + format!("sha256:{:x}", digest.finalize()) +} + +impl FromStr for ModelProfile { + type Err = OcrError; + + fn from_str(value: &str) -> Result { + match value { + "pp-ocrv6-small" | "small" => Ok(Self::PpOcrV6Small), + "pp-ocrv6-medium" | "medium" => Ok(Self::PpOcrV6Medium), + _ => Err(OcrError::ModelUnavailable(format!( + "unknown profile {value:?}; expected pp-ocrv6-small or pp-ocrv6-medium" + ))), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct ArtifactSpec { + role: &'static str, + name: &'static str, + size: u64, + sha256: &'static str, +} + +const DICTIONARY: ArtifactSpec = ArtifactSpec { + role: "dictionary", + name: "ppocrv6_dict.txt", + size: 74_947, + sha256: "b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d", +}; + +const SMALL_ARTIFACTS: [ArtifactSpec; 3] = [ + ArtifactSpec { + role: "detector", + name: "pp-ocrv6_small_det.onnx", + size: 9_880_512, + sha256: "d73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e", + }, + ArtifactSpec { + role: "recognizer", + name: "pp-ocrv6_small_rec.onnx", + size: 21_159_378, + sha256: "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + }, + DICTIONARY, +]; + +const MEDIUM_ARTIFACTS: [ArtifactSpec; 3] = [ + ArtifactSpec { + role: "detector", + name: "pp-ocrv6_medium_det.onnx", + size: 62_032_837, + sha256: "eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1", + }, + ArtifactSpec { + role: "recognizer", + name: "pp-ocrv6_medium_rec.onnx", + size: 76_554_979, + sha256: "9c09abf0957f7968c7586464b7397b84ad2387a0497a351af40e9acc71b673ba", + }, + DICTIONARY, +]; + +#[derive(Clone, Debug)] +pub struct ModelFiles { + pub detector: PathBuf, + pub recognizer: PathBuf, + pub dictionary: PathBuf, + pub identity: OcrProfileIdentity, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ModelStatus { + pub profile: String, + pub installed: bool, + pub verified: bool, + pub bytes: u64, + pub license: String, +} + +pub trait ArtifactFetcher { + fn fetch(&self, url: &str, max_bytes: u64) -> Result, OcrError>; +} + +#[derive(Clone)] +pub struct HttpsArtifactFetcher { + agent: ureq::Agent, +} + +impl Default for HttpsArtifactFetcher { + fn default() -> Self { + let config = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(15 * 60))) + .timeout_connect(Some(Duration::from_secs(30))) + .max_redirects(0) + .build(); + Self { + agent: config.into(), + } + } +} + +impl ArtifactFetcher for HttpsArtifactFetcher { + fn fetch(&self, url: &str, max_bytes: u64) -> Result, OcrError> { + let mut current = url.to_owned(); + for redirect in 0..=3 { + validate_model_url(¤t, redirect > 0)?; + let response = self + .agent + .get(¤t) + .header("User-Agent", USER_AGENT) + .call() + .map_err(|error| OcrError::ModelUnavailable(bounded_error(&error.to_string())))?; + if response.status().is_redirection() { + if redirect == 3 { + return Err(OcrError::ModelVerification( + "model download exceeded the redirect limit".to_owned(), + )); + } + let location = response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| { + OcrError::ModelVerification( + "model download redirect has no valid location".to_owned(), + ) + })?; + if location.len() > MODEL_REDIRECT_MAX_BYTES { + return Err(OcrError::ModelVerification( + "model download redirect exceeds its byte limit".to_owned(), + )); + } + current = location.to_owned(); + continue; + } + if !response.status().is_success() { + return Err(OcrError::ModelUnavailable(format!( + "model host returned HTTP {}", + response.status() + ))); + } + let limit = max_bytes + .checked_add(1) + .ok_or_else(|| OcrError::ModelVerification("model size overflow".to_owned()))?; + return Ok(Box::new( + response + .into_body() + .into_with_config() + .limit(limit) + .reader(), + )); + } + Err(OcrError::ModelVerification( + "model download redirect handling failed".to_owned(), + )) + } +} + +fn validate_model_url(url: &str, redirected: bool) -> Result<(), OcrError> { + let parsed = ureq::http::Uri::try_from(url) + .map_err(|error| OcrError::ModelVerification(error.to_string()))?; + let allowed_host = if redirected { + matches!(parsed.host(), Some(MODEL_HOST) | Some(MODEL_ASSET_HOST)) + } else { + parsed.host() == Some(MODEL_HOST) + }; + let unsafe_authority = parsed + .authority() + .is_none_or(|authority| authority.as_str().contains('@')); + let unsafe_port = parsed.port_u16().is_some_and(|port| port != 443); + if parsed.scheme_str() != Some("https") || !allowed_host || unsafe_authority || unsafe_port { + return Err(OcrError::ModelVerification( + "model URL is outside the HTTPS host allowlist".to_owned(), + )); + } + Ok(()) +} + +fn bounded_error(message: &str) -> String { + message.chars().take(MODEL_ERROR_MAX_CHARS).collect() +} + +#[derive(Debug)] +struct ModelInstallGuard { + file: File, +} + +impl ModelInstallGuard { + fn acquire(directory: &Path) -> Result { + Self::acquire_with_timeout(directory, MODEL_INSTALL_LOCK_WAIT) + } + + fn acquire_with_timeout(directory: &Path, timeout: Duration) -> Result { + let path = directory.join(MODEL_INSTALL_LOCK); + if let Ok(metadata) = fs::symlink_metadata(&path) + && (!metadata.is_file() || metadata.file_type().is_symlink()) + { + return Err(OcrError::ModelVerification(format!( + "model install lock is not a regular file: {}", + path.display() + ))); + } + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let file = options.open(&path).map_err(|source| OcrError::Io { + path: path.clone(), + source, + })?; + let deadline = Instant::now() + timeout; + loop { + match file.try_lock() { + Ok(()) => return Ok(Self { file }), + Err(std::fs::TryLockError::WouldBlock) if Instant::now() < deadline => { + thread::sleep(MODEL_INSTALL_LOCK_RETRY); + } + Err(std::fs::TryLockError::WouldBlock) => { + return Err(OcrError::ModelUnavailable(format!( + "timed out waiting for another model installation at {}; retry `compass models install`", + path.display() + ))); + } + Err(std::fs::TryLockError::Error(source)) => { + return Err(OcrError::Io { path, source }); + } + } + } + } +} + +impl Drop for ModelInstallGuard { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} + +#[derive(Clone, Debug)] +pub struct ModelCache { + root: PathBuf, +} + +impl ModelCache { + #[must_use] + pub fn new(root: PathBuf) -> Self { + Self { root } + } + + pub fn from_environment() -> Result { + if let Some(root) = std::env::var_os("COMPASS_CACHE_DIR") { + return Ok(Self::new(PathBuf::from(root).join("models/ocr"))); + } + if let Some(root) = std::env::var_os("XDG_CACHE_HOME") { + return Ok(Self::new(PathBuf::from(root).join("compass/models/ocr"))); + } + if cfg!(windows) + && let Some(root) = std::env::var_os("LOCALAPPDATA") + { + return Ok(Self::new(PathBuf::from(root).join("Compass/models/ocr"))); + } + std::env::var_os("HOME") + .map(PathBuf::from) + .map(|root| Self::new(root.join(".cache/compass/models/ocr"))) + .ok_or_else(|| { + OcrError::ModelUnavailable( + "could not determine Compass's model cache directory".to_owned(), + ) + }) + } + + fn profile_dir(&self, profile: ModelProfile) -> PathBuf { + self.root.join(profile.name()).join(ENGINE_VERSION) + } + + pub fn install( + &self, + profile: ModelProfile, + fetcher: &dyn ArtifactFetcher, + ) -> Result { + let directory = self.profile_dir(profile); + fs::create_dir_all(&directory).map_err(|source| OcrError::Io { + path: directory.clone(), + source, + })?; + let _guard = ModelInstallGuard::acquire(&directory)?; + for artifact in profile.artifacts() { + ensure_artifact(&directory, artifact, fetcher)?; + } + let files = verify_artifacts(&directory, profile)?; + write_verified_marker(&directory, profile)?; + Ok(files) + } + + pub fn verify(&self, profile: ModelProfile) -> Result { + let directory = self.profile_dir(profile); + if !verify_marker(&directory, profile)? { + return Err(OcrError::ModelUnavailable(format!( + "profile {} has no current verification marker; run `compass models install {}`", + profile.name(), + profile.name() + ))); + } + verify_artifacts(&directory, profile) + } + + #[must_use] + pub fn status(&self, profile: ModelProfile) -> ModelStatus { + let installed = profile + .artifacts() + .iter() + .all(|artifact| is_regular_file(&self.profile_dir(profile).join(artifact.name))); + ModelStatus { + profile: profile.name().to_owned(), + installed, + verified: self.verify(profile).is_ok(), + bytes: profile + .artifacts() + .iter() + .map(|artifact| artifact.size) + .sum(), + license: MODEL_LICENSE.to_owned(), + } + } +} + +fn verify_artifacts(directory: &Path, profile: ModelProfile) -> Result { + let mut digests = BTreeMap::new(); + for artifact in profile.artifacts() { + let path = directory.join(artifact.name); + if !verify_artifact(&path, artifact)? { + return Err(OcrError::ModelUnavailable(format!( + "profile {} is missing or invalid; run `compass models install {}`", + profile.name(), + profile.name() + ))); + } + digests.insert(artifact.role.to_owned(), artifact.sha256.to_owned()); + } + Ok(ModelFiles { + detector: directory.join(profile.artifacts()[0].name), + recognizer: directory.join(profile.artifacts()[1].name), + dictionary: directory.join(profile.artifacts()[2].name), + identity: OcrProfileIdentity { + engine: "oar-ocr".to_owned(), + engine_version: ENGINE_VERSION.to_owned(), + profile: profile.name().to_owned(), + model_digests: digests, + languages: vec!["mul".to_owned()], + preprocessing_version: OCR_PREPROCESSING_VERSION, + }, + }) +} + +fn verify_marker(directory: &Path, profile: ModelProfile) -> Result { + let path = directory.join("verified.json"); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(source) => return Err(OcrError::Io { path, source }), + }; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > VERIFIED_MARKER_MAX_BYTES + { + return Ok(false); + } + let file = File::open(&path).map_err(|source| OcrError::Io { + path: path.clone(), + source, + })?; + let opened_metadata = file.metadata().map_err(|source| OcrError::Io { + path: path.clone(), + source, + })?; + if !opened_metadata.is_file() || opened_metadata.len() > VERIFIED_MARKER_MAX_BYTES { + return Ok(false); + } + let mut bytes = Vec::with_capacity(opened_metadata.len() as usize); + file.take(VERIFIED_MARKER_MAX_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|source| OcrError::Io { + path: path.clone(), + source, + })?; + if bytes.len() as u64 > VERIFIED_MARKER_MAX_BYTES { + return Ok(false); + } + let Ok(value) = serde_json::from_slice::(&bytes) else { + return Ok(false); + }; + Ok(value == expected_verified_marker(profile)) +} + +fn write_verified_marker(directory: &Path, profile: ModelProfile) -> Result<(), OcrError> { + let destination = directory.join("verified.json"); + let marker = expected_verified_marker(profile); + let mut temporary = + tempfile::NamedTempFile::new_in(directory).map_err(|source| OcrError::Io { + path: directory.to_path_buf(), + source, + })?; + serde_json::to_writer(&mut temporary, &marker) + .map_err(|error| OcrError::ModelVerification(error.to_string()))?; + temporary.flush().map_err(|source| OcrError::Io { + path: destination.clone(), + source, + })?; + temporary + .as_file() + .sync_all() + .map_err(|source| OcrError::Io { + path: destination.clone(), + source, + })?; + temporary + .persist(&destination) + .map_err(|error| OcrError::Io { + path: destination, + source: error.error, + })?; + sync_directory(directory)?; + Ok(()) +} + +fn expected_verified_marker(profile: ModelProfile) -> VerifiedProfileMarker { + VerifiedProfileMarker { + schema: VERIFIED_MARKER_SCHEMA.to_owned(), + profile: profile.name().to_owned(), + engine: "oar-ocr".to_owned(), + engine_version: ENGINE_VERSION.to_owned(), + repository: MODEL_REPOSITORY.to_owned(), + revision: MODEL_REVISION.to_owned(), + manifest_digest: profile_manifest_digest(profile), + license: MODEL_LICENSE.to_owned(), + model_card: MODEL_CARD.to_owned(), + } +} + +pub fn install_profile(profile: ModelProfile) -> Result { + ModelCache::from_environment()?.install(profile, &HttpsArtifactFetcher::default()) +} + +pub fn verify_profile(profile: ModelProfile) -> Result { + ModelCache::from_environment()?.verify(profile) +} + +pub fn list_profiles() -> Result, OcrError> { + let cache = ModelCache::from_environment()?; + Ok([ModelProfile::PpOcrV6Small, ModelProfile::PpOcrV6Medium] + .into_iter() + .map(|profile| cache.status(profile)) + .collect()) +} + +fn ensure_artifact( + directory: &Path, + artifact: &ArtifactSpec, + fetcher: &dyn ArtifactFetcher, +) -> Result<(), OcrError> { + let destination = directory.join(artifact.name); + if verify_artifact(&destination, artifact)? { + return Ok(()); + } + let url = format!( + "https://{MODEL_HOST}/{MODEL_REPOSITORY}/releases/download/{MODEL_REVISION}/{}", + artifact.name + ); + let mut reader = fetcher.fetch(&url, artifact.size)?; + let mut temporary = + tempfile::NamedTempFile::new_in(directory).map_err(|source| OcrError::Io { + path: directory.to_path_buf(), + source, + })?; + let mut hasher = Sha256::new(); + let mut total = 0_u64; + let mut buffer = vec![0_u8; 1024 * 1024]; + loop { + let read = reader.read(&mut buffer).map_err(|source| OcrError::Io { + path: destination.clone(), + source, + })?; + if read == 0 { + break; + } + total = total + .checked_add(read as u64) + .ok_or_else(|| OcrError::ModelVerification("model size overflow".to_owned()))?; + if total > artifact.size { + return Err(OcrError::ModelVerification(format!( + "{} exceeds its declared size", + artifact.name + ))); + } + hasher.update(&buffer[..read]); + temporary + .write_all(&buffer[..read]) + .map_err(|source| OcrError::Io { + path: destination.clone(), + source, + })?; + } + let digest = format!("{:x}", hasher.finalize()); + if total != artifact.size || digest != artifact.sha256 { + return Err(OcrError::ModelVerification(format!( + "{} failed size or SHA-256 verification", + artifact.name + ))); + } + temporary + .as_file() + .sync_all() + .map_err(|source| OcrError::Io { + path: destination.clone(), + source, + })?; + temporary + .persist(&destination) + .map_err(|error| OcrError::Io { + path: destination.clone(), + source: error.error, + })?; + sync_directory(directory)?; + Ok(()) +} + +fn verify_artifact(path: &Path, artifact: &ArtifactSpec) -> Result { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(source) => { + return Err(OcrError::Io { + path: path.to_path_buf(), + source, + }); + } + }; + if !metadata.is_file() || metadata.file_type().is_symlink() || metadata.len() != artifact.size { + return Ok(false); + } + let file = File::open(path).map_err(|source| OcrError::Io { + path: path.to_path_buf(), + source, + })?; + let opened_metadata = file.metadata().map_err(|source| OcrError::Io { + path: path.to_path_buf(), + source, + })?; + if !opened_metadata.is_file() || opened_metadata.len() != artifact.size { + return Ok(false); + } + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 1024 * 1024]; + let mut total = 0_u64; + let mut bounded = file.take(artifact.size.saturating_add(1)); + loop { + let read = bounded.read(&mut buffer).map_err(|source| OcrError::Io { + path: path.to_path_buf(), + source, + })?; + if read == 0 { + break; + } + total = total.saturating_add(read as u64); + hasher.update(&buffer[..read]); + } + Ok(total == artifact.size && format!("{:x}", hasher.finalize()) == artifact.sha256) +} + +fn is_regular_file(path: &Path) -> bool { + fs::symlink_metadata(path) + .is_ok_and(|metadata| metadata.is_file() && !metadata.file_type().is_symlink()) +} + +fn sync_directory(path: &Path) -> Result<(), OcrError> { + #[cfg(unix)] + { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(|source| OcrError::Io { + path: path.to_path_buf(), + source, + }) + } + #[cfg(not(unix))] + { + let _ = path; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::io::Cursor; + + use super::*; + + struct StaticFetcher { + body: Vec, + calls: Cell, + } + + impl ArtifactFetcher for StaticFetcher { + fn fetch(&self, _url: &str, _max_bytes: u64) -> Result, OcrError> { + self.calls.set(self.calls.get() + 1); + Ok(Box::new(Cursor::new(self.body.clone()))) + } + } + + #[test] + fn profiles_are_pinned_and_complete() { + for profile in [ModelProfile::PpOcrV6Small, ModelProfile::PpOcrV6Medium] { + assert_eq!(profile.artifacts().len(), 3); + assert!(profile.artifacts().iter().all(|artifact| { + artifact.size > 0 + && artifact.sha256.len() == 64 + && artifact.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + })); + } + } + + #[test] + fn model_urls_require_https_and_the_fixed_release_hosts() { + assert!( + validate_model_url( + "https://github.com/GreatV/oar-ocr/releases/download/v0.7.0/model.onnx", + false + ) + .is_ok() + ); + assert!(validate_model_url( + "https://release-assets.githubusercontent.com/github-production-release-asset/model", + true + ) + .is_ok()); + assert!(validate_model_url("http://github.com/model", false).is_err()); + assert!(validate_model_url("https://example.com/model", true).is_err()); + assert!(validate_model_url("https://user@github.com/model", true).is_err()); + assert!(validate_model_url("https://github.com:444/model", true).is_err()); + } + + #[test] + fn concurrent_model_install_lock_is_bounded_and_reusable() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let first = + ModelInstallGuard::acquire_with_timeout(directory.path(), Duration::from_secs(1))?; + let blocked = ModelInstallGuard::acquire_with_timeout(directory.path(), Duration::ZERO); + assert!(matches!(blocked, Err(OcrError::ModelUnavailable(_)))); + drop(first); + let second = ModelInstallGuard::acquire_with_timeout(directory.path(), Duration::ZERO)?; + drop(second); + Ok(()) + } + + #[test] + fn wrong_download_never_publishes() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let fetcher = StaticFetcher { + body: b"wrong".to_vec(), + calls: Cell::new(0), + }; + let result = ensure_artifact(directory.path(), &DICTIONARY, &fetcher); + assert!(result.is_err()); + assert!(!directory.path().join(DICTIONARY.name).exists()); + assert_eq!(fetcher.calls.get(), 1); + Ok(()) + } + + #[test] + fn verified_download_is_atomic_reusable_and_revision_marked() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let artifact = ArtifactSpec { + role: "fixture", + name: "fixture.bin", + size: 7, + sha256: "f16d05ec6b29248d2c61adb1e9263f78e4f7bace1b955014a2d17872cfe4064d", + }; + let fetcher = StaticFetcher { + body: b"fixture".to_vec(), + calls: Cell::new(0), + }; + ensure_artifact(directory.path(), &artifact, &fetcher)?; + ensure_artifact(directory.path(), &artifact, &fetcher)?; + assert_eq!(fetcher.calls.get(), 1); + assert_eq!(fs::read(directory.path().join("fixture.bin"))?, b"fixture"); + + write_verified_marker(directory.path(), ModelProfile::PpOcrV6Small)?; + let marker: serde_json::Value = + serde_json::from_slice(&fs::read(directory.path().join("verified.json"))?)?; + assert_eq!(marker["schema"], VERIFIED_MARKER_SCHEMA); + assert_eq!(marker["revision"], "v0.7.0"); + assert_eq!( + marker["manifest_digest"], + profile_manifest_digest(ModelProfile::PpOcrV6Small) + ); + assert!(verify_marker(directory.path(), ModelProfile::PpOcrV6Small)?); + let mut unknown = marker.clone(); + unknown["unexpected"] = serde_json::json!(true); + fs::write( + directory.path().join("verified.json"), + serde_json::to_vec(&unknown)?, + )?; + assert!(!verify_marker( + directory.path(), + ModelProfile::PpOcrV6Small + )?); + fs::write( + directory.path().join("verified.json"), + serde_json::to_vec(&marker)?, + )?; + let mut stale = marker; + stale["revision"] = serde_json::json!("mutable-branch"); + fs::write( + directory.path().join("verified.json"), + serde_json::to_vec(&stale)?, + )?; + assert!(!verify_marker( + directory.path(), + ModelProfile::PpOcrV6Small + )?); + let oversized = File::create(directory.path().join("verified.json"))?; + oversized.set_len(VERIFIED_MARKER_MAX_BYTES + 1)?; + assert!(!verify_marker( + directory.path(), + ModelProfile::PpOcrV6Small + )?); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn verification_rejects_symlinked_artifacts_and_markers() + -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let directory = tempfile::tempdir()?; + let target = directory.path().join("target.bin"); + fs::write(&target, b"fixture")?; + let link = directory.path().join("fixture.bin"); + symlink(&target, &link)?; + let artifact = ArtifactSpec { + role: "fixture", + name: "fixture.bin", + size: 7, + sha256: "f16d05ec6b29248d2c61adb1e9263f78e4f7bace1b955014a2d17872cfe4064d", + }; + assert!(!verify_artifact(&link, &artifact)?); + + let marker_target = directory.path().join("marker-target.json"); + fs::write( + &marker_target, + serde_json::to_vec(&expected_verified_marker(ModelProfile::PpOcrV6Small))?, + )?; + symlink(&marker_target, directory.path().join("verified.json"))?; + assert!(!verify_marker( + directory.path(), + ModelProfile::PpOcrV6Small + )?); + Ok(()) + } +} diff --git a/crates/compass-ocr/tests/model_acceptance.rs b/crates/compass-ocr/tests/model_acceptance.rs new file mode 100644 index 00000000..3090be14 --- /dev/null +++ b/crates/compass-ocr/tests/model_acceptance.rs @@ -0,0 +1,159 @@ +use compass_ocr::{ + ManagedOarEngine, ModelProfile, OCR_SCHEMA, OcrEngine, OcrRequest, OcrSourceKind, + PreparedRaster, prepared_raster_digest, +}; +use image::{Rgb, RgbImage}; + +#[test] +#[ignore = "requires an already-installed verified pp-ocrv6-small profile"] +fn installed_profile_runs_without_external_runtime_dependencies() +-> Result<(), Box> { + let engine = match ManagedOarEngine::load(ModelProfile::PpOcrV6Small) { + Ok(engine) => engine, + Err(error) if error.to_string().contains("models install pp-ocrv6-small") => { + eprintln!("SKIP: exact pp-ocrv6-small profile is not installed"); + return Ok(()); + } + Err(error) => return Err(error.into()), + }; + let image = RgbImage::from_pixel(256, 128, Rgb([255, 255, 255])); + let raster = PreparedRaster { + image, + width: 256, + height: 128, + }; + let request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: "installed-model-acceptance".to_owned(), + source_kind: OcrSourceKind::EmbeddedImage, + width: 256, + height: 128, + language_hints: Vec::new(), + image_digest: prepared_raster_digest(&raster), + }; + let response = engine.recognize(&request, &raster)?; + response.validate_for(&request)?; + assert_eq!(response.profile.profile, "pp-ocrv6-small"); + + let image = synthetic_clean_english("COMPASS OCR 2026"); + let raster = PreparedRaster { + width: image.width(), + height: image.height(), + image, + }; + let request = OcrRequest { + schema: OCR_SCHEMA.to_owned(), + request_id: "installed-model-clean-english".to_owned(), + source_kind: OcrSourceKind::RasterImage, + width: raster.width, + height: raster.height, + language_hints: vec!["en".to_owned()], + image_digest: prepared_raster_digest(&raster), + }; + let response = engine.recognize(&request, &raster)?; + let actual = response + .observations + .iter() + .map(|observation| observation.text.as_str()) + .collect::>() + .join(" "); + let expected = normalize("COMPASS OCR 2026"); + let actual = normalize(&actual); + let distance = edit_distance(&expected, &actual); + let cer_bps = distance.saturating_mul(10_000) / expected.chars().count().max(1); + eprintln!("clean-English expected={expected:?} actual={actual:?} cer_bps={cer_bps}"); + assert!(cer_bps <= 500, "clean English CER exceeds 5% release gate"); + Ok(()) +} + +fn synthetic_clean_english(text: &str) -> RgbImage { + const SCALE: u32 = 8; + let mut image = RgbImage::from_pixel(900, 160, Rgb([255, 255, 255])); + let mut x = 34_u32; + for character in text.chars() { + if character == ' ' { + x += 4 * SCALE; + continue; + } + if let Some(rows) = glyph(character) { + for (row, bits) in rows.iter().enumerate() { + for column in 0..5_u32 { + if bits & (1 << (4 - column)) != 0 { + for dy in 0..SCALE { + for dx in 0..SCALE { + image.put_pixel( + x + column * SCALE + dx, + 48 + u32::try_from(row).unwrap_or(0) * SCALE + dy, + Rgb([0, 0, 0]), + ); + } + } + } + } + } + } + x += 6 * SCALE; + } + image +} + +fn glyph(character: char) -> Option<[u8; 7]> { + match character { + 'A' => Some([ + 0b01110, 0b10001, 0b10001, 0b11111, 0b10001, 0b10001, 0b10001, + ]), + 'C' => Some([ + 0b01110, 0b10001, 0b10000, 0b10000, 0b10000, 0b10001, 0b01110, + ]), + 'M' => Some([ + 0b10001, 0b11011, 0b10101, 0b10101, 0b10001, 0b10001, 0b10001, + ]), + 'O' => Some([ + 0b01110, 0b10001, 0b10001, 0b10001, 0b10001, 0b10001, 0b01110, + ]), + '0' => Some([ + 0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110, + ]), + 'P' => Some([ + 0b11110, 0b10001, 0b10001, 0b11110, 0b10000, 0b10000, 0b10000, + ]), + 'R' => Some([ + 0b11110, 0b10001, 0b10001, 0b11110, 0b10100, 0b10010, 0b10001, + ]), + 'S' => Some([ + 0b01111, 0b10000, 0b10000, 0b01110, 0b00001, 0b00001, 0b11110, + ]), + '2' => Some([ + 0b01110, 0b10001, 0b00001, 0b00010, 0b00100, 0b01000, 0b11111, + ]), + '6' => Some([ + 0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110, + ]), + _ => None, + } +} + +fn normalize(value: &str) -> String { + value + .chars() + .filter(|character| character.is_alphanumeric()) + .flat_map(char::to_uppercase) + .collect() +} + +fn edit_distance(left: &str, right: &str) -> usize { + let right = right.chars().collect::>(); + let mut previous = (0..=right.len()).collect::>(); + for (row, left_character) in left.chars().enumerate() { + let mut current = vec![row + 1]; + for (column, right_character) in right.iter().enumerate() { + current.push( + (previous[column + 1] + 1) + .min(current[column] + 1) + .min(previous[column] + usize::from(left_character != *right_character)), + ); + } + previous = current; + } + previous.last().copied().unwrap_or(0) +} diff --git a/crates/compass-semantic/src/lib.rs b/crates/compass-semantic/src/lib.rs index b69dad67..e62dda61 100644 --- a/crates/compass-semantic/src/lib.rs +++ b/crates/compass-semantic/src/lib.rs @@ -13,6 +13,7 @@ use std::io::{Read, Write}; use std::net::{IpAddr, ToSocketAddrs}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::Arc; use std::sync::OnceLock; use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::thread; @@ -106,6 +107,8 @@ pub enum SemanticError { InvalidProviderResponse(String), #[error("invalid provider configuration: {0}")] InvalidProviderConfiguration(String), + #[error("document processing failed: {0}")] + DocumentProcessing(String), #[error("provider transport failed: {0}")] Transport(String), #[error("semantic cache failed: {0}")] @@ -509,12 +512,12 @@ pub fn build_untrusted_prompt(sources: &[EvidenceSource<'_>], root: &Path) -> St .strip_prefix(root) .unwrap_or(source.path) .to_string_lossy(); - let capped = source + let bounded = source .content .chars() .take(FILE_CHAR_CAP) .collect::(); - Some(wrap_untrusted_source(&relative, &capped)) + Some(wrap_untrusted_source(&relative, &bounded)) }) .collect::>() .join("\n\n") @@ -742,6 +745,32 @@ pub fn strip_partial_markers(result: &mut Value) { pub enum SemanticUnit { File(PathBuf), Slice(FileSlice), + DocumentSlice(DocumentSlice), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DocumentSlice { + pub path: PathBuf, + pub content: Arc, + pub start: usize, + pub end: usize, + pub ordinal: u32, + pub total: u32, + pub schema: &'static str, + pub normalizer_version: u32, +} + +impl DocumentSlice { + pub fn text(&self) -> Result<&str, String> { + if self.start > self.end + || self.end > self.content.len() + || !self.content.is_char_boundary(self.start) + || !self.content.is_char_boundary(self.end) + { + return Err("document slice has invalid UTF-8 boundaries".to_owned()); + } + Ok(&self.content[self.start..self.end]) + } } impl SemanticUnit { @@ -750,6 +779,7 @@ impl SemanticUnit { match self { Self::File(path) => path, Self::Slice(slice) => &slice.path, + Self::DocumentSlice(slice) => &slice.path, } } } @@ -799,10 +829,25 @@ pub fn read_semantic_units(units: &[SemanticUnit], root: &Path) -> SemanticReadR }; read_slice_text(&safe_slice).ok() } + SemanticUnit::DocumentSlice(slice) => match slice.text() { + Ok(content) => Some(content.to_owned()), + Err(error) => { + result.warnings.push(format!( + "could not read semantic document slice {}: {error}", + path.display() + )); + None + } + }, SemanticUnit::File(_) => match extract_text(&resolved_path) { Ok(content) => Some(content), - Err(_) if is_compat_binary_document(&resolved_path) => Some(String::new()), - Err(_) => None, + Err(error) => { + result.warnings.push(format!( + "could not decode semantic source {}: {error}", + path.display() + )); + None + } }, }; let Some(content) = loaded else { @@ -811,7 +856,6 @@ pub fn read_semantic_units(units: &[SemanticUnit], root: &Path) -> SemanticReadR .push(format!("could not read semantic source {}", path.display())); continue; }; - let content = content.chars().take(FILE_CHAR_CAP).collect::(); result.sources.push(LoadedSemanticSource { path: resolved_path, relative_path, @@ -827,16 +871,6 @@ pub fn read_semantic_units(units: &[SemanticUnit], root: &Path) -> SemanticReadR result } -fn is_compat_binary_document(path: &Path) -> bool { - path.extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| { - ["pdf", "docx", "xlsx"] - .iter() - .any(|candidate| extension.eq_ignore_ascii_case(candidate)) - }) -} - mod orchestration; pub use orchestration::*; /// True for OpenAI reasoning-model families that reject explicit temperature. diff --git a/crates/compass-semantic/src/orchestration.rs b/crates/compass-semantic/src/orchestration.rs index b9e70487..db95bd19 100644 --- a/crates/compass-semantic/src/orchestration.rs +++ b/crates/compass-semantic/src/orchestration.rs @@ -94,8 +94,38 @@ pub fn extract_semantic_units_custom( /// Expand oversized splittable documents into complete, gap-free slices. #[must_use] pub fn expand_oversized_semantic_files(paths: &[PathBuf], max_chars: usize) -> Vec { + let prepared = PreparedDocumentInputs::default(); + paths + .iter() + .flat_map(|path| { + expand_semantic_files_with_prepared_documents( + std::slice::from_ref(path), + max_chars, + &prepared, + ) + .unwrap_or_else(|_| vec![SemanticUnit::File(path.clone())]) + }) + .collect() +} + +/// Expand semantic files from application-prepared document text. This layer +/// never runs OCR or creates a second document cache. +pub fn expand_semantic_files_with_prepared_documents( + paths: &[PathBuf], + max_chars: usize, + prepared_documents: &PreparedDocumentInputs, +) -> Result, SemanticError> { let mut units = Vec::new(); for path in paths { + if is_rich_document(path) { + let slices = prepare_document_slices( + path, + max_chars, + prepared_documents.documents.get(path).cloned(), + )?; + units.extend(slices.into_iter().map(SemanticUnit::DocumentSlice)); + continue; + } let splittable = path .extension() .and_then(|extension| extension.to_str()) @@ -115,7 +145,80 @@ pub fn expand_oversized_semantic_files(paths: &[PathBuf], max_chars: usize) -> V _ => units.push(SemanticUnit::File(path.clone())), } } - units + Ok(units) +} + +fn is_rich_document(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + ["pdf", "docx", "xlsx", "pptx"] + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }) +} + +fn prepare_document_slices( + path: &Path, + max_chars: usize, + prepared_content: Option>, +) -> Result, SemanticError> { + if max_chars == 0 { + return Err(SemanticError::DocumentProcessing( + "document slice size must be positive".to_owned(), + )); + } + let content = match prepared_content { + Some(content) => content, + None => { + let bytes = fs::read(path).map_err(|source| SemanticError::Read { + path: path.to_path_buf(), + source, + })?; + let artifact = compass_media::decode_document(path, &bytes).map_err(|error| { + SemanticError::DocumentProcessing(format!("{}: {error}", path.display())) + })?; + Arc::::from(compass_media::render_document_markdown(&artifact).map_err( + |error| SemanticError::DocumentProcessing(format!("{}: {error}", path.display())), + )?) + } + }; + let mut boundaries = vec![0_usize]; + let mut chars = 0_usize; + for (byte, _) in content.char_indices() { + if chars == max_chars { + boundaries.push(byte); + chars = 0; + } + chars = chars.saturating_add(1); + } + if boundaries.last().copied() != Some(content.len()) { + boundaries.push(content.len()); + } + if boundaries.len() == 1 { + boundaries.push(content.len()); + } + let total = u32::try_from(boundaries.len().saturating_sub(1)).map_err(|_| { + SemanticError::DocumentProcessing("document slice count overflow".to_owned()) + })?; + boundaries + .windows(2) + .enumerate() + .map(|(index, bounds)| { + Ok(DocumentSlice { + path: path.to_path_buf(), + content: Arc::clone(&content), + start: bounds[0], + end: bounds[1], + ordinal: u32::try_from(index).map_err(|_| { + SemanticError::DocumentProcessing("document slice ordinal overflow".to_owned()) + })?, + total, + schema: compass_media::DOCUMENT_SCHEMA, + normalizer_version: compass_media::DOCUMENT_NORMALIZER_VERSION, + }) + }) + .collect() } /// Estimate prompt cost using Compass's deterministic chars-per-token @@ -127,6 +230,10 @@ pub fn estimate_semantic_unit_tokens(unit: &SemanticUnit) -> usize { } let chars = match unit { SemanticUnit::Slice(slice) => slice.end.saturating_sub(slice.start).min(FILE_CHAR_CAP), + SemanticUnit::DocumentSlice(slice) => slice + .text() + .map(|text| text.chars().count()) + .unwrap_or_default(), SemanticUnit::File(path) => match fs::metadata(path) { Ok(metadata) => usize::try_from(metadata.len()) .unwrap_or(usize::MAX) @@ -271,6 +378,26 @@ fn split_semantic_chunk( vec![SemanticUnit::Slice(right)], )); } + if let [SemanticUnit::DocumentSlice(slice)] = chunk { + let text = slice.text().ok()?; + if text.chars().count() < 2 { + return None; + } + let midpoint_chars = text.chars().count() / 2; + let relative = text + .char_indices() + .nth(midpoint_chars) + .map(|(byte, _)| byte)?; + let midpoint = slice.start.checked_add(relative)?; + let mut left = slice.clone(); + left.end = midpoint; + let mut right = slice.clone(); + right.start = midpoint; + return Some(( + vec![SemanticUnit::DocumentSlice(left)], + vec![SemanticUnit::DocumentSlice(right)], + )); + } if chunk.len() <= 1 { return None; } @@ -465,6 +592,25 @@ pub struct CorpusExtractionOptions { pub max_retry_depth: usize, } +#[derive(Clone, Debug, PartialEq)] +pub struct PreparedDocumentInputs { + pub documents: BTreeMap>, + pub cache_identity: String, +} + +impl Default for PreparedDocumentInputs { + fn default() -> Self { + Self { + documents: BTreeMap::new(), + cache_identity: format!( + "schema={};normalizer={};mode=off", + compass_media::DOCUMENT_SCHEMA, + compass_media::DOCUMENT_NORMALIZER_VERSION + ), + } + } +} + impl Default for CorpusExtractionOptions { fn default() -> Self { Self { @@ -494,6 +640,7 @@ pub struct CorpusExtractionResult { #[derive(Clone, Debug, PartialEq)] pub struct CachedCorpusExtractionOptions { pub extraction: CorpusExtractionOptions, + pub prepared_documents: PreparedDocumentInputs, pub deep_mode: bool, pub force: bool, pub cache_enabled: bool, @@ -506,6 +653,7 @@ impl Default for CachedCorpusExtractionOptions { fn default() -> Self { Self { extraction: CorpusExtractionOptions::default(), + prepared_documents: PreparedDocumentInputs::default(), deep_mode: false, force: false, cache_enabled: true, @@ -620,7 +768,34 @@ where F: Fn(&[SemanticUnit]) -> Result + Sync, P: FnMut(usize, usize, &[SemanticUnit], &Value) + Send, { - let units = expand_oversized_semantic_files(files, FILE_CHAR_CAP); + extract_corpus_parallel_with_progress_prepared_documents( + files, + root, + options, + &PreparedDocumentInputs::default(), + environment, + extract, + on_chunk_done, + ) +} + +/// Progress-aware corpus extraction with an explicit local document OCR +/// profile. OCR and document normalization complete before worker dispatch. +pub fn extract_corpus_parallel_with_progress_prepared_documents( + files: &[PathBuf], + root: &Path, + options: &CorpusExtractionOptions, + prepared_documents: &PreparedDocumentInputs, + environment: &HashMap, + extract: &F, + on_chunk_done: &mut P, +) -> Result +where + F: Fn(&[SemanticUnit]) -> Result + Sync, + P: FnMut(usize, usize, &[SemanticUnit], &Value) + Send, +{ + let units = + expand_semantic_files_with_prepared_documents(files, FILE_CHAR_CAP, prepared_documents)?; let chunks = if let Some(token_budget) = options.token_budget { pack_semantic_chunks(&units, token_budget)? } else { @@ -749,7 +924,7 @@ where F: Fn(&[SemanticUnit]) -> Result + Sync, P: FnMut(usize, usize, &[SemanticUnit], &Value) + Send, { - let prompt = extraction_prompt(options.deep_mode); + let prompt = semantic_cache_prompt(options.deep_mode, &options.prepared_documents); let cache_enabled = options.cache_enabled && !environment.contains_key("COMPASS_NO_INCREMENTAL_CACHE"); let cache_options = cache_root.map_or_else( @@ -817,10 +992,11 @@ where } on_chunk_done(index, total, chunk, fragment); }; - extract_corpus_parallel_with_progress( + extract_corpus_parallel_with_progress_prepared_documents( &checked.uncached, root, &options.extraction, + &options.prepared_documents, environment, extract, &mut checkpoint, @@ -1016,7 +1192,7 @@ impl SemanticCacheSaveOptions { allowed_source_files: None, partial_source_files: Vec::new(), deep_mode, - prompt: extraction_prompt(deep_mode), + prompt: semantic_cache_prompt(deep_mode, &PreparedDocumentInputs::default()), } } } @@ -1043,6 +1219,17 @@ fn semantic_cache_kind(deep_mode: bool) -> CacheKind { } } +pub(crate) fn semantic_cache_prompt( + deep_mode: bool, + prepared_documents: &PreparedDocumentInputs, +) -> String { + format!( + "{}\n[compass-document-cache {}]", + extraction_prompt(deep_mode), + prepared_documents.cache_identity + ) +} + /// Replay complete per-file semantic entries produced by the same prompt. pub fn check_semantic_cache( cache: &mut Cache, diff --git a/crates/compass-semantic/src/tests.rs b/crates/compass-semantic/src/tests.rs index ffaf4d32..58119f5d 100644 --- a/crates/compass-semantic/src/tests.rs +++ b/crates/compass-semantic/src/tests.rs @@ -965,7 +965,7 @@ fn cached_corpus_orchestration_checkpoints_replays_and_prunes() -> Result<(), Bo cache, &[chunk[0].path().to_path_buf()], false, - &extraction_prompt(false), + &semantic_cache_prompt(false, &PreparedDocumentInputs::default()), ) .ok() }); diff --git a/crates/compass-semantic/tests/orchestration_coverage.rs b/crates/compass-semantic/tests/orchestration_coverage.rs index aca0a9f0..420a71bd 100644 --- a/crates/compass-semantic/tests/orchestration_coverage.rs +++ b/crates/compass-semantic/tests/orchestration_coverage.rs @@ -1,13 +1,15 @@ use std::collections::HashMap; use std::error::Error; use std::fs; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use compass_files::{Cache, CacheOptions}; use compass_semantic::{ - CachedCorpusExtractionOptions, CorpusExtractionOptions, SemanticCacheSaveOptions, - SemanticError, SemanticUnit, check_semantic_cache, effective_semantic_concurrency, - estimate_semantic_unit_tokens, expand_oversized_semantic_files, extract_corpus_cached_with, + CachedCorpusExtractionOptions, CorpusExtractionOptions, PreparedDocumentInputs, + SemanticCacheSaveOptions, SemanticError, SemanticUnit, check_semantic_cache, + effective_semantic_concurrency, estimate_semantic_unit_tokens, expand_oversized_semantic_files, + expand_semantic_files_with_prepared_documents, extract_corpus_cached_with, extract_corpus_parallel_with, extract_corpus_parallel_with_progress, extract_with_adaptive_retry, merge_semantic_results, pack_semantic_chunks, reconcile_semantic_scope, save_semantic_cache, @@ -22,6 +24,30 @@ fn node_for(unit: &SemanticUnit, id: String) -> Value { }) } +#[test] +fn prepared_rich_documents_are_sliced_losslessly_past_the_legacy_cap() -> Result<(), Box> +{ + let directory = tempfile::tempdir()?; + let path = directory.path().join("large.docx"); + let content = Arc::::from(format!("{}END-🧭", "文档🙂".repeat(7_500))); + let prepared = PreparedDocumentInputs { + documents: std::collections::BTreeMap::from([(path.clone(), Arc::clone(&content))]), + cache_identity: "fixture-document-cache-v1".to_owned(), + }; + let units = expand_semantic_files_with_prepared_documents(&[path], 2_003, &prepared)?; + assert!(units.len() > 10); + let rebuilt = units + .iter() + .map(|unit| match unit { + SemanticUnit::DocumentSlice(slice) => slice.text(), + _ => Err("expected only prepared document slices".to_owned()), + }) + .collect::>()?; + assert_eq!(rebuilt, content.as_ref()); + assert!(rebuilt.ends_with("END-🧭")); + Ok(()) +} + #[test] fn packing_estimation_and_adaptive_retry_cover_slices_images_limits_and_partial_results() -> Result<(), Box> { @@ -363,6 +389,7 @@ fn slice_retry_cache_disabled_and_deep_partial_merges_cover_public_orchestration let cached_options = CachedCorpusExtractionOptions { extraction: options, + prepared_documents: compass_semantic::PreparedDocumentInputs::default(), deep_mode: true, force: false, cache_enabled: false, diff --git a/docs/README.md b/docs/README.md index cec1a3ec..534514e9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,7 @@ to yours: - [Integrate Compass with other tools](guides/integrating-compass.md) - [Set up a coding assistant](guides/assistant-setup.md) - [Use versioned graph history](guides/versioned-history.md) +- [Process PDF and Office documents](reference/document-formats.md) - [Review pull requests in GitHub](guides/github-pr-review.md) - [Operate watch, serve, hooks, and providers](guides/operations.md) - [Solve a concrete problem](cookbook/README.md) @@ -104,6 +105,7 @@ They are not evidence that an uncompleted design has shipped. | Document | Purpose | | --- | --- | | [Universal evidence implementation](implementation/universal-evidence.md) | Current universal evidence pipeline, resolution order, and failure classes | +| [Document OCR qualification](implementation/document-ocr-qualification.md) | Offline contract gate and opt-in pinned-model acceptance procedure | | [Evidence resolution framework technical design](implementation/evidence-resolution-framework-technical-design.md) | Target ownership, components, interfaces, and invariants for rearchitecting the resolver | | [Evidence resolution framework execution plan](implementation/evidence-resolution-framework-phased-execution-plan.md) | Phased, commit-oriented implementation and verification plan | | [Architecture graph hardening technical design](implementation/architecture-graph-hardening-phased-technical-design.md) | Project-specific architecture projection, quality contract, and phased delivery plan | @@ -154,10 +156,11 @@ document and open a documentation issue. ## Product status in one paragraph Compass is a Rust workspace that ships the `compass` executable. Structural -code extraction and graph queries run locally and do not require Python, -embeddings, a vector database, or runtime grammar downloads. Semantic -extraction for documents and other non-code sources is optional and may contact -the provider you explicitly configure. The current release packaging and +code and document extraction and graph queries run locally and do not require +Python, embeddings, a vector database, or runtime grammar downloads. OCR is +optional and local with Compass-managed verified models. Semantic provider +enrichment is separate and may contact the provider you explicitly configure. +The current release packaging and platform guarantees are recorded in the [compatibility ledger](../COMPATIBILITY.md), not inferred from what happens to compile on one developer machine. diff --git a/docs/design/document-processing.md b/docs/design/document-processing.md index 80317428..dea86e7f 100644 --- a/docs/design/document-processing.md +++ b/docs/design/document-processing.md @@ -4,8 +4,8 @@ meta: title: Structural document processing navLabel: Document Processing category: Design - overview: How Compass turns Markdown and HTML bytes into bounded, deterministic graph evidence. - goal: Define the ownership, provenance, and cache rules for local text documents. + overview: How Compass turns text, PDF, and Office bytes into bounded native and OCR evidence. + goal: Define ownership, provenance, cache, OCR, and graph rules for local documents. audience: - Compass contributors - technical evaluators @@ -20,9 +20,10 @@ meta: # Structural document processing Compass treats a document as an ordered source artifact, not as a bag of -extracted strings. The current structural implementation is Markdown-first: -the same bounded bytes read by the build pipeline are parsed into a document -root, structural blocks, and provenance-preserving relationships. +extracted strings. Markdown and HTML retain exact source ranges. PDF, DOCX, +PPTX, and XLSX use the versioned `compass.document/1` intermediate artifact, +typed logical locators, and one shared projection into graph blocks and +semantic slices. ## Ownership and data flow @@ -175,16 +176,61 @@ partial response cannot replace a deterministic structural realization. ## Other document formats -File discovery recognizes several document extensions, but recognition is not -the same as structural extraction. DOCX and XLSX retain their bounded media -conversion surfaces; PPTX and RTF remain future format adapters. See the -[document format reference](../reference/document-formats.md) for the current -matrix. Markdown and HTML links may point at those formats, but Compass does -not fetch or execute a linked resource during extraction. +PDF and OOXML packages are decoded in pure Rust under centralized raw-byte, +archive-member, expansion-ratio, XML-depth, block, link, row, cell, page, and +raster limits. DOCX body order, PPTX relationship slide order, and sparse XLSX +coordinates are preserved. Spreadsheet formulas are evidence and are never +executed. External OOXML relationships remain inert. + +OCR is an optional derived layer and is off by default. `auto` selects PDF +pages with little native text and eligible embedded Office images; `always` +selects every bounded candidate. Native text remains authoritative and OCR +observations retain the owning page/image locator, polygon, confidence, exact +engine version, profile, model digests, and preprocessing version. OCR never +replaces or silently deduplicates native blocks. + +Preprocessing version 2 applies declared EXIF orientation exactly once, +composites alpha onto white, resizes with the fixed triangle filter, and tiles +rasters above the 2,048-pixel engine side with 128 pixels of overlap. Tile +regions are mapped back to the normalized source raster and equal overlapping +regions are deduplicated deterministically. Decoding, PDF page rendering, +candidate iteration, and inference boundaries honor cancellation; the document +deadline is 600 seconds. The in-process runtime uses one inter-op and one +intra-op thread. + +`compass-core` prepares each rich document once. Complete artifacts are cached +atomically by source SHA-256 plus document schema, normalizer, renderer, OCR +policy, profile manifest, preprocessing version, and language hints. Corrupt +or incompatible entries fail explicitly; partial OCR is never finalized as a +complete cache entry. The same prepared artifact feeds structural publication +and gap-free Unicode-safe semantic slices. The semantic layer does not load an +OCR engine or maintain a second document cache. + +The PP-OCRv6 runtime is compiled with Compass. Users install no Python, +Tesseract, office suite, Poppler, Java, or system ONNX package. Model weights +are deliberately separate: `compass models install pp-ocrv6-small` is the only +download path and validates a fixed allowlisted HTTPS source, declared size, +SHA-256, and atomic verified marker. Inspection and extraction never download +or prompt. + +The production engine identity is OAR-OCR 0.9.2 with its in-process ONNX +backend. The model source is the immutable GreatV/OAR-OCR `v0.7.0` GitHub +release: the small profile is 31,114,837 bytes and the medium profile is +138,662,763 bytes across detector, recognizer, and shared dictionary. Each +artifact has a compiled SHA-256. Hayro 0.7.1 is the sole PDF renderer and is +fingerprinted as `hayro/0.7.1@300dpi`. Neither boundary invokes a system tool. + +The checked installed-model smoke gate currently establishes only clean, +synthetic English recognition and runtime availability; it recorded 0 CER for +`COMPASS OCR 2026` on the development aarch64 macOS host. This is not a broad +multilingual, photographed-document, handwriting, or comparative quality +claim. Release promotion for additional input classes and architectures must +come from the corpus and gates in the qualification guide. ## Related pages - [Document format reference](../reference/document-formats.md) - [Language architecture](language-architecture.md) - [Extraction pipeline](../implementation/extraction-pipeline.md) +- [Document OCR qualification](../implementation/document-ocr-qualification.md) - [Graph model](../concepts/graph-model.md) diff --git a/docs/design/security-and-privacy.md b/docs/design/security-and-privacy.md index e2cceff7..35193ba0 100644 --- a/docs/design/security-and-privacy.md +++ b/docs/design/security-and-privacy.md @@ -283,6 +283,33 @@ Safe patterns include: When adding a subprocess, test timeout, nonzero exit, oversized output, malformed UTF-8, and missing executable behavior. +## Local document and OCR boundary + +PDF and Office bytes, ZIP members, XML, raster dimensions, OCR observations, +downloaded model bytes, and document-cache JSON are untrusted. Compass checks +raw and expanded sizes before allocation and keeps raw file/cache reads bounded +at the stream even if a file grows after its metadata check. It rejects archive +traversal, unsafe relationships, unknown cache fields, and incoherent OCR +origin, profile, completeness, or geometry. Formulas/macros/OLE stay inert, +document URLs are never followed, and every OCR request/result is validated +against the normalized raster. + +OCR extraction is network-disabled by construction. `compass models install` +is the sole download command; it uses fixed HTTPS release hosts, at most three +validated redirects, immutable `v0.7.0` artifact names, exact sizes and +SHA-256, temporary same-directory files, and atomic publication with parent +directory synchronization. A bounded per-profile lock serializes concurrent +installers. Symlinked model artifacts, verified markers, and install locks are +rejected. Extraction, inspection, watch, and history verify local artifacts and +fail explicitly when a selected profile is absent. They never downgrade to +native-only processing after OCR was requested. + +The ONNX runtime and pure-Rust PDF renderer are linked into Compass. Users do +not configure an executable or install Python, Tesseract, an office suite, +Poppler, Java, or a system ONNX library. Native text remains authoritative; +model output is separately identified derived evidence and cannot execute or +replace source content. + ## Threat-informed operating checklist ### Fully local code graph diff --git a/docs/implementation/document-ocr-qualification.md b/docs/implementation/document-ocr-qualification.md new file mode 100644 index 00000000..39dae6bf --- /dev/null +++ b/docs/implementation/document-ocr-qualification.md @@ -0,0 +1,52 @@ +--- +meta: + contentType: Guide + title: Qualifying document OCR + navLabel: Document OCR Qualification + category: Implementation + overview: Reproduce the offline document and optional installed-model OCR gates. +--- + +# Qualifying document OCR + +The default gate is offline and model-free: + +```bash +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-document-ocr \ + scripts/qualify_document_ocr_v1.sh --fixtures-only +``` + +It checks the versioned OCR and document contracts, archive and raster limits, +native DOCX/PPTX/XLSX order, selective OCR policy, response rejection, +provenance, EXIF orientation, alpha compositing, deterministic resize/tiling +and overlap reassembly, cancellation, partial completeness, prepared-document +caching, Unicode-safe semantic slicing, CLI JSON, and offline model +verification. It must pass even when the Compass model cache is empty. + +Install and verify a profile only for the opt-in native-runtime gate: + +```bash +compass models install pp-ocrv6-small +compass models verify pp-ocrv6-small +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-document-ocr \ + scripts/qualify_document_ocr_v1.sh --with-installed-model +``` + +Installation is the only network step. The gate itself performs no download. +It verifies the immutable OAR-OCR 0.9.2 / `v0.7.0` profile, executes inference, +validates geometry and profile identity, and enforces at most 5% CER on the +deterministic clean-English smoke raster. The smoke raster is CC0 synthetic. + +The v1 manifest intentionally does not claim photographed pages, handwriting, +table reconstruction, or any script/platform absent from measured fixtures. +A release claim for those classes requires checked ground truth, CER/WER, +region IoU, reading-order and native-duplicate metrics, hostile-input results, +and equivalent recognized content on every supported CPU architecture. Vendor +benchmark figures are not Compass qualification evidence. + +The manifest machine-reports the remaining blocking, unmeasured release gates: +the pinned Tesseract degraded-English comparison, PP-OCRv6 medium and `ocrs` +candidate comparison, per-script quality/geometry, x86_64/aarch64 equivalence, +and hostile-corpus RSS/timeout evidence. The implementation is usable with the +narrow clean-English gate, but Plan 022 must not be marked `DONE` until those +external qualification runs and its prerequisite plan statuses are reviewed. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 4e45e72b..234cfccd 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -96,6 +96,9 @@ compass extract [PATH] [--backend NAME] [--model MODEL] [--mode deep] + [--ocr off|auto|always] + [--ocr-profile NAME] + [--ocr-language BCP47] [--token-budget N] [--max-concurrency N] [--max-workers N] @@ -121,6 +124,34 @@ the scanned file inventory. Program IR is opt-in with `--program`; `--program-artifact` also enables it. `--no-program` is retained for callers that already use the structural-only spelling. +OCR is off by default. `auto` processes scanned/low-text PDF pages and eligible +embedded Office images; `always` processes every bounded visual candidate. +Both are local and require an explicitly installed verified profile. Extraction +never downloads a model. `--ocr-language` is repeatable, and +`--allow-partial` also authorizes visibly incomplete OCR coverage. + +### `document` and `models` + +```text +compass document inspect FILE + [--format text|json] + [--ocr off|auto|always] + [--ocr-profile NAME] + [--ocr-language BCP47] + [--allow-partial] + +compass models list [--format text|json] +compass models install pp-ocrv6-small|pp-ocrv6-medium +compass models verify pp-ocrv6-small|pp-ocrv6-medium +``` + +`document inspect` is read-only and does not publish a graph. JSON uses +`compass.document.inspect/1`; text marks OCR-derived evidence visibly. Native +PDF, DOCX, PPTX, and XLSX processing requires no additional installation. +`models install` is the only command here that uses the network. It downloads +only pinned artifacts from the Compass allowlist, validates size and SHA-256, +and publishes an atomic verification marker. `list` and `verify` are offline. + `update`, `extract`, and watch rebuilds may succeed with a warning that Compass published a partial graph. The warning reports exact omitted node, omitted edge, and identity-collision counts. The retained `graph.json` remains strictly diff --git a/docs/reference/document-formats.md b/docs/reference/document-formats.md index 09aa731b..ce96e493 100644 --- a/docs/reference/document-formats.md +++ b/docs/reference/document-formats.md @@ -92,15 +92,41 @@ lexically against the validated source/base URL, never fetched. | Format | Discovery classification | Structural extractor in this release | | --- | --- | --- | | HTML / HTM | document | structural Tree-sitter adapter and shared ingestion renderer | -| DOCX | document/media | media conversion surface; no native block graph | -| PPTX | not a general local document adapter | not yet | +| PDF | document | native page/text blocks; optional page OCR | +| DOCX | document | ordered paragraphs, headings, lists, tables, notes, links, embedded-image OCR | +| PPTX | document | relationship-ordered slides, shapes, tables, notes, links, embedded-image OCR | | RTF | not a general local document adapter | not yet | -| XLSX | document/media | media conversion surface; no native block graph | +| XLSX | document | sparse typed sheets/rows/cells, formulas as inert metadata, embedded-image OCR | | TXT / RST | document | generic/document fallback only | -This distinction keeps product claims honest: future office and rich-text work -must add bounded parsing, exact or explicitly normalized locators, security -tests, and cache/version contracts before it becomes graph evidence. +PDF and Office adapters emit `compass.document/1`. Locators are typed as PDF +page/item, OOXML package part/path, slide/shape, sheet/row/column, or OCR owner +plus pixel polygon. Structural graph nodes preserve the serialized locator and +document origin. Unknown schema or normalizer versions fail rather than being +flattened through a compatibility adapter. + +### OCR commands and defaults + +```text +compass document inspect FILE --ocr off|auto|always --format text|json +compass extract PATH --ocr off|auto|always +compass models list|install|verify +``` + +OCR defaults to `off`. Native extraction needs no model. `auto` and `always` +require a verified local profile and never download implicitly. Use +`--ocr-language TAG` repeatedly for bounded language hints and +`--allow-partial` only when incomplete visual coverage is acceptable. JSON +inspection uses `compass.document.inspect/1` and includes the policy, artifact, +limits, diagnostics, visual coverage, and exact OCR profile identity. + +Document and cache files are capped while streaming, so size checks still hold +if an input changes during a read. PDF rasterization reserves aggregate pixels +before rendering each page, and tiled recognition checks the document deadline +before and after each inference unit. One native inference call cannot be +preempted midway; its result is discarded as a timeout if the deadline has +elapsed. Concurrent installation of the same model profile waits on a bounded +lock, and verification rejects symlinked artifacts or markers. ## Related contracts diff --git a/scripts/qualify_document_ocr_v1.sh b/scripts/qualify_document_ocr_v1.sh new file mode 100755 index 00000000..949729de --- /dev/null +++ b/scripts/qualify_document_ocr_v1.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu + +mode=fixtures +case "${1:-}" in + ""|--fixtures-only) mode=fixtures ;; + --with-installed-model) mode=model ;; + *) + echo "usage: scripts/qualify_document_ocr_v1.sh [--fixtures-only|--with-installed-model]" >&2 + exit 2 + ;; +esac + +if [ -z "${CARGO_TARGET_DIR:-}" ]; then + echo "CARGO_TARGET_DIR must name this checkout's directory under /Volumes/Workspace/crabbuild-target" >&2 + exit 2 +fi +case "$CARGO_TARGET_DIR" in + /Volumes/Workspace/crabbuild-target/*) ;; + *) + echo "CARGO_TARGET_DIR must be below /Volumes/Workspace/crabbuild-target" >&2 + exit 2 + ;; +esac + +cargo test -p compass-ocr -p compass-media --lib --locked +cargo test -p compass-core document --lib --locked +cargo test -p compass-semantic --test orchestration_coverage --locked +cargo test -p compass-cli --test document_cli --locked +python3 scripts/validate_document_ocr_manifest.py + +if [ "$mode" = model ]; then + cargo test -p compass-ocr --test model_acceptance --locked -- --ignored --nocapture +fi + +printf '{"schema":"compass.document-ocr.qualification-result/1","mode":"%s","status":"passed","network_during_gate":false}\n' "$mode" diff --git a/scripts/validate_document_ocr_manifest.py b/scripts/validate_document_ocr_manifest.py new file mode 100644 index 00000000..c4ebb4d8 --- /dev/null +++ b/scripts/validate_document_ocr_manifest.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Validate the bounded, redistributable document-OCR qualification manifest.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = ROOT / "tests/qualification/document-ocr/v1/manifest.json" + + +def fail(message: str) -> None: + raise SystemExit(f"document OCR manifest error: {message}") + + +def main() -> None: + value = json.loads(MANIFEST.read_text(encoding="utf-8")) + if value.get("schema") != "compass.document-ocr.qualification/1": + fail("unsupported schema") + if value.get("license") != "CC0-1.0 synthetic fixtures": + fail("fixture license is not the reviewed CC0 identity") + if value.get("offline_default") is not True or value.get("network_during_tests") is not False: + fail("offline/network policy changed") + required_formats = {"pdf", "docx", "pptx", "xlsx", "png"} + if set(value.get("formats", [])) != required_formats: + fail("format set changed") + if value.get("engine") != "oar-ocr/0.9.2": + fail("engine identity changed") + if value.get("model_revision") != "GreatV/oar-ocr@v0.7.0": + fail("model revision is not immutable v0.7.0") + if value.get("renderer") != "hayro/0.7.1@300dpi": + fail("renderer identity changed") + preprocessing = value.get("preprocessing") + if preprocessing != { + "version": 2, + "exif_orientation": True, + "alpha_background": "white", + "resize_filter": "triangle", + "engine_max_side": 2048, + "tile_overlap": 128, + "engine_threads": 1, + "document_timeout_seconds": 600, + }: + fail("preprocessing identity changed") + gate = value.get("installed_model_gate") + if not isinstance(gate, dict) or gate.get("maximum_cer_bps") != 500: + fail("clean-English CER gate changed") + blockers = value.get("blocking_unmeasured_release_gates") + if not isinstance(blockers, list) or len(blockers) != 5 or not all( + isinstance(item, str) and item for item in blockers + ): + fail("unmeasured release blockers are not explicit") + print(json.dumps({ + "schema": value["schema"], + "manifest": str(MANIFEST.relative_to(ROOT)), + "offline": True, + "engine": value["engine"], + "model_revision": value["model_revision"], + "renderer": value["renderer"], + "clean_english_maximum_cer_bps": gate["maximum_cer_bps"], + "preprocessing_version": preprocessing["version"], + "blocking_unmeasured_release_gates": blockers, + }, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/tests/qualification/document-ocr/v1/README.md b/tests/qualification/document-ocr/v1/README.md new file mode 100644 index 00000000..fbe82bf1 --- /dev/null +++ b/tests/qualification/document-ocr/v1/README.md @@ -0,0 +1,32 @@ +# Document OCR v1 qualification corpus + +This directory is the reviewable manifest for the license-safe document/OCR +gate. Deterministic unit and CLI fixtures generate their minimal PDF/OOXML/image +bytes in memory so ZIP metadata and raster bytes remain reproducible and no +opaque office document is committed. + +The fixture-only gate covers native DOCX order, sparse typed XLSX cells, +relationship-ordered PPTX parsing, bounded PDF raster dimensions, archive/XML +rejection, selective `auto` policy, provenance-preserving fake OCR, partial +coverage, EXIF orientation, deterministic tiling/overlap reassembly, +cancellation, artifact cache replay/corruption, Unicode-safe semantic slicing, +CLI schemas, missing-model diagnostics, and offline model +listing/verification. + +The optional installed-model gate validates the exact pinned OAR/PP-OCR runtime +without downloading. It skips only when `pp-ocrv6-small` is not already +verified in the Compass model cache, and enforces the 5% CER ceiling for the +checked clean-English synthetic raster. + +Run: + +```bash +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-document-ocr \ + scripts/qualify_document_ocr_v1.sh --fixtures-only +``` + +Fixture provenance and expected policies are machine-readable in +`manifest.json`. The gate makes no broader multilingual, degraded-document, +handwriting, or cross-platform quality claim from its synthetic smoke fixture. +The same manifest lists every unmeasured release blocker so a narrow passing +smoke test cannot be mistaken for full Plan-022 qualification. diff --git a/tests/qualification/document-ocr/v1/manifest.json b/tests/qualification/document-ocr/v1/manifest.json new file mode 100644 index 00000000..4b9b2811 --- /dev/null +++ b/tests/qualification/document-ocr/v1/manifest.json @@ -0,0 +1,43 @@ +{ + "schema": "compass.document-ocr.qualification/1", + "license": "CC0-1.0 synthetic fixtures", + "generation": "deterministic in-memory Rust test fixtures", + "formats": ["pdf", "docx", "pptx", "xlsx", "png"], + "policies": ["off", "auto", "always", "allow_partial"], + "required_contracts": [ + "compass.document/1", + "compass.document.inspect/1", + "compass.ocr/1", + "compass.ocr.protocol/1" + ], + "offline_default": true, + "network_during_tests": false, + "model_profile": "pp-ocrv6-small", + "engine": "oar-ocr/0.9.2", + "model_revision": "GreatV/oar-ocr@v0.7.0", + "renderer": "hayro/0.7.1@300dpi", + "preprocessing": { + "version": 2, + "exif_orientation": true, + "alpha_background": "white", + "resize_filter": "triangle", + "engine_max_side": 2048, + "tile_overlap": 128, + "engine_threads": 1, + "document_timeout_seconds": 600 + }, + "installed_model_gate": { + "fixture": "synthetic-clean-english-5x7-v1", + "expected": "COMPASS OCR 2026", + "maximum_cer_bps": 500, + "license": "CC0-1.0 synthetic fixture" + }, + "quality_claim": "Clean synthetic English only; no multilingual, degraded, handwriting, or broad accuracy claim", + "blocking_unmeasured_release_gates": [ + "degraded-English comparison against pinned Tesseract 5 tessdata_best", + "PP-OCRv6 medium and ocrs candidate comparison", + "per-script CER/WER and region-IoU corpus", + "x86_64 versus aarch64 semantic equivalence", + "hostile-corpus peak RSS and timeout termination" + ] +}