diff --git a/bun.lock b/bun.lock index f099b266..abebbbcd 100644 --- a/bun.lock +++ b/bun.lock @@ -6,27 +6,19 @@ "name": "@workglow/sec", "dependencies": { "@huggingface/transformers": "^4.3.0", - "@huggingface/transformers-structured-output": "^4.3.0", - "@modelcontextprotocol/sdk": "^1.30.0", "@workglow/cli": "0.6.7", "cheerio": "^1.2.0", - "cheerio-json-mapper": "^1.0.4", "commander": "^15.0.0", - "compromise": "^14.17.0", "csv-parse": "^7.0.2", "fast-xml-parser": "^5.11.1", "html-entities": "^2.6.0", - "pdf2json": "^4.1.0", "pg": "^8.23.0", "typebox": "1.3.34", - "words-to-numbers": "^1.5.1", "workglow": "0.6.7", - "xml2js": "^0.6.2", }, "devDependencies": { "@types/bun": "1.4.2", "@types/pg": "^8.23.1", - "@types/xml2js": "^0.4.14", "bunset": "1.1.2", "concurrently": "^10.0.5", "oxfmt": "0.68.0", @@ -408,8 +400,6 @@ "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], - "@types/xml2js": ["@types/xml2js@0.4.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ=="], - "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], @@ -538,8 +528,6 @@ "auto-bind": ["auto-bind@5.0.1", "", {}, "sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg=="], - "babel-runtime": ["babel-runtime@6.26.0", "", { "dependencies": { "core-js": "^2.4.0", "regenerator-runtime": "^0.11.0" } }, "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g=="], - "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="], @@ -566,8 +554,6 @@ "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], - "cheerio-json-mapper": ["cheerio-json-mapper@1.0.4", "", { "dependencies": { "cheerio": "^1.0.0" } }, "sha512-EZANYyrHLy4hf2fRnuQwIudcmrAZ+6iibn+DqAeeKdFikUw4u69j8NEH41sBgwEuHeOtVx4tnh8cHcBvTqzApg=="], - "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], "chmodrp": ["chmodrp@1.0.2", "", {}, "sha512-TdngOlFV1FLTzU0o1w8MB6/BFywhtLC0SzRTGJU7T9lmdjlCWeMRt1iVo0Ki+ldwNk0BqNiKoc8xpLZEQ8mY1w=="], @@ -586,8 +572,6 @@ "cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="], - "clj-fuzzy": ["clj-fuzzy@0.3.3", "", {}, "sha512-9cyh9A8+OphDZeKIG21MgyDHWDkWxTvagwvFLVjtdi6eToFENF7iDLlKwhHrnBQRSQwprKNhazG053nE/UgwfQ=="], - "cmake-js": ["cmake-js@8.0.0", "", { "dependencies": { "debug": "^4.4.3", "fs-extra": "^11.3.3", "node-api-headers": "^1.8.0", "rc": "1.2.8", "semver": "^7.7.3", "tar": "^7.5.6", "url-join": "^4.0.1", "which": "^6.0.0", "yargs": "^17.7.2" }, "bin": { "cmake-js": "bin/cmake-js" } }, "sha512-YbUP88RDwCvoQkZhRtGURYm9RIpWdtvZuhT87fKNoLjk8kIFIFeARpKfuZQGdwfH99GZpUmqSfcDrK62X7lTgg=="], "code-excerpt": ["code-excerpt@4.0.0", "", { "dependencies": { "convert-to-spaces": "^2.0.1" } }, "sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA=="], @@ -598,8 +582,6 @@ "commander": ["commander@15.0.0", "", {}, "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg=="], - "compromise": ["compromise@14.17.0", "", { "dependencies": { "efrt": "2.7.0", "grad-school": "0.0.5", "suffix-thumb": "5.0.3" } }, "sha512-zw9iEcts/8tMDASNopMQEs3Pclkx2Xk7XCltAb/oV1LEZcCQpDaalAtFmvuxYo+wnVDsW3H3oT16mw9qVHpkqA=="], - "concurrently": ["concurrently@10.0.5", "", { "dependencies": { "chalk": "5.6.2", "rxjs": "7.8.2", "shell-quote": "1.9.0", "supports-color": "10.2.2", "tree-kill": "1.2.2", "yargs": "18.0.0" }, "bin": { "conc": "dist/bin/index.js", "concurrently": "dist/bin/index.js" } }, "sha512-JaP/CoftUrCcAFW/g//RbgEGwlelnEae6cfBLgH6ZdO6s8jPkn6p9SB9u6pdVxYXoiSnFqseOlHfrEfF82TVOg=="], "content-disposition": ["content-disposition@1.1.0", "", {}, "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g=="], @@ -612,8 +594,6 @@ "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], - "core-js": ["core-js@2.6.12", "", {}, "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ=="], - "cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="], "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], @@ -654,8 +634,6 @@ "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], - "efrt": ["efrt@2.7.0", "", {}, "sha512-/RInbCy1d4P6Zdfa+TMVsf/ufZVotat5hCw3QXmWtjU+3pFEOvOQ7ibo3aIxyCJw2leIeAMjmPj+1SLJiCpdrQ=="], - "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], @@ -764,8 +742,6 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "grad-school": ["grad-school@0.0.5", "", {}, "sha512-rXunEHF9M9EkMydTBux7+IryYXEZinRk6g8OBOGDBzo/qWJjhTxy86i5q7lQYpCLHN8Sqv1XX3OIOc7ka2gtvQ=="], - "guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="], "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], @@ -784,7 +760,7 @@ "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], - "iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], "idn-hostname": ["idn-hostname@15.1.19", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-pRGRbFpQWCptOBWw9Gn5DXtPdern5/XtdkcXdoEavjd+sh72jNm2eVI1eeysp64UisXPfaqoQrnLUKpSN4hJWw=="], @@ -816,9 +792,7 @@ "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - - "its-set": ["its-set@1.2.3", "", { "dependencies": { "babel-runtime": "6.x.x", "lodash.get": "^4.4.2" } }, "sha512-UQc+xLLn+0a8KKRXRj3OS2kERK8G7zcayPpPULqZnPwuJ1hGWEO8+j0T5eycu7DKXYjezw3pyF8oV1fJkJxV5w=="], + "isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], "jose": ["jose@6.2.12", "", {}, "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw=="], @@ -866,8 +840,6 @@ "lodash.debounce": ["lodash.debounce@4.0.8", "", {}, "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="], - "lodash.get": ["lodash.get@4.4.2", "", {}, "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ=="], - "log-symbols": ["log-symbols@7.0.1", "", { "dependencies": { "is-unicode-supported": "^2.0.0", "yoctocolors": "^2.1.1" } }, "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg=="], "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], @@ -970,8 +942,6 @@ "path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], - "pdf2json": ["pdf2json@4.1.0", "", { "bin": { "pdf2json": "bin/pdf2json.js" } }, "sha512-Nmmw7uxYUspUEBMs3Dx2/wcnAJxPGxlSYtNR/Owp/iOLKk0qK4hzUThnGSQSQshHvoNR3N4LEVKo0mHx2i7a0g=="], - "pg": ["pg@8.23.0", "", { "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.16.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg=="], "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="], @@ -1034,8 +1004,6 @@ "react-reconciler": ["react-reconciler@0.33.0", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.0" } }, "sha512-KetWRytFv1epdpJc3J4G75I4WrplZE5jOL7Yq0p34+OVOKF4Se7WrdIdVC45XsSSmUTlht2FM/fM1FZb1mfQeA=="], - "regenerator-runtime": ["regenerator-runtime@0.11.1", "", {}, "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="], - "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], @@ -1054,8 +1022,6 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - "sax": ["sax@1.6.1", "", {}, "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q=="], - "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], @@ -1124,8 +1090,6 @@ "strnum": ["strnum@2.4.2", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw=="], - "suffix-thumb": ["suffix-thumb@5.0.3", "", {}, "sha512-d77avV91FwJkDA0juRQ19XjE1lE1cNCVIWS0ZRicXqdMN28yjO5LltzEkZBNAWS7qHpn37c1fU4PkIJ/rjJQEA=="], - "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], "tagged-tag": ["tagged-tag@1.0.0", "", {}, "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng=="], @@ -1192,14 +1156,12 @@ "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], "widest-line": ["widest-line@6.0.0", "", { "dependencies": { "string-width": "^8.1.0" } }, "sha512-U89AsyEeAsyoF0zVJBkG9zBgekjgjK7yk9sje3F4IQpXBJ10TF6ByLlIfjMhcmHMJgHZI4KHt4rdNfktzxIAMA=="], - "words-to-numbers": ["words-to-numbers@1.5.1", "", { "dependencies": { "babel-runtime": "6.x.x", "clj-fuzzy": "^0.3.2", "its-set": "^1.1.5" } }, "sha512-uvz7zSCKmmA7o5f5zp4Z5l24RQhy6HSNu10URhNxQWv1I82RsFaZX3qD07RLFUMJsCV38oAuaca13AvhO+9yGw=="], - "workglow": ["workglow@0.6.7", "", { "dependencies": { "@workglow/a2a": "0.6.7", "@workglow/ai": "0.6.7", "@workglow/anthropic": "0.6.7", "@workglow/browser-control": "0.6.7", "@workglow/chrome-ai": "0.6.7", "@workglow/deepseek": "0.6.7", "@workglow/duckdb": "0.6.7", "@workglow/google-gemini": "0.6.7", "@workglow/huggingface-inference": "0.6.7", "@workglow/huggingface-transformers": "0.6.7", "@workglow/javascript": "0.6.7", "@workglow/job-queue": "0.6.7", "@workglow/knowledge-base": "0.6.7", "@workglow/mcp": "0.6.7", "@workglow/node-llama-cpp": "0.6.7", "@workglow/ollama": "0.6.7", "@workglow/openai": "0.6.7", "@workglow/openrouter": "0.6.7", "@workglow/postgres": "0.6.7", "@workglow/sqlite": "0.6.7", "@workglow/storage": "0.6.7", "@workglow/task-graph": "0.6.7", "@workglow/tasks": "0.6.7", "@workglow/tf-mediapipe": "0.6.7", "@workglow/triggers": "0.6.7", "@workglow/util": "0.6.7", "@workglow/xai": "0.6.7", "tslog": "^5.2.0" } }, "sha512-lc3ciau6XLpge9tLpJHDmCeolN6X2x3Yo0mJNyF3ylfHIz2il0NgTAso/s5z5vRm14atGMy4Owoot851lNTwjQ=="], "wrap-ansi": ["wrap-ansi@10.0.1", "", { "dependencies": { "ansi-styles": "^6.2.3", "string-width": "^8.2.0" } }, "sha512-M0N4xzyzosiIok3svYlEo1sdLZts/8FPgYH/GPC3wvlmPoRvnoManGMrE54waYj3tISA8w6lsdesfVv67qSr8Q=="], @@ -1210,10 +1172,6 @@ "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], - "xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="], - - "xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], - "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], @@ -1238,15 +1196,15 @@ "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], + "body-parser/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], + "cli-truncate/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "cliui/wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], - "cmake-js/which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="], - "cmake-js/yargs": ["yargs@17.7.3", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g=="], - "encoding-sniffer/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + "cross-spawn/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "gcp-metadata/google-logging-utils": ["google-logging-utils@1.2.0", "", {}, "sha512-WE9av4wKDZgRjBwgVUabocx8T6/7o3Ca1Fat46FXDhXVAFibzNadedcOXrdgd1Kzmk8tsk/9ZH89Wyf/SqeZ3A=="], @@ -1270,8 +1228,6 @@ "node-llama-cpp/slice-ansi": ["slice-ansi@8.0.0", "", { "dependencies": { "ansi-styles": "^6.2.3", "is-fullwidth-code-point": "^5.1.0" } }, "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg=="], - "node-llama-cpp/which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="], - "node-llama-cpp/yargs": ["yargs@17.7.3", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g=="], "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.31.0-dev.20260911-2a43ec07e", "", {}, "sha512-gBuF6U32YErKIAt+yD7DeGBpjRxhJ1uJwako6ygjokSZULDU6Pd+KWComX8Tumk1hYVHRxjPj7mdMnD0ryAPOw=="], @@ -1286,29 +1242,27 @@ "proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], + "raw-body/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], + "serialize-error/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], "stdout-update/ansi-escapes": ["ansi-escapes@6.2.1", "", {}, "sha512-4nJ3yixlEthEJ9Rk4vPcdBRkZvQZlYyu8j4/Mqz5sgIkddmEnH2Yj2ZrnP9S3tQOvSNRUIgVNF/1yPpRAGNRig=="], "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], - "whatwg-encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], - "widest-line/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], - "cmake-js/which/isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], - "cmake-js/yargs/cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], "cmake-js/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "cmake-js/yargs/yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], - "ipull/pretty-ms/parse-ms": ["parse-ms@3.0.0", "", {}, "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw=="], + "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "node-llama-cpp/which/isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], + "ipull/pretty-ms/parse-ms": ["parse-ms@3.0.0", "", {}, "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw=="], "node-llama-cpp/yargs/cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], diff --git a/package.json b/package.json index f557e636..aed51143 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "homepage": "https://workglow.dev", "description": "A worked example of the Workglow libraries: SEC EDGAR and Form ADV ingestion, filing-to-markdown conversion, and retrieval-grounded question answering over what filings say.", "scripts": { - "release-checks": "bun run format && bun run lint && bun run typecheck && bun run build && bun run prepack-check", + "release-checks": "bun run format-check && bun run lint && bun run typecheck && bun run build && bun run test && bun run prepack-check", "release": "bun run release-checks && bunset --auto --push --commit --tag", "prepack-check": "bun ./scripts/checkPackedContents.ts", "dev": "concurrently -c 'auto' -n 'sec:' 'bun:dev-*'", @@ -52,27 +52,19 @@ }, "dependencies": { "@huggingface/transformers": "^4.3.0", - "@huggingface/transformers-structured-output": "^4.3.0", - "@modelcontextprotocol/sdk": "^1.30.0", "@workglow/cli": "0.6.7", "cheerio": "^1.2.0", - "cheerio-json-mapper": "^1.0.4", "commander": "^15.0.0", - "compromise": "^14.17.0", "csv-parse": "^7.0.2", "fast-xml-parser": "^5.11.1", "html-entities": "^2.6.0", - "pdf2json": "^4.1.0", "pg": "^8.23.0", "typebox": "1.3.34", - "words-to-numbers": "^1.5.1", - "workglow": "0.6.7", - "xml2js": "^0.6.2" + "workglow": "0.6.7" }, "devDependencies": { "@types/bun": "1.4.2", "@types/pg": "^8.23.1", - "@types/xml2js": "^0.4.14", "bunset": "1.1.2", "concurrently": "^10.0.5", "oxfmt": "0.68.0", diff --git a/src/config/kbIndexedStamp.sqlite.test.ts b/src/config/kbIndexedStamp.sqlite.test.ts new file mode 100644 index 00000000..819ac810 --- /dev/null +++ b/src/config/kbIndexedStamp.sqlite.test.ts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { DocumentNode } from "workglow"; +import { Document, globalServiceRegistry, NodeKind } from "workglow"; +import { getSecKnowledgeBase, resetSecKnowledgeBaseForTesting } from "../kb/secKnowledgeBase"; +import { + FILING_DOCUMENT_REPOSITORY_TOKEN, + type FilingDocument, +} from "../storage/document/FilingDocumentSchema"; +import { kbDocIdFor } from "../task/kb/selectDocumentsToIndex"; +import { selectDocumentsToIndex } from "../task/kb/selectDocumentsToIndex"; +import { getDb } from "../util/db"; +import { syncKbIndexedStamp } from "./kbIndexedStamp"; +import { withSqliteDb } from "./testing/withSqliteDb"; + +const accession = (index: number) => `0000320193-26-${String(index).padStart(6, "0")}`; + +const doc = (index: number, over: Partial = {}): FilingDocument => ({ + cik: 320193, + accession_number: accession(index), + doc_file: "primary.htm", + doc_type: "10-K", + description: null, + sequence: 1, + is_primary: true, + form: "10-K", + filing_date: "2026-03-01", + title: `Filing ${index}`, + section_count: 1, + char_count: 100, + converter_version: "1", + converted_at: "2026-01-01T00:00:00.000Z", + kb_indexed_at: null, + ...over, +}); + +/** + * The stamp is a cache of "is there a `kb_document` row for this", and the three + * properties below are what make it safe to read: it never claims more than the + * anti-join, `db setup` never leaves it empty, and the steady state stops + * touching the table. + */ +describe("kb_indexed_at stamp (sqlite)", () => { + withSqliteDb("kb_indexed_stamp", [FILING_DOCUMENT_REPOSITORY_TOKEN]); + + beforeEach(async () => { + await resetSecKnowledgeBaseForTesting(); + delete process.env.SEC_EMBEDDING_MODEL; + }); + afterEach(async () => { + await resetSecKnowledgeBaseForTesting(); + }); + + async function seed(count: number) { + const repo = globalServiceRegistry.get(FILING_DOCUMENT_REPOSITORY_TOKEN); + for (let i = 1; i <= count; i += 1) await repo.put(doc(i) as never); + } + + async function putInKb(index: number) { + const kb = await getSecKnowledgeBase(); + const title = `Filing ${index}`; + const root = { kind: NodeKind.DOCUMENT, title, children: [] } as unknown as DocumentNode; + await kb.upsertDocument( + new Document(root, { title } as never, [], kbDocIdFor(accession(index), "primary.htm")) + ); + } + + const stampOf = (index: number): string | null => + ( + getDb() + .prepare("SELECT `kb_indexed_at` AS s FROM `filing_document` WHERE `accession_number` = ?") + .get(accession(index)) as { s: string | null } | undefined + )?.s ?? null; + + it("backfills a database whose documents were indexed before the column existed", async () => { + // The state every existing deployment is in: rows in the knowledge base, + // no stamp. Left un-backfilled the partial index would cover the whole + // table and the selection would walk it, which is the plan this replaces. + await seed(3); + await putInKb(1); + await putInKb(2); + + syncKbIndexedStamp(getDb()); + + expect(stampOf(1)).not.toBeNull(); + expect(stampOf(2)).not.toBeNull(); + expect(stampOf(3)).toBeNull(); + }); + + it("is idempotent, and does not restamp a row on a second run", async () => { + await seed(1); + await putInKb(1); + + syncKbIndexedStamp(getDb()); + const first = stampOf(1); + syncKbIndexedStamp(getDb()); + + expect(stampOf(1)).toBe(first); + }); + + it("creates the partial index, and the steady state reads it instead of the table", async () => { + await seed(3); + for (const i of [1, 2, 3]) await putInKb(i); + syncKbIndexedStamp(getDb()); + + const plan = getDb() + .prepare( + `EXPLAIN QUERY PLAN + SELECT d.* FROM \`filing_document\` d + WHERE d.\`section_count\` > 0 AND d.\`kb_indexed_at\` IS NULL + ORDER BY d.\`filing_date\` DESC, d.\`accession_number\` DESC` + ) + .all() as Array<{ detail?: string }>; + const detail = plan.map((row) => row.detail ?? "").join(" | "); + + expect(detail).toContain("filing_document_kb_unindexed"); + // The point of the partial index: no table scan, and no temp B-tree to + // sort what a scan would have produced. + expect(detail).not.toContain("SCAN d\n"); + expect(detail).not.toContain("TEMP B-TREE"); + }); + + it("never widens the selection: an unstamped document already in the kb is still skipped", async () => { + // The stamp is a narrowing of the anti-join, never a replacement for it. + // If a stamp goes missing — a crash between the upsert and the stamp, a + // database restored from before the backfill — the anti-join still has to + // keep that document out, because indexing it twice spends embedding calls. + await seed(2); + await putInKb(1); + // Deliberately NOT stamped. + expect(stampOf(1)).toBeNull(); + + const picked = await selectDocumentsToIndex({ limit: 10 }); + + expect(picked.map((d) => d.accession_number)).toEqual([accession(2)]); + }); +}); diff --git a/src/config/kbIndexedStamp.ts b/src/config/kbIndexedStamp.ts new file mode 100644 index 00000000..15a222a6 --- /dev/null +++ b/src/config/kbIndexedStamp.ts @@ -0,0 +1,88 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Sqlite } from "workglow"; +import { KB_DOCUMENT_TABLE } from "../kb/secKbTables"; + +/** The partial index the steady-state selection walks. */ +const PARTIAL_INDEX = "filing_document_kb_unindexed"; + +/** + * Brings `filing_document.kb_indexed_at` up to date and indexes the nulls. + * + * Two statements that have to travel together, which is why they are one + * function rather than a registry entry. The column is a cache of "is there a + * `kb_document` row for this", and a cache that is added empty is worse than no + * cache: every row reads as unindexed, so the partial index below covers the + * whole table and the planner walks it in `filing_date` order, issuing a random + * probe per row — the access path that measured 11x slower than the plain scan + * it replaced. Backfilling in the same pass that creates the index is what + * keeps that state from existing. + * + * Both statements are idempotent and cheap on a database already in step: the + * `UPDATE` matches nothing once every indexed document is stamped, and + * `CREATE INDEX IF NOT EXISTS` is a catalog read. + * + * Callers must have applied the schema's columns first — the column is added + * generically from the registry, not here — so this no-ops when it is absent + * rather than assuming an order it cannot see. + */ +export function syncKbIndexedStamp(db: Sqlite.Database): void { + if (!hasColumn(db, "filing_document", "kb_indexed_at")) return; + if (!tableExists(db, KB_DOCUMENT_TABLE)) { + // No knowledge base yet, so nothing is indexed and every row's null is + // already the truth. The index still pays off on the first `ask`. + createPartialIndex(db); + return; + } + + // `converted_at` rather than a timestamp taken here: the row's own history is + // a better answer than "whenever someone ran db setup", and it keeps the + // backfill deterministic, so two runs cannot disagree about a row. + db.prepare( + `UPDATE \`filing_document\` + SET \`kb_indexed_at\` = \`converted_at\` + WHERE \`kb_indexed_at\` IS NULL + AND EXISTS ( + SELECT 1 FROM \`${KB_DOCUMENT_TABLE}\` k + WHERE k.\`doc_id\` = \`filing_document\`.\`accession_number\` + || ':' || \`filing_document\`.\`doc_file\` + )` + ).run(); + + createPartialIndex(db); +} + +/** + * Indexed on the order the selection reads — newest first, ties by accession — + * so the same walk serves both the filter and the `ORDER BY`. Partial, because + * the rows it must not contain are the ones that accumulate: a corpus that is + * fully indexed leaves it empty, which is the point. + */ +function createPartialIndex(db: Sqlite.Database): void { + db.prepare( + `CREATE INDEX IF NOT EXISTS \`${PARTIAL_INDEX}\` + ON \`filing_document\` (\`filing_date\` DESC, \`accession_number\` DESC) + WHERE \`kb_indexed_at\` IS NULL` + ).run(); +} + +function hasColumn(db: Sqlite.Database, table: string, column: string): boolean { + const rows = db.prepare(`PRAGMA table_info(\`${table}\`)`).all() as Array<{ name?: unknown }>; + return rows.some((row) => row.name === column); +} + +function tableExists(db: Sqlite.Database, table: string): boolean { + return ( + db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").all(table) + .length > 0 + ); +} + +/** Whether the fast path is available — the column exists, so setup backfilled it. */ +export function kbIndexedStampAvailable(db: Sqlite.Database): boolean { + return hasColumn(db, "filing_document", "kb_indexed_at"); +} diff --git a/src/config/setupAllDatabases.ts b/src/config/setupAllDatabases.ts index f3947901..0542f96f 100644 --- a/src/config/setupAllDatabases.ts +++ b/src/config/setupAllDatabases.ts @@ -15,6 +15,7 @@ import { } from "./addMissingColumns"; import { alignPostgresColumnTypes } from "./alignPostgresColumnTypes"; import { dropStaleCheckConstraints } from "./dropStaleCheckConstraints"; +import { syncKbIndexedStamp } from "./kbIndexedStamp"; import { SEC_STORAGE_REGISTRY } from "./storageRegistry"; import { SEC_DB_FOLDER, SEC_DB_TYPE } from "./tokens"; @@ -49,12 +50,14 @@ export async function setupAllDatabases(): Promise { // Skipped under --dry-run: these passes issue DDL through raw SQL, which the // repositories' ReadOnlyTabularStorage wrapper cannot intercept. - if ( - dbType === "sqlite" && - globalServiceRegistry.has(SEC_DB_FOLDER) && - shouldAddMissingColumns("sqlite") - ) { - addMissingColumnsSqlite(getDb()); + if (dbType === "sqlite" && globalServiceRegistry.has(SEC_DB_FOLDER)) { + if (shouldAddMissingColumns("sqlite")) { + addMissingColumnsSqlite(getDb()); + } + // After the column pass, because the stamp it maintains is one of the + // columns that pass adds — and it must be backfilled in the same run that + // creates its index, never left empty. + syncKbIndexedStamp(getDb()); } if (dbType === "postgres" && !isDryRun()) { if (shouldAddMissingColumns("postgres")) await addMissingColumnsPostgres(); diff --git a/src/storage/document/FilingDocumentSchema.ts b/src/storage/document/FilingDocumentSchema.ts index d8875758..56e38dff 100644 --- a/src/storage/document/FilingDocumentSchema.ts +++ b/src/storage/document/FilingDocumentSchema.ts @@ -90,6 +90,24 @@ export const FilingDocumentSchema = Type.Object({ */ converter_version: Type.String({ maxLength: 32, description: "Converter version stamp" }), converted_at: Type.String({ description: "ISO 8601 timestamp" }), + /** + * When this document entered the knowledge base, or null while it has not. + * + * A denormalisation of "is there a `kb_document` row for this", and it earns + * its keep in the one regime the index selection spends its life in. `ask` + * pre-indexes before every question, so after the first question the + * selection matches nothing — and a `LIMIT` that never fills has to visit + * every candidate to learn that. Against a partial index over the nulls, the + * steady state is an empty index scan instead of the whole table. + * + * It is a fast path, never the authority: the anti-join against + * `kb_document` still runs, so a stamp that is missing or stale costs a + * traversal rather than a document indexed twice. That direction matters, + * because indexing twice spends embedding calls. + */ + kb_indexed_at: TypeNullable( + Type.String({ description: "ISO 8601 timestamp the document entered the knowledge base" }) + ), }); export type FilingDocument = Static; diff --git a/src/task/document/ConvertFilingDocumentTask.ts b/src/task/document/ConvertFilingDocumentTask.ts index efce89ca..57129714 100644 --- a/src/task/document/ConvertFilingDocumentTask.ts +++ b/src/task/document/ConvertFilingDocumentTask.ts @@ -387,6 +387,11 @@ export class ConvertFilingDocumentTask extends Task< char_count: doc.charCount, converter_version: FILING_CONVERTER_VERSION, converted_at: convertedAt, + // Freshly converted markdown is not in the knowledge base yet, and a + // RE-conversion at a newer converter version has replaced the text the + // knowledge base holds — so both start unindexed. Narrower than the + // anti-join beside it, never wider, which is the safe direction. + kb_indexed_at: null, }); } diff --git a/src/task/document/selectFilingsToConvert.sqlite.test.ts b/src/task/document/selectFilingsToConvert.sqlite.test.ts index bf43b10f..6d61600b 100644 --- a/src/task/document/selectFilingsToConvert.sqlite.test.ts +++ b/src/task/document/selectFilingsToConvert.sqlite.test.ts @@ -39,6 +39,7 @@ const documentRow = (accession: string, docFile: string, isPrimary: boolean) => char_count: 900, converter_version: VERSION, converted_at: "2026-03-02T00:00:00.000Z", + kb_indexed_at: null, }); /** diff --git a/src/task/document/selectFilingsToConvert.test.ts b/src/task/document/selectFilingsToConvert.test.ts index a6799f28..7f1a52a6 100644 --- a/src/task/document/selectFilingsToConvert.test.ts +++ b/src/task/document/selectFilingsToConvert.test.ts @@ -61,6 +61,7 @@ async function markConverted( char_count: 100, converter_version: version, converted_at: "2026-08-01T00:00:00.000Z", + kb_indexed_at: null, }); } diff --git a/src/task/kb/IndexFilingSectionsTask.test.ts b/src/task/kb/IndexFilingSectionsTask.test.ts index e46fe74b..c91f9f1d 100644 --- a/src/task/kb/IndexFilingSectionsTask.test.ts +++ b/src/task/kb/IndexFilingSectionsTask.test.ts @@ -34,6 +34,7 @@ const header = (index: number, filingDate: string): FilingDocument => ({ char_count: 100, converter_version: "1", converted_at: "2026-01-01T00:00:00.000Z", + kb_indexed_at: null, }); /** @@ -213,4 +214,28 @@ describe("IndexFilingSectionsTask under --dry-run", () => { expect(upsert).toHaveBeenCalledTimes(1); expect(out).toMatchObject({ success: true, indexed: 1, sections: 1 }); }); + + it("stamps kb_indexed_at on what it indexed, and leaves it alone on a dry run", async () => { + // The stamp is what lets the next run answer "anything to index?" from a + // partial index rather than a traversal. A dry run indexes nothing, so it + // must claim nothing either. + const kb = await getSecKnowledgeBase(); + vi.spyOn(kb, "upsert").mockResolvedValue({ + doc_id: "0000320193-26-000000:primary.htm", + } as never); + const stamp = () => + ( + getDb().prepare("SELECT `kb_indexed_at` AS s FROM `filing_document` LIMIT 1").get() as + | { s: string | null } + | undefined + )?.s ?? null; + + globalServiceRegistry.registerInstance(SEC_DRY_RUN, true); + await new IndexFilingSectionsTask().run({}); + expect(stamp()).toBeNull(); + + globalServiceRegistry.registerInstance(SEC_DRY_RUN, false); + await new IndexFilingSectionsTask().run({}); + expect(stamp()).not.toBeNull(); + }); }); diff --git a/src/task/kb/IndexFilingSectionsTask.ts b/src/task/kb/IndexFilingSectionsTask.ts index b37fba2c..6e99c627 100644 --- a/src/task/kb/IndexFilingSectionsTask.ts +++ b/src/task/kb/IndexFilingSectionsTask.ts @@ -16,7 +16,10 @@ import { } from "workglow"; import { isDryRun } from "../../cli/isDryRun"; import { getSecKnowledgeBase } from "../../kb/secKnowledgeBase"; -import type { FilingDocument } from "../../storage/document/FilingDocumentSchema"; +import { + FILING_DOCUMENT_REPOSITORY_TOKEN, + type FilingDocument, +} from "../../storage/document/FilingDocumentSchema"; import { FILING_SECTION_REPOSITORY_TOKEN, type FilingSection, @@ -171,6 +174,9 @@ export class IndexFilingSectionsTask extends Task< ): Promise> { const kb = await getSecKnowledgeBase(); const sectionRepo = globalServiceRegistry.get(FILING_SECTION_REPOSITORY_TOKEN); + const documentRepo = globalServiceRegistry.has(FILING_DOCUMENT_REPOSITORY_TOKEN) + ? globalServiceRegistry.get(FILING_DOCUMENT_REPOSITORY_TOKEN) + : undefined; const scope = { cik: input.cik, @@ -226,6 +232,12 @@ export class IndexFilingSectionsTask extends Task< document.doc_id = docId; await new KbAddDocumentTask().run({ knowledgeBase: kb, document } as never); + // Stamped only after the document is in, and never treated as the + // authority: the selection still anti-joins `kb_document`, so a stamp + // this fails to write costs a traversal on the next run rather than a + // document that never gets indexed. + await documentRepo?.put({ ...header, kb_indexed_at: new Date().toISOString() }); + indexed += 1; sectionTotal += sections.length; await context.updateProgress( diff --git a/src/task/kb/selectDocumentsToIndex.sqlite.test.ts b/src/task/kb/selectDocumentsToIndex.sqlite.test.ts index 8451fe63..dc3b0d05 100644 --- a/src/task/kb/selectDocumentsToIndex.sqlite.test.ts +++ b/src/task/kb/selectDocumentsToIndex.sqlite.test.ts @@ -31,6 +31,7 @@ const doc = (index: number, over: Partial = {}): FilingDocument char_count: 100, converter_version: "1", converted_at: "2026-01-01T00:00:00.000Z", + kb_indexed_at: null, ...over, }); diff --git a/src/task/kb/selectDocumentsToIndex.ts b/src/task/kb/selectDocumentsToIndex.ts index c36da210..199bff9f 100644 --- a/src/task/kb/selectDocumentsToIndex.ts +++ b/src/task/kb/selectDocumentsToIndex.ts @@ -6,6 +6,7 @@ import type { PageCursor } from "workglow"; import { globalServiceRegistry } from "workglow"; +import { kbIndexedStampAvailable } from "../../config/kbIndexedStamp"; import { KB_DOCUMENT_TABLE } from "../../kb/secKbTables"; import { FILING_DOCUMENT_REPOSITORY_TOKEN, @@ -113,7 +114,20 @@ export async function selectDocumentsToIndex( // transaction as the section rows, so it is the same answer reading them // gives. clauses.push("d.`section_count` > 0"); - if (antiJoin) clauses.push("k.`doc_id` IS NULL"); + if (antiJoin) { + clauses.push("k.`doc_id` IS NULL"); + // The fast path, and only ever a narrowing of the anti-join beside it. In + // the regime `ask` lives in — pre-index before every question, so nothing + // is left to do — the anti-join alone still has to visit every candidate + // before a LIMIT that never fills can conclude it is empty. This clause + // answers that from a partial index over the nulls, which a fully indexed + // corpus leaves empty. + // + // Conditional on the column existing because `db setup` backfills it in the + // same pass that creates the index, so its presence is what says the values + // can be trusted. Where it is absent the query is exactly what it was. + if (kbIndexedStampAvailable(getDb())) clauses.push("d.`kb_indexed_at` IS NULL"); + } // SQLite numbers `?` by position, so the limit binds last because it is // written last. if (options.limit !== undefined) params.push(options.limit);