diff --git a/.depalignrc.json b/.depalignrc.json index 8aeaf9562..5b8e60e05 100644 --- a/.depalignrc.json +++ b/.depalignrc.json @@ -1,5 +1,6 @@ { "ignore": { - "node-fetch": ["^3.3.2", "^2.6.11"] + "node-fetch": ["^3.3.2", "^2.6.11"], + "js-yaml": ["^5.2.2", "^4.1.1"] } } diff --git a/.prettierignore b/.prettierignore index 73b9a5ffe..fe7b3b3b6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ resources/cidrs.json +packages/mongodb-schema/examples/fanclub.json diff --git a/package-lock.json b/package-lock.json index d7eb612cc..8ba5a1f13 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6685,6 +6685,226 @@ "react": "^17.0.2" } }, + "node_modules/@mongodb-js/compass-test-server": { + "version": "0.3.22", + "resolved": "https://registry.npmjs.org/@mongodb-js/compass-test-server/-/compass-test-server-0.3.22.tgz", + "integrity": "sha512-iYoLWFePqjSgU0AW7I5EgOF0WKcm9Me4VWJYY4Nz5dN67S4JpiLA5YLF/7HMBPRwg0ymmPKOmEMknUUWoMUCZQ==", + "dev": true, + "license": "SSPL", + "dependencies": { + "mongodb-runner": "^5.8.0" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/@mongodb-js/mongodb-downloader": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.5.0.tgz", + "integrity": "sha512-N0ijRRNygm1xTQ3IhAMFcnZNhhF1CTwzFHqMk+tttm6LKD576t7j6olfuqSf8vFQ+tVpO6Z1JWOcquTKrpwOlQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.7.0", + "node-fetch": "^2.7.0", + "tar": "^6.1.15" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/mongodb": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", + "integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.2" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.3.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/mongodb-connection-string-url": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/mongodb-runner": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-5.11.1.tgz", + "integrity": "sha512-vgp9pKqnfRr5aXUCYqh0DHw0I3LmRjD+LvoiGi2scp1BCgTeftSEZGy9py8xkzeWxCGBr7S0uwdmfcUPiVYusg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/mongodb-downloader": "^0.5.0", + "@mongodb-js/saslprep": "^1.3.2", + "debug": "^4.4.0", + "mongodb": "^6.9.0", + "mongodb-connection-string-url": "^3.0.0", + "yargs": "^17.7.2" + }, + "bin": { + "mongodb-runner": "bin/runner.js" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@mongodb-js/compass-test-server/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/@mongodb-js/device-id": { "resolved": "packages/device-id", "link": true @@ -6729,6 +6949,10 @@ "resolved": "packages/mongodb-downloader", "link": true }, + "node_modules/@mongodb-js/mongodb-schema": { + "resolved": "packages/mongodb-schema", + "link": true + }, "node_modules/@mongodb-js/mongodb-server-log-checker": { "resolved": "packages/mongodb-server-log-checker", "link": true @@ -9706,6 +9930,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/reservoir": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@types/reservoir/-/reservoir-0.1.6.tgz", + "integrity": "sha512-2Eo3hcBaWYtrr0DwJLa8X/V/d8YQw72s7OuE9N+MrnYx2MQ72tftXX9lDQwnPKMeRiJrBXFbhfJ4FRMrs2+39w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -13101,44 +13332,6 @@ "node-fetch": "^2.7.0" } }, - "node_modules/cross-fetch/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/cross-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/cross-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/cross-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -21741,6 +21934,48 @@ "node": ">=10.5.0" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-machine-id": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/node-machine-id/-/node-machine-id-1.1.12.tgz", @@ -28467,32 +28702,6 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, - "packages/dl-center/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "packages/dl-center/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "packages/dl-center/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -28500,22 +28709,6 @@ "dev": true, "license": "MIT" }, - "packages/dl-center/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "packages/dl-center/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "packages/download-url": { "name": "mongodb-download-url", "version": "1.8.15", @@ -28558,26 +28751,6 @@ "undici-types": ">=7.24.0 <7.24.7" } }, - "packages/download-url/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "packages/download-url/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -28590,12 +28763,6 @@ "node": ">=10" } }, - "packages/download-url/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "packages/download-url/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -28603,22 +28770,6 @@ "dev": true, "license": "MIT" }, - "packages/download-url/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "packages/download-url/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "packages/get-os-info": { "name": "@mongodb-js/get-os-info", "version": "0.7.1", @@ -28874,31 +29025,6 @@ "@types/sinon": "*" } }, - "packages/mongodb-downloader/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "packages/mongodb-downloader/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "packages/mongodb-downloader/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -28906,20 +29032,6 @@ "dev": true, "license": "MIT" }, - "packages/mongodb-downloader/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "packages/mongodb-downloader/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "packages/mongodb-log-writer": { "version": "2.5.14", "license": "Apache-2.0", @@ -29217,6 +29329,309 @@ "node": "^20.19.0 || ^22.12.0 || >=23" } }, + "packages/mongodb-schema": { + "name": "@mongodb-js/mongodb-schema", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "reservoir": "^0.1.2" + }, + "bin": { + "mongodb-schema": "bin/mongodb-schema" + }, + "devDependencies": { + "@mongodb-js/compass-test-server": "^0.3.2", + "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mocha-config-devtools": "^1.1.3", + "@mongodb-js/prettier-config-devtools": "^1.0.4", + "@mongodb-js/tsconfig-devtools": "^1.1.3", + "@types/chai": "^4.2.21", + "@types/json-schema": "^7.0.15", + "@types/mocha": "^9.1.1", + "@types/node": "^25.5.2", + "@types/reservoir": "^0.1.0", + "@types/sinon-chai": "^4.0.0", + "ajv": "^8.18.0", + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", + "chai": "^4.5.0", + "depcheck": "^1.4.7", + "eslint": "^7.25.0 || ^8.0.0", + "gen-esm-wrapper": "^1.1.3", + "mocha": "^8.4.0", + "mongodb": "^7.5.0", + "nyc": "^15.1.0", + "prettier": "^3.8.1", + "sinon": "^9.2.3", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + }, + "optionalDependencies": { + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", + "cli-table": "^0.3.4", + "js-yaml": "^5.2.2", + "mongodb": "^6.9.0 || ^7.0.0", + "mongodb-ns": "^3.2.1", + "numeral": "^2.0.6", + "progress": "^2.0.3", + "stats-lite": "^2.0.0", + "yargs": "^18.0.0" + } + }, + "packages/mongodb-schema/node_modules/@types/node": { + "version": "25.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.5.tgz", + "integrity": "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "packages/mongodb-schema/node_modules/@types/sinon-chai": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/sinon-chai/-/sinon-chai-4.0.0.tgz", + "integrity": "sha512-Uar+qk3TmeFsUWCwtqRNqNUE7vf34+MCJiQJR5M2rd4nCbhtE8RgTiHwN/mVwbfCjhmO6DiOel/MgzHkRMJJFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "*", + "@types/sinon": "*" + } + }, + "packages/mongodb-schema/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "packages/mongodb-schema/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "packages/mongodb-schema/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "packages/mongodb-schema/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0", + "optional": true + }, + "packages/mongodb-schema/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, + "packages/mongodb-schema/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "packages/mongodb-schema/node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/mongodb-schema/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT", + "optional": true + }, + "packages/mongodb-schema/node_modules/js-yaml": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz", + "integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.mjs" + } + }, + "packages/mongodb-schema/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "packages/mongodb-schema/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "license": "MIT", + "optional": true, + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/mongodb-schema/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "packages/mongodb-schema/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "packages/mongodb-schema/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "packages/mongodb-schema/node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/mongodb-schema/node_modules/yargs": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", + "integrity": "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==", + "license": "MIT", + "optional": true, + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "packages/mongodb-schema/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "optional": true, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "packages/mongodb-server-log-checker": { "name": "@mongodb-js/mongodb-server-log-checker", "version": "0.2.18", @@ -29275,11 +29690,11 @@ "version": "0.8.2", "license": "Apache-2.0", "dependencies": { + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/ts-autocomplete": "^0.5.13", "@mongosh/shell-api": "^5.1.6", "debug": "^4.4.0", "lodash": "^4.17.21", - "mongodb-schema": "^12.6.2", "node-cache": "^5.1.2", "typescript": "^5.9.3" }, @@ -29669,6 +30084,7 @@ }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/prettier-config-devtools": "^1.0.4", "@mongodb-js/shell-bson-parser": "^1.5.14", "@mongodb-js/tsconfig-devtools": "^1.1.3", @@ -29682,7 +30098,6 @@ "js-yaml": "^4.1.1", "jsdom": "^24.1.3", "json-schema-to-zod": "^2.6.0", - "mongodb-schema": "^12.6.2", "prettier": "^3.8.1", "ts-node": "^10.9.2", "typescript": "^5.9.3", @@ -29775,10 +30190,20 @@ "license": "MIT" }, "packages/mql-typescript/node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -30528,26 +30953,6 @@ "node": ">=16 || 14 >=14.17" } }, - "packages/sbom-tools/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "packages/sbom-tools/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -30622,12 +31027,6 @@ "node": ">=10" } }, - "packages/sbom-tools/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "packages/sbom-tools/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -30635,22 +31034,6 @@ "dev": true, "license": "MIT" }, - "packages/sbom-tools/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "packages/sbom-tools/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "packages/shell-bson-parser": { "name": "@mongodb-js/shell-bson-parser", "version": "1.5.14", @@ -36036,6 +36419,153 @@ "react-window": "^1.8.6" } }, + "@mongodb-js/compass-test-server": { + "version": "0.3.22", + "resolved": "https://registry.npmjs.org/@mongodb-js/compass-test-server/-/compass-test-server-0.3.22.tgz", + "integrity": "sha512-iYoLWFePqjSgU0AW7I5EgOF0WKcm9Me4VWJYY4Nz5dN67S4JpiLA5YLF/7HMBPRwg0ymmPKOmEMknUUWoMUCZQ==", + "dev": true, + "requires": { + "mongodb-runner": "^5.8.0" + }, + "dependencies": { + "@mongodb-js/mongodb-downloader": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.5.0.tgz", + "integrity": "sha512-N0ijRRNygm1xTQ3IhAMFcnZNhhF1CTwzFHqMk+tttm6LKD576t7j6olfuqSf8vFQ+tVpO6Z1JWOcquTKrpwOlQ==", + "dev": true, + "requires": { + "debug": "^4.4.0", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.7.0", + "node-fetch": "^2.7.0", + "tar": "^6.1.15" + } + }, + "@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "dev": true, + "requires": { + "@types/webidl-conversions": "*" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mongodb": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", + "integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==", + "dev": true, + "requires": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.2" + } + }, + "mongodb-connection-string-url": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "dev": true, + "requires": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" + } + }, + "mongodb-runner": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-5.11.1.tgz", + "integrity": "sha512-vgp9pKqnfRr5aXUCYqh0DHw0I3LmRjD+LvoiGi2scp1BCgTeftSEZGy9py8xkzeWxCGBr7S0uwdmfcUPiVYusg==", + "dev": true, + "requires": { + "@mongodb-js/mongodb-downloader": "^0.5.0", + "@mongodb-js/saslprep": "^1.3.2", + "debug": "^4.4.0", + "mongodb": "^6.9.0", + "mongodb-connection-string-url": "^3.0.0", + "yargs": "^17.7.2" + } + }, + "tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "dev": true + } + } + }, + "yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true + } + } + }, "@mongodb-js/device-id": { "version": "file:packages/device-id", "requires": { @@ -36302,38 +36832,11 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } }, @@ -36538,18 +37041,169 @@ "@types/sinon": "*" } }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true + } + } + }, + "@mongodb-js/mongodb-schema": { + "version": "file:packages/mongodb-schema", + "requires": { + "@mongodb-js/compass-test-server": "^0.3.2", + "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mocha-config-devtools": "^1.1.3", + "@mongodb-js/prettier-config-devtools": "^1.0.4", + "@mongodb-js/tsconfig-devtools": "^1.1.3", + "@types/chai": "^4.2.21", + "@types/json-schema": "^7.0.15", + "@types/mocha": "^9.1.1", + "@types/node": "^25.5.2", + "@types/reservoir": "^0.1.0", + "@types/sinon-chai": "^4.0.0", + "ajv": "^8.18.0", + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", + "chai": "^4.5.0", + "cli-table": "^0.3.4", + "depcheck": "^1.4.7", + "eslint": "^7.25.0 || ^8.0.0", + "gen-esm-wrapper": "^1.1.3", + "js-yaml": "^5.2.2", + "mocha": "^8.4.0", + "mongodb": "^7.5.0", + "mongodb-ns": "^3.2.1", + "numeral": "^2.0.6", + "nyc": "^15.1.0", + "prettier": "^3.8.1", + "progress": "^2.0.3", + "reservoir": "^0.1.2", + "sinon": "^9.2.3", + "stats-lite": "^2.0.0", + "ts-node": "^10.9.2", + "typescript": "^5.9.3", + "yargs": "^18.0.0" + }, + "dependencies": { + "@types/node": { + "version": "25.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.5.tgz", + "integrity": "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==", + "dev": true, "requires": { - "whatwg-url": "^5.0.0" + "undici-types": ">=7.24.0 <7.24.7" } }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + "@types/sinon-chai": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/sinon-chai/-/sinon-chai-4.0.0.tgz", + "integrity": "sha512-Uar+qk3TmeFsUWCwtqRNqNUE7vf34+MCJiQJR5M2rd4nCbhtE8RgTiHwN/mVwbfCjhmO6DiOel/MgzHkRMJJFg==", + "dev": true, + "requires": { + "@types/chai": "*", + "@types/sinon": "*" + } + }, + "ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + } + }, + "ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "optional": true + }, + "ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "optional": true + }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "optional": true + }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true + }, + "cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "optional": true, + "requires": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "dependencies": { + "string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "optional": true, + "requires": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + } + } + } + }, + "emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "optional": true + }, + "js-yaml": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz", + "integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==", + "optional": true, + "requires": { + "argparse": "^2.0.1" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "optional": true, + "requires": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + } + }, + "strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "optional": true, + "requires": { + "ansi-regex": "^6.2.2" + } }, "undici-types": { "version": "7.24.6", @@ -36557,19 +37211,49 @@ "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + "wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "optional": true, + "requires": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "dependencies": { + "string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "optional": true, + "requires": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + } + } + } }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "yargs": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", + "integrity": "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==", + "optional": true, "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" } + }, + "yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "optional": true } } }, @@ -36628,6 +37312,7 @@ "requires": { "@mongodb-js/eslint-config-devtools": "^0.11.8", "@mongodb-js/mocha-config-devtools": "^1.1.3", + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/mql-typescript": "^0.6.2", "@mongodb-js/prettier-config-devtools": "^1.0.4", "@mongodb-js/ts-autocomplete": "^0.5.13", @@ -36646,7 +37331,6 @@ "lodash": "^4.17.21", "mocha": "^8.4.0", "mongodb": "^7.5.0", - "mongodb-schema": "^12.6.2", "node-cache": "^5.1.2", "nyc": "^15.1.0", "prettier": "^3.8.1", @@ -36884,6 +37568,7 @@ "version": "file:packages/mql-typescript", "requires": { "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/prettier-config-devtools": "^1.0.4", "@mongodb-js/shell-bson-parser": "^1.5.14", "@mongodb-js/tsconfig-devtools": "^1.1.3", @@ -36899,7 +37584,6 @@ "jsdom": "^24.1.3", "json-schema-to-zod": "^2.6.0", "mongodb": "^7.5.0", - "mongodb-schema": "^12.6.2", "prettier": "^3.8.1", "ts-node": "^10.9.2", "typescript": "^5.9.3", @@ -36965,9 +37649,9 @@ "dev": true }, "js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", "dev": true, "requires": { "argparse": "^2.0.1" @@ -37547,14 +38231,6 @@ "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", "dev": true }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -37596,30 +38272,11 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==" }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } }, @@ -39906,6 +40563,12 @@ "integrity": "sha512-L0+4KOC47zH0AE/27YhzECRlE4HxHycQy0ng8P4SMmsYIdUqJFzKlk4LameozRQ2mRozPRwRAzpXsDsUgb6ezA==", "dev": true }, + "@types/reservoir": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@types/reservoir/-/reservoir-0.1.6.tgz", + "integrity": "sha512-2Eo3hcBaWYtrr0DwJLa8X/V/d8YQw72s7OuE9N+MrnYx2MQ72tftXX9lDQwnPKMeRiJrBXFbhfJ4FRMrs2+39w==", + "dev": true + }, "@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -42327,35 +42990,6 @@ "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==", "requires": { "node-fetch": "^2.7.0" - }, - "dependencies": { - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - } } }, "cross-spawn": { @@ -48285,43 +48919,16 @@ "undici-types": ">=7.24.0 <7.24.7" } }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, "semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==" }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } }, @@ -48899,6 +49506,35 @@ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "node-machine-id": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/node-machine-id/-/node-machine-id-1.1.12.tgz", diff --git a/packages/mongodb-schema/.depcheckrc b/packages/mongodb-schema/.depcheckrc new file mode 100644 index 000000000..400223b76 --- /dev/null +++ b/packages/mongodb-schema/.depcheckrc @@ -0,0 +1,13 @@ +ignores: + - '@mongodb-js/prettier-config-devtools' + - '@mongodb-js/tsconfig-devtools' + - '@types/chai' + - '@types/json-schema' + - '@types/reservoir' + - '@types/sinon-chai' + - 'chai' + - 'json-schema' + - 'reservoir' + - 'sinon' +ignore-patterns: + - 'dist' diff --git a/packages/mongodb-schema/.eslintignore b/packages/mongodb-schema/.eslintignore new file mode 100644 index 000000000..2b1ce0295 --- /dev/null +++ b/packages/mongodb-schema/.eslintignore @@ -0,0 +1,6 @@ +.nyc_output +coverage +dist +docs +examples/fanclub.json +.esm-wrapper.mjs diff --git a/packages/mongodb-schema/.eslintrc.js b/packages/mongodb-schema/.eslintrc.js new file mode 100644 index 000000000..83296d735 --- /dev/null +++ b/packages/mongodb-schema/.eslintrc.js @@ -0,0 +1,8 @@ +module.exports = { + root: true, + extends: ['@mongodb-js/eslint-config-devtools'], + parserOptions: { + tsconfigRootDir: __dirname, + project: ['./tsconfig-lint.json'], + }, +}; diff --git a/packages/mongodb-schema/.mocharc.js b/packages/mongodb-schema/.mocharc.js new file mode 100644 index 000000000..64afeb1f9 --- /dev/null +++ b/packages/mongodb-schema/.mocharc.js @@ -0,0 +1 @@ +module.exports = require('@mongodb-js/mocha-config-devtools'); diff --git a/packages/mongodb-schema/.prettierignore b/packages/mongodb-schema/.prettierignore new file mode 100644 index 000000000..2b1ce0295 --- /dev/null +++ b/packages/mongodb-schema/.prettierignore @@ -0,0 +1,6 @@ +.nyc_output +coverage +dist +docs +examples/fanclub.json +.esm-wrapper.mjs diff --git a/packages/mongodb-schema/.prettierrc.json b/packages/mongodb-schema/.prettierrc.json new file mode 100644 index 000000000..dfae21d04 --- /dev/null +++ b/packages/mongodb-schema/.prettierrc.json @@ -0,0 +1 @@ +"@mongodb-js/prettier-config-devtools" diff --git a/packages/mongodb-schema/LICENSE b/packages/mongodb-schema/LICENSE new file mode 100644 index 000000000..5e0fd33cb --- /dev/null +++ b/packages/mongodb-schema/LICENSE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "{}" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright {yyyy} {name of copyright owner} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/mongodb-schema/README.md b/packages/mongodb-schema/README.md new file mode 100644 index 000000000..ee5eacf3e --- /dev/null +++ b/packages/mongodb-schema/README.md @@ -0,0 +1,353 @@ +# @mongodb-js/mongodb-schema [![][npm_img]][npm_url] + +Infer a probabilistic schema for a MongoDB collection. + +## Usage + +`mongodb-schema` can be used as a command line tool or programmatically in your application as a node module. + +### Command line + +To install mongodb-schema for command line use, run `npm install -g @mongodb-js/mongodb-schema`. This will add a new +shell script which you can run directly from the command line. + +The command line tool expects a MongoDB connection URI and a namespace in the form `.`. +Without further arguments, it will sample 100 random documents from the collection and print a schema of +the collection in JSON format to stdout. + +``` +mongodb-schema mongodb://localhost:27017 mongodb.fanclub +``` + +Additional arguments change the number of samples (`--number`), print additional statistics about the +schema analysis (`--stats`), switch to a different output format (`--format`), or let you suppress the +schema output altogether (`--no-output`) if you are only interested in the schema statistics, semantic +type discovery (`--semantic-types`), and the ability to disable value collection (`--no-values`). + +For more information, run + +``` +mongodb-schema --help +``` + +### API + +The following example demonstrates how `mongodb-schema` can be used programmatically from +your node application. You need to additionally install the MongoDB node driver to follow +along with this example. + +Make sure you have a `mongod` running on localhost on port 27017 (or change the example +below accordingly). + +1. From your application folder, install the driver and `mongodb-schema` locally: + + ``` + npm install --save mongodb @mongodb-js/mongodb-schema + ``` + +2. (optional) If you don't have any data in your MongoDB instance yet, you can create a + `test.data` collection with this command: + + ``` + mongosh --eval "db.data.insertMany([{_id: 1, a: true}, {_id: 2, a: 'true'}, {_id: 3, a: 1}, {_id: 4}])" localhost:27017/test + ``` + +3. Create a new file `parse-schema.js` and paste in the following code: + + ```javascript + const { parseSchema } = require('@mongodb-js/mongodb-schema'); + const { MongoClient } = require('mongodb'); + + const dbName = 'test'; + const uri = `mongodb://localhost:27017/${dbName}`; + const client = new MongoClient(uri); + + async function run() { + try { + const database = client.db(dbName); + const documentStream = database.collection('data').find(); + + // Here we are passing in a cursor as the first argument. You can + // also pass in a stream or an array of documents directly. + const schema = await parseSchema(documentStream); + + console.log(JSON.stringify(schema, null, 2)); + } finally { + await client.close(); + } + } + + run().catch(console.dir); + ``` + +4. When we run the above with `node ./parse-schema.js`, we'll see output + similar to this (some fields not present here for clarity): + +```javascript +{ + "count": 4, // parsed 4 documents + "fields": [ // an array of Field objects, @see `./src/schema-analyzer.ts` + { + "name": "_id", + "count": 4, // 4 documents counted with _id + "type": "Number", // the type of _id is `Number` + "probability": 1, // all documents had an _id field + "hasDuplicates": false, // therefore no duplicates + "types": [ // an array of Type objects, @see `./src/schema-analyzer.ts` + { + "name": "Number", // name of the type + "count": 4, // 4 numbers counted + "probability": 1, + "unique": 4, + "values": [ // array of encountered values + 1, + 2, + 3, + 4 + ] + } + ] + }, + { + "name": "a", + "count": 3, // only 3 documents with field `a` counted + "probability": 0.75, // hence probability 0.75 + "type": [ // found these types + "Boolean", + "String", + "Number", + "Undefined" // for convenience, we treat Undefined as its own type + ], + "hasDuplicates": false, // there were no duplicate values + "types": [ + { + "name": "Boolean", + "count": 1, + "probability": 0.25, // probabilities for types are calculated factoring in Undefined + "unique": 1, + "values": [ + true + ] + }, + { + "name": "String", + "count": 1, + "probability": 0.25, + "unique": 1, + "values": [ + "true" + ] + }, + { + "name": "Number", + "count": 1, + "probability": 0.25, + "unique": 1, + "values": [ + 1 + ] + }, + { + "name": "Undefined", + "count": 1, + "probability": 0.25, + "unique": 0 + } + ] + } + ] +} +``` + +A high-level view of the schema tree structure is as follows: + +![](./docs/mongodb-schema_diagram.png) + +## BSON Types + +`mongodb-schema` supports all [BSON types][bson-types]. +Checkout [the tests][tests] for more usage examples. + +## Semantic Types + +As of version 6.1.0, mongodb-schema has a new feature called "Semantic Type Detection". It allows to override the type identification of a value. This enables users to provide specific domain knowledge of their data, while still using +the underlying flexible BSON representation and nested documents and arrays. + +One of the built-in semantic types is GeoJSON, which traditionally would just be detected as "Document" type. With the new option `semanticTypes` enabled, these sub-documents are now considered atomic values with a type "GeoJSON". The original BSON type name is still available under the `bsonType` field. + +To enable this mode, use the `-t` or `--semantic-types` flag at the command line. When using the API, pass an option object as the second parameter with the `semanticTypes` flag set to `true`: + +```javascript +parseSchema(db.collection('data').find(), {semanticTypes: true}, function(err, schema) { + ... +}); +``` + +This mode is disabled by default. + +#### Custom Semantic Types + +It is also possible to provide custom semantic type detector functions. This is useful to provide domain +knowledge, for example to detect trees or graphs, special string encodings of data, etc. + +The detector function is called with `value` and `path` (the full field path in dot notation) +as arguments, and must return a truthy value if the data type applies to this field or value. + +Here is an example to detect email addresses: + +```javascript + +var emailRegex = /[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/; + +function emailDetector(value, path) { + return emailRegex.test(value); +}; + +parseSchema(db.collection('data').find(), { semanticTypes: { EmailAddress: emailDetector } }, function(err, schema) { + ... +}); +``` + +This returns a schema with the following content (only partially shown): + +``` +{ + "name": "email", + "path": "email", + "count": 100, + "types": [ + { + "name": "EmailAddress", // custom type "EmailAddress" was recognized + "bsonType": "String", // original BSON type available as well + "path": "email", + "count": 100, + "values": [ + "twinbutterfly28@aim.com", + "funkymoney45@comcast.net", + "beauty68@msn.com", + "veryberry8@hotmail.com", + +``` + +As can be seen, the field name "email" was correctly identified as a custom type "EmailAddress". + +## Value Sampling + +As of version 6.1.0, mongodb-schema supports analysing only the structure of the documents, without collection data samples. To enable this mode, use the `--no-values` flag at the command line. When using the API, pass an option object as the second parameter with the `storeValues` flag set to `false`. + +This mode is enabled by default. + +## Schema Statistics + +To compare schemas quantitatively we introduce the following measurable metrics on a schema: + +#### Schema Depth + +The schema depth is defined as the maximum number of nested levels of keys in the schema. It does not matter if the subdocuments are nested directly or as elements of an array. An empty document has a depth of 0, whereas a document with some top-level keys but no nested subdocuments has a depth of 1. + +#### Schema Width + +The schema width is defined as the number of individual keys, added up over all nesting levels of the schema. Array values do not count towards the schema width. + +#### Examples + +```js +{ +} +``` + +| Statistic | Value | +| :----------- | :---: | +| Schema Depth | 0 | +| Schema Width | 0 | + +```js +{ + one: 1; +} +``` + +| Statistic | Value | +| :----------- | :---: | +| Schema Depth | 1 | +| Schema Width | 1 | + +```js +{ + one: [ + "foo", + "bar", + { + two: { + three: 3 + } + }, + "baz" + ], + foo: "bar" +} +``` + +| Statistic | Value | +| :----------- | :---: | +| Schema Depth | 3 | +| Schema Width | 4 | + +```js +{ + a: 1, + b: false, + one: { + c: null, + two: { + three: { + four: 4, + e: "deepest nesting level" + } + } + }, + f: { + g: "not the deepest level" + } +} +``` + +| Statistic | Value | +| :----------- | :---: | +| Schema Depth | 4 | +| Schema Width | 10 | + +```js +// first document +{ + foo: [ + { + bar: [1, 2, 3] + } + ] +}, +// second document +{ + foo: 0 +} +``` + +| Statistic | Value | +| :----------- | :---: | +| Schema Depth | 2 | +| Schema Width | 2 | + +## Testing + +``` +npm test +``` + +## License + +Apache 2.0 + +[bson-types]: http://docs.mongodb.org/manual/reference/bson-types/ +[tests]: https://github.com/mongodb-js/devtools-shared/tree/main/packages/mongodb-schema/test +[npm_img]: https://img.shields.io/npm/v/@mongodb-js/mongodb-schema.svg +[npm_url]: https://www.npmjs.org/package/@mongodb-js/mongodb-schema diff --git a/packages/mongodb-schema/bin/mongodb-schema b/packages/mongodb-schema/bin/mongodb-schema new file mode 100755 index 000000000..c62e74197 --- /dev/null +++ b/packages/mongodb-schema/bin/mongodb-schema @@ -0,0 +1,231 @@ +#!/usr/bin/env node + +/* eslint-disable no-console */ +/* eslint-disable @typescript-eslint/no-var-requires */ + +const { SchemaAnalyzer, schemaStats: _schemaStats } = require('../'); +const schemaStats = _schemaStats.default; + +const { MongoClient } = require('mongodb'); +const toNS = require('mongodb-ns'); +const yaml = require('js-yaml'); +const pkg = require('../package.json'); +const Table = require('cli-table'); +const numeral = require('numeral'); +const { EJSON } = require('bson'); +const ProgressBar = require('progress'); + +const yargs = require('yargs/yargs'); +const { hideBin } = require('yargs/helpers'); + +const argv = yargs(hideBin(process.argv)) + .strict() + .usage('Usage: $0 [--format= --number=]') + .demand(2) + .option('n', { + alias: 'number', + default: 100, + describe: 'The number of documents to return.', + }) + .option('f', { + alias: 'format', + default: 'json', + describe: 'The output format.', + choices: ['json', 'yaml', 'table'], + }) + .option('o', { + alias: 'output', + type: 'boolean', + describe: 'Print the computed schema to stdout.', + default: true, + }) + .option('s', { + alias: 'stats', + type: 'boolean', + describe: 'print schema statistics to stderr', + }) + .option('p', { + alias: 'promote', + type: 'boolean', + default: true, + describe: 'promote values to Javascript numbers.', + }) + .options('t', { + alias: 'semantic-types', + type: 'boolean', + default: false, + describe: + 'semantic type detection, currently supported are emails and geojson', + }) + .option('values', { + type: 'boolean', + default: true, + describe: 'enables the collection of sample values', + }) + .option('sampling', { + type: 'boolean', + default: true, + describe: 'use random sampling on the collection.', + }) + .describe('version', 'Show version.') + .alias('h', 'help') + .describe('h', 'Show this screen.') + .help('h') + .wrap(100) + .example( + '$0 localhost:27017 mongodb.fanclub --number 1000 --stats ' + '--no-output', + 'analyze 1000 docs from the mongodb.fanclub ' + + 'collection and only show statistics.', + ) + .example( + '$0 localhost:27017 test.foo --format table', + 'analyze 100 docs from the test.foo collection and print ' + + 'the schema in table form.', + ).argv; + +let uri = argv._[0]; +if (!uri.startsWith('mongodb://') && !uri.startsWith('mongodb+srv://')) { + uri = 'mongodb://' + uri; +} +const sampleSize = parseInt(argv.number, 10); + +if (argv.version) { + console.error(pkg.version); + process.exit(1); +} + +function addTableRow(table, field) { + table.push([ + field.path, + field.type, + numeral(field.probability).format('0.000%'), + ]); + + if (field.fields) { + field.fields.map(function (child) { + addTableRow(table, child); + }); + } + + if (field.arrayFields) { + field.arrayFields.map(function (child) { + addTableRow(table, child); + }); + } +} + +function getTable(schema) { + const table = new Table({ + head: ['Path', 'Type', 'Probability'], + colWidths: [50, 30, 20], + }); + schema.fields.map(function (field) { + addTableRow(table, field); + }); + return table; +} + +const bar = new ProgressBar('analyzing [:bar] :percent :etas ', { + total: argv.number, + width: 60, + complete: '=', + incomplete: ' ', + clear: true, +}); + +function sample(collection, size = 1000) { + return collection.aggregate( + [ + { + $sample: { + size, + }, + }, + ], + { + allowDiskUse: true, + }, + ); +} + +const client = new MongoClient(uri); + +(async function main() { + try { + await client.connect(); + } catch (err) { + if (err) { + console.error('Failed to connect to MongoDB: ', err); + process.exit(1); + } + } + + const ns = toNS(argv._[1]); + const db = client.db(ns.database); + let ts; + + const options = { + size: sampleSize, + query: {}, + promoteValues: argv.promote, + }; + + const schemaOptions = { + storeValues: argv.values, + semanticTypes: argv.semanticTypes, + }; + + const analyzer = new SchemaAnalyzer(schemaOptions); + try { + const input = argv.sampling + ? sample(db.collection(ns.collection), sampleSize) + : db + .collection(ns.collection) + .find(options.query, { + promoteValues: options.promoteValues, + }) + .limit(options.size); + + for await (const doc of input) { + bar.tick(); + analyzer.analyzeDoc(doc); + } + } catch (err) { + console.error('error:', err.message); + process.exit(1); + } + + const schema = analyzer.getResult(); + + if (argv.output) { + let output = ''; + if (argv.format === 'yaml') { + output = yaml.dump(schema); + } else if (argv.format === 'table') { + output = getTable(schema).toString(); + } else { + output = EJSON.stringify(schema, null, 2); + } + console.log(output); + } + + if (argv.stats) { + let branchOutput = '['; + const branchingFactors = schemaStats.branch(schema); + if (branchingFactors.length > 20) { + branchOutput += `${branchingFactors.slice(0, 20).join(',')},...] (top 20 shown)`; + } else { + branchOutput += branchingFactors.join(',') + ']'; + } + + console.error('toplevel fields:', schema.fields.length); + console.error('branching factors:', branchOutput); + console.error('schema width: ' + schemaStats.width(schema)); + console.error('schema depth: ' + schemaStats.depth(schema)); + } + + console.dir(analyzer.getSchemaPaths()); + console.dir(analyzer.getSimplifiedSchema(), { depth: null }); + + client.close(); +})(); diff --git a/packages/mongodb-schema/docs/mongodb-schema_diagram.graffle b/packages/mongodb-schema/docs/mongodb-schema_diagram.graffle new file mode 100644 index 000000000..4720a7925 Binary files /dev/null and b/packages/mongodb-schema/docs/mongodb-schema_diagram.graffle differ diff --git a/packages/mongodb-schema/docs/mongodb-schema_diagram.png b/packages/mongodb-schema/docs/mongodb-schema_diagram.png new file mode 100644 index 000000000..9de892dee Binary files /dev/null and b/packages/mongodb-schema/docs/mongodb-schema_diagram.png differ diff --git a/packages/mongodb-schema/examples/fanclub.json b/packages/mongodb-schema/examples/fanclub.json new file mode 100644 index 000000000..aa44f8344 --- /dev/null +++ b/packages/mongodb-schema/examples/fanclub.json @@ -0,0 +1,100 @@ +{"_id":{"$oid":"50319491fe4dce143835c552"},"membership_status":"ACTIVE","name":"Ellie J Clarke","gender":"male","age":36,"phone_no":"+19786213180","last_login":{"$date":"2014-01-31T22:26:33.000Z"},"address":{"city":"El Paso, Texas","street":"133 Aloha Ave","postal_code":50017,"country":"USA","location":{"type":"Point","coordinates":[-106.3974968970265,31.79689833641156]}},"favorite_feature":"Auth","email":"corefinder88@hotmail.com"} +{"_id":{"$oid":"5031bb65fe4dce143635c960"},"membership_status":"ACTIVE","name":"Ava Z Robinson","gender":"female","age":22,"phone_no":"+451634929422","last_login":{"$date":"2015-05-12T02:39:24.000Z"},"address":{"city":"El Paso, Texas","street":"290 East George Rd","postal_code":"18851-1275","country":"USA","location":{"type":"Point","coordinates":[-106.49044778805387,31.7957361699594]}},"email":"bubbles51@gmail.com"} +{"_id":{"$oid":"5031ce4bfe4dce143735c6df"},"membership_status":"PENDING","name":"Jayden V Allen","gender":"male","age":14,"phone_no":"+16632418722","last_login":{"$date":"2012-09-14T14:44:21.000Z"},"address":{"city":"Dallas, Texas","street":"369 Interurban Pkwy","postal_code":60240,"country":"USA","location":{"type":"Point","coordinates":[-96.80626011661555,32.76331397593709]}},"email":"netfreak22@msn.com"} +{"_id":{"$oid":"5031f557fe4dce143835c9b0"},"membership_status":"INACTIVE","name":"Joshua Evans","gender":"female","age":40,"phone_no":{"$numberLong":"6451617244"},"last_login":{"$date":"2012-06-22T13:19:16.000Z"},"address":{"city":"Chula Vista, California","street":"208 West Boston St","postal_code":"88662","country":"Australia","location":{"type":"Point","coordinates":[-117.11388821760998,32.707530991493854]}},"favorite_feature":"Document Validation","email":"takeaway45@icloud.com"} +{"_id":{"$oid":"5031fd5ffe4dce143835c242"},"membership_status":"ACTIVE","name":"Landon Jackson","gender":"male","phone_no":"+459395024796","last_login":{"$date":"2011-02-17T20:04:57.000Z"},"address":{"city":"Lincoln, Nebraska","street":"370 South Irving St","postal_code":"61657-9116","country":"UK","location":{"type":"Point","coordinates":[-96.6486323045409,40.749246312984]}},"favorite_feature":"Wired Tiger","email":"veryberry15@yahoo.com"} +{"_id":{"$oid":"50326538fe4dce143635c611"},"membership_status":"ACTIVE","name":"Madison J Harris","gender":"female","age":49,"phone_no":"+12347596128","last_login":{"$date":"2014-02-16T14:31:07.000Z"},"address":{"city":"Bakersfield, California","street":"86 West Prospect Ave","postal_code":97086,"country":"USA","location":{"type":"Point","coordinates":[-119.07966704760815,35.36248673002892]}},"email":"cookie25@comcast.net"} +{"_id":{"$oid":"50328e08fe4dce143535c7a9"},"membership_status":"ACTIVE","name":"Sophia Cooper","gender":"female","age":45,"phone_no":{"$numberLong":"6451029082"},"last_login":{"$date":"2013-02-26T14:35:11.000Z"},"address":{"city":"El Paso, Texas","street":"250 Clyde St","postal_code":87682,"country":"USA","location":{"type":"Point","coordinates":[-106.45620334201676,31.792055786355498]}},"favorite_feature":"Replication","email":"fiddle7@hotmail.com"} +{"_id":{"$oid":"5032a2c2fe4dce143835c32d"},"membership_status":"ACTIVE","name":"Andrew Wilson","phone_no":"+11774153168","last_login":{"$date":"2011-03-07T16:13:20.000Z"},"address":{"city":"Mesa, Arizona","street":"327 Olympic St","postal_code":"23996","country":"USA","location":{"type":"Point","coordinates":[-111.78826676616428,33.42673545202268]}},"favorite_feature":"Text Search","email":"dandy22@hotmail.com"} +{"_id":{"$oid":"5032c9effe4dce143735c442"},"membership_status":"ACTIVE","name":"Lucy U Martin","gender":"male","age":12,"phone_no":"+498384871227","last_login":{"$date":"2011-08-22T16:52:13.000Z"},"address":{"city":"Orlando, Florida","street":"82 South Edmunds Rd","postal_code":69727,"country":"USA","location":{"type":"Point","coordinates":[-81.3859647858629,28.521979342926883]}},"favorite_feature":"Wired Tiger","email":"shortcircuit81@comcast.net"} +{"_id":{"$oid":"5032edb1fe4dce143635cac3"},"membership_status":"ACTIVE","name":"Addison James","gender":"male","phone_no":"+1109044312","last_login":{"$date":"2014-03-05T13:44:52.000Z"},"address":{"city":"Columbus, Ohio","street":"334 Woodland Rd","postal_code":"72727","country":"USA","location":{"type":"Point","coordinates":[-83.03214264765697,39.913457135337374]}},"email":"mindprobe79@gmail.com"} +{"_id":{"$oid":"50331c53fe4dce143635c90a"},"membership_status":"ACTIVE","name":"Jacob White","gender":"male","age":29,"phone_no":"+16742335892","last_login":{"$date":"2010-09-27T08:07:21.000Z"},"address":{"city":"Riverside, California","street":"152 Northeast Reny St","postal_code":"50337","country":"Germany","location":{"type":"Point","coordinates":[-117.36917639593297,33.95725787096601]}},"email":"bliss35@comcast.net"} +{"_id":{"$oid":"5033325dfe4dce143735c81d"},"membership_status":"ACTIVE","name":"Ella Harris","gender":"male","age":43,"phone_no":"+14751027659","last_login":{"$date":"2010-12-19T09:50:38.000Z"},"address":{"city":"Boston, Massachusetts","street":"228 Perimeter Ave","postal_code":"77790","country":"USA","location":{"type":"Point","coordinates":[-71.06248839321384,42.36517500946935]}},"favorite_feature":"Document Validation","email":"sugarycake76@hotmail.com"} +{"_id":{"$oid":"50337271fe4dce143835c529"},"membership_status":"ACTIVE","name":"Sebastian C King","gender":"male","phone_no":"+18729615410","last_login":{"$date":"2014-04-28T09:51:46.000Z"},"address":{"city":"Charlotte, North Carolina","street":"296 Northeast Ring St","postal_code":39929,"country":"USA","location":{"type":"Point","coordinates":[-80.84749920829343,35.26558124444622]}},"favorite_feature":"Document Validation","email":"mencity14@verizon.net"} +{"_id":{"$oid":"5033ac9afe4dce143835c9ef"},"membership_status":"ACTIVE","name":"Zoey A James","phone_no":"+12275857418","last_login":{"$date":"2011-10-21T23:20:05.000Z"},"address":{"city":"Wichita, Kansas","street":"240 East Valley St","postal_code":"82275","country":"USA","location":{"type":"Point","coordinates":[-97.35401816658857,37.679287359830084]}},"favorite_feature":"Partial Indexes","email":"cybergypsy9@gmx.net"} +{"_id":{"$oid":"50341db4fe4dce143735c833"},"membership_status":"PENDING","name":"Jackson B Bell","gender":"female","phone_no":"+19688538381","last_login":{"$date":"2013-05-20T17:00:55.000Z"},"address":{"city":"Nashville-Davidson, Tennessee","street":"197 Birch St","postal_code":"82254","country":"USA","location":{"type":"Point","coordinates":[-86.73538620987506,36.151079202231024]}},"favorite_feature":"Aggregation","email":"mr.america12@yahoo.com"} +{"_id":{"$oid":"5034782ffe4dce143635c9c7"},"membership_status":"PENDING","name":"Ethan James","gender":"female","age":43,"phone_no":"+13103764856","last_login":{"$date":"2015-05-11T18:08:36.000Z"},"address":{"city":"Sacramento, California","street":"61 Southwest Dakota Ave","postal_code":"80362","country":"USA","location":{"type":"Point","coordinates":[-121.51720950611428,38.58973166161231]}},"email":"hypnosis57@comcast.net"} +{"_id":{"$oid":"5034d1d6fe4dce143735c1c4"},"membership_status":"INACTIVE","name":"Ryan Lee","gender":"male","phone_no":"+19345986970","last_login":{"$date":"2013-10-24T14:50:59.000Z"},"address":{"city":"Scottsdale, Arizona","street":"167 South Carver St","postal_code":"78105","country":"USA","location":{"type":"Point","coordinates":[-111.95440344669727,33.510098974798275]}},"favorite_feature":"Aggregation","email":"sugarycake98@msn.com"} +{"_id":{"$oid":"5034ef31fe4dce143535c313"},"membership_status":"ACTIVE","name":"Mason C Williams","gender":"male","age":49,"phone_no":"+611382078320","last_login":{"$date":"2011-12-06T15:08:12.000Z"},"address":{"city":"Memphis, Tennessee","street":"250 Northeast Larchmount St","postal_code":"55813","country":"Australia","location":{"type":"Point","coordinates":[-90.0590125637064,35.13977080908803]}},"favorite_feature":"Auth","email":"cyberking8@aim.com"} +{"_id":{"$oid":"50350163fe4dce143835ca46"},"membership_status":"PENDING","name":"Ava Watson","gender":"male","age":31,"phone_no":"+498289859715","last_login":{"$date":"2014-05-14T14:10:05.000Z"},"address":{"city":"San Diego, California","street":"156 Aloha St","postal_code":"87651","country":"USA","location":{"type":"Point","coordinates":[-117.15234182944589,32.70511752957363]}},"email":"dixie0@hotmail.com"} +{"_id":{"$oid":"503517c6fe4dce143635c2cf"},"membership_status":"PENDING","name":"Hannah Harris","gender":"female","age":35,"phone_no":"+16491667120","last_login":{"$date":"2015-02-02T08:47:46.000Z"},"address":{"city":"St. Paul, Minnesota","street":"116 Christensen Ave","postal_code":70051,"country":"USA","location":{"type":"Point","coordinates":[-93.11363402069591,44.93880912265563]}},"favorite_feature":"Document Validation","email":"jaycee87@gmail.com"} +{"_id":{"$oid":"503525aefe4dce143735c628"},"membership_status":"ACTIVE","name":"Gabriel Lewis","age":36,"phone_no":{"$numberLong":"5705326570"},"last_login":{"$date":"2013-04-01T19:41:53.000Z"},"address":{"city":"Fresno, California","street":"102 Northwest Spring Fork Rd","postal_code":"48866","country":"USA","location":{"type":"Point","coordinates":[-119.81614942751808,36.72190998438775]}},"email":"coolstrawberry18@gmx.net"} +{"_id":{"$oid":"50353108fe4dce143835c73a"},"membership_status":"PENDING","name":"Lucas T Scott","gender":"male","phone_no":{"$numberLong":"4638336714"},"last_login":{"$date":"2012-06-17T20:02:48.000Z"},"address":{"city":"Wichita, Kansas","street":"213 Union Oil Company Ln","postal_code":"82023","country":"USA","location":{"type":"Point","coordinates":[-97.34389510412187,37.694450426102286]}},"favorite_feature":"Document Validation","email":"broomspun45@icloud.com"} +{"_id":{"$oid":"50354f8afe4dce143535c72b"},"membership_status":"PENDING","name":"Ryan C Clark","gender":"female","age":47,"phone_no":"+17581816982","last_login":{"$date":"2013-06-12T04:27:02.000Z"},"address":{"city":"Kansas City, Missouri","street":"158 Northeast Ohio Ave","postal_code":22179,"country":"USA","location":{"type":"Point","coordinates":[-94.61018096397198,39.02076719209666]}},"favorite_feature":"Document Validation","email":"littletrout73@icloud.com"} +{"_id":{"$oid":"5035aabbfe4dce143735c3fe"},"membership_status":"ACTIVE","name":"Elijah Edwards","age":33,"phone_no":{"$numberLong":"9508380915"},"last_login":{"$date":"2012-09-25T21:16:35.000Z"},"address":{"city":"El Paso, Texas","street":"29 Northwest Elford Ln","postal_code":"72220","country":"USA","location":{"type":"Point","coordinates":[-106.38527383924198,31.771914753348636]}},"email":"mencity29@aim.com"} +{"_id":{"$oid":"5035add2fe4dce143635c4a5"},"membership_status":"ACTIVE","name":"Audrey O Phillips","gender":"female","phone_no":"+14238559669","last_login":{"$date":"2012-10-23T07:56:08.000Z"},"address":{"city":"Austin, Texas","street":"317 Homeview St","postal_code":"19525","country":"UK","location":{"type":"Point","coordinates":[-97.72969424683606,30.267998395327737]}},"email":"born-confused35@aim.com"} +{"_id":{"$oid":"5035b81cfe4dce143635c249"},"membership_status":"ACTIVE","name":"Oliver O Wilson","gender":"male","age":44,"phone_no":1444775294,"last_login":{"$date":"2014-07-08T03:18:58.000Z"},"address":{"city":"Tucson, Arizona","street":"148 Mallard Ln","postal_code":"63394","country":"UK","location":{"type":"Point","coordinates":[-110.92346399038425,32.24217493236621]}},"email":"cooldora15@verizon.net"} +{"_id":{"$oid":"5035bf1efe4dce143635c9a3"},"membership_status":"PENDING","name":"Samuel B Thompson","gender":"female","phone_no":"+14240959525","last_login":{"$date":"2013-04-23T22:18:46.000Z"},"address":{"city":"Omaha, Nebraska","street":"121 Tolt Rd","postal_code":18920,"country":"UK","location":{"type":"Point","coordinates":[-95.9882005313852,41.243236002762096]}},"favorite_feature":"Wired Tiger","email":"cookie93@icloud.com"} +{"_id":{"$oid":"503614c4fe4dce143835c347"},"membership_status":"ACTIVE","name":"Nora P Turner","gender":"female","phone_no":"+15238242224","last_login":{"$date":"2013-08-21T02:32:30.000Z"},"address":{"city":"New Orleans, Louisiana","street":"301 East Louisa Ln","postal_code":"98397","country":"USA","location":{"type":"Point","coordinates":[-90.1258060814679,29.949670711760216]}},"email":"gozmit56@gmail.com"} +{"_id":{"$oid":"50362919fe4dce143635c18a"},"membership_status":"ACTIVE","name":"Oliver Williams","gender":"female","age":34,"phone_no":"+13911696939","last_login":{"$date":"2011-06-30T00:49:33.000Z"},"address":{"city":"Colorado Springs, Colorado","street":"28 Colonial St","postal_code":"64215","country":"USA","location":{"type":"Point","coordinates":[-104.84322652446603,38.830924227214595]}},"email":"techbro68@comcast.net"} +{"_id":{"$oid":"50363146fe4dce143535c24a"},"membership_status":"ACTIVE","name":"Aiden F Watson","gender":"female","age":23,"phone_no":"+11798558233","last_login":{"$date":"2010-01-23T00:14:28.000Z"},"address":{"city":"Baton Rouge, Louisiana","street":"240 South Nebraska Rd","postal_code":"92026","country":"USA","location":{"type":"Point","coordinates":[-91.16401489428318,30.455760139092977]}},"favorite_feature":"Document Validation","email":"beloved27@yahoo.com"} +{"_id":{"$oid":"503631f2fe4dce143735c66c"},"membership_status":"ACTIVE","name":"Caleb Z Robinson","age":33,"phone_no":"+18309799235","last_login":{"$date":"2014-11-11T13:51:03.000Z"},"address":{"city":"Toledo, Ohio","street":"5 South Nebraska Rd","postal_code":"50946","country":"France","location":{"type":"Point","coordinates":[-83.55678741720347,41.68113909282523]}},"email":"brownie34@msn.com"} +{"_id":{"$oid":"5036425afe4dce143535c32e"},"membership_status":"ACTIVE","name":"Avery Hall","gender":"female","phone_no":"+447393926788","last_login":{"$date":"2012-02-01T10:31:27.000Z"},"address":{"city":"Tampa, Florida","street":"395 Whitman Rd","postal_code":20322,"country":"France","location":{"type":"Point","coordinates":[-82.44397018158193,27.907286098758533]}},"favorite_feature":"Document Validation","email":"braindead97@hotmail.com"} +{"_id":{"$oid":"503678c6fe4dce143635c79b"},"membership_status":"PENDING","name":"Madison Clark","phone_no":"+1909542687","last_login":{"$date":"2013-10-03T05:45:25.000Z"},"address":{"city":"Lincoln, Nebraska","street":"212 Erben Rd","postal_code":"34224","country":"USA","location":{"type":"Point","coordinates":[-96.72098855282341,40.79275323018288]}},"email":"couldnt_find_good_name79@yahoo.com"} +{"_id":{"$oid":"5036fc0afe4dce143535c891"},"membership_status":"ACTIVE","name":"James Young","gender":"female","age":52,"phone_no":"+18770127752","last_login":{"$date":"2012-12-14T08:15:53.000Z"},"address":{"city":"Riverside, California","street":"374 Southwest Sullivan Ln","postal_code":"46994","country":"USA","location":{"type":"Point","coordinates":[-117.38986103778586,33.92300743235786]}},"favorite_feature":"Partial Indexes","email":"fiddle29@verizon.net"} +{"_id":{"$oid":"50370aaefe4dce143735c679"},"membership_status":"INACTIVE","name":"Logan Phillips","gender":"male","age":23,"phone_no":1272641335,"last_login":{"$date":"2011-02-16T02:11:10.000Z"},"address":{"city":"Oklahoma City, Oklahoma","street":"276 Crescent Lake Rd","postal_code":"48963","country":"USA","location":{"type":"Point","coordinates":[-97.47815616033876,35.41129245670654]}},"favorite_feature":"Aggregation","email":"takeaway34@verizon.net"} +{"_id":{"$oid":"50372c54fe4dce143835ca20"},"membership_status":"ACTIVE","name":"Ethan G Turner","gender":"male","age":37,"phone_no":"+44654308202","last_login":{"$date":"2014-07-04T02:37:43.000Z"},"address":{"city":"Austin, Texas","street":"325 Alexander Rd","postal_code":"71696","country":"USA","location":{"type":"Point","coordinates":[-97.7120214873177,30.21646994084725]}},"favorite_feature":"Partial Indexes","email":"lilokie78@aim.com"} +{"_id":{"$oid":"503743edfe4dce143635ca6f"},"membership_status":"ACTIVE","name":"Emma P Lewis","gender":"female","age":25,"phone_no":"+13612660749","last_login":{"$date":"2014-10-03T11:02:11.000Z"},"address":{"city":"Chesapeake, Virginia","street":"63 Southwest Jacobsen Rd","postal_code":31117,"country":"USA","location":{"type":"Point","coordinates":[-76.25802025410239,36.77630401530377]}},"favorite_feature":"Document Validation","email":"hockeywain66@comcast.net"} +{"_id":{"$oid":"50374ee3fe4dce143835c369"},"membership_status":"ACTIVE","name":"Logan Anderson","gender":"female","phone_no":{"$numberLong":"7886157051"},"last_login":{"$date":"2010-12-27T02:23:02.000Z"},"address":{"city":"Chicago, Illinois","street":"72 Northeast Mary Lou St","postal_code":"78647-4033","country":"USA","location":{"type":"Point","coordinates":[-87.64403496388532,41.917318177067]}},"favorite_feature":"Auth","email":"awesomegirl90@gmx.net"} +{"_id":{"$oid":"503766ccfe4dce143535c716"},"membership_status":"ACTIVE","name":"Sebastian Taylor","gender":"female","age":30,"phone_no":"+11264294482","last_login":{"$date":"2013-09-19T12:30:54.000Z"},"address":{"city":"El Paso, Texas","street":"107 Southwest Francis Rd","postal_code":"97603","country":"UK","location":{"type":"Point","coordinates":[-106.49261738244712,31.805160294856076]}},"favorite_feature":"Document Validation","email":"gozmit91@icloud.com"} +{"_id":{"$oid":"50378103fe4dce143735c939"},"membership_status":"PENDING","name":"Owen U Lewis","gender":"female","age":44,"phone_no":"+445622359175","last_login":{"$date":"2013-12-03T04:48:40.000Z"},"address":{"city":"Reno, Nevada","street":"247 Southwest Normandy Pkwy","postal_code":15907,"country":"USA","location":{"type":"Point","coordinates":[-119.81237626153762,39.57997164096322]}},"favorite_feature":"Sharding","email":"cybergypsy26@hotmail.com"} +{"_id":{"$oid":"5037ab4cfe4dce143535c20b"},"membership_status":"PENDING","name":"Matthew L Hughes","gender":"male","age":39,"phone_no":"+11297282707","last_login":{"$date":"2010-08-15T02:36:45.000Z"},"address":{"city":"Arlington, Texas","street":"257 Northeast White Horse Rd","postal_code":"36444","country":"USA","location":{"type":"Point","coordinates":[-97.08165811415708,32.7476245290993]}},"favorite_feature":"Wired Tiger","email":"awesomegirl68@yahoo.com"} +{"_id":{"$oid":"5037c02cfe4dce143835c2b0"},"membership_status":"ACTIVE","name":"Dylan G Davis","phone_no":"+451104340553","last_login":{"$date":"2010-04-02T16:46:57.000Z"},"address":{"city":"Milwaukee, Wisconsin","street":"188 Terrace St","postal_code":"59470","country":"USA","location":{"type":"Point","coordinates":[-87.9166769189508,43.0732589209909]}},"favorite_feature":"Aggregation","email":"coolwhip11@gmx.net"} +{"_id":{"$oid":"5038287dfe4dce143635c82c"},"membership_status":"PENDING","name":"Landon K Thompson","gender":"female","phone_no":"+18052943426","last_login":{"$date":"2010-05-10T12:04:46.000Z"},"address":{"city":"Laredo, Texas","street":"90 South Rainbow Rd","postal_code":92861,"country":"UK","location":{"type":"Point","coordinates":[-99.48901986890544,27.53698988541721]}},"favorite_feature":"Document Validation","email":"funkymoney32@yahoo.com"} +{"_id":{"$oid":"5038419efe4dce143735c7e0"},"membership_status":"PENDING","name":"Scarlett N Anderson","gender":"male","age":42,"phone_no":"+19824671192","last_login":{"$date":"2010-09-19T14:18:20.000Z"},"address":{"city":"Tulsa, Oklahoma","street":"228 Tye River Ave","postal_code":12337,"country":"USA","location":{"type":"Point","coordinates":[-95.9594459276981,36.173443213456366]}},"email":"neurotic33@aim.com"} +{"_id":{"$oid":"5038662ffe4dce143835c319"},"membership_status":"ACTIVE","name":"Emma Z Harris","gender":"male","phone_no":"+15102966031","last_login":{"$date":"2013-06-17T03:56:30.000Z"},"address":{"city":"San Francisco, California","street":"195 Baker Rd","postal_code":"46954","country":"USA","location":{"type":"Point","coordinates":[-122.45504480822119,37.78345435889993]}},"email":"coolstrawberry57@icloud.com"} +{"_id":{"$oid":"503872d4fe4dce143735c155"},"membership_status":"ACTIVE","name":"Aubrey Z Lewis","gender":"female","phone_no":"+12931558514","last_login":{"$date":"2013-04-29T04:44:29.000Z"},"address":{"city":"Denver, Colorado","street":"197 Northeast Ocean Rd","postal_code":"32469","country":"USA","location":{"type":"Point","coordinates":[-105.04661760670464,39.73991078943252]}},"favorite_feature":"Auth","email":"heavenlybeliever67@msn.com"} +{"_id":{"$oid":"5038758dfe4dce143835c1b5"},"membership_status":"ACTIVE","name":"James C Johnson","gender":"female","phone_no":"+445979910409","last_login":{"$date":"2013-08-05T07:36:50.000Z"},"address":{"city":"Chesapeake, Virginia","street":"115 Northwest Firwood Ln","postal_code":"98778","country":"USA","location":{"type":"Point","coordinates":[-76.3289057935534,36.767456583381815]}},"email":"bubbles78@gmx.net"} +{"_id":{"$oid":"503883fafe4dce143635c60e"},"membership_status":"ACTIVE","name":"Grace B Robinson","gender":"female","phone_no":"+12018846177","last_login":{"$date":"2013-01-28T23:18:55.000Z"},"address":{"city":"Boston, Massachusetts","street":"243 Andover Ln","postal_code":"91337","country":"USA","location":{"type":"Point","coordinates":[-71.00580541060702,42.273106022754966]}},"favorite_feature":"Wired Tiger","email":"override56@icloud.com"} +{"_id":{"$oid":"5038af20fe4dce143535c4d0"},"membership_status":"ACTIVE","name":"Addison K Roberts","gender":"male","phone_no":"+444202064667","last_login":{"$date":"2014-07-05T05:38:41.000Z"},"address":{"city":"Lexington-Fayette, Kentucky","street":"328 Hanna Park Rd","postal_code":"38847","country":"Australia","location":{"type":"Point","coordinates":[-84.51005328061954,38.033158533515454]}},"email":"crashtest8@msn.com"} +{"_id":{"$oid":"5038cd94fe4dce143535c2f7"},"membership_status":"ACTIVE","name":"Jayden Brown","gender":"female","age":39,"phone_no":"+13005615088","last_login":{"$date":"2011-07-28T18:21:31.000Z"},"address":{"city":"Lincoln, Nebraska","street":"173 Skagit Ave","postal_code":59448,"country":"USA","location":{"type":"Point","coordinates":[-96.68606033674155,40.823391244737735]}},"favorite_feature":"Partial Indexes","email":"bullfrog67@gmail.com"} +{"_id":{"$oid":"5038f20ffe4dce143735c5f5"},"membership_status":"ACTIVE","name":"Harper M Hall","age":36,"phone_no":{"$numberLong":"6012853977"},"last_login":{"$date":"2013-06-28T09:55:48.000Z"},"address":{"city":"New York, New York","street":"397 Homeview St","postal_code":"77008-7338","country":"USA","location":{"type":"Point","coordinates":[-74.01711966985118,40.69069738086477]}},"favorite_feature":"Sharding","email":"angelwonderland13@comcast.net"} +{"_id":{"$oid":"5038f3d4fe4dce143635c8e9"},"membership_status":"PENDING","name":"Jacob C Allen","gender":"female","phone_no":"+16290756822","last_login":{"$date":"2010-04-13T21:44:47.000Z"},"address":{"city":"El Paso, Texas","street":"101 Fern Hollow Ave","postal_code":"52129","country":"USA","location":{"type":"Point","coordinates":[-106.44942793972183,31.72755655037713]}},"favorite_feature":"Sharding","email":"chiquitita85@gmail.com"} +{"_id":{"$oid":"503911cbfe4dce143535c556"},"membership_status":"ACTIVE","name":"Carter H Green","gender":"male","age":27,"phone_no":{"$numberLong":"4146224337"},"last_login":{"$date":"2011-03-08T09:34:19.000Z"},"address":{"city":"Baton Rouge, Louisiana","street":"283 Hedlund Rd","postal_code":"25371-2506","country":"USA","location":{"type":"Point","coordinates":[-91.1410148115081,30.4462164714689]}},"favorite_feature":"Sharding","email":"brownie79@comcast.net"} +{"_id":{"$oid":"503986e4fe4dce143835c15b"},"membership_status":"ACTIVE","name":"Matthew P Scott","gender":"male","age":29,"phone_no":"+445428835071","last_login":{"$date":"2013-04-23T16:14:30.000Z"},"address":{"city":"Fort Wayne, Indiana","street":"193 South Tillicum Ln","postal_code":"26305","country":"USA","location":{"type":"Point","coordinates":[-85.14886341838589,41.05923884850483]}},"email":"delicious72@gmx.net"} +{"_id":{"$oid":"50399796fe4dce143835c3fd"},"membership_status":"ACTIVE","name":"Ryan C Thompson","age":40,"phone_no":"+17725961350","last_login":{"$date":"2012-07-21T02:42:21.000Z"},"address":{"city":"Chicago, Illinois","street":"8 Woodlake St","postal_code":84957,"country":"Germany","location":{"type":"Point","coordinates":[-87.5908109568315,41.90314381173272]}},"favorite_feature":"Sharding","email":"happinesskingdom2@comcast.net"} +{"_id":{"$oid":"5039e165fe4dce143735ca01"},"membership_status":"ACTIVE","name":"Samuel Hall","gender":"male","phone_no":"+613118264437","last_login":{"$date":"2011-03-17T19:33:41.000Z"},"address":{"city":"Jacksonville, Florida","street":"398 South Redwing St","postal_code":"24576","country":"USA","location":{"type":"Point","coordinates":[-81.63928472883148,30.319041086004702]}},"favorite_feature":"Wired Tiger","email":"beauty36@comcast.net"} +{"_id":{"$oid":"5039f325fe4dce143635caf3"},"membership_status":"ACTIVE","name":"Aria B James","gender":"female","age":42,"phone_no":"+448099839354","last_login":{"$date":"2015-05-22T16:08:54.000Z"},"address":{"city":"Boston, Massachusetts","street":"329 Northeast Oddfellows St","postal_code":"81320-4562","country":"USA","location":{"type":"Point","coordinates":[-71.05506508831115,42.388184299141216]}},"email":"override73@comcast.net"} +{"_id":{"$oid":"5039f3a6fe4dce143835c6b1"},"membership_status":"PENDING","name":"Scarlett Robinson","phone_no":{"$numberLong":"4322399992"},"last_login":{"$date":"2010-12-24T15:15:18.000Z"},"address":{"city":"Las Vegas, Nevada","street":"184 West Barrett Ln","postal_code":"11790-4943","country":"UK","location":{"type":"Point","coordinates":[-115.18518799507171,36.15646878805473]}},"favorite_feature":"Aggregation","email":"max78@gmx.net"} +{"_id":{"$oid":"503a0c42fe4dce143835cad0"},"membership_status":"PENDING","name":"Owen M Ward","gender":"male","phone_no":{"$numberLong":"2613305652"},"last_login":{"$date":"2012-07-20T06:12:21.000Z"},"address":{"city":"Modesto, California","street":"148 Oregon Rd","postal_code":"23280","country":"USA","location":{"type":"Point","coordinates":[-121.00074370342853,37.661484144467074]}},"favorite_feature":"Document Validation","email":"born2pizza89@verizon.net"} +{"_id":{"$oid":"503a2d71fe4dce143535c968"},"membership_status":"ACTIVE","name":"Daniel Jackson","gender":"female","phone_no":"+1231970440","last_login":{"$date":"2013-04-01T17:09:31.000Z"},"address":{"city":"Plano, Texas","street":"166 Skagit Rd","postal_code":61737,"country":"USA","location":{"type":"Point","coordinates":[-96.67194563553288,33.051754178837186]}},"email":"buttercup78@hotmail.com"} +{"_id":{"$oid":"503a3992fe4dce143635c23c"},"membership_status":"ACTIVE","name":"Lillian N Morgan","gender":"male","age":32,"phone_no":"+444233082982","last_login":{"$date":"2013-10-25T23:40:19.000Z"},"address":{"city":"Chula Vista, California","street":"305 Southwest Orchard Ave","postal_code":"33875","country":"USA","location":{"type":"Point","coordinates":[-117.08959008455346,32.637427567346386]}},"favorite_feature":"Wired Tiger","email":"wokie46@gmx.net"} +{"_id":{"$oid":"503a5ecafe4dce143835c173"},"membership_status":"ACTIVE","name":"Lillian F Watson","gender":"male","age":36,"phone_no":"+18106420364","last_login":{"$date":"2010-08-21T16:51:45.000Z"},"address":{"city":"Reno, Nevada","street":"302 Northeast Miller St","postal_code":"12985","country":"USA","location":{"type":"Point","coordinates":[-119.83369942991055,39.49746754215054]}},"favorite_feature":"Partial Indexes","email":"lamb48@verizon.net"} +{"_id":{"$oid":"503a6039fe4dce143635c6f7"},"membership_status":"ACTIVE","name":"Logan E Morgan","gender":"female","age":30,"phone_no":"+14717544344","last_login":{"$date":"2010-05-15T20:32:02.000Z"},"address":{"city":"Henderson, Nevada","street":"82 Northwest Sammamish St","postal_code":24722,"country":"USA","location":{"type":"Point","coordinates":[-114.99377231450256,36.061844769494364]}},"favorite_feature":"Partial Indexes","email":"cyberwarrior31@hotmail.com"} +{"_id":{"$oid":"503a6992fe4dce143535c92b"},"membership_status":"ACTIVE","name":"Leah U Allen","gender":"male","age":32,"phone_no":"+15032549920","last_login":{"$date":"2013-07-21T22:14:04.000Z"},"address":{"city":"Portland, Oregon","street":"177 Robin Hood Rd","postal_code":"57759","country":"USA","location":{"type":"Point","coordinates":[-122.66916448512227,45.557548287657134]}},"favorite_feature":"Document Validation","email":"braindead24@verizon.net"} +{"_id":{"$oid":"503aa5e9fe4dce143835c9b2"},"membership_status":"ACTIVE","name":"Amelie D Mitchell","gender":"female","age":46,"phone_no":1283342351,"last_login":{"$date":"2011-08-30T14:30:37.000Z"},"address":{"city":"Washington, District of Columbia","street":"64 Prospect Rd","postal_code":"21301","country":"France","location":{"type":"Point","coordinates":[-76.9947302232287,38.884057570681485]}},"email":"exotica10@msn.com"} +{"_id":{"$oid":"503ac400fe4dce143735c1d6"},"membership_status":"ACTIVE","name":"Mia B Morgan","gender":"female","phone_no":"+14530966355","last_login":{"$date":"2013-04-09T16:22:31.000Z"},"address":{"city":"Sacramento, California","street":"106 East Commercial Rd","postal_code":"86084","country":"USA","location":{"type":"Point","coordinates":[-121.43642684095471,38.53404730943275]}},"favorite_feature":"Document Validation","email":"hangman40@gmx.net"} +{"_id":{"$oid":"503ad4fffe4dce143535ca4b"},"membership_status":"ACTIVE","name":"Scarlett M Phillips","gender":"female","age":37,"phone_no":"+497348601728","last_login":{"$date":"2010-09-18T23:50:12.000Z"},"address":{"city":"Tampa, Florida","street":"353 Entwistle St","postal_code":"33035","country":"USA","location":{"type":"Point","coordinates":[-82.49890668718594,27.875646539283746]}},"email":"cheekygirl84@comcast.net"} +{"_id":{"$oid":"503b01b7fe4dce143835c66d"},"membership_status":"PENDING","name":"Avery H Hughes","gender":"female","age":34,"phone_no":"+18516876471","last_login":{"$date":"2011-02-06T00:40:37.000Z"},"address":{"city":"Sacramento, California","street":"4 South Bradford St","postal_code":49836,"country":"USA","location":{"type":"Point","coordinates":[-121.49176171143955,38.58426237703769]}},"email":"dixie29@verizon.net"} +{"_id":{"$oid":"503b0ad2fe4dce143835c79e"},"membership_status":"ACTIVE","name":"Harper Smith","gender":"male","age":17,"phone_no":{"$numberLong":"8825640869"},"last_login":{"$date":"2014-04-17T01:41:13.000Z"},"address":{"city":"Stockton, California","street":"388 Northeast Meadowmeer St","postal_code":"67876","country":"USA","location":{"type":"Point","coordinates":[-121.30857723467344,37.97817618770525]}},"email":"coolguy48@msn.com"} +{"_id":{"$oid":"503b12f7fe4dce143535c318"},"membership_status":"ACTIVE","name":"Avery Martin","gender":"female","age":43,"phone_no":"+16700326504","last_login":{"$date":"2013-07-02T02:50:57.000Z"},"address":{"city":"Birmingham, Alabama","street":"383 Southwest Lisabuela St","postal_code":11988,"country":"USA","location":{"type":"Point","coordinates":[-86.86582181058897,33.487512379704626]}},"favorite_feature":"Sharding","email":"brads62@yahoo.com"} +{"_id":{"$oid":"503b1bb1fe4dce143535c8e3"},"membership_status":"ACTIVE","name":"Carter K James","gender":"female","age":46,"phone_no":"+615078400838","last_login":{"$date":"2010-09-01T10:55:39.000Z"},"address":{"city":"Boston, Massachusetts","street":"397 South Glacier St","postal_code":"66192","country":"USA","location":{"type":"Point","coordinates":[-71.10961286815105,42.374431032294524]}},"email":"beautiful64@comcast.net"} +{"_id":{"$oid":"503b290ffe4dce143835c147"},"membership_status":"ACTIVE","name":"Ryan Patel","gender":"male","phone_no":"+17438917865","last_login":{"$date":"2014-11-20T12:00:33.000Z"},"address":{"city":"Tulsa, Oklahoma","street":"361 Fairweather St","postal_code":"54522","country":"USA","location":{"type":"Point","coordinates":[-95.96451471507615,36.14748935864255]}},"favorite_feature":"Wired Tiger","email":"sugarypie57@verizon.net"} +{"_id":{"$oid":"503b4be4fe4dce143835cacc"},"membership_status":"PENDING","name":"Ellie Thompson","gender":"female","phone_no":"+17518179206","last_login":{"$date":"2012-05-05T08:03:06.000Z"},"address":{"city":"Las Vegas, Nevada","street":"88 Main Blvd","postal_code":"55245","country":"UK","location":{"type":"Point","coordinates":[-115.05775753288418,36.25014524785538]}},"email":"corefinder3@yahoo.com"} +{"_id":{"$oid":"503b4cb5fe4dce143535c3c5"},"membership_status":"ACTIVE","name":"Victoria B Wilson","gender":"male","age":51,"phone_no":"+14108799192","last_login":{"$date":"2011-06-06T08:18:39.000Z"},"address":{"city":"Columbus, Ohio","street":"198 South Taft St","postal_code":"83136","country":"USA","location":{"type":"Point","coordinates":[-82.9785189467614,39.92755073919848]}},"email":"fiddle59@hotmail.com"} +{"_id":{"$oid":"503b5c05fe4dce143535c89c"},"membership_status":"ACTIVE","name":"Sofia Scott","gender":"female","age":30,"phone_no":"+17566267862","last_login":{"$date":"2015-04-08T18:39:16.000Z"},"address":{"city":"Las Vegas, Nevada","street":"345 Driftwood St","postal_code":"52317","country":"USA","location":{"type":"Point","coordinates":[-115.10721160033528,36.16309462623908]}},"favorite_feature":"Text Search","email":"exotica92@verizon.net"} +{"_id":{"$oid":"503ba46ffe4dce143835c7fd"},"membership_status":"PENDING","name":"David Wood","gender":"male","age":35,"phone_no":"+452641424347","last_login":{"$date":"2013-08-17T01:14:50.000Z"},"address":{"city":"Boston, Massachusetts","street":"51 South Bayview Pkwy","postal_code":"97362","country":"USA","location":{"type":"Point","coordinates":[-71.11036616565663,42.325196359227974]}},"email":"entertain-me32@icloud.com"} +{"_id":{"$oid":"503bb442fe4dce143735c517"},"membership_status":"PENDING","name":"Sofia C Lewis","gender":"male","age":16,"phone_no":"+15615547175","last_login":{"$date":"2010-07-21T10:13:14.000Z"},"address":{"city":"Lubbock, Texas","street":"258 Northeast Monsaas St","postal_code":85936,"country":"USA","location":{"type":"Point","coordinates":[-101.87915639556758,33.56066575051059]}},"favorite_feature":"Document Validation","email":"cutie4@hotmail.com"} +{"_id":{"$oid":"503bb4edfe4dce143635c93b"},"membership_status":"ACTIVE","name":"Aubrey Jackson","gender":"female","age":39,"phone_no":"+443466760466","last_login":{"$date":"2012-05-04T18:46:38.000Z"},"address":{"city":"Stockton, California","street":"206 Southwest Monroe Ln","postal_code":"46020","country":"USA","location":{"type":"Point","coordinates":[-121.26483150067776,38.016984803522384]}},"favorite_feature":"Document Validation","email":"cutie31@verizon.net"} +{"_id":{"$oid":"503c0411fe4dce143635cae6"},"membership_status":"ACTIVE","name":"Isabella F Edwards","gender":"female","phone_no":"+446173137697","last_login":{"$date":"2013-08-17T18:45:38.000Z"},"address":{"city":"Baton Rouge, Louisiana","street":"374 Valley View St","postal_code":"68015","country":"France","location":{"type":"Point","coordinates":[-91.13453114306226,30.416559952695952]}},"favorite_feature":"Wired Tiger","email":"delicious45@aim.com"} +{"_id":{"$oid":"503c0c5ffe4dce143535c836"},"membership_status":"PENDING","name":"Sebastian Davies","gender":"male","phone_no":"+12660424548","last_login":{"$date":"2010-05-31T17:50:56.000Z"},"address":{"city":"Plano, Texas","street":"174 West Thomas Pkwy","postal_code":"70038","country":"USA","location":{"type":"Point","coordinates":[-96.6953271803223,33.032081774581656]}},"favorite_feature":"Document Validation","email":"happysky25@comcast.net"} +{"_id":{"$oid":"503c2376fe4dce143635c28a"},"membership_status":"ACTIVE","name":"Amelie J Robinson","gender":"female","age":40,"phone_no":"+19744516860","last_login":{"$date":"2011-07-22T12:30:43.000Z"},"address":{"city":"Huntington, New York","street":"338 Observation Rd","postal_code":"11950","country":"USA","location":{"type":"Point","coordinates":[-73.4446279457308,40.89674015263909]}},"favorite_feature":"Aggregation","email":"blossom86@gmail.com"} +{"_id":{"$oid":"503c35d0fe4dce143535c340"},"membership_status":"ACTIVE","name":"Luke W Allen","gender":"male","phone_no":"+496367386048","last_login":{"$date":"2011-02-13T02:36:54.000Z"},"address":{"city":"Omaha, Nebraska","street":"172 Northwest Far Country Ln","postal_code":"28762","country":"USA","location":{"type":"Point","coordinates":[-95.99510478645988,41.27130441732978]}},"favorite_feature":"Partial Indexes","email":"manicpsycho28@yahoo.com"} +{"_id":{"$oid":"503c6720fe4dce143735c7ab"},"membership_status":"ACTIVE","name":"David King","gender":"female","phone_no":"+14304837434","last_login":{"$date":"2013-04-11T21:00:09.000Z"},"address":{"city":"Rochester, New York","street":"33 Westlake St","postal_code":"77991","country":"USA","location":{"type":"Point","coordinates":[-77.6663650168254,43.14447753179987]}},"favorite_feature":"Auth","email":"born-confused54@aim.com"} +{"_id":{"$oid":"503cb09dfe4dce143635c835"},"membership_status":"ACTIVE","name":"Jack N Hill","gender":"female","age":49,"phone_no":"+19430658243","last_login":{"$date":"2012-02-09T08:37:00.000Z"},"address":{"city":"Fort Worth, Texas","street":"380 East Mc Gilvra Pkwy","postal_code":"17225","country":"USA","location":{"type":"Point","coordinates":[-97.31943062253647,32.70516207948639]}},"favorite_feature":"Sharding","email":"dogbone36@hotmail.com"} +{"_id":{"$oid":"503d09bdfe4dce143735c8c7"},"membership_status":"PENDING","name":"Lillian L Allen","gender":"female","phone_no":{"$numberLong":"2263259627"},"last_login":{"$date":"2013-05-09T16:01:52.000Z"},"address":{"city":"Tampa, Florida","street":"48 East St","postal_code":"15360","country":"UK","location":{"type":"Point","coordinates":[-82.46544367437033,27.96402082116518]}},"email":"heavenlybeliever44@verizon.net"} +{"_id":{"$oid":"503d1fd8fe4dce143735cac4"},"membership_status":"ACTIVE","name":"Jack V Clarke","age":33,"phone_no":"+13575306196","last_login":{"$date":"2013-03-06T14:19:58.000Z"},"address":{"city":"Glendale, Arizona","street":"164 S Raymond Rd","postal_code":"54344","country":"USA","location":{"type":"Point","coordinates":[-112.18110072377546,33.50037945391298]}},"favorite_feature":"Text Search","email":"jaycee13@verizon.net"} +{"_id":{"$oid":"503d250afe4dce143535ca17"},"membership_status":"PENDING","name":"Audrey Morgan","gender":"male","phone_no":"+11340017394","last_login":{"$date":"2011-12-23T16:19:19.000Z"},"address":{"city":"Arlington, Virginia","street":"228 Slater Rd","postal_code":37862,"country":"USA","location":{"type":"Point","coordinates":[-77.04282363427767,38.87417702609198]}},"favorite_feature":"Sharding","email":"deathnut91@comcast.net"} +{"_id":{"$oid":"503d29d5fe4dce143535c995"},"membership_status":"PENDING","name":"Olivia S Smith","gender":"female","phone_no":"+17398485292","last_login":{"$date":"2014-04-18T01:45:48.000Z"},"address":{"city":"Newark, New Jersey","street":"233 South Corgiat Rd","postal_code":87499,"country":"USA","location":{"type":"Point","coordinates":[-74.19361383949821,40.706032965635124]}},"email":"theprophet92@gmail.com"} +{"_id":{"$oid":"503d5149fe4dce143635c8d0"},"membership_status":"ACTIVE","name":"Daniel L Cooper","gender":"female","age":47,"phone_no":"+15103711432","last_login":{"$date":"2011-03-05T18:18:41.000Z"},"address":{"city":"Modesto, California","street":"358 Bogny Rd","postal_code":"54889-1361","country":"USA","location":{"type":"Point","coordinates":[-121.00586018229355,37.67783487422151]}},"email":"enchanted96@gmx.net"} +{"_id":{"$oid":"503d5342fe4dce143635c36d"},"membership_status":"ACTIVE","name":"Isabella O Roberts","gender":"female","age":46,"phone_no":"+11043160480","last_login":{"$date":"2011-07-26T01:00:19.000Z"},"address":{"city":"Fort Wayne, Indiana","street":"226 Saint Andrew St","postal_code":16610,"country":"USA","location":{"type":"Point","coordinates":[-85.1563327927226,41.068453483114865]}},"favorite_feature":"Auth","email":"soultaker7@comcast.net"} +{"_id":{"$oid":"503d6d1cfe4dce143635c518"},"membership_status":"PENDING","name":"Ellie Thomas","phone_no":"+15875624428","last_login":{"$date":"2010-01-18T11:05:52.000Z"},"address":{"city":"St. Petersburg, Florida","street":"19 Valley Greene St","postal_code":"61361","country":"USA","location":{"type":"Point","coordinates":[-82.61392915925177,27.72042178110669]}},"favorite_feature":"Partial Indexes","email":"buttercup12@aim.com"} +{"_id":{"$oid":"503d94ccfe4dce143835c859"},"membership_status":"ACTIVE","name":"Evelyn Green","gender":"female","phone_no":1623174675,"last_login":{"$date":"2011-01-14T08:48:10.000Z"},"address":{"city":"Plano, Texas","street":"29 West Plymouth Ave","postal_code":71530,"country":"UK","location":{"type":"Point","coordinates":[-96.75408243594752,32.983625650168875]}},"email":"princess82@gmx.net"} +{"_id":{"$oid":"503db4eafe4dce143535ca5a"},"membership_status":"ACTIVE","name":"Matthew P Brown","phone_no":"+11572864401","last_login":{"$date":"2011-01-14T05:51:17.000Z"},"address":{"city":"Dallas, Texas","street":"115 Daley Rd","postal_code":"98752","country":"USA","location":{"type":"Point","coordinates":[-96.83882843334047,32.834341406278085]}},"favorite_feature":"Document Validation","email":"girlydarkhorse72@icloud.com"} +{"_id":{"$oid":"503db5a9fe4dce143635c1c8"},"membership_status":"ACTIVE","name":"Sofia Davies","gender":"female","age":40,"phone_no":"+499224943326","last_login":{"$date":"2014-09-17T09:02:03.000Z"},"address":{"city":"Portland, Oregon","street":"383 Northeast Roberts Pkwy","postal_code":"30799","country":"USA","location":{"type":"Point","coordinates":[-122.69747685630946,45.51563791543689]}},"favorite_feature":"Document Validation","email":"battledoom48@hotmail.com"} +{"_id":{"$oid":"503df455fe4dce143635c8d9"},"membership_status":"ACTIVE","name":"Isaac L Edwards","gender":"female","age":37,"phone_no":"+18941414539","last_login":{"$date":"2013-12-10T07:22:19.000Z"},"address":{"city":"Portland, Oregon","street":"12 Dellwood St","postal_code":"77489","country":"USA","location":{"type":"Point","coordinates":[-122.69333925736352,45.46897176755713]}},"favorite_feature":"Partial Indexes","email":"swampmasher72@verizon.net"} +{"_id":{"$oid":"503e4fa0fe4dce143735c377"},"membership_status":"ACTIVE","name":"Brooklyn L Lewis","gender":"female","phone_no":"+19956277557","last_login":{"$date":"2011-10-06T13:08:54.000Z"},"address":{"city":"Arlington, Virginia","street":"87 Triland Ln","postal_code":"66952","country":"Germany","location":{"type":"Point","coordinates":[-77.16704297630906,38.8399002683331]}},"favorite_feature":"Document Validation","email":"dollyhitchhiker18@gmx.net"} +{"_id":{"$oid":"503e539afe4dce143535caef"},"membership_status":"ACTIVE","name":"Nora U Clarke","gender":"male","age":32,"phone_no":"+18545273552","last_login":{"$date":"2012-10-03T20:38:41.000Z"},"address":{"city":"Baltimore, Maryland","street":"393 West Lawton St","postal_code":"53525","country":"USA","location":{"type":"Point","coordinates":[-76.59382285943863,39.275085894191044]}},"favorite_feature":"Auth","email":"battledoom9@yahoo.com"} +{"_id":{"$oid":"503e77c9fe4dce143535c639"},"membership_status":"ACTIVE","name":"Isaac Allen","gender":"female","phone_no":"+15398291195","last_login":{"$date":"2014-08-09T06:25:19.000Z"},"address":{"city":"Birmingham, Alabama","street":"297 South Spokane Ln","postal_code":"78351","country":"UK","location":{"type":"Point","coordinates":[-86.80728835562441,33.57523144252437]}},"favorite_feature":"Document Validation","email":"gozmit95@gmx.net"} +{"_id":{"$oid":"503e82b0fe4dce143535c5df"},"membership_status":"ACTIVE","name":"David J Thomas","gender":"female","phone_no":{"$numberLong":"4134701337"},"last_login":{"$date":"2010-03-30T09:12:51.000Z"},"address":{"city":"Denver, Colorado","street":"120 Northeast Maple St","postal_code":"38462","country":"UK","location":{"type":"Point","coordinates":[-104.97010545365144,39.74992324364473]}},"email":"dogbone54@verizon.net"} +{"_id":{"$oid":"503e930bfe4dce143835c8f8"},"membership_status":"ACTIVE","name":"Gabriel U Lee","gender":"female","age":39,"phone_no":"+16297373635","last_login":{"$date":"2014-10-05T04:08:16.000Z"},"address":{"city":"Columbus, Ohio","street":"266 East Spring Ave","postal_code":16643,"country":"UK","location":{"type":"Point","coordinates":[-82.95153341097446,39.95414671849552]}},"email":"cozybutton100@aim.com"} diff --git a/packages/mongodb-schema/examples/parse-from-file.ts b/packages/mongodb-schema/examples/parse-from-file.ts new file mode 100644 index 000000000..c0c235401 --- /dev/null +++ b/packages/mongodb-schema/examples/parse-from-file.ts @@ -0,0 +1,58 @@ +/* eslint no-console: 0 */ + +import { pipeline as callbackPipeline, PassThrough, Transform } from 'stream'; +import path from 'path'; +import fs from 'fs'; +import { promisify } from 'util'; +import { parseSchema } from '../src'; + +const schemaFileName = path.join(__dirname, './fanclub.json'); + +function createFileStreamLineParser() { + let currentLine: undefined | string; + + return new Transform({ + readableObjectMode: true, + transform(chunk, encoding, next) { + const lines = ( + (currentLine !== undefined ? currentLine : '') + String(chunk) + ).split(/\r?\n/); + + currentLine = lines.pop(); + + for (const line of lines) { + this.push(JSON.parse(line)); + } + + next(); + }, + + flush(done) { + if (currentLine) { + this.push(JSON.parse(currentLine)); + } + + done(); + }, + }); +} + +async function parseFromFile(fileName: string) { + const startTime = Date.now(); + + const fileReadStream = fs.createReadStream(fileName, { + flags: 'r', // Open for reading, error if doesn't exist. + }); + + const dest = new PassThrough({ objectMode: true }); + const resultPromise = parseSchema(dest); + const pipeline = promisify(callbackPipeline); + await pipeline(fileReadStream, createFileStreamLineParser(), dest); + const res = await resultPromise; + + const dur = Date.now() - startTime; + console.log(res); + console.log(`Schema analysis took ${dur}ms.`); // Log time it took to parse. +} + +void parseFromFile(schemaFileName); diff --git a/packages/mongodb-schema/examples/parse-schema.ts b/packages/mongodb-schema/examples/parse-schema.ts new file mode 100644 index 000000000..84fd4c1cc --- /dev/null +++ b/packages/mongodb-schema/examples/parse-schema.ts @@ -0,0 +1,27 @@ +/* eslint no-console: 0 */ + +import { MongoClient } from 'mongodb'; + +import parseSchema from '../src'; + +const dbName = 'test'; + +const client = new MongoClient(`mongodb://localhost:27017/${dbName}`); + +async function run() { + const db = client.db(dbName); + + try { + const docs = db.collection('test-schema').find().limit(100); + + const parsedSchema = await parseSchema(docs, {}); + + console.log(JSON.stringify(parsedSchema, null, 2)); + } catch (err) { + return console.error(err); + } finally { + await client.close(); + } +} + +void run(); diff --git a/packages/mongodb-schema/package.json b/packages/mongodb-schema/package.json new file mode 100644 index 000000000..b8c5501e5 --- /dev/null +++ b/packages/mongodb-schema/package.json @@ -0,0 +1,110 @@ +{ + "name": "@mongodb-js/mongodb-schema", + "description": "Infer a probabilistic schema for a MongoDB collection.", + "author": { + "name": "MongoDB Inc", + "email": "compass@mongodb.com" + }, + "publishConfig": { + "access": "public" + }, + "bugs": { + "url": "https://jira.mongodb.org/projects/COMPASS/issues", + "email": "compass@mongodb.com" + }, + "homepage": "https://github.com/mongodb-js/devtools-shared", + "version": "0.1.0", + "repository": { + "type": "git", + "url": "https://github.com/mongodb-js/devtools-shared.git" + }, + "keywords": [ + "mongodb", + "schema" + ], + "bin": { + "mongodb-schema": "bin/mongodb-schema" + }, + "files": [ + "LICENSE", + "dist", + "bin", + "package.json", + "README.md", + ".esm-wrapper.mjs" + ], + "license": "Apache-2.0", + "main": "dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "require": { + "default": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "import": { + "default": "./.esm-wrapper.mjs", + "types": "./dist/index.d.ts" + } + }, + "scripts": { + "bootstrap": "npm run compile", + "prepublishOnly": "npm run compile", + "compile": "tsc -p tsconfig.json && gen-esm-wrapper . ./.esm-wrapper.mjs", + "typecheck": "tsc --noEmit", + "eslint": "eslint", + "prettier": "prettier", + "lint": "npm run eslint . && npm run prettier -- --check .", + "depcheck": "depcheck", + "check": "npm run typecheck && npm run lint && npm run depcheck", + "check-ci": "npm run check", + "test": "npm run compile && npm run test-only", + "test-only": "mocha", + "test-cov": "nyc -x \"**/*.spec.*\" -x \"**/*.test.*\" --reporter=lcov --reporter=text --reporter=html npm run test", + "test-watch": "npm run test-only -- --watch", + "test-ci": "npm run test-cov", + "test-example-parse-from-file": "ts-node examples/parse-from-file.ts", + "test-example-parse-schema": "ts-node examples/parse-schema.ts", + "test-time": "ts-node ./scripts/time-testing.ts", + "reformat": "npm run prettier -- --write ." + }, + "dependencies": { + "reservoir": "^0.1.2" + }, + "optionalDependencies": { + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", + "cli-table": "^0.3.4", + "js-yaml": "^5.2.2", + "mongodb": "^6.9.0 || ^7.0.0", + "mongodb-ns": "^3.2.1", + "numeral": "^2.0.6", + "progress": "^2.0.3", + "stats-lite": "^2.0.0", + "yargs": "^18.0.0" + }, + "devDependencies": { + "@mongodb-js/compass-test-server": "^0.3.2", + "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mocha-config-devtools": "^1.1.3", + "@mongodb-js/prettier-config-devtools": "^1.0.4", + "@mongodb-js/tsconfig-devtools": "^1.1.3", + "@types/chai": "^4.2.21", + "@types/json-schema": "^7.0.15", + "@types/mocha": "^9.1.1", + "@types/node": "^25.5.2", + "@types/reservoir": "^0.1.0", + "@types/sinon-chai": "^4.0.0", + "ajv": "^8.18.0", + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", + "chai": "^4.5.0", + "depcheck": "^1.4.7", + "eslint": "^7.25.0 || ^8.0.0", + "gen-esm-wrapper": "^1.1.3", + "mocha": "^8.4.0", + "mongodb": "^7.5.0", + "nyc": "^15.1.0", + "prettier": "^3.8.1", + "sinon": "^9.2.3", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } +} diff --git a/packages/mongodb-schema/scripts/time-testing.ts b/packages/mongodb-schema/scripts/time-testing.ts new file mode 100644 index 000000000..26237af5b --- /dev/null +++ b/packages/mongodb-schema/scripts/time-testing.ts @@ -0,0 +1,45 @@ +/* eslint no-console: 0 */ + +import { MongoClient } from 'mongodb'; + +import parseSchema from '../src'; + +const dbName = 'test'; +const collectionName = 'test-schema'; +// const collectionName = 'complex'; +// const collectionName = 'listings'; +// const collectionName = 'all-bson-types'; + +const namespace = `${dbName}.${collectionName}`; + +const client = new MongoClient(`mongodb://localhost:27017/${dbName}`); + +async function run() { + const db = client.db(dbName); + + try { + const docsLimit = 10000; + const docs = await db + .collection(collectionName) + .find() + .limit(docsLimit) + .toArray(); + + console.log( + `Analyzing the schema of ${docs.length} documents from the '${namespace}' namespace...`, + ); + + const startTime = Date.now(); + + await parseSchema(docs, {}); + + const elapsed = Date.now() - startTime; + console.log(`Done. Time elapsed: ${elapsed} ms.`); + } catch (err) { + return console.error(err); + } finally { + await client.close(); + } +} + +void run(); diff --git a/packages/mongodb-schema/src/index.ts b/packages/mongodb-schema/src/index.ts new file mode 100644 index 000000000..7d95f4289 --- /dev/null +++ b/packages/mongodb-schema/src/index.ts @@ -0,0 +1,108 @@ +import type { SchemaAccessor } from './schema-accessor'; +import { InternalSchemaBasedAccessor } from './schema-accessor'; +import { getCompletedSchemaAnalyzer, SchemaAnalyzer } from './schema-analyzer'; +import type { + ArraySchemaType, + BaseSchemaType, + ConstantSchemaType, + DocumentSchemaType, + PrimitiveSchemaType, + SchemaType, + Schema as InternalSchema, + SchemaField, + SchemaParseOptions, + SimplifiedSchemaBaseType, + SimplifiedSchemaArrayType, + SimplifiedSchemaDocumentType, + SimplifiedSchemaType, + SimplifiedSchemaField, + SimplifiedSchema, +} from './schema-analyzer'; +import { convertInternalToExpanded } from './schema-converters/internal-to-expanded'; +import { convertInternalToMongodb } from './schema-converters/internal-to-mongodb'; +import { convertInternalToStandard } from './schema-converters/internal-to-standard'; +import * as schemaStats from './stats'; +import type { + AnyIterable, + StandardJSONSchema, + MongoDBJSONSchema, + ExpandedJSONSchema, + JSONSchema, +} from './types'; +import { toTypescriptTypeDefinition } from './to-typescript'; + +/** + * Analyze documents - schema can be retrieved in different formats. + */ +async function analyzeDocuments( + source: AnyIterable, + options?: SchemaParseOptions, +): Promise { + const internalSchema = ( + await getCompletedSchemaAnalyzer(source, options) + ).getResult(); + return new InternalSchemaBasedAccessor(internalSchema, { + internalToStandard: convertInternalToStandard, + internalToMongoDB: convertInternalToMongodb, + internalToExpanded: convertInternalToExpanded, + }); +} + +/** + * Convenience shortcut for parsing schemas. Accepts an array, stream or + * MongoDB cursor object to parse documents` from. + */ +async function parseSchema( + source: AnyIterable, + options?: Partial, +): Promise { + return (await getCompletedSchemaAnalyzer(source, options)).getResult(); +} + +// Convenience shortcut for getting schema paths. +async function getSchemaPaths(source: AnyIterable): Promise { + return (await getCompletedSchemaAnalyzer(source)).getSchemaPaths(); +} + +// Convenience shortcut for getting the simplified schema. +async function getSimplifiedSchema( + source: AnyIterable, +): Promise { + return (await getCompletedSchemaAnalyzer(source)).getSimplifiedSchema(); +} + +export default parseSchema; + +export type { + ArraySchemaType, + BaseSchemaType, + ConstantSchemaType, + DocumentSchemaType, + PrimitiveSchemaType, + SchemaType, + InternalSchema as Schema, + InternalSchema, + SchemaField, + SchemaParseOptions, + SimplifiedSchemaBaseType, + SimplifiedSchemaArrayType, + SimplifiedSchemaDocumentType, + SimplifiedSchemaType, + SimplifiedSchemaField, + SimplifiedSchema, + StandardJSONSchema, + MongoDBJSONSchema, + ExpandedJSONSchema, + JSONSchema, + SchemaAccessor, +}; + +export { + parseSchema, + analyzeDocuments, + getSchemaPaths, + getSimplifiedSchema, + SchemaAnalyzer, + schemaStats, + toTypescriptTypeDefinition, +}; diff --git a/packages/mongodb-schema/src/schema-accessor.ts b/packages/mongodb-schema/src/schema-accessor.ts new file mode 100644 index 000000000..05ec280c7 --- /dev/null +++ b/packages/mongodb-schema/src/schema-accessor.ts @@ -0,0 +1,80 @@ +import type { Schema as InternalSchema } from './schema-analyzer'; +import type { + ExpandedJSONSchema, + MongoDBJSONSchema, + SchemaConverterFn, + StandardJSONSchema, +} from './types'; + +type Options = { + signal?: AbortSignal; +}; + +export interface SchemaAccessor { + getStandardJsonSchema: (options?: Options) => Promise; + getMongoDBJsonSchema: (options?: Options) => Promise; + getExpandedJSONSchema: (options?: Options) => Promise; + getInternalSchema: (options?: Options) => Promise; +} + +/** + * Accessor for different schema formats. + * Internal schema is provided at initialization, + * the others are converted lazily and memoized. + * Conversion can be aborted. + */ + +export type InternalConverters = { + internalToStandard: SchemaConverterFn; + internalToExpanded: SchemaConverterFn; + internalToMongoDB: SchemaConverterFn; +}; +export class InternalSchemaBasedAccessor implements SchemaAccessor { + private internalSchema: InternalSchema; + private standardJSONSchema?: StandardJSONSchema; + private mongodbJSONSchema?: MongoDBJSONSchema; + private expandedJSONSchema?: ExpandedJSONSchema; + private converters: InternalConverters; + + constructor(internalSchema: InternalSchema, converters: InternalConverters) { + this.internalSchema = internalSchema; + this.converters = converters; + } + + getInternalSchema(): Promise { + return Promise.resolve(this.internalSchema); + } + + /** + * Get standard JSON Schema - as per + * https://json-schema.org/draft/2020-12/schema + */ + async getStandardJsonSchema( + options: Options = {}, + ): Promise { + return (this.standardJSONSchema ??= + await this.converters.internalToStandard(this.internalSchema, options)); + } + + /** + * Get MongoDB's $jsonSchema + */ + async getMongoDBJsonSchema( + options: Options = {}, + ): Promise { + return (this.mongodbJSONSchema ??= await this.converters.internalToMongoDB( + this.internalSchema, + options, + )); + } + + /** + * Get expanded JSON Schema - with additional properties + */ + async getExpandedJSONSchema( + options: Options = {}, + ): Promise { + return (this.expandedJSONSchema ??= + await this.converters.internalToExpanded(this.internalSchema, options)); + } +} diff --git a/packages/mongodb-schema/src/schema-analyzer.ts b/packages/mongodb-schema/src/schema-analyzer.ts new file mode 100644 index 000000000..e8b373d20 --- /dev/null +++ b/packages/mongodb-schema/src/schema-analyzer.ts @@ -0,0 +1,775 @@ +import Reservoir from 'reservoir'; +import type { + Document, + ObjectId, + MinKey, + MaxKey, + Long, + Double, + Int32, + Decimal128, + BSONRegExp, + BSONSymbol, + Timestamp, +} from 'bson'; +import { Binary, Code } from 'bson'; + +import semanticTypes from './semantic-types'; +import type { AnyIterable } from './types'; +import { allowAbort, ALLOW_ABORT_INTERVAL_COUNT } from './util'; + +type TypeCastMap = { + Array: unknown[]; + Binary: Binary; + Boolean: boolean; + Code: Code; + CodeWScope: Code; + Date: Date; + Decimal128: Decimal128; + Double: Double; + Int32: Int32; + Int64: Long; + MaxKey: MaxKey; + MinKey: MinKey; + Null: null; + Object: Record; // Note: In the resulting schema we rename `Object` to `Document`. + ObjectId: ObjectId; + BSONRegExp: BSONRegExp; + String: string; + BSONSymbol: BSONSymbol; + Timestamp: Timestamp; + Undefined: undefined; +}; + +type TypeCastTypes = keyof TypeCastMap; +type BSONValue = TypeCastMap[TypeCastTypes]; + +export type BaseSchemaType = { + path: string[]; + name: string; + count: number; + probability: number; + bsonType: string; + + // As `values` is from a sample reservoir this isn't a true check for duplicates/uniqueness. + // We cannot compute `unique` and `hasDuplicates` when `storeValues` is false. + hasDuplicates?: boolean; + unique?: number; +}; + +export type ConstantSchemaType = BaseSchemaType & { + name: 'Null' | 'Undefined'; +}; + +export type PrimitiveSchemaType = BaseSchemaType & { + name: + | 'String' + | 'Number' + | 'Int32' + | 'Boolean' + | 'Decimal128' + | 'Long' + | 'ObjectId' + | 'Date' + | 'RegExp' + | 'Symbol' + | 'MaxKey' + | 'MinKey' + | 'Binary' + | 'Code' + | 'Timestamp' + | 'DBRef'; + values: BSONValue[]; +}; + +export type ArraySchemaType = BaseSchemaType & { + name: 'Array'; + lengths: number[]; + averageLength: number; + totalCount: number; + // eslint-disable-next-line no-use-before-define + types: SchemaType[]; +}; + +export type DocumentSchemaType = BaseSchemaType & { + name: 'Document'; + // eslint-disable-next-line no-use-before-define + fields: SchemaField[]; +}; + +// We include the base schema type to make the `semantic-types` usage +// easier to type. +export type SchemaType = + | BaseSchemaType + | ConstantSchemaType + | PrimitiveSchemaType + | ArraySchemaType + | DocumentSchemaType; + +export type SchemaField = { + name: string; + count: number; + path: string[]; + type: string | string[]; + probability: number; + hasDuplicates: boolean; + types: SchemaType[]; +}; + +export type Schema = { + count: number; + fields: SchemaField[]; +}; + +type SchemaBSONType = Exclude | 'Document'; + +type SchemaAnalysisBaseType = { + name: string; + path: string[]; + bsonType: SchemaBSONType; + count: number; + values?: ReturnType; +}; + +type SchemaAnalysisNullType = SchemaAnalysisBaseType & { + name: 'Null'; +}; + +type SchemaAnalysisPrimitiveType = SchemaAnalysisBaseType & { + name: + | 'String' + | 'Number' + | 'Int32' + | 'Boolean' + | 'Decimal128' + | 'Long' + | 'ObjectId' + | 'Date' + | 'RegExp' + | 'Symbol' + | 'MaxKey' + | 'MinKey' + | 'Binary' + | 'Code' + | 'Timestamp' + | 'DBRef'; +}; + +type SchemaAnalysisArrayType = SchemaAnalysisBaseType & { + name: 'Array'; + lengths: number[]; + // eslint-disable-next-line no-use-before-define + types: SchemaAnalysisFieldTypes; +}; + +type SchemaAnalysisDocumentType = SchemaAnalysisBaseType & { + name: 'Document'; + // eslint-disable-next-line no-use-before-define + fields: SchemaAnalysisFieldsMap; +}; + +// We include the base schema type to make the `semantic-types` usage +// easier to type. +type SchemaAnalysisType = + | SchemaAnalysisBaseType + | SchemaAnalysisNullType + | SchemaAnalysisPrimitiveType + | SchemaAnalysisArrayType + | SchemaAnalysisDocumentType; + +type SchemaAnalysisFieldTypes = { + [fieldName: string]: SchemaAnalysisType; +}; + +type SchemaAnalysisField = { + name: string; + path: string[]; + count: number; + types: SchemaAnalysisFieldTypes; +}; + +// This is used for the current state of the schema analysis. +type SchemaAnalysisFieldsMap = { + [fieldName: string]: SchemaAnalysisField; +}; + +type SchemaAnalysisRoot = { + fields: SchemaAnalysisFieldsMap; + count: number; +}; + +type SemanticTypeFunction = (value: BSONValue, path?: string[]) => boolean; +type SemanticTypeMap = { + [typeName: string]: SemanticTypeFunction | boolean; +}; + +type AllSchemaParseOptions = { + semanticTypes: boolean | SemanticTypeMap; + storeValues: boolean; + signal?: AbortSignal; + storedValuesLengthLimit: number; + /** Complexity limit: + * The analysis will be aborted if the threshold is exceeded. */ + distinctFieldsAbortThreshold?: number; +}; +export type SchemaParseOptions = Partial; + +const defaultSchemaParseOptions: AllSchemaParseOptions = { + semanticTypes: false, + storeValues: true, + storedValuesLengthLimit: 10000, +}; + +/** + * Extracts a Javascript string from a BSON type to compute unique values. + */ +function extractStringValueFromBSON(value: any): string { + if (value?._bsontype) { + if (['Decimal128', 'Long'].includes(value._bsontype)) { + return value.toString(); + } + if (['Double', 'Int32'].includes(value._bsontype)) { + return String(value.value); + } + } + if (typeof value === 'string') { + return value; + } + return String(value); +} + +function fieldComparator( + a: { + name: string; + }, + b: { + name: string; + }, +) { + // Make sure _id is always at top, even in presence of uppercase fields. + const aName = a.name; + const bName = b.name; + if (aName === '_id') { + return -1; + } + if (bName === '_id') { + return 1; + } + // Otherwise sort case-insensitively. + return aName.toLowerCase() < bName.toLowerCase() ? -1 : 1; +} + +/** + * Returns the type of value as a string. BSON type aware. Replaces `Object` + * with `Document` to avoid naming conflicts with javascript Objects. + */ +function getBSONType(value: any): SchemaBSONType { + const bsonType = value?._bsontype + ? value._bsontype + : Object.prototype.toString.call(value).replace(/^\[object (\w+)\]$/, '$1'); + if (bsonType === 'Object') { + // In the resulting schema we rename `Object` to `Document`. + return 'Document'; + } + if (bsonType === 'Code' && (value as Code).scope) { + return 'CodeWScope'; + } + return bsonType; +} + +function isNullType(type: SchemaAnalysisType): type is SchemaAnalysisNullType { + return (type as SchemaAnalysisNullType).name === 'Null'; +} + +function isArrayType( + type: SchemaAnalysisType, +): type is SchemaAnalysisArrayType { + return (type as SchemaAnalysisArrayType).name === 'Array'; +} + +function isDocumentType( + type: SchemaAnalysisType, +): type is SchemaAnalysisDocumentType { + return (type as SchemaAnalysisDocumentType).name === 'Document'; +} + +/** + * Recursively extracts all of the schema field paths as string arrays. + */ +function schemaToPaths( + fields: SchemaAnalysisFieldsMap, + parent: string[] = [], +): string[][] { + const paths: string[][] = []; + const sortedFields = Object.values(fields).sort(fieldComparator); + + for (const field of sortedFields) { + const path = [...parent, field.name]; + paths.push(path); + + // Recurse on doc. + const doc = Object.values(field.types).find( + (f) => f.bsonType === 'Document', + ) as SchemaAnalysisDocumentType | undefined; + + if (doc) { + paths.push(...schemaToPaths(doc.fields, path)); + } + + // Recurse on array. + const array = Object.values(field.types).find( + (f) => f.bsonType === 'Array', + ) as SchemaAnalysisArrayType | undefined; + if (array) { + const arrayDoc = Object.values(array.types).find( + (f) => f.bsonType === 'Document', + ) as SchemaAnalysisDocumentType | undefined; + + if (arrayDoc) { + paths.push(...schemaToPaths(arrayDoc.fields, path)); + } + } + } + + return paths; +} + +export type SimplifiedSchemaBaseType = { + bsonType: SchemaBSONType; +}; +export type SimplifiedSchemaArrayType = SimplifiedSchemaBaseType & { + bsonType: 'Array'; + // eslint-disable-next-line no-use-before-define + types: SimplifiedSchemaType[]; +}; +export type SimplifiedSchemaDocumentType = SimplifiedSchemaBaseType & { + bsonType: 'Document'; + // eslint-disable-next-line no-use-before-define + fields: SimplifiedSchema; +}; +export type SimplifiedSchemaType = + | SimplifiedSchemaBaseType + | SimplifiedSchemaArrayType + | SimplifiedSchemaDocumentType; +export type SimplifiedSchemaField = { + types: SimplifiedSchemaType[]; +}; +export type SimplifiedSchema = { + [fieldName: string]: SimplifiedSchemaField; +}; + +function simplifiedSchema(fields: SchemaAnalysisFieldsMap): SimplifiedSchema { + function finalizeSchemaFieldTypes( + types: SchemaAnalysisFieldTypes, + ): SimplifiedSchemaType[] { + return Object.values(types) + .sort((a: SchemaAnalysisType, b: SchemaAnalysisType) => { + // Sort the types by what occurs most frequent first. + return b.count - a.count; + }) + .map((type: SchemaAnalysisType) => { + return { + bsonType: type.bsonType, // Note: `Object` is replaced with `Document`. + ...(isArrayType(type) + ? { + types: finalizeSchemaFieldTypes(type.types), + } + : {}), + ...(isDocumentType(type) + ? { fields: finalizeDocumentFieldSchema(type.fields) } + : {}), + }; + }); + } + + function finalizeDocumentFieldSchema( + fieldMap: SchemaAnalysisFieldsMap, + ): SimplifiedSchema { + const fieldSchema: SimplifiedSchema = Object.create(null); + Object.values(fieldMap).forEach((field: SchemaAnalysisField) => { + const fieldTypes = finalizeSchemaFieldTypes(field.types); + + fieldSchema[field.name] = { + types: fieldTypes, + }; + }); + return fieldSchema; + } + + return finalizeDocumentFieldSchema(fields); +} + +function cropString(value: string, limit: number) { + if (limit < 1) return ''; + return value.charCodeAt(limit - 1) === value.codePointAt(limit - 1) + ? value.slice(0, limit) + : value.slice(0, limit - 1); +} + +function getCappedValue( + bsonType: SchemaBSONType, + value: BSONValue, + limit: number, +) { + if (bsonType === 'String') { + return cropString(value as string, limit); + } + if (bsonType === 'Binary') { + value = value as Binary; + return value.buffer.length > limit + ? new Binary(value.buffer.slice(0, limit), value.sub_type) + : value; + } + if (bsonType === 'Code') { + value = value as Code; + return value.code.length >= limit + ? new Code(cropString(value.code, limit), value.scope) + : value; + } + return value; +} + +function computeHasDuplicatesForType( + type: SchemaAnalysisType, + unique?: number, +) { + if (isNullType(type)) { + return type.count > 0; + } + + if (!type.values) { + return undefined; + } + + return unique !== type.values.length; +} + +function computeUniqueForType(type: SchemaAnalysisType) { + if (isNullType(type)) { + return type.count === 0 ? 0 : 1; + } + + if (!type.values) { + return undefined; + } + + // As `values` is from a sample reservoir this isn't a true check for uniqueness. + return new Set(type.values.map(extractStringValueFromBSON)).size; +} + +/** + * Final pass through the result to add missing information: + * - Compute `probability`, `unique`, `hasDuplicates` and + * `averageLength` fields. + * - Add `Undefined` pseudo-types. + * - Collapse `type` arrays to single string if length 1. + * - Turn fields and types objects into arrays to conform with original + * schema parser. + */ +function finalizeSchema(schemaAnalysis: SchemaAnalysisRoot): SchemaField[] { + function finalizeArrayFieldProperties(type: SchemaAnalysisArrayType) { + const totalCount = Object.values(type.types) + .map((v) => v.count) + .reduce((p: number, c: number) => p + c, 0); + + const types = finalizeSchemaFieldTypes(type.types, totalCount); + + return { + types, + totalCount, + lengths: type.lengths, + averageLength: totalCount / type.lengths.length, + }; + } + + function finalizeSchemaFieldTypes( + types: SchemaAnalysisFieldTypes, + parentCount: number, + ): SchemaType[] { + return Object.values(types) + .map((type) => { + const unique = computeUniqueForType(type); + + return { + name: type.name, + path: type.path, + count: type.count, + probability: type.count / parentCount, + unique, + hasDuplicates: computeHasDuplicatesForType(type, unique), + values: isNullType(type) ? undefined : type.values, + bsonType: type.bsonType, // Note: `Object` is replaced with `Document`. + ...(isArrayType(type) ? finalizeArrayFieldProperties(type) : {}), + ...(isDocumentType(type) + ? { fields: finalizeDocumentFieldSchema(type.fields, type.count) } + : {}), + }; + }) + .sort((a, b) => b.probability - a.probability); + } + + function finalizeDocumentFieldSchema( + fieldMap: SchemaAnalysisFieldsMap, + parentCount: number, + ): SchemaField[] { + return Object.values(fieldMap) + .map((field: SchemaAnalysisField): SchemaField => { + const fieldTypes = finalizeSchemaFieldTypes(field.types, parentCount); + + const undefinedCount = parentCount - field.count; + if (undefinedCount > 0) { + // Add the `Undefined` pseudo-type. + fieldTypes.push({ + name: 'Undefined', + bsonType: 'Undefined', + unique: undefinedCount > 1 ? 0 : 1, + hasDuplicates: undefinedCount > 1, + path: field.path, + count: undefinedCount, + probability: undefinedCount / parentCount, + }); + } + + return { + name: field.name, + path: field.path, + count: field.count, + type: + fieldTypes.length === 1 + ? fieldTypes[0].name + : fieldTypes.map((v: SchemaType) => v.name), // Or one value or array. + probability: field.count / parentCount, + hasDuplicates: !!fieldTypes.find((v: SchemaType) => v.hasDuplicates), + types: fieldTypes, + }; + }) + .sort(fieldComparator); + } + + return finalizeDocumentFieldSchema( + schemaAnalysis.fields, + schemaAnalysis.count, + ); +} + +export class SchemaAnalyzer { + semanticTypes: SemanticTypeMap; + options: AllSchemaParseOptions; + documentsAnalyzed = 0; + fieldsCount = 0; + schemaAnalysisRoot: SchemaAnalysisRoot = { + fields: Object.create(null), + count: 0, + }; + + finalized = true; + schemaResult: Schema = { + count: 0, + fields: [], + }; + + // Increments when every field or type is analyzed. + // Useful for occasionally checking if the analysis should be aborted. + fieldAndTypeAnalysisCounter = 0; + + constructor(options?: SchemaParseOptions) { + // Set default options. + this.options = { ...defaultSchemaParseOptions, ...options }; + + this.semanticTypes = { + ...semanticTypes, + }; + + if (typeof this.options.semanticTypes === 'object') { + // Enable existing semantic types that evaluate to true. + const enabledTypes = Object.entries(this.options.semanticTypes) + .filter(([, v]) => typeof v === 'boolean' && v) + .map(([k]) => k.toLowerCase()); + + this.semanticTypes = { + ...Object.entries(this.semanticTypes) + .filter(([k]) => enabledTypes.includes(k.toLowerCase())) + .reduce((p, [k, v]) => ({ ...p, [k]: v }), Object.create(null)), + }; + + Object.entries(this.options.semanticTypes) + .filter(([, v]) => typeof v === 'function') + .forEach(([k, v]) => { + this.semanticTypes[k] = v; + }); + } + } + + allowAbortDuringAnalysis(): Promise { + // Allow aborting the analysis. + if (this.fieldAndTypeAnalysisCounter++ % ALLOW_ABORT_INTERVAL_COUNT === 0) { + void allowAbort(); + } + return Promise.resolve(); + } + + increaseFieldCount() { + if (!this.options.distinctFieldsAbortThreshold) return; + this.fieldsCount++; + if (this.fieldsCount > this.options.distinctFieldsAbortThreshold) { + throw new Error( + `Schema analysis aborted: Fields count above ${this.options.distinctFieldsAbortThreshold}`, + ); + } + } + + getSemanticType(value: BSONValue, path: string[]) { + // Pass value to semantic type detectors, return first match or undefined. + const returnValue = Object.entries(this.semanticTypes) + .filter(([, v]) => { + return (v as SemanticTypeFunction)(value, path); + }) + .map(([k]) => k)[0]; + + return returnValue; + } + + async analyzeDoc(doc: Document) { + this.finalized = false; + /** + * Takes a field value, determines the correct type, handles recursion into + * nested arrays and documents, and passes the value down to `addToValue`. + * Note: This mutates the `schema` argument. + */ + const addToType = async ( + path: string[], + value: BSONValue, + schema: SchemaAnalysisFieldTypes, + ) => { + await this.allowAbortDuringAnalysis(); + const bsonType = getBSONType(value); + // If semantic type detection is enabled, the type is the semantic type + // or the original bson type if no semantic type was detected. If disabled, + // it is always the bson type. + const typeName = this.options.semanticTypes + ? this.getSemanticType(value, path) || bsonType + : bsonType; + if (!schema[typeName]) { + schema[typeName] = { + name: typeName, + bsonType: bsonType, + path, + count: 0, + }; + } + const type = schema[typeName]; + type.count++; + + if (isArrayType(type)) { + // Recurse into arrays by calling `addToType` for each element. + type.types = type.types ?? Object.create(null); + type.lengths = type.lengths ?? []; + type.lengths.push((value as BSONValue[]).length); + for (const v of value as BSONValue[]) { + await addToType(path, v, type.types); + } + } else if (isDocumentType(type)) { + // Recurse into nested documents by calling `addToField` for all sub-fields. + type.fields = type.fields ?? Object.create(null); + + for (const [fieldName, v] of Object.entries(value as Document)) { + await addToField(fieldName, [...path, fieldName], v, type.fields); + } + } else if (this.options.storeValues && !isNullType(type)) { + // When the `storeValues` option is enabled, store some example values. + if (!type.values) { + type.values = ['String', 'Binary', 'Code'].includes(bsonType) + ? Reservoir(100) + : Reservoir(10000); + } + + type.values.pushSome( + getCappedValue( + type.bsonType, + value, + this.options.storedValuesLengthLimit, + ), + ); + } + }; + + /** + * Handles a field from a document. Passes the value to `addToType`. + * Note: This mutates the `schema` argument. + */ + const addToField = async ( + fieldName: string, + path: string[], + value: BSONValue, + schema: SchemaAnalysisFieldsMap, + ) => { + await this.allowAbortDuringAnalysis(); + if (!schema[fieldName]) { + schema[fieldName] = { + name: fieldName, + path: path, + count: 0, + types: Object.create(null), + }; + this.increaseFieldCount(); + } + const field = schema[fieldName]; + + field.count++; + await addToType(path, value, field.types); + }; + + for (const key of Object.keys(doc)) { + await addToField(key, [key], doc[key], this.schemaAnalysisRoot.fields); + } + this.schemaAnalysisRoot.count += 1; + } + + getResult(): Schema { + if (this.finalized) { + return this.schemaResult; + } + + this.schemaResult = { + count: this.schemaAnalysisRoot.count, + fields: finalizeSchema(this.schemaAnalysisRoot), + }; + + this.finalized = true; + return this.schemaResult; + } + + getSchemaPaths(): string[][] { + return schemaToPaths(this.schemaAnalysisRoot.fields); + } + + /** + * Returns a simplified schema result, this has no type metadata. + */ + getSimplifiedSchema(): SimplifiedSchema { + return simplifiedSchema(this.schemaAnalysisRoot.fields); + } +} + +export function verifyStreamSource(source: AnyIterable): AnyIterable { + if (!(Symbol.iterator in source) && !(Symbol.asyncIterator in source)) { + throw new Error( + 'Unknown input type for `docs`. Must be an array, ' + + 'stream or MongoDB Cursor.', + ); + } + + return source; +} + +export async function getCompletedSchemaAnalyzer( + source: AnyIterable, + options?: SchemaParseOptions, +): Promise { + const analyzer = new SchemaAnalyzer(options); + for await (const doc of verifyStreamSource(source)) { + if (options?.signal?.aborted) throw options.signal.reason; + await analyzer.analyzeDoc(doc); + } + return analyzer; +} diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-expanded.test.ts b/packages/mongodb-schema/src/schema-converters/internal-to-expanded.test.ts new file mode 100644 index 000000000..a00354114 --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-expanded.test.ts @@ -0,0 +1,1283 @@ +import assert from 'assert'; +import { RELAXED_EJSON_DEFINITIONS } from './internal-to-standard'; +import { convertInternalToExpanded } from './internal-to-expanded'; + +describe('internalSchemaToExpanded', function () { + describe('Converts: ', function () { + it('various types', async function () { + const internal = { + count: 1, + fields: [ + // types with ref + { + name: '_id', + path: ['_id'], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: ['_id'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766b7300158b1f22e972'], + bsonType: 'ObjectId', + }, + ], + }, + { + name: 'binaries', + path: ['binaries'], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['binaries'], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'binaryOld', + path: ['binaries', 'binaryOld'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'binaryOld'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + ], + }, + ], + }, + { + name: 'binData', + path: ['binData'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binData'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['AQID'], + bsonType: 'Binary', + }, + ], + }, + // type with different standard and bsonType + { + name: 'boolean', + path: ['boolean'], + count: 1, + type: 'Boolean', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Boolean', + path: ['boolean'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [true], + bsonType: 'Boolean', + }, + ], + }, + { + name: 'decimal', + path: ['decimal'], + count: 1, + type: 'Decimal128', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Decimal128', + path: ['decimal'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088', + }, + ], + bsonType: 'Decimal128', + }, + ], + }, + { + name: 'double', + path: ['double'], + count: 1, + type: 'Double', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['double'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [1.2], + bsonType: 'Double', + }, + ], + }, + { + name: 'javascript', + path: ['javascript'], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: ['javascript'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + }, + ], + bsonType: 'Code', + }, + ], + }, + { + name: 'long', + path: ['long'], + count: 1, + type: 'Long', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Long', + path: ['long'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + low: -1395630315, + high: 28744523, + unsigned: false, + }, + ], + bsonType: 'Long', + }, + ], + }, + { + name: 'maxKey', + path: ['maxKey'], + count: 1, + type: 'MaxKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MaxKey', + path: ['maxKey'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [{}], + bsonType: 'MaxKey', + }, + ], + }, + { + name: 'regex', + path: ['regex'], + count: 1, + type: 'BSONRegExp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONRegExp', + path: ['regex'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + pattern: 'pattern', + options: 'i', + }, + ], + bsonType: 'BSONRegExp', + }, + ], + }, + { + name: 'timestamp', + path: ['timestamp'], + count: 1, + type: 'Timestamp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Timestamp', + path: ['timestamp'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $timestamp: '7218556297505931265', + }, + ], + bsonType: 'Timestamp', + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + const expectedDefinitions: Partial = { + ...RELAXED_EJSON_DEFINITIONS, + }; + delete expectedDefinitions.BSONSymbol; + delete expectedDefinitions.CodeWScope; + delete expectedDefinitions.DBPointer; + delete expectedDefinitions.DBRef; + delete expectedDefinitions.Date; + delete expectedDefinitions.MinKey; + delete expectedDefinitions.Undefined; + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: [], + $defs: expectedDefinitions, + properties: { + _id: { + $ref: '#/$defs/ObjectId', + 'x-bsonType': 'objectId', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': ['642d766b7300158b1f22e972'], + }, + binData: { + $ref: '#/$defs/Binary', + 'x-bsonType': 'binData', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': ['AQID'], + }, + binaries: { + type: 'object', + 'x-bsonType': 'object', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + properties: { + binaryOld: { + $ref: '#/$defs/Binary', + 'x-bsonType': 'binData', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': ['//8='], + }, + }, + required: [], + }, + boolean: { + type: 'boolean', + 'x-bsonType': 'bool', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [true], + }, + decimal: { + $ref: '#/$defs/Decimal128', + 'x-bsonType': 'decimal', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088', + }, + ], + }, + double: { + $ref: '#/$defs/Double', + 'x-bsonType': 'double', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [1.2], + }, + javascript: { + $ref: '#/$defs/Code', + 'x-bsonType': 'javascript', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [ + { + code: 'function() {}', + }, + ], + }, + long: { + type: 'integer', + 'x-bsonType': 'long', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [ + { + low: -1395630315, + high: 28744523, + unsigned: false, + }, + ], + }, + maxKey: { + $ref: '#/$defs/MaxKey', + 'x-bsonType': 'maxKey', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [{}], + }, + regex: { + $ref: '#/$defs/RegExp', + 'x-bsonType': 'regex', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [ + { + options: 'i', + pattern: 'pattern', + }, + ], + }, + timestamp: { + $ref: '#/$defs/Timestamp', + 'x-bsonType': 'timestamp', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': [ + { + $timestamp: '7218556297505931265', + }, + ], + }, + }, + }); + }); + + it('nested document/object', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'author', + path: ['author'], + count: 1, + type: ['Document', 'Undefined'], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['author'], + count: 1, + probability: 0.5, + bsonType: 'Document', + fields: [ + { + name: 'name', + path: ['author', 'name'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['author', 'name'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['Peter Sonder'], + bsonType: 'String', + }, + ], + }, + { + name: 'rating', + path: ['author', 'rating'], + count: 1, + type: 'Double', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['author', 'rating'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [1.3], + bsonType: 'Double', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['author'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + const expectedDefinitions = { + Double: RELAXED_EJSON_DEFINITIONS.Double, + }; + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: ['author'], + $defs: expectedDefinitions, + properties: { + author: { + type: 'object', + 'x-bsonType': 'object', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 1, + }, + required: ['name', 'rating'], + properties: { + name: { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 1, + }, + 'x-sampleValues': ['Peter Sonder'], + }, + rating: { + $ref: '#/$defs/Double', + 'x-bsonType': 'double', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 1, + }, + 'x-sampleValues': [1.3], + }, + }, + }, + }, + }); + }); + + describe('arrays', function () { + it('array - single type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 1, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + ], + totalCount: 2, + lengths: [2], + averageLength: 2, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: [], + $defs: {}, + properties: { + genres: { + type: 'array', + 'x-bsonType': 'array', + 'x-metadata': { + probability: 0.5, + hasDuplicates: false, + count: 1, + }, + items: { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 2, + probability: 1, + hasDuplicates: false, + }, + 'x-sampleValues': ['crimi', 'comedy'], + }, + }, + }, + }); + }); + + it('array - mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['Array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + { + name: 'Document', + path: ['genres'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'long', + path: ['genres', 'long'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'long'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['science fiction'], + bsonType: 'String', + }, + ], + }, + { + name: 'short', + path: ['genres', 'short'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'short'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['scifi'], + bsonType: 'String', + }, + ], + }, + ], + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: [], + $defs: {}, + properties: { + genres: { + type: 'array', + 'x-bsonType': 'array', + 'x-metadata': { + probability: 0.5, + hasDuplicates: false, + count: 1, + }, + items: { + anyOf: [ + { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 2, + probability: 0.6666666666666666, + hasDuplicates: false, + }, + 'x-sampleValues': ['crimi', 'comedy'], + }, + { + type: 'object', + 'x-bsonType': 'object', + 'x-metadata': { + count: 1, + probability: 0.3333333333333333, + }, + required: ['long', 'short'], + properties: { + long: { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 1, + probability: 1, + hasDuplicates: false, + }, + 'x-sampleValues': ['science fiction'], + }, + short: { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 1, + probability: 1, + hasDuplicates: false, + }, + 'x-sampleValues': ['scifi'], + }, + }, + }, + ], + }, + }, + }, + }); + }); + + it('array - simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'arrayMixedType', + path: ['arrayMixedType'], + count: 1, + type: 'Array', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['arrayMixedType'], + count: 1, + probability: 1, + bsonType: 'Array', + types: [ + { + name: 'int32', + path: ['arrayMixedType'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [1, 3], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['arrayMixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['2'], + bsonType: 'String', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: ['arrayMixedType'], + $defs: {}, + properties: { + arrayMixedType: { + type: 'array', + 'x-bsonType': 'array', + 'x-metadata': { + count: 1, + probability: 1, + hasDuplicates: false, + }, + items: { + anyOf: [ + { + type: 'integer', + 'x-bsonType': 'int', + 'x-metadata': { + count: 2, + hasDuplicates: false, + probability: 0.6666666666666666, + }, + 'x-sampleValues': [1, 3], + }, + { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.3333333333333333, + }, + 'x-sampleValues': ['2'], + }, + ], + }, + }, + }, + }); + }); + }); + + describe('mixed types', function () { + it('simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: ['mixedType'], + count: 2, + type: ['Int32', 'String', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [1], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['abc'], + bsonType: 'String', + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: [], + $defs: {}, + properties: { + mixedType: { + 'x-metadata': { + probability: 0.6666666666666666, + hasDuplicates: false, + count: 2, + }, + anyOf: [ + { + type: 'integer', + 'x-bsonType': 'int', + 'x-metadata': { + probability: 0.3333333333333333, + hasDuplicates: false, + count: 1, + }, + 'x-sampleValues': [1], + }, + { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + probability: 0.3333333333333333, + hasDuplicates: false, + count: 1, + }, + 'x-sampleValues': ['abc'], + }, + ], + }, + }, + }); + }); + + it('complex mixed type (with array and object)', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: [], + $defs: {}, + properties: { + mixedComplexType: { + 'x-metadata': { + probability: 0.6666666666666666, + hasDuplicates: false, + count: 2, + }, + anyOf: [ + { + type: 'array', + 'x-bsonType': 'array', + 'x-metadata': { + count: 1, + probability: 0.3333333333333333, + }, + items: { + type: 'integer', + 'x-bsonType': 'int', + 'x-metadata': { + probability: 1, + hasDuplicates: false, + count: 3, + }, + 'x-sampleValues': [1, 2, 3], + }, + }, + { + type: 'object', + 'x-bsonType': 'object', + required: ['a'], + 'x-metadata': { + count: 1, + probability: 0.3333333333333333, + }, + properties: { + a: { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + probability: 1, + hasDuplicates: false, + count: 1, + }, + 'x-sampleValues': ['bc'], + }, + }, + }, + ], + }, + }, + }); + }); + + it('complex mixed type (with $refs)', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: ['mixedType'], + count: 2, + type: ['String', 'ObjectId'], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['abc'], + bsonType: 'String', + }, + { + name: 'ObjectId', + path: ['objectId'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766c7300158b1f22e975'], + bsonType: 'ObjectId', + }, + ], + }, + ], + }; + const expanded = await convertInternalToExpanded(internal); + const expectedDefinitions = { + ObjectId: RELAXED_EJSON_DEFINITIONS.ObjectId, + }; + assert.deepStrictEqual(expanded, { + type: 'object', + 'x-bsonType': 'object', + required: ['mixedType'], + $defs: expectedDefinitions, + properties: { + mixedType: { + 'x-metadata': { + count: 2, + hasDuplicates: false, + probability: 1, + }, + anyOf: [ + { + type: 'string', + 'x-bsonType': 'string', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.3333333333333333, + }, + 'x-sampleValues': ['abc'], + }, + { + $ref: '#/$defs/ObjectId', + 'x-bsonType': 'objectId', + 'x-metadata': { + count: 1, + hasDuplicates: false, + probability: 0.8, + }, + 'x-sampleValues': ['642d766c7300158b1f22e975'], + }, + ], + }, + }, + }); + }); + }); + + it('can be aborted', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const abortController = new AbortController(); + const promise = convertInternalToExpanded(internal, { + signal: abortController.signal, + }); + abortController.abort(new Error("Too long, didn't wait.")); + await assert.rejects(promise, { + name: 'Error', + message: "Too long, didn't wait.", + }); + }); + }); +}); diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-expanded.ts b/packages/mongodb-schema/src/schema-converters/internal-to-expanded.ts new file mode 100644 index 000000000..769342347 --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-expanded.ts @@ -0,0 +1,151 @@ +import type { + ArraySchemaType, + DocumentSchemaType, + Schema as InternalSchema, + SchemaType, + SchemaField, +} from '../schema-analyzer'; +import { type ExpandedJSONSchema } from '../types'; +import { + InternalTypeToStandardTypeMap, + RELAXED_EJSON_DEFINITIONS, +} from './internal-to-standard'; +import { InternalTypeToBsonTypeMap } from './internal-to-mongodb'; +import { allowAbort } from '../util'; + +const createConvertInternalToExpanded = function () { + const usedDefinitions = new Set(); + + function getUsedDefinitions() { + const filteredDefinitions = Object.fromEntries( + Object.entries(RELAXED_EJSON_DEFINITIONS).filter(([key]) => + usedDefinitions.has(key), + ), + ); + return Object.freeze(filteredDefinitions); + } + + function markUsedDefinition(ref: string) { + usedDefinitions.add(ref.split('/')[2]); + } + + function getStandardType(internalType: string) { + const type = InternalTypeToStandardTypeMap[internalType]; + if (!type) throw new Error(`Encountered unknown type: ${internalType}`); + return { ...type }; + } + + function getBsonType(internalType: string) { + const type = InternalTypeToBsonTypeMap[internalType]; + if (!type) throw new Error(`Encountered unknown type: ${internalType}`); + return type; + } + + async function parseType( + type: SchemaType, + signal?: AbortSignal, + ): Promise { + await allowAbort(signal); + const schema: ExpandedJSONSchema = { + ...getStandardType(type.bsonType), + 'x-bsonType': getBsonType(type.bsonType), + 'x-metadata': getMetadata(type), + }; + if ('values' in type && type.values) { + schema['x-sampleValues'] = type.values; + } + if (schema.$ref) markUsedDefinition(schema.$ref); + switch (type.bsonType) { + case 'Array': + schema.items = await parseTypes( + (type as ArraySchemaType).types, + signal, + ); + break; + case 'Document': + Object.assign( + schema, + await parseFields((type as DocumentSchemaType).fields, signal), + ); + break; + } + return schema; + } + + function getMetadata({ + hasDuplicates, + probability, + count, + }: TType) { + return { + ...(typeof hasDuplicates === 'boolean' ? { hasDuplicates } : {}), + probability, + count, + }; + } + + async function parseTypes( + types: SchemaType[], + signal?: AbortSignal, + ): Promise { + await allowAbort(signal); + const definedTypes = types.filter( + (type) => type.bsonType.toLowerCase() !== 'undefined', + ); + const isSingleType = definedTypes.length === 1; + if (isSingleType) { + return parseType(definedTypes[0], signal); + } + const parsedTypes = await Promise.all( + definedTypes.map((type) => parseType(type, signal)), + ); + return { + anyOf: parsedTypes, + }; + } + + async function parseFields( + fields: DocumentSchemaType['fields'], + signal?: AbortSignal, + ): Promise<{ + required: ExpandedJSONSchema['required']; + properties: ExpandedJSONSchema['properties']; + }> { + const required = []; + const properties: ExpandedJSONSchema['properties'] = {}; + for (const field of fields) { + if (field.probability === 1) required.push(field.name); + properties[field.name] = { + ...(await parseTypes(field.types, signal)), + 'x-metadata': getMetadata(field), + }; + } + + return { required, properties }; + } + + return async function convert( + internalSchema: InternalSchema, + options: { signal?: AbortSignal } = {}, + ): Promise { + const { required, properties } = await parseFields( + internalSchema.fields, + options.signal, + ); + const schema: ExpandedJSONSchema = { + type: 'object', + 'x-bsonType': 'object', + required, + properties, + $defs: getUsedDefinitions(), + }; + return schema; + }; +}; + +export function convertInternalToExpanded( + internalSchema: InternalSchema, + options: { signal?: AbortSignal } = {}, +): Promise { + return createConvertInternalToExpanded()(internalSchema, options); +} diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.test.ts b/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.test.ts new file mode 100644 index 000000000..63ecd9ad7 --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.test.ts @@ -0,0 +1,1403 @@ +import assert from 'assert'; +import { convertInternalToMongodb } from './internal-to-mongodb'; + +describe('internalSchemaToMongoDB', function () { + describe('Converts: ', function () { + it('all the types', async function () { + const internal = { + count: 1, + fields: [ + { + name: '_id', + path: ['_id'], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: ['_id'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766b7300158b1f22e972'], + bsonType: 'ObjectId', + }, + ], + }, + { + name: 'array', + path: ['array'], + count: 1, + type: 'Array', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['array'], + count: 1, + probability: 0.8, + bsonType: 'Array', + types: [ + { + name: 'Number', + path: ['array'], + count: 3, + probability: 0.8, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Number', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + ], + }, + { + name: 'binaries', + path: ['binaries'], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['binaries'], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'binaryOld', + path: ['binaries', 'binaryOld'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'binaryOld'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'compressedTimeSeries', + path: ['binaries', 'compressedTimeSeries'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'compressedTimeSeries'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'custom', + path: ['binaries', 'custom'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'custom'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'encrypted', + path: ['binaries', 'encrypted'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'encrypted'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'functionData', + path: ['binaries', 'functionData'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'functionData'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'generic', + path: ['binaries', 'generic'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'generic'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['AQID'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'md5', + path: ['binaries', 'md5'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'md5'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'uuid', + path: ['binaries', 'uuid'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'uuid'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'uuidOld', + path: ['binaries', 'uuidOld'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'uuidOld'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + ], + }, + ], + }, + { + name: 'binData', + path: ['binData'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binData'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['AQID'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'boolean', + path: ['boolean'], + count: 1, + type: 'Boolean', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Boolean', + path: ['boolean'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [true], + bsonType: 'Boolean', + }, + ], + }, + { + name: 'date', + path: ['date'], + count: 1, + type: 'Date', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Date', + path: ['date'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['2023-04-05T13:25:08.445Z'], + bsonType: 'Date', + }, + ], + }, + { + name: 'dbRef', + path: ['dbRef'], + count: 1, + type: 'DBRef', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'DBRef', + path: ['dbRef'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $ref: 'namespace', + $id: '642d76b4b7ebfab15d3c4a78', + }, + ], + bsonType: 'DBRef', + }, + ], + }, + { + name: 'decimal', + path: ['decimal'], + count: 1, + type: 'Decimal128', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Decimal128', + path: ['decimal'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088', + }, + ], + bsonType: 'Decimal128', + }, + ], + }, + { + name: 'double', + path: ['double'], + count: 1, + type: 'Double', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['double'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [1.2], + bsonType: 'Double', + }, + ], + }, + { + name: 'int', + path: ['int'], + count: 1, + type: 'Int32', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: ['int'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [12345], + bsonType: 'Int32', + }, + ], + }, + { + name: 'javascript', + path: ['javascript'], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: ['javascript'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + }, + ], + bsonType: 'Code', + }, + ], + }, + { + name: 'javascriptWithScope', + path: ['javascriptWithScope'], + count: 1, + type: 'CodeWScope', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'CodeWScope', + path: ['javascriptWithScope'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + scope: { + foo: 1, + bar: 'a', + }, + }, + ], + bsonType: 'CodeWScope', + }, + ], + }, + { + name: 'long', + path: ['long'], + count: 1, + type: 'Long', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Long', + path: ['long'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + low: -1395630315, + high: 28744523, + unsigned: false, + }, + ], + bsonType: 'Long', + }, + ], + }, + { + name: 'maxKey', + path: ['maxKey'], + count: 1, + type: 'MaxKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MaxKey', + path: ['maxKey'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [{}], + bsonType: 'MaxKey', + }, + ], + }, + { + name: 'minKey', + path: ['minKey'], + count: 1, + type: 'MinKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MinKey', + path: ['minKey'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [{}], + bsonType: 'MinKey', + }, + ], + }, + { + name: 'null', + path: ['null'], + count: 1, + type: 'Null', + probability: 0.8, + hasDuplicates: true, + types: [ + { + name: 'Null', + path: ['null'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: true, + bsonType: 'Null', + }, + ], + }, + { + name: 'object', + path: ['object'], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['object'], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'key', + path: ['object', 'key'], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['object', 'key'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['value'], + bsonType: 'String', + }, + ], + }, + ], + }, + ], + }, + { + name: 'objectId', + path: ['objectId'], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: ['objectId'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766c7300158b1f22e975'], + bsonType: 'ObjectId', + }, + ], + }, + { + name: 'regex', + path: ['regex'], + count: 1, + type: 'BSONRegExp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONRegExp', + path: ['regex'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + pattern: 'pattern', + options: 'i', + }, + ], + bsonType: 'BSONRegExp', + }, + ], + }, + { + name: 'string', + path: ['string'], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['string'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['Hello, world!'], + bsonType: 'String', + }, + ], + }, + { + name: 'symbol', + path: ['symbol'], + count: 1, + type: 'BSONSymbol', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONSymbol', + path: ['symbol'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['symbol'], + bsonType: 'BSONSymbol', + }, + ], + }, + { + name: 'timestamp', + path: ['timestamp'], + count: 1, + type: 'Timestamp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Timestamp', + path: ['timestamp'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $timestamp: '7218556297505931265', + }, + ], + bsonType: 'Timestamp', + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + properties: { + _id: { + bsonType: 'objectId', + }, + array: { + bsonType: 'array', + items: { + bsonType: 'double', + }, + }, + binData: { + bsonType: 'binData', + }, + binaries: { + bsonType: 'object', + properties: { + binaryOld: { + bsonType: 'binData', + }, + compressedTimeSeries: { + bsonType: 'binData', + }, + custom: { + bsonType: 'binData', + }, + encrypted: { + bsonType: 'binData', + }, + functionData: { + bsonType: 'binData', + }, + generic: { + bsonType: 'binData', + }, + md5: { + bsonType: 'binData', + }, + uuid: { + bsonType: 'binData', + }, + uuidOld: { + bsonType: 'binData', + }, + }, + }, + boolean: { + bsonType: 'bool', + }, + date: { + bsonType: 'date', + }, + dbRef: { + bsonType: 'dbPointer', + }, + decimal: { + bsonType: 'decimal', + }, + double: { + bsonType: 'double', + }, + int: { + bsonType: 'int', + }, + javascript: { + bsonType: 'javascript', + }, + javascriptWithScope: { + bsonType: 'javascriptWithScope', + }, + long: { + bsonType: 'long', + }, + maxKey: { + bsonType: 'maxKey', + }, + minKey: { + bsonType: 'minKey', + }, + null: { + bsonType: 'null', + }, + object: { + bsonType: 'object', + properties: { + key: { + bsonType: 'string', + }, + }, + }, + objectId: { + bsonType: 'objectId', + }, + regex: { + bsonType: 'regex', + }, + string: { + bsonType: 'string', + }, + symbol: { + bsonType: 'symbol', + }, + timestamp: { + bsonType: 'timestamp', + }, + }, + }); + }); + + it('nested document/object', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'author', + path: ['author'], + count: 1, + type: ['Document', 'Undefined'], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['author'], + count: 1, + probability: 0.5, + bsonType: 'Document', + fields: [ + { + name: 'name', + path: ['author', 'name'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['author', 'name'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['Peter Sonder'], + bsonType: 'String', + }, + ], + }, + { + name: 'rating', + path: ['author', 'rating'], + count: 1, + type: 'Double', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['author', 'rating'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [1.3], + bsonType: 'Double', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['author'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + required: ['author'], + properties: { + author: { + bsonType: 'object', + required: ['name', 'rating'], + properties: { + name: { + bsonType: 'string', + }, + rating: { + bsonType: 'double', + }, + }, + }, + }, + }); + }); + + describe('arrays', function () { + it('array - single type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 1, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + ], + totalCount: 2, + lengths: [2], + averageLength: 2, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + properties: { + genres: { + bsonType: 'array', + items: { + bsonType: 'string', + }, + }, + }, + }); + }); + + it('array - complex mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['Array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + { + name: 'Document', + path: ['genres'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'long', + path: ['genres', 'long'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'long'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['science fiction'], + bsonType: 'String', + }, + ], + }, + { + name: 'short', + path: ['genres', 'short'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'short'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['scifi'], + bsonType: 'String', + }, + ], + }, + ], + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + properties: { + genres: { + bsonType: 'array', + items: { + anyOf: [ + { + bsonType: 'string', + }, + { + bsonType: 'object', + required: ['long', 'short'], + properties: { + long: { + bsonType: 'string', + }, + short: { + bsonType: 'string', + }, + }, + }, + ], + }, + }, + }, + }); + }); + + it('array - simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'arrayMixedType', + path: ['arrayMixedType'], + count: 1, + type: 'Array', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['arrayMixedType'], + count: 1, + probability: 1, + bsonType: 'Array', + types: [ + { + name: 'int32', + path: ['arrayMixedType'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [1, 3], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['arrayMixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['2'], + bsonType: 'String', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + required: ['arrayMixedType'], + properties: { + arrayMixedType: { + bsonType: 'array', + items: { + bsonType: ['int', 'string'], + }, + }, + }, + }); + }); + }); + + describe('mixed types', function () { + it('simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: ['mixedType'], + count: 2, + type: ['Int32', 'String', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [1], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['abc'], + bsonType: 'String', + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + properties: { + mixedType: { + bsonType: ['int', 'string'], + }, + }, + }); + }); + + it('complex mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const mongodb = await convertInternalToMongodb(internal); + assert.deepStrictEqual(mongodb, { + bsonType: 'object', + properties: { + mixedComplexType: { + anyOf: [ + { + bsonType: 'array', + items: { + bsonType: 'int', + }, + }, + { + bsonType: 'object', + required: ['a'], + properties: { + a: { + bsonType: 'string', + }, + }, + }, + ], + }, + }, + }); + }); + }); + + it('can be aborted', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const abortController = new AbortController(); + const promise = convertInternalToMongodb(internal, { + signal: abortController.signal, + }); + abortController.abort(new Error("Too long, didn't wait.")); + await assert.rejects(promise, { + name: 'Error', + message: "Too long, didn't wait.", + }); + }); + }); +}); diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.ts b/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.ts new file mode 100644 index 000000000..9ff1dd837 --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-mongodb.ts @@ -0,0 +1,145 @@ +/** + * Transforms the internal schema to $jsonSchema + */ +import type { + ArraySchemaType, + DocumentSchemaType, + Schema as InternalSchema, + SchemaType, +} from '../schema-analyzer'; +import type { MongoDBJSONSchema } from '../types'; +import { allowAbort } from '../util'; + +// The rule's `__proto__: null` fix does not typecheck against `Record`, +// and this map is only ever read with keys derived from our own schema types. +// eslint-disable-next-line @mongodb-js/devtools/no-plain-object-records +export const InternalTypeToBsonTypeMap: Record< + SchemaType['name'] | 'Double' | 'BSONSymbol', + string +> = { + Double: 'double', + Number: 'double', + String: 'string', + Document: 'object', + Array: 'array', + Binary: 'binData', + Undefined: 'undefined', + ObjectId: 'objectId', + Boolean: 'bool', + Date: 'date', + Null: 'null', + RegExp: 'regex', + BSONRegExp: 'regex', + DBRef: 'dbPointer', + BSONSymbol: 'symbol', + Symbol: 'symbol', + Code: 'javascript', + CodeWScope: 'javascriptWithScope', + Int32: 'int', + Timestamp: 'timestamp', + Long: 'long', + Decimal128: 'decimal', + MinKey: 'minKey', + MaxKey: 'maxKey', +}; + +const convertInternalType = (type: string) => { + const bsonType = InternalTypeToBsonTypeMap[type]; + if (!bsonType) throw new Error(`Encountered unknown type: ${type}`); + return bsonType; +}; + +async function parseType( + type: SchemaType, + signal?: AbortSignal, +): Promise { + await allowAbort(signal); + const schema: MongoDBJSONSchema = { + bsonType: convertInternalType(type.bsonType), + }; + + if (type.bsonType === 'Array') { + const items = await parseTypes((type as ArraySchemaType).types); + // Don't include empty bson type arrays (it's invalid). + if (!items.bsonType || items.bsonType?.length > 0) { + schema.items = items; + } + } else if (type.bsonType === 'Document') { + Object.assign( + schema, + await parseFields((type as DocumentSchemaType).fields, signal), + ); + } + + return schema; +} + +function isPlainTypesOnly( + types: MongoDBJSONSchema[], +): types is { bsonType: string }[] { + return types.every( + (definition) => + !!definition.bsonType && Object.keys(definition).length === 1, + ); +} + +async function parseTypes( + types: SchemaType[], + signal?: AbortSignal, +): Promise { + await allowAbort(signal); + const definedTypes = types.filter( + (type) => type.bsonType.toLowerCase() !== 'undefined', + ); + const isSingleType = definedTypes.length === 1; + if (isSingleType) { + return parseType(definedTypes[0], signal); + } + const parsedTypes = await Promise.all( + definedTypes.map((type) => parseType(type, signal)), + ); + if (isPlainTypesOnly(parsedTypes)) { + return { + bsonType: parsedTypes.map(({ bsonType }) => bsonType), + }; + } + return { + anyOf: parsedTypes, + }; +} + +async function parseFields( + fields: DocumentSchemaType['fields'], + signal?: AbortSignal, +): Promise<{ + required?: MongoDBJSONSchema['required']; + properties: MongoDBJSONSchema['properties']; +}> { + const required: string[] = []; + const properties: MongoDBJSONSchema['properties'] = {}; + for (const field of fields) { + if (field.probability === 1) required.push(field.name); + properties[field.name] = await parseTypes(field.types, signal); + } + + return { properties, ...(required.length > 0 ? { required } : {}) }; +} + +export async function convertInternalToMongodb( + internalSchema: InternalSchema, + options: { + signal?: AbortSignal; + } = {}, +): Promise { + const { required, properties } = await parseFields( + internalSchema.fields, + options.signal, + ); + const schema: MongoDBJSONSchema = { + bsonType: 'object', + // Prevent adding an empty required array as it isn't valid. + ...(required === undefined ? {} : { required }), + properties, + }; + return schema; +} diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-standard.test.ts b/packages/mongodb-schema/src/schema-converters/internal-to-standard.test.ts new file mode 100644 index 000000000..bcc85c65b --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-standard.test.ts @@ -0,0 +1,1506 @@ +import assert from 'assert'; +import Ajv2020 from 'ajv/dist/2020'; +import { + convertInternalToStandard, + RELAXED_EJSON_DEFINITIONS, +} from './internal-to-standard'; + +describe('internalSchemaToStandard', function () { + const ajv = new Ajv2020(); + + describe('Converts: ', function () { + it('all the types', async function () { + const internal = { + count: 1, + fields: [ + { + name: '_id', + path: ['_id'], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: ['_id'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766b7300158b1f22e972'], + bsonType: 'ObjectId', + }, + ], + }, + { + name: 'array', + path: ['array'], + count: 1, + type: 'Array', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['array'], + count: 1, + probability: 0.8, + bsonType: 'Array', + types: [ + { + name: 'Number', + path: ['array'], + count: 3, + probability: 0.8, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Number', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + ], + }, + { + name: 'binaries', + path: ['binaries'], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['binaries'], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'binaryOld', + path: ['binaries', 'binaryOld'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'binaryOld'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'compressedTimeSeries', + path: ['binaries', 'compressedTimeSeries'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'compressedTimeSeries'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'custom', + path: ['binaries', 'custom'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'custom'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'encrypted', + path: ['binaries', 'encrypted'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'encrypted'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'functionData', + path: ['binaries', 'functionData'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'functionData'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['//8='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'generic', + path: ['binaries', 'generic'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'generic'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['AQID'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'md5', + path: ['binaries', 'md5'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'md5'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + { + name: 'uuid', + path: ['binaries', 'uuid'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'uuid'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'uuidOld', + path: ['binaries', 'uuidOld'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binaries', 'uuidOld'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['c//SZESzTGmQ6OfR38A11A=='], + bsonType: 'Binary', + }, + ], + }, + ], + }, + ], + }, + { + name: 'binData', + path: ['binData'], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: ['binData'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['AQID'], + bsonType: 'Binary', + }, + ], + }, + { + name: 'boolean', + path: ['boolean'], + count: 1, + type: 'Boolean', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Boolean', + path: ['boolean'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [true], + bsonType: 'Boolean', + }, + ], + }, + { + name: 'date', + path: ['date'], + count: 1, + type: 'Date', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Date', + path: ['date'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['2023-04-05T13:25:08.445Z'], + bsonType: 'Date', + }, + ], + }, + { + name: 'dbRef', + path: ['dbRef'], + count: 1, + type: 'DBRef', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'DBRef', + path: ['dbRef'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $ref: 'namespace', + $id: '642d76b4b7ebfab15d3c4a78', + }, + ], + bsonType: 'DBRef', + }, + ], + }, + { + name: 'decimal', + path: ['decimal'], + count: 1, + type: 'Decimal128', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Decimal128', + path: ['decimal'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088', + }, + ], + bsonType: 'Decimal128', + }, + ], + }, + { + name: 'double', + path: ['double'], + count: 1, + type: 'Double', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['double'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [1.2], + bsonType: 'Double', + }, + ], + }, + { + name: 'int', + path: ['int'], + count: 1, + type: 'Int32', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: ['int'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [12345], + bsonType: 'Int32', + }, + ], + }, + { + name: 'javascript', + path: ['javascript'], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: ['javascript'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + }, + ], + bsonType: 'Code', + }, + ], + }, + { + name: 'javascriptWithScope', + path: ['javascriptWithScope'], + count: 1, + type: 'CodeWScope', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'CodeWScope', + path: ['javascriptWithScope'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + scope: { + foo: 1, + bar: 'a', + }, + }, + ], + bsonType: 'CodeWScope', + }, + ], + }, + { + name: 'long', + path: ['long'], + count: 1, + type: 'Long', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Long', + path: ['long'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + low: -1395630315, + high: 28744523, + unsigned: false, + }, + ], + bsonType: 'Long', + }, + ], + }, + { + name: 'maxKey', + path: ['maxKey'], + count: 1, + type: 'MaxKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MaxKey', + path: ['maxKey'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [{}], + bsonType: 'MaxKey', + }, + ], + }, + { + name: 'minKey', + path: ['minKey'], + count: 1, + type: 'MinKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MinKey', + path: ['minKey'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [{}], + bsonType: 'MinKey', + }, + ], + }, + { + name: 'null', + path: ['null'], + count: 1, + type: 'Null', + probability: 0.8, + hasDuplicates: true, + types: [ + { + name: 'Null', + path: ['null'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: true, + bsonType: 'Null', + }, + ], + }, + { + name: 'object', + path: ['object'], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['object'], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'key', + path: ['object', 'key'], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['object', 'key'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['value'], + bsonType: 'String', + }, + ], + }, + ], + }, + ], + }, + { + name: 'objectId', + path: ['objectId'], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: ['objectId'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766c7300158b1f22e975'], + bsonType: 'ObjectId', + }, + ], + }, + { + name: 'regex', + path: ['regex'], + count: 1, + type: 'BSONRegExp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONRegExp', + path: ['regex'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + pattern: 'pattern', + options: 'i', + }, + ], + bsonType: 'BSONRegExp', + }, + ], + }, + { + name: 'string', + path: ['string'], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['string'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['Hello, world!'], + bsonType: 'String', + }, + ], + }, + { + name: 'symbol', + path: ['symbol'], + count: 1, + type: 'BSONSymbol', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONSymbol', + path: ['symbol'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['symbol'], + bsonType: 'BSONSymbol', + }, + ], + }, + { + name: 'timestamp', + path: ['timestamp'], + count: 1, + type: 'Timestamp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Timestamp', + path: ['timestamp'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $timestamp: '7218556297505931265', + }, + ], + bsonType: 'Timestamp', + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + const expectedDefinitions: any = { + ...RELAXED_EJSON_DEFINITIONS, + }; + delete expectedDefinitions.Undefined; + delete expectedDefinitions.DBPointer; + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: [], + $defs: expectedDefinitions, + properties: { + _id: { + $ref: '#/$defs/ObjectId', + }, + array: { + type: 'array', + items: { + $ref: '#/$defs/Double', + }, + }, + binData: { + $ref: '#/$defs/Binary', + }, + binaries: { + type: 'object', + properties: { + binaryOld: { + $ref: '#/$defs/Binary', + }, + compressedTimeSeries: { + $ref: '#/$defs/Binary', + }, + custom: { + $ref: '#/$defs/Binary', + }, + encrypted: { + $ref: '#/$defs/Binary', + }, + functionData: { + $ref: '#/$defs/Binary', + }, + generic: { + $ref: '#/$defs/Binary', + }, + md5: { + $ref: '#/$defs/Binary', + }, + uuid: { + $ref: '#/$defs/Binary', + }, + uuidOld: { + $ref: '#/$defs/Binary', + }, + }, + required: [], + }, + boolean: { + type: 'boolean', + }, + date: { + $ref: '#/$defs/Date', + }, + dbRef: { + $ref: '#/$defs/DBRef', + }, + decimal: { + $ref: '#/$defs/Decimal128', + }, + double: { + $ref: '#/$defs/Double', + }, + int: { + type: 'integer', + }, + javascript: { + $ref: '#/$defs/Code', + }, + javascriptWithScope: { + $ref: '#/$defs/CodeWScope', + }, + long: { + type: 'integer', + }, + maxKey: { + $ref: '#/$defs/MaxKey', + }, + minKey: { + $ref: '#/$defs/MinKey', + }, + null: { + type: 'null', + }, + object: { + type: 'object', + properties: { + key: { + type: 'string', + }, + }, + required: [], + }, + objectId: { + $ref: '#/$defs/ObjectId', + }, + regex: { + $ref: '#/$defs/RegExp', + }, + string: { + type: 'string', + }, + symbol: { + $ref: '#/$defs/BSONSymbol', + }, + timestamp: { + $ref: '#/$defs/Timestamp', + }, + }, + }); + }); + + it('nested document/object', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'author', + path: ['author'], + count: 1, + type: ['Document', 'Undefined'], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: ['author'], + count: 1, + probability: 0.5, + bsonType: 'Document', + fields: [ + { + name: 'name', + path: ['author', 'name'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['author', 'name'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['Peter Sonder'], + bsonType: 'String', + }, + ], + }, + { + name: 'rating', + path: ['author', 'rating'], + count: 1, + type: 'Double', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: ['author', 'rating'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [1.3], + bsonType: 'Double', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['author'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + const expectedDefinitions = { + Double: RELAXED_EJSON_DEFINITIONS.Double, + }; + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: ['author'], + $defs: expectedDefinitions, + properties: { + author: { + type: 'object', + required: ['name', 'rating'], + properties: { + name: { + type: 'string', + }, + rating: { + $ref: '#/$defs/Double', + }, + }, + }, + }, + }); + }); + + describe('arrays', function () { + it('array - single type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 1, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + ], + totalCount: 2, + lengths: [2], + averageLength: 2, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: [], + $defs: {}, + properties: { + genres: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }); + }); + + it('array - complex mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: ['genres'], + count: 1, + type: ['Array', 'Undefined'], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['genres'], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: ['genres'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: ['crimi', 'comedy'], + bsonType: 'String', + }, + { + name: 'Document', + path: ['genres'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'long', + path: ['genres', 'long'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'long'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['science fiction'], + bsonType: 'String', + }, + ], + }, + { + name: 'short', + path: ['genres', 'short'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['genres', 'short'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['scifi'], + bsonType: 'String', + }, + ], + }, + ], + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['genres'], + count: 1, + probability: 0.5, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: [], + $defs: {}, + properties: { + genres: { + type: 'array', + items: { + anyOf: [ + { + type: 'string', + }, + { + type: 'object', + required: ['long', 'short'], + properties: { + long: { + type: 'string', + }, + short: { + type: 'string', + }, + }, + }, + ], + }, + }, + }, + }); + }); + + it('array - simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'arrayMixedType', + path: ['arrayMixedType'], + count: 1, + type: 'Array', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['arrayMixedType'], + count: 1, + probability: 1, + bsonType: 'Array', + types: [ + { + name: 'int32', + path: ['arrayMixedType'], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [1, 3], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['arrayMixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['2'], + bsonType: 'String', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: ['arrayMixedType'], + $defs: {}, + properties: { + arrayMixedType: { + type: 'array', + items: { + type: ['integer', 'string'], + }, + }, + }, + }); + }); + }); + + describe('mixed types', function () { + it('simple mixed type', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: ['mixedType'], + count: 2, + type: ['Int32', 'String', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [1], + bsonType: 'Int32', + }, + { + name: 'String', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['abc'], + bsonType: 'String', + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: [], + $defs: {}, + properties: { + mixedType: { + type: ['integer', 'string'], + }, + }, + }); + }); + + it('complex mixed type (with array and object)', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: [], + $defs: {}, + properties: { + mixedComplexType: { + anyOf: [ + { + type: 'array', + items: { + type: 'integer', + }, + }, + { + type: 'object', + required: ['a'], + properties: { + a: { + type: 'string', + }, + }, + }, + ], + }, + }, + }); + }); + + it('complex mixed type (with $refs)', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: ['mixedType'], + count: 2, + type: ['String', 'ObjectId'], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedType'], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: ['abc'], + bsonType: 'String', + }, + { + name: 'ObjectId', + path: ['objectId'], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: ['642d766c7300158b1f22e975'], + bsonType: 'ObjectId', + }, + ], + }, + ], + }; + const standard = await convertInternalToStandard(internal); + void ajv.validateSchema(standard); + const expectedDefinitions = { + ObjectId: RELAXED_EJSON_DEFINITIONS.ObjectId, + }; + assert.deepStrictEqual(standard, { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required: ['mixedType'], + $defs: expectedDefinitions, + properties: { + mixedType: { + anyOf: [ + { + type: 'string', + }, + { + $ref: '#/$defs/ObjectId', + }, + ], + }, + }, + }); + }); + }); + + it('can be aborted', async function () { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: ['mixedComplexType'], + count: 2, + type: ['Array', 'Document', 'Undefined'], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: ['mixedComplexType'], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [1, 2, 3], + bsonType: 'Int32', + }, + ], + totalCount: 3, + lengths: [3], + averageLength: 3, + }, + { + name: 'Document', + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: ['mixedComplexType', 'a'], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: ['mixedComplexType', 'a'], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: ['bc'], + bsonType: 'String', + }, + ], + }, + ], + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: ['mixedComplexType'], + count: 1, + probability: 0.3333333333333333, + }, + ], + }, + ], + }; + const abortController = new AbortController(); + const promise = convertInternalToStandard(internal, { + signal: abortController.signal, + }); + abortController.abort(new Error("Too long, didn't wait.")); + await assert.rejects(promise, { + name: 'Error', + message: "Too long, didn't wait.", + }); + }); + }); +}); diff --git a/packages/mongodb-schema/src/schema-converters/internal-to-standard.ts b/packages/mongodb-schema/src/schema-converters/internal-to-standard.ts new file mode 100644 index 000000000..6eccee356 --- /dev/null +++ b/packages/mongodb-schema/src/schema-converters/internal-to-standard.ts @@ -0,0 +1,377 @@ +import type { JSONSchema4TypeName } from 'json-schema'; +import type { + ArraySchemaType, + DocumentSchemaType, + Schema as InternalSchema, + SchemaType, +} from '../schema-analyzer'; +import type { StandardJSONSchema } from '../types'; +import { allowAbort } from '../util'; + +type StandardTypeDefinition = + | { type: JSONSchema4TypeName; $ref?: never } + | { $ref: string; type?: never }; + +type TypeToDefinitionMap = Record< + SchemaType['name'] | 'Double' | 'BSONSymbol', + StandardTypeDefinition +>; +export const InternalTypeToStandardTypeMap: TypeToDefinitionMap = { + Double: { $ref: '#/$defs/Double' }, + Number: { $ref: '#/$defs/Double' }, + String: { type: 'string' }, + Document: { type: 'object' }, + Array: { type: 'array' }, + Binary: { $ref: '#/$defs/Binary' }, + Undefined: { $ref: '#/$defs/Undefined' }, + ObjectId: { $ref: '#/$defs/ObjectId' }, + Boolean: { type: 'boolean' }, + Date: { $ref: '#/$defs/Date' }, + Null: { type: 'null' }, + RegExp: { $ref: '#/$defs/RegExp' }, + BSONRegExp: { $ref: '#/$defs/RegExp' }, + DBRef: { $ref: '#/$defs/DBRef' }, + DBPointer: { $ref: '#/$defs/DBPointer' }, + BSONSymbol: { $ref: '#/$defs/BSONSymbol' }, + Code: { $ref: '#/$defs/Code' }, + CodeWScope: { $ref: '#/$defs/CodeWScope' }, + Int32: { type: 'integer' }, + Timestamp: { $ref: '#/$defs/Timestamp' }, + Long: { type: 'integer' }, + Decimal128: { $ref: '#/$defs/Decimal128' }, + MinKey: { $ref: '#/$defs/MinKey' }, + MaxKey: { $ref: '#/$defs/MaxKey' }, +}; + +export const RELAXED_EJSON_DEFINITIONS = { + ObjectId: { + type: 'object', + properties: { + $oid: { + type: 'string', + pattern: '^[0-9a-fA-F]{24}$', + }, + }, + required: ['$oid'], + additionalProperties: false, + }, + BSONSymbol: { + type: 'object', + properties: { + $symbol: { + type: 'string', + }, + }, + required: ['$symbol'], + additionalProperties: false, + }, + Double: { + oneOf: [ + { type: 'number' }, + { + type: 'object', + properties: { + $numberDouble: { + enum: ['Infinity', '-Infinity', 'NaN'], + }, + }, + }, + ], + }, + Decimal128: { + type: 'object', + properties: { + $numberDecimal: { + type: 'string', + }, + }, + required: ['$numberDecimal'], + additionalProperties: false, + }, + Binary: { + type: 'object', + properties: { + $binary: { + type: 'object', + properties: { + base64: { + type: 'string', + }, + subType: { + type: 'string', + pattern: '^[0-9a-fA-F]{1,2}$', // BSON binary type as a one- or two-character hex string + }, + }, + required: ['base64', 'subType'], + additionalProperties: false, + }, + }, + required: ['$binary'], + additionalProperties: false, + }, + Code: { + type: 'object', + properties: { + $code: { + type: 'string', + }, + }, + required: ['$code'], + additionalProperties: false, + }, + CodeWScope: { + type: 'object', + properties: { + $code: { + type: 'string', + }, + $scope: { + type: 'object', + }, + }, + required: ['$code', '$scope'], + additionalProperties: false, + }, + Timestamp: { + type: 'object', + properties: { + $timestamp: { + type: 'object', + properties: { + t: { + type: 'integer', + minimum: 0, + }, + i: { + type: 'integer', + minimum: 0, + }, + }, + required: ['t', 'i'], + additionalProperties: false, + }, + }, + required: ['$timestamp'], + additionalProperties: false, + }, + RegExp: { + type: 'object', + properties: { + $regularExpression: { + type: 'object', + properties: { + pattern: { + type: 'string', + }, + options: { + type: 'string', + pattern: '^[gimuy]*$', + }, + }, + required: ['pattern'], + additionalProperties: false, + }, + }, + required: ['$regularExpression'], + additionalProperties: false, + }, + DBPointer: { + type: 'object', + properties: { + $dbPointer: { + type: 'object', + properties: { + $ref: { + type: 'string', + }, + $id: { + $ref: '#/$defs/ObjectId', + }, + }, + required: ['$ref', '$id'], + additionalProperties: false, + }, + }, + required: ['$dbPointer'], + additionalProperties: false, + }, + Date: { + type: 'object', + properties: { + $date: { + type: 'string', + format: 'date-time', + }, + }, + required: ['$date'], + additionalProperties: false, + }, + DBRef: { + type: 'object', + properties: { + $ref: { + type: 'string', + }, + $id: {}, + $db: { + type: 'string', + }, + }, + required: ['$ref', '$id'], + additionalProperties: true, + }, + MinKey: { + type: 'object', + properties: { + $minKey: { + type: 'integer', + const: 1, + }, + }, + required: ['$minKey'], + additionalProperties: false, + }, + MaxKey: { + type: 'object', + properties: { + $maxKey: { + type: 'integer', + const: 1, + }, + }, + required: ['$maxKey'], + additionalProperties: false, + }, + Undefined: { + type: 'object', + properties: { + $undefined: { + type: 'boolean', + const: true, + }, + }, + required: ['$undefined'], + additionalProperties: false, + }, +}; + +const createConvertInternalToStandard = function () { + const usedDefinitions = new Set(); + + function getUsedDefinitions() { + const filteredDefinitions = Object.fromEntries( + Object.entries(RELAXED_EJSON_DEFINITIONS).filter(([key]) => + usedDefinitions.has(key), + ), + ); + return Object.freeze(filteredDefinitions); + } + + function markUsedDefinition(ref: string) { + usedDefinitions.add(ref.split('/')[2]); + } + + function convertInternalType(internalType: string) { + const type = InternalTypeToStandardTypeMap[internalType]; + if (!type) throw new Error(`Encountered unknown type: ${internalType}`); + return { ...type }; + } + + async function parseType( + type: SchemaType, + signal?: AbortSignal, + ): Promise { + await allowAbort(signal); + const schema: StandardJSONSchema = convertInternalType(type.bsonType); + if (schema.$ref) markUsedDefinition(schema.$ref); + switch (type.bsonType) { + case 'Array': + schema.items = await parseTypes( + (type as ArraySchemaType).types, + signal, + ); + break; + case 'Document': + Object.assign( + schema, + await parseFields((type as DocumentSchemaType).fields, signal), + ); + break; + } + return schema; + } + + function isPlainTypesOnly( + types: StandardJSONSchema[], + ): types is { type: JSONSchema4TypeName }[] { + return types.every( + (definition) => !!definition.type && Object.keys(definition).length === 1, + ); + } + + async function parseTypes( + types: SchemaType[], + signal?: AbortSignal, + ): Promise { + await allowAbort(signal); + const definedTypes = types.filter( + (type) => type.bsonType.toLowerCase() !== 'undefined', + ); + const isSingleType = definedTypes.length === 1; + if (isSingleType) { + return parseType(definedTypes[0], signal); + } + const parsedTypes = await Promise.all( + definedTypes.map((type) => parseType(type, signal)), + ); + if (isPlainTypesOnly(parsedTypes)) { + return { + type: parsedTypes.map(({ type }) => type), + }; + } + return { + anyOf: parsedTypes, + }; + } + + async function parseFields( + fields: DocumentSchemaType['fields'], + signal?: AbortSignal, + ): Promise<{ + required: StandardJSONSchema['required']; + properties: StandardJSONSchema['properties']; + }> { + const required = []; + const properties: StandardJSONSchema['properties'] = {}; + for (const field of fields) { + if (field.probability === 1) required.push(field.name); + properties[field.name] = await parseTypes(field.types, signal); + } + return { required, properties }; + } + + return async function convert( + internalSchema: InternalSchema, + options: { signal?: AbortSignal } = {}, + ): Promise { + const { required, properties } = await parseFields( + internalSchema.fields, + options.signal, + ); + const schema: StandardJSONSchema = { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + required, + properties, + $defs: getUsedDefinitions(), + }; + return schema; + }; +}; + +export function convertInternalToStandard( + internalSchema: InternalSchema, + options: { signal?: AbortSignal } = {}, +): Promise { + return createConvertInternalToStandard()(internalSchema, options); +} diff --git a/packages/mongodb-schema/src/semantic-types/email.ts b/packages/mongodb-schema/src/semantic-types/email.ts new file mode 100644 index 000000000..d60b3dc9f --- /dev/null +++ b/packages/mongodb-schema/src/semantic-types/email.ts @@ -0,0 +1,7 @@ +// from http://stackoverflow.com/questions/46155/validate-email-address-in-javascript +const emailRegex = + /[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/; + +export function isEmail(value: any) { + return emailRegex.test(value); +} diff --git a/packages/mongodb-schema/src/semantic-types/index.ts b/packages/mongodb-schema/src/semantic-types/index.ts new file mode 100644 index 000000000..905b6ede3 --- /dev/null +++ b/packages/mongodb-schema/src/semantic-types/index.ts @@ -0,0 +1,5 @@ +import { isEmail } from './email'; + +export default { + Email: isEmail, +}; diff --git a/packages/mongodb-schema/src/stats.ts b/packages/mongodb-schema/src/stats.ts new file mode 100644 index 000000000..8a6ad3c5b --- /dev/null +++ b/packages/mongodb-schema/src/stats.ts @@ -0,0 +1,105 @@ +import type { + Schema, + ArraySchemaType, + SchemaField, + SchemaType, +} from './schema-analyzer'; + +function widthRecursive(schema?: Schema) { + let width = 0; + if (!schema) { + return width; + } + if (schema.fields === undefined) { + return width; + } + + width += schema.fields.length; + + width += schema.fields + .map((field) => { + const doc = field.types.find((v) => v.name === 'Document'); + return widthRecursive(doc as any); + }) + .reduce((p, c) => p + c || 0, 0); + + width += + schema.fields + .map((field: SchemaField) => { + const arr = field.types.find((v: SchemaType) => v.name === 'Array'); + if (arr) { + const doc = (arr as ArraySchemaType).types.find( + (v) => v.name === 'Document', + ); + return widthRecursive(doc as Schema | undefined); + } + }) + .reduce((p, c) => (p ?? 0) + (c ?? 0) || 0, 0) ?? 0; + return width; +} + +function depthRecursive(schema?: Schema) { + if (!schema) { + return 0; + } + let maxChildDepth = 0; + if (schema.fields !== undefined && schema.fields.length > 0) { + maxChildDepth = + 1 + + Math.max( + Math.max( + ...schema.fields.map((field) => { + const doc = field.types.find((v) => v.name === 'Document'); + return depthRecursive(doc as Schema | undefined); + }), + ), + Math.max( + ...schema.fields.map((field) => { + const arr = field.types.find((v) => v.name === 'Array'); + if (arr) { + const doc = (arr as ArraySchemaType).types.find( + (v) => v.name === 'Document', + ); + return depthRecursive(doc as Schema | undefined); + } + return 0; + }), + ), + ); + } + return maxChildDepth; +} + +function branchingFactors(schema?: Schema) { + const branchArray: any[] = []; + let res; + if (!schema) { + return branchArray; + } + if (schema.fields !== undefined && schema.fields.length > 0) { + branchArray.push(schema.fields.length); + res = schema.fields.map(function (field) { + const doc = field.types.find((v) => v.name === 'Document'); + return branchingFactors(doc as Schema | undefined); + }); + branchArray.push(...res.flat(Infinity)); + res = schema.fields.map(function (field) { + const arr = field.types.find((v) => v.name === 'Array'); + if (arr) { + const doc = (arr as ArraySchemaType).types.find( + (v) => v.name === 'Document', + ); + return branchingFactors(doc as Schema | undefined); + } + return []; + }); + branchArray.push(...res.flat(Infinity)); + } + return branchArray.sort().reverse(); +} + +export default { + width: widthRecursive, + depth: depthRecursive, + branch: branchingFactors, +}; diff --git a/packages/mongodb-schema/src/to-typescript.spec.ts b/packages/mongodb-schema/src/to-typescript.spec.ts new file mode 100644 index 000000000..6b85e83f6 --- /dev/null +++ b/packages/mongodb-schema/src/to-typescript.spec.ts @@ -0,0 +1,281 @@ +import type { MongoDBJSONSchema, StandardJSONSchema } from './types'; +import { analyzeDocuments } from './index'; +import { toTypescriptTypeDefinition } from './to-typescript'; + +import assert from 'assert/strict'; + +import { + BSONRegExp, + Binary, + Code, + DBRef, + Decimal128, + Double, + Int32, + Long, + MaxKey, + MinKey, + ObjectId, + Timestamp, + UUID, + BSONSymbol, +} from 'bson'; + +import { inspect } from 'util'; + +function convertAndCompare(schema: MongoDBJSONSchema, expected: string) { + const code = toTypescriptTypeDefinition(schema); + + try { + assert.equal(code, expected); + } catch (err: any) { + // While the diff you get when this fails is handy, it is much easier to + // just copy/paste the actual code to the expected result once you've + // confirmed it is correct. Also it is nice to be able to see what schema + // you're working with. + + // eslint-disable-next-line no-console + console.log(inspect(schema, { depth: null })); + + // eslint-disable-next-line no-console + console.log(code); + + throw err; + } +} + +describe('toTypescriptTypeDefinition', function () { + it('converts a MongoDB JSON schema to TypeScript', async function () { + const docs = [ + { + _id: new ObjectId('642d766b7300158b1f22e972'), + double: new Double(1.2), // Double, 1, double + doubleThatIsAlsoAnInteger: new Double(1), // Double, 1, double + string: 'Hello, world!', // String, 2, string + object: { key: 'value' }, // Object, 3, object + array: [1, 2, 3], // Array, 4, array + binData: new Binary(Buffer.from([1, 2, 3])), // Binary data, 5, binData + // Undefined, 6, undefined (deprecated) + objectId: new ObjectId('642d766c7300158b1f22e975'), // ObjectId, 7, objectId + boolean: true, // Boolean, 8, boolean + date: new Date('2023-04-05T13:25:08.445Z'), // Date, 9, date + null: null, // Null, 10, null + regex: new BSONRegExp('pattern', 'i'), // Regular Expression, 11, regex + // DBPointer, 12, dbPointer (deprecated) + javascript: new Code('function() {}'), // JavaScript, 13, javascript + symbol: new BSONSymbol('symbol'), // Symbol, 14, symbol (deprecated) + javascriptWithScope: new Code('function() {}', { foo: 1, bar: 'a' }), // JavaScript code with scope 15 "javascriptWithScope" Deprecated in MongoDB 4.4. + int: new Int32(12345), // 32-bit integer, 16, "int" + timestamp: new Timestamp(new Long('7218556297505931265')), // Timestamp, 17, timestamp + long: new Long('123456789123456789'), // 64-bit integer, 18, long + decimal: new Decimal128( + Buffer.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + ), // Decimal128, 19, decimal + minKey: new MinKey(), // Min key, -1, minKey + maxKey: new MaxKey(), // Max key, 127, maxKey + + binaries: { + generic: new Binary(Buffer.from([1, 2, 3]), 0), // 0 + functionData: new Binary(Buffer.from('//8='), 1), // 1 + binaryOld: new Binary(Buffer.from('//8='), 2), // 2 + uuidOld: new Binary(Buffer.from('c//SZESzTGmQ6OfR38A11A=='), 3), // 3 + uuid: new UUID('AAAAAAAA-AAAA-4AAA-AAAA-AAAAAAAAAAAA'), // 4 + md5: new Binary(Buffer.from('c//SZESzTGmQ6OfR38A11A=='), 5), // 5 + encrypted: new Binary(Buffer.from('c//SZESzTGmQ6OfR38A11A=='), 6), // 6 + compressedTimeSeries: new Binary( + Buffer.from( + 'CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==', + 'base64', + ), + 7, + ), // 7 + custom: new Binary(Buffer.from('//8='), 128), // 128 + }, + + dbRef: new DBRef('namespace', new ObjectId('642d76b4b7ebfab15d3c4a78')), // not actually a separate type, just a convention + + // TODO: what about arrays of objects or arrays of arrays or heterogynous types in general + }, + ]; + const analyzedDocuments = await analyzeDocuments(docs); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + + convertAndCompare( + schema, + `{ + _id?: bson.ObjectId; + array?: bson.Double | number)[]; + binaries?: { + binaryOld?: bson.Binary; + compressedTimeSeries?: bson.Binary; + custom?: bson.Binary; + encrypted?: bson.Binary; + functionData?: bson.Binary; + generic?: bson.Binary; + md5?: bson.Binary; + uuid?: bson.Binary; + uuidOld?: bson.Binary; + }; + binData?: bson.Binary; + boolean?: boolean; + date?: bson.Date; + dbRef?: bson.DBPointer; + decimal?: bson.Decimal128; + double?: bson.Double | number; + doubleThatIsAlsoAnInteger?: bson.Double | number; + int?: bson.Int32 | number; + javascript?: bson.Code; + javascriptWithScope?: bson.Code; + long?: bson.Long | number; + maxKey?: bson.MaxKey; + minKey?: bson.MinKey; + null?: null; + object?: { + key?: string; + }; + objectId?: bson.ObjectId; + regex?: bson.BSONRegExp; + string?: string; + symbol?: bson.BSONSymbol; + timestamp?: bson.Timestamp; +}`, + ); + }); + + it('converts a standard JSON schema to TypeScript', function () { + // from https://json-schema.org/learn/miscellaneous-examples#complex-object-with-nested-properties + const schema: StandardJSONSchema = { + $id: 'https://example.com/complex-object.schema.json', + $schema: 'https://json-schema.org/draft/2020-12/schema', + title: 'Complex Object', + type: 'object', + properties: { + name: { + type: 'string', + }, + age: { + type: 'integer', + minimum: 0, + }, + address: { + type: 'object', + properties: { + street: { + type: 'string', + }, + city: { + type: 'string', + }, + state: { + type: 'string', + }, + postalCode: { + type: 'string', + pattern: '\\d{5}', + }, + }, + required: ['street', 'city', 'state', 'postalCode'], + }, + hobbies: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + required: ['name', 'age'], + }; + + convertAndCompare( + schema, + `{ + name?: string; + age?: number; + address?: { + street?: string; + city?: string; + state?: string; + postalCode?: string; + }; + hobbies?: string[]; +}`, + ); + }); + + it('deals with inconsistent types', async function () { + const docs = [ + { + a: 1, + }, + { + a: 'foo', + }, + { + a: true, + }, + { + a: null, + }, + ]; + + const analyzedDocuments = await analyzeDocuments(docs); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + + convertAndCompare( + schema, + `{ + a?: bson.Double | number | string | boolean | null; +}`, + ); + }); + + it('deals with nested arrays', async function () { + const docs = [ + { + a: [['foo']], + b: [[{ b: 'foo' }]], + }, + ]; + + const analyzedDocuments = await analyzeDocuments(docs); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + + convertAndCompare( + schema, + `{ + a?: any[]; + b?: any[]; +}`, + ); + }); + + it('deals with nested objects', async function () { + const docs = [ + { + a: { + foo: { bar: 'baz' }, + }, + b: [ + { + foo: { bar: 'baz' }, + }, + ], + }, + ]; + + const analyzedDocuments = await analyzeDocuments(docs); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + + convertAndCompare( + schema, + `{ + a?: { + foo?: { + bar?: string; + }; + }; + b?: any[]; +}`, + ); + }); +}); diff --git a/packages/mongodb-schema/src/to-typescript.ts b/packages/mongodb-schema/src/to-typescript.ts new file mode 100644 index 000000000..7b2c181e6 --- /dev/null +++ b/packages/mongodb-schema/src/to-typescript.ts @@ -0,0 +1,147 @@ +import type { JSONSchema } from './types'; + +function getBSONType(property: JSONSchema): string | string[] | undefined { + return property.bsonType || property.type; +} + +function assertIsDefined( + value: T, + message: string, +): asserts value is NonNullable { + if (value === undefined || value === null) { + throw new Error(message); + } +} + +function toTypeName(type: string): string | string[] { + switch (type) { + // JSON Schema types + case 'string': + return 'string'; + + case 'number': + case 'integer': + return 'number'; + + case 'boolean': + return 'boolean'; + + case 'null': + return 'null'; + + // BSON types + // see InternalTypeToBsonTypeMap in mongodb-schema: + // https://github.com/mongodb-js/mongodb-schema/blob/5ca185a6967e0f0d1bb20f75555d3f4f1f9c24fe/src/schema-converters/internalToMongoDB.ts#L8 + case 'double': + return ['bson.Double', 'number']; + + case 'binData': + return 'bson.Binary'; + case 'objectId': + return 'bson.ObjectId'; + + case 'bool': + return 'boolean'; + + case 'date': + return 'bson.Date'; + + case 'regex': + return 'bson.BSONRegExp'; + + case 'symbol': + return 'bson.BSONSymbol'; + + case 'javascript': + case 'javascriptWithScope': + return 'bson.Code'; + + case 'int': + return ['bson.Int32', 'number']; + + case 'timestamp': + return 'bson.Timestamp'; + + case 'long': + return ['bson.Long', 'number']; + + case 'decimal': + return 'bson.Decimal128'; + + case 'minKey': + return 'bson.MinKey'; + + case 'maxKey': + return 'bson.MaxKey'; + + case 'dbPointer': + return 'bson.DBPointer'; + + case 'undefined': + return 'undefined'; + + default: + return 'any'; + } +} + +function uniqueTypes(property: JSONSchema): Set { + const type = getBSONType(property); + const types = (Array.isArray(type) ? type : [type ?? 'any']) + .map((t) => toTypeName(t)) + .flat(); + return new Set(types.flat()); +} + +function indentSpaces(indent: number) { + const spaces = []; + for (let i = 0; i < indent; i++) { + spaces.push(' '); + } + return spaces.join(''); +} + +function arrayType(types: string[]) { + if (types.length === 1) { + return `${types[0]}[]`; + } + return `${types.join(' | ')})[]`; +} + +function toTypescriptType( + properties: Record, + indent: number, +): string { + const eachFieldDefinition = Object.entries(properties).map( + ([propertyName, schema]) => { + switch (getBSONType(schema)) { + case 'array': + assertIsDefined(schema.items, 'schema.items must be defined'); + return `${indentSpaces(indent)}${propertyName}?: ${arrayType([ + ...uniqueTypes(schema.items), + ])}`; + case 'object': + assertIsDefined( + schema.properties, + 'schema.properties must be defined', + ); + return `${indentSpaces(indent)}${propertyName}?: ${toTypescriptType( + schema.properties as Record, + indent + 1, + )}`; + default: + return `${indentSpaces(indent)}${propertyName}?: ${[ + ...uniqueTypes(schema), + ].join(' | ')}`; + } + }, + ); + + return `{\n${eachFieldDefinition.join(';\n')};\n${indentSpaces(indent - 1)}}`; +} + +export function toTypescriptTypeDefinition(schema: JSONSchema): string { + assertIsDefined(schema.properties, 'schema.properties must be defined'); + + return toTypescriptType(schema.properties as Record, 1); +} diff --git a/packages/mongodb-schema/src/types.ts b/packages/mongodb-schema/src/types.ts new file mode 100644 index 000000000..b69cb152f --- /dev/null +++ b/packages/mongodb-schema/src/types.ts @@ -0,0 +1,44 @@ +import { type JSONSchema4 } from 'json-schema'; +import type { InternalSchema } from '.'; + +export type StandardJSONSchema = JSONSchema4; + +export type MongoDBJSONSchema = Pick< + StandardJSONSchema, + 'title' | 'required' | 'description' +> & { + bsonType?: string | string[]; + properties?: Record; + items?: MongoDBJSONSchema | MongoDBJSONSchema[]; + anyOf?: MongoDBJSONSchema[]; +}; + +export type ExpandedJSONSchema = StandardJSONSchema & { + ['x-bsonType']?: string | string[]; + ['x-metadata']?: { + hasDuplicates?: boolean; + probability: number; + count: number; + }; + ['x-sampleValues']?: any[]; + properties?: Record; + items?: ExpandedJSONSchema | ExpandedJSONSchema[]; + anyOf?: ExpandedJSONSchema[]; +}; + +export type JSONSchema = Partial & MongoDBJSONSchema; + +export type AnyIterable = Iterable | AsyncIterable; + +type AnySchema = + | InternalSchema + | StandardJSONSchema + | MongoDBJSONSchema + | ExpandedJSONSchema; +export type SchemaConverterFn< + InputSchema = AnySchema, + OutputSchema = AnySchema, +> = ( + input: InputSchema, + options: { signal?: AbortSignal }, +) => Promise; diff --git a/packages/mongodb-schema/src/util.ts b/packages/mongodb-schema/src/util.ts new file mode 100644 index 000000000..cf5e17a24 --- /dev/null +++ b/packages/mongodb-schema/src/util.ts @@ -0,0 +1,11 @@ +export const ALLOW_ABORT_INTERVAL_COUNT = 1000; + +export async function allowAbort(signal?: AbortSignal) { + return new Promise((resolve, reject) => + setTimeout(() => { + if (signal?.aborted) + return reject(signal?.reason || new Error('Operation aborted')); + resolve(); + }), + ); +} diff --git a/packages/mongodb-schema/test/all-bson-types-fixture.ts b/packages/mongodb-schema/test/all-bson-types-fixture.ts new file mode 100644 index 000000000..87d55ee40 --- /dev/null +++ b/packages/mongodb-schema/test/all-bson-types-fixture.ts @@ -0,0 +1,79 @@ +import { + BSONRegExp, + Binary, + Code, + DBRef, + Decimal128, + Double, + Int32, + Long, + MaxKey, + MinKey, + ObjectId, + Timestamp, + UUID, + BSONSymbol, +} from 'bson'; + +export const allBSONTypesDoc = { + _id: new ObjectId('642d766b7300158b1f22e972'), + double: new Double(1.2), // Double, 1, double + string: 'Hello, world!', // String, 2, string + object: { key: 'value' }, // Object, 3, object + array: [1, 2, 3], // Array, 4, array + binData: new Binary(Buffer.from([1, 2, 3])), // Binary data, 5, binData + // Undefined, 6, undefined (deprecated) + objectId: new ObjectId('642d766c7300158b1f22e975'), // ObjectId, 7, objectId + boolean: true, // Boolean, 8, boolean + date: new Date('2023-04-05T13:25:08.445Z'), // Date, 9, date + null: null, // Null, 10, null + regex: new BSONRegExp('pattern', 'i'), // Regular Expression, 11, regex + // DBPointer, 12, dbPointer (deprecated) + javascript: new Code('function() {}'), // JavaScript, 13, javascript + symbol: new BSONSymbol('symbol'), // Symbol, 14, symbol (deprecated) + javascriptWithScope: new Code('function() {}', { foo: 1, bar: 'a' }), // JavaScript code with scope 15 "javascriptWithScope" Deprecated in MongoDB 4.4. + int: new Int32(12345), // 32-bit integer, 16, "int" + timestamp: new Timestamp(new Long('7218556297505931265')), // Timestamp, 17, timestamp + long: new Long('123456789123456789'), // 64-bit integer, 18, long + decimal: new Decimal128( + Buffer.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + ), // Decimal128, 19, decimal + minKey: new MinKey(), // Min key, -1, minKey + maxKey: new MaxKey(), // Max key, 127, maxKey + + binaries: { + generic: new Binary(Buffer.from([1, 2, 3]), 0), // 0 + functionData: Binary.createFromBase64('//8=', 1), // 1 + binaryOld: Binary.createFromBase64('//8=', 2), // 2 + uuidOld: Binary.createFromBase64('c//SZESzTGmQ6OfR38A11A==', 3), // 3 + uuid: new UUID('AAAAAAAA-AAAA-4AAA-AAAA-AAAAAAAAAAAA'), // 4 + md5: Binary.createFromBase64('c//SZESzTGmQ6OfR38A11A==', 5), // 5 + encrypted: Binary.createFromBase64('c//SZESzTGmQ6OfR38A11A==', 6), // 6 + compressedTimeSeries: new Binary( + Buffer.from( + 'CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==', + 'base64', + ), + 7, + ), // 7 + custom: Binary.createFromBase64('//8=', 128), // 128 + }, + + dbRef: new DBRef('namespace', new ObjectId('642d76b4b7ebfab15d3c4a78')), // not actually a separate type, just a convention +}; + +const { dbRef, ...allValidBSONTypesDoc } = allBSONTypesDoc; + +// Includes some edge cases like empty objects, nested empty arrays, etc. +export const allValidBSONTypesWithEdgeCasesDoc = { + ...allValidBSONTypesDoc, + emptyObject: {}, + objectWithNestedEmpty: { + nestedEmpty: {}, + }, + emptyArray: [], + arrayOfEmptyArrays: [[], []], + infinityNum: Infinity, + negativeInfinityNum: -Infinity, + NaNNum: NaN, +}; diff --git a/packages/mongodb-schema/test/all-bson-types.test.ts b/packages/mongodb-schema/test/all-bson-types.test.ts new file mode 100644 index 000000000..cf0357546 --- /dev/null +++ b/packages/mongodb-schema/test/all-bson-types.test.ts @@ -0,0 +1,46 @@ +import assert from 'assert'; + +import type { Schema } from '../src/schema-analyzer'; +import getSchema from '../src'; +import { allBSONTypesDoc } from './all-bson-types-fixture'; + +describe('using a document with all bson types', function () { + let schema: Schema; + before(async function () { + schema = await getSchema([allBSONTypesDoc]); + }); + + it('contains all of the types', function () { + assert.equal(schema.count, 1); + + const fieldTypes = [ + 'Array', + 'Binary', + 'Boolean', + 'Code', + 'CodeWScope', + 'Date', + 'Decimal128', + 'Double', + 'Int32', + 'Long', + 'MaxKey', + 'MinKey', + 'Null', + 'Document', + 'ObjectId', + 'BSONRegExp', + 'String', + 'BSONSymbol', + 'Timestamp', + ]; + + for (const type of fieldTypes) { + assert.equal( + !!schema.fields.find((schemaField) => schemaField.type === type), + true, + `Cannot find type "${type}" in schemaField types: ${JSON.stringify(schema.fields, null, 2)}`, + ); + } + }); +}); diff --git a/packages/mongodb-schema/test/array-object-types.test.ts b/packages/mongodb-schema/test/array-object-types.test.ts new file mode 100644 index 000000000..470b57390 --- /dev/null +++ b/packages/mongodb-schema/test/array-object-types.test.ts @@ -0,0 +1,104 @@ +import getSchema from '../src'; +import assert from 'assert'; +import type { + ArraySchemaType, + Schema, + SchemaField, + SchemaType, +} from '../src/schema-analyzer'; + +describe('arrays and objects as type (INT-203 restructuring)', function () { + const docs = [ + { + x: [1, 2, 3], + }, + { + x: 'foo', + }, + { + x: { + b: 1, + }, + }, + { + x: ['bar', null, false], + }, + { + x: [ + { + c: 1, + d: 1, + }, + { + c: 2, + }, + ], + }, + { + e: 1, + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + describe('Field', function () { + let x: SchemaField | undefined; + before(function () { + x = schema.fields.find((v) => v.name === 'x'); + }); + it('have the right type distribution of x', function () { + const names = x?.types.map((v) => v.name); + const probabilities = x?.types.map((v) => v.probability); + const dist = names?.reduce( + (p, c, i) => ({ ...p, [c]: probabilities?.[i] }), + {}, + ); + assert.deepEqual(dist, { + Array: 3 / 6, + String: 1 / 6, + Document: 1 / 6, + Undefined: 1 / 6, + }); + }); + }); + + describe('Nested Array', function () { + let arr: SchemaType | undefined; + + before(function () { + const types = schema.fields.find((v) => v.name === 'x')?.types; + arr = types?.find((v) => v.name === 'Array'); + }); + + it('should return the lengths of all encountered arrays', function () { + assert.deepEqual((arr as ArraySchemaType).lengths, [3, 3, 2]); + }); + + it('should return the probability of x being an array', function () { + assert.equal(arr?.probability, 3 / 6); + }); + + it('should return the total count of all containing values', function () { + assert.equal((arr as ArraySchemaType).totalCount, 8); + }); + + it('should return the type distribution inside an array', function () { + const names = (arr as ArraySchemaType).types.map((v) => v.name); + const probabilities = (arr as ArraySchemaType).types.map( + (v) => v.probability, + ); + const arrDist = names.reduce( + (p, c, i) => ({ ...p, [c]: probabilities[i] }), + {}, + ); + assert.deepEqual(arrDist, { + Number: 3 / 8, + String: 1 / 8, + Null: 1 / 8, + Boolean: 1 / 8, + Document: 2 / 8, + }); + }); + }); +}); diff --git a/packages/mongodb-schema/test/basic-embedded-array.test.ts b/packages/mongodb-schema/test/basic-embedded-array.test.ts new file mode 100644 index 000000000..de2004912 --- /dev/null +++ b/packages/mongodb-schema/test/basic-embedded-array.test.ts @@ -0,0 +1,57 @@ +import assert from 'assert'; +import BSON from 'bson'; + +import getSchema from '../src'; +import type { ArraySchemaType } from '../src/schema-analyzer'; + +describe('basic embedded array', function () { + let followingIds: ArraySchemaType; + const docs = [ + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d71'), + followingIds: [new BSON.ObjectId('55582407aafa8fbbc57196e2')], + }, + { + _id: new BSON.ObjectId('55582407aafa8fbbc57196e2'), + followingIds: [ + new BSON.ObjectId('55581e0a9bf712d0c2b48d71'), + '55581e0a9bf712d0c2b48d71', + ], + }, + ]; + + before(async function () { + const res = await getSchema(docs); + const types = res.fields.find((v) => v.name === 'followingIds')?.types; + followingIds = types?.find((v) => v.name === 'Array') as ArraySchemaType; + }); + + it('should have 2 lengths for followingIds', function () { + assert.deepEqual(followingIds.lengths, [1, 2]); + }); + + it('should have an average length of 1.5 for followingIds', function () { + assert.equal(followingIds.averageLength, 1.5); + }); + + it('should have a sum of probability for followingIds of 1', function () { + const expectedSum = followingIds.types + .map((v) => v.probability) + .reduce((p, c) => p + c || 0, 0); + assert.equal(expectedSum, 1); + }); + + it('should have 33% String for followingIds', function () { + assert.equal( + followingIds.types.find((v) => v.name === 'String')?.probability, + 1 / 3, + ); + }); + + it('should have 66% ObjectId for followingIds', function () { + assert.equal( + followingIds.types.find((v) => v.name === 'ObjectId')?.probability, + 2 / 3, + ); + }); +}); diff --git a/packages/mongodb-schema/test/basic-embedded-documents.test.ts b/packages/mongodb-schema/test/basic-embedded-documents.test.ts new file mode 100644 index 000000000..ffa29caec --- /dev/null +++ b/packages/mongodb-schema/test/basic-embedded-documents.test.ts @@ -0,0 +1,75 @@ +import assert from 'assert'; +import BSON from 'bson'; + +import getSchema from '../src'; +import type { + Schema, + DocumentSchemaType, + SchemaField, +} from '../src/schema-analyzer'; + +describe('basic embedded documents', function () { + const docs = [ + { + _id: new BSON.ObjectId('55582407aafa8fbbc57196e2'), + name: 'Brett Flowers', + email: { + _id: 'gohu@pum.io', + is_verified: false, + }, + twitter: { + username: '@lekbisova', + }, + stats: { + friends: 7584, + }, + push_token: { + android: undefined, + apple: + '4e2e068cd281cfe924ff3174dfe363bd3108a5852ca5197f37c40c1bca6e1a4c', + }, + last_address: { + latitude: null, + longitude: null, + }, + created_at: new Date(), + }, + ]; + + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should detect all fields names and nested paths', function () { + const fieldNames = [ + '_id', + 'created_at', + 'email', + 'last_address', + 'name', + 'push_token', + 'stats', + 'twitter', + ]; + + const nestedFieldPaths = [ + ['push_token', 'android'], + ['push_token', 'apple'], + ]; + + assert.deepEqual( + schema.fields.map((v) => v.name), + fieldNames, + ); + + const types = schema.fields.find((v) => v.name === 'push_token')?.types; + const pushTokens = ( + types?.find((v) => v.name === 'Document') as DocumentSchemaType + )?.fields; + assert.deepEqual( + pushTokens?.map((v: SchemaField) => v.path), + nestedFieldPaths, + ); + }); +}); diff --git a/packages/mongodb-schema/test/basic-probability.test.ts b/packages/mongodb-schema/test/basic-probability.test.ts new file mode 100644 index 000000000..12163b37b --- /dev/null +++ b/packages/mongodb-schema/test/basic-probability.test.ts @@ -0,0 +1,36 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import type { Schema } from '../src/schema-analyzer'; + +describe('simple probability', function () { + const docs = [ + { + _id: 1, + registered: true, + }, + { + _id: 2, + }, + ]; + + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + + if (!schema.fields.find((v) => v.name === 'registered')) { + throw new Error('Did not pick up `registered` field'); + } + }); + it('should have a probability of 50% for `registered` to be a boolean', function () { + const types = schema.fields.find((v) => v.name === 'registered')?.types; + assert.equal(types?.find((v) => v.name === 'Boolean')?.probability, 1 / 2); + }); + it('should have a probability of 50% for `registered` to be undefined', function () { + const types = schema.fields.find((v) => v.name === 'registered')?.types; + assert.equal( + types?.find((v) => v.name === 'Undefined')?.probability, + 1 / 2, + ); + }); +}); diff --git a/packages/mongodb-schema/test/basic-unique.test.ts b/packages/mongodb-schema/test/basic-unique.test.ts new file mode 100644 index 000000000..533a5f22a --- /dev/null +++ b/packages/mongodb-schema/test/basic-unique.test.ts @@ -0,0 +1,134 @@ +/* eslint-disable mocha/max-top-level-suites */ +import assert from 'assert'; +import bson from 'bson'; + +import getSchema from '../src'; +import type { Schema, PrimitiveSchemaType } from '../src/schema-analyzer'; + +describe('hasDuplicates', function () { + const docs: { + num: number; + str: string; + }[] = []; + for (let i = 0; i <= 11111; i++) { + docs.push({ + num: i, + str: String(i), + }); + } + + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should not have duplicates', function () { + assert.equal( + schema.fields.find((v) => v.name === 'num')?.hasDuplicates, + false, + ); + }); + + it('should have 10000 number values for the `num` field', function () { + const types = schema.fields.find((v) => v.name === 'num')?.types; + assert.equal( + (types?.find((v) => v.name === 'Number') as PrimitiveSchemaType)?.values + .length, + 10000, + ); + }); + + it('should have 100 string values for the `str` field', function () { + const types = schema.fields.find((v) => v.name === 'str')?.types; + assert.equal( + (types?.find((v) => v.name === 'String') as PrimitiveSchemaType)?.values + .length, + 100, + ); + }); +}); + +describe('unique', function () { + const docs = [ + { + _id: 1, + registered: true, + b: false, + int32: new bson.Int32(5), + date: new Date('2016-01-01'), + }, + { + _id: 2, + registered: true, + code: null, + b: 'false', + int32: new bson.Int32(5), + date: new Date('2016-01-01'), + }, + { + _id: 3, + code: null, + int32: new bson.Int32(9), + date: new Date('2011-11-11'), + }, + ]; + + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have count of 3 for `_id`', function () { + assert.equal(schema.fields.find((v) => v.name === '_id')?.count, 3); + }); + + it('should have unique of 3 for `_id`', function () { + const types = schema.fields.find((v) => v.name === '_id')?.types; + assert.equal(types?.find((v) => v.name === 'Number')?.unique, 3); + }); + + it('should not have duplicates for `_id`', function () { + assert.equal( + schema.fields.find((v) => v.name === '_id')?.hasDuplicates, + false, + ); + }); + + it('should have count of 2 for `registered`', function () { + assert.equal(schema.fields.find((v) => v.name === 'registered')?.count, 2); + }); + + it('should have unique of 1 for `registered` type Boolean', function () { + const types = schema.fields.find((v) => v.name === 'registered')?.types; + assert.equal(types?.find((v) => v.name === 'Boolean')?.unique, 1); + }); + + it('should have unique of 1 for `code`', function () { + const types = schema.fields.find((v) => v.name === 'code')?.types; + assert.equal(types?.find((v) => v.name === 'Null')?.unique, 1); + }); + + it('should have unique of 2 for `int32`', function () { + const types = schema.fields.find((v) => v.name === 'int32')?.types; + assert.equal(types?.find((v) => v.name === 'Int32')?.unique, 2); + }); + + it('should have unique of 2 for `date`', function () { + const types = schema.fields.find((v) => v.name === 'date')?.types; + assert.equal(types?.find((v) => v.name === 'Date')?.unique, 2); + }); + + it('should not have duplicate values for b', function () { + assert.equal( + schema.fields.find((v) => v.name === 'b')?.hasDuplicates, + false, + ); + }); + + it('should have duplicates for `registered`', function () { + assert.equal( + schema.fields.find((v) => v.name === 'registered')?.hasDuplicates, + true, + ); + }); +}); diff --git a/packages/mongodb-schema/test/basic.test.ts b/packages/mongodb-schema/test/basic.test.ts new file mode 100644 index 000000000..ab28cb3a9 --- /dev/null +++ b/packages/mongodb-schema/test/basic.test.ts @@ -0,0 +1,91 @@ +import assert from 'assert'; +import BSON from 'bson'; + +import type { Schema } from '../src/schema-analyzer'; +import getSchema from '../src'; + +describe('using only basic fields', function () { + const docs = [ + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d71'), + email: 'tupjud@weigehib.gov', + is_verified: false, + twitter_username: '@zaetisi', + name: 'Hunter Maxwell', + stats_friends: 2163, + apple_push_token: + 'd4b4e7f3361cec05fae848575d7e6e1da2f0dccdf8ccc86a8ff2124d8b0542f6', + android_push_token: undefined, + last_address_latitude: null, + last_address_longitude: null, + created_at: new Date(), + length: 29, + 'name[]': 'Annabeth Frankie', + toString: 42, + }, + ]; + + let users: Schema; + it('should detect all fields', function () { + const fieldNames = [ + '_id', + 'android_push_token', + 'apple_push_token', + 'created_at', + 'email', + 'is_verified', + 'last_address_latitude', + 'last_address_longitude', + 'length', + 'name', + 'stats_friends', + 'twitter_username', + 'name[]', + 'toString', + ]; + assert.deepEqual(users.fields.map((v) => v.name).sort(), fieldNames.sort()); + }); + + before(async function () { + users = await getSchema(docs); + }); + + it('should detect the correct type for each field', function () { + assert.equal(users.fields.find((v) => v.name === '_id')?.type, 'ObjectId'); + assert.equal( + users.fields.find((v) => v.name === 'apple_push_token')?.type, + 'String', + ); + assert.equal( + users.fields.find((v) => v.name === 'created_at')?.type, + 'Date', + ); + assert.equal(users.fields.find((v) => v.name === 'email')?.type, 'String'); + assert.equal( + users.fields.find((v) => v.name === 'is_verified')?.type, + 'Boolean', + ); + assert.equal(users.fields.find((v) => v.name === 'length')?.type, 'Number'); + assert.equal( + users.fields.find((v) => v.name === 'last_address_latitude')?.type, + 'Null', + ); + assert.equal( + users.fields.find((v) => v.name === 'last_address_longitude')?.type, + 'Null', + ); + assert.equal(users.fields.find((v) => v.name === 'name')?.type, 'String'); + assert.equal( + users.fields.find((v) => v.name === 'stats_friends')?.type, + 'Number', + ); + assert.equal( + users.fields.find((v) => v.name === 'twitter_username')?.type, + 'String', + ); + assert.equal( + users.fields.find((v) => v.name === 'toString')?.type, + 'Number', + ); + }); +}); diff --git a/packages/mongodb-schema/test/bloated.test.ts b/packages/mongodb-schema/test/bloated.test.ts new file mode 100644 index 000000000..eeec6d347 --- /dev/null +++ b/packages/mongodb-schema/test/bloated.test.ts @@ -0,0 +1,142 @@ +import assert from 'assert'; +import { Binary, Code } from 'bson'; + +import type { PrimitiveSchemaType } from '../src/schema-analyzer'; +import getSchema from '../src'; + +function generateRandomString(length: number) { + const chars = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < length; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +describe('bloated documents', function () { + describe('sizeable sample values', function () { + it('really long string is cropped', async function () { + const documents = [ + { + str: generateRandomString(20000), + }, + ]; + const schema = await getSchema(documents); + const stringLength = ( + (schema.fields[0].types[0] as PrimitiveSchemaType).values[0] as string + ).length; + assert.ok(stringLength <= 10000); + }); + + it('really long code is cropped', async function () { + const documents = [ + { + code: new Code(generateRandomString(20000)), + }, + ]; + const schema = await getSchema(documents); + const codeLength = ( + (schema.fields[0].types[0] as PrimitiveSchemaType).values[0] as Code + ).code.length; + assert.ok(codeLength <= 10000); + }); + + it('really long binary is cropped', async function () { + const documents = [ + { + binData: new Binary(Buffer.from(generateRandomString(20000)), 2), + }, + ]; + const schema = await getSchema(documents); + const binary = (schema.fields[0].types[0] as PrimitiveSchemaType) + .values[0] as Binary; + assert.ok(binary.length() <= 10000); + assert.strictEqual(binary.sub_type, 2); + }); + + it('the limit is configurable', async function () { + const documents = [ + { + str: generateRandomString(20000), + }, + ]; + const schema = await getSchema(documents, { storedValuesLengthLimit: 5 }); + const stringLength = ( + (schema.fields[0].types[0] as PrimitiveSchemaType).values[0] as string + ).length; + assert.ok(stringLength === 5); + }); + }); + + describe('high complexity', function () { + it('aborts after reaching the given limit', async function () { + const documents = [ + { + field1: 'abc', + field2: 'bca', + field3: 'cba', + field4: 'cab', + field5: 'bac', + }, + ]; + try { + await getSchema(documents, { distinctFieldsAbortThreshold: 4 }); + assert.fail('Analysis did not throw'); + } catch (error) { + assert.strictEqual( + (error as Error).message, + 'Schema analysis aborted: Fields count above 4', + ); + } + }); + + it('aborts after reaching the given limit - nested', async function () { + const documents = [ + { + field1: { + field2: { + field3: 'abc', + field4: 'bca', + }, + field5: 'cab', + }, + }, + ]; + try { + await getSchema(documents, { distinctFieldsAbortThreshold: 4 }); + assert.fail('Analysis did not throw'); + } catch (error) { + assert.strictEqual( + (error as Error).message, + 'Schema analysis aborted: Fields count above 4', + ); + } + }); + + it('does not count the same field in different documents', async function () { + const documents = [ + { + field1: { + field2: { + field3: 'abc', + }, + }, + }, + { + field1: { + field2: { + field3: 'bca', + }, + }, + }, + ]; + try { + await getSchema(documents, { distinctFieldsAbortThreshold: 4 }); + assert.ok('Analysis finished'); + } catch (error) { + assert.fail('Analysis aborted unexpectedly'); + } + }); + }); +}); diff --git a/packages/mongodb-schema/test/field-order.test.ts b/packages/mongodb-schema/test/field-order.test.ts new file mode 100644 index 000000000..538438d9f --- /dev/null +++ b/packages/mongodb-schema/test/field-order.test.ts @@ -0,0 +1,37 @@ +import assert from 'assert'; + +import getSchema from '../src'; + +describe('order of fields', function () { + it('should have _id fields always at top, even with uppercase fields', async function () { + const docs = [ + { + FOO: 1, + _id: 1, + BAR: 1, + zoo: 1, + }, + ]; + const schema = await getSchema(docs); + assert.deepEqual( + schema.fields.map((v) => v.name), + ['_id', 'BAR', 'FOO', 'zoo'], + ); + }); + it('should sort keys in case-insensitive manner', async function () { + const docs = [ + { + cb: 1, + Ca: 1, + cC: 1, + a: 1, + b: 1, + }, + ]; + const schema = await getSchema(docs); + assert.deepEqual( + schema.fields.map((v) => v.name), + ['a', 'b', 'Ca', 'cb', 'cC'], + ); + }); +}); diff --git a/packages/mongodb-schema/test/get-schema-paths.test.ts b/packages/mongodb-schema/test/get-schema-paths.test.ts new file mode 100644 index 000000000..ed536cb95 --- /dev/null +++ b/packages/mongodb-schema/test/get-schema-paths.test.ts @@ -0,0 +1,113 @@ +import assert from 'assert'; +import type { Document } from 'bson'; + +import { getSchemaPaths } from '../src'; + +describe('getSchemaPaths', function () { + let schemaPaths: string[][]; + + describe('with fields with dots in them', function () { + const docs = [ + { + pineapple: { + orange: { + apple: 1, + }, + 'orange.with.dot': { + bah: 2, + }, + }, + }, + ]; + + before(async function () { + schemaPaths = await getSchemaPaths(docs); + }); + + it('returns an array of the fields', function () { + assert.deepEqual(schemaPaths, [ + ['pineapple'], + ['pineapple', 'orange'], + ['pineapple', 'orange', 'apple'], + ['pineapple', 'orange.with.dot'], + ['pineapple', 'orange.with.dot', 'bah'], + ]); + }); + }); + + describe('with multiple documents with different fields', function () { + const docs = [ + { + pineapple: { + orange: { + apple: 1, + }, + }, + }, + { + pineapple: { + orange: 'ok', + }, + }, + { + pineapple: ['test', '123'], + clementine: false, + }, + ]; + + before(async function () { + schemaPaths = await getSchemaPaths(docs); + }); + + it('returns all of the field paths (sorted)', function () { + assert.deepEqual(schemaPaths, [ + ['clementine'], + ['pineapple'], + ['pineapple', 'orange'], + ['pineapple', 'orange', 'apple'], + ]); + }); + }); + + describe('with nested array documents', function () { + const docs = [ + { + orangutan: [ + { + tuatara: 'yes', + lizard: { + snakes: false, + birds: false, + }, + }, + ], + }, + ]; + + before(async function () { + schemaPaths = await getSchemaPaths(docs); + }); + + it('returns all of the field paths (sorted)', function () { + assert.deepEqual(schemaPaths, [ + ['orangutan'], + ['orangutan', 'lizard'], + ['orangutan', 'lizard', 'birds'], + ['orangutan', 'lizard', 'snakes'], + ['orangutan', 'tuatara'], + ]); + }); + }); + + describe('with no documents', function () { + const docs: Document[] = []; + + before(async function () { + schemaPaths = await getSchemaPaths(docs); + }); + + it('returns no paths', function () { + assert.deepEqual(schemaPaths, []); + }); + }); +}); diff --git a/packages/mongodb-schema/test/integration/generate-and-validate.test.ts b/packages/mongodb-schema/test/integration/generate-and-validate.test.ts new file mode 100644 index 000000000..2d71a6cac --- /dev/null +++ b/packages/mongodb-schema/test/integration/generate-and-validate.test.ts @@ -0,0 +1,235 @@ +/* eslint-disable no-console */ +/* eslint-disable mocha/max-top-level-suites */ +import Ajv2020 from 'ajv/dist/2020'; +import assert from 'assert'; +import { Double, Int32, ObjectId, EJSON } from 'bson'; +import { MongoClient, type Db } from 'mongodb'; +import { mochaTestServer } from '@mongodb-js/compass-test-server'; + +import { allValidBSONTypesWithEdgeCasesDoc } from '../all-bson-types-fixture'; +import { analyzeDocuments } from '../../src'; + +const bsonDocuments = [ + { + _id: new ObjectId('67863e82fb817085a6b0ebad'), + title: 'My book', + year: new Int32(1983), + genres: [ + 'crimi', + 'comedy', + { + short: 'scifi', + long: 'science fiction', + }, + ], + number: Double.fromString('Infinity'), + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebae'), + title: 'Other book', + year: new Int32('1999'), + author: { + name: 'Peter Sonder', + rating: new Double(1.3), + }, + }, +]; + +describe('Documents -> Generate schema -> Validate Documents against the schema', function () { + it('Standard JSON Schema with Relaxed EJSON', async function () { + const ajv = new Ajv2020(); + // First we get the JSON schema from BSON + const analyzedDocuments = await analyzeDocuments(bsonDocuments); + const schema = await analyzedDocuments.getStandardJsonSchema(); + const validate = ajv.compile(schema); + for (const doc of bsonDocuments) { + // Then we get EJSON + const relaxedEJSONDoc = EJSON.serialize(doc, { relaxed: true }); + // And validate it agains the JSON Schema + const valid = validate(relaxedEJSONDoc); + if (validate.errors) console.error('Validation failed', validate.errors); + assert.strictEqual(valid, true); + } + }); +}); + +describe('With a MongoDB Cluster', function () { + let client: MongoClient; + let db: Db; + const cluster = mochaTestServer(); + + before(async function () { + // Connect to the mongodb instance. + const connectionString = cluster().connectionString; + client = new MongoClient(connectionString); + await client.connect(); + db = client.db('test'); + }); + + after(async function () { + await client?.close(); + }); + + describe('Documents -> Generate basic schema -> Use schema in validation rule in MongoDB -> Validate documents against the schema', function () { + before(async function () { + // Create the schema validation rule. + const analyzedDocuments = await analyzeDocuments(bsonDocuments); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + const validationRule = { + $jsonSchema: schema, + }; + + // Create a collection with the schema validation. + await db.createCollection('books', { + validator: validationRule, + }); + }); + + it('allows inserting valid documents', async function () { + await db.collection('books').insertMany(bsonDocuments); + }); + + it('prevents inserting invalid documents', async function () { + const invalidDocs = [ + { + _id: new ObjectId('67863e82fb817085a6b0ebba'), + title: 'Pineapple 1', + year: new Int32(1983), + genres: [ + 'crimi', + 'comedy', + { + short: 'scifi', + long: 'science fiction', + }, + ], + number: 'an invalid string', + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbb'), + title: 'Pineapple 2', + year: 'year a string', + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbc'), + title: 123, + year: new Int32('1999'), + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbc'), + title: 'No year', + }, + ]; + + for (const doc of invalidDocs) { + try { + await db.collection('books').insertOne(doc); + + throw new Error('This should not be reached'); + } catch (e: any) { + const expectedMessage = 'Document failed validation'; + assert.ok( + e.message.includes(expectedMessage), + `Expected error ${String(e.message)} message to include "${expectedMessage}", doc: ${String(doc._id)}`, + ); + } + } + }); + }); + + describe('[All Types] Documents -> Generate basic schema -> Use schema in validation rule in MongoDB -> Validate documents against the schema', function () { + const allTypesCollection = 'allTypes'; + + before(async function () { + await db + .collection(allTypesCollection) + .insertOne(allValidBSONTypesWithEdgeCasesDoc); + const docsFromCollection = await db + .collection(allTypesCollection) + .find({}, { promoteValues: false }) + .toArray(); + + // Create the schema validation rule. + const analyzedDocuments = await analyzeDocuments(docsFromCollection); + const schema = await analyzedDocuments.getMongoDBJsonSchema(); + const validationRule = { + $jsonSchema: schema, + }; + // Update the collection with the schema validation. + await db.command({ + collMod: allTypesCollection, + validator: validationRule, + }); + }); + + it('allows inserting valid documents (does not error)', async function () { + const docs = [ + { + ...allValidBSONTypesWithEdgeCasesDoc, + _id: new ObjectId(), + }, + { + ...allValidBSONTypesWithEdgeCasesDoc, + _id: new ObjectId(), + }, + ]; + + try { + await db.collection(allTypesCollection).insertMany(docs); + } catch (err) { + console.error( + 'Error inserting documents', + EJSON.stringify(err, undefined, 2), + ); + throw err; + } + }); + + it('prevents inserting invalid documents', async function () { + const invalidDocs = [ + { + _id: new ObjectId('67863e82fb817085a6b0ebba'), + title: 'Pineapple 1', + year: new Int32(1983), + genres: [ + 'crimi', + 'comedy', + { + short: 'scifi', + long: 'science fiction', + }, + ], + number: 'an invalid string', + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbb'), + title: 'Pineapple 2', + year: 'year a string', + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbc'), + title: 123, + year: new Int32('1999'), + }, + { + _id: new ObjectId('67863eacfb817085a6b0ebbc'), + title: 'No year', + }, + ]; + + for (const doc of invalidDocs) { + try { + await db.collection(allTypesCollection).insertOne(doc); + + throw new Error('This should not be reached'); + } catch (e: any) { + const expectedMessage = 'Document failed validation'; + assert.ok( + e.message.includes(expectedMessage), + `Expected error ${String(e.message)} message to include "${expectedMessage}", doc: ${String(doc._id)}`, + ); + } + } + }); + }); +}); diff --git a/packages/mongodb-schema/test/mixed-type-evolving-schema.test.ts b/packages/mongodb-schema/test/mixed-type-evolving-schema.test.ts new file mode 100644 index 000000000..2604d5b3c --- /dev/null +++ b/packages/mongodb-schema/test/mixed-type-evolving-schema.test.ts @@ -0,0 +1,81 @@ +import assert from 'assert'; +import BSON from 'bson'; + +import type { SchemaField } from '../src/schema-analyzer'; +import getSchema from '../src'; + +describe('evolving schema', function () { + // The hardest case and really why this module exists at all: proper + // handling for polymorphic schemas. Consider the following scenario: + // + // 1. started out with schema in `only basic fields`. + // 2. then read a blog post about how awesome; embedded documents are. + // 3. then realized what a pain embedded documents are. + let applePushToken: SchemaField | undefined; + const docs = [ + { + _id: new BSON.ObjectId('55582407aafa8fbbc57196e2'), + name: 'Brett Flowers', + email: { + _id: 'gohu@pum.io', + is_verified: false, + }, + twitter: { + username: '@lekbisova', + }, + stats: { + friends: 7584, + }, + push_token: { + android: undefined, + apple: + '4e2e068cd281cfe924ff3174dfe363bd3108a5852ca5197f37c40c1bca6e1a4c', + }, + last_address: { + latitude: null, + longitude: null, + }, + created_at: new Date(), + }, + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d71'), + email: 'tupjud@weigehib.gov', + is_verified: false, + twitter_username: '@zaetisi', + name: 'Hunter Maxwell', + stats_friends: 2163, + apple_push_token: + 'd4b4e7f3361cec05fae848575d7e6e1da2f0dccdf8ccc86a8ff2124d8b0542f6', + android_push_token: undefined, + last_address_latitude: null, + last_address_longitude: null, + created_at: new Date(), + }, + ]; + + before(async function () { + const users = await getSchema(docs); + applePushToken = users.fields.find((v) => v.name === 'apple_push_token'); + }); + it('should have the `apple_push_token` field', function () { + assert.ok(applePushToken); + }); + it('should have seen `apple_push_token` 1 time', function () { + assert.equal(applePushToken?.count, 1); + }); + it('should have seen `apple_push_token` in 50% of documents', function () { + assert.equal(applePushToken?.probability, 0.5); + }); + it('should have seen `apple_push_token` 1 time as a string', function () { + assert.equal( + applePushToken?.types.find((v) => v.name === 'String')?.count, + 1, + ); + }); + it('should have seen 1 unique string value for `apple_push_token`', function () { + assert.equal( + applePushToken?.types.find((v) => v.name === 'String')?.unique, + 1, + ); + }); +}); diff --git a/packages/mongodb-schema/test/mixed-type-nested.test.ts b/packages/mongodb-schema/test/mixed-type-nested.test.ts new file mode 100644 index 000000000..a6227a92b --- /dev/null +++ b/packages/mongodb-schema/test/mixed-type-nested.test.ts @@ -0,0 +1,101 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import type { + DocumentSchemaType, + Schema, + SchemaField, +} from '../src/schema-analyzer'; + +describe('mixed types nested', function () { + const docs = [ + { + _id: 1, + address: { + valid: 0, + }, + }, + { + _id: 2, + address: { + valid: false, + }, + }, + { + _id: 3, + address: { + valid: 'None', + }, + }, + { + _id: 4, + address: {}, + }, + { + _id: 5, + address: { + valid: true, + toString: { value: false }, + }, + }, + ] as const; + + let schema: Schema; + let valid: SchemaField | undefined; + + before(async function () { + schema = await getSchema(docs); + if (!schema.fields.find((v) => v.name === '_id')) { + throw new Error('Did not pick up `_id` field'); + } + + valid = ( + schema.fields + .find((v) => v.name === 'address') + ?.types.find((x) => x.name === 'Document') as DocumentSchemaType + )?.fields.find((v) => v.name === 'valid'); + if (!valid) { + throw new Error('Did not pick up `address.valid` field'); + } + if (!valid.types.find((v) => v.name === 'Undefined')) { + throw new Error('Missing Undefined type on `address.valid`'); + } + }); + + it('should see the `address` field is always present', function () { + assert.equal( + schema.fields.find((v) => v.name === 'address')?.probability, + 1, + ); + }); + it('should see the `valid` field in 80% of documents', function () { + assert.equal(valid?.probability, 0.8); + }); + it('should see there are 4 possible types for `valid`', function () { + assert.equal(valid?.types.length, 4); + }); + it('should see `Number` was used in 20% of documents', function () { + assert.equal( + valid?.types.find((v) => v.name === 'Number')?.probability, + 0.2, + ); + }); + it('should see `Boolean` was used in 40% of documents', function () { + assert.equal( + valid?.types.find((v) => v.name === 'Boolean')?.probability, + 0.4, + ); + }); + it('should see `Undefined` was used in 20% of documents', function () { + assert.equal( + valid?.types.find((v) => v.name === 'Undefined')?.probability, + 0.2, + ); + }); + it('should see `String` was used in 20% of documents', function () { + assert.equal( + valid?.types.find((v) => v.name === 'String')?.probability, + 0.2, + ); + }); +}); diff --git a/packages/mongodb-schema/test/mixed-type-order.test.ts b/packages/mongodb-schema/test/mixed-type-order.test.ts new file mode 100644 index 000000000..c52198dd4 --- /dev/null +++ b/packages/mongodb-schema/test/mixed-type-order.test.ts @@ -0,0 +1,47 @@ +import assert from 'assert'; + +import type { SchemaField } from '../src/schema-analyzer'; +import getSchema from '../src'; + +describe('mixed type order', function () { + const docs = [ + { + _id: 2, + }, + { + _id: 3, + registered: '1', + }, + { + _id: 4, + registered: '1', + }, + { + _id: 1, + registered: 1, + }, + ]; + + let registered: SchemaField | undefined; + before(async function () { + const schema = await getSchema(docs); + registered = schema.fields.find((v) => v.name === 'registered'); + + if (!registered) { + throw new Error('Did not pick up `registered` field'); + } + if (!registered.types.find((v) => v.name === 'Undefined')) { + throw new Error('Missing Undefined type on `registered`'); + } + }); + it('should have 3 types for `registered`', function () { + assert.equal(registered?.types.length, 3); + }); + it('should return the order of types as ["String", "Number", "Undefined"]', function (done) { + assert.deepEqual( + registered?.types.map((v) => v.name), + ['String', 'Number', 'Undefined'], + ); + done(); + }); +}); diff --git a/packages/mongodb-schema/test/mixed-type-probability.test.ts b/packages/mongodb-schema/test/mixed-type-probability.test.ts new file mode 100644 index 000000000..737d43a5b --- /dev/null +++ b/packages/mongodb-schema/test/mixed-type-probability.test.ts @@ -0,0 +1,74 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import type { Schema, SchemaField } from '../src/schema-analyzer'; + +describe('mixed type probability', function () { + const docs = [ + { + _id: 1, + registered: 1, + }, + { + _id: 2, + registered: '1', + }, + { + _id: 3, + registered: true, + }, + { + _id: 4, + assigned: true, + }, + ]; + + let schema: Schema; + let registered: SchemaField | undefined; + before(async function () { + schema = await getSchema(docs); + registered = schema.fields.find((v) => v.name === 'registered'); + + if (!registered) { + throw new Error('Did not pick up `registered` field'); + } + if (!registered.types.find((v) => v.name === 'Undefined')) { + throw new Error('Missing Undefined type on `registered`'); + } + }); + it('should have 4 types for `registered`', function () { + assert.equal(registered?.types.length, 4); + }); + it('should have a probability of 25% for `registered` to be a boolean', function () { + assert.equal( + registered?.types.find((v) => v.name === 'Boolean')?.probability, + 1 / 4, + ); + }); + it('should have a probability of 25% for `registered` to be a number', function () { + assert.equal( + registered?.types.find((v) => v.name === 'Number')?.probability, + 1 / 4, + ); + }); + it('should have a probability of 25% for `registered` to be a string', function () { + assert.equal( + registered?.types.find((v) => v.name === 'String')?.probability, + 1 / 4, + ); + }); + it('should have a probability of 25% for `registered` to be undefined', function () { + assert.equal( + registered?.types.find((v) => v.name === 'Undefined')?.probability, + 1 / 4, + ); + }); + it('should compensate for missed Undefined values', function () { + const assigned = schema.fields.find((v) => v.name === 'assigned'); + assert.equal(assigned?.probability, 0.25); + assert.equal( + assigned?.types.find((v) => v.name === 'Boolean')?.probability, + 0.25, + ); + }); +}); diff --git a/packages/mongodb-schema/test/nested-document-path.test.ts b/packages/mongodb-schema/test/nested-document-path.test.ts new file mode 100644 index 000000000..04ef4fba3 --- /dev/null +++ b/packages/mongodb-schema/test/nested-document-path.test.ts @@ -0,0 +1,49 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import type { Schema, DocumentSchemaType } from '../src/schema-analyzer'; + +describe('nested document path', function () { + let schema: Schema; + const docs = [ + { + foo: { + bar: { + baz: 1, + }, + 'bar.with.dot': { + bah: 2, + }, + }, + }, + ]; + + before(async function () { + schema = await getSchema(docs); + }); + + it('should assemble the path correctly fields with dots in their names', function () { + const foo = schema.fields.find((v) => v.name === 'foo'); + const bar = ( + foo?.types.find((v) => v.name === 'Document') as DocumentSchemaType + )?.fields.find((v) => v.name === 'bar'); + const baz = ( + bar?.types.find((v) => v.name === 'Document') as DocumentSchemaType + )?.fields.find((v) => v.name === 'baz'); + assert.ok(foo); + assert.ok(bar); + assert.ok(baz); + assert.deepEqual(baz.path, ['foo', 'bar', 'baz']); + + const barWithDot = ( + foo.types.find((v) => v.name === 'Document') as DocumentSchemaType + )?.fields.find((v) => v.name === 'bar.with.dot'); + assert.ok(barWithDot); + const bah = ( + barWithDot.types.find((v) => v.name === 'Document') as DocumentSchemaType + )?.fields.find((v) => v.name === 'bah'); + assert.ok(bah); + assert.deepEqual(barWithDot.path, ['foo', 'bar.with.dot']); + assert.deepEqual(bah.path, ['foo', 'bar.with.dot', 'bah']); + }); +}); diff --git a/packages/mongodb-schema/test/no-node.test.ts b/packages/mongodb-schema/test/no-node.test.ts new file mode 100644 index 000000000..57486fa40 --- /dev/null +++ b/packages/mongodb-schema/test/no-node.test.ts @@ -0,0 +1,96 @@ +import assert from 'assert'; +import vm from 'vm'; +import fs from 'fs'; +import path from 'path'; + +import type * as mongodbSchema from '..'; + +function createMockModuleSystem() { + // `setTimeout` is available in every JS environment (browsers included), but + // is not part of the ECMAScript spec and so is absent from a bare vm context. + const context = vm.createContext( + Object.assign(Object.create(null), { setTimeout }), + ); + class Module { + exports = {}; + } + const modules = new Map(); + // Tiny (incomplete) CommonJS module system mock + function makeRequire(basename: string) { + return function require(identifier: string): any { + if ( + !identifier.startsWith('./') && + !identifier.startsWith('../') && + !path.isAbsolute(identifier) + ) { + let current = path.dirname(basename); + let previous: string; + do { + const nodeModulesEntry = path.resolve( + current, + 'node_modules', + identifier, + ); + previous = current; + current = path.dirname(current); + if (fs.existsSync(nodeModulesEntry)) { + return require(nodeModulesEntry); + } + } while (previous !== current); + throw new Error( + `mock require() does not support Node.js built-ins (${identifier})`, + ); + } + let file = path.resolve(path.dirname(basename), identifier); + if (!fs.existsSync(file) && fs.existsSync(`${file}.js`)) { + file = `${file}.js`; + } else if (fs.statSync(file).isDirectory()) { + if (fs.existsSync(`${file}/package.json`)) { + const pjson = JSON.parse( + fs.readFileSync(`${file}/package.json`, 'utf8'), + ); + file = path.resolve(file, pjson.main || 'index.js'); + } else if (fs.existsSync(`${file}/index.js`)) { + file = path.resolve(file, 'index.js'); + } + } + const existing = modules.get(file); + if (existing) { + return existing.exports; + } + const module = new Module(); + const source = fs.readFileSync(file); + vm.runInContext( + `(function(require, module, exports, __filename, __dirname) {\n${source.toString('utf8')}\n})`, + context, + )(makeRequire(file), module, module.exports, file, path.dirname(file)); + modules.set(file, module); + return module.exports; + }; + } + return makeRequire; +} + +describe('getSchema should work in plain JS environment without Node.js or browser dependencies', function () { + const docs = [ + { foo: 'bar' }, + { country: 'Croatia' }, + { country: 'Croatia' }, + { country: 'England' }, + ]; + + it('Check if return value is a promise', async function () { + const makeRequire = createMockModuleSystem(); + + const { parseSchema } = makeRequire(__filename)( + '../dist/index.js', + ) as typeof mongodbSchema; + + const result = await parseSchema(docs); + assert.strictEqual(result.count, 4); + assert.strictEqual( + result.fields.map((f) => f.name).join(','), + 'country,foo', + ); + }); +}); diff --git a/packages/mongodb-schema/test/promise.test.ts b/packages/mongodb-schema/test/promise.test.ts new file mode 100644 index 000000000..81e09cf21 --- /dev/null +++ b/packages/mongodb-schema/test/promise.test.ts @@ -0,0 +1,56 @@ +import assert from 'assert'; + +import type { Schema } from '../src/schema-analyzer'; +import getSchema from '../src'; + +describe('getSchema should return promise', function () { + const docs = [ + { foo: 'bar' }, + { country: 'Croatia' }, + { country: 'Croatia' }, + { country: 'England' }, + ]; + + it('Check if return value is a promise', function () { + const result = getSchema(docs); + assert.strictEqual(result instanceof Promise, true); + }); + + it('Check that promise returns expected schema', async function () { + const result = await getSchema(docs); + const fieldNames = result.fields.map((v) => v.name); + assert.deepStrictEqual(fieldNames, ['country', 'foo']); + }); + + describe('Check promise success', function () { + let response: Schema; + + before(async function () { + response = await getSchema(docs); + }); + + it('Using promise should succeed', function () { + assert.equal(response.count, 4); + assert.equal(response.fields.length, 2); + }); + }); + + describe('Check failure', function () { + let errMsg: string | undefined; + + before(async function () { + try { + await getSchema({ foo: 'bar' } as any); + } catch (err: any) { + errMsg = err?.message; + } + }); + + it('Using promise should give an error message', function () { + assert.strictEqual( + errMsg, + 'Unknown input type for `docs`. Must be an array, stream or MongoDB Cursor.', + ); + }); + }); +}); diff --git a/packages/mongodb-schema/test/regression-strings-have-same-probability.test.ts b/packages/mongodb-schema/test/regression-strings-have-same-probability.test.ts new file mode 100644 index 000000000..8e644f04d --- /dev/null +++ b/packages/mongodb-schema/test/regression-strings-have-same-probability.test.ts @@ -0,0 +1,37 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import type { Schema, PrimitiveSchemaType } from '../src/schema-analyzer'; + +describe('regression', function () { + describe('strings have same probability', function () { + const docs = [ + { + _id: 1, + value: 'DUPE', + }, + { + _id: 2, + value: 'DUPE', + }, + { + _id: 3, + value: 'DUPE', + }, + ]; + + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should not dedupe values but return all 3 of them', function () { + const types = schema.fields.find((v) => v.name === 'value')?.types; + assert.equal( + (types?.find((v) => v.name === 'String') as PrimitiveSchemaType)?.values + .length, + 3, + ); + }); + }); +}); diff --git a/packages/mongodb-schema/test/schema-accessor.test.ts b/packages/mongodb-schema/test/schema-accessor.test.ts new file mode 100644 index 000000000..212fa3177 --- /dev/null +++ b/packages/mongodb-schema/test/schema-accessor.test.ts @@ -0,0 +1,61 @@ +import { type InternalSchema } from '../src'; +import sinon from 'sinon'; +import assert from 'assert'; +import type { InternalConverters } from '../src/schema-accessor'; +import { InternalSchemaBasedAccessor } from '../src/schema-accessor'; + +describe('analyzeDocuments', function () { + const internalSchema: InternalSchema = { + count: 1, + fields: [], + }; + let sandbox: sinon.SinonSandbox; + let converters: InternalConverters; + + before(function () { + sandbox = sinon.createSandbox(); + }); + + beforeEach(function () { + converters = { + internalToStandard: sandbox.stub(), + internalToMongoDB: sandbox.stub(), + internalToExpanded: sandbox.stub(), + }; + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('Converts lazily', async function () { + const schema = new InternalSchemaBasedAccessor(internalSchema, converters); + assert.strictEqual( + (converters.internalToStandard as sinon.SinonStub).called, + false, + ); + await schema.getStandardJsonSchema(); + assert.strictEqual( + (converters.internalToStandard as sinon.SinonStub).calledOnce, + true, + ); + }); + + it('Only converts the same format once', async function () { + const schema = new InternalSchemaBasedAccessor(internalSchema, converters); + assert.strictEqual( + (converters.internalToExpanded as sinon.SinonStub).called, + false, + ); + (converters.internalToExpanded as sinon.SinonStub).returns({ + abc: 'string', + }); + await schema.getExpandedJSONSchema(); + await schema.getExpandedJSONSchema(); + await schema.getExpandedJSONSchema(); + assert.strictEqual( + (converters.internalToExpanded as sinon.SinonStub).calledOnce, + true, + ); + }); +}); diff --git a/packages/mongodb-schema/test/semantic-types.test.ts b/packages/mongodb-schema/test/semantic-types.test.ts new file mode 100644 index 000000000..7b8004ac0 --- /dev/null +++ b/packages/mongodb-schema/test/semantic-types.test.ts @@ -0,0 +1,212 @@ +import assert from 'assert'; +import BSON from 'bson'; + +import getSchema from '../src'; +import type { Schema, PrimitiveSchemaType } from '../src/schema-analyzer'; + +describe('options', function () { + const docs = [ + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d71'), + email: 'tick@duck.org', + shape: { + type: 'Point', + coordinates: [20.0, 30.0], + }, + is_verified: false, + }, + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d72'), + email: 'trick@duck.org', + shape: { + type: 'LineString', + coordinates: [ + [20.0, 30.0], + [30.0, 40.0], + ], + }, + is_verified: false, + }, + { + _id: new BSON.ObjectId('55581e0a9bf712d0c2b48d73'), + email: 'track@duck.org', + shape: { + type: 'Polygon', + coordinates: [ + [ + [20.0, 30.0], + [30.0, 40.0], + [15.0, 50.0], + [20.0, 30.0], + ], + ], + }, + is_verified: true, + }, + ]; + + let schema: Schema; + context('when using default options', function () { + beforeEach(async function () { + schema = await getSchema(docs); + }); + + it('stores values by default', function () { + assert.equal( + (schema.fields[0].types[0] as PrimitiveSchemaType).values.length, + 3, + ); + }); + + it('does not use semantic type detection', function () { + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'String', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + assert.equal( + schema.fields.find((v) => v.name === 'shape')?.types[0].name, + 'Document', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'shape')?.types[0] as any) + .bsonType, + 'Document', + ); + }); + }); + + context('when `storeValues` is false', function () { + beforeEach(async function () { + schema = await getSchema(docs, { storeValues: false }); + }); + it('does not store values when `storeValues` is false', function () { + assert.ok(!(schema.fields[0].types[0] as PrimitiveSchemaType).values); + }); + }); + + context('when `semanticTypes` is true', function () { + beforeEach(async function () { + schema = await getSchema(docs, { semanticTypes: true }); + }); + it('calls semantic type detection', function () { + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'Email', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + }); + }); + context('when `semanticTypes` is an object', function () { + context('and all values are boolean', function () { + beforeEach(async function () { + schema = await getSchema(docs, { semanticTypes: { email: true } }); + }); + it('only uses the enabled type detectors', function () { + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'Email', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + }); + }); + context('and values are mixed upper/lower case', function () { + beforeEach(async function () { + schema = await getSchema(docs, { semanticTypes: { eMaIl: true } }); + }); + it('uses the enabled type detectors', function () { + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'Email', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + }); + }); + + context('and all values are custom detector functions', function () { + beforeEach(async function () { + schema = await getSchema(docs, { + semanticTypes: { + Verification: function (value, key) { + return !!key?.pop?.()?.match(/verified/); + }, + }, + }); + }); + it('uses the custom type detectors', function () { + assert.equal( + schema.fields.find((v) => v.name === 'is_verified')?.types[0].name, + 'Verification', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'is_verified')?.types[0] as any) + .bsonType, + 'Boolean', + ); + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'String', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + }); + }); + + context( + 'and values are mixed booleans and custom detector functions', + function () { + beforeEach(async function () { + schema = await getSchema(docs, { + semanticTypes: { + email: true, + Verification: function (value, key) { + return !!key?.pop?.()?.match(/verified/); + }, + }, + }); + }); + it('uses the enabled and custom type detectors', function () { + assert.equal( + schema.fields.find((v) => v.name === 'is_verified')?.types[0].name, + 'Verification', + ); + assert.equal( + ( + schema.fields.find((v) => v.name === 'is_verified') + ?.types[0] as any + ).bsonType, + 'Boolean', + ); + assert.equal( + schema.fields.find((v) => v.name === 'email')?.types[0].name, + 'Email', + ); + assert.equal( + (schema.fields.find((v) => v.name === 'email')?.types[0] as any) + .bsonType, + 'String', + ); + }); + }, + ); + }); +}); diff --git a/packages/mongodb-schema/test/simplified-schema.test.ts b/packages/mongodb-schema/test/simplified-schema.test.ts new file mode 100644 index 000000000..2d9e0465a --- /dev/null +++ b/packages/mongodb-schema/test/simplified-schema.test.ts @@ -0,0 +1,330 @@ +import assert from 'assert'; + +import { getSimplifiedSchema } from '../src'; +import type { SimplifiedSchema } from '../src/schema-analyzer'; +import { allBSONTypesDoc } from './all-bson-types-fixture'; + +const docsFixture = [ + { + foo: 1, + bar: 25, + }, + { + foo: 2, + bar: 'test', + baz: true, + }, + { + foo: 3, + bar: 'another test', + }, + allBSONTypesDoc, +]; + +const expected = { + _id: { + types: [ + { + bsonType: 'ObjectId', + }, + ], + }, + double: { + types: [ + { + bsonType: 'Double', + }, + ], + }, + string: { + types: [ + { + bsonType: 'String', + }, + ], + }, + object: { + types: [ + { + bsonType: 'Document', + fields: { + key: { + types: [ + { + bsonType: 'String', + }, + ], + }, + }, + }, + ], + }, + array: { + types: [ + { + bsonType: 'Array', + types: [ + { + bsonType: 'Number', + }, + ], + }, + ], + }, + binData: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + objectId: { + types: [ + { + bsonType: 'ObjectId', + }, + ], + }, + boolean: { + types: [ + { + bsonType: 'Boolean', + }, + ], + }, + date: { + types: [ + { + bsonType: 'Date', + }, + ], + }, + null: { + types: [ + { + bsonType: 'Null', + }, + ], + }, + regex: { + types: [ + { + bsonType: 'BSONRegExp', + }, + ], + }, + javascript: { + types: [ + { + bsonType: 'Code', + }, + ], + }, + symbol: { + types: [ + { + bsonType: 'BSONSymbol', + }, + ], + }, + javascriptWithScope: { + types: [ + { + bsonType: 'CodeWScope', + }, + ], + }, + int: { + types: [ + { + bsonType: 'Int32', + }, + ], + }, + timestamp: { + types: [ + { + bsonType: 'Timestamp', + }, + ], + }, + long: { + types: [ + { + bsonType: 'Long', + }, + ], + }, + decimal: { + types: [ + { + bsonType: 'Decimal128', + }, + ], + }, + minKey: { + types: [ + { + bsonType: 'MinKey', + }, + ], + }, + maxKey: { + types: [ + { + bsonType: 'MaxKey', + }, + ], + }, + + binaries: { + types: [ + { + bsonType: 'Document', + fields: { + generic: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + functionData: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + binaryOld: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + uuidOld: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + uuid: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + md5: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + encrypted: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + compressedTimeSeries: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + custom: { + types: [ + { + bsonType: 'Binary', + }, + ], + }, + }, + }, + ], + }, + dbRef: { + types: [ + { + bsonType: 'DBRef', + }, + ], + }, + foo: { + types: [ + { + bsonType: 'Number', + }, + ], + }, + bar: { + types: [ + { + bsonType: 'String', + }, + { + bsonType: 'Number', + }, + ], + }, + baz: { + types: [ + { + bsonType: 'Boolean', + }, + ], + }, +}; + +describe('simplified schema', function () { + let schema: SimplifiedSchema; + + before(async function () { + schema = await getSimplifiedSchema(docsFixture); + }); + + it('should assemble a simplified schema', function () { + assert.deepEqual(schema, expected); + }); + + it('contains all of the types', function () { + const fieldTypes = [ + 'Array', + 'Binary', + 'Boolean', + 'Code', + 'Date', + 'Decimal128', + 'Double', + 'Int32', + 'Long', + 'MaxKey', + 'MinKey', + 'Null', + 'Document', + 'ObjectId', + 'BSONRegExp', + 'String', + 'BSONSymbol', + 'Timestamp', + ]; + + for (const fieldType of fieldTypes) { + assert.equal( + !!Object.values(schema).find((schemaType) => + schemaType.types.find( + (schemaType) => schemaType.bsonType === fieldType, + ), + ), + true, + `Cannot find type "${fieldType}" in schemaField types: ${JSON.stringify(schema.fields, null, 2)}`, + ); + } + }); +}); diff --git a/packages/mongodb-schema/test/stats.test.ts b/packages/mongodb-schema/test/stats.test.ts new file mode 100644 index 000000000..8016ec60b --- /dev/null +++ b/packages/mongodb-schema/test/stats.test.ts @@ -0,0 +1,178 @@ +import assert from 'assert'; + +import getSchema from '../src'; +import stats from '../src/stats'; +import type { Schema } from '../src/schema-analyzer'; + +describe('schema statistics', function () { + describe('empty doc', function () { + const docs = [{}]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 0); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 0); + }); + }); + describe('doc with one key', function () { + const docs = [ + { + foo: 'bar', + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 1); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 1); + }); + }); + describe('example 1', function () { + const docs = [ + { + one: [ + 'foo', + 'bar', + { + two: { + three: 3, + }, + }, + 'baz', + ], + foo: 'bar', + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 4); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 3); + }); + }); + describe('example 2', function () { + const docs = [ + { + x: [1, 2, 3], + }, + { + x: 'foo', + }, + { + x: { + b: 1, + }, + }, + { + x: ['bar', null, false], + }, + { + x: [ + { + c: 1, + d: 1, + }, + { + c: 2, + }, + ], + }, + { + e: 1, + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 5); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 2); + }); + }); + describe('example 3', function () { + const docs = [ + { + a: 1, + b: false, + one: { + c: null, + two: { + three: { + four: 4, + e: 'deepest nesting level', + }, + }, + }, + f: { + g: 'not the deepest level', + }, + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 10); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 4); + }); + }); + describe('example 4', function () { + const docs = [ + { + a: { + b: [ + { + c: { + d: [ + { + e: { + f: [ + { + g: [1, 2, 3], + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ]; + let schema: Schema; + before(async function () { + schema = await getSchema(docs); + }); + + it('should have the correct schema width', function () { + assert.equal(stats.width(schema), 7); + }); + it('should have the correct schema depth', function () { + assert.equal(stats.depth(schema), 7); + }); + }); +}); diff --git a/packages/mongodb-schema/test/wrapper.test.ts b/packages/mongodb-schema/test/wrapper.test.ts new file mode 100644 index 000000000..832cb5888 --- /dev/null +++ b/packages/mongodb-schema/test/wrapper.test.ts @@ -0,0 +1,36 @@ +import assert from 'assert'; + +import getSchema from '../src'; + +describe('Convenience Wrapper', function () { + it('should be available as a function', function () { + assert.ok(getSchema); + assert.equal(typeof getSchema, 'function'); + }); + + it('should accept documents as an array and return a schema', async function () { + const res = await getSchema([ + { + a: 1, + }, + { + a: 2, + b: false, + }, + ]); + assert.ok(res); + assert.equal(typeof res, 'object'); + assert.ok(!!res.count); + assert.ok(!!res.fields); + assert.equal(res.count, 2); + }); + + it('should return an error when no documents are provided', async function () { + try { + await getSchema('' as any); + assert(false); // Should error. + } catch (err) { + assert.ok(err); + } + }); +}); diff --git a/packages/mongodb-schema/tsconfig-lint.json b/packages/mongodb-schema/tsconfig-lint.json new file mode 100644 index 000000000..6bdef84f3 --- /dev/null +++ b/packages/mongodb-schema/tsconfig-lint.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/mongodb-schema/tsconfig.json b/packages/mongodb-schema/tsconfig.json new file mode 100644 index 000000000..d9e86b4ab --- /dev/null +++ b/packages/mongodb-schema/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@mongodb-js/tsconfig-devtools/tsconfig.common.json", + "exclude": ["src/**/*.spec.*", "src/**/*.test.*"] +} diff --git a/packages/mongodb-ts-autocomplete/package.json b/packages/mongodb-ts-autocomplete/package.json index 1ff8d04e5..6e0e4f332 100644 --- a/packages/mongodb-ts-autocomplete/package.json +++ b/packages/mongodb-ts-autocomplete/package.json @@ -53,11 +53,11 @@ "extract-types": "ts-node scripts/extract-types.ts" }, "dependencies": { + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/ts-autocomplete": "^0.5.13", "@mongosh/shell-api": "^5.1.6", "debug": "^4.4.0", "lodash": "^4.17.21", - "mongodb-schema": "^12.6.2", "node-cache": "^5.1.2", "typescript": "^5.9.3" }, diff --git a/packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts b/packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts index 6a5f9fca1..cbaeb72c7 100644 --- a/packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts +++ b/packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts @@ -1,7 +1,7 @@ import * as ts from 'typescript'; import { MongoDBAutocompleter } from './index'; import type { AutocompletionContext } from './autocompletion-context'; -import { analyzeDocuments } from 'mongodb-schema'; +import { analyzeDocuments } from '@mongodb-js/mongodb-schema'; import { expect } from 'chai'; import { relativeNodePath } from '@mongodb-js/ts-autocomplete'; diff --git a/packages/mongodb-ts-autocomplete/src/autocompleter.ts b/packages/mongodb-ts-autocomplete/src/autocompleter.ts index bd6e643b8..a695bc521 100644 --- a/packages/mongodb-ts-autocomplete/src/autocompleter.ts +++ b/packages/mongodb-ts-autocomplete/src/autocompleter.ts @@ -7,8 +7,8 @@ import autocompleteTypes from './types/autocomplete-types'; import { api as ShellApiText } from '@mongosh/shell-api/api'; import { replaceImports } from './utils'; -import type { JSONSchema } from 'mongodb-schema'; -import { toTypescriptTypeDefinition } from 'mongodb-schema'; +import type { JSONSchema } from '@mongodb-js/mongodb-schema'; +import { toTypescriptTypeDefinition } from '@mongodb-js/mongodb-schema'; import { inferCollectionNameFromFunctionCall, diff --git a/packages/mongodb-ts-autocomplete/src/autocompletion-context.ts b/packages/mongodb-ts-autocomplete/src/autocompletion-context.ts index 6fbbf11ce..204f8578a 100644 --- a/packages/mongodb-ts-autocomplete/src/autocompletion-context.ts +++ b/packages/mongodb-ts-autocomplete/src/autocompletion-context.ts @@ -1,6 +1,6 @@ import NodeCache from 'node-cache'; -import type { JSONSchema } from 'mongodb-schema'; +import type { JSONSchema } from '@mongodb-js/mongodb-schema'; type CacheOptions = { databaseCollectionsTTL: number; diff --git a/packages/mql-typescript/package.json b/packages/mql-typescript/package.json index 0d9dbb633..fe2069348 100644 --- a/packages/mql-typescript/package.json +++ b/packages/mql-typescript/package.json @@ -71,6 +71,7 @@ }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "^0.11.8", + "@mongodb-js/mongodb-schema": "^0.1.0", "@mongodb-js/prettier-config-devtools": "^1.0.4", "@mongodb-js/shell-bson-parser": "^1.5.14", "@mongodb-js/tsconfig-devtools": "^1.1.3", @@ -84,7 +85,6 @@ "js-yaml": "^4.1.1", "jsdom": "^24.1.3", "json-schema-to-zod": "^2.6.0", - "mongodb-schema": "^12.6.2", "prettier": "^3.8.1", "ts-node": "^10.9.2", "typescript": "^5.9.3", diff --git a/packages/mql-typescript/src/driverSchema/docs-crawler.ts b/packages/mql-typescript/src/driverSchema/docs-crawler.ts index 2854321a4..d2ef86e18 100644 --- a/packages/mql-typescript/src/driverSchema/docs-crawler.ts +++ b/packages/mql-typescript/src/driverSchema/docs-crawler.ts @@ -1,6 +1,6 @@ import { removeNewlines, removeTrailingComments } from '../utils'; -import { getSimplifiedSchema } from 'mongodb-schema'; -import type { SimplifiedSchema } from 'mongodb-schema'; +import { getSimplifiedSchema } from '@mongodb-js/mongodb-schema'; +import type { SimplifiedSchema } from '@mongodb-js/mongodb-schema'; import { JSDOM, VirtualConsole } from 'jsdom'; import parse, { ParseMode } from '@mongodb-js/shell-bson-parser'; diff --git a/packages/mql-typescript/src/driverSchema/static-schemas.ts b/packages/mql-typescript/src/driverSchema/static-schemas.ts index 9452aeed0..c807500aa 100644 --- a/packages/mql-typescript/src/driverSchema/static-schemas.ts +++ b/packages/mql-typescript/src/driverSchema/static-schemas.ts @@ -1,4 +1,4 @@ -import type { SimplifiedSchema } from 'mongodb-schema'; +import type { SimplifiedSchema } from '@mongodb-js/mongodb-schema'; interface SchemaInfo { collectionName: string; diff --git a/packages/mql-typescript/src/testGenerator/test-generator.ts b/packages/mql-typescript/src/testGenerator/test-generator.ts index 96db46d5f..82839d62e 100644 --- a/packages/mql-typescript/src/testGenerator/test-generator.ts +++ b/packages/mql-typescript/src/testGenerator/test-generator.ts @@ -8,7 +8,7 @@ import type { SimplifiedSchema, SimplifiedSchemaType, SimplifiedSchemaBaseType, -} from 'mongodb-schema'; +} from '@mongodb-js/mongodb-schema'; import { unsupportedAggregations } from './unsupported-aggregations'; import * as bson from 'bson';