diff --git a/.env.example b/.env.example index 8dfaf08..3ac389f 100644 --- a/.env.example +++ b/.env.example @@ -1,21 +1,24 @@ NODE_ENV="development" TZ="UTC" -DATABASE_URL="postgresql://localhost/postgres?schema=public&connection_limit=50" +DATABASE_URL="postgresql://postgres:postgres@localhost:8099/postgres?schema=public&connection_limit=50" IP_INFO_TOKEN="" # used for https://ipinfo.io endpoints GLIF_API_BASE_URL="https://api.node.glif.io/rpc" -FILSCAN_API_BASE_URL="https://api-v2.filscan.io/api" PROMETHEUS_AUTH_TOKEN="" -PROMETHEUS_METRICS_ENV="staging" +PROMETHEUS_METRICS_ENV="" + # PO REP -PO_REP_CHAIN_ID=314 # Chain ID on which Po-Rep contracts live -PO_REP_ARCHIVE_RPC_URL="https://filecoin.chain.love/fvm-archive/rpc/v1" # URL of archive node RPC -PO_REP_ARCHIVE_RPC_AUTH_TOKEN= # (Optional) Bearer token that will be included in headers of requests made to archive node -PO_REP_RECENT_RPC_URL= # (Optional) URL of recent node RPC, if not provided default URL configured in viem will be used -PO_REP_RECENT_RPC_AUTH_TOKEN= # (Optional) Bearer token that will be included in headers of requests made to recent node -SP_REGISTRY_CONTRACT_ADDRESS="" # Address of SPRegistry contract -PO_REP_MARKET_CONTRACT_ADDRESS="" # Address of PoRepMarket contract -FILECOIN_PAY_CONTRACT_ADDRESS="" # Address of FilecoinPayV1 contract +PO_REP_CHAIN_ID=314 # Chain ID on which Po-Rep contracts live + +PO_REP_ARCHIVE_RPC_URL="https://filecoin.chain.love/fvm-archive/rpc/v1" # URL of archive node RPC +PO_REP_ARCHIVE_RPC_AUTH_TOKEN="" # (Optional) Bearer token that will be included in headers of requests made to archive node + +PO_REP_RECENT_RPC_URL="" # (Optional) URL of recent node RPC, if not provided default URL configured in viem will be used +PO_REP_RECENT_RPC_AUTH_TOKEN="" # (Optional) Bearer token that will be included in headers of requests made to recent node + +SP_REGISTRY_CONTRACT_ADDRESS="" # Address of SPRegistry contract +PO_REP_MARKET_CONTRACT_ADDRESS="" # Address of PoRepMarket contract +FILECOIN_PAY_CONTRACT_ADDRESS="" # Address of FilecoinPayV1 contract diff --git a/docker-compose.yml b/docker-compose.yml index b144738..7d03686 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,15 +17,6 @@ services: timeout: 5s retries: 2 - autoheal: - restart: always - image: willfarrell/autoheal - environment: - - AUTOHEAL_START_PERIOD=30 - - AUTOHEAL_INTERVAL=30 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - volumes: postgres18_data: driver: local diff --git a/package-lock.json b/package-lock.json index 07cac3d..132b074 100644 --- a/package-lock.json +++ b/package-lock.json @@ -320,7 +320,8 @@ "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.4.3.tgz", "integrity": "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ==", "devOptional": true, - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/@electric-sql/pglite-socket": { "version": "0.1.3", @@ -925,7 +926,6 @@ "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", "license": "ISC", - "peer": true, "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -943,7 +943,6 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -956,7 +955,6 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -968,15 +966,13 @@ "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "license": "MIT", - "peer": true, "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -994,7 +990,6 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1010,7 +1005,6 @@ "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -1135,6 +1129,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-4.0.1.tgz", "integrity": "sha512-68pFJgu+/AZbWkGu65Z3r55bTsCPlgyKaV4BSG8yUAD72q1PPuyVRgUwFv6BxdnibTUHlyxm06FmYWNC+bjN7A==", "license": "MIT", + "peer": true, "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", "axios": "^1.3.1", @@ -1204,6 +1199,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.9.tgz", "integrity": "sha512-zDntUTReRbAThIfSp3dQZ9kKqI+LjgLp5YZN5c1bgNRDuoeLySAoZg46Bg1a+uV8TMgIRziHocglKGNzr6l+bQ==", "license": "MIT", + "peer": true, "dependencies": { "file-type": "21.1.0", "iterare": "1.2.1", @@ -1269,6 +1265,7 @@ "integrity": "sha512-a00B0BM4X+9z+t3UxJqIZlemIwCQdYoPKrMcM+ky4z3pkqqG1eTWexjs+YXpGObnLnjtMPVKWlcZHp3adDYvUw==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@nuxt/opencollective": "0.4.1", "fast-safe-stringify": "2.1.1", @@ -1329,6 +1326,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.9.tgz", "integrity": "sha512-GVd3+0lO0mJq2m1kl9hDDnVrX3Nd4oH3oDfklz0pZEVEVS0KVSp63ufHq2Lu9cyPdSBuelJr9iPm2QQ1yX+Kmw==", "license": "MIT", + "peer": true, "dependencies": { "cors": "2.8.5", "express": "5.1.0", @@ -1546,6 +1544,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/typeorm/-/typeorm-11.0.0.tgz", "integrity": "sha512-SOeUQl70Lb2OfhGkvnh4KXWlsd+zA08RuuQgT7kKbzivngxzSo1Oc7Usu5VxCxACQC9wc2l9esOHILSJeK7rJA==", "license": "MIT", + "peer": true, "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", "@nestjs/core": "^10.0.0 || ^11.0.0", @@ -1756,6 +1755,7 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz", "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -2006,6 +2006,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -2027,6 +2028,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.30.1.tgz", "integrity": "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=14" }, @@ -2063,6 +2065,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.57.2.tgz", "integrity": "sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/api-logs": "0.57.2", "@types/shimmer": "^1.2.0", @@ -2613,6 +2616,7 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=14" } @@ -2644,7 +2648,6 @@ "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "license": "MIT", "optional": true, - "peer": true, "engines": { "node": ">=14" } @@ -2688,6 +2691,7 @@ "resolved": "https://registry.npmjs.org/@prisma/client/-/client-7.10.0.tgz", "integrity": "sha512-Ubw/QS9JGIBSBUsyxAUQuK/Jcu0Tsva7le7QbLd91Kix9yJvYDdj5QkwgEbbZniH80dd+sziQcALPc+HnvQC8Q==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@prisma/client-runtime-utils": "7.10.0" }, @@ -3504,8 +3508,7 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.5.tgz", "integrity": "sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@standard-schema/spec": { "version": "1.0.0", @@ -3704,6 +3707,7 @@ "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "*", "@types/json-schema": "*" @@ -3813,6 +3817,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -3855,6 +3860,7 @@ "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -3984,6 +3990,7 @@ "integrity": "sha512-PC0PDZfJg8sP7cmKe6L3QIL8GZwU5aRvUFedqSIpw3B+QjRSUZeeITC2M5XKeMXEzL6wccN196iy3JLwKNvDVA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.1", "@typescript-eslint/types": "8.48.1", @@ -4524,6 +4531,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4601,6 +4609,7 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4700,7 +4709,6 @@ "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", "license": "MIT", - "peer": true, "engines": { "node": ">= 6.0.0" } @@ -4760,7 +4768,6 @@ "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", "license": "MIT", - "peer": true, "dependencies": { "possible-typed-array-names": "^1.0.0" }, @@ -4786,6 +4793,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz", "integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.15.11", "form-data": "^4.0.5", @@ -5055,6 +5063,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -5154,6 +5163,7 @@ "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-7.2.5.tgz", "integrity": "sha512-Y5LF7olTrcKJn1NoKiWPOvjEiO5DfDVPxqZHETCRMaliC60KBNb4Ge/vEYep5TyaqpXvnpnPPo8zauCe6UzZwA==", "license": "MIT", + "peer": true, "dependencies": { "@cacheable/utils": "^2.3.0", "keyv": "^5.5.4" @@ -5164,7 +5174,6 @@ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", "license": "MIT", - "peer": true, "dependencies": { "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", @@ -5315,6 +5324,7 @@ "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "readdirp": "^4.0.1" }, @@ -5345,13 +5355,15 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/class-validator": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.3.tgz", "integrity": "sha512-rXXekcjofVN1LTOSw+u4u9WXVEUvNBVjORW154q/IdmYWy1nMbOU9aNtZB0t8m+FJQ9q91jlr2f9CwwUFdFMRA==", "license": "MIT", + "peer": true, "dependencies": { "@types/validator": "^13.15.3", "libphonenumber-js": "^1.11.1", @@ -5443,7 +5455,6 @@ "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "license": "ISC", - "peer": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -5458,7 +5469,6 @@ "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -5905,7 +5915,6 @@ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", "license": "MIT", - "peer": true, "engines": { "node": ">= 12" } @@ -5923,8 +5932,7 @@ "version": "1.11.19", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/debug": { "version": "4.4.3", @@ -5948,7 +5956,6 @@ "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", "license": "MIT", - "peer": true, "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, @@ -6002,7 +6009,6 @@ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", "license": "MIT", - "peer": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -6111,6 +6117,7 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", "license": "BSD-2-Clause", + "peer": true, "engines": { "node": ">=12" }, @@ -6163,8 +6170,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", @@ -6369,6 +6375,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6425,6 +6432,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -6886,7 +6894,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -7065,7 +7072,6 @@ "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "license": "MIT", - "peer": true, "dependencies": { "is-callable": "^1.2.7" }, @@ -7186,7 +7192,6 @@ "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", "license": "MIT", - "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -7442,7 +7447,6 @@ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", "license": "MIT", - "peer": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -7660,6 +7664,7 @@ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz", "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==", "license": "MIT", + "peer": true, "dependencies": { "@ioredis/commands": "1.4.0", "cluster-key-slot": "^1.1.0", @@ -7708,7 +7713,6 @@ "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.4" }, @@ -7823,7 +7827,6 @@ "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "license": "MIT", - "peer": true, "dependencies": { "which-typed-array": "^1.1.16" }, @@ -7851,8 +7854,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", @@ -7898,7 +7900,6 @@ "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "license": "BlueOak-1.0.0", - "peer": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -8105,6 +8106,7 @@ "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.5.5.tgz", "integrity": "sha512-FA5LmZVF1VziNc0bIdCSA1IoSVnDCqE8HJIZZv2/W8YmoAM50+tnUgJR/gQZwEeIMleuIOnRnHA/UaZRNeV4iQ==", "license": "MIT", + "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -8685,6 +8687,7 @@ "integrity": "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", @@ -8768,7 +8771,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=10.5.0" } @@ -9084,8 +9086,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "license": "BlueOak-1.0.0", - "peer": true + "license": "BlueOak-1.0.0" }, "node_modules/parent-module": { "version": "1.0.1", @@ -9222,6 +9223,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.9.1", "pg-pool": "^3.10.1", @@ -9320,6 +9322,7 @@ "resolved": "https://registry.npmjs.org/pg-query-stream/-/pg-query-stream-4.10.3.tgz", "integrity": "sha512-h2utrzpOIzeT9JfaqfvBbVuvCfBjH86jNfVrGGTbyepKAIOyTfDew0lAt8bbJjs9n/I5bGDl7S2sx6h5hPyJxw==", "license": "MIT", + "peer": true, "dependencies": { "pg-cursor": "^2.15.3" }, @@ -9576,7 +9579,6 @@ "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.4" } @@ -9650,6 +9652,7 @@ "integrity": "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -9680,6 +9683,7 @@ "devOptional": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@prisma/config": "7.10.0", "@prisma/dev": "0.24.17", @@ -9824,6 +9828,7 @@ "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" @@ -10096,7 +10101,8 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/remeda": { "version": "2.33.4", @@ -10113,7 +10119,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -10339,6 +10344,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -10406,8 +10412,7 @@ "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", "devOptional": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/schema-utils": { "version": "3.3.0", @@ -10434,6 +10439,7 @@ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -10548,7 +10554,6 @@ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", "license": "MIT", - "peer": true, "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -10572,7 +10577,6 @@ "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.12.tgz", "integrity": "sha512-8LzC5+bvI45BjpfXU8V5fdU2mfeKiQe1D1gIMn7XUlF3OTUrpdJpPPH4EMAnF0DsHHdSZqCdSss5qCmJKuiO3w==", "license": "(MIT AND BSD-3-Clause)", - "peer": true, "dependencies": { "inherits": "^2.0.4", "safe-buffer": "^5.2.1", @@ -10767,7 +10771,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=14" } @@ -10841,7 +10844,6 @@ "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "license": "MIT", - "peer": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -10869,7 +10871,6 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -11164,7 +11165,6 @@ "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", "integrity": "sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==", "license": "MIT", - "peer": true, "dependencies": { "isarray": "^2.0.5", "safe-buffer": "^5.2.1", @@ -11254,6 +11254,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -11373,7 +11374,6 @@ "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", "license": "MIT", - "peer": true, "dependencies": { "call-bound": "^1.0.3", "es-errors": "^1.3.0", @@ -11497,7 +11497,6 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", "license": "BSD-2-Clause", - "peer": true, "engines": { "node": ">=12" }, @@ -11510,7 +11509,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "license": "ISC", - "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -11530,15 +11528,13 @@ "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/typeorm/node_modules/path-scurry": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "license": "BlueOak-1.0.0", - "peer": true, "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -11556,6 +11552,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -11709,7 +11706,6 @@ "https://github.com/sponsors/ctavan" ], "license": "MIT", - "peer": true, "bin": { "uuid": "dist/esm/bin/uuid" } @@ -11867,7 +11863,6 @@ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", "license": "MIT", - "peer": true, "engines": { "node": ">= 8" } @@ -11878,6 +11873,7 @@ "integrity": "sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -12046,7 +12042,6 @@ "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", "license": "MIT", - "peer": true, "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", @@ -12106,7 +12101,6 @@ "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -12130,6 +12124,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -12160,7 +12155,6 @@ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -12170,7 +12164,6 @@ "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "license": "MIT", - "peer": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -12245,6 +12238,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 91955ae..9adebcc 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,7 @@ "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "migration:deploy": "prisma migrate deploy", "migration:generate": "prisma generate --sql", - "migration:create_migration": "npx prisma migrate dev --schema prisma/schema.prisma --name MIGRATION_NAME", - "prisma:generate": "prisma generate" + "migration:create_migration": "npx prisma migrate dev --schema prisma/schema.prisma --name MIGRATION_NAME" }, "dependencies": { "@glif/filecoin-address": "^4.0.0", diff --git a/prisma/migrations/20261001221920_remove_unused_tables/migration.sql b/prisma/migrations/20261001221920_remove_unused_tables/migration.sql new file mode 100644 index 0000000..0c3e12b --- /dev/null +++ b/prisma/migrations/20261001221920_remove_unused_tables/migration.sql @@ -0,0 +1,12 @@ +/* + Warnings: + + - You are about to drop the `client_provider_distribution` table. If the table is not empty, all the data it contains will be lost. + - You are about to drop the `provider` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropTable +DROP TABLE "client_provider_distribution"; + +-- DropTable +DROP TABLE "provider"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index fa05941..31991e4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -16,26 +16,6 @@ datasource db { provider = "postgresql" } -model provider { - id String @id - num_of_deals Int - total_deal_size BigInt - num_of_clients Int - last_deal_height Int - first_deal_height Int @default(0) -} - -model client_provider_distribution { - client String - provider String - total_deal_size BigInt - unique_data_size BigInt - claims_count BigInt? - - @@id([client, provider]) - @@index([provider]) -} - model ipni_publisher_advertisement { id String @id previous_id String? @unique @@ -70,17 +50,17 @@ model po_rep_indexer_run { // Defaults are there to insert providers based on registration event which // contains only the ID. Rest of the field will get updated based on other events model po_rep_storage_provider { - providerId BigInt @id - organization String @default("0x0000000000000000000000000000000000000000") - payee String @default("0x0000000000000000000000000000000000000000") - paused Boolean @default(false) - blocked Boolean @default(false) - availableBytes BigInt @default(0) - committedBytes BigInt @default(0) - pendingBytes BigInt @default(0) - pricePerSectorPerMonth Decimal @default(0) @db.Decimal(78, 0) - minDealDurationDays Int @default(0) - maxDealDurationDays Int @default(0) + providerId BigInt @id + organization String @default("0x0000000000000000000000000000000000000000") + payee String @default("0x0000000000000000000000000000000000000000") + paused Boolean @default(false) + blocked Boolean @default(false) + availableBytes BigInt @default(0) + committedBytes BigInt @default(0) + pendingBytes BigInt @default(0) + pricePerSectorPerMonth Decimal @default(0) @db.Decimal(78, 0) + minDealDurationDays Int @default(0) + maxDealDurationDays Int @default(0) registeredAtBlock BigInt deals po_rep_deal[] offers po_rep_offer[] diff --git a/prisma/sql/getIpniReportingWeekly.sql b/prisma/sql/getIpniReportingWeekly.sql index 9ba51f5..9a3378c 100644 --- a/prisma/sql/getIpniReportingWeekly.sql +++ b/prisma/sql/getIpniReportingWeekly.sql @@ -4,8 +4,7 @@ with "with_week" as (select "date", "not_reporting", "total", date_trunc('week', "date") as "week" - from "ipni_reporting_daily" -) + from "ipni_reporting_daily") -- select distinct on ("week") "week", "ok", diff --git a/src/aggregation/aggregation-runner.ts b/src/aggregation/aggregation-runner.ts deleted file mode 100644 index 48a816d..0000000 --- a/src/aggregation/aggregation-runner.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { PostgresService } from 'src/db/postgres.service'; - -import { PrismaService } from 'src/db/prisma.service'; -import { PrometheusMetricService } from 'src/prometheus'; -import { StorageProviderService } from '../service/storage-provider/storage-provider.service'; -import { AggregationTable } from './aggregation-table'; -import { PoRepService } from '../service/po-rep/po-rep.service'; -import { IpniMisreportingCheckerService } from 'src/service/ipni-misreporting-checker/ipni-misreporting-checker.service'; - -export type AggregationRunnerRunServices = { - prismaService: PrismaService; - postgresService?: PostgresService; - prometheusMetricService?: PrometheusMetricService; - storageProviderService: StorageProviderService; - ipniMisreportingCheckerService: IpniMisreportingCheckerService; - porepService: PoRepService; -}; - -export interface AggregationRunner { - run(services: AggregationRunnerRunServices): Promise; - - getFilledTables(): AggregationTable[]; - - getDependingTables(): AggregationTable[]; -} diff --git a/src/aggregation/aggregation-table.ts b/src/aggregation/aggregation-table.ts deleted file mode 100644 index d8a1d79..0000000 --- a/src/aggregation/aggregation-table.ts +++ /dev/null @@ -1,31 +0,0 @@ -export enum AggregationTable { - AllocatorClientBookkeeping, - AllocatorRegistry, - AllocatorsWeeklyAcc, - CidSharing, - Client, - ClientAllocatorDistributionWeekly, - ClientAllocatorDistributionWeeklyAcc, - ClientClaimsHourly, - ClientProviderDistribution, - ClientProviderDistributionWeekly, - ClientProviderDistributionWeeklyAcc, - ClientReplicaDistribution, - IpniReportingDaily, - OldDatacapBalanceNv22, - OldDatacapBalanceWeekly, - OldDatacapClientBalanceNv22, - OldDatacapClientBalanceWeekly, - ProviderFirstClient, - ProviderIpInfo, - ProviderRetrievabilityDaily, - ProvidersWeekly, - ProvidersWeeklyAcc, - UnifiedVerifiedDealHourly, - Allocator, - Provider, - ClientDatacapAllocation, - AllocatorRegistryArchive, - AllocatorsClientStatsDaily, - ClientsStatsDaily, -} diff --git a/src/aggregation/aggregation-tasks.service.ts b/src/aggregation/aggregation-tasks.service.ts deleted file mode 100644 index 18b55c1..0000000 --- a/src/aggregation/aggregation-tasks.service.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { Inject, Injectable, Logger } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { - HealthCheckError, - HealthIndicator, - HealthIndicatorResult, -} from '@nestjs/terminus'; -import { PrometheusMetricService } from 'src/prometheus'; -import { sleep } from 'src/utils/utils'; -import { PostgresService } from '../db/postgres.service'; -import { PrismaService } from '../db/prisma.service'; -import { IpniMisreportingCheckerService } from '../service/ipni-misreporting-checker/ipni-misreporting-checker.service'; -import { PoRepService } from '../service/po-rep/po-rep.service'; -import { StorageProviderService } from '../service/storage-provider/storage-provider.service'; -import { AggregationRunner } from './aggregation-runner'; -import { AggregationTable } from './aggregation-table'; - -@Injectable() -export class AggregationTasksService extends HealthIndicator { - private readonly logger = new Logger(AggregationTasksService.name); - private jobInProgress = false; - private healthy = true; - private unhealthyReason: string = null; - private lastSuccess: Date = null; - private lastRun: Date = null; - - constructor( - private readonly prismaService: PrismaService, - private readonly postgresService: PostgresService, - - @Inject('AggregationRunner') - private readonly aggregationRunners: AggregationRunner[], - private readonly prometheusMetricService: PrometheusMetricService, - private readonly ipniMisreportingCheckerService: IpniMisreportingCheckerService, - private readonly storageProviderService: StorageProviderService, - private readonly porepService: PoRepService, - ) { - super(); - } - - public async getHealth(): Promise { - const result = this.getStatus(AggregationTasksService.name, this.healthy, { - lastSuccess: this.lastSuccess, - lastRun: this.lastRun, - unhealthyReason: this.healthy ? null : this.unhealthyReason, - }); - - if (this.healthy) return result; - throw new HealthCheckError('Healthcheck failed', result); - } - - @Cron(CronExpression.EVERY_HOUR) - public async runAggregationJob() { - if (!this.jobInProgress) { - this.jobInProgress = true; - const endAllAggregationsTimer = - this.prometheusMetricService.aggregateMetrics.startAggregateTimer(); - - try { - this.logger.log('Starting aggregations'); - this.lastRun = new Date(); - this.healthy = true; - - await this.runAggregations(); - - this.lastSuccess = new Date(); - this.logger.log('Finished aggregations'); - } catch (err) { - this.healthy = false; - this.unhealthyReason = err.message || 'Unknown error'; - - this.logger.error( - `Error during aggregation job: ${err.message}`, - // err.cause?.stack || err.stack, - ); - } finally { - endAllAggregationsTimer(); - this.jobInProgress = false; - } - } else { - this.logger.warn( - 'Aggregations job still in progress - skipping next execution', - ); - } - } - - public async runAggregations() { - const filledTables: AggregationTable[] = []; - const pendingAggregationRunners = Object.assign( - [], - this.aggregationRunners, - ); - - while (pendingAggregationRunners.length > 0) { - let executedRunners = 0; - - for (const aggregationRunner of this.aggregationRunners) { - if ( - pendingAggregationRunners.indexOf(aggregationRunner) > -1 && - aggregationRunner - .getDependingTables() - .every((p) => filledTables.includes(p)) - ) { - // execute runner - const aggregationRunnerName = aggregationRunner.constructor.name; - this.logger.debug(`Starting aggregation: ${aggregationRunnerName}`); - - // start transaction timer - const endSingleAggregationTransactionTimer = - this.prometheusMetricService.aggregateMetrics.startTimerByRunnerNameMetric( - aggregationRunnerName, - ); - - try { - await this.executeWithRetries( - 3, - () => - // prettier-ignore - aggregationRunner.run({ - prismaService: this.prismaService, - postgresService: this.postgresService, - prometheusMetricService: this.prometheusMetricService, - ipniMisreportingCheckerService: this.ipniMisreportingCheckerService, - storageProviderService: this.storageProviderService, - porepService: this.porepService, - }), - aggregationRunnerName, - ); - } catch (err) { - throw new Error( - `Error running ${aggregationRunnerName}: ${err.message || err.code || err}`, - { cause: err }, - ); - } finally { - endSingleAggregationTransactionTimer(); - } - - this.logger.debug(`Finished aggregation: ${aggregationRunnerName}`); - - executedRunners++; - - // store filled tables - filledTables.push(...aggregationRunner.getFilledTables()); - - // remove from pending runners - pendingAggregationRunners.splice( - pendingAggregationRunners.indexOf(aggregationRunner), - 1, - ); - } - } - - if (executedRunners === 0) { - this.logger.error( - 'Cannot execute runners - impossible dependencies defined', - ); - - break; - } - } - } - - private async executeWithRetries( - maxTries: number, - fn: () => Promise, - aggregationRunnerName: string, - ) { - let success = false; - let executionNumber = 0; - let lastErr: Error = null; - - while (!success && executionNumber < maxTries) { - try { - await fn(); - success = true; - } catch (err) { - lastErr = err; - executionNumber++; - - this.logger.warn( - `Error during aggregation job: ${aggregationRunnerName}, execution ${executionNumber}/${maxTries}: ${err.message || err.code || err}`, - ); - - if (executionNumber !== maxTries) await sleep(90000); // 90 seconds - } - } - - if (!success) throw lastErr; - } -} diff --git a/src/aggregation/runners/ipni-reporting-daily.runner.ts b/src/aggregation/runners/ipni-reporting-daily.runner.ts deleted file mode 100644 index 8a87f27..0000000 --- a/src/aggregation/runners/ipni-reporting-daily.runner.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { DateTime } from 'luxon'; -import { isTodayUTC } from 'src/utils/utils'; -import { - AggregationRunner, - AggregationRunnerRunServices, -} from '../aggregation-runner'; -import { AggregationTable } from '../aggregation-table'; - -export class IpniReportingDailyRunner implements AggregationRunner { - public async run({ - prismaService, - prometheusMetricService, - ipniMisreportingCheckerService, - }: AggregationRunnerRunServices): Promise { - const { - startGetDataTimerByRunnerNameMetric, - startStoreDataTimerByRunnerNameMetric, - } = prometheusMetricService.aggregateMetrics; - - const latestStored = await prismaService.ipni_reporting_daily.findFirst({ - select: { - date: true, - }, - orderBy: { - date: 'desc', - }, - }); - - const latestStoredDate = latestStored - ? DateTime.fromJSDate(latestStored.date, { zone: 'UTC' }) - : null; - - // skip if we already did aggregation today - if (!!latestStoredDate && isTodayUTC(latestStoredDate)) { - return; - } - - const getDataEndTimerMetric = startGetDataTimerByRunnerNameMetric( - IpniReportingDailyRunner.name, - ); - - const result = - await ipniMisreportingCheckerService.getAggregatedProvidersReportingStatus(); - - const data = { - not_reporting: result.notReporting, - misreporting: result.misreporting, - ok: result.ok, - total: result.total, - }; - - getDataEndTimerMetric(); - - const storeDataEndTimerMetric = startStoreDataTimerByRunnerNameMetric( - IpniReportingDailyRunner.name, - ); - - await prismaService.ipni_reporting_daily.create({ data: data }); - - storeDataEndTimerMetric(); - } - - getFilledTables(): AggregationTable[] { - return [AggregationTable.IpniReportingDaily]; - } - - getDependingTables(): AggregationTable[] { - return [AggregationTable.ClientProviderDistribution]; - } -} diff --git a/src/app.module.ts b/src/app.module.ts index 629046d..2accf4f 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -7,11 +7,9 @@ import { ScheduleModule } from '@nestjs/schedule'; import { TerminusModule } from '@nestjs/terminus'; import axios from 'axios'; import axiosBetterStacktrace from 'axios-better-stacktrace'; -import { AggregationTasksService } from './aggregation/aggregation-tasks.service'; import { AppController } from './controller/app/app.controller'; import { FilecoinPayController } from './controller/filecoin-pay/filecoin-pay.controller'; import { PoRepController } from './controller/po-rep/po-rep.controller'; -import { StorageProvidersController } from './controller/storage-providers/storage-providers.controller'; import { PostgresService } from './db/postgres.service'; import { PrismaService } from './db/prisma.service'; import { IpniAdvertisementFetcherJobService } from './jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service'; @@ -19,22 +17,15 @@ import { ErrorHandlerMiddleware } from './middleware/error-handler.middleware'; import { RequestLoggerMiddleware } from './middleware/request-logger.middleware'; import { PoRepIndexerModule } from './po-rep-indexer'; import { PrometheusMetricModule } from './prometheus'; -import { StorageProviderService } from './service/storage-provider/storage-provider.service'; - import { CidContactService } from './service/cid-contact/cid-contact.service'; import { ERC20TokenInfoService } from './service/erc20-token-info/erc20-token-info.service'; import { EthApiService } from './service/eth-api/eth-api.service'; -import { IpniMisreportingCheckerService } from './service/ipni-misreporting-checker/ipni-misreporting-checker.service'; -import { LocationService } from './service/location/location.service'; import { LotusApiService } from './service/lotus-api/lotus-api.service'; import { PoRepPriceOracleService } from './service/po-rep-price-oracle/po-rep-price-oracle.service'; import { PoRepService } from './service/po-rep/po-rep.service'; - import { queryBuilderProviders } from './db'; - -const AGGREGATION_RUNNERS = []; - -const AGGREGATION_RUNNERS_RUN_ONLY = []; +import { IpniReportingDailyRunnerService } from 'src/jobs/ipni-reporting-daily-runner/ipni-reporting-daily-runner.service'; +import { IPNIController } from 'src/controller/ipni/ipni.controller'; @Module({ imports: [ @@ -47,25 +38,19 @@ const AGGREGATION_RUNNERS_RUN_ONLY = []; PoRepIndexerModule, ], controllers: [ - StorageProvidersController, PoRepController, FilecoinPayController, AppController, + IPNIController, ], providers: [ - ...(AGGREGATION_RUNNERS_RUN_ONLY.length - ? AGGREGATION_RUNNERS_RUN_ONLY - : AGGREGATION_RUNNERS), - AggregationTasksService, IpniAdvertisementFetcherJobService, + IpniReportingDailyRunnerService, PrismaService, - StorageProviderService, CidContactService, - LocationService, ERC20TokenInfoService, EthApiService, LotusApiService, - IpniMisreportingCheckerService, PostgresService, PoRepPriceOracleService, PoRepService, @@ -79,13 +64,6 @@ const AGGREGATION_RUNNERS_RUN_ONLY = []; return axiosInstance; }, }, - { - provide: 'AggregationRunner', - useFactory: (...runners) => runners, - inject: AGGREGATION_RUNNERS_RUN_ONLY.length - ? AGGREGATION_RUNNERS_RUN_ONLY - : AGGREGATION_RUNNERS, - }, ...queryBuilderProviders, ], }) diff --git a/src/controller/app/app.controller.ts b/src/controller/app/app.controller.ts index 7eaa3ce..8bedc55 100644 --- a/src/controller/app/app.controller.ts +++ b/src/controller/app/app.controller.ts @@ -1,11 +1,9 @@ -import { HttpService } from '@nestjs/axios'; import { Cache, CACHE_MANAGER, CacheTTL } from '@nestjs/cache-manager'; import { Controller, Get, Inject, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ApiExcludeEndpoint } from '@nestjs/swagger'; import { HealthCheck, - HealthCheckError, HealthCheckResult, HealthCheckService, HealthIndicator, @@ -13,10 +11,10 @@ import { HttpHealthIndicator, TypeOrmHealthIndicator, } from '@nestjs/terminus'; -import { firstValueFrom } from 'rxjs'; import { PostgresService } from 'src/db/postgres.service'; import { IpniAdvertisementFetcherJobService } from 'src/jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service'; import { Cacheable } from 'src/utils/cacheable'; +import { IpniReportingDailyRunnerService } from 'src/jobs/ipni-reporting-daily-runner/ipni-reporting-daily-runner.service'; @Controller() export class AppController extends HealthIndicator { @@ -32,7 +30,7 @@ export class AppController extends HealthIndicator { private readonly postgresService: PostgresService, private readonly configService: ConfigService, private readonly ipniAdvertisementFetcherJobService: IpniAdvertisementFetcherJobService, - private readonly httpService: HttpService, + private readonly ipniReportingDailyRunnerService: IpniReportingDailyRunnerService, ) { super(); } @@ -40,7 +38,7 @@ export class AppController extends HealthIndicator { @Get() @ApiExcludeEndpoint() public getRoot(): string { - return 'Hyperion'; + return 'Hyperion API'; } @Get('/health') @@ -83,30 +81,6 @@ export class AppController extends HealthIndicator { ); } - @Cacheable({ ttl: 1000 * 60 * 60 }) // 1 hour - private async _httpPingCheckFilscan(): Promise { - let healthy = false; - - try { - const endpoint = `${this.configService.get('FILSCAN_API_BASE_URL')}/v1/TotalIndicators`; - - const { data } = await firstValueFrom( - this.httpService.post(endpoint, {}), - ); - - healthy = !!data?.['result']; - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (_) { - healthy = false; - } - - const result = this.getStatus('filscan-api', healthy); - - if (healthy) return result; - throw new HealthCheckError('Healthcheck failed', result); - } - private async _httpPingCheckGlifApi(): Promise { const url = `${this.configService.get('GLIF_API_BASE_URL')}/v1`; @@ -136,6 +110,7 @@ export class AppController extends HealthIndicator { timeout: 5000, }), () => this.ipniAdvertisementFetcherJobService.getHealth(), + () => this.ipniReportingDailyRunnerService.getHealth(), ]); } } diff --git a/src/controller/filecoin-pay/filecoin-pay.controller.ts b/src/controller/filecoin-pay/filecoin-pay.controller.ts index 5aad809..75c0fe7 100644 --- a/src/controller/filecoin-pay/filecoin-pay.controller.ts +++ b/src/controller/filecoin-pay/filecoin-pay.controller.ts @@ -30,8 +30,7 @@ import { class BigIntTransform implements PipeTransform { transform(value: string): bigint { try { - const parsedValue = BigInt(value); - return parsedValue; + return BigInt(value); } catch { const message = `'${value}' is not a valid rail id`; throw new BadRequestException(message, message); diff --git a/src/controller/ipni/ipni.controller.ts b/src/controller/ipni/ipni.controller.ts new file mode 100644 index 0000000..d6976bd --- /dev/null +++ b/src/controller/ipni/ipni.controller.ts @@ -0,0 +1,43 @@ +import { Cache, CACHE_MANAGER, CacheTTL } from '@nestjs/cache-manager'; +import { Controller, Get, Inject } from '@nestjs/common'; +import { ApiOkResponse, ApiOperation } from '@nestjs/swagger'; +import { ControllerBase } from '../base/controller-base'; +import { PrismaService } from 'src/db/prisma.service'; +import { getIpniReportingWeekly } from '../../generated/prisma/sql'; +import { AggregatedProvidersIPNIReportingStatusWeekly } from './types.ipni'; + +@Controller('ipni') +@CacheTTL(1000 * 60 * 60) // 1 hour +export class IPNIController extends ControllerBase { + constructor( + @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, + private readonly prismaService: PrismaService, + ) { + super(); + } + + @Get('/aggregated-ipni-status-weekly') + @CacheTTL(1000 * 60 * 60) // 1 hour + @ApiOperation({ + summary: 'Get aggregated storage providers IPNI reporting status over time', + }) + @ApiOkResponse({ + description: 'Aggregated storage providers IPNI reporting status over time', + type: AggregatedProvidersIPNIReportingStatusWeekly, + }) + public async getAggregatedProvidersIPNIReportingStatusWeekly(): Promise { + const result = await this.prismaService.$queryRawTyped( + getIpniReportingWeekly(), + ); + + return { + results: result.map((r) => ({ + week: r.week, + total: r.total, + misreporting: r.misreporting, + notReporting: r.not_reporting, + ok: r.ok, + })), + }; + } +} diff --git a/src/service/ipni-misreporting-checker/types.ipni-misreporting-checker.ts b/src/controller/ipni/types.ipni.ts similarity index 54% rename from src/service/ipni-misreporting-checker/types.ipni-misreporting-checker.ts rename to src/controller/ipni/types.ipni.ts index ba76f0b..ee4f7b1 100644 --- a/src/service/ipni-misreporting-checker/types.ipni-misreporting-checker.ts +++ b/src/controller/ipni/types.ipni.ts @@ -1,11 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -export enum StorageProviderIpniReportingStatus { - MISREPORTING = 'MISREPORTING', - NOT_REPORTING = 'NOT_REPORTING', - OK = 'OK', -} - export class AggregatedProvidersIPNIReportingStatusWeek { @ApiProperty({ description: 'ISO format', @@ -45,32 +39,3 @@ export class AggregatedProvidersIPNIReportingStatusWeekly { }) results: AggregatedProvidersIPNIReportingStatusWeek[]; } - -export class AggregatedProvidersIPNIReportingStatus { - @ApiProperty({ - description: 'Number of storage providers misreporting the IPNI data', - }) - misreporting: number; - - @ApiProperty({ - description: 'Number of storage providers reporting no IPNI data', - }) - notReporting: number; - - @ApiProperty({ - description: - 'Number of storage providers reporting the IPNI data correctly', - }) - ok: number; - - @ApiProperty({ - description: 'Total number of storage providers', - }) - total: number; -} - -export class ProviderIPNIReportingStatus { - status: StorageProviderIpniReportingStatus; - actualClaimsCount: number; - ipniReportedClaimsCount: number | null; -} diff --git a/src/controller/po-rep/po-rep.controller.ts b/src/controller/po-rep/po-rep.controller.ts index 6725b34..077e3e8 100644 --- a/src/controller/po-rep/po-rep.controller.ts +++ b/src/controller/po-rep/po-rep.controller.ts @@ -9,7 +9,6 @@ import { ValidationPipe, } from '@nestjs/common'; import { ApiOkResponse, ApiOperation } from '@nestjs/swagger'; -import { PrismaService } from 'src/db/prisma.service'; import { PoRepService } from 'src/service/po-rep/po-rep.service'; import { PoRepDealsList, @@ -28,7 +27,6 @@ import { ControllerBase } from '../base/controller-base'; export class PoRepController extends ControllerBase { constructor( @Inject(CACHE_MANAGER) private _cacheManager: Cache, - private readonly prismaService: PrismaService, private readonly poRepService: PoRepService, ) { super(); diff --git a/src/controller/storage-providers/storage-providers.controller.ts b/src/controller/storage-providers/storage-providers.controller.ts deleted file mode 100644 index ccacd2f..0000000 --- a/src/controller/storage-providers/storage-providers.controller.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { CacheTTL } from '@nestjs/cache-manager'; -import { Controller, Logger } from '@nestjs/common'; - -import { ControllerBase } from '../base/controller-base'; - -@Controller('storage-providers') -@CacheTTL(1000 * 60 * 30) // 30 minutes -export class StorageProvidersController extends ControllerBase { - private readonly logger = new Logger(StorageProvidersController.name); - - constructor() { - super(); - } -} diff --git a/src/jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service.ts b/src/jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service.ts index 3bbccb5..931353d 100644 --- a/src/jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service.ts +++ b/src/jobs/ipni-advertisement-fetcher-job/ipni-advertisement-fetcher-job.service.ts @@ -36,7 +36,7 @@ export class IpniAdvertisementFetcherJobService extends HealthIndicator { throw new HealthCheckError('Healthcheck failed', result); } - @Cron(CronExpression.EVERY_DAY_AT_8PM) + @Cron(CronExpression.EVERY_DAY_AT_3PM) public async runIPNIAdvertisementFetcherJob() { if (!this.jobInProgress) { this.jobInProgress = true; @@ -47,7 +47,7 @@ export class IpniAdvertisementFetcherJobService extends HealthIndicator { await this._runIPNIAdvertisementFetcherJob(); - this.logger.log(`Finishing IPNI Advertisement Fetcher job`); + this.logger.log(`Finished IPNI Advertisement Fetcher job`); } catch (err) { this.healthy = false; this.logger.error( @@ -84,9 +84,7 @@ export class IpniAdvertisementFetcherJobService extends HealthIndicator { } } - private async fetchAndStoreAdvertisementsByProvider( - provider: IPNIProvider, - ): Promise { + private async fetchAndStoreAdvertisementsByProvider(provider: IPNIProvider) { const parsedAddress = this.cidContactService.extractMultiaddrAndBuildPublisherBaseUrl( provider.Publisher.Addrs[0], @@ -113,7 +111,7 @@ export class IpniAdvertisementFetcherJobService extends HealthIndicator { currentAd: IPNIAdvertisement, baseUrl: string, adLimit?: number, - ): Promise { + ) { do { try { await this.prismaService.ipni_publisher_advertisement.create({ diff --git a/src/service/ipni-misreporting-checker/ipni-misreporting-checker.service.ts b/src/jobs/ipni-reporting-daily-runner/ipni-reporting-daily-runner.service.ts similarity index 52% rename from src/service/ipni-misreporting-checker/ipni-misreporting-checker.service.ts rename to src/jobs/ipni-reporting-daily-runner/ipni-reporting-daily-runner.service.ts index ecaa00d..def7080 100644 --- a/src/service/ipni-misreporting-checker/ipni-misreporting-checker.service.ts +++ b/src/jobs/ipni-reporting-daily-runner/ipni-reporting-daily-runner.service.ts @@ -1,38 +1,98 @@ import { Injectable, Logger } from '@nestjs/common'; -import { getIpniReportingWeekly } from '../../generated/prisma/sql'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { + HealthCheckError, + HealthIndicator, + HealthIndicatorResult, +} from '@nestjs/terminus'; import { PrismaService } from 'src/db/prisma.service'; -import { bigIntToNumber } from 'src/utils/utils'; -import { LotusApiService } from '../lotus-api/lotus-api.service'; -import { LotusStateMinerInfoResponse } from '../lotus-api/types.lotus-api'; -import { StorageProviderService } from '../storage-provider/storage-provider.service'; +import { LotusApiService } from 'src/service/lotus-api/lotus-api.service'; +import { PoRepService } from 'src/service/po-rep/po-rep.service'; +import { bigIntToNumber, F0Id } from 'src/utils/utils'; import { AggregatedProvidersIPNIReportingStatus, - AggregatedProvidersIPNIReportingStatusWeekly, ProviderIPNIReportingStatus, StorageProviderIpniReportingStatus, -} from './types.ipni-misreporting-checker'; +} from './types.ipni-reporting-daily-runner'; +import { LotusStateMinerInfoResponse } from 'src/service/lotus-api/types.lotus-api'; @Injectable() -export class IpniMisreportingCheckerService { - private readonly logger = new Logger(IpniMisreportingCheckerService.name); +export class IpniReportingDailyRunnerService extends HealthIndicator { + private readonly logger = new Logger(IpniReportingDailyRunnerService.name); + private healthy = true; + private jobInProgress = false; constructor( private readonly prismaService: PrismaService, - private readonly storageProviderService: StorageProviderService, private readonly lotusApiService: LotusApiService, - ) {} + private readonly poRepService: PoRepService, + ) { + super(); + } + + public async getHealth(): Promise { + const result = this.getStatus( + IpniReportingDailyRunnerService.name, + this.healthy, + {}, + ); + + if (this.healthy) return result; + throw new HealthCheckError('Healthcheck failed', result); + } + + @Cron(CronExpression.EVERY_MINUTE) + public async runIPNIReportingDailyRunnerJob() { + if (!this.jobInProgress) { + this.jobInProgress = true; + + try { + this.logger.log('Starting IPNI Reporting Daily Runner job'); + this.healthy = true; + + await this._runIPNIReportingDailyRunnerJob(); + + this.logger.log(`Finished IPNI Reporting Daily Runner job`); + } catch (err) { + this.healthy = false; + this.logger.error( + `Error while running IPNI Reporting Daily Runner job: ${err.message}`, + err.cause?.stack || err.stack, + ); + } finally { + this.jobInProgress = false; + } + } else { + this.logger.warn( + 'IPNI Reporting Daily Runner job is already in progress - skipping next execution', + ); + } + } + + public async _runIPNIReportingDailyRunnerJob() { + const result = await this.getAggregatedProvidersReportingStatus(); + + const data = { + not_reporting: result.notReporting, + misreporting: result.misreporting, + ok: result.ok, + total: result.total, + }; + + await this.prismaService.ipni_reporting_daily.create({ data: data }); + } // because of lotus api rate limiting, this function first tries to get all providers status in parallel // and then retries sequentially for failed requests // throws error if sequential retry fails for any provider public async getAggregatedProvidersReportingStatus(): Promise { - const storageProviders = await this.storageProviderService.getProviders(); + const storageProviders = await this.poRepService.getProviders(); const result: ProviderIPNIReportingStatus[] = []; // try to execute all in parallel const promiseResults = await Promise.allSettled( storageProviders.map((storageProvider) => - this.getProviderReportingStatus(storageProvider.id), + this.getProviderReportingStatus(storageProvider), ), ); @@ -42,9 +102,7 @@ export class IpniMisreportingCheckerService { result.push((promiseResults[i] as PromiseFulfilledResult).value); } else { // retry sequentially for failed requests - result.push( - await this.getProviderReportingStatus(storageProviders[i].id), - ); + result.push(await this.getProviderReportingStatus(storageProviders[i])); } } @@ -62,27 +120,13 @@ export class IpniMisreportingCheckerService { }; } - public async getAggregatedProvidersReportingStatusWeekly(): Promise { - const result = await this.prismaService.$queryRawTyped( - getIpniReportingWeekly(), - ); - - return { - results: result.map((r) => ({ - week: r.week, - total: r.total, - misreporting: r.misreporting, - notReporting: r.not_reporting, - ok: r.ok, - })), - }; - } - public async getProviderReportingStatus( - storageProviderId: string, + storageProviderId: F0Id, minerInfo?: LotusStateMinerInfoResponse, ): Promise { - minerInfo ??= await this.lotusApiService.getMinerInfo(storageProviderId); + minerInfo ??= await this.lotusApiService.getMinerInfo( + storageProviderId.toString(), + ); const actualClaimsCount = await this.getProviderActualClaimsCount(storageProviderId); @@ -106,19 +150,19 @@ export class IpniMisreportingCheckerService { } private async getProviderActualClaimsCount( - storageProviderId: string, + storageProviderId: F0Id, ): Promise { return bigIntToNumber( ( - await this.prismaService.client_provider_distribution.aggregate({ - _sum: { - claims_count: true, - }, - where: { - provider: storageProviderId, - }, - }) - )._sum.claims_count, + await this.prismaService.$queryRaw<{ + count: bigint; + }>` + SELECT count("po_rep_deal_pieces"."piece_cid") + FROM "po_rep_deal_pieces" + JOIN "po_rep_deal" ON "po_rep_deal"."dealId" = "po_rep_deal_pieces"."deal_id" + WHERE "po_rep_deal"."providerId" = ${storageProviderId.toBigInt()}; + ` + )?.[0]?.count ?? 0n, ); } diff --git a/src/jobs/ipni-reporting-daily-runner/types.ipni-reporting-daily-runner.ts b/src/jobs/ipni-reporting-daily-runner/types.ipni-reporting-daily-runner.ts new file mode 100644 index 0000000..d069e5c --- /dev/null +++ b/src/jobs/ipni-reporting-daily-runner/types.ipni-reporting-daily-runner.ts @@ -0,0 +1,36 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export enum StorageProviderIpniReportingStatus { + MISREPORTING = 'MISREPORTING', + NOT_REPORTING = 'NOT_REPORTING', + OK = 'OK', +} + +export class AggregatedProvidersIPNIReportingStatus { + @ApiProperty({ + description: 'Number of storage providers misreporting the IPNI data', + }) + misreporting: number; + + @ApiProperty({ + description: 'Number of storage providers reporting no IPNI data', + }) + notReporting: number; + + @ApiProperty({ + description: + 'Number of storage providers reporting the IPNI data correctly', + }) + ok: number; + + @ApiProperty({ + description: 'Total number of storage providers', + }) + total: number; +} + +export class ProviderIPNIReportingStatus { + status: StorageProviderIpniReportingStatus; + actualClaimsCount: number; + ipniReportedClaimsCount: number | null; +} diff --git a/src/service/cid-contact/cid-contact.service.ts b/src/service/cid-contact/cid-contact.service.ts index db7b5f9..ad572f0 100644 --- a/src/service/cid-contact/cid-contact.service.ts +++ b/src/service/cid-contact/cid-contact.service.ts @@ -4,8 +4,11 @@ import { AxiosRequestConfig } from 'axios'; import { decodeAllSync } from 'cbor'; import { Multiaddr } from 'multiaddr'; import { lastValueFrom } from 'rxjs'; -import { Address } from '../location/types.location'; -import { IPNIAdvertisement, IPNIProvider } from './types.cid-contact'; +import { + IPAddress, + IPNIAdvertisement, + IPNIProvider, +} from './types.cid-contact'; const base64Regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/; @@ -17,6 +20,16 @@ export class CidContactService { constructor(private readonly httpService: HttpService) {} public async getIPNIProviders(): Promise { + try { + return await this._getIPNIProviders(); + } catch (err) { + throw new Error(`Error fetching IPNI providers: ${err.message}`, { + cause: err, + }); + } + } + + private async _getIPNIProviders(): Promise { const endpoint = 'https://cid.contact/providers'; const { data } = await lastValueFrom(this.httpService.get(endpoint)); return data; @@ -45,9 +58,7 @@ export class CidContactService { if (!advertisementId) return null; const endpoint = `${baseUrl}/ipni/v1/ad/${advertisementId}`; - const { data } = await lastValueFrom(this.httpService.get(endpoint)); - return { ...data, ID: advertisementId }; } @@ -102,7 +113,6 @@ export class CidContactService { if (isCurioBaseUrl) { const decodedCborCurio = decodeAllSync(data); // decode as CBOR - entriesCount += decodedCborCurio[0]?.Entries?.length; nextEntriesData = decodedCborCurio[0]?.Next?.['/']; } else { @@ -121,7 +131,7 @@ export class CidContactService { public extractMultiaddrAndBuildPublisherBaseUrl(publisherAddress: string): { multiaddrString: string; publisherBaseUrl: string; - multiaddr: Address; + multiaddr: IPAddress; } { let finalMultiAddrToParse = publisherAddress; @@ -136,12 +146,12 @@ export class CidContactService { let curioSuffix = ''; - // TODO temporary fix needed because multiaddr library does not support /dns/ prefix + // fix needed because multiaddr library does not support /dns/ prefix if (finalMultiAddrToParse.startsWith('/dns/')) { finalMultiAddrToParse = finalMultiAddrToParse.replace('/dns/', '/dns4/'); } - // TODO temporary fix needed because multiaddr library does not support /http-path/ and /ipni-provider/ sections - curio includes this in their multiaddrs + // fix needed because multiaddr library does not support /http-path/ and /ipni-provider/ sections - curio includes this in their multiaddrs if ( finalMultiAddrToParse.includes('http-path') && finalMultiAddrToParse.includes('ipni-provider') @@ -164,7 +174,7 @@ export class CidContactService { cleanedAddress.length, ); - // Add missing STANDARD parts of multiaddr to curio multiaddr - curio omits tcp/port before http/https + // add missing STANDARD parts of multiaddr to curio multiaddr - curio omits tcp/port before http/https if (newMultiAddrCurio.endsWith('https')) { newMultiAddrCurio = newMultiAddrCurio.replace('https', 'tcp/443/https'); } else if (newMultiAddrCurio.endsWith('/http')) { @@ -176,7 +186,7 @@ export class CidContactService { const multiaddrInstance = new Multiaddr(finalMultiAddrToParse); - const publisherAddressInstance: Address = { + const publisherAddressInstance: IPAddress = { address: multiaddrInstance.nodeAddress().address, port: multiaddrInstance.nodeAddress().port, protocol: multiaddrInstance.protos()[0].name, diff --git a/src/service/cid-contact/types.cid-contact.ts b/src/service/cid-contact/types.cid-contact.ts index 0d31102..0ec8e76 100644 --- a/src/service/cid-contact/types.cid-contact.ts +++ b/src/service/cid-contact/types.cid-contact.ts @@ -46,3 +46,10 @@ export interface IPNIAdvertisement { }; }; } + +export interface IPAddress { + address: string; + port: number; + protocol: string; + isHttps?: boolean; +} diff --git a/src/service/location/location.service.ts b/src/service/location/location.service.ts deleted file mode 100644 index 4a6925c..0000000 --- a/src/service/location/location.service.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { HttpService } from '@nestjs/axios'; -import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager'; -import { Inject, Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { resolve4, resolve6 } from 'dns/promises'; -import { firstValueFrom } from 'rxjs'; -import { Cacheable } from 'src/utils/cacheable'; -import { CidContactService } from '../cid-contact/cid-contact.service'; -import { Address, IPResponse } from './types.location'; - -@Injectable() -export class LocationService { - private readonly logger = new Logger(LocationService.name); - - constructor( - private configService: ConfigService, - private readonly httpService: HttpService, - private readonly cidContactService: CidContactService, - @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, - ) {} - - public async getLocation(multiAddrs?: string[]): Promise { - if (!multiAddrs) return null; - - try { - return await this._getLocation(multiAddrs); - } catch (err) { - this.logger.warn( - `Error getting location for ${multiAddrs}: ${err.message}`, - // err.cause?.stack || err.stack, - ); - - return null; - } - } - - @Cacheable({ ttl: 1000 * 60 * 60 * 12 }) // 12 hours - public async resolveAddress(address: Address): Promise { - let result: string[]; - - switch (address.protocol) { - case 'dns4': - result = await resolve4(address.address); - break; - case 'dns6': - result = await resolve6(address.address); - break; - case 'ip4': - case 'ip6': - result = [address.address]; - break; - default: - this.logger.error(`Unknown address / protocol: ${address}`); - result = []; - } - - return result[0]; - } - - // returns the location of the first non-bogon IP - private async _getLocation(multiAddrs: string[]): Promise { - const ips: string[] = []; - - for (const multiaddr of multiAddrs) { - const parsedMultiaddr = - this.cidContactService.extractMultiaddrAndBuildPublisherBaseUrl( - multiaddr, - ); - - if (!parsedMultiaddr.multiaddr) continue; - - ips.push(await this.resolveAddress(parsedMultiaddr.multiaddr)); - } - - return await this.getLocationDetails(ips); - } - - @Cacheable({ ttl: 1000 * 60 * 60 * 12 }) // 12 hours - private async _getLocationDetails(ip: string): Promise { - const ipInfoToken = this.configService.get('IP_INFO_TOKEN'); - - const { data } = await firstValueFrom( - this.httpService.get( - `https://ipinfo.io/${ip}?token=${ipInfoToken}`, - ), - ); - - return data; - } - - // returns the first non-bogon IP response - private async getLocationDetails(ips: string[]): Promise { - for (const ip of ips) { - const data = await this._getLocationDetails(ip); - if (data.bogon === true) continue; - - return data; - } - - return null; - } -} diff --git a/src/service/location/types.location.ts b/src/service/location/types.location.ts deleted file mode 100644 index 876e77a..0000000 --- a/src/service/location/types.location.ts +++ /dev/null @@ -1,18 +0,0 @@ -export interface IPResponse { - ip: string; - city: string; - region: string; - country: string; - loc?: string; - org?: string; - postal?: string; - timezone: string; - bogon?: boolean; -} - -export interface Address { - address: string; - port: number; - protocol: string; - isHttps?: boolean; -} diff --git a/src/service/lotus-api/lotus-api.service.ts b/src/service/lotus-api/lotus-api.service.ts index 852e206..d8173c9 100644 --- a/src/service/lotus-api/lotus-api.service.ts +++ b/src/service/lotus-api/lotus-api.service.ts @@ -6,10 +6,7 @@ import { firstValueFrom } from 'rxjs'; import { Cacheable } from 'src/utils/cacheable'; import { Retryable } from 'src/utils/retryable'; import { EthApiService } from '../eth-api/eth-api.service'; -import { - LotusStateMinerInfoResponse, - LotusStateVerifiedClientStatusResponse, -} from './types.lotus-api'; +import { LotusStateMinerInfoResponse } from './types.lotus-api'; @Injectable() export class LotusApiService { @@ -36,6 +33,15 @@ export class LotusApiService { } } + @Cacheable({ ttl: 1000 * 60 * 60 * 12 }) // 12 hours + public async listMiners(): Promise { + try { + return await this._listMiners(); + } catch (err) { + throw new Error(`Error listing miners: ${err.message}`, { cause: err }); + } + } + @Retryable({ retries: 3, delay: 5000 }) // 5 seconds private async _getMinerInfo( storageProviderId: string, @@ -67,26 +73,21 @@ export class LotusApiService { }; } - public async getClientDatacap(clientId: string): Promise { + @Retryable({ retries: 3, delay: 5000 }) // 5 seconds + private async _listMiners(): Promise { const endpoint = `${this.configService.get('GLIF_API_BASE_URL')}/v1`; const { data } = await firstValueFrom( - this.httpService.post(endpoint, { + this.httpService.post<{ result: string[] }>(endpoint, { jsonrpc: '2.0', - method: 'Filecoin.StateVerifiedClientStatus', - params: [clientId, []], - id: 0, + id: 1, + method: 'Filecoin.StateListMiners', + params: [null], }), ); - if (data.error || !data.result) { - this.logger.warn( - `Glif API returned an error for StateVerifiedClientStatus with clientId ${clientId}: ${JSON.stringify(data)}`, - ); - - return null; - } + if (!data?.result) throw new Error(`No data`); - return BigInt(data.result); + return data.result; } } diff --git a/src/service/lotus-api/types.lotus-api.ts b/src/service/lotus-api/types.lotus-api.ts index 0e846bc..1213de0 100644 --- a/src/service/lotus-api/types.lotus-api.ts +++ b/src/service/lotus-api/types.lotus-api.ts @@ -1,21 +1,9 @@ -export interface LotusStateVerifiedClientStatusResponse { - jsonrpc: string; - result?: string; - error?: LotusError; - id: number; -} - export interface LotusStateMinerInfoResponse { jsonrpc: string; result: LotusStateMinerInfoResult; id: number; } -export interface LotusError { - code: number; - message: string; -} - export interface LotusStateMinerInfoResult { Beneficiary: string; BeneficiaryTerm: BeneficiaryTerm; diff --git a/src/service/po-rep/po-rep.service.ts b/src/service/po-rep/po-rep.service.ts index 339eb31..21a64c8 100644 --- a/src/service/po-rep/po-rep.service.ts +++ b/src/service/po-rep/po-rep.service.ts @@ -187,6 +187,17 @@ export class PoRepService { }; } + public async getProviders(): Promise { + const results = await this.prismaService.po_rep_deal.findMany({ + select: { + providerId: true, + }, + distinct: ['providerId'], + }); + + return results.map((result) => F0Id.from(result.providerId)); + } + public async getProviderStorageStatistics( providerId: F0Id | F0IdInput, ): Promise { @@ -580,8 +591,7 @@ export class PoRepService { }); const tokenDetailsResponses = await Promise.all(tokenDetailsRequests); - const tokenDetailsMap = new Map(tokenDetailsResponses); - return tokenDetailsMap; + return new Map(tokenDetailsResponses); } private async getTokensExchangeRatesUSD( @@ -595,8 +605,6 @@ export class PoRepService { }); const tokensExchangeRates = await Promise.all(tokenExchangeRateRequests); - const tokensExchangeRateMap = new Map(tokensExchangeRates); - - return tokensExchangeRateMap; + return new Map(tokensExchangeRates); } } diff --git a/src/service/storage-provider/storage-provider.service.ts b/src/service/storage-provider/storage-provider.service.ts deleted file mode 100644 index 37eb192..0000000 --- a/src/service/storage-provider/storage-provider.service.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Cache, CACHE_MANAGER } from '@nestjs/cache-manager'; -import { Inject, Injectable, Logger } from '@nestjs/common'; -import { PrismaService } from 'src/db/prisma.service'; - -@Injectable() -export class StorageProviderService { - private readonly logger = new Logger(StorageProviderService.name); - - constructor( - private readonly prismaService: PrismaService, - @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, - ) {} - - public async getProviders() { - return this.prismaService.provider.findMany({}); - } -} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 869c99c..101773c 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -66,7 +66,7 @@ export function envSet(value?: any): boolean { return !envNotSet(value); } -export async function sleep(ms: number): Promise { +export async function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); }