diff --git a/API.md b/API.md index c15fb177e..628b9e841 100644 --- a/API.md +++ b/API.md @@ -2886,6 +2886,7 @@ new sources.NpmJs(props?: NpmJsProps) | metricLastSeq | The last sequence number that was processed. | | metricLateChangeCount | The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. | | metricLateChangeLag | For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. | +| metricMetadataFetchFailures | The number of change entries whose registry metadata could not be fetched (after transient-error retries). | | metricNpmJsChangeAge | *No description.* | | metricPackageVersionAge | The age of the oldest package version that was processed. | | metricPackageVersionCount | The total count of package versions that were inspected. | @@ -3040,6 +3041,23 @@ and can be used to tune the scan window. --- +##### `metricMetadataFetchFailures` + +```typescript +public metricMetadataFetchFailures(opts?: MetricOptions): Metric +``` + +The number of change entries whose registry metadata could not be fetched (after transient-error retries). + +These entries are not receipted, so a +later scan retries them. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricNpmJsChangeAge` ```typescript diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 3586de3a4..5e3412976 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -711,7 +711,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -9974,7 +9974,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10859,7 +10859,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -14790,7 +14790,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -24427,7 +24427,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -25312,7 +25312,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -29181,7 +29181,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -38444,7 +38444,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -39329,7 +39329,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -43346,7 +43346,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -52618,7 +52618,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -53490,7 +53490,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -57696,7 +57696,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -66775,7 +66775,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -67660,7 +67660,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 69aa58539..a985a1ac9 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -773,7 +773,7 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -11249,7 +11249,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "5890a5f32d4ca5288bf1b5299a0b6cb482820a5711468d81598bf681d7d34909.zip", + "S3Key": "d58fed793fe662f0b62aaceaede6cf0fc861f804fd2c8236871b3eecb5c18567.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -12128,7 +12128,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", + "S3Key": "4ffbb107d5c61355d7d0b2e1675dc33fa6ff3617685ff538e3a1e6cd07136f29.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts index 1bcc876c9..36e57d225 100644 --- a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -1,4 +1,4 @@ -import { gunzipSync } from 'zlib'; +import { gunzipSync, gzipSync } from 'zlib'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { GetObjectCommand, @@ -522,6 +522,116 @@ test('records interim checkpoints so an interrupted scan resumes mid-window', as expect(saved.lastDeepScanAt).toBeUndefined(); }); +test('retries truncated gzip responses from the registry', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { + seq: 1_005, + id: 'flaky-package', + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + }, + ], + last_seq: 1_005, + }); + // First packument response is a truncated gzip body (Z_BUF_ERROR), the + // retry succeeds. + const fullBody = gzipSync( + Buffer.from( + JSON.stringify(packageDoc('flaky-package', '1-aaaaaa', '1.0.0')) + ) + ); + nock(REGISTRY) + .get('/flaky-package') + .reply(200, fullBody.subarray(0, fullBody.length - 8), { + 'content-encoding': 'gzip', + }); + nock(REGISTRY) + .get('/flaky-package') + .reply(200, packageDoc('flaky-package', '1-aaaaaa', '1.0.0')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'flaky-package', version: '1.0.0' }, + ]); + expect(savedState().has(1_005)).toBe(true); +}); + +test('does not receipt entries whose metadata fetch failed, so a later scan retries them', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { + seq: 1_004, + id: 'broken-package', + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + }, + { + seq: 1_006, + id: 'good-package', + changes: [{ rev: '1-bbbbbb' }], + deleted: false, + }, + ], + last_seq: 1_006, + }); + // A client error is used so the request is not retried until the + // transient-error deadline. + nock(REGISTRY).get('/broken-package').reply(403, 'forbidden'); + nock(REGISTRY) + .get('/good-package') + .reply(200, packageDoc('good-package', '1-bbbbbb', '1.0.0')); + + await handler(event, context); + + // The healthy entry was processed and receipted; the failed one was not + // receipted, so the next scan will retry it. + expect(stagedPackages()).toEqual([ + { name: 'good-package', version: '1.0.0' }, + ]); + const state = savedState(); + expect(state.has(1_006)).toBe(true); + expect(state.has(1_004)).toBe(false); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.METADATA_FETCH_FAILURES, + 1, + Unit.Count + ); +}); + +test('persists its state even when the scan fails part-way', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + + // The feed itself rejects the request: the scan throws. (A client error + // is used so the request is not retried until the transient-error deadline.) + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(403, 'forbidden'); + + await expect(handler(event, context)).rejects.toThrow(/HTTP 403/); + + // The state (here: the seeded checkpoint) was saved regardless. + expect(savedState().newestCheckpointSeq()).toBe(1_000); +}); + test('resets its position when the feed head regresses below the newest checkpoint', async () => { const now = Date.now(); // The feed head (3000) is below our newest checkpoint (5000). diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index 0c60da3a7..c11b3aa71 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -392,6 +392,12 @@ export class NpmJs implements IPackageSource { }), 0 ), + fillMetric( + this.metricMetadataFetchFailures({ + label: 'Metadata Fetch Failures', + }), + 0 + ), ], leftYAxis: { min: 0 }, right: [ @@ -522,6 +528,21 @@ export class NpmJs implements IPackageSource { }); } + /** + * The number of change entries whose registry metadata could not be fetched + * (after transient-error retries). These entries are not receipted, so a + * later scan retries them. + */ + public metricMetadataFetchFailures(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.METADATA_FETCH_FAILURES, + namespace: METRICS_NAMESPACE, + }); + } + /** * The number of packages for which the registry packument is still behind * the revision announced by the `_changes` feed. The versions served so far diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index caf354cd8..ea5f09ef6 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -32,6 +32,13 @@ export const enum MetricName { */ LATE_CHANGE_LAG = 'LateChangeLag', + /** + * Number of change entries whose registry metadata could not be fetched + * (after transient-error retries). These entries are not receipted, so a + * later scan retries them. + */ + METADATA_FETCH_FAILURES = 'MetadataFetchFailures', + /** * The number of packages for which the registry packument is still behind * the revision announced by the `_changes` feed ("laggy packuments"). The diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts index dc15e114a..243ba742b 100644 --- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts +++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts @@ -120,13 +120,31 @@ export class CouchChanges extends EventEmitter { */ public async attachAllMetadata( changes: readonly DatabaseChange[] - ): Promise { - const outcomes = await Promise.all( - changes.map((change) => this.attachMetadata(change)) - ); - return outcomes.filter( - (outcome): outcome is AttachedChange => outcome !== undefined + ): Promise { + const ok = new Array(); + const failedSeqs = new Array(); + await Promise.all( + changes.map(async (change) => { + try { + const outcome = await this.attachMetadata(change); + if (outcome !== undefined) { + ok.push(outcome); + } + } catch (error) { + // One unreachable packument must not fail the whole batch. The + // entry is reported as failed so the caller does not record a + // receipt for it; a later scan retries it. + console.error( + `Failed to fetch metadata for ${change.id}, a later scan will retry it: ${error}` + ); + const seq = Number(change.seq); + if (!isNaN(seq)) { + failedSeqs.push(seq); + } + } + }) ); + return { ok, failedSeqs }; } private async attachMetadata( @@ -301,7 +319,13 @@ function readResponseJson( class RetryableError extends Error {} function isRetryableError(e: Error): boolean { - return e instanceof RetryableError || (e as any).code === 'ECONNRESET'; + return ( + e instanceof RetryableError || + (e as any).code === 'ECONNRESET' || + // A truncated response body (the gunzip stream ends unexpectedly). Seen + // repeatedly from registry.npmjs.org; transient like a connection reset. + (e as any).code === 'Z_BUF_ERROR' + ); } async function sleep(ms: number) { @@ -352,6 +376,22 @@ export interface DatabaseChanges { readonly totalCount: number; } +/** + * The result of attaching registry metadata to a batch of change entries. + */ +export interface AttachedMetadata { + /** + * The change entries whose metadata could be fetched. + */ + readonly ok: AttachedChange[]; + + /** + * The sequence numbers of change entries whose metadata fetch failed. No + * receipt must be recorded for these, so that a later scan retries them. + */ + readonly failedSeqs: number[]; +} + /** * A change entry with its registry metadata attached. */ diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index a9ca3acd3..3c1c5ddaf 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -161,51 +161,56 @@ export async function handler(event: ScheduledEvent, context: Context) { state.addCheckpoint(head); } - // Re-check laggy packuments (packages whose registry packument was behind - // the revision announced by the feed when we first saw them). - const laggyStaged = await retryLaggyPackuments( - context, - npm, - state, - stagingFunction, - denyList, - licenseList, - knownVersions - ); - writeKnownVersionsFile ||= laggyStaged > 0; - - // Scan the trailing window of the feed. - const scanStaged = await runScan( - context, - npm, - state, - head, - stagingFunction, - denyList, - licenseList, - knownVersions - ); - writeKnownVersionsFile ||= scanStaged > 0; - - // The laggy packument gauge is emitted at the end of the run, so it - // reflects the expectations recorded by this run's scan (emitting it during - // the retry phase would always read the trough: after recoveries, before - // new expectations are recorded). - await metricScope((metrics) => async () => { - metrics.setDimensions({}); - metrics.putMetric( - MetricName.LAGGY_PACKUMENTS, - state.laggyPackumentCount, - Unit.Count + try { + // Re-check laggy packuments (packages whose registry packument was behind + // the revision announced by the feed when we first saw them). + const laggyStaged = await retryLaggyPackuments( + context, + npm, + state, + stagingFunction, + denyList, + licenseList, + knownVersions ); - })(); + writeKnownVersionsFile ||= laggyStaged > 0; + + // Scan the trailing window of the feed. + const scanStaged = await runScan( + context, + npm, + state, + head, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + writeKnownVersionsFile ||= scanStaged > 0; - await Promise.all([ - saveFollowerState(context, stagingBucket, state), - ...(writeKnownVersionsFile - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ]); + // The laggy packument gauge is emitted at the end of the run, so it + // reflects the expectations recorded by this run's scan (emitting it during + // the retry phase would always read the trough: after recoveries, before + // new expectations are recorded). + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS, + state.laggyPackumentCount, + Unit.Count + ); + })(); + } finally { + // Persist the state even when a scan failed part-way: receipts, interim + // checkpoints, and staged known versions represent completed work that + // the next run should not redo. + await Promise.all([ + saveFollowerState(context, stagingBucket, state), + ...(writeKnownVersionsFile + ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] + : []), + ]); + } console.log('All done here, we have success!'); @@ -364,7 +369,12 @@ async function processBatch( ); } - const attached = await npm.attachAllMetadata(fresh); + const { ok: attached, failedSeqs } = await npm.attachAllMetadata(fresh); + metrics.putMetric( + MetricName.METADATA_FETCH_FAILURES, + failedSeqs.length, + Unit.Count + ); // The most recent "modified" timestamp observed in the batch. let lastModified: Date | undefined; @@ -423,8 +433,11 @@ async function processBatch( } // Record receipts for every entry received (including deleted or - // unreachable packages: they were received and deliberately skipped). - state.addSeqs(batch.seqs); + // unpublished packages: they were received and deliberately skipped) - + // except entries whose metadata fetch failed, so a later scan retries + // them. + const failed = new Set(failedSeqs); + state.addSeqs(batch.seqs.filter((seq) => !failed.has(seq))); } finally { metrics.putMetric(MetricName.LAST_SEQ, head, Unit.None); metrics.putMetric(