From ebe985e0303e8425c68584816b64b7a10de2d117 Mon Sep 17 00:00:00 2001 From: Aditya Jain Date: Wed, 23 Sep 2026 02:13:37 -0700 Subject: [PATCH] fix(toolkit-lib): cdk watch loses log events when one log group is throttled The watch log monitor reads all log groups in parallel with Promise.all. When one FilterLogEvents call failed (usually a ThrottlingException on stacks with many functions), Promise.all rejected and the whole tick was discarded. The other log groups had already moved their start time past the events they read, so those events were never printed. The user only saw "Error occurred while monitoring logs: ThrottlingException: Rate exceeded". Errors are now handled per log group. The events of the other log groups are printed. A throttled log group keeps its start time, so its events are read on the next tick, and the throttle is reported at debug level instead of as an error. Other errors are still reported. Fixes #879 Co-Authored-By: Claude Opus 5.5 --- .../lib/api/logs-monitor/logs-monitor.ts | 34 +++++++++--- .../api/logs-monitor/logs-monitor.test.ts | 54 +++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/logs-monitor/logs-monitor.ts b/packages/@aws-cdk/toolkit-lib/lib/api/logs-monitor/logs-monitor.ts index 7d5f06666..900467467 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/logs-monitor/logs-monitor.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/logs-monitor/logs-monitor.ts @@ -3,7 +3,6 @@ import * as util from 'node:util'; import type * as cxapi from '@aws-cdk/cloud-assembly-api'; import chalk from 'chalk'; import type { CloudWatchLogEvent } from '../../payloads/logs-monitor'; -import { flatten } from '../../util'; import type { SDK } from '../aws-auth/private'; import { IO } from '../io/private'; import type { IoHelper } from '../io/private'; @@ -156,9 +155,14 @@ export class CloudWatchLogEventMonitor { /* c8 ignore stop */ try { - const events = flatten(await this.readNewEvents()); - for (const event of events) { - await this.print(event); + for (const result of await this.readNewEvents()) { + if ('error' in result) { + await this.reportError(result.error); + continue; + } + for (const event of result.events) { + await this.print(event); + } } // We might have been stop()ped while the network call was in progress. @@ -166,21 +170,35 @@ export class CloudWatchLogEventMonitor { return; } } catch (e: any) { - await this.ioHelper.notify(IO.CDK_TOOLKIT_E5035.msg(`Error occurred while monitoring logs: ${String(e)}`, { error: e })); + await this.reportError(e); } this.scheduleNextTick(); } + private async reportError(e: any): Promise { + if (e?.name === 'ThrottlingException') { + // The log group keeps its start time, so the events are read on the next tick. + await this.ioHelper.defaults.debug(`Throttled while monitoring logs, will retry: ${String(e)}`); + return; + } + await this.ioHelper.notify(IO.CDK_TOOLKIT_E5035.msg(`Error occurred while monitoring logs: ${String(e)}`, { error: e })); + } + /** * Reads all new log events from a set of CloudWatch Log Groups * in parallel + * + * A failure for one log group does not discard the events of the other log groups. */ - private async readNewEvents(): Promise>> { - const promises: Array>> = []; + private async readNewEvents(): Promise> { + const promises: Array> = []; for (const settings of this.envsLogGroupsAccessSettings.values()) { for (const group of Object.keys(settings.logGroupsStartTimes)) { - promises.push(this.readEventsFromLogGroup(settings, group)); + promises.push(this.readEventsFromLogGroup(settings, group).then( + (events) => ({ events }), + (error) => ({ error }), + )); } } // Limited set of log groups diff --git a/packages/@aws-cdk/toolkit-lib/test/api/logs-monitor/logs-monitor.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/logs-monitor/logs-monitor.test.ts index 99452dffa..ecdf02dda 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/logs-monitor/logs-monitor.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/logs-monitor/logs-monitor.test.ts @@ -90,6 +90,60 @@ test('process truncated events', async () => { ); }); +describe('when reading one log group fails', () => { + const env = { name: 'name', account: '11111111111', region: 'us-east-1' }; + const throttle = Object.assign(new Error('Rate exceeded'), { name: 'ThrottlingException' }); + + function messages(): string[] { + return ioHost.notifySpy.mock.calls.map((call) => stripAnsi(call[0].message)); + } + + test('a throttled log group does not discard the events of the other log groups, and is read again later', async () => { + // GIVEN + mockCloudWatchClient.on(FilterLogEventsCommand, { logGroupName: 'group-a' }) + .resolvesOnce({ events: [event(102, 'message-a', new Date(T102))] }) + .resolves({ events: [] }); + mockCloudWatchClient.on(FilterLogEventsCommand, { logGroupName: 'group-b' }) + .rejectsOnce(throttle) + .resolvesOnce({ events: [event(103, 'message-b', new Date(T102))] }) + .resolves({ events: [] }); + monitor.addLogGroups(env, sdk, ['group-a', 'group-b']); + + // WHEN + await monitor.activate(); + await sleep(2500); + + // THEN + expect(messages()).toEqual([ + expect.stringContaining('[group-a]'), + expect.stringContaining('[group-b]'), + ]); + expect(messages()[0]).toContain('message-a'); + expect(messages()[1]).toContain('message-b'); + }); + + test('other errors are still reported, without discarding the events of the other log groups', async () => { + // GIVEN + mockCloudWatchClient.on(FilterLogEventsCommand, { logGroupName: 'group-a' }) + .resolvesOnce({ events: [event(102, 'message-a', new Date(T102))] }) + .resolves({ events: [] }); + mockCloudWatchClient.on(FilterLogEventsCommand, { logGroupName: 'group-b' }) + .rejectsOnce(new Error('Access denied')) + .resolves({ events: [] }); + monitor.addLogGroups(env, sdk, ['group-a', 'group-b']); + + // WHEN + await monitor.activate(); + await sleep(500); + + // THEN + expect(messages()).toEqual([ + expect.stringContaining('message-a'), + expect.stringContaining('Error occurred while monitoring logs: Error: Access denied'), + ]); + }); +}); + const T0 = 1597837230504; const T100 = T0 + 100 * 1000; const T102 = T0 + 102 * 1000;