feat(cli): emit telemetry for the validate action

packages/@aws-cdk-testing/cli-integ/tests/telemetry-integ-tests/cdk-validate-telemetry.integtest.ts

@@ -0,0 +1,41 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { integTest, withSpecificFixture } from '../../lib';
+
+integTest(
+  'cdk validate records offlineWouldFailDeploy on the SYNTH telemetry event',
+  withSpecificFixture('validate-app', async (fixture) => {
+    const telemetryFile = path.join(fixture.integTestDir, `telemetry-validate-${Date.now()}.json`);
+
+    // --no-online keeps the run deterministic; offline validation still runs
+    // during synthesis, and its deploy-blocking outcome is recorded on the
+    // SYNTH event as offlineWouldFailDeploy (visible for synth/deploy/validate).
+    await fixture.cdk(
+      ['--unstable=validate', 'validate', fixture.fullStackName('validate'), '--no-online', `--telemetry-file=${telemetryFile}`],
+      {
+        verboseLevel: 3, // trace mode
+        allowErrExit: true, // violations make validate exit non-zero
+      },
+    );
+
+    // Assert against the local telemetry file: the CLI dispatches the batch to
+    // the endpoint from a background process, so its own output only confirms
+    // dispatch, not delivery. Real endpoint delivery is covered separately by
+    // cdk-telemetry-reaches-the-endpoint.integtest.ts.
+    const json = fs.readJSONSync(telemetryFile);
+    const synthEvent = json.find((e: any) => e.event?.eventType === 'SYNTH');
+    expect(synthEvent).toBeDefined();
+
+    // The app's single S3 bucket makes SecurityPlugin report a fatal/error
+    // violation (what would have failed a deploy) and a warning-severity
+    // violation (counted by offlineValidationWarnings, separate from annotation warnings).
+    expect(synthEvent.counters).toEqual(
+      expect.objectContaining({
+        offlineWouldFailDeploy: 1,
+        offlineValidationWarnings: 1,
+      }),
+    );
+
+    fs.unlinkSync(telemetryFile);
+  }),
+);

packages/@aws-cdk/toolkit-lib/docs/message-registry.md

@@ -158,6 +158,8 @@ Please let us know by [opening an issue](https://github.com/aws/aws-cdk-cli/issu
 | `CDK_TOOLKIT_E9600` | Policy validation failed | `error` | {@link ValidateResult} |
 | `CDK_TOOLKIT_I9601` | No policy validation report found | `info` | n/a |
 | `CDK_TOOLKIT_W9602` | Online validation could not be completed for a stack | `warn` | n/a |
+| `CDK_TOOLKIT_I9603` | Online validation is starting | `trace` | {@link StackSelectionDetails} |
+| `CDK_TOOLKIT_I9604` | Online validation has finished. Provides online validation timing, outcome counters, and any engine failure. | `trace` | {@link OnlineValidationResult} |
 | `CDK_TOOLKIT_I0100` | Notices decoration (the header or footer of a list of notices) | `info` | n/a |
 | `CDK_TOOLKIT_W0101` | A notice that is marked as a warning | `warn` | n/a |
 | `CDK_TOOLKIT_E0101` | A notice that is marked as an error | `error` | n/a |

packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts

@@ -27,6 +27,7 @@ import type {
   ContextProviderMessageSource,
   Duration,
   ErrorPayload,
+  OnlineValidationResult,
   Operation,
   SingleStack,
   StackAndAssemblyData,
@@ -563,6 +564,18 @@ export const IO = {
     description: 'Online validation could not be completed for a stack',
   }),
 
+  CDK_TOOLKIT_I9603: make.trace<StackSelectionDetails>({
+    code: 'CDK_TOOLKIT_I9603',
+    description: 'Online validation is starting',
+    interface: 'StackSelectionDetails',
+  }),
+
+  CDK_TOOLKIT_I9604: make.trace<OnlineValidationResult>({
+    code: 'CDK_TOOLKIT_I9604',
+    description: 'Online validation has finished. Provides online validation timing, outcome counters, and any engine failure.',
+    interface: 'OnlineValidationResult',
+  }),
+
   // Notices
   CDK_TOOLKIT_I0100: make.info({
     code: 'CDK_TOOLKIT_I0100',
@@ -734,4 +747,9 @@ export const SPAN = {
     start: IO.CDK_TOOLKIT_I5400,
     end: IO.CDK_TOOLKIT_I5410,
   },
+  VALIDATE_ONLINE: {
+    name: 'Online validation',
+    start: IO.CDK_TOOLKIT_I9603,
+    end: IO.CDK_TOOLKIT_I9604,
+  },
 } satisfies Record<string, SpanDefinition<any, any>>;

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts

@@ -1,7 +1,7 @@
 import type { WorkNode, StackNode, AssetBuildNode, AssetPublishNode, MarkerNode } from './work-graph-types';
 import { DeploymentState } from './work-graph-types';
 import { ToolkitError } from '../../toolkit/toolkit-error';
-import { parallelPromises } from '../../util';
+import { parallelPromises, sum } from '../../util';
 import type { IoHelper } from '../io/private';
 export type Concurrency = number | Record<WorkNode['type'], number>;
 
@@ -416,14 +416,6 @@ export interface WorkGraphActions {
   marker: (markerNode: MarkerNode) => Promise<void>;
 }
 
-function sum(xs: number[]) {
-  let ret = 0;
-  for (const x of xs) {
-    ret += x;
-  }
-  return ret;
-}
-
 function retainOnly<A>(xs: A[], pred: (x: A) => boolean) {
   xs.splice(0, xs.length, ...xs.filter(pred));
 }

packages/@aws-cdk/toolkit-lib/lib/payloads/types.ts

@@ -100,6 +100,34 @@ export interface Operation extends Duration {
   readonly error?: Error;
 }
 
+/**
+ * The result of the online (CloudFormation change set) validation phase of the
+ * `validate` action.
+ *
+ * This payload is only emitted when online validation actually runs. `duration`
+ * times the online phase and `counters` describe only its outcome. Offline
+ * validation runs during synthesis and its counters are reported separately.
+ */
+export interface OnlineValidationResult extends Duration {
+  /**
+   * Counters describing the outcome of the online validation phase.
+   *
+   * @default - no counters
+   */
+  readonly counters?: Record<string, number>;
+
+  /**
+   * Set when the online validation engine could not run at all.
+   *
+   * Online validation finding template problems is not an error; this is only
+   * set when the validation itself could not be performed (for example, when
+   * the CloudFormation calls failed for every selected stack).
+   *
+   * @default - online validation ran
+   */
+  readonly error?: Error;
+}
+
 /**
  * Generic payload of a simple yes/no question.
  *

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-assembly-results.ts

@@ -1,13 +1,27 @@
+import * as path from 'path';
 import type * as cxapi from '@aws-cdk/cloud-assembly-api';
 import { SynthesisMessageLevel } from '@aws-cdk/cloud-assembly-api';
+import type { PluginReportJson } from '@aws-cdk/cloud-assembly-schema';
+import { Manifest } from '@aws-cdk/cloud-assembly-schema';
+import * as fs from 'fs-extra';
 import type { IMessageSpan } from '../../api/io/private/span';
+import { sum } from '../../util';
+
+const VALIDATION_REPORT_FILE = 'validation-report.json';
 
 export function countAssemblyResults(span: IMessageSpan<any>, assembly: cxapi.CloudAssembly) {
   const stacksRecursively = assembly.stacksRecursively;
+  const summary = offlineValidationSummary(assembly);
+  const messageWarnings = sum(stacksRecursively.map(s => s.messages.filter(m => m.level === SynthesisMessageLevel.WARNING).length));
   span.incCounter('stacks', stacksRecursively.length);
   span.incCounter('assemblies', asmCount(assembly));
   span.incCounter('errorAnns', sum(stacksRecursively.map(s => s.messages.filter(m => m.level === SynthesisMessageLevel.ERROR).length)));
-  span.incCounter('warnings', sum(stacksRecursively.map(s => s.messages.filter(m => m.level === SynthesisMessageLevel.WARNING).length)));
+  // Construct annotation warnings live in stack.messages for legacy assemblies
+  // and in the validation report (report-only mode) for newer ones; the two are
+  // mutually exclusive, so summing them counts each once.
+  span.incCounter('warnings', messageWarnings + summary.reportAnnotationWarnings);
+  span.incCounter('offlineValidationWarnings', summary.offlineValidationWarnings);
+  span.incCounter('offlineWouldFailDeploy', summary.wouldFailDeploy ? 1 : 0);
 
   const annotationErrorCodes = stacksRecursively
     .flatMap(s => Object.values(s.metadata ?? {})
@@ -21,8 +35,73 @@ export function countAssemblyResults(span: IMessageSpan<any>, assembly: cxapi.Cl
   }
 }
 
-function sum(xs: number[]) {
-  return xs.reduce((a, b) => a + b, 0);
+export interface OfflineValidationSummary {
+  /**
+   * Whether the offline validation results would have failed a default `cdk deploy`
+   *
+   * Mirrors `wouldFailDeploy` at the default 'error' threshold over the whole
+   * assembly (independent of any `--strict`/`--ignore-errors` on the current
+   * command): a deploy fails if there are error-level construct annotations or a
+   * policy plugin reported a failure.
+   */
+  readonly wouldFailDeploy: boolean;
+
+  /**
+   * The number of warning-severity violations reported by policy plugins
+   *
+   * Construct annotation warnings are excluded (they are counted by the
+   * `warnings` counter), so this only reflects policy validation plugins.
+   */
+  readonly offlineValidationWarnings: number;
+
+  /**
+   * The number of warning-severity construct annotations recorded in the
+   * validation report (report-only assemblies)
+   *
+   * These are folded into the `warnings` counter, not `offlineValidationWarnings`,
+   * so report-only annotation warnings are counted like message-based ones.
+   */
+  readonly reportAnnotationWarnings: number;
+}
+
+/**
+ * Summarize the offline validation outcome (policy report + construct annotations) for the whole assembly
+ */
+export function offlineValidationSummary(assembly: cxapi.CloudAssembly): OfflineValidationSummary {
+  const hasErrorAnnotations = assembly.stacksRecursively.some(
+    s => s.messages.some(m => m.level === SynthesisMessageLevel.ERROR),
+  );
+
+  const pluginReports = loadValidationReport(assembly);
+
+  const warningsFor = (isAnnotation: boolean) => pluginReports
+    .filter(r => (r.pluginName === CONSTRUCT_ANNOTATIONS_PLUGINNAME) === isAnnotation)
+    .reduce((acc, r) => acc + r.violations.filter(v => v.severity === 'warning').length, 0);
+
+  return {
+    wouldFailDeploy: hasErrorAnnotations || pluginReports.some(r => r.conclusion === 'failure'),
+    offlineValidationWarnings: warningsFor(false),
+    reportAnnotationWarnings: warningsFor(true),
+  };
+}
+
+/**
+ * Load the policy validation report, if any
+ *
+ * These counters are best-effort telemetry that run on every synth, so a
+ * missing or malformed report must never fail the command: on any read or
+ * schema error we behave as if there were no report.
+ */
+function loadValidationReport(assembly: cxapi.CloudAssembly): PluginReportJson[] {
+  const reportPath = path.join(assembly.directory, VALIDATION_REPORT_FILE);
+  if (!fs.existsSync(reportPath)) {
+    return [];
+  }
+  try {
+    return Manifest.loadValidationReport(reportPath).pluginReports;
+  } catch {
+    return [];
+  }
 }
 
 /**
@@ -31,3 +110,11 @@ function sum(xs: number[]) {
  * Do not change, obviously.
  */
 const ANNOTATION_ERROR_CODE_TYPE = 'aws:cdk:error-code';
+
+/**
+ * The name of the plugin that emits construct annotations into the validation report.
+ *
+ * Its warnings are counted by the `warnings` counter, so they are excluded from
+ * the policy warning count.
+ */
+const CONSTRUCT_ANNOTATIONS_PLUGINNAME = 'Construct Annotations';

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts

@@ -0,0 +1,17 @@
+import type { PluginReportJson } from '@aws-cdk/cloud-assembly-schema';
+import type { IMessageSpan } from '../../api/io/private/span';
+import { sum } from '../../util';
+
+/**
+ * Add counters describing the online validation outcome to the given span
+ *
+ * `onlineViolations` counts the violations online validation reported, and
+ * `online:stacksIncomplete` records how many selected stacks could not be
+ * validated (0 on a clean run; every selected stack when the engine could not
+ * run at all, e.g. a setup failure before validation started). Both are always
+ * emitted so consumers can distinguish zero from missing data.
+ */
+export function countOnlineValidationResults(span: IMessageSpan<any>, onlineReports: PluginReportJson[] | undefined, incompleteStacks: number) {
+  span.incCounter('onlineViolations', sum((onlineReports ?? []).map((r) => r.violations.length)));
+  span.incCounter('online:stacksIncomplete', incompleteStacks);
+}

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts

@@ -88,26 +88,36 @@ export async function throwIfValidationFailures(
   const result: ValidateResult = { conclusion, pluginReports };
   await ioHelper.notify(hostMessageFromValidation(process.cwd(), result));
 
+  if (!wouldFailDeploy(pluginReports, failAt)) {
+    return;
+  }
+
+  if (failAt === 'warn') {
+    const error = AssemblyError.withStacks('Synthesis finished with warnings (--strict mode)', stacks.stackArtifacts);
+    error.attachSynthesisErrorCode('StrictAnnotationWarnings');
+    throw error;
+  }
+
+  const error = AssemblyError.withStacks('Synthesis finished with errors', stacks.stackArtifacts);
+  error.attachSynthesisErrorCode('AnnotationErrors');
+  throw error;
+}
+
+/**
+ * Whether the given validation reports make a deploy-like action fail at the given severity threshold
+ *
+ * This is the exact predicate applied by `throwIfValidationFailures`.
+ */
+export function wouldFailDeploy(pluginReports: PluginReportJson[], failAt: MinimumSeverity): boolean {
   switch (failAt) {
     case 'error':
-      if (conclusion === 'failure') {
-        const error = AssemblyError.withStacks('Synthesis finished with errors', stacks.stackArtifacts);
-        error.attachSynthesisErrorCode('AnnotationErrors');
-        throw error;
-      }
-      break;
+      return combineConclusions(pluginReports) === 'failure';
     case 'warn':
-      // if we're failing at 'warn', then both warnings and errors cause failure, so the initial conclusion is correct
-      if (conclusion === 'failure' || hasWarnings(pluginReports)) {
-        const error = AssemblyError.withStacks('Synthesis finished with warnings (--strict mode)', stacks.stackArtifacts);
-        error.attachSynthesisErrorCode('StrictAnnotationWarnings');
-        throw error;
-      }
-
-      break;
+      // if we're failing at 'warn', then both warnings and errors cause failure
+      return combineConclusions(pluginReports) === 'failure' || hasWarnings(pluginReports);
     case 'none':
       // if we're not failing at all, then the conclusion is always success
-      break;
+      return false;
   }
 }