diff --git a/packages/@aws-cdk/toolkit-lib/docs/message-registry.md b/packages/@aws-cdk/toolkit-lib/docs/message-registry.md index 22e921d10..7e4e850db 100644 --- a/packages/@aws-cdk/toolkit-lib/docs/message-registry.md +++ b/packages/@aws-cdk/toolkit-lib/docs/message-registry.md @@ -117,6 +117,7 @@ Please let us know by [opening an issue](https://github.com/aws/aws-cdk-cli/issu | `CDK_TOOLKIT_I5900` | Deployment results on success | `result` | {@link SuccessfulDeployStackResult} | | `CDK_TOOLKIT_I5901` | Generic deployment success messages | `info` | n/a | | `CDK_TOOLKIT_W5902` | Express Mode deployment completed with resources still stabilizing | `warn` | n/a | +| `CDK_TOOLKIT_W5903` | Deployment includes a replacement that CloudFormation does not support while rollback is disabled | `warn` | {@link ReplacementRequiresRollback} | | `CDK_TOOLKIT_W5400` | Hotswap disclosure message | `warn` | n/a | | `CDK_TOOLKIT_E5001` | No stacks found | `error` | n/a | | `CDK_TOOLKIT_E5500` | Stack Monitoring error | `error` | {@link ErrorPayload} | diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts index 36e00a616..f634e79f7 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts @@ -5,6 +5,7 @@ import { diffTemplate } from '@aws-cdk/cloudformation-diff'; import type { CreateChangeSetCommandInput, CreateStackCommandInput, + DescribeChangeSetCommandOutput, ExecuteChangeSetCommandInput, UpdateStackCommandInput, Tag, @@ -27,6 +28,7 @@ import { determineAllowCrossAccountAssetPublishing } from './checks'; import type { DeployStackResult, SuccessfulDeployStackResult } from './deployment-result'; import type { ChangeSetDeployment, DeploymentMethod, DirectDeployment, ExecuteChangeSetDeployment } from '../../actions/deploy'; import { DEFAULT_DEPLOY_CHANGE_SET_NAME } from '../../actions/deploy/private/deployment-method'; +import type { ReplacedResource } from '../../payloads/deploy'; import { DeploymentError, DeploymentErrorCodes, ToolkitError } from '../../toolkit/toolkit-error'; import type { StabilizingResource } from '../../toolkit/types'; import { formatErrorMessage } from '../../util'; @@ -43,6 +45,7 @@ import { HotswapPropertyOverrides, ICON, createHotswapPropertyOverrides } from ' import { tryHotswapDeployment } from '../hotswap/hotswap-deployments'; import { invalidateHotswapTemplateCache, readHotswapTemplateCache } from '../hotswap/hotswap-template-cache'; import type { IoHelper } from '../io/private'; +import { IO } from '../io/private'; import type { ResourcesToImport } from '../resource-import'; import { StackActivityMonitor } from '../stack-events'; import type { ResourceErrors } from '../stack-events/resource-errors'; @@ -522,7 +525,7 @@ class FullCloudFormationDeployment { } // If there are replacements in the changeset, check the rollback flag and stack status - return this.checkAndExecuteChangeSet(changeSetReport); + return this.checkAndExecuteChangeSet(changeSetReport, { preExistingChangeSet: false }); } private async executeExistingChangeSet(deploymentMethod: ExecuteChangeSetDeployment): Promise { @@ -538,45 +541,192 @@ class FullCloudFormationDeployment { changeSetNameOrArn: deploymentMethod.changeSetName, }).describeForExecution({ diagnoser: this.diagnoser }); - return this.checkAndExecuteChangeSet(changeSetReport); + return this.checkAndExecuteChangeSet(changeSetReport, { preExistingChangeSet: true }); + } + + private replacementRecovery(): ReplacementRecovery { + if (!this.cloudFormationStack.stackStatus.isRollbackable) { + return 'none'; + } + switch (this.cloudFormationStack.stackStatus.name) { + case 'UPDATE_FAILED': + return 'replay'; + case 'CREATE_FAILED': + return 'recreate'; + default: + return 'resolve-state'; + } + } + + private rollbackDisabled(): boolean { + return this.options.express ? this.options.rollback !== true : this.options.rollback === false; } private deployConfig(): DeploymentConfig { + if (!this.options.express) { + return { Mode: 'STANDARD' }; + } + return { - Mode: this.options.express ? 'EXPRESS' : 'STANDARD', - ...(this.options.express && this.options.rollback == true ? { DisableRollback: false } : undefined), + Mode: 'EXPRESS', + ...(this.rollbackDisabled() ? undefined : { DisableRollback: false }), }; } - /** - * Check rollback/replacement constraints and execute the change set if all checks pass. - */ - private async checkAndExecuteChangeSet(changeSetReport: ChangeSetReport): Promise { - const replacement = hasReplacement(changeSetReport); + private async findAllReplacements(changeSet: DescribeChangeSetCommandOutput): Promise { + const visited = new Set(); + const uninspected: string[] = []; + + const collect = async (current: DescribeChangeSetCommandOutput, depth: number): Promise => { + const replacements = findReplacements(current); + const nestedStackChanges = (current.Changes ?? []) + .map((change) => change.ResourceChange) + .filter((nested) => nested?.ResourceType === 'AWS::CloudFormation::Stack') + .filter((nested) => nested!.Action !== 'Remove'); + + if (nestedStackChanges.length > 0 && depth >= MAX_NESTED_CHANGE_SET_DEPTH) { + uninspected.push(format( + 'nested stacks below depth %d (%s) were not inspected', + depth, + nestedStackChanges.map((nested) => nested!.LogicalResourceId ?? '').join(', '), + )); + return replacements; + } + + for (const nested of nestedStackChanges) { + const logicalId = nested!.LogicalResourceId ?? ''; + + if (!nested!.ChangeSetId) { + uninspected.push(format('nested stack %s reported no change set to inspect', logicalId)); + continue; + } + + if (visited.has(nested!.ChangeSetId)) { + continue; + } + visited.add(nested!.ChangeSetId); + + let child: DescribeChangeSetCommandOutput; + try { + child = await new ChangeSetDescriber({ + cfn: this.cfn, + ioHelper: this.ioHelper, + stackNameOrArn: nested!.PhysicalResourceId ?? logicalId, + changeSetNameOrArn: nested!.ChangeSetId, + }).waitForSettled(); + } catch (e: any) { + uninspected.push(format('nested stack %s could not be described (%s)', logicalId, formatErrorMessage(e))); + continue; + } + + if (child.Status !== 'CREATE_COMPLETE') { + uninspected.push(format( + 'nested stack %s has change set status %s, so its changes could not be read', + logicalId, + child.Status ?? '', + )); + continue; + } + + replacements.push(...await collect(child, depth + 1)); + } + + return replacements; + }; + + return { replacements: await collect(changeSet, 0), uninspected }; + } + + private rollbackWillBeDisabled(changeSet: DescribeChangeSetCommandOutput, persistedRollbackDisabled: boolean | undefined): boolean { + if (persistedRollbackDisabled !== undefined) { + return persistedRollbackDisabled; + } + if (changeSet.DeploymentConfig === undefined && this.options.express) { + return this.rollbackDisabled(); + } + return this.options.rollback === false; + } + + private async checkAndExecuteChangeSet( + changeSetReport: ChangeSetReport, + opts: { preExistingChangeSet: boolean }, + ): Promise { + const changeSet = changeSetReport.changeSet; + + const persistedMode = changeSet.DeploymentConfig?.Mode; + const isExpress = persistedMode !== undefined ? persistedMode === 'EXPRESS' : (this.options.express ?? false); + const persistedRollbackDisabled = expressRollbackDisabled(changeSet.DeploymentConfig); + const requestedRollbackDisabled = this.rollbackDisabled(); + + const scan = await this.findAllReplacements(changeSet); + const replacements = scan.replacements; const isPausedFailState = this.cloudFormationStack.stackStatus.isRollbackable; const rollback = this.options.rollback ?? true; - // For express mode deployments, don't check paused and failed, since express mode stacks cannot use rollback API - if (!this.options.express) { - if (isPausedFailState && replacement) { + if (!isExpress) { + if (isPausedFailState && replacements.length > 0) { return { type: 'failpaused-need-rollback-first', reason: 'replacement', status: this.cloudFormationStack.stackStatus.name }; } if (isPausedFailState && rollback) { return { type: 'failpaused-need-rollback-first', reason: 'not-norollback', status: this.cloudFormationStack.stackStatus.name }; } - if (!rollback && replacement) { - return { type: 'replacement-requires-rollback' }; + } + + const rollbackWillBeDisabled = this.rollbackWillBeDisabled(changeSet, persistedRollbackDisabled); + + if (replacements.length > 0 && rollbackWillBeDisabled) { + if (isExpress) { + const guidance = replacementRoutingMessage({ + rejected: false, + recovery: this.replacementRecovery(), + status: this.cloudFormationStack.stackStatus.name, + }); + + await this.ioHelper.notify(IO.CDK_TOOLKIT_W5903.msg(guidance, { + stackName: this.stackName, + changeSetId: changeSet.ChangeSetId, + replacements, + detectedBy: 'change-set', + })); + + if (opts.preExistingChangeSet && persistedRollbackDisabled === true) { + throw new ToolkitError( + 'ReplacementRequiresRecreateChangeSet', + changeSetRecreateForReplacementMessage(changeSet.ChangeSetName), + ); + } + + if (isPausedFailState) { + throw new ToolkitError('ReplacementRequiresUnwedge', guidance); + } } + return { type: 'replacement-requires-rollback' }; + } + + if (rollbackWillBeDisabled && scan.uninspected.length > 0) { + throw new ToolkitError( + 'NestedChangeSetInspectionIncomplete', + nestedInspectionIncompleteMessage(scan.uninspected), + ); + } + + if (persistedRollbackDisabled !== undefined && persistedRollbackDisabled !== requestedRollbackDisabled) { + await this.ioHelper.defaults.warn( + changeSetPolicyMismatchMessage(changeSet.ChangeSetName, persistedRollbackDisabled), + ); } - const changeSet = changeSetReport.changeSet; await this.ioHelper.defaults.debug(format('Initiating execution of changeset %s on stack %s', changeSet.ChangeSetId, this.stackName)); + const { DisableRollback, ...sharedExecuteOptions } = this.commonExecuteOptions(); + const rollbackFlagStillDecides = persistedRollbackDisabled === undefined; + await this.cfn.executeChangeSet({ StackName: changeSet.StackId ?? this.stackName, ChangeSetName: changeSet.ChangeSetId!, ClientRequestToken: `exec${this.uuid}`, - ...this.commonExecuteOptions(), + ...sharedExecuteOptions, + ...(rollbackFlagStillDecides && DisableRollback !== undefined ? { DisableRollback } : undefined), }); await this.ioHelper.defaults.debug( @@ -726,6 +876,18 @@ class FullCloudFormationDeployment { await monitor.start(); let finalState: CloudFormationStack; + let monitorStopped = false; + + // `monitor.stop()` performs a final poll, and that poll is what fills `monitor.errors` with the resource-level + // failures CloudFormation reported. Everything that reads those errors has to run after it. `stop()` is not + // idempotent (it emits a completion message and polls again), so it must run exactly once. + const stopMonitor = async () => { + if (!monitorStopped) { + monitorStopped = true; + await monitor.stop(); + } + }; + try { const successStack = await waitForStackDeploy(this.cfn, this.ioHelper, stackArn, this.options.stackEventPollingInterval); @@ -735,6 +897,10 @@ class FullCloudFormationDeployment { } finalState = successStack; } catch (e: any) { + await stopMonitor(); + + await this.routeReplacementRejectedWithRollbackDisabled(e, monitor.errors); + // Deployment errors get replaced by a diagnosis of the underlying resource failures, which says more. // Any other error, and any failure to diagnose, leaves `e` to propagate as it is. if (ToolkitError.isDeploymentError(e)) { @@ -743,7 +909,7 @@ class FullCloudFormationDeployment { throw e; } finally { - await monitor.stop(); + await stopMonitor(); } await this.ioHelper.defaults.debug(format('Stack %s has completed updating', this.stackName)); return { @@ -756,6 +922,42 @@ class FullCloudFormationDeployment { }; } + private async routeReplacementRejectedWithRollbackDisabled(error: any, errors: ResourceErrors): Promise { + if (!this.options.express || !this.rollbackDisabled()) { + return; + } + + const rejected = errors.all.filter((e) => mentionsReplacementRejection(e.message)); + const matched = rejected.length > 0 || mentionsReplacementRejection(error?.message ?? ''); + + if (!matched) { + const reported = errors.allErrorMessages.filter((m) => m.trim() !== ''); + await this.ioHelper.defaults.debug(format( + 'Deployment failed with rollback disabled but no reported error mentioned %j, so no replacement guidance was emitted. Reported reasons: %s', + CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON, + reported.length > 0 ? reported.join(' | ') : '(none)', + )); + return; + } + + await this.ioHelper.notify(IO.CDK_TOOLKIT_W5903.msg( + replacementRoutingMessage({ + rejected: true, + recovery: this.update ? 'replay' : 'recreate', + }), + { + stackName: this.stackName, + replacements: rejected + .filter((e) => e.logicalId !== undefined) + .map((e) => ({ + logicalId: e.logicalId, + resourceType: e.resourceType, + })), + detectedBy: 'service-error', + }, + )); + } + /** * Throw a `DeploymentError` describing why the deployment failed, if we can establish that * @@ -776,7 +978,7 @@ class FullCloudFormationDeployment { } const diagnosis = await this.diagnoser.diagnoseFromErrorCollection(errors, deployedState, true, { - rollbackEnabled: this.options.rollback !== false, + rollbackEnabled: !this.rollbackDisabled(), }); diagnosis.throwOnError(); } @@ -803,11 +1005,11 @@ class FullCloudFormationDeployment { * deployed everywhere yet. */ private commonExecuteOptions(): Partial> { - const shouldDisableRollback = this.options.rollback === false; + const shouldSendDisableRollbackFlag = this.options.rollback === false; return { StackName: this.stackName, - ...(shouldDisableRollback ? { DisableRollback: true } : undefined), + ...(shouldSendDisableRollbackFlag ? { DisableRollback: true } : undefined), }; } } @@ -1015,9 +1217,159 @@ function arrayEquals(a: any[], b: any[]): boolean { return a.every((item) => b.includes(item)) && b.every((item) => a.includes(item)); } -function hasReplacement(report: ChangeSetReport) { - return (report.changeSet.Changes ?? []).some(c => { - const a = c.ResourceChange?.PolicyAction; - return a === 'ReplaceAndDelete' || a === 'ReplaceAndRetain' || a === 'ReplaceAndSnapshot'; +/** + * Find the resource changes in a change set that CloudFormation would perform by replacement + */ +function findReplacements(changeSet: DescribeChangeSetCommandOutput): ReplacedResource[] { + return (changeSet.Changes ?? []).flatMap((c) => { + const change = c.ResourceChange; + const policyAction = change?.PolicyAction; + const replacesResource = policyAction === 'ReplaceAndDelete' + || policyAction === 'ReplaceAndRetain' + || policyAction === 'ReplaceAndSnapshot'; + + if (!change || !replacesResource) { + return []; + } + + return [{ + logicalId: change.LogicalResourceId, + resourceType: change.ResourceType, + replacement: change.Replacement, + policyAction, + }]; }); } + +/** + * The reason CloudFormation reports when it refuses a replacement because rollback is disabled. + */ +export const CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON = 'Replacement type updates not supported on stack with disable-rollback'; + +function mentionsReplacementRejection(message: string): boolean { + return message.toLowerCase().includes(CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON.toLowerCase()); +} + +/** + * Whether a persisted change set `DeploymentConfig` pins the rollback choice, and if so which way. + */ +function expressRollbackDisabled(config: DeploymentConfig | undefined): boolean | undefined { + if (config?.Mode !== 'EXPRESS') { + return undefined; + } + return config.DisableRollback !== false; +} + +/** + * Explain that an existing change set's rollback policy cannot be changed by executing it differently + */ +function changeSetPolicyMismatchMessage(changeSetName: string | undefined, persistedRollbackDisabled: boolean): string { + const named = changeSetName ? ` ${chalk.blue(changeSetName)}` : ''; + const persisted = persistedRollbackDisabled ? 'disabled' : 'enabled'; + const requested = persistedRollbackDisabled ? 'enabled' : 'disabled'; + const recreateWith = chalk.blue(`cdk deploy --express${persistedRollbackDisabled ? ' --rollback' : ''}`); + + return [ + `Change set${named} was created with rollback ${persisted}, but this deployment asks for rollback ${requested}.`, + 'CloudFormation fixes that choice when the change set is created and executing it cannot change it, so this', + 'deployment would silently do the opposite of what you asked for.', + '', + `Create a new change set with the flags you want rather than executing this one: ${recreateWith}`, + ].join('\n'); +} + +type ReplacementRecovery = 'none' | 'replay' | 'recreate' | 'resolve-state'; + +/** + * The outcome of scanning a change set hierarchy for replacements. + * + * `uninspected` is non-empty when part of the hierarchy could not be read, in which case an empty `replacements` + * does NOT mean there are none. + */ +interface ReplacementScan { + readonly replacements: ReplacedResource[]; + readonly uninspected: string[]; +} + +function nestedInspectionIncompleteMessage(uninspected: string[]): string { + const withRollback = chalk.blue('cdk deploy --express --rollback'); + + return [ + 'Rollback is disabled for this deployment, so a replacement would be rejected mid-execution and leave the stack', + 'in UPDATE_FAILED. Part of the nested stack hierarchy could not be inspected, so it cannot be confirmed that this', + 'deployment contains no replacement:', + ...uninspected.map((reason) => ` - ${reason}`), + '', + `Deploy with rollback enabled instead, which allows replacements: ${withRollback}`, + ].join('\n'); +} + +const MAX_NESTED_CHANGE_SET_DEPTH = 10; + +/** + * Explain that a replacement cannot be deployed by executing this change set, whatever flags are passed. + */ +function changeSetRecreateForReplacementMessage(changeSetName: string | undefined): string { + const named = changeSetName ? ` ${chalk.blue(changeSetName)}` : ''; + const recreateWith = chalk.blue('cdk deploy --express --rollback'); + + return [ + `Change set${named} was created with rollback disabled and replaces a resource, which CloudFormation does not`, + 'support. A change set fixes its rollback policy when it is created and executing it cannot change that, so there', + 'is no way to execute this change set successfully.', + '', + `Create a new change set with rollback enabled instead: ${recreateWith}`, + ].join('\n'); +} + +/** + * Explain how to deploy a replacement when rollback is disabled. + */ +function replacementRoutingMessage(opts: { rejected: boolean; recovery: ReplacementRecovery; status?: string }): string { + const withRollback = chalk.blue('cdk deploy --express --rollback'); + const direct = chalk.blue('cdk deploy --express --method=direct'); + + const headline = opts.rejected + ? [ + 'CloudFormation refused a replacement because rollback is disabled for this stack.', + 'Express Mode disables rollback by default; replacements themselves are supported.', + ] + : [ + 'This deployment replaces a resource, which CloudFormation does not support while rollback is disabled.', + 'Express Mode disables rollback unless you ask for it with --rollback; replacements themselves are supported.', + ]; + + switch (opts.recovery) { + case 'none': + return headline.join('\n'); + + case 'replay': + return [ + ...headline, + '', + `${opts.rejected ? 'The stack may now be' : 'This stack is'} in a failed state, which ${withRollback} cannot update. To recover:`, + ' 1. Revert your change so your app matches the last configuration that deployed successfully.', + ` 2. Run ${direct} - this should replay that configuration as a no-op`, + ' and return the stack to a terminal state.', + ` 3. Re-apply your change and deploy it with ${withRollback}.`, + ].join('\n'); + + case 'recreate': + return [ + ...headline, + '', + `This stack never completed a deployment, so there is no previous configuration to replay and ${withRollback}`, + 'cannot update it either. Delete the stack and deploy again.', + ].join('\n'); + + case 'resolve-state': + return [ + ...headline, + '', + `This stack is in ${opts.status ?? 'a failed state'}, which ${withRollback} cannot update, and it is not a state`, + 'this command can recover from. Resolve it in CloudFormation first, then deploy the replacement with rollback', + 'enabled.', + ].join('\n'); + } +} + diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts b/packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts index 6001cbca9..e7a41179d 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts @@ -6,7 +6,7 @@ import type { ValidateResult } from '../../../actions/validate'; import type { StackDiff, DiffResult } from '../../../payloads'; import type { BootstrapEnvironmentProgress } from '../../../payloads/bootstrap-environment-progress'; import type { MissingContext, UpdatedContext } from '../../../payloads/context'; -import type { BuildAsset, DeployConfirmationRequest, PublishAsset, PublishAssetEvent, StackDeployProgress, SuccessfulDeployStackResult } from '../../../payloads/deploy'; +import type { BuildAsset, DeployConfirmationRequest, PublishAsset, PublishAssetEvent, ReplacementRequiresRollback, StackDeployProgress, SuccessfulDeployStackResult } from '../../../payloads/deploy'; import type { StackDestroy, StackDestroyProgress } from '../../../payloads/destroy'; import type { DriftResultPayload } from '../../../payloads/drift'; import type { FeatureFlagChangeRequest } from '../../../payloads/flags'; @@ -334,6 +334,11 @@ export const IO = { code: 'CDK_TOOLKIT_W5902', description: 'Express Mode deployment completed with resources still stabilizing', }), + CDK_TOOLKIT_W5903: make.warn({ + code: 'CDK_TOOLKIT_W5903', + description: 'Deployment includes a replacement that CloudFormation does not support while rollback is disabled', + interface: 'ReplacementRequiresRollback', + }), CDK_TOOLKIT_W5400: make.warn({ code: 'CDK_TOOLKIT_W5400', description: 'Hotswap disclosure message', diff --git a/packages/@aws-cdk/toolkit-lib/lib/payloads/deploy.ts b/packages/@aws-cdk/toolkit-lib/lib/payloads/deploy.ts index a63eeb5dc..8298dfff4 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/payloads/deploy.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/payloads/deploy.ts @@ -75,3 +75,60 @@ export interface PublishAssetEvent { */ readonly asset?: IManifestEntry; } + +/** + * A resource that CloudFormation reported it would replace + */ +export interface ReplacedResource { + /** + * Logical ID of the resource being replaced + * + * Absent when CloudFormation reported the change or failure without naming a resource. + */ + readonly logicalId?: string; + + /** + * CloudFormation resource type, when known + */ + readonly resourceType?: string; + + /** + * The `Replacement` field CloudFormation reported for this change, when known + */ + readonly replacement?: string; + + /** + * The `PolicyAction` CloudFormation reported for this change, when known + */ + readonly policyAction?: string; +} + +/** + * A deployment includes a replacement that CloudFormation will not perform while rollback is disabled + */ +export interface ReplacementRequiresRollback { + /** + * The stack being deployed + */ + readonly stackName: string; + + /** + * The change set the replacement was found in, if it was found in one + */ + readonly changeSetId?: string; + + /** + * The resources that would be replaced + * + * Empty when CloudFormation reported the rejection without naming a resource. + */ + readonly replacements: ReplacedResource[]; + + /** + * How the replacement was detected + * + * `change-set` means the pre-flight check found it and nothing was submitted. `service-error` means we only found + * out from CloudFormation's failure, after the update had already been submitted. + */ + readonly detectedBy: 'change-set' | 'service-error'; +} diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts index 93c154586..04297806e 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts @@ -1032,7 +1032,9 @@ export class Toolkit extends CloudAssemblySourceBuilder { } case 'replacement-requires-rollback': { - const motivation = 'Change includes a replacement which cannot be deployed with "--no-rollback"'; + const motivation = options.express + ? 'Change includes a replacement, which CloudFormation does not support while rollback is disabled (the default for Express Mode)' + : 'Change includes a replacement which cannot be deployed with "--no-rollback"'; const question = `${motivation}. Perform a deployment with rollback enabled`; const confirmed = await ioHelper.requestResponse(IO.CDK_TOOLKIT_I5050.req(question, { diff --git a/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.md b/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.md index 46bef874c..92a926975 100644 --- a/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.md +++ b/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.md @@ -68,9 +68,19 @@ Tests that use the fake should use fake timers to advance time. 3. API returns `{ StackId }` immediately. 4. Stack status: `UPDATE_IN_PROGRESS`. 5. After delay: - - If any resource has `Fail: true` → `UPDATE_FAILED` (if - `DisableRollback`) or `UPDATE_ROLLBACK_IN_PROGRESS` → + - If any resource has `Fail: true` → `UPDATE_FAILED` (if rollback is + disabled) or `UPDATE_ROLLBACK_IN_PROGRESS` → `UPDATE_ROLLBACK_COMPLETE`. + - Rollback counts as disabled when `DisableRollback: true` is passed, **or** + when `DeploymentConfig.Mode === 'EXPRESS'` and the call did not explicitly + re-enable it with `DeploymentConfig.DisableRollback: false`. This mirrors + Express Mode having rollback disabled server-side by default, and is what + makes a failed express update strand the stack in `UPDATE_FAILED`. + - Every failing resource that also has `FailReason: '...'` emits + resource-level `UPDATE_IN_PROGRESS` and `UPDATE_FAILED` events, with that + string as the `ResourceStatusReason` (this is where real CloudFormation + reports why an operation failed). Without `FailReason`, only stack-level + events are emitted. `ExecuteChangeSet` failures do the same. - Otherwise → `UPDATE_COMPLETE`. ### DeleteStack diff --git a/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.ts b/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.ts index 3a208bd4d..be22e5484 100644 --- a/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.ts +++ b/packages/@aws-cdk/toolkit-lib/test/_helpers/fake-aws/fake-cloudformation.ts @@ -10,6 +10,7 @@ import { type DeleteChangeSetCommandOutput, type DeleteStackCommandInput, type DeleteStackCommandOutput, + type DeploymentConfig, type DescribeChangeSetCommandInput, type DescribeChangeSetCommandOutput, type DescribeEventsCommandInput, @@ -119,6 +120,7 @@ interface InMemoryChangeSet { capabilities: string[]; description?: string; changes: Change[]; + deploymentConfig?: DeploymentConfig; creationTime: Date; changeSetFailureEvents: OperationEvent[]; earlyValidationErrors: EarlyValidationErrorPrime[]; @@ -323,7 +325,7 @@ export class FakeCloudFormation { const operationId = randomUUID(); const { id, stack, template } = this.initCreateStack(input, operationId); this.scheduleAsync(() => { - this.finalizeCreateStack(stack, template, input.DisableRollback, operationId); + this.finalizeCreateStack(stack, template, this.rollbackIsDisabled(input), operationId); }); return { StackId: id, $metadata: {} }; } @@ -347,7 +349,8 @@ export class FakeCloudFormation { this.scheduleAsync(() => { if (this.shouldFail(template)) { - if (input.DisableRollback) { + this.addFailedUpdateResourceEvents(stack, template, operationId); + if (this.rollbackIsDisabled(input)) { this.transitionStack(stack, 'UPDATE_FAILED', 'Resource update failed', operationId); } else { this.transitionStack(stack, 'UPDATE_ROLLBACK_IN_PROGRESS', 'Resource update failed', operationId); @@ -435,6 +438,7 @@ export class FakeCloudFormation { Tags?: Tag[]; Capabilities?: string[]; Description?: string; + DeploymentConfig?: DeploymentConfig; }): CreateChangeSetCommandOutput { const stackName = input.StackName; const stack = this.requireStack(stackName); @@ -470,6 +474,7 @@ export class FakeCloudFormation { capabilities: input.Capabilities ?? [], description: input.Description, changes: changes ?? [], + deploymentConfig: input.DeploymentConfig, creationTime: new Date(), changeSetFailureEvents: [], earlyValidationErrors: [], @@ -511,6 +516,7 @@ export class FakeCloudFormation { Capabilities: cs.capabilities as any, Description: cs.description, CreationTime: cs.creationTime, + ...(cs.deploymentConfig ? { DeploymentConfig: cs.deploymentConfig } : undefined), NextToken: nextToken, $metadata: {}, }; @@ -523,6 +529,20 @@ export class FakeCloudFormation { cfnError('InvalidChangeSetStatus', `ChangeSet [${cs.name}] is in ${cs.executionStatus} state and cannot be executed`); } + const persistedRollbackDisabled = cs.deploymentConfig?.Mode === 'EXPRESS' + ? cs.deploymentConfig.DisableRollback !== false + : undefined; + if ( + input.DisableRollback !== undefined && + persistedRollbackDisabled !== undefined && + input.DisableRollback !== persistedRollbackDisabled + ) { + cfnError( + 'ValidationError', + 'DisableRollback specified on ExecuteChangeSet conflicts with the value DisableRollback the ChangeSet was created with.', + ); + } + // Remove the executed change set from the stack's list. Real CloudFormation // also deletes all other change sets, but we skip that to avoid interfering // with concurrent operations on the same stack in tests. @@ -550,7 +570,8 @@ export class FakeCloudFormation { if (this.shouldFail(cs.template)) { const failedStatus = isCreate ? 'CREATE_FAILED' : 'UPDATE_FAILED'; - if (input.DisableRollback) { + this.addFailedUpdateResourceEvents(stack, cs.template, operationId, isCreate ? 'CREATE' : 'UPDATE'); + if (this.rollbackIsDisabled({ ...input, DeploymentConfig: cs.deploymentConfig })) { this.transitionStack(stack, failedStatus, 'Resource operation failed', operationId); } else { const rollbackStatus = isCreate ? 'ROLLBACK_IN_PROGRESS' : 'UPDATE_ROLLBACK_IN_PROGRESS'; @@ -865,6 +886,7 @@ export class FakeCloudFormation { capabilities: (input.Capabilities as string[]) ?? [], description: input.Description, changes: [], + deploymentConfig: input.DeploymentConfig, creationTime: new Date(), changeSetFailureEvents: [], earlyValidationErrors: [], @@ -1153,7 +1175,17 @@ export class FakeCloudFormation { }); } - private addResourceEvent(stack: InMemoryStack, logicalId: string, resourceType: string, status: string, operationId?: string) { + private rollbackIsDisabled(input: { DisableRollback?: boolean; DeploymentConfig?: DeploymentConfig }): boolean { + if (input.DisableRollback) { + return true; + } + if (input.DeploymentConfig?.Mode === 'EXPRESS') { + return input.DeploymentConfig.DisableRollback !== false; + } + return false; + } + + private addResourceEvent(stack: InMemoryStack, logicalId: string, resourceType: string, status: string, operationId?: string, reason?: string) { stack.events.unshift({ StackId: stack.id, StackName: stack.name, @@ -1162,11 +1194,26 @@ export class FakeCloudFormation { PhysicalResourceId: `fake-${logicalId}-${uid()}`, ResourceType: resourceType, ResourceStatus: status as any, + ResourceStatusReason: reason, OperationId: operationId, Timestamp: new Date(), }); } + private addFailedUpdateResourceEvents(stack: InMemoryStack, template: Record, operationId?: string, verb: 'UPDATE' | 'CREATE' = 'UPDATE') { + for (const [logicalId, res] of Object.entries(templateResources(template))) { + const r = res as any; + const reason = r.Properties?.FailReason; + if (reason === undefined) { + continue; + } + if (this.alwaysFailResources || r.Properties?.Fail === true) { + this.addResourceEvent(stack, logicalId, r.Type, `${verb}_IN_PROGRESS`, operationId); + this.addResourceEvent(stack, logicalId, r.Type, `${verb}_FAILED`, operationId, reason); + } + } + } + private toStackDescription(stack: InMemoryStack): Stack { return { StackName: stack.name, diff --git a/packages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts b/packages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts index d52d4898f..cc49a9723 100644 --- a/packages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts +++ b/packages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts @@ -68,14 +68,19 @@ export class TestIoHost implements IIoHost { return spyResponse ?? msg.defaultResponse; } - public expectMessage(m: { containing: string; level?: IoMessageLevel }) { + public expectMessage(m: { containing: string; level?: IoMessageLevel; code?: IoMessageCode }) { expect(this.messages).toContainEqual(expect.objectContaining({ ...m.level ? { level: m.level } : undefined, + ...m.code ? { code: m.code } : undefined, // Can be a partial string as well message: expect.stringContaining(m.containing), })); } + public messagesWithCode(code: IoMessageCode): Array> { + return this.messages.filter((m) => m.code === code); + } + /** * Mocks the response for a given message code. * diff --git a/packages/@aws-cdk/toolkit-lib/test/actions/deploy.test.ts b/packages/@aws-cdk/toolkit-lib/test/actions/deploy.test.ts index e601943b9..1ce555aee 100644 --- a/packages/@aws-cdk/toolkit-lib/test/actions/deploy.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/actions/deploy.test.ts @@ -620,6 +620,68 @@ IAM Statement Changes // THEN successfulDeployment(); }); + + test('replacement-requires-rollback under --express explains that rollback is disabled, and retries with it enabled', async () => { + // GIVEN + mockDeployStack.mockImplementation(async (params) => { + if (params.rollback === true) { + return { + type: 'did-deploy-stack', + stackArn: 'arn:aws:cloudformation:region:account:stack/test-stack', + outputs: {}, + noOp: false, + deleteFailures: [], + stabilizingResources: [], + } satisfies DeployStackResult; + } + return { type: 'replacement-requires-rollback' } satisfies DeployStackResult; + }); + + // WHEN + const cx = await cdkOutFixture(toolkit, 'stack-with-role'); + await toolkit.deploy(cx, { express: true }); + + // THEN + expect(ioHost.requestSpy).toHaveBeenCalledWith(expect.objectContaining({ + code: 'CDK_TOOLKIT_I5050', + data: expect.objectContaining({ + motivation: 'Change includes a replacement, which CloudFormation does not support while rollback is disabled (the default for Express Mode)', + }), + })); + + expect(mockDeployStack).toHaveBeenCalledWith(expect.objectContaining({ express: true, rollback: true })); + successfulDeployment(); + }); + + test('replacement-requires-rollback without --express keeps the --no-rollback wording', async () => { + // GIVEN + mockDeployStack.mockImplementation(async (params) => { + if (params.rollback === true) { + return { + type: 'did-deploy-stack', + stackArn: 'arn:aws:cloudformation:region:account:stack/test-stack', + outputs: {}, + noOp: false, + deleteFailures: [], + stabilizingResources: [], + } satisfies DeployStackResult; + } + return { type: 'replacement-requires-rollback' } satisfies DeployStackResult; + }); + + // WHEN + const cx = await cdkOutFixture(toolkit, 'stack-with-role'); + await toolkit.deploy(cx, { rollback: false }); + + // THEN + expect(ioHost.requestSpy).toHaveBeenCalledWith(expect.objectContaining({ + code: 'CDK_TOOLKIT_I5050', + data: expect.objectContaining({ + motivation: 'Change includes a replacement which cannot be deployed with "--no-rollback"', + }), + })); + successfulDeployment(); + }); }); test('deploy returns stack information', async () => { diff --git a/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack-express-replacement.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack-express-replacement.test.ts new file mode 100644 index 000000000..54c4508aa --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack-express-replacement.test.ts @@ -0,0 +1,1315 @@ +import type { CloudFormationStackArtifact } from '@aws-cdk/cloud-assembly-api'; +import type { Change, CreateChangeSetCommandInput, DeploymentConfig, Stack } from '@aws-sdk/client-cloudformation'; +import { + CreateChangeSetCommand, + DescribeChangeSetCommand, + ExecuteChangeSetCommand, + StackStatus, + UpdateStackCommand, +} from '@aws-sdk/client-cloudformation'; +import type { DeployStackOptions as DeployStackApiOptions } from '../../../lib/api/deployments/deploy-stack'; +import { CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON, deployStack } from '../../../lib/api/deployments/deploy-stack'; +import { CloudFormationStackDiagnoser } from '../../../lib/api/diagnosing/stack-diagnoser'; +import { NoBootstrapStackEnvironmentResources } from '../../../lib/api/environment'; +import { IO } from '../../../lib/api/io/private'; +import { StackArtifactSourceTracer } from '../../../lib/api/source-tracing/private/stack-source-tracing'; +import { testStack } from '../../_helpers/assembly'; +import { FakeCloudFormation } from '../../_helpers/fake-aws/fake-cloudformation'; +import { advanceTime } from '../../_helpers/fake-time'; +import { + mockCloudFormationClient, + mockResolvedEnvironment, + MockSdk, + MockSdkProvider, + restoreSdkMocksToDefault, +} from '../../_helpers/mock-sdk'; +import { TestIoHost } from '../../_helpers/test-io-host'; + +const W5903 = IO.CDK_TOOLKIT_W5903.code; + +let ioHost = new TestIoHost('debug', true); +let ioHelper = ioHost.asHelper('deploy'); + +function testDeployStack(options: DeployStackApiOptions) { + return advanceTime(deployStack(options, ioHelper)); +} + +jest.mock('../../../lib/api/deployments/checks', () => ({ + determineAllowCrossAccountAssetPublishing: jest.fn().mockResolvedValue(true), +})); + +function startTemplate() { + return { + Description: 'Start template in deploy-stack-express-replacement.test.ts', + Resources: { + MyResource: { + Type: 'Test::Resource::Type', + Properties: { Foo: 'Foo' }, + }, + }, + }; +} + +function targetTemplate() { + return { + Description: 'Start template in deploy-stack-express-replacement.test.ts', + Resources: { + MyResource: { + Type: 'Test::Resource::Type', + Properties: { Bar: 'Bar' }, + }, + }, + }; +} + +function templateRejectingReplacement() { + return { + Resources: { + MyResource: { + Type: 'Test::Resource::Type', + Properties: { + Bar: 'Bar', + Fail: true, + FailReason: `${CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON}.`, + }, + }, + }, + }; +} + +const FAKE_STACK = testStack({ + stackName: 'withouterrors', + template: targetTemplate(), +}); + +const FAKE_STACK_REJECTING_REPLACEMENT = testStack({ + stackName: 'withouterrors', + template: templateRejectingReplacement(), +}); + +const baseResponse = { + StackName: 'mock-stack-name', + StackId: 'mock-stack-id', + CreationTime: new Date(), + StackStatus: StackStatus.CREATE_COMPLETE, + EnableTerminationProtection: false, +}; + +let sdk: MockSdk; +let sdkProvider: MockSdkProvider; +const fakeCfn = new FakeCloudFormation(); + +beforeEach(() => { + fakeCfn.reset(); + + ioHost = new TestIoHost('debug', true); + ioHelper = ioHost.asHelper('deploy'); + + sdkProvider = new MockSdkProvider(); + sdk = new MockSdk(); + sdk.getUrlSuffix = () => Promise.resolve('amazonaws.com'); + jest.resetAllMocks(); + + restoreSdkMocksToDefault(); + fakeCfn.installUsingAwsMock(mockCloudFormationClient); + + jest.useFakeTimers(); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +function standardDeployStackArguments(stack: CloudFormationStackArtifact = FAKE_STACK): DeployStackApiOptions { + const resolvedEnvironment = mockResolvedEnvironment(); + return { + stack, + sdk, + sdkProvider, + resolvedEnvironment, + envResources: new NoBootstrapStackEnvironmentResources(resolvedEnvironment, sdk, ioHelper), + diagnoser: new CloudFormationStackDiagnoser({ + sdk, + sourceTracer: new StackArtifactSourceTracer(stack), + ioHelper, + topLevelStackHierarchicalId: stack.hierarchicalId, + }), + }; +} + +function givenStackExists(overrides: Partial & { StackName?: string } = {}) { + const stackName = overrides.StackName ?? 'withouterrors'; + fakeCfn.createStackSync({ + ...baseResponse, + StackName: stackName, + ...overrides, + }); + fakeCfn.accessStack(stackName).template = startTemplate(); +} + +function policyActionReplacementChange(logicalId = 'TaskDef54694570'): Change { + return { + Type: 'Resource', + ResourceChange: { + PolicyAction: 'ReplaceAndDelete', + Action: 'Modify', + LogicalResourceId: logicalId, + ResourceType: 'AWS::ECS::TaskDefinition', + Replacement: 'True', + Scope: ['Properties'], + Details: [ + { + Target: { Attribute: 'Properties', Name: 'Memory', RequiresRecreation: 'Always' }, + Evaluation: 'Static', + ChangeSource: 'DirectModification', + }, + ], + }, + }; +} + +function updateChange(logicalId = 'Queue4A7E3555'): Change { + return { + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: logicalId, + ResourceType: 'AWS::SQS::Queue', + Replacement: 'False', + }, + }; +} + +function conditionalChange(logicalId = 'CDKMetadata'): Change { + return { + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: logicalId, + ResourceType: 'AWS::CDK::Metadata', + Replacement: 'Conditional', + Scope: ['Properties'], + Details: [ + { + Target: { Attribute: 'Properties', Name: 'Analytics', RequiresRecreation: 'Conditionally' }, + Evaluation: 'Static', + ChangeSource: 'DirectModification', + }, + ], + }, + }; +} + +function replacementOnlyChange(logicalId = 'TaskDef54694570'): Change { + return { + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: logicalId, + ResourceType: 'AWS::ECS::TaskDefinition', + Replacement: 'True', + Scope: ['Properties'], + }, + }; +} + +function failOnAnyStackMutation() { + mockCloudFormationClient.on(ExecuteChangeSetCommand).callsFake(() => { + throw new Error('ExecuteChangeSet must not be called when the replacement guard trips'); + }); + mockCloudFormationClient.on(UpdateStackCommand).callsFake(() => { + throw new Error('UpdateStack must not be called when the replacement guard trips'); + }); +} + +function expectNoStackMutation() { + expect(mockCloudFormationClient).not.toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(mockCloudFormationClient).not.toHaveReceivedCommand(UpdateStackCommand); +} + +function expectUnwedgeBeforeRollbackSuggestion(message: string) { + const state = message.indexOf('in a failed state'); + const step1 = message.indexOf('Revert your change'); + const step2 = message.indexOf('cdk deploy --express --method=direct'); + const suggestion = message.indexOf('Re-apply your change and deploy it with'); + const firstRollbackMention = message.indexOf('cdk deploy --express --rollback'); + + expect(state).toBeGreaterThanOrEqual(0); + expect(firstRollbackMention).toBeGreaterThan(state); + expect(step1).toBeGreaterThan(state); + expect(step2).toBeGreaterThan(step1); + expect(suggestion).toBeGreaterThan(step2); +} + +function expectRecreateGuidance(message: string) { + expect(message).toMatch(/no previous configuration to replay/); + expect(message).toMatch(/Delete the stack and deploy again/); + expect(message).not.toMatch(/Revert your change/); + expect(message).not.toMatch(/Re-apply your change and deploy it with/); + expect(message).not.toMatch(/--method=direct/); +} + +describe('change set path, replacement reported as policy action with Replacement=True', () => { + const replacementChange = policyActionReplacementChange; + test('express with rollback disabled is gated before ExecuteChangeSet', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [replacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('replacement-requires-rollback'); + expectNoStackMutation(); + + ioHost.expectMessage({ + level: 'warn', + code: W5903, + containing: 'does not support while rollback is disabled', + }); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'unless you ask for it with --rollback' }); + + expect(ioHost.messagesWithCode(W5903)[0].message).not.toContain('To recover'); + expect(ioHost.messagesWithCode(W5903)[0].message).not.toContain('cdk deploy --express --rollback'); + + expect(ioHost.messagesWithCode(W5903)[0].data).toEqual(expect.objectContaining({ + stackName: 'withouterrors', + detectedBy: 'change-set', + replacements: [expect.objectContaining({ + logicalId: 'TaskDef54694570', + resourceType: 'AWS::ECS::TaskDefinition', + })], + })); + }); + + test('express with rollback enabled deploys the replacement with DisableRollback: false', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [replacementChange()]; + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + rollback: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(mockCloudFormationClient).toHaveReceivedCommandWith(CreateChangeSetCommand, { + ...expect.anything, + DeploymentConfig: { + Mode: 'EXPRESS', + DisableRollback: false, + }, + } as CreateChangeSetCommandInput); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('non-express --no-rollback is gated, without the express-specific guidance', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [replacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + rollback: false, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('replacement-requires-rollback'); + expectNoStackMutation(); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('non-express paused fail state still requires a rollback first', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_FAILED }); + fakeCfn.overrideChangeSetChanges = [replacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + rollback: false, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('failpaused-need-rollback-first'); + expectNoStackMutation(); + }); + + test('express from an already-failed stack fails terminally instead of offering a doomed retry', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_FAILED }); + fakeCfn.overrideChangeSetChanges = [replacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(/in a failed state/); + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresUnwedge' })); + await expect(deployment).rejects.toThrow(/Revert your change/); + await expect(deployment).rejects.toThrow(/cdk deploy --express --method=direct/); + expectUnwedgeBeforeRollbackSuggestion(await deployment.then(() => '', (e) => e.message)); + expectNoStackMutation(); + + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'Revert your change' }); + }); +}); + +describe('change set path', () => { + test('express with rollback disabled and no replacement deploys unchanged', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [updateChange()]; + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(mockCloudFormationClient).toHaveReceivedCommandWith(CreateChangeSetCommand, { + ...expect.anything, + DeploymentConfig: { Mode: 'EXPRESS' }, + } as CreateChangeSetCommandInput); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a replacement reported without a policy action is not gated up front (#1971)', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [replacementOnlyChange()]; + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a replacement on the second page of DescribeChangeSet results is still gated', async () => { + // GIVEN + fakeCfn.reset({ pageSize: 1 }); + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [updateChange('First'), policyActionReplacementChange('Second')]; + failOnAnyStackMutation(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('replacement-requires-rollback'); + expectNoStackMutation(); + expect(mockCloudFormationClient.commandCalls(DescribeChangeSetCommand).length).toBeGreaterThan(1); + }); + + test('a plain deployment with a replacement deploys unchanged', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [policyActionReplacementChange()]; + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + }); + + test('a replacement the change set did not declare is still routed after CloudFormation rejects it', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [conditionalChange()]; + + // WHEN + await expect(testDeployStack({ + ...standardDeployStackArguments(FAKE_STACK_REJECTING_REPLACEMENT), + express: true, + forceDeployment: true, + })).rejects.toThrow(CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON); + + // THEN + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'CloudFormation refused a replacement' }); + expect(ioHost.messagesWithCode(W5903)[0].data).toEqual(expect.objectContaining({ + detectedBy: 'service-error', + })); + }); +}); + +describe('REGRESSION aws/aws-cdk-cli#1931: the express replacement guard must not be removed or rescoped', () => { + test.each([ + ['rollback not specified (express disables rollback by default)', undefined], + ['rollback explicitly disabled', false], + ] satisfies Array<[string, boolean | undefined]>)('%s', async (_name, rollback) => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [policyActionReplacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + rollback, + forceDeployment: true, + }); + + // THEN + expect(result).toEqual({ type: 'replacement-requires-rollback' }); + expectNoStackMutation(); + expect(fakeCfn.accessStack('withouterrors').status).toEqual(StackStatus.UPDATE_COMPLETE); + }); + + test('rollback: true is the one express case that is allowed through', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + fakeCfn.overrideChangeSetChanges = [policyActionReplacementChange()]; + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + express: true, + rollback: true, + forceDeployment: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + }); +}); + +describe('direct path', () => { + test('a non-express --no-rollback rejection is not routed towards Express Mode', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + + // WHEN + await expect(testDeployStack({ + ...standardDeployStackArguments(FAKE_STACK_REJECTING_REPLACEMENT), + deploymentMethod: { method: 'direct' }, + rollback: false, + forceDeployment: true, + })).rejects.toThrow(CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON); + + // THEN + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a rejected replacement strands the stack, and the user is routed to --express --rollback', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + + // WHEN + await expect(testDeployStack({ + ...standardDeployStackArguments(FAKE_STACK_REJECTING_REPLACEMENT), + deploymentMethod: { method: 'direct' }, + express: true, + forceDeployment: true, + })).rejects.toThrow(CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON); + + // THEN + expect(fakeCfn.accessStack('withouterrors').status).toEqual(StackStatus.UPDATE_FAILED); + + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'CloudFormation refused a replacement' }); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'Revert your change' }); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'cdk deploy --express --method=direct' }); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'cdk deploy --express --rollback' }); + expectUnwedgeBeforeRollbackSuggestion(ioHost.messagesWithCode(W5903)[0].message); + expect(ioHost.messagesWithCode(W5903)[0].data).toEqual(expect.objectContaining({ + stackName: 'withouterrors', + detectedBy: 'service-error', + replacements: [expect.objectContaining({ logicalId: 'MyResource' })], + })); + }); + + test('with rollback enabled the stack rolls back and no guidance is emitted', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + + // WHEN + await expect(testDeployStack({ + ...standardDeployStackArguments(FAKE_STACK_REJECTING_REPLACEMENT), + deploymentMethod: { method: 'direct' }, + express: true, + rollback: true, + forceDeployment: true, + })).rejects.toThrow(CFN_REPLACEMENT_WITH_ROLLBACK_DISABLED_REASON); + + // THEN + expect(fakeCfn.accessStack('withouterrors').status).toEqual(StackStatus.UPDATE_ROLLBACK_COMPLETE); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a no-op replay against a hand-seeded stranded stack is not blocked', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_FAILED }); + fakeCfn.accessStack('withouterrors').template = targetTemplate(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + deploymentMethod: { method: 'direct' }, + express: true, + forceDeployment: true, + }); + + // THEN + expect(result).toEqual(expect.objectContaining({ type: 'did-deploy-stack', noOp: true })); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a failure that does not mention the rejection logs what it did see', async () => { + // GIVEN + const otherFailure = testStack({ + stackName: 'withouterrors', + template: { + Resources: { + MyResource: { + Type: 'Test::Resource::Type', + Properties: { Bar: 'Bar', Fail: true, FailReason: 'Some other service error' }, + }, + }, + }, + }); + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + + // WHEN + await expect(testDeployStack({ + ...standardDeployStackArguments(otherFailure), + deploymentMethod: { method: 'direct' }, + express: true, + forceDeployment: true, + })).rejects.toThrow('Some other service error'); + + // THEN + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + ioHost.expectMessage({ level: 'debug', containing: 'no reported error mentioned' }); + ioHost.expectMessage({ level: 'debug', containing: 'Some other service error' }); + }); +}); + +describe('executing a change set created by an earlier invocation', () => { + function givenExpressChangeSetExists(opts: { rollbackDisabled: boolean; changes: Change[] }) { + fakeCfn.createChangeSetSync({ + StackName: 'withouterrors', + ChangeSetName: 'prepared', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'AVAILABLE', + Changes: opts.changes, + DeploymentConfig: opts.rollbackDisabled + ? { Mode: 'EXPRESS' } + : { Mode: 'EXPRESS', DisableRollback: false }, + }); + } + + const executePrepared: Partial = { + deploymentMethod: { method: 'execute-change-set', changeSetName: 'prepared' }, + forceDeployment: true, + }; + + test('a rollback-disabled change set with no replacement is executed, with the ignored flag reported', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: true, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + rollback: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + ioHost.expectMessage({ level: 'warn', containing: 'created with rollback disabled' }); + }); + + test('a rollback-disabled change set containing a replacement is never executed', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: true, changes: [policyActionReplacementChange()] }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + rollback: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresRecreateChangeSet' })); + await expect(deployment).rejects.toThrow(/Create a new change set with rollback enabled/); + expectNoStackMutation(); + expect(fakeCfn.accessStack('withouterrors').status).toEqual(StackStatus.UPDATE_COMPLETE); + }); + + test('omitting --express does not make a persisted Express change set look safe', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: true, changes: [policyActionReplacementChange()] }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresRecreateChangeSet' })); + expectNoStackMutation(); + }); + + test('the opposite mismatch is reported too: rollback-enabled change set executed as rollback-disabled', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: false, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + ioHost.expectMessage({ level: 'warn', containing: 'created with rollback enabled' }); + }); + + test('a matching rollback-disabled change set with a replacement is terminal, not offered a retry', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: true, changes: [policyActionReplacementChange()] }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresRecreateChangeSet' })); + expectNoStackMutation(); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'does not support while rollback is disabled' }); + }); + + function givenChangeSetExists(opts: { deploymentConfig?: DeploymentConfig; changes: Change[] }) { + fakeCfn.createChangeSetSync({ + StackName: 'withouterrors', + ChangeSetName: 'prepared', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'AVAILABLE', + Changes: opts.changes, + DeploymentConfig: opts.deploymentConfig, + }); + } + + function givenNestedChangeSetExists(opts: { rollbackDisabled: boolean; childChanges: Change[] }) { + const childStackName = 'withouterrors-NestedChild-ABC123'; + const deploymentConfig: DeploymentConfig = opts.rollbackDisabled + ? { Mode: 'EXPRESS' } + : { Mode: 'EXPRESS', DisableRollback: false }; + + fakeCfn.createStackSync({ StackName: childStackName, StackStatus: StackStatus.UPDATE_COMPLETE }); + const child = fakeCfn.createChangeSetSync({ + StackName: childStackName, + ChangeSetName: 'prepared-nested-child', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'UNAVAILABLE', + Changes: opts.childChanges, + DeploymentConfig: deploymentConfig, + }); + + givenChangeSetExists({ + deploymentConfig, + changes: [{ + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: 'NestedChild', + PhysicalResourceId: childStackName, + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: child.Id, + }, + }], + }); + + return { childStackName, childChangeSetId: child.Id }; + } + + test('a replacement inside a nested stack is gated, not executed', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenNestedChangeSetExists({ rollbackDisabled: true, childChanges: [policyActionReplacementChange()] }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresRecreateChangeSet' })); + expectNoStackMutation(); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'does not support while rollback is disabled' }); + }); + + test('a nested stack with no replacement executes normally', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenNestedChangeSetExists({ rollbackDisabled: true, childChanges: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + function givenRootWithNestedStackChange(nested: Record, opts: { rollbackDisabled?: boolean } = {}) { + givenChangeSetExists({ + deploymentConfig: opts.rollbackDisabled === false + ? { Mode: 'EXPRESS', DisableRollback: false } + : { Mode: 'EXPRESS' }, + changes: [{ + Type: 'Resource', + ResourceChange: { + LogicalResourceId: 'NestedChild', + ResourceType: 'AWS::CloudFormation::Stack', + ...nested, + }, + }], + }); + } + + function givenNestedChain(levels: number, deepestChanges: Change[], opts: { rollbackDisabled?: boolean } = {}) { + const deploymentConfig: DeploymentConfig = { Mode: 'EXPRESS' }; + let child: { id: string; stackName: string } | undefined; + + for (let i = levels; i >= 1; i--) { + const stackName = `withouterrors-Nested${i}`; + fakeCfn.createStackSync({ StackName: stackName, StackStatus: StackStatus.UPDATE_COMPLETE }); + const changes: Change[] = child + ? [{ + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: `Nested${i + 1}`, + PhysicalResourceId: child.stackName, + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: child.id, + }, + }] + : deepestChanges; + + const cs = fakeCfn.createChangeSetSync({ + StackName: stackName, + ChangeSetName: `prepared-nested-${i}`, + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'UNAVAILABLE', + Changes: changes, + DeploymentConfig: deploymentConfig, + }); + child = { id: cs.Id!, stackName }; + } + + givenRootWithNestedStackChange({ + Action: 'Modify', + LogicalResourceId: 'Nested1', + PhysicalResourceId: child!.stackName, + Replacement: 'False', + ChangeSetId: child!.id, + }, opts); + } + + function givenFailedChildChangeSet(opts: { rollbackDisabled?: boolean } = {}) { + const childStackName = 'withouterrors-NestedChild-FAILED'; + fakeCfn.createStackSync({ StackName: childStackName, StackStatus: StackStatus.UPDATE_COMPLETE }); + const child = fakeCfn.createChangeSetSync({ + StackName: childStackName, + ChangeSetName: 'prepared-nested-failed', + Status: 'CREATE_FAILED', + StatusReason: 'Insufficient permissions to describe the nested template', + ExecutionStatus: 'UNAVAILABLE', + DeploymentConfig: { Mode: 'EXPRESS' }, + }); + givenRootWithNestedStackChange({ + Action: 'Modify', + PhysicalResourceId: childStackName, + Replacement: 'False', + ChangeSetId: child.Id, + }, opts); + } + + async function expectBlockedAsIncomplete(deployment: Promise, containing: string) { + await expect(deployment).rejects.toThrow(expect.objectContaining({ + name: 'NestedChangeSetInspectionIncomplete', + message: expect.stringContaining(containing), + })); + expectNoStackMutation(); + } + + test('a nested stack change carrying no child change set blocks instead of assuming no replacement', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenRootWithNestedStackChange({ Action: 'Modify', PhysicalResourceId: 'some-child', Replacement: 'False' }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expectBlockedAsIncomplete(deployment, 'reported no change set to inspect'); + }); + + test('a REMOVED nested stack legitimately has no child change set and does not block', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenRootWithNestedStackChange({ Action: 'Remove', PhysicalResourceId: 'some-child' }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + }); + + test('a malformed or not-found child change set blocks instead of assuming no replacement', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenRootWithNestedStackChange({ + Action: 'Modify', + PhysicalResourceId: 'some-child', + Replacement: 'False', + ChangeSetId: 'malformed-or-not-found', + }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expectBlockedAsIncomplete(deployment, 'could not be described'); + }); + + test('a child DescribeChangeSet failure blocks instead of assuming no replacement', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + const { childChangeSetId } = givenNestedChangeSetExists({ + rollbackDisabled: true, + childChanges: [policyActionReplacementChange()], + }); + mockCloudFormationClient + .on(DescribeChangeSetCommand, { ChangeSetName: childChangeSetId }) + .rejects(new Error('Rate exceeded')); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expectBlockedAsIncomplete(deployment, 'Rate exceeded'); + }); + + test('a child change set in CREATE_FAILED blocks instead of reading its absent changes as empty', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenFailedChildChangeSet(); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expectBlockedAsIncomplete(deployment, 'has change set status CREATE_FAILED'); + }); + + test('a hierarchy deeper than the traversal cap blocks instead of skipping the unread levels', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenNestedChain(11, [policyActionReplacementChange()]); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expectBlockedAsIncomplete(deployment, 'were not inspected'); + }); + + test.each([ + ['a missing child change set', () => givenRootWithNestedStackChange( + { Action: 'Modify', PhysicalResourceId: 'some-child', Replacement: 'False' }, + { rollbackDisabled: false }, + )], + ['a child that cannot be described', () => { + const { childChangeSetId } = givenNestedChangeSetExists({ + rollbackDisabled: false, + childChanges: [updateChange()], + }); + mockCloudFormationClient + .on(DescribeChangeSetCommand, { ChangeSetName: childChangeSetId }) + .rejects(new Error('Rate exceeded')); + }], + ['a child change set that is not CREATE_COMPLETE', () => givenFailedChildChangeSet({ rollbackDisabled: false })], + ['a hierarchy deeper than the traversal cap', () => givenNestedChain( + 11, + [policyActionReplacementChange()], + { rollbackDisabled: false }, + )], + ])('rollback enabled deploys normally despite %s', async (_name, setup) => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + setup(); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + rollback: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + }); + + test('a cycle with no replacement is cut short by the visited set rather than walked to the depth cap', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + const childStackName = 'withouterrors-NestedCycleClean'; + fakeCfn.createStackSync({ StackName: childStackName, StackStatus: StackStatus.UPDATE_COMPLETE }); + const root = fakeCfn.createChangeSetSync({ + StackName: 'withouterrors', + ChangeSetName: 'prepared', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'AVAILABLE', + DeploymentConfig: { Mode: 'EXPRESS' }, + Changes: [{ + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: 'NestedCycleClean', + PhysicalResourceId: childStackName, + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: 'cycle-child-clean', + }, + }], + }); + fakeCfn.createChangeSetSync({ + StackName: childStackName, + ChangeSetName: 'cycle-child-clean', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'UNAVAILABLE', + DeploymentConfig: { Mode: 'EXPRESS' }, + Changes: [ + updateChange(), + { + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: 'BackToRoot', + PhysicalResourceId: 'withouterrors', + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: root.Id, + }, + }, + ], + }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + }); + + test('a replacement is still reported when it is reached through a cycle', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + const childStackName = 'withouterrors-NestedCycle'; + fakeCfn.createStackSync({ StackName: childStackName, StackStatus: StackStatus.UPDATE_COMPLETE }); + const root = fakeCfn.createChangeSetSync({ + StackName: 'withouterrors', + ChangeSetName: 'prepared', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'AVAILABLE', + DeploymentConfig: { Mode: 'EXPRESS' }, + Changes: [{ + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: 'NestedCycle', + PhysicalResourceId: childStackName, + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: 'cycle-child', + }, + }], + }); + fakeCfn.createChangeSetSync({ + StackName: childStackName, + ChangeSetName: 'cycle-child', + Status: 'CREATE_COMPLETE', + ExecutionStatus: 'UNAVAILABLE', + DeploymentConfig: { Mode: 'EXPRESS' }, + Changes: [ + policyActionReplacementChange(), + { + Type: 'Resource', + ResourceChange: { + Action: 'Modify', + LogicalResourceId: 'BackToRoot', + PhysicalResourceId: 'withouterrors', + ResourceType: 'AWS::CloudFormation::Stack', + Replacement: 'False', + ChangeSetId: root.Id, + }, + }, + ], + }); + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresRecreateChangeSet' })); + expectNoStackMutation(); + ioHost.expectMessage({ level: 'warn', code: W5903, containing: 'does not support while rollback is disabled' }); + }); + + const POLICY_MATRIX = [ + [true, { express: true, rollback: true }, 'reported'], + [true, { express: true }, 'silent'], + [true, { express: true, rollback: false }, 'silent'], + [true, { rollback: true }, 'reported'], + [true, {}, 'reported'], + [true, { rollback: false }, 'silent'], + [false, { express: true, rollback: true }, 'silent'], + [false, { express: true }, 'reported'], + [false, { express: true, rollback: false }, 'reported'], + [false, { rollback: true }, 'silent'], + [false, {}, 'silent'], + [false, { rollback: false }, 'reported'], + ] as Array<[boolean, Partial, 'silent' | 'reported']>; + + test('the policy matrix covers every express/rollback/persisted combination', () => { + expect(POLICY_MATRIX).toHaveLength(12); + }); + + test('no DisableRollback is sent when the change set already pins the policy', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: false, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + rollback: false, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + const sent = mockCloudFormationClient.commandCalls(ExecuteChangeSetCommand)[0].args[0].input; + expect(sent).not.toHaveProperty('DisableRollback'); + }); + + test('DisableRollback is still sent when the change set records no policy', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenChangeSetExists({ deploymentConfig: { Mode: 'STANDARD' }, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + rollback: false, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + const sent = mockCloudFormationClient.commandCalls(ExecuteChangeSetCommand)[0].args[0].input; + expect(sent.DisableRollback).toEqual(true); + }); + + describe.each(POLICY_MATRIX)( + 'persisted rollbackDisabled=%s executed with %j', + (persistedRollbackDisabled, flags, expected) => { + test(`is ${expected}`, async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: persistedRollbackDisabled, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + ...flags, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + + const warnings = ioHost.notifySpy.mock.calls + .map((c) => c[0]) + .filter((m: any) => m.level === 'warn' && /was created with rollback/.test(m.message)); + expect(warnings).toHaveLength(expected === 'reported' ? 1 : 0); + }); + }, + ); + + test('--express does not imply rollback-disabled for a change set that is not Express', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenChangeSetExists({ deploymentConfig: { Mode: 'STANDARD' }, changes: [policyActionReplacementChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); + + test('a failed initial create is not told to replay a configuration that never existed', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.CREATE_FAILED }); + fakeCfn.overrideChangeSetChanges = [policyActionReplacementChange()]; + failOnAnyStackMutation(); + + // WHEN + const deployment = testDeployStack({ + ...standardDeployStackArguments(), + express: true, + forceDeployment: true, + }); + + // THEN + await expect(deployment).rejects.toThrow(expect.objectContaining({ name: 'ReplacementRequiresUnwedge' })); + await expect(deployment).rejects.toThrow(/no previous configuration to replay/); + await expect(deployment).rejects.toThrow(/Delete the stack and deploy again/); + await expect(deployment).rejects.not.toThrow(/Revert your change/); + expectRecreateGuidance(await deployment.then(() => '', (e) => e.message)); + expectNoStackMutation(); + }); + + test('a matching change set without a replacement executes normally', async () => { + // GIVEN + givenStackExists({ StackStatus: StackStatus.UPDATE_COMPLETE }); + givenExpressChangeSetExists({ rollbackDisabled: true, changes: [updateChange()] }); + + // WHEN + const result = await testDeployStack({ + ...standardDeployStackArguments(), + ...executePrepared, + express: true, + }); + + // THEN + expect(result.type).toEqual('did-deploy-stack'); + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(ioHost.messagesWithCode(W5903)).toEqual([]); + }); +}); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts index 7325f66b4..4a1560085 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts @@ -1603,13 +1603,8 @@ test.each([ // CloudFormation's RollbackStack API is not supported for stacks last deployed // with express mode, so `cdk deploy --express` (without an explicit `--rollback`) -// must never route through the rollback path. It always fix-forwards via the -// change set / UpdateStack and lets CloudFormation surface any error (including a -// rejected replacement on a disable-rollback stack). `--express --rollback` -// explicitly opts back into the rollback-enabled path. test.each([ - // --express alone (rollback defaults off): always fix-forward, never divert to rollback - ['express, no explicit rollback', { express: true } as Partial, 'did-deploy-stack'], + ['express, no explicit rollback', { express: true } as Partial, 'replacement-requires-rollback'], // --express --rollback: rollback is explicitly enabled, so the replacement deploys directly ['express with rollback=true', { express: true, rollback: true } as Partial, 'did-deploy-stack'], ] satisfies Array<[string, Partial, string]>)( @@ -1631,16 +1626,20 @@ test.each([ // THEN expect(result.type).toEqual(expectedType); + + if (expectedType === 'replacement-requires-rollback') { + expect(mockCloudFormationClient).not.toHaveReceivedCommand(ExecuteChangeSetCommand); + expect(mockCloudFormationClient).not.toHaveReceivedCommand(UpdateStackCommand); + } else { + expect(mockCloudFormationClient).toHaveReceivedCommand(ExecuteChangeSetCommand); + } }, ); // A stack last deployed with express mode that is in a paused fail state // (UPDATE_FAILED) cannot be recovered via the RollbackStack API. `cdk deploy -// --express` must therefore always fix-forward via createChangeSet/UpdateStack -// instead of routing to the rollback path. test.each([ ['express, no explicit rollback, no-replacement', { express: true } as Partial, 'no-replacement', 'did-deploy-stack'], - ['express, no explicit rollback, replacement', { express: true } as Partial, 'replacement', 'did-deploy-stack'], ['express with rollback=true, no-replacement', { express: true, rollback: true } as Partial, 'no-replacement', 'did-deploy-stack'], ['express with rollback=true, replacement', { express: true, rollback: true } as Partial, 'replacement', 'did-deploy-stack'], ] satisfies Array<[string, Partial, 'replacement' | 'no-replacement', string]>)( diff --git a/packages/aws-cdk/lib/cli/cdk-toolkit.ts b/packages/aws-cdk/lib/cli/cdk-toolkit.ts index 857a3f43b..946f76937 100644 --- a/packages/aws-cdk/lib/cli/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cli/cdk-toolkit.ts @@ -491,6 +491,7 @@ export class CdkToolkit { roleArn: options.roleArn, forceDeployment: options.force, rollback: options.rollback, + express: options.express, reuseAssets: options.reuseAssets, concurrency: options.concurrency, traceLogs: options.traceLogs, @@ -2565,7 +2566,9 @@ class WorkGraphDeploymentActions implements WorkGraphActions { } case 'replacement-requires-rollback': { - const motivation = 'Change includes a replacement which cannot be deployed with "--no-rollback"'; + const motivation = this.options.express + ? 'Change includes a replacement, which CloudFormation does not support while rollback is disabled (the default for Express Mode)' + : 'Change includes a replacement which cannot be deployed with "--no-rollback"'; if (this.options.force) { await this.ioHost.asIoHelper().defaults.warn(`${motivation}. Proceeding with deployment with rollback enabled (--force).`); diff --git a/packages/aws-cdk/test/cli/cdk-toolkit.test.ts b/packages/aws-cdk/test/cli/cdk-toolkit.test.ts index 349893c6c..b9ae13e69 100644 --- a/packages/aws-cdk/test/cli/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cli/cdk-toolkit.test.ts @@ -866,6 +866,35 @@ describe('deploy', () => { expect(mockCfnDeployments.deployStack).not.toHaveBeenCalled(); expect(mockCfnDeployments.prepareStack).not.toHaveBeenCalled(); }); + + test.each([ + ['plain --express', { express: true }, { express: true, rollback: undefined }], + ['--express --rollback', { express: true, rollback: true }, { express: true, rollback: true }], + ['--express --no-rollback', { express: true, rollback: false }, { express: true, rollback: false }], + ['no --express', { rollback: false }, { express: undefined, rollback: false }], + ])('forwards the rollback policy flags to toolkit-lib: %s', async (_name, flags, expected) => { + // GIVEN + const mockCfnDeployments = instanceMockFrom(Deployments); + const toolkitDeploySpy = jest.spyOn(Toolkit.prototype, 'deploy').mockResolvedValue(undefined as any); + + const cdkToolkit = new CdkToolkit({ + ioHost, + cloudExecutable, + configuration: cloudExecutable.configuration, + sdkProvider: cloudExecutable.sdkProvider, + deployments: mockCfnDeployments, + }); + + // WHEN + await cdkToolkit.deploy({ + selector: selectWithUpstream('Test-Stack-A-Display-Name'), + deploymentMethod: { method: 'execute-change-set', changeSetName: 'MyCS' }, + ...flags, + }); + + // THEN + expect(toolkitDeploySpy).toHaveBeenCalledWith(cloudExecutable, expect.objectContaining(expected)); + }); }); test('fails when no valid stack names are given', async () => {