feat(toolkit): add '--reuse-asset' option (#1918)

Reusing assets avoids rebuilding an asset and just reuses the currently
deployed one. Especially helpful for Docker containers that take a long
time to build.

Fixes #1916

Author: rix0rrr

packages/aws-cdk/bin/cdk.ts

@@ -6,14 +6,14 @@ import fs = require('fs-extra');
 import util = require('util');
 import yargs = require('yargs');
 
-import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, SDK } from '../lib';
+import { bootstrapEnvironment, destroyStack, SDK } from '../lib';
 import { environmentsFromDescriptors, globEnvironmentsFromStacks } from '../lib/api/cxapp/environments';
 import { execProgram } from '../lib/api/cxapp/exec';
 import { AppStacks, ExtendedStackSelection, listStackNames } from '../lib/api/cxapp/stacks';
-import { CloudFormationDeploymentTarget } from '../lib/api/deployment-target';
+import { CloudFormationDeploymentTarget, DEFAULT_TOOLKIT_STACK_NAME } from '../lib/api/deployment-target';
 import { leftPad } from '../lib/api/util/string-manipulation';
 import { CdkToolkit } from '../lib/cdk-toolkit';
-import { printSecurityDiff, RequireApproval } from '../lib/diff';
+import { RequireApproval } from '../lib/diff';
 import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init';
 import { interactive } from '../lib/interactive';
 import { data, debug, error, highlight, print, setVerbose, success } from '../lib/logging';
@@ -27,8 +27,6 @@ import { VERSION } from '../lib/version';
 const promptly = require('promptly');
 const confirm = util.promisify(promptly.confirm);
 
-const DEFAULT_TOOLKIT_STACK_NAME = 'CDKToolkit';
-
 // tslint:disable:no-shadowed-variable max-line-length
 async function parseCommandLineArguments() {
   const initTemplateLanuages = await availableInitLanguages;
@@ -61,6 +59,7 @@ async function parseCommandLineArguments() {
       .option('numbered', { type: 'boolean', alias: 'n', desc: 'prefix filenames with numbers to indicate deployment ordering' }))
     .command('bootstrap [ENVIRONMENTS..]', 'Deploys the CDK toolkit stack into an AWS environment')
     .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', yargs => yargs
+      .option('build-exclude', { type: 'array', alias: 'E', nargs: 1, desc: 'do not rebuild asset with the given ID. Can be specified multiple times.', default: [] })
       .option('exclusively', { type: 'boolean', alias: 'e', desc: 'only deploy requested stacks, don\'t include dependencies' })
       .option('require-approval', { type: 'string', choices: [RequireApproval.Never, RequireApproval.AnyChange, RequireApproval.Broadening], desc: 'what security-sensitive changes need manual approval' }))
       .option('ci', { type: 'boolean', desc: 'Force CI detection. Use --no-ci to disable CI autodetection.', default: process.env.CI !== undefined })
@@ -87,6 +86,7 @@ async function parseCommandLineArguments() {
     ].join('\n\n'))
     .argv;
 }
+
 if (!process.stdout.isTTY) {
   colors.disable();
 }
@@ -191,7 +191,15 @@ async function initCommandLine() {
         return await cliBootstrap(args.ENVIRONMENTS, toolkitStackName, args.roleArn);
 
       case 'deploy':
-        return await cliDeploy(args.STACKS, args.exclusively, toolkitStackName, args.roleArn, configuration.settings.get(['requireApproval']), args.ci);
+        return await cli.deploy({
+          stackNames: args.STACKS,
+          exclusively: args.exclusively,
+          toolkitStackName,
+          roleArn: args.roleArn,
+          requireApproval: configuration.settings.get(['requireApproval']),
+          ci: args.ci,
+          reuseAssets: args['build-exclude']
+        });
 
       case 'destroy':
         return await cliDestroy(args.STACKS, args.exclusively, args.force, args.roleArn);
@@ -340,73 +348,6 @@ async function initCommandLine() {
     return 0; // exit-code
   }
 
-  async function cliDeploy(stackNames: string[],
-                           exclusively: boolean,
-                           toolkitStackName: string,
-                           roleArn: string | undefined,
-                           requireApproval: RequireApproval,
-                           ci: boolean) {
-    if (requireApproval === undefined) { requireApproval = RequireApproval.Broadening; }
-
-    const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Upstream);
-
-    for (const stack of stacks) {
-      if (stacks.length !== 1) { highlight(stack.name); }
-      if (!stack.environment) {
-        // tslint:disable-next-line:max-line-length
-        throw new Error(`Stack ${stack.name} does not define an environment, and AWS credentials could not be obtained from standard locations or no region was configured.`);
-      }
-      const toolkitInfo = await loadToolkitInfo(stack.environment, aws, toolkitStackName);
-
-      if (requireApproval !== RequireApproval.Never) {
-        const currentTemplate = await provisioner.readCurrentTemplate(stack);
-        if (printSecurityDiff(currentTemplate, stack, requireApproval)) {
-
-          // only talk to user if we STDIN is a terminal (otherwise, fail)
-          if (!process.stdin.isTTY) {
-            throw new Error(
-              '"--require-approval" is enabled and stack includes security-sensitive updates, ' +
-              'but terminal (TTY) is not attached so we are unable to get a confirmation from the user');
-          }
-
-          const confirmed = await confirm(`Do you wish to deploy these changes (y/n)?`);
-          if (!confirmed) { throw new Error('Aborted by user'); }
-        }
-      }
-
-      if (stack.name !== stack.originalName) {
-        print('%s: deploying... (was %s)', colors.bold(stack.name), colors.bold(stack.originalName));
-      } else {
-        print('%s: deploying...', colors.bold(stack.name));
-      }
-
-      try {
-        const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName: stack.name, roleArn, ci });
-        const message = result.noOp
-          ? ` ✅  %s (no changes)`
-          : ` ✅  %s`;
-
-        success('\n' + message, stack.name);
-
-        if (Object.keys(result.outputs).length > 0) {
-          print('\nOutputs:');
-        }
-
-        for (const name of Object.keys(result.outputs)) {
-          const value = result.outputs[name];
-          print('%s.%s = %s', colors.cyan(stack.name), colors.cyan(name), colors.underline(colors.cyan(value)));
-        }
-
-        print('\nStack ARN:');
-
-        data(result.stackArn);
-      } catch (e) {
-        error('\n ❌  %s failed: %s', colors.bold(stack.name), e);
-        throw e;
-      }
-    }
-  }
-
   async function cliDestroy(stackNames: string[], exclusively: boolean, force: boolean, roleArn: string | undefined) {
     const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Downstream);
 

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -31,6 +31,7 @@ export interface DeployStackOptions {
   deployName?: string;
   quiet?: boolean;
   ci?: boolean;
+  reuseAssets?: string[];
 }
 
 const LARGE_TEMPLATE_SIZE_KB = 50;
@@ -40,7 +41,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
     throw new Error(`The stack ${options.stack.name} does not have an environment`);
   }
 
-  const params = await prepareAssets(options.stack, options.toolkitInfo, options.ci);
+  const params = await prepareAssets(options.stack, options.toolkitInfo, options.ci, options.reuseAssets);
 
   const deployName = options.deployName || options.stack.name;
 

packages/aws-cdk/lib/api/deployment-target.ts

@@ -2,8 +2,12 @@ import cxapi = require('@aws-cdk/cx-api');
 import { debug } from '../logging';
 import { deserializeStructure } from '../serialize';
 import { Mode } from './aws-auth/credentials';
+import { deployStack, DeployStackResult } from './deploy-stack';
+import { loadToolkitInfo } from './toolkit-info';
 import { SDK } from './util/sdk';
 
+export const DEFAULT_TOOLKIT_STACK_NAME = 'CDKToolkit';
+
 export type Template = { [key: string]: any };
 
 /**
@@ -13,6 +17,17 @@ export type Template = { [key: string]: any };
  */
 export interface IDeploymentTarget {
   readCurrentTemplate(stack: cxapi.SynthesizedStack): Promise<Template>;
+  deployStack(options: DeployStackOptions): Promise<DeployStackResult>;
+}
+
+export interface DeployStackOptions {
+  stack: cxapi.SynthesizedStack;
+  roleArn?: string;
+  deployName?: string;
+  quiet?: boolean;
+  ci?: boolean;
+  toolkitStackName?: string;
+  reuseAssets?: string[];
 }
 
 export interface ProvisionerProps {
@@ -23,13 +38,16 @@ export interface ProvisionerProps {
  * Default provisioner (applies to CloudFormation).
  */
 export class CloudFormationDeploymentTarget implements IDeploymentTarget {
-  constructor(private readonly props: ProvisionerProps) {
+  private readonly aws: SDK;
+
+  constructor(props: ProvisionerProps) {
+    this.aws = props.aws;
   }
 
   public async readCurrentTemplate(stack: cxapi.SynthesizedStack): Promise<Template> {
     debug(`Reading existing template for stack ${stack.name}.`);
 
-    const cfn = await this.props.aws.cloudFormation(stack.environment, Mode.ForReading);
+    const cfn = await this.aws.cloudFormation(stack.environment, Mode.ForReading);
     try {
       const response = await cfn.getTemplate({ StackName: stack.name }).promise();
       return (response.TemplateBody && deserializeStructure(response.TemplateBody)) || {};
@@ -41,4 +59,18 @@ export class CloudFormationDeploymentTarget implements IDeploymentTarget {
       }
     }
   }
+
+  public async deployStack(options: DeployStackOptions): Promise<DeployStackResult> {
+    const toolkitInfo = await loadToolkitInfo(options.stack.environment, this.aws, options.toolkitStackName || DEFAULT_TOOLKIT_STACK_NAME);
+    return deployStack({
+      stack: options.stack,
+      deployName: options.deployName,
+      roleArn: options.roleArn,
+      quiet: options.quiet,
+      sdk: this.aws,
+      ci: options.ci,
+      reuseAssets: options.reuseAssets,
+      toolkitInfo,
+    });
+  }
 }

packages/aws-cdk/lib/assets.ts

@@ -10,8 +10,11 @@ import { zipDirectory } from './archive';
 import { prepareContainerAsset } from './docker';
 import { debug, success } from './logging';
 
-export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
+// tslint:disable-next-line:max-line-length
+export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo, ci?: boolean, reuse?: string[]): Promise<CloudFormation.Parameter[]> {
+  reuse = reuse || [];
   const assets = findAssets(stack.metadata);
+
   if (assets.length === 0) {
     return [];
   }
@@ -21,40 +24,51 @@ export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: Toolk
     throw new Error(`This stack uses assets, so the toolkit stack must be deployed to the environment (Run "${colors.blue("cdk bootstrap " + stack.environment!.name)}")`);
   }
 
-  debug('Preparing assets');
   let params = new Array<CloudFormation.Parameter>();
   for (const asset of assets) {
-    debug(` - ${asset.path} (${asset.packaging})`);
+    // FIXME: Should have excluded by construct path here instead of by unique ID, preferably using
+    // minimatch so we can support globs. Maybe take up during artifact refactoring.
+    const reuseAsset = reuse.indexOf(asset.id) > -1;
+
+    if (reuseAsset) {
+      debug(`Preparing asset ${asset.id}: ${JSON.stringify(asset)} (reusing)`);
+    } else {
+      debug(`Preparing asset ${asset.id}: ${JSON.stringify(asset)}`);
+    }
 
-    params = params.concat(await prepareAsset(asset, toolkitInfo, ci));
+    params = params.concat(await prepareAsset(asset, toolkitInfo, reuseAsset, ci));
   }
 
   return params;
 }
 
-async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
-  debug('Preparing asset', JSON.stringify(asset));
+// tslint:disable-next-line:max-line-length
+async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, reuse: boolean, ci?: boolean): Promise<CloudFormation.Parameter[]> {
   switch (asset.packaging) {
     case 'zip':
-      return await prepareZipAsset(asset, toolkitInfo);
+      return await prepareZipAsset(asset, toolkitInfo, reuse);
     case 'file':
-      return await prepareFileAsset(asset, toolkitInfo);
+      return await prepareFileAsset(asset, toolkitInfo, reuse);
     case 'container-image':
-      return await prepareContainerAsset(asset, toolkitInfo, ci);
+      return await prepareContainerAsset(asset, toolkitInfo, reuse, ci);
     default:
       // tslint:disable-next-line:max-line-length
       throw new Error(`Unsupported packaging type: ${(asset as any).packaging}. You might need to upgrade your aws-cdk toolkit to support this asset type.`);
   }
 }
 
-async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise<CloudFormation.Parameter[]> {
+async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: ToolkitInfo, reuse: boolean): Promise<CloudFormation.Parameter[]> {
+  if (reuse) {
+    return await prepareFileAsset(asset, toolkitInfo, reuse);
+  }
+
   debug('Preparing zip asset from directory:', asset.path);
   const staging = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-assets'));
   try {
     const archiveFile = path.join(staging, 'archive.zip');
     await zipDirectory(asset.path, archiveFile);
     debug('zip archive:', archiveFile);
-    return await prepareFileAsset(asset, toolkitInfo, archiveFile, 'application/zip');
+    return await prepareFileAsset(asset, toolkitInfo, reuse, archiveFile, 'application/zip');
   } finally {
     await fs.remove(staging);
   }
@@ -69,9 +83,17 @@ async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: Toolk
 async function prepareFileAsset(
     asset: FileAssetMetadataEntry,
     toolkitInfo: ToolkitInfo,
+    reuse: boolean,
     filePath?: string,
     contentType?: string): Promise<CloudFormation.Parameter[]> {
 
+  if (reuse) {
+    return [
+      { ParameterKey: asset.s3BucketParameter, UsePreviousValue: true },
+      { ParameterKey: asset.s3KeyParameter, UsePreviousValue: true },
+    ];
+  }
+
   filePath = filePath || asset.path;
   debug('Preparing file asset:', filePath);