fix(toolkit): support diff on multiple stacks (#1855)

If there are stack dependencies, 'diff' would fail because it does not
know how to diff multiple stacks. Make diff support the same stack
selection mechanisms as 'cdk deploy'.

Move 'stack rename' facilities into the class that deals with the
CDK app, which is the source of thruth for stacks. This way, all
downstream code doesn't have to deal with the renames every time.

Start factoring out toolkit code into logical layers. Introducing
the class `CdkToolkit`, which represents the toolkit logic and
forms the bridge between `AppStacks` which deals with the CDK
model source (probably needs to be renamed to something better)
and `CfnProvisioner`, which deals with the deployed stacks.

N.B.: The indirection to a provisioner class with an interface is
because the interface is going to be complex (therefore, interface
over a set of functions that take callbacks) and we want to depend
just on the interface so it's easy to stub out for testing.

Author: rix0rrr

packages/@aws-cdk/cloudformation-diff/lib/format.ts

@@ -11,6 +11,10 @@ import { SecurityGroupChanges } from './network/security-group-changes';
 // tslint:disable-next-line:no-var-requires
 const { structuredPatch } = require('diff');
 
+export interface FormatStream extends NodeJS.WritableStream {
+  columns?: number;
+}
+
 /**
  * Renders template differences to the process' console.
  *
@@ -20,7 +24,7 @@ const { structuredPatch } = require('diff');
  *                         case there is no aws:cdk:path metadata in the template.
  * @param context          the number of context lines to use in arbitrary JSON diff (defaults to 3).
  */
-export function formatDifferences(stream: NodeJS.WriteStream,
+export function formatDifferences(stream: FormatStream,
                                   templateDiff: TemplateDiff,
                                   logicalToPathMap: { [logicalId: string]: string } = { },
                                   context: number = 3) {
@@ -72,7 +76,7 @@ const UPDATE   = colors.yellow('[~]');
 const REMOVAL  = colors.red('[-]');
 
 class Formatter {
-  constructor(private readonly stream: NodeJS.WriteStream,
+  constructor(private readonly stream: FormatStream,
               private readonly logicalToPathMap: { [logicalId: string]: string },
               diff?: TemplateDiff,
               private readonly context: number = 3) {

packages/aws-cdk/bin/cdk.ts

@@ -1,24 +1,25 @@
 #!/usr/bin/env node
 import 'source-map-support/register';
 
-import cxapi = require('@aws-cdk/cx-api');
 import colors = require('colors/safe');
 import fs = require('fs-extra');
 import util = require('util');
 import yargs = require('yargs');
 
-import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, Mode, SDK } from '../lib';
+import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, SDK } from '../lib';
 import { environmentsFromDescriptors, globEnvironmentsFromStacks } from '../lib/api/cxapp/environments';
 import { execProgram } from '../lib/api/cxapp/exec';
 import { AppStacks, ExtendedStackSelection, listStackNames } from '../lib/api/cxapp/stacks';
+import { CloudFormationDeploymentTarget } from '../lib/api/deployment-target';
 import { leftPad } from '../lib/api/util/string-manipulation';
-import { printSecurityDiff, printStackDiff, RequireApproval } from '../lib/diff';
+import { CdkToolkit } from '../lib/cdk-toolkit';
+import { printSecurityDiff, RequireApproval } from '../lib/diff';
 import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init';
 import { interactive } from '../lib/interactive';
 import { data, debug, error, highlight, print, setVerbose, success } from '../lib/logging';
 import { PluginHost } from '../lib/plugin';
 import { parseRenames } from '../lib/renames';
-import { deserializeStructure, serializeStructure } from '../lib/serialize';
+import { serializeStructure } from '../lib/serialize';
 import { Configuration, Settings } from '../lib/settings';
 import { VERSION } from '../lib/version';
 
@@ -66,7 +67,8 @@ async function parseCommandLineArguments() {
     .command('destroy [STACKS..]', 'Destroy the stack(s) named STACKS', yargs => yargs
       .option('exclusively', { type: 'boolean', alias: 'x', desc: 'only deploy requested stacks, don\'t include dependees' })
       .option('force', { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' }))
-    .command('diff [STACK]', 'Compares the specified stack with the deployed stack or a local template file, and returns with status 1 if any difference is found', yargs => yargs
+    .command('diff [STACKS..]', 'Compares the specified stack with the deployed stack or a local template file, and returns with status 1 if any difference is found', yargs => yargs
+      .option('exclusively', { type: 'boolean', alias: 'e', desc: 'only diff requested stacks, don\'t include dependencies' })
       .option('context-lines', { type: 'number', desc: 'number of context lines to include in arbitrary JSON diff rendering', default: 3 })
       .option('template', { type: 'string', desc: 'the path to the CloudFormation template to compare with' })
       .option('strict', { type: 'boolean', desc: 'do not filter out AWS::CDK::Metadata resources', default: false }))
@@ -107,13 +109,17 @@ async function initCommandLine() {
   const configuration = new Configuration(argv);
   await configuration.load();
 
+  const provisioner = new CloudFormationDeploymentTarget({ aws });
+
   const appStacks = new AppStacks({
     verbose: argv.trace || argv.verbose,
     ignoreErrors: argv.ignoreErrors,
     strict: argv.strict,
-    configuration, aws, synthesizer: execProgram });
-
-  const renames = parseRenames(argv.rename);
+    configuration,
+    aws,
+    synthesizer: execProgram,
+    renames: parseRenames(argv.rename)
+  });
 
   /** Function to load plug-ins, using configurations additively. */
   function loadPlugins(...settings: Settings[]) {
@@ -165,13 +171,21 @@ async function initCommandLine() {
     args.STACKS = args.STACKS || [];
     args.ENVIRONMENTS = args.ENVIRONMENTS || [];
 
+    const cli = new CdkToolkit({ appStacks, provisioner });
+
     switch (command) {
       case 'ls':
       case 'list':
         return await cliList({ long: args.long });
 
       case 'diff':
-        return await diffStack(await findStack(args.STACK), args.template, args.strict, args.contextLines);
+        return await cli.diff({
+          stackNames: args.STACKS,
+          exclusively: args.exclusively,
+          templatePath: args.template,
+          strict: args.strict,
+          contextLines: args.contextLines
+        });
 
       case 'bootstrap':
         return await cliBootstrap(args.ENVIRONMENTS, toolkitStackName, args.roleArn);
@@ -258,7 +272,6 @@ async function initCommandLine() {
     const autoSelectDependencies = !exclusively && outputDir !== undefined;
 
     const stacks = await appStacks.selectStacks(stackNames, autoSelectDependencies ? ExtendedStackSelection.Upstream : ExtendedStackSelection.None);
-    renames.validateSelectedStacks(stacks);
 
     if (doInteractive) {
       if (stacks.length !== 1) {
@@ -294,9 +307,8 @@ async function initCommandLine() {
 
     let i = 0;
     for (const stack of stacks) {
-      const finalName = renames.finalName(stack.name);
       const prefix = numbered ? leftPad(`${i}`, 3, '0') + '.' : '';
-      const fileName = `${outputDir}/${prefix}${finalName}.template.${json ? 'json' : 'yaml'}`;
+      const fileName = `${outputDir}/${prefix}${stack.name}.template.${json ? 'json' : 'yaml'}`;
       highlight(fileName);
       await fs.writeFile(fileName, toJsonOrYaml(stack.template));
       i++;
@@ -337,7 +349,6 @@ async function initCommandLine() {
     if (requireApproval === undefined) { requireApproval = RequireApproval.Broadening; }
 
     const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Upstream);
-    renames.validateSelectedStacks(stacks);
 
     for (const stack of stacks) {
       if (stacks.length !== 1) { highlight(stack.name); }
@@ -346,10 +357,9 @@ async function initCommandLine() {
         throw new Error(`Stack ${stack.name} does not define an environment, and AWS credentials could not be obtained from standard locations or no region was configured.`);
       }
       const toolkitInfo = await loadToolkitInfo(stack.environment, aws, toolkitStackName);
-      const deployName = renames.finalName(stack.name);
 
       if (requireApproval !== RequireApproval.Never) {
-        const currentTemplate = await readCurrentTemplate(stack);
+        const currentTemplate = await provisioner.readCurrentTemplate(stack);
         if (printSecurityDiff(currentTemplate, stack, requireApproval)) {
 
           // only talk to user if we STDIN is a terminal (otherwise, fail)
@@ -364,14 +374,14 @@ async function initCommandLine() {
         }
       }
 
-      if (deployName !== stack.name) {
-        print('%s: deploying... (was %s)', colors.bold(deployName), colors.bold(stack.name));
+      if (stack.name !== stack.originalName) {
+        print('%s: deploying... (was %s)', colors.bold(stack.name), colors.bold(stack.originalName));
       } else {
         print('%s: deploying...', colors.bold(stack.name));
       }
 
       try {
-        const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName, roleArn, ci });
+        const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName: stack.name, roleArn, ci });
         const message = result.noOp
           ? ` ✅  %s (no changes)`
           : ` ✅  %s`;
@@ -384,7 +394,7 @@ async function initCommandLine() {
 
         for (const name of Object.keys(result.outputs)) {
           const value = result.outputs[name];
-          print('%s.%s = %s', colors.cyan(deployName), colors.cyan(name), colors.underline(colors.cyan(value)));
+          print('%s.%s = %s', colors.cyan(stack.name), colors.cyan(name), colors.underline(colors.cyan(value)));
         }
 
         print('\nStack ARN:');
@@ -403,8 +413,6 @@ async function initCommandLine() {
     // The stacks will have been ordered for deployment, so reverse them for deletion.
     stacks.reverse();
 
-    renames.validateSelectedStacks(stacks);
-
     if (!force) {
       // tslint:disable-next-line:max-line-length
       const confirmed = await confirm(`Are you sure you want to delete: ${colors.blue(stacks.map(s => s.name).join(', '))} (y/n)?`);
@@ -414,59 +422,17 @@ async function initCommandLine() {
     }
 
     for (const stack of stacks) {
-      const deployName = renames.finalName(stack.name);
-
-      success('%s: destroying...', colors.blue(deployName));
+      success('%s: destroying...', colors.blue(stack.name));
       try {
-        await destroyStack({ stack, sdk: aws, deployName, roleArn });
-        success('\n ✅  %s: destroyed', colors.blue(deployName));
+        await destroyStack({ stack, sdk: aws, deployName: stack.name, roleArn });
+        success('\n ✅  %s: destroyed', colors.blue(stack.name));
       } catch (e) {
-        error('\n ❌  %s: destroy failed', colors.blue(deployName), e);
+        error('\n ❌  %s: destroy failed', colors.blue(stack.name), e);
         throw e;
       }
     }
   }
 
-  async function diffStack(stackName: string, templatePath: string | undefined, strict: boolean, context: number): Promise<number> {
-    const stack = await appStacks.synthesizeStack(stackName);
-    const currentTemplate = await readCurrentTemplate(stack, templatePath);
-    if (printStackDiff(currentTemplate, stack, strict, context) === 0) {
-      return 0;
-    } else {
-      return 1;
-    }
-  }
-
-  async function readCurrentTemplate(stack: cxapi.SynthesizedStack, templatePath?: string): Promise<{ [key: string]: any }> {
-    if (templatePath) {
-      if (!await fs.pathExists(templatePath)) {
-        throw new Error(`There is no file at ${templatePath}`);
-      }
-      const fileContent = await fs.readFile(templatePath, { encoding: 'UTF-8' });
-      return parseTemplate(fileContent);
-    } else {
-      const stackName = renames.finalName(stack.name);
-      debug(`Reading existing template for stack ${stackName}.`);
-
-      const cfn = await aws.cloudFormation(stack.environment, Mode.ForReading);
-      try {
-        const response = await cfn.getTemplate({ StackName: stackName }).promise();
-        return (response.TemplateBody && parseTemplate(response.TemplateBody)) || {};
-      } catch (e) {
-        if (e.code === 'ValidationError' && e.message === `Stack with id ${stackName} does not exist`) {
-          return {};
-        } else {
-          throw e;
-        }
-      }
-    }
-
-    /* Attempt to parse YAML, fall back to JSON. */
-    function parseTemplate(text: string): any {
-      return deserializeStructure(text);
-    }
-  }
-
   /**
    * Match a single stack from the list of available stacks
    */

packages/aws-cdk/lib/api/cxapp/stacks.ts

@@ -3,6 +3,7 @@ import colors = require('colors/safe');
 import minimatch = require('minimatch');
 import contextproviders = require('../../context-providers');
 import { debug, error, print, warning } from '../../logging';
+import { Renames } from '../../renames';
 import { Configuration } from '../../settings';
 import cdkUtil = require('../../util');
 import { SDK } from '../util/sdk';
@@ -42,6 +43,11 @@ export interface AppStacksProps {
    */
   aws: SDK;
 
+  /**
+   * Renames to apply
+   */
+  renames?: Renames;
+
   /**
    * Callback invoked to synthesize the actual stacks
    */
@@ -59,8 +65,10 @@ export class AppStacks {
    * we can invoke it once and cache the response for subsequent calls.
    */
   private cachedResponse?: cxapi.SynthesizeResponse;
+  private readonly renames: Renames;
 
   constructor(private readonly props: AppStacksProps) {
+    this.renames = props.renames || new Renames({});
   }
 
   /**
@@ -69,7 +77,7 @@ export class AppStacks {
    * It's an error if there are no stacks to select, or if one of the requested parameters
    * refers to a nonexistant stack.
    */
-  public async selectStacks(selectors: string[], extendedSelection: ExtendedStackSelection): Promise<cxapi.SynthesizedStack[]> {
+  public async selectStacks(selectors: string[], extendedSelection: ExtendedStackSelection): Promise<SelectedStack[]> {
     selectors = selectors.filter(s => s != null); // filter null/undefined
 
     const stacks: cxapi.SynthesizedStack[] = await this.listStacks();
@@ -79,7 +87,7 @@ export class AppStacks {
 
     if (selectors.length === 0) {
       debug('Stack name not specified, so defaulting to all available stacks: ' + listStackNames(stacks));
-      return stacks;
+      return this.applyRenames(stacks);
     }
 
     const allStacks = new Map<string, cxapi.SynthesizedStack>();
@@ -118,7 +126,7 @@ export class AppStacks {
 
     // Only check selected stacks for errors
     this.processMessages(selectedList);
-    return selectedList;
+    return this.applyRenames(selectedList);
   }
 
   /**
@@ -128,6 +136,8 @@ export class AppStacks {
    * topologically sorted order. If there are dependencies that are not in the
    * set, they will be ignored; it is the user's responsibility that the
    * non-selected stacks have already been deployed previously.
+   *
+   * Renames are *NOT* applied in list mode.
    */
   public async listStacks(): Promise<cxapi.SynthesizedStack[]> {
     const response = await this.synthesizeStacks();
@@ -137,13 +147,13 @@ export class AppStacks {
   /**
    * Synthesize a single stack
    */
-  public async synthesizeStack(stackName: string): Promise<cxapi.SynthesizedStack> {
+  public async synthesizeStack(stackName: string): Promise<SelectedStack> {
     const resp = await this.synthesizeStacks();
     const stack = resp.stacks.find(s => s.name === stackName);
     if (!stack) {
       throw new Error(`Stack ${stackName} not found`);
     }
-    return stack;
+    return this.applyRenames([stack])[0];
   }
 
   /**
@@ -253,6 +263,21 @@ export class AppStacks {
       logFn(`  ${entry.trace.join('\n  ')}`);
     }
   }
+
+  private applyRenames(stacks: cxapi.SynthesizedStack[]): SelectedStack[] {
+    this.renames.validateSelectedStacks(stacks);
+
+    const ret = [];
+    for (const stack of stacks) {
+      ret.push({
+        ...stack,
+        originalName: stack.name,
+        name: this.renames.finalName(stack.name),
+      });
+    }
+
+    return ret;
+  }
 }
 
 /**
@@ -335,4 +360,11 @@ function includeUpstreamStacks(selectedStacks: Map<string, cxapi.SynthesizedStac
   if (added.length > 0) {
     print('Including dependency stacks: %s', colors.bold(added.join(', ')));
   }
+}
+
+export interface SelectedStack extends cxapi.SynthesizedStack {
+  /**
+   * The original name of the stack before renaming
+   */
+  originalName: string;
 }