fix(toolkit): increase number of retries (#2053)

This will help in situations where the default retry behavior is
not enough to cover throttling errors.

Fixes #1647.

Author: rix0rrr

packages/aws-cdk/lib/api/bootstrap-environment.ts

@@ -12,7 +12,7 @@ export async function bootstrapEnvironment(environment: Environment, aws: SDK, t
     environment,
     metadata: {},
     template: {
-      Description: "The CDK Toolkit Stack. It cas created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
+      Description: "The CDK Toolkit Stack. It was created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
       Resources: {
         StagingBucket: {
           Type: "AWS::S3::Bucket",

packages/aws-cdk/lib/api/util/sdk.ts

@@ -50,6 +50,18 @@ export class SDK {
   private readonly credentialsCache: CredentialsCache;
   private readonly profile?: string;
 
+  /**
+   * Default retry options for SDK clients
+   *
+   * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be
+   * a little more tenacious than the defaults, and with a little more breathing
+   * room between calls (defaults are {retries=3, base=100}).
+   *
+   * I've left this running in a tight loop for an hour and the throttle errors
+   * haven't escaped the retry mechanism.
+   */
+  private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
+
   constructor(options: SDKOptions = {}) {
     this.profile = options.profile;
 
@@ -78,41 +90,47 @@ export class SDK {
 
   public async cloudFormation(environment: Environment, mode: Mode): Promise<AWS.CloudFormation> {
     return new AWS.CloudFormation({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });
   }
 
   public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.EC2> {
     return new AWS.EC2({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode)
     });
   }
 
   public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.SSM> {
     return new AWS.SSM({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode)
     });
   }
 
   public async s3(environment: Environment, mode: Mode): Promise<AWS.S3> {
     return new AWS.S3({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });
   }
 
   public async route53(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.Route53> {
     return new AWS.Route53({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode),
     });
   }
 
   public async ecr(environment: Environment, mode: Mode): Promise<AWS.ECR> {
     return new AWS.ECR({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });