feat(aws-ecs): instance autoscaling and drain hook (#1192)

Make it easy to configure EC2 instance autoscaling for your cluster,
and automatically add a Lifecylce Hook Lambda that will delay
instance termination until all ECS tasks have drained from the instance.

Fixes #1162.

Author: rix0rrr

packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts

@@ -28,13 +28,15 @@ export interface AutoScalingGroupProps {
 
   /**
    * Minimum number of instances in the fleet
+   *
    * @default 1
    */
   minSize?: number;
 
   /**
    * Maximum number of instances in the fleet
-   * @default 1
+   *
+   * @default desiredCapacity
    */
   maxSize?: number;
 
@@ -234,9 +236,12 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
 
     launchConfig.addDependency(this.role);
 
+    const desiredCapacity =
+        (props.desiredCapacity !== undefined ? props.desiredCapacity :
+        (props.minSize !== undefined ? props.minSize :
+        (props.maxSize !== undefined ? props.maxSize : 1)));
     const minSize = props.minSize !== undefined ? props.minSize : 1;
-    const maxSize = props.maxSize !== undefined ? props.maxSize : 1;
-    const desiredCapacity = props.desiredCapacity !== undefined ? props.desiredCapacity : 1;
+    const maxSize = props.maxSize !== undefined ? props.maxSize : desiredCapacity;
 
     if (desiredCapacity < minSize || desiredCapacity > maxSize) {
       throw new Error(`Should have minSize (${minSize}) <= desiredCapacity (${desiredCapacity}) <= maxSize (${maxSize})`);
@@ -322,8 +327,8 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
   /**
    * Scale out or in based on time
    */
-  public scaleOnSchedule(id: string, props: BasicScheduledActionProps) {
-    new ScheduledAction(this, `ScheduledAction${id}`, {
+  public scaleOnSchedule(id: string, props: BasicScheduledActionProps): ScheduledAction {
+    return new ScheduledAction(this, `ScheduledAction${id}`, {
       autoScalingGroup: this,
       ...props,
     });
@@ -332,7 +337,7 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
   /**
    * Scale out or in to achieve a target CPU utilization
    */
-  public scaleOnCpuUtilization(id: string, props: CpuUtilizationScalingProps) {
+  public scaleOnCpuUtilization(id: string, props: CpuUtilizationScalingProps): TargetTrackingScalingPolicy {
     return new TargetTrackingScalingPolicy(this, `ScalingPolicy${id}`, {
       autoScalingGroup: this,
       predefinedMetric: PredefinedMetric.ASGAverageCPUUtilization,
@@ -344,7 +349,7 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
   /**
    * Scale out or in to achieve a target network ingress rate
    */
-  public scaleOnIncomingBytes(id: string, props: NetworkUtilizationScalingProps) {
+  public scaleOnIncomingBytes(id: string, props: NetworkUtilizationScalingProps): TargetTrackingScalingPolicy {
     return new TargetTrackingScalingPolicy(this, `ScalingPolicy${id}`, {
       autoScalingGroup: this,
       predefinedMetric: PredefinedMetric.ASGAverageNetworkIn,
@@ -356,7 +361,7 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
   /**
    * Scale out or in to achieve a target network egress rate
    */
-  public scaleOnOutgoingBytes(id: string, props: NetworkUtilizationScalingProps) {
+  public scaleOnOutgoingBytes(id: string, props: NetworkUtilizationScalingProps): TargetTrackingScalingPolicy {
     return new TargetTrackingScalingPolicy(this, `ScalingPolicy${id}`, {
       autoScalingGroup: this,
       predefinedMetric: PredefinedMetric.ASGAverageNetworkOut,
@@ -371,7 +376,7 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
    * The AutoScalingGroup must have been attached to an Application Load Balancer
    * in order to be able to call this.
    */
-  public scaleOnRequestCount(id: string, props: RequestCountScalingProps) {
+  public scaleOnRequestCount(id: string, props: RequestCountScalingProps): TargetTrackingScalingPolicy {
     if (this.albTargetGroup === undefined) {
       throw new Error('Attach the AutoScalingGroup to an Application Load Balancer before calling scaleOnRequestCount()');
     }
@@ -389,13 +394,14 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
     // Target tracking policy can only be created after the load balancer has been
     // attached to the targetgroup (because we need its ARN).
     policy.addDependency(this.albTargetGroup.loadBalancerDependency());
+    return policy;
   }
 
   /**
    * Scale out or in in order to keep a metric around a target value
    */
-  public scaleToTrackMetric(id: string, props: MetricTargetTrackingProps) {
-    new TargetTrackingScalingPolicy(this, `ScalingPolicy${id}`, {
+  public scaleToTrackMetric(id: string, props: MetricTargetTrackingProps): TargetTrackingScalingPolicy {
+    return new TargetTrackingScalingPolicy(this, `ScalingPolicy${id}`, {
       autoScalingGroup: this,
       customMetric: props.metric,
       ...props
@@ -405,7 +411,7 @@ export class AutoScalingGroup extends cdk.Construct implements IAutoScalingGroup
   /**
    * Scale out or in, in response to a metric
    */
-  public scaleOnMetric(id: string, props: BasicStepScalingPolicyProps) {
+  public scaleOnMetric(id: string, props: BasicStepScalingPolicyProps): StepScalingPolicy {
     return new StepScalingPolicy(this, id, { ...props, autoScalingGroup: this });
   }
 
@@ -658,6 +664,41 @@ export interface IAutoScalingGroup {
    * The name of the AutoScalingGroup
    */
   readonly autoScalingGroupName: string;
+
+  /**
+   * Send a message to either an SQS queue or SNS topic when instances launch or terminate
+   */
+  onLifecycleTransition(id: string, props: BasicLifecycleHookProps): LifecycleHook;
+
+  /**
+   * Scale out or in based on time
+   */
+  scaleOnSchedule(id: string, props: BasicScheduledActionProps): ScheduledAction;
+
+  /**
+   * Scale out or in to achieve a target CPU utilization
+   */
+  scaleOnCpuUtilization(id: string, props: CpuUtilizationScalingProps): TargetTrackingScalingPolicy;
+
+  /**
+   * Scale out or in to achieve a target network ingress rate
+   */
+  scaleOnIncomingBytes(id: string, props: NetworkUtilizationScalingProps): TargetTrackingScalingPolicy;
+
+  /**
+   * Scale out or in to achieve a target network egress rate
+   */
+  scaleOnOutgoingBytes(id: string, props: NetworkUtilizationScalingProps): TargetTrackingScalingPolicy;
+
+  /**
+   * Scale out or in in order to keep a metric around a target value
+   */
+  scaleToTrackMetric(id: string, props: MetricTargetTrackingProps): TargetTrackingScalingPolicy;
+
+  /**
+   * Scale out or in, in response to a metric
+   */
+  scaleOnMetric(id: string, props: BasicStepScalingPolicyProps): StepScalingPolicy;
 }
 
 /**

packages/@aws-cdk/aws-autoscaling/lib/lifecycle-hook.ts

@@ -27,7 +27,7 @@ export interface BasicLifecycleHookProps {
    *
    * If the lifecycle hook times out, perform the action in DefaultResult.
    */
-  heartbeatTimeout?: number;
+  heartbeatTimeoutSec?: number;
 
   /**
    * The state of the Amazon EC2 instance to which you want to attach the lifecycle hook.
@@ -87,7 +87,7 @@ export class LifecycleHook extends cdk.Construct implements api.ILifecycleHook {
     const resource = new cloudformation.LifecycleHookResource(this, 'Resource', {
       autoScalingGroupName: props.autoScalingGroup.autoScalingGroupName,
       defaultResult: props.defaultResult,
-      heartbeatTimeout: props.heartbeatTimeout,
+      heartbeatTimeout: props.heartbeatTimeoutSec,
       lifecycleHookName: props.lifecycleHookName,
       lifecycleTransition: props.lifecycleTransition,
       notificationMetadata: props.notificationMetadata,

packages/@aws-cdk/aws-autoscaling/test/test.auto-scaling-group.ts

@@ -149,6 +149,78 @@ export = {
     test.done();
   },
 
+  'can specify only min capacity'(test: Test) {
+    // GIVEN
+    const stack = new cdk.Stack();
+    const vpc = mockVpc(stack);
+
+    // WHEN
+    new autoscaling.AutoScalingGroup(stack, 'MyFleet', {
+      instanceType: new ec2.InstanceTypePair(ec2.InstanceClass.M4, ec2.InstanceSize.Micro),
+      machineImage: new ec2.AmazonLinuxImage(),
+      vpc,
+      minSize: 10
+    });
+
+    // THEN
+    expect(stack).to(haveResource("AWS::AutoScaling::AutoScalingGroup", {
+        MinSize: "10",
+        MaxSize: "10",
+        DesiredCapacity: "10",
+      }
+    ));
+
+    test.done();
+  },
+
+  'can specify only max capacity'(test: Test) {
+    // GIVEN
+    const stack = new cdk.Stack();
+    const vpc = mockVpc(stack);
+
+    // WHEN
+    new autoscaling.AutoScalingGroup(stack, 'MyFleet', {
+      instanceType: new ec2.InstanceTypePair(ec2.InstanceClass.M4, ec2.InstanceSize.Micro),
+      machineImage: new ec2.AmazonLinuxImage(),
+      vpc,
+      maxSize: 10
+    });
+
+    // THEN
+    expect(stack).to(haveResource("AWS::AutoScaling::AutoScalingGroup", {
+        MinSize: "1",
+        MaxSize: "10",
+        DesiredCapacity: "10",
+      }
+    ));
+
+    test.done();
+  },
+
+  'can specify only desiredCount'(test: Test) {
+    // GIVEN
+    const stack = new cdk.Stack();
+    const vpc = mockVpc(stack);
+
+    // WHEN
+    new autoscaling.AutoScalingGroup(stack, 'MyFleet', {
+      instanceType: new ec2.InstanceTypePair(ec2.InstanceClass.M4, ec2.InstanceSize.Micro),
+      machineImage: new ec2.AmazonLinuxImage(),
+      vpc,
+      desiredCapacity: 10
+    });
+
+    // THEN
+    expect(stack).to(haveResource("AWS::AutoScaling::AutoScalingGroup", {
+        MinSize: "1",
+        MaxSize: "10",
+        DesiredCapacity: "10",
+      }
+    ));
+
+    test.done();
+  },
+
   'addToRolePolicy can be used to add statements to the role policy'(test: Test) {
     const stack = new cdk.Stack(undefined, 'MyStack', { env: { region: 'us-east-1', account: '1234' }});
     const vpc = mockVpc(stack);

packages/@aws-cdk/aws-ecs/README.md

@@ -236,10 +236,32 @@ containers are running on for you. If you're running an ECS cluster however,
 your EC2 instances might fill up as your number of Tasks goes up.
 
 To avoid placement errors, you will want to configure AutoScaling for your
-EC2 instance group so that your instance count scales with demand.
+EC2 instance group so that your instance count scales with demand. To keep
+your EC2 instances halfway loaded, scaling up to a maximum of 30 instances
+if required:
+
+```ts
+const autoScalingGroup = cluster.addDefaultAutoScalingGroupCapacity({
+  instanceType: new ec2.InstanceType("t2.xlarge"),
+  minCapacity: 3,
+  maxCapacity: 30
+  instanceCount: 3,
+
+  // Give instances 5 minutes to drain running tasks when an instance is
+  // terminated. This is the default, turn this off by specifying 0 or
+  // change the timeout up to 900 seconds.
+  taskDrainTimeSec: 300,
+});
+
+autoScalingGroup.scaleOnCpuUtilization('KeepCpuHalfwayLoaded', {
+  targetUtilizationPercent: 50
+});
+```
+
+See the `@aws-cdk/aws-autoscaling` library for more autoscaling options
+you can configure on your instances.
 
 ### Roadmap
 
-- [ ] Instance AutoScaling
 - [ ] Service Discovery Integration
 - [ ] Private registry authentication

packages/@aws-cdk/aws-ecs/lib/cluster.ts

@@ -3,6 +3,7 @@ import cloudwatch = require ('@aws-cdk/aws-cloudwatch');
 import ec2 = require('@aws-cdk/aws-ec2');
 import iam = require('@aws-cdk/aws-iam');
 import cdk = require('@aws-cdk/cdk');
+import { InstanceDrainHook } from './drain-hook/instance-drain-hook';
 import { cloudformation } from './ecs.generated';
 
 /**
@@ -70,19 +71,23 @@ export class Cluster extends cdk.Construct implements ICluster {
 
   /**
    * Add a default-configured AutoScalingGroup running the ECS-optimized AMI to this Cluster
+   *
+   * Returns the AutoScalingGroup so you can add autoscaling settings to it.
    */
-  public addDefaultAutoScalingGroupCapacity(options: AddDefaultAutoScalingGroupOptions) {
+  public addDefaultAutoScalingGroupCapacity(options: AddDefaultAutoScalingGroupOptions): autoscaling.AutoScalingGroup {
     const autoScalingGroup = new autoscaling.AutoScalingGroup(this, 'DefaultAutoScalingGroup', {
       vpc: this.vpc,
       instanceType: options.instanceType,
       machineImage: new EcsOptimizedAmi(),
       updateType: autoscaling.UpdateType.ReplacingUpdate,
-      minSize: 0,
-      maxSize: options.instanceCount || 1,
-      desiredCapacity: options.instanceCount || 1
+      minSize: options.minCapacity,
+      maxSize: options.maxCapacity,
+      desiredCapacity: options.instanceCount,
     });
 
-    this.addAutoScalingGroupCapacity(autoScalingGroup);
+    this.addAutoScalingGroupCapacity(autoScalingGroup, options);
+
+    return autoScalingGroup;
   }
 
   /**
@@ -118,6 +123,15 @@ export class Cluster extends cdk.Construct implements ICluster {
       "logs:CreateLogStream",
       "logs:PutLogEvents"
     ).addAllResources());
+
+    // 0 disables, otherwise forward to underlying implementation which picks the sane default
+    if (options.taskDrainTimeSeconds !== 0) {
+      new InstanceDrainHook(autoScalingGroup, 'DrainECSHook', {
+        autoScalingGroup,
+        cluster: this,
+        drainTimeSec: options.taskDrainTimeSeconds
+      });
+    }
   }
 
   /**
@@ -291,12 +305,25 @@ export interface AddAutoScalingGroupCapacityOptions {
    * @default false
    */
   containersAccessInstanceRole?: boolean;
+
+  /**
+   * Give tasks this many seconds to complete when instances are being scaled in.
+   *
+   * Task draining adds a Lambda and a Lifecycle hook to your AutoScalingGroup
+   * that will delay instance termination until all ECS tasks have drained from
+   * the instance.
+   *
+   * Set to 0 to disable task draining.
+   *
+   * @default 300
+   */
+  taskDrainTimeSeconds?: number;
 }
 
 /**
  * Properties for adding autoScalingGroup
  */
-export interface AddDefaultAutoScalingGroupOptions {
+export interface AddDefaultAutoScalingGroupOptions extends AddAutoScalingGroupCapacityOptions {
 
   /**
    * The type of EC2 instance to launch into your Autoscaling Group
@@ -309,4 +336,18 @@ export interface AddDefaultAutoScalingGroupOptions {
    * @default 1
    */
   instanceCount?: number;
+
+  /**
+   * Maximum number of instances
+   *
+   * @default Same as instanceCount
+   */
+  maxCapacity?: number;
+
+  /**
+   * Minimum number of instances
+   *
+   * @default Same as instanceCount
+   */
+  minCapacity?: number;
 }