AWS ECS EC2 with CodeDeploy: Task Placement Failure During Blue/Green Deployment

Question

I'm working on a Terraform infrastructure using ECS EC2, ECR, and CodeDeploy for zero-downtime deployments. Here's my current setup and issue: Current Setup:

ECS cluster running on EC2 instances Using CodeDeploy for blue/green deployments Terraform manages the entire infrastructure

Problem: When I trigger a CodeDeploy deployment using AWS CLI, the blue/green deployment works, but it creates a new task and runs it on the same existing EC2 instance. This forces me to oversize the EC2 instances to handle two tasks running simultaneously during deployment.

Desired Behavior: I want CodeDeploy to:

Spin up a new EC2 instance

Run the new task on this new instance

Wait for the new instance's health check to pass

Terminate the original task

Terminate the original EC2 instance

Question: Is there a way to achieve this behavior using Terraform? How can I configure CodeDeploy or ECS to ensure a new EC2 instance is created for each new deployment?

Here is all my terraform code.

# alb
resource "aws_alb" "demo-app_alb" {
  name               = "demo-app-devt-alb"
  load_balancer_type = "application"
  subnets            = [var.public_subnet_1a.id, var.public_subnet_1b.id]
  security_groups    = [var.alb_sn.id]
}

# Modify the ALB listener to use blue target group by default
resource "aws_alb_listener" "alb_listener_demo-app" {
  load_balancer_arn = aws_alb.demo-app_alb.arn
  port              = "80"
  protocol          = "HTTP"

  default_action {
    type             = "forward"
    target_group_arn = aws_alb_target_group.blue.arn
  }

  lifecycle {
    ignore_changes = [ default_action ]
  }
}

output "alb_listener_demo-app" {
  value = aws_alb_listener.alb_listener_demo-app
}

# Create two target groups for blue-green deployment
resource "aws_alb_target_group" "blue" {
  name_prefix = "blue-"
  vpc_id      = var.vpc_id
  protocol    = "HTTP"
  port        = 8000
  target_type = "ip"

  health_check {
    enabled             = true
    path                = "/"
    port                = 8000
    matcher             = 200
    interval            = 30
    timeout             = 5
    healthy_threshold   = 2
    unhealthy_threshold = 3
  }

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_alb_target_group" "green" {
  name_prefix = "green-"
  vpc_id      = var.vpc_id
  protocol    = "HTTP"
  port        = 8000
  target_type = "ip"

  health_check {
    enabled             = true
    path                = "/"
    port                = 8000
    matcher             = 200
    interval            = 30
    timeout             = 5
    healthy_threshold   = 2
    unhealthy_threshold = 3
  }

  lifecycle {
    create_before_destroy = true
  }
}

# ecs & codedeploy

resource "aws_autoscaling_policy" "codedeploy_scaling_policy" {
  name = "CodeDeployDeploymentScalingPolicy"
  autoscaling_group_name = aws_autoscaling_group.ecs_asg.name

  policy_type            = "TargetTrackingScaling"

  target_tracking_configuration {
    predefined_metric_specification {
      predefined_metric_type = "ASGAverageCPUUtilization"
    }
    target_value = 75.0
  }

  estimated_instance_warmup = 300
}

resource "aws_autoscaling_group" "ecs_asg" {
  name_prefix         = "demo-app-ecs-asg-devt-"
  vpc_zone_identifier = [var.ecs_asg_subnet.id, var.public_subnet_1b.id]
  desired_capacity    = 1
  max_size            = 2
  min_size            = 1

  launch_template {
    id      = aws_launch_template.demo-app_ecs_ec2_devt_lt.id
    version = "$Latest"
  }

  lifecycle {
    create_before_destroy = true
  }

  tag {
    key                 = "Name"
    value               = "demo-app-ecs-cluster-devt"
    propagate_at_launch = true
  }

  tag {
    key                 = "AmazonECSManaged"
    value               = true
    propagate_at_launch = true
  }
}

resource "aws_ecs_capacity_provider" "ecs_cp" {
  name = "demo-app-ecs-cp-devt"

  auto_scaling_group_provider {
    auto_scaling_group_arn = aws_autoscaling_group.ecs_asg.arn
    managed_scaling {
      maximum_scaling_step_size = 2
      minimum_scaling_step_size = 1
      status                    = "ENABLED"
      target_capacity           = 100 # NEW
    }
  }
}

resource "aws_ecs_cluster_capacity_providers" "ecs_ccp" {
  cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name

  capacity_providers = [aws_ecs_capacity_provider.ecs_cp.name]
}

resource "aws_ecs_cluster" "demo-app_ecs_cluster" {
  name = "DEMO-APP_ECS_CLUSTER_DEVT"

  tags = {
    Name = "DEMO-APP_ECS_CLUSTER_DEVT"
  }
}

data "aws_iam_policy_document" "assume_by_codedeploy" {
  statement {
    sid     = ""
    effect  = "Allow"
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["codedeploy.amazonaws.com"]
    }
  }
}

resource "aws_iam_role" "codedeploy" {
  name               = "codedeploy"
  assume_role_policy = data.aws_iam_policy_document.assume_by_codedeploy.json
}

data "aws_iam_policy_document" "codedeploy" {
  statement {
    sid    = "AllowLoadBalancingAndECSModifications"
    effect = "Allow"

    actions = [
      "ecs:CreateTaskSet",
      "ecs:DeleteTaskSet",
      "ecs:DescribeServices",
      "ecs:UpdateServicePrimaryTaskSet",
      "elasticloadbalancing:DescribeListeners",
      "elasticloadbalancing:DescribeRules",
      "elasticloadbalancing:DescribeTargetGroups",
      "elasticloadbalancing:ModifyListener",
      "elasticloadbalancing:ModifyRule",
      "s3:GetObject"
    ]

    resources = ["*"]
  }
  statement {
    sid    = "AllowPassRole"
    effect = "Allow"

    actions = ["iam:PassRole"]

    resources = [
      aws_iam_role.codedeploy.arn,
      "arn:aws:iam::088342693028:role/DEMO-APP_ECS_TaskExecutionRole",
      "arn:aws:iam::088342693028:role/*" 
    ]
  }

  statement {
    sid    = "DeployService"
    effect = "Allow"

    actions = [
      "ecs:DescribeServices",
      "ecs:CreateTaskSet",
      "ecs:UpdateServicePrimaryTaskSet",
      "ecs:DeleteTaskSet",
      "codedeploy:GetDeploymentGroup",
      "codedeploy:CreateDeployment",
      "codedeploy:GetDeployment",
      "codedeploy:GetDeploymentConfig",
      "codedeploy:RegisterApplicationRevision",
      "codedeploy:GetApplicationRevision"
    ]

    resources = ["*"]
  }
}

resource "aws_iam_role_policy" "codedeploy" {
  role   = aws_iam_role.codedeploy.name
  policy = data.aws_iam_policy_document.codedeploy.json
}

resource "aws_codedeploy_app" "ecs_app" {
  compute_platform = "ECS"
  name             = "demo-app-ecs-app"
}

# CodeDeploy Deployment Group
resource "aws_codedeploy_deployment_group" "ecs_dg" {
  app_name               = aws_codedeploy_app.ecs_app.name
  deployment_group_name  = "demo-app-ecs-dg"
  service_role_arn       = aws_iam_role.codedeploy.arn
  deployment_config_name = "CodeDeployDefault.ECSAllAtOnce"

  depends_on = [ aws_ecs_service.demo-app_ecs_service ]
  
  ecs_service {
    cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name
    service_name = aws_ecs_service.demo-app_ecs_service.name
  }

  deployment_style {
    deployment_option = "WITH_TRAFFIC_CONTROL"
    deployment_type   = "BLUE_GREEN"
  }

  auto_rollback_configuration {
    enabled = true
    events  = ["DEPLOYMENT_FAILURE"]
  }

  blue_green_deployment_config {
    deployment_ready_option {
      action_on_timeout = "CONTINUE_DEPLOYMENT"
    }

    terminate_blue_instances_on_deployment_success {
      action                           = "TERMINATE"
      termination_wait_time_in_minutes = 2
    }
  }

  load_balancer_info {
    target_group_pair_info {
      prod_traffic_route {
        listener_arns = [var.alb_listener_demo-app.arn]
      }

      target_group {
        name = var.blue_target_group.name
      }

      target_group {
        name = var.green_target_group.name
      }
    }
  }
}

data "aws_ami" "amazon_linux_2" {
  most_recent = true

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }

  filter {
    name   = "owner-alias"
    values = ["amazon"]
  }

  filter {
    name   = "name"
    values = ["amzn2-ami-ecs-hvm-*-x86_64-ebs"]
  }

  owners = ["amazon"]
}

# Data source to fetch the existing key pair
data "aws_key_pair" "demo-app_terraform_dev" {
  key_name = "demo-app-terraform-dev"
}

resource "aws_launch_template" "demo-app_ecs_ec2_devt_lt" {
  name_prefix            = "demo-app-ecs-ec2-devt-lt-"
  description            = "launch template for demo-app devt"
  image_id               = data.aws_ami.amazon_linux_2.image_id
  instance_type          = "t2.large"
  vpc_security_group_ids = [var.ecs_node_sg.id]

  iam_instance_profile {
    arn = var.ecs_node_instance_role_profile.arn
  }

  # Reference the key pair here
  key_name = data.aws_key_pair.demo-app_terraform_dev.key_name

  user_data = base64encode(templatefile("${path.module}/user_data.tftpl", {
    ecs_cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name
  }))

  monitoring {
    enabled = true
  }
}

data "aws_ecs_task_definition" "latest_task" {
  task_definition = aws_ecs_task_definition.default.family
}

resource "aws_ecs_service" "demo-app_ecs_service" {
  name            = "demo-app-ecs-devt-service"
  cluster         = aws_ecs_cluster.demo-app_ecs_cluster.id
  task_definition = "${aws_ecs_task_definition.default.family}:${max(aws_ecs_task_definition.default.revision, data.aws_ecs_task_definition.latest_task.revision)}"
  deployment_maximum_percent         = 200
  deployment_minimum_healthy_percent = 100
  desired_count   = 1

  tags = {
    Name = "DEMO-APP_ECS_Service"
  }

  deployment_controller {
    type = "CODE_DEPLOY"
  }

  force_new_deployment = true

  load_balancer {
    target_group_arn = var.blue_target_group.arn
    container_name   = var.service_name 
    container_port   = var.container_port // 8000
  }

  network_configuration {
    subnets         = [var.ecs_asg_subnet.id]
    security_groups = [var.ecs_node_sg.id]
    assign_public_ip = false
  }

  lifecycle {
    ignore_changes = [task_definition, load_balancer, desired_count]
  }

  capacity_provider_strategy {
    capacity_provider = aws_ecs_capacity_provider.ecs_cp.name
    weight            = 100
  }
}

resource "aws_cloudwatch_log_group" "log_group" {
  name              = "/ecs/demo-app-backend-ec2-terraform-devt"
  retention_in_days = 7
}

module "ecr" {
  source = "../ecr"
}

data "aws_ecr_image" "service_image" {
  repository_name = "mock-demo-app-backend-dev"
  image_tag       = "latest"
}

resource "aws_ecs_task_definition" "default" {
  family                   = "DEMO-APP_ECS_Devt_TaskDefinition"
  execution_role_arn       = var.ecs_task_execution_role.arn
  task_role_arn            = var.ecs_task_iam_role.arn
  requires_compatibilities = ["EC2"]
  network_mode = "awsvpc"
  cpu       = 1024
  memory    = 4096

  runtime_platform {
    operating_system_family = "LINUX"
    cpu_architecture        = "X86_64"
  }

  container_definitions = jsonencode([
    {
      name      = "demo-app-backend-app-devt"
      image = "${var.ecs_container_repo.repository_url}@${data.aws_ecr_image.service_image.image_digest}"
      cpu       = 1024
      memory    = 4096
      essential = true
      portMappings = [
        {
          containerPort = 8000
          hostPort      = 8000
          protocol      = "tcp"
          appProtocol   = "http"
        }
      ]
   
      logConfiguration = {
        logDriver = "awslogs",
        options = {
          "awslogs-region"        = "ap-southeast-1",
          "awslogs-group"         = aws_cloudwatch_log_group.log_group.name,
          "awslogs-stream-prefix" = "ecs"
        }
      }
    }
  ])
}