AWS ECS Fargate run task: Essential container in task exited

Question

Goal:

Create an interactive shell within an ECS Fargate container

Problem:

After running a task within the ECS service, the task status immediately goes to STOPPED after Pending and gives the following stopped reason: Essential container in task exited. Since the task is stopped, creating an interactive shell with the aws ecs execute-command is not feasible.

Background:

• Using a custom ECR image for the target container
• Cloudwatch logs show that the ECR image associated entrypoint.sh was successful

Dockerfile:

FROM python:3.9-alpine AS build

ARG TERRAFORM_VERSION=1.0.2
ARG TERRAGRUNT_VERSION=0.31.0
ARG TFLINT_VERSION=0.23.0
ARG TFSEC_VERSION=0.36.11
ARG TFDOCS_VERSION=0.10.1
ARG GIT_CHGLOG_VERSION=0.14.2
ARG SEMTAG_VERSION=0.1.1
ARG GH_VERSION=2.2.0
ARG TFENV_VERSION=2.2.2

ENV VIRTUAL_ENV=/opt/venv
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

WORKDIR /src/

COPY install.sh ./install.sh
COPY requirements.txt ./requirements.txt

RUN chmod u+x ./install.sh \
    && sh ./install.sh

FROM python:3.9-alpine

ENV VIRTUAL_ENV=/opt/venv
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PIP_DISABLE_PIP_VERSION_CHECK=1
ENV PATH="/usr/local/.tfenv/bin:$PATH"

WORKDIR /src/

COPY --from=build /usr/local /usr/local
COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV

ENV PATH="$VIRTUAL_ENV/bin:$VIRTUAL_ENV/lib/python3.9/site-packages:$PATH"

RUN apk update \
    && apk add --virtual .runtime \
    bash \
    git \
    curl \
    jq \
    # needed for bats --pretty formatter
    ncurses \
    openssl \
    grep \
    # needed for pcregrep
    pcre-tools \
    coreutils \
    postgresql-client \
    libgcc \
    libstdc++ \
    ncurses-libs \
    docker \
&& ln -sf python3 /usr/local/bin/python \
&& git config --global advice.detachedHead false \
&& git config --global user.email testing_user@users.noreply.github.com \
&& git config --global user.name testing_user

COPY entrypoint.sh ./entrypoint.sh

ENTRYPOINT ["bash", "entrypoint.sh"]
CMD ["/bin/bash"]

entrypoint.sh:

if [ -n "$ADDITIONAL_PATH" ]; then
    echo "Adding to PATH: $ADDITIONAL_PATH"
    export PATH="$ADDITIONAL_PATH:$PATH"
fi

source $VIRTUAL_ENV/bin/activate
pip install -e /src

echo "done"

Terraform configurations for ECS: (Using this AWS blog post as a reference)

data "aws_caller_identity" "current" {}
data "aws_region" "current" {}

module "vpc" {
  source = "terraform-aws-modules/vpc/aws"

  name = local.mut_id
  cidr = "10.0.0.0/16"
  azs  = ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"]
  enable_dns_hostnames = true

  public_subnets = local.public_subnets

  create_database_subnet_group   = true
  database_dedicated_network_acl = true
  database_inbound_acl_rules = [
    {
      rule_number = 1
      rule_action = "allow"
      from_port   = 5432
      to_port     = 5432
      protocol    = "tcp"
      cidr_block  = local.private_subnets[0]
    }
  ]
  database_subnet_group_name = "metadb"
  database_subnets           = local.database_subnets

  private_subnets               = local.private_subnets
  private_dedicated_network_acl = true
  private_outbound_acl_rules = [
    {
      rule_number = 1
      rule_action = "allow"
      from_port   = 5432
      to_port     = 5432
      protocol    = "tcp"
      cidr_block  = local.database_subnets[0]
    }
  ]

  enable_nat_gateway     = true
  single_nat_gateway     = true
  one_nat_gateway_per_az = false
}

module "ecr_testing_img" {
  source = "github.com/marshall7m/terraform-aws-ecr/modules//ecr-docker-img"

  create_repo = true
  source_path = "${path.module}/../.."
  repo_name   = "${local.mut_id}-integration-testing"
  tag         = "latest"
  trigger_build_paths = [
    "${path.module}/../../Dockerfile",
    "${path.module}/../../entrypoint.sh",
    "${path.module}/../../install.sh"
  ]
}

module "testing_kms" {
  source                           = "github.com/marshall7m/terraform-aws-kms/modules//cmk"
  trusted_admin_arns               = [data.aws_caller_identity.current.arn]
  trusted_service_usage_principals = ["ecs-tasks.amazonaws.com"]
}

module "testing_ecs_task_role" {
  source           = "github.com/marshall7m/terraform-aws-iam/modules//iam-role"
  role_name        = "${local.mut_id}-task"
  trusted_services = ["ecs-tasks.amazonaws.com"]
  statements = [
    {
      effect    = "Allow"
      actions   = ["kms:Decrypt"]
      resources = [module.testing_kms.arn]
    },
    {
      effect = "Allow"
      actions = [
        "ssmmessages:CreateControlChannel",
        "ssmmessages:CreateDataChannel",
        "ssmmessages:OpenControlChannel",
        "ssmmessages:OpenDataChannel"
      ]
      resources = ["*"]
    }
  ]
}

module "testing_ecs_execution_role" {
  source                  = "github.com/marshall7m/terraform-aws-iam/modules//iam-role"
  role_name               = "${local.mut_id}-exec"
  trusted_services        = ["ecs-tasks.amazonaws.com"]
  custom_role_policy_arns = ["arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"]
}

resource "aws_ecs_cluster" "testing" {
  name = "${local.mut_id}-integration-testing"

  configuration {
    execute_command_configuration {
      kms_key_id = module.testing_kms.arn
            logging = "DEFAULT"
    }
  }
}

resource "aws_ecs_service" "testing" {
  name                   = "${local.mut_id}-integration-testing"
  task_definition        = aws_ecs_task_definition.testing.arn
  cluster                = aws_ecs_cluster.testing.id
  desired_count          = 0
  enable_execute_command = true
  launch_type            = "FARGATE"
  platform_version       = "1.4.0"
  network_configuration {
    subnets         = [module.vpc.public_subnets[0]]
    security_groups = [aws_security_group.testing.id]
    assign_public_ip = true
  }
  wait_for_steady_state = true
}

resource "aws_cloudwatch_log_group" "testing" {
  name = "${local.mut_id}-ecs"
}

resource "aws_ecs_task_definition" "testing" {
  family                   = "integration-testing"
  requires_compatibilities = ["FARGATE"]
  task_role_arn            = module.testing_ecs_task_role.role_arn
  execution_role_arn       = module.testing_ecs_execution_role.role_arn
  network_mode             = "awsvpc"
  cpu                      = 256
  memory                   = 512
  container_definitions = jsonencode([{
    name  = "testing"
    image = module.ecr_testing_img.full_image_url
    linuxParameters = {
      initProcessEnabled = true
    }
        logConfiguration = {
            logDriver = "awslogs",
            options = {
                awslogs-group = aws_cloudwatch_log_group.testing.name
                awslogs-region = data.aws_region.current.name
                awslogs-stream-prefix = "testing"
            }
        }
    cpu    = 256
    memory = 512
  }])
  runtime_platform {
    operating_system_family = "LINUX"
    cpu_architecture        = "X86_64"
  }
}

resource "aws_security_group" "testing" {
  name        = "${local.mut_id}-integration-testing-ecs"
  description = "Allows internet access request from testing container"
  vpc_id      = module.vpc.vpc_id

  egress {
    description = "Allows outbound HTTP access for installing packages within container"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    description = "Allows outbound HTTPS access for installing packages within container"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

Snippet of Bash script that runs the ECS task and execute command within container:

task_id=$(aws ecs run-task \
        --cluster "$cluster_arn"  \
        --task-definition "$task_arn" \
        --launch-type FARGATE \
        --platform-version '1.4.0' \
        --enable-execute-command \
        --network-configuration awsvpcConfiguration="{subnets=[$subnet_id],securityGroups=[$sg_id],assignPublicIp=ENABLED}" \
        --region $AWS_REGION | jq -r '.tasks[0].taskArn | split("/") | .[-1]')
    
    echo "Task ID: $task_id"
    
    if [ "$run_ecs_exec_check" == true ]; then
        bash <( curl -Ls https://raw.githubusercontent.com/aws-containers/amazon-ecs-exec-checker/main/check-ecs-exec.sh ) "$cluster_arn" "$task_id"
    fi

    sleep_time=10
    status=""
    echo ""
    echo "Waiting for task to be running"
    while [ "$status" != "RUNNING" ]; do
        echo "Checking status in $sleep_time seconds..."
        sleep $sleep_time

        status=$(aws ecs describe-tasks \
            --cluster "$cluster_arn" \
            --region $AWS_REGION \
            --tasks "$task_id" | jq -r '.tasks[0].containers[0].managedAgents[] | select(.name == "ExecuteCommandAgent") | .lastStatus')

        echo "Status: $status"

        if [ "$status" == "STOPPED" ]; then
            aws ecs describe-tasks \
            --cluster "$cluster_arn" \
            --region $AWS_REGION \
            --tasks "$task_id"
            exit 1
        fi

        # sleep_time=$(( $sleep_time * 2 ))
    done

    echo "Running interactive shell within container"
    
    aws ecs execute-command  \
        --region $AWS_REGION \
        --cluster "$cluster_arn" \
        --task "$task_id" \
        --command "/bin/bash" \
        --interactive

Answer 1

As soon as the last command in your entrypoint.sh finishes, the docker container is going to exit. Just like if you ran the docker container locally. I suggest working on getting a docker container to run locally without exiting first, and then deploying that to ECS.

A command like tail -f /dev/null will work if you just want the container to sit there doing nothing.