nginx unable to resolve container name when deployed to ECS, works locally

Question

I have a ECS configuration I am launching. However, the nginx frontend container is failing citing:

nginx: [emerg] host not found in upstream “backend”

This works fine locally, but it seems that ECS cannot resolve the docker container names.

I am deploying using ecs-cli. Launch type is EC2. The network mode is bridge.

command:

ecs-cli compose \
--cluster mycluster \
--file docker-compose.yml \
--ecs-params ecs-params.yml service up \
--deployment-min-healthy-percent=50 --force-deployment \
--target-groups targetGroupArn=<load-balancer>,containerName=frontend,containerPort=80 \
--health-check-grace-period 60 \
--role <my-role> \
--timeout 30

ecs-params.yml

task_definition:
  task_role_arn: <my-arn>
  task_execution_role: <my-exec-role>
  services:
    backend:
      essential: true
      mem_reservation: 1024m
    frontend:
      essential: true
      mem_reservation: 1024m

nginx.conf:

events {
  worker_connections 1024;
}

http {
  server_tokens off;
  upstream backend_server {
        server backend:8001;
    }

  server {
    listen 80;
    listen [::]:80;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
        try_files $uri $uri/ /index.html;
    }

    location /api {
      proxy_pass      http://backend_server/api;
    }

  }
}

docker-compose:

version: '3'

services:
  backend:
    image: <backend-image>
    ports:
      - 8001:8001

  frontend:
    image: <frontend-image, built locally with the nginx conf>
    ports:
      - 80:80

Task definition:

{
  "ipcMode": null,
  "executionRoleArn": <exec-role>,
  "containerDefinitions": [
    {
      "dnsSearchDomains": [],
      "environmentFiles": null,
      "logConfiguration": {
        "logDriver": "awslogs",
        "secretOptions": null,
        "options": { <log-options>
        }
      },
      "entryPoint": [],
      "portMappings": [
        {
          "hostPort": 8001,
          "protocol": "tcp",
          "containerPort": 8001
        }
      ],
      "command": [],
      "linuxParameters": {
        "capabilities": {
          "add": null,
          "drop": null
        },
        "sharedMemorySize": null,
        "tmpfs": null,
        "devices": [],
        "maxSwap": null,
        "swappiness": null,
        "initProcessEnabled": null
      },
      "cpu": 0,
      "environment": [
      ],
      "resourceRequirements": null,
      "ulimits": null,
      "dnsServers": [],
      "mountPoints": [],
      "workingDirectory": null,
      "secrets": null,
      "dockerSecurityOptions": [],
      "memory": null,
      "memoryReservation": 1024,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": <backend-image>,
      "startTimeout": null,
      "firelensConfiguration": null,
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": [],
      "hostname": null,
      "extraHosts": [],
      "pseudoTerminal": false,
      "user": null,
      "readonlyRootFilesystem": false,
      "dockerLabels": null,
      "systemControls": null,
      "privileged": false,
      "name": "backend"
    },
    {
      "dnsSearchDomains": [],
      "environmentFiles": null,
      "logConfiguration": {
        "logDriver": "awslogs",
        "secretOptions": null,
        "options": { <log-options>
        }
      },
      "entryPoint": [],
      "portMappings": [
        {
          "hostPort": 80,
          "protocol": "tcp",
          "containerPort": 80
        }
      ],
      "command": [],
      "linuxParameters": {
        "capabilities": {
          "add": null,
          "drop": null
        },
        "sharedMemorySize": null,
        "tmpfs": null,
        "devices": [],
        "maxSwap": null,
        "swappiness": null,
        "initProcessEnabled": null
      },
      "cpu": 0,
      "environment": [],
      "resourceRequirements": null,
      "ulimits": null,
      "dnsServers": [],
      "mountPoints": [],
      "workingDirectory": null,
      "secrets": null,
      "dockerSecurityOptions": [],
      "memory": null,
      "memoryReservation": 1024,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": <frontend-image>,
      "startTimeout": null,
      "firelensConfiguration": null,
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": [],
      "hostname": null,
      "extraHosts": [],
      "pseudoTerminal": false,
      "user": null,
      "readonlyRootFilesystem": false,
      "dockerLabels": null,
      "systemControls": null,
      "privileged": false,
      "name": "frontend"
    }
  ],
  "placementConstraints": [],
  "memory": null,
  "taskRoleArn": <task-role-arn>,
  "compatibilities": [
    "EXTERNAL",
    "EC2"
  ],
  "taskDefinitionArn": <definition>,
  "family": "<my-family>",
  "requiresAttributes": [
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.execution-role-awslogs"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.ecr-auth"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.17"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.task-iam-role"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.execution-role-ecr-pull"
    }
  ],
  "pidMode": null,
  "requiresCompatibilities": [],
  "networkMode": null,
  "runtimePlatform": null,
  "cpu": null,
  "revision": 75,
  "status": "ACTIVE",
  "inferenceAccelerators": null,
  "proxyConfiguration": null,
  "volumes": []
}

Answer 1

@MarkB was correct. I needed to add links. The way to do this was through the docker-compose.yml:

version: '3'

services:
  backend:
    image: <backend-image>
    ports:
      - 8001:8001

  frontend:
    image: <frontend-image, built locally with the nginx conf>
    links: ["backend"]
    ports:
      - 80:80

I also added a healthcheck to the ecs-params.yml to make sure that the backend was starting up before the frontend.

The reason this happens is because the ECS network bridge is a default bridge. Which does not allow containers to be resolved by their name (hence the links). I had assumed it was a user-defined bridge. More info can be found on Dockers documentation.