Terraform - Client can't reach instances behind application load balancer

Question

I'm trying to deploy a simple web app to test application load balancing while the application is deployed on 3 instances on 3 different availability zones with Terraform.

In the beginning while I was testing the Terraform script I didn't had any problem accessing the app from the public IPs of the instances. But when I try to put an Application Load Balancer in front of the instances I receive the error message Hmmm… can't reach this page alb-1860663581.eu-central-1.elb.amazonaws.com refused to connect.

Main.tf

#PROVIDERS
provider "aws" {
  access_key = var.aws_access_key
  secret_key = var.aws_secret_key
  region     = var.aws_region
}

#DATA

data "aws_ssm_parameter" "ami" {
  name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2"
}

#RESOURCES

#KEY PAIR

##NETWORKING
resource "aws_vpc" "vpc" {
  cidr_block = "172.32.0.0/16"
  enable_dns_hostnames = true
  enable_dns_support = true
  tags = local.common_tags
}

resource "aws_internet_gateway" "igw" {
  vpc_id = aws_vpc.vpc.id

  tags = local.common_tags
}

resource "aws_subnet" "public_subnet" {
  count                   = var.subnet_count.public
  cidr_block              = var.public_subnet_cidr_blocks[count.index]
  vpc_id                  = aws_vpc.vpc.id
  map_public_ip_on_launch = true
  availability_zone       = var.public_subnet_availability_zones[count.index]
  tags                    = local.common_tags
}

resource "aws_subnet" "private_subnets" {
  count             = var.subnet_count.private
  cidr_block        = var.private_subnet_cidr_blocks[count.index]
  vpc_id            = aws_vpc.vpc.id
  availability_zone = var.private_subnet_availability_zones[count.index]
  tags              = local.common_tags
}

##ROUTING
resource "aws_route_table" "rtb-pub" {
  vpc_id = aws_vpc.vpc.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw.id
  }

  tags = local.common_tags
}

resource "aws_route_table_association" "rta-public" {
  count          = var.subnet_count.public
  subnet_id      = aws_subnet.public_subnet[count.index].id
  route_table_id = aws_route_table.rtb-pub.id
}


resource "aws_route_table" "rtb-priv" {
  vpc_id = aws_vpc.vpc.id

  tags = local.common_tags
}

resource "aws_route_table_association" "rta-priv" {
  count          = var.subnet_count.private
  subnet_id      = aws_subnet.private_subnets[count.index].id
  route_table_id = aws_route_table.rtb-priv.id
}

#SECURITY GROUPS
#PHP-SG

resource "aws_security_group" "php-sg" {
  name   = "php-sg"
  vpc_id = aws_vpc.vpc.id

  # HTTP access from anywhere which also includes the LB's listener port
  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    description = "Allow SSH from my PC"
    from_port   = "22"
    to_port     = "22"
    protocol    = "tcp"
    cidr_blocks = ["${var.my_ip}/32"]
  }

  ingress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  # ingress {
  #   description = "Load balancer listener port"
  #   from_port   = 90
  #   to_port     = 90
  #   protocol    = "tcp"
  #   cidr_blocks = ["0.0.0.0/0"]
  # }

  # outbound internet access
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = local.common_tags
}

resource "aws_security_group" "rds-sg" {
  name   = "rds-sg"
  vpc_id = aws_vpc.vpc.id

  ingress {
    description     = "Allow MySQL traffic from only the php sg"
    from_port       = "3306"
    to_port         = "3306"
    protocol        = "tcp"
    security_groups = [aws_security_group.php-sg.id]
  }

  tags = local.common_tags
}

# APPLICATION LOAD BALANCER RESOURCES#
resource "aws_lb_target_group" "tg" {
  name = "targetGroup"
  port = 80
  target_type = "instance"
  vpc_id = aws_vpc.vpc.id
  protocol = "HTTP"
}

resource "aws_lb_target_group_attachment" "tgatt" {
  count = var.subnet_count.public
  target_group_arn = aws_lb_target_group.tg.arn
  port = 80
  target_id = aws_instance.phpserver[count.index].id
  #availability_zone = var.public_subnet_availability_zones
}

resource "aws_lb" "lb" {
  name = "ALB"
  internal = false
  load_balancer_type = "application"
  security_groups = [ aws_security_group.php-sg.id ]
  subnets = aws_subnet.public_subnet.*.id
}

resource "aws_lb_listener" "listener" {
  load_balancer_arn = aws_lb.lb.arn
  port = "80"
  protocol = "HTTP"

  default_action {
    type = "redirect"

    redirect {
      port = "443"
      protocol = "HTTPS"
      status_code = "HTTP_301"
    }
  }
}

resource "aws_lb_listener_rule" "listener_rule" {
  listener_arn = aws_lb_listener.listener.arn
  priority = 100

  action {
    type = "forward"
    target_group_arn = aws_lb_target_group.tg.arn
  }

  condition {
    path_pattern {
      values = [ "/var/www/html/index.php" ]
    }
  }
}


# INSTANCE #
resource "aws_instance" "phpserver" {
  count                       = var.subnet_count.public
  ami                         = nonsensitive(data.aws_ssm_parameter.ami.value)
  instance_type               = "t3.micro"
  subnet_id                   = aws_subnet.public_subnet[count.index].id
  associate_public_ip_address = true
  vpc_security_group_ids      = [aws_security_group.php-sg.id]
  key_name = "someKeyName"
  user_data = templatefile("${path.module}/startup_script.tftpl", {
    endpoint = aws_db_instance.default.endpoint
    db_name  = aws_db_instance.default.db_name
  })

  tags = local.common_tags

  depends_on = [
    aws_db_instance.default
  ]
}

# RDS Subnet group # 

resource "aws_db_subnet_group" "db_subnet_group" {
  name       = "db_subnet_group"
  subnet_ids = [for subnet in aws_subnet.private_subnets : subnet.id]
}

# RDS #
resource "aws_db_instance" "default" {
  db_subnet_group_name   = "db_subnet_group"
  allocated_storage      = 20
  db_name                = "livanoDB"
  engine                 = "mysql"
  instance_class         = "db.t3.micro"
  username               = var.db_username
  password               = var.db_password
  skip_final_snapshot    = true
  vpc_security_group_ids = [aws_security_group.rds-sg.id]

  tags = local.common_tags
  depends_on = [
    aws_db_subnet_group.db_subnet_group
  ]
}

Variables.tf

variable "aws_access_key" {
  type        = string
  description = "AWS Access key"
  sensitive   = true
}

variable "aws_secret_key" {
  type        = string
  description = "AWS Secret key"
  sensitive   = true
}

variable "db_username" {
  description = "Database administrator username"
  type        = string
  sensitive   = true
}

variable "db_password" {
  description = "Database administrator password"
  type        = string
  sensitive   = true
}

variable "aws_region" {
  type        = string
  description = "AWS Region to use for resources"
  default     = "eu-central-1"
}

variable "my_ip" {
  description = "my IP Address"
  type        = string
  sensitive   = true
}

variable "owner" {
  type        = string
  description = "Owner of the deployed resource in the VPC"
  default     = "livano"
}

variable "public_subnet_cidr_blocks" {
  description = "CIDR blocks for public subnets"
  type        = list(string)
  default     = ["172.32.0.0/19", "172.32.32.0/19", "172.32.64.0/19", "172.32.96.0/19"]
}

variable "private_subnet_cidr_blocks" {
  description = "CIDR blocks for private subnets"
  type        = list(string)
  default     = ["172.32.224.0/19", "172.32.192.0/19", "172.32.160.0/19", "172.32.128.0/19"]
}

variable "private_subnet_availability_zones" {
  description = "list of eu-central-1 availability zones"
  type        = list(string)
  default     = ["eu-central-1a", "eu-central-1b", "eu-central-1c", ]
}

variable "public_subnet_availability_zones" {
  description = "list of eu-central-1 availability zones"
  type        = list(string)
  default     = ["eu-central-1a", "eu-central-1b", "eu-central-1c", ]
}

variable "subnet_count" {
  description = "Number of subnets"
  type        = map(number)
  default = {
    public  = 3,
    private = 2
  }
}

I tried changing the Application LB's listener port to a different one and then whitelist it in the security group. I tried removing the public IPs of the instances and make them all private. Telnet from the instances to the load balancer's DNS is failing in all of the scenarios. Health checks for the servers are Healthy. I'm starting to think that I make some unnoticeable mistake, because I'm facing the same issue when doing the simple AWS tutorial of putting an ELB in front of instances with private ips.

Answer 1

A few things are wrong about your infra here :

• You have only 1 listener on port 80 and its default action is to redirect on port 443. But you don't listene to HTTP(S):443. Try something like :

   resource "aws_lb_listener" "listener-http-80" {
     load_balancer_arn = aws_lb.lb.arn
     port = "80"
     protocol = "HTTP"
  
     default_action {
       type = "redirect"
  
       redirect {
         port = "443"
         protocol = "HTTPS"
         status_code = "HTTP_301"
       }
     }
   }
  
   resource "aws_lb_listener" "listener-https-443" {
     load_balancer_arn = aws_lb.lb.arn
     port = "443"
     protocol = "HTTPS"
     certificate_arn = <Your listener certificate ARN : https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener_certificate>
  
     default_action {
       type = "redirect"
  
       redirect {
         port = "443"
         protocol = "HTTPS"
         status_code = "HTTP_301"
         path = "/var/www/html/index.php"
       }
     }
  
   }
  
   resource "aws_lb_listener_rule" "listener_rule" {
     listener_arn = aws_lb_listener.listener-https-443.arn
     priority = 100
  
     action {
       type = "forward"
       target_group_arn = aws_lb_target_group.tg.arn
     }
  
     condition {
       path_pattern {
         values = [ "/var/www/html/index.php" ]
       }
     }
   }
  

• Also, your private subnets doesn't route trafic outside AWS. The priv routing table has only the local route now. You should add a NAT gateway in the public subnet and add a route to it in the private subnets :

  resource "aws_nat_gateway" "nat" {
     subnet_id         = aws_subnet.public_subnet[0].id
  }
  
  resource "aws_route_table" "rtb-priv" {
    vpc_id = aws_vpc.vpc.id
  
    route {
      cidr_block = "0.0.0.0/0"
      nat_gateway_id = aws_nat_gateway.nat.id
    }
    tags = local.common_tags
  }