Allow resources in a private subnet access access between them and egress to internet

Question

I got a VPC with

• 1 VPC
• 1 private subnet attached to VPC
• 1 NAT gateway
• Security group with egress and ingress rules attached to my VPC
• ACL with egress and ingress rules attached to both VPC and subnet

The resources I'm trying to get connected are:

• 1 Elastic Cache cluster, that got a aws_elasticache_subnet_group attached to the subnet
• 1 CodeBuild instance that got VPC configuration for same VPC and subnet ids
• Fargate instances that already got access to internet through public subnets attached, but should communicate with the Elastic Cache node through the private subnet

When I try to run CodeBuild, it fails to connect to S3 to download build source. This wasn't happening before editing VPC config on the build project. In other words, if I remove vpc and subnet configs from CodeBuild, it will work instantly, but I need connection setup.

Particularly, the error I'm getting is:

dial tcp 52.216.129.171:443: i/o timeout for primary source and source version arn:aws:s3:::blog-us-setup/blog-us-pipe/source_out/8ADWIXv"

I've also set other config like:

• Route table
• Route
• Route table association
• Network ACL

No luck so far. Can't get the first important resource connected to the internet and can't even be sure if it will connect to the Elastic Cache cluster in same private VPC

This is my networking config definition in terraform:

resource "aws_subnet" "audible_blog_resources" {
  vpc_id     = "vpc-xxxxx"
  cidr_block = "10.0.2.0/24"
}

resource "aws_eip" "nat_eip" {
  vpc = true
}

resource "aws_route_table" "private" {
  vpc_id = "vpc-xxxxx"
}

resource "aws_nat_gateway" "nat" {
  allocation_id = aws_eip.nat_eip.id
  subnet_id     = aws_subnet.audible_blog_resources.id
}

resource "aws_route" "private_nat_gateway" {
  route_table_id         = aws_route_table.private.id
  destination_cidr_block = "0.0.0.0/0"
  nat_gateway_id         = aws_nat_gateway.nat.id
}

resource "aws_route_table_association" "private" {
  subnet_id      = aws_subnet.audible_blog_resources.id
  route_table_id = aws_route_table.private.id
}


resource "aws_security_group" "allow_redis_ingress" {
  name        = "allow_redis_ingress"
  description = "Allow Redis inbound traffic"
  vpc_id      = "vpc-xxxxx"

  ingress {
    description      = "Redis"
    from_port        = 6379
    to_port          = 6379
    protocol         = "tcp"
    cidr_blocks      = ["10.0.2.0/24"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
}

resource "aws_network_acl" "main" {
  vpc_id = "vpc-xxxxx"
  subnet_ids = [ aws_subnet.audible_blog_resources.id ]
  egress {
    rule_no    = "200" 
    protocol   = "tcp"
    action     = "allow"
    cidr_block = "0.0.0.0/0"
    from_port  = 0
    to_port    = 65535
  }

  ingress {
    rule_no    = "100" 
    protocol   = "tcp"
    action     = "allow"
    cidr_block = "10.0.2.0/24"
    from_port  = 6379
    to_port    = 6379
  }
}

output "networking_details" {
  value = {
    subnet = {
      arn = aws_subnet.audible_blog_resources.arn
      id = aws_subnet.audible_blog_resources.id
    }
    security_group = {
      arn = aws_security_group.allow_redis_ingress.arn
      id = aws_security_group.allow_redis_ingress.id 
    }
  }
}

Edit 1

Added a public subnet, route table for it and NAT moved to same public subnet:

resource "aws_subnet" "audible_blog_resources" {
  vpc_id     = "vpc-xxxxx"
  cidr_block = "10.0.2.0/24"
}

resource "aws_subnet" "public" {
  vpc_id     = "vpc-xxxxx"
  cidr_block = "10.0.5.0/24"
}

resource "aws_eip" "nat_eip" {
  vpc = true
}

resource "aws_route_table" "private" {
  vpc_id = "vpc-xxxxx"
}

resource "aws_route_table" "public" {
  vpc_id = "vpc-xxxxx"
}

resource "aws_nat_gateway" "nat" {
  allocation_id = aws_eip.nat_eip.id
  subnet_id     = aws_subnet.public.id
}

resource "aws_route" "public_to_internet" {
  route_table_id          = aws_route_table.public.id
  destination_cidr_block  = "0.0.0.0/0"
  gateway_id              = "igw-igw-id"
}

resource "aws_route_table_association" "public" {
  subnet_id      = aws_subnet.public.id
  route_table_id = aws_route_table.public.id
}

resource "aws_route" "private_nat_gateway" {
  route_table_id         = aws_route_table.private.id
  destination_cidr_block = "0.0.0.0/0"
  nat_gateway_id         = aws_nat_gateway.nat.id
}

resource "aws_route_table_association" "private" {
  subnet_id      = aws_subnet.audible_blog_resources.id
  route_table_id = aws_route_table.private.id
}


resource "aws_security_group" "allow_redis_ingress" {
  name        = "allow_redis_ingress"
  description = "Allow Redis inbound traffic"
  vpc_id      = "vpc-xxxxx"

  ingress {
    description      = "Redis"
    from_port        = 6379
    to_port          = 6379
    protocol         = "tcp"
    cidr_blocks      = ["10.0.2.0/24"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
}

resource "aws_network_acl" "main" {
  vpc_id = "vpc-xxxxx"
  subnet_ids = [ aws_subnet.audible_blog_resources.id ]
  egress {
    rule_no    = "200" 
    protocol   = "tcp"
    action     = "allow"
    cidr_block = "0.0.0.0/0"
    from_port  = 0
    to_port    = 65535
  }

  ingress {
    rule_no    = "100" 
    protocol   = "tcp"
    action     = "allow"
    cidr_block = "10.0.2.0/24"
    from_port  = 6379
    to_port    = 6379
  }
}

output "networking_details" {
  value = {
    subnet = {
      arn = aws_subnet.audible_blog_resources.arn
      id = aws_subnet.audible_blog_resources.id
    }
    security_group = {
      arn = aws_security_group.allow_redis_ingress.arn
      id = aws_security_group.allow_redis_ingress.id 
    }
  }
}

Answer 1

Your VPC design is incorrect:

• missing public subnet and internet gateway,
• no route tables for public subnets to internet,
• NAT is misplaced - it must be in public subnet, not private.

Modified version below, but SGs and NACLs may still require further review (I removed it from the code).

resource "aws_vpc" "vpc" {

  cidr_block = "10.0.0.0/16"

}
resource "aws_subnet" "public" {
  vpc_id     = aws_vpc.vpc.id
  cidr_block = "10.0.1.0/24"
}

resource "aws_subnet" "audible_blog_resources" {
  vpc_id     = aws_vpc.vpc.id
  cidr_block = "10.0.2.0/24"
}

resource "aws_internet_gateway" "gw" {
  vpc_id = aws_vpc.vpc.id
}

resource "aws_eip" "nat_eip" {
  vpc = true
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.vpc.id
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.vpc.id
}

resource "aws_nat_gateway" "nat" {
  allocation_id = aws_eip.nat_eip.id
  subnet_id     = aws_subnet.public.id
}

resource "aws_route" "public_to_internet" {
  route_table_id         = aws_route_table.public.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id         = aws_internet_gateway.gw.id
}

resource "aws_route_table_association" "public" {
  subnet_id      = aws_subnet.public.id
  route_table_id = aws_route_table.public.id
}

resource "aws_route" "private_nat_gateway" {
  route_table_id         = aws_route_table.private.id
  destination_cidr_block = "0.0.0.0/0"
  nat_gateway_id         = aws_nat_gateway.nat.id
}

resource "aws_route_table_association" "private" {
  subnet_id      = aws_subnet.audible_blog_resources.id
  route_table_id = aws_route_table.private.id
}


resource "aws_security_group" "allow_redis_ingress" {
  name        = "allow_redis_ingress"
  description = "Allow Redis inbound traffic"
  vpc_id      = aws_vpc.vpc.id

  ingress {
    description      = "Redis"
    from_port        = 6379
    to_port          = 6379
    protocol         = "tcp"
    cidr_blocks      = ["10.0.2.0/24"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
}


output "networking_details" {
  value = {
    subnet = {
      arn = aws_subnet.audible_blog_resources.arn
      id = aws_subnet.audible_blog_resources.id
    }
    security_group = {
      arn = aws_security_group.allow_redis_ingress.arn
      id = aws_security_group.allow_redis_ingress.id 
    }
  }
}