AWS VPC peering issue between lambda and ec2

Question

I have a lambda deployed in a private subnet under VPC #1 and i have a mongodb hosted in an EC2 udner VPC #2. When I try to trigger the lambda through api gateway it successfully goes through my ALB but when ALB sends the request to the target group which is the lambda in VPC #1, it always return a socket timeout. I already set the route tables for each VPC and allowed inbound rules for the EC2 but to still can't establish a connection between them.

These are the TF files i used to deploy the services:

VPC:

resource "aws_vpc" "aws_backend_vpc" {
  cidr_block       = var.vpc_cidr_block
  enable_dns_support = true
  enable_dns_hostnames = true

  tags = {
    Name        = "aws-backend-vpc"
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_subnet" "aws_backend_private_subnet1" {
  vpc_id            = aws_vpc.aws_backend_vpc.id
  cidr_block        = var.private_subnet1_cidr_block
  availability_zone = var.private_subnet1_az
  map_public_ip_on_launch = false

  tags = {
    Name        = "aws-backend-private-subnet-1"
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_subnet" "aws_backend_private_subnet2" {
  vpc_id            = aws_vpc.aws_backend_vpc.id
  cidr_block        = var.private_subnet2_cidr_block
  availability_zone = var.private_subnet2_az
  map_public_ip_on_launch = false

  tags = {
    Name        = "aws-backend-private-subnet-2"
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_subnet" "aws_backend_public_subnet1" {
  vpc_id            = aws_vpc.aws_backend_vpc.id
  cidr_block        = var.public_subnet1_cidr_block
  availability_zone = var.public_subnet1_az
  map_public_ip_on_launch = false

  tags = {
    Name        = "aws-backend-public-subnet-1"
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_subnet" "aws_backend_public_subnet2" {
  vpc_id            = aws_vpc.aws_backend_vpc.id
  cidr_block        = var.public_subnet2_cidr_block
  availability_zone = var.public_subnet2_az
  map_public_ip_on_launch = false

  tags = {
    Name        = "aws-backend-public-subnet-2"
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_security_group" "aws_backend_security_group1" {
  name        = "aws-backend-sg-1"
  description = "Enable access to ALB"
  vpc_id      = aws_vpc.aws_backend_vpc.id

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_security_group" "aws_backend_security_group2" {
  name        = "aws-backend-sg-2"
  description = "Enable access to Lambda"
  vpc_id      = aws_vpc.aws_backend_vpc.id

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_vpc_endpoint" "aws_backend_vpc_endpoint" {
  vpc_id              = aws_vpc.aws_backend_vpc.id
  vpc_endpoint_type   = "Interface"
  service_name        = "com.amazonaws.${var.aws_region}.execute-api"
  private_dns_enabled = true
  subnet_ids          = [
    aws_subnet.aws_backend_public_subnet1.id, 
    aws_subnet.aws_backend_public_subnet2.id
  ]
  security_group_ids  = [
    aws_security_group.aws_backend_security_group1.id
  ]
}

resource "aws_vpc_peering_connection" "aws_backend_vpc_peering_connection" {
  vpc_id        = aws_vpc.aws_backend_vpc.id
  peer_vpc_id   = var.vpc_id_to_peer

  tags = {
    Name = "aws-backend-vpc-peering"
  }
}

resource "aws_route_table" "aws_backend_vpc_route_table" {
  vpc_id = aws_vpc.aws_backend_vpc.id

  tags = {
    Name = "aws-backend-vpc-route-table"
  }
}

resource "aws_route" "aws_backend_vpc_route" {
  route_table_id              = aws_route_table.aws_backend_vpc_route_table.id
  destination_cidr_block      = var.cidr_block_of_vpc_to_peer
  vpc_peering_connection_id   = aws_vpc_peering_connection.aws_backend_vpc_peering_connection.id
}

resource "aws_route_table_association" "aws_backend_private_subnet1_association" {
  subnet_id      = aws_subnet.aws_backend_private_subnet1.id
  route_table_id = aws_route_table.aws_backend_vpc_route_table.id
}

resource "aws_route_table_association" "aws_backend_private_subnet2_association" {
  subnet_id      = aws_subnet.aws_backend_private_subnet2.id
  route_table_id = aws_route_table.aws_backend_vpc_route_table.id
}

ALB:

resource "aws_lb" "aws_backend_load_balancer" {
  name               = "aws-backend-lb"
  internal           = true
  security_groups    = [
    var.aws_backend_security_group1_id
    ]
  subnets            = [
    var.aws_backend_private_subnet1_id,
    var.aws_backend_private_subnet2_id,
    var.aws_backend_public_subnet1_id,
    var.aws_backend_public_subnet2_id,
  ]
  tags = {
    Name       = "aws-backend-lb"
    CostCenter = var.cost_center_tag
    Environment = var.environment_tag
  }
}

resource "aws_lb_target_group" "aws_backend_load_balancer_target_group1" {
  name     = "aws-backend-lb-target-group-1"
  port     = 80
  protocol = "HTTP"
  target_type = "lambda"
}

resource "aws_lb_target_group_attachment" "lambda_attachment" {
  target_group_arn = aws_lb_target_group.aws_backend_load_balancer_target_group1.arn
  target_id        = var.mongodb_roa_function_arn
}

resource "aws_lb_listener" "aws_backend_load_balancer_listener" {
  load_balancer_arn = aws_lb.aws_backend_load_balancer.arn
  port              = 80
  protocol          = "HTTP"

  default_action {
    type             = "fixed-response"
    fixed_response {
      content_type    = "application/json"
      status_code     = "404"
      message_body    = "The URL you requested could not be found."
    }
  }
}

resource "aws_lb_listener_rule" "aws_backend_listener_rule1" {
  listener_arn = aws_lb_listener.aws_backend_load_balancer_listener.arn
  priority     = 1

  action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.aws_backend_load_balancer_target_group1.arn
  }

  condition {
    path_pattern {
      values = ["/${var.aws_environment}/${var.path_part}"]
    }
  }
}

LAMBDA

resource "aws_iam_role" "mongodb_roa_function_role" {
  name = "MongoDbRoaFunctionRole"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Service = "lambda.amazonaws.com"
        }
        Action = "sts:AssumeRole"
      }
    ]
  })

  managed_policy_arns = [
    "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
    "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole",
  ]
}

resource "aws_lambda_function" "mongodb_roa_function" {
  function_name    = var.lambda_function_name
  handler          = "app.lambda_handler"
  runtime          = "python3.9"
  timeout          = 60
  role             = aws_iam_role.mongodb_roa_function_role.arn
  filename         = "${path.module}/code/${var.lambda_function_name}.zip"

  environment {
    variables = {
      MONGODB_URI   = var.mongodb_url
      DATABASE_NAME = var.mongodb_name
    }
  }

  tags = {
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
    Project     = var.project_tag
  }

  vpc_config {
    security_group_ids = [
      var.aws_backend_security_group2_id
    ]
    subnet_ids = [
      var.aws_backend_private_subnet1_id,
      var.aws_backend_private_subnet2_id
    ]
  }
}

resource "aws_cloudwatch_log_group" "mongodb_roa_function_log_group" {
  name = "/aws/lambda/${aws_lambda_function.mongodb_roa_function.function_name}"
}

resource "aws_lambda_permission" "mongodb_roa_function_invoke_permission" {
  statement_id  = "AllowExecutionFromELB"
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.mongodb_roa_function.arn
  principal     = "elasticloadbalancing.amazonaws.com"
}

API GATEWAY

resource "aws_apigatewayv2_vpc_link" "mongodb_roa_vpc_link" {
  name = "mongodb-roa-vpc-link"
  security_group_ids = [
    var.aws_backend_security_group1_id
  ]
  subnet_ids = [
    var.aws_backend_public_subnet1_id,
    var.aws_backend_public_subnet2_id
  ]
}

resource "aws_apigatewayv2_api" "mongodb_roa_api" {
  name          = "mongodb-roa-api"
  protocol_type = "HTTP"

  tags = {
    CostCenter  = var.cost_center_tag
    Environment = var.environment_tag
    Project     = var.project_tag
  }
}

resource "aws_cloudwatch_log_group" "mongodb_roa_api_log_group" {
  name = "/aws/api-gateway/${aws_apigatewayv2_api.mongodb_roa_api.name}"
}

resource "aws_apigatewayv2_stage" "mongodb_roa_api_stage" {
  api_id      = aws_apigatewayv2_api.mongodb_roa_api.id
  auto_deploy = true
  name        = var.aws_environment

  access_log_settings {
    destination_arn = aws_cloudwatch_log_group.mongodb_roa_api_log_group.arn
    format          = "{ \"requestId\": \"$context.requestId\", \"ip\": \"$context.identity.sourceIp\", \"requestTime\": \"$context.requestTime\", \"httpMethod\": \"$context.httpMethod\", \"routeKey\": \"$context.routeKey\", \"status\": \"$context.status\", \"protocol\": \"$context.protocol\", \"responseLength\": \"$context.responseLength\" }"
  }
}

resource "aws_apigatewayv2_integration" "mongodb_roa_api_integration" {
  api_id                 = aws_apigatewayv2_api.mongodb_roa_api.id
  connection_type        = "VPC_LINK"
  connection_id          = aws_apigatewayv2_vpc_link.mongodb_roa_vpc_link.id
  integration_method     = "GET"
  integration_type       = "HTTP_PROXY"
  integration_uri        = var.aws_backend_load_balancer_listener_id
  payload_format_version = "1.0"
}

resource "aws_apigatewayv2_route" "mongodb_roa_api_route" {
  api_id             = aws_apigatewayv2_api.mongodb_roa_api.id
  route_key          = "GET /${var.path_part}"
  target             = "integrations/${aws_apigatewayv2_integration.mongodb_roa_api_integration.id}"
  authorization_type = "AWS_IAM"
}

resource "aws_apigatewayv2_deployment" "mongodb_roa_api_deployment" {
  api_id = aws_apigatewayv2_api.mongodb_roa_api.id

  depends_on = [
    aws_apigatewayv2_route.mongodb_roa_api_route
  ]
}

I have enabled all inbound and outbound rules as all traffic but still no luck.