How can I create an AWS Kinesis Firehose connected to S3 using Terraform?

Question

I'm trying to use the kinesis_firehose_delivery_stream resource to create a Kinesis Firehose with a Direct PUT source, no data transformation, and an extended_s3 destination.

I've modified the code in this example (to remove the lambda function) so now it looks like this:

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

Terraform is able to successfully apply everything but Firehose doesn't seem to be able to write to S3.

Am I missing something in my IAM role? and if so how can I fix it?

Edit

I've updated my terraform file to update the IAM policy, per @Marcin 's answer, to give Firehose permission to write to S3.

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-west-2"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",  
  "Statement":
  [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF

  inline_policy {
    name = "kinesis-s3-inline-policy"
    policy = jsonencode({
      Version = "2012-10-17"
      Statement = [
        {      
          Effect = "Allow",      
          Action = [
            "s3:AbortMultipartUpload",
            "s3:GetBucketLocation",
            "s3:GetObject",
            "s3:ListBucket",
            "s3:ListBucketMultipartUploads",
            "s3:PutObject"
          ]      
          Resource = [        
            "arn:aws:s3:::test-kinesis-destination-bucket",
            "arn:aws:s3:::test-kinesis-destination-bucket/*"            
          ]    
        },
        {
          Effect = "Allow"
          Action = [
            "kinesis:DescribeStream",
            "kinesis:GetShardIterator",
            "kinesis:GetRecords",
            "kinesis:ListShards"
          ]
          Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn
        }
      ]
    })
  }
}

But when I run terraform plan I get the following error:

$ terraform plan
╷
│ Error: Cycle: aws_kinesis_firehose_delivery_stream.kinesis_event_stream, aws_iam_role.firehose_role
│ 
│ 
|

How can I reference Firehose's ARN inside its IAM policy?

Answer 1

To avoid the cycle terraform is complaining about, you can convert your inline_policy into a aws_iam_role_policy instead.

With this approach, you can reference the aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn without the cycle error.

For example

resource "aws_iam_role" "firehose_role" {
  name = "${local.service}-${var.environment}-kinesis-firehose-role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement":
  [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "firehose_role_policy" {
  name = "${local.service}-${var.environment}-kinesis-firehose-inline-policy"
  role = aws_iam_role.firehose_role.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow",
        Action = [
          "s3:AbortMultipartUpload",
          "s3:GetBucketLocation",
          "s3:GetObject",
          "s3:ListBucket",
          "s3:ListBucketMultipartUploads",
          "s3:PutObject"
        ]
        Resource = [
          "arn:aws:s3:::${aws_s3_bucket.bucket.id}",
          "arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
        ]
      },
      {
        Effect = "Allow"
        Action = [
          "kinesis:DescribeStream",
          "kinesis:GetShardIterator",
          "kinesis:GetRecords",
          "kinesis:ListShards"
        ]
        Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn
      }
    ]
  })
}

ref

• Difference between aws_iam_policy and aws_iam_role_policy

Answer 2

@A Poor: But when I run terraform plan I get the following error:

You are in loop:

• aws_kinesis_firehose_delivery_stream is wating aws_iam_role to get aws_iam_role.firehose_role.arn

• aws_iam_role is waiting: aws_kinesis_firehose_delivery_stream to get aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn

So I use: "Resource": "arn:aws:kinesis:ap-southeast-1:${var.account_id}:stream/${var.kinesis_firehose_delivery_stream_name}"

Answer 3

You have created role firehose_role with only trust relatinship, but no actual S3 permissions. Your role should have the following permissions as explained in the docs (you can trim it down if you don't use lambda with kinesis, kms or other services that firehose can use):

{
    "Version": "2012-10-17",  
    "Statement":
    [    
        {      
            "Effect": "Allow",      
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:GetBucketLocation",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads",
                "s3:PutObject"
            ],      
            "Resource": [        
                "arn:aws:s3:::bucket-name",
                "arn:aws:s3:::bucket-name/*"            
            ]    
        },        
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:DescribeStream",
                "kinesis:GetShardIterator",
                "kinesis:GetRecords",
                "kinesis:ListShards"
            ],
            "Resource": "arn:aws:kinesis:region:account-id:stream/stream-name"
        },
        {
           "Effect": "Allow",
           "Action": [
               "kms:Decrypt",
               "kms:GenerateDataKey"
           ],
           "Resource": [
               "arn:aws:kms:region:account-id:key/key-id"           
           ],
           "Condition": {
               "StringEquals": {
                   "kms:ViaService": "s3.region.amazonaws.com"
               },
               "StringLike": {
                   "kms:EncryptionContext:aws:s3:arn": "arn:aws:s3:::bucket-name/prefix*"
               }
           }
        },
        {
           "Effect": "Allow",
           "Action": [
               "logs:PutLogEvents"
           ],
           "Resource": [
               "arn:aws:logs:region:account-id:log-group:log-group-name:log-stream:log-stream-name"
           ]
        },
        {
           "Effect": "Allow", 
           "Action": [
               "lambda:InvokeFunction", 
               "lambda:GetFunctionConfiguration" 
           ],
           "Resource": [
               "arn:aws:lambda:region:account-id:function:function-name:function-version"
           ]
        }
    ]
}