From 93e906c401b2f67bb8fdfca02d5188db56444b1f Mon Sep 17 00:00:00 2001 From: Brian Price Date: Mon, 4 Sep 2023 09:22:32 -0500 Subject: [PATCH 1/5] Fixing management of policies to prevent constant adding and deleteting. --- modules/firehose/main.tf | 156 ++++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 59 deletions(-) diff --git a/modules/firehose/main.tf b/modules/firehose/main.tf index 7c337b2e..a484f006 100644 --- a/modules/firehose/main.tf +++ b/modules/firehose/main.tf @@ -79,13 +79,13 @@ resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" { } resource "aws_iam_role" "firehose_to_coralogix" { - tags = local.tags - name = "${var.firehose_stream}-firehose" + tags = local.tags + name = "${var.firehose_stream}-firehose" assume_role_policy = jsonencode({ - "Version" = "2012-10-17", + "Version" = "2012-10-17", "Statement" = [ { - "Action" = "sts:AssumeRole", + "Action" = "sts:AssumeRole", "Principal" = { "Service" = "firehose.amazonaws.com" }, @@ -94,9 +94,9 @@ resource "aws_iam_role" "firehose_to_coralogix" { ] }) inline_policy { - name = "${var.firehose_stream}-firehose" + name = "${var.firehose_stream}-firehose" policy = jsonencode({ - "Version" = "2012-10-17", + "Version" = "2012-10-17", "Statement" = [ { "Effect" = "Allow", @@ -137,6 +137,7 @@ resource "aws_iam_role" "firehose_to_coralogix" { } } + ################################################################################ # Firehose Logs Stream ################################################################################ @@ -148,7 +149,9 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" { count = var.logs_enable == true ? 1 : 0 dynamic "kinesis_source_configuration" { - for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [1] : [] + for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [ + 1 + ] : [] content { kinesis_stream_arn = var.kinesis_stream_arn role_arn = aws_iam_role.firehose_to_coralogix.arn @@ -239,6 +242,93 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" { # Firehose Metrics Stream ################################################################################ +data "aws_iam_policy_document" "firehose_to_coralogix_metric_policy" { + statement { + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ] + resources = [ + aws_s3_bucket.firehose_bucket.arn, + "${aws_s3_bucket.firehose_bucket.arn}/*" + ] + } + statement { + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey" + ] + resources = [ + "arn:aws:kms:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:key/key-id" + ] + condition { + test = "StringEquals" + values = ["s3.${data.aws_region.current_region.name}.amazonaws.com"] + variable = "kms:ViaService" + } + condition { + test = "StringLike" + values = ["${aws_s3_bucket.firehose_bucket.arn}/prefix*"] + variable = "kms:EncryptionContext:aws:s3:arn" + } + + } + statement { + effect = "Allow" + + actions = [ + "kinesis:DescribeStream", + "kinesis:GetShardIterator", + "kinesis:GetRecords", + "kinesis:ListShards" + ] + resources = [ + "arn:aws:kinesis:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:stream/*" + ] + } + statement { + effect = "Allow" + + actions = [ + "logs:PutLogEvents" + ] + resources = [ + aws_cloudwatch_log_group.firehose_loggroup.arn + ] + } + statement { + effect = "Allow" + + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration" + ] + resources = [ + "${aws_lambda_function.lambda_processor[count.index].arn}:*" + ] + } +} + +resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" { + count = var.metric_enable == true ? 1 : 0 + name = "Coralogix-firehose_metric_policy" + policy = data.aws_iam_policy_document.firehose_to_coralogix_metric_policy.json +} + +resource "aws_iam_role_policy_attachment" "firehose_to_coralogix_metric_policy" { + count = var.metric_enable == true ? 1 : 0 + policy_arn = aws_iam_policy.firehose_to_coralogix_metric_policy[count.index].arn + role = aws_iam_role.firehose_to_coralogix.name +} + resource "aws_iam_role_policy" "firehose_to_coralogix_metric_policy" { count = var.metric_enable == true ? 1 : 0 name = "${var.firehose_stream}-metrics-addon" @@ -248,58 +338,6 @@ resource "aws_iam_role_policy" "firehose_to_coralogix_metric_policy" { "Version": "2012-10-17", "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject" - ], - "Resource": [ - "${aws_s3_bucket.firehose_bucket.arn}", - "${aws_s3_bucket.firehose_bucket.arn}/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "kms:Decrypt", - "kms:GenerateDataKey" - ], - "Resource": [ - "arn:aws:kms:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:key/key-id" - ], - "Condition": { - "StringEquals": { - "kms:ViaService": "s3.${data.aws_region.current_region.name}.amazonaws.com" - }, - "StringLike": { - "kms:EncryptionContext:aws:s3:arn": "${aws_s3_bucket.firehose_bucket.arn}/prefix*" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "kinesis:DescribeStream", - "kinesis:GetShardIterator", - "kinesis:GetRecords", - "kinesis:ListShards" - ], - "Resource": "arn:aws:kinesis:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:stream/*" - }, - { - "Effect": "Allow", - "Action": [ - "logs:PutLogEvents" - ], - "Resource": [ - "${aws_cloudwatch_log_group.firehose_loggroup.arn}" - ] - }, { "Effect": "Allow", "Action": [ From 5579bff3af51cc7ee982e9dd0d4af605754bd35d Mon Sep 17 00:00:00 2001 From: Brian Price Date: Mon, 4 Sep 2023 09:29:16 -0500 Subject: [PATCH 2/5] Fixing tags and policies to prevent endless updates each time Terraform is ran. --- modules/firehose/main.tf | 149 ++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 89 deletions(-) diff --git a/modules/firehose/main.tf b/modules/firehose/main.tf index a484f006..86849d89 100644 --- a/modules/firehose/main.tf +++ b/modules/firehose/main.tf @@ -32,7 +32,7 @@ locals { terraform-module = "kinesis-firehose-to-coralogix" terraform-module-version = "v0.1.0" managed-by = "coralogix-terraform" - custom_endpoint = var.coralogix_firehose_custom_endpoint != null ? var.coralogix_firehose_custom_endpoint : "" + custom_endpoint = var.coralogix_firehose_custom_endpoint != null ? var.coralogix_firehose_custom_endpoint : "_default_" }) # default namings @@ -242,102 +242,67 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" { # Firehose Metrics Stream ################################################################################ -data "aws_iam_policy_document" "firehose_to_coralogix_metric_policy" { - statement { - effect = "Allow" - - actions = [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject" - ] - resources = [ - aws_s3_bucket.firehose_bucket.arn, - "${aws_s3_bucket.firehose_bucket.arn}/*" - ] - } - statement { - effect = "Allow" - - actions = [ - "kms:Decrypt", - "kms:GenerateDataKey" - ] - resources = [ - "arn:aws:kms:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:key/key-id" - ] - condition { - test = "StringEquals" - values = ["s3.${data.aws_region.current_region.name}.amazonaws.com"] - variable = "kms:ViaService" - } - condition { - test = "StringLike" - values = ["${aws_s3_bucket.firehose_bucket.arn}/prefix*"] - variable = "kms:EncryptionContext:aws:s3:arn" - } - - } - statement { - effect = "Allow" - - actions = [ - "kinesis:DescribeStream", - "kinesis:GetShardIterator", - "kinesis:GetRecords", - "kinesis:ListShards" - ] - resources = [ - "arn:aws:kinesis:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:stream/*" - ] - } - statement { - effect = "Allow" - - actions = [ - "logs:PutLogEvents" - ] - resources = [ - aws_cloudwatch_log_group.firehose_loggroup.arn - ] - } - statement { - effect = "Allow" - - actions = [ - "lambda:InvokeFunction", - "lambda:GetFunctionConfiguration" - ] - resources = [ - "${aws_lambda_function.lambda_processor[count.index].arn}:*" - ] - } -} - resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" { count = var.metric_enable == true ? 1 : 0 name = "Coralogix-firehose_metric_policy" - policy = data.aws_iam_policy_document.firehose_to_coralogix_metric_policy.json -} - -resource "aws_iam_role_policy_attachment" "firehose_to_coralogix_metric_policy" { - count = var.metric_enable == true ? 1 : 0 - policy_arn = aws_iam_policy.firehose_to_coralogix_metric_policy[count.index].arn - role = aws_iam_role.firehose_to_coralogix.name -} - -resource "aws_iam_role_policy" "firehose_to_coralogix_metric_policy" { - count = var.metric_enable == true ? 1 : 0 - name = "${var.firehose_stream}-metrics-addon" - role = aws_iam_role.firehose_to_coralogix.id + tags = local.tags policy = < Date: Tue, 5 Sep 2023 10:58:55 +0800 Subject: [PATCH 3/5] fmt --- modules/firehose/main.tf | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/firehose/main.tf b/modules/firehose/main.tf index 86849d89..780249b3 100644 --- a/modules/firehose/main.tf +++ b/modules/firehose/main.tf @@ -79,13 +79,13 @@ resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" { } resource "aws_iam_role" "firehose_to_coralogix" { - tags = local.tags - name = "${var.firehose_stream}-firehose" + tags = local.tags + name = "${var.firehose_stream}-firehose" assume_role_policy = jsonencode({ - "Version" = "2012-10-17", + "Version" = "2012-10-17", "Statement" = [ { - "Action" = "sts:AssumeRole", + "Action" = "sts:AssumeRole", "Principal" = { "Service" = "firehose.amazonaws.com" }, @@ -94,9 +94,9 @@ resource "aws_iam_role" "firehose_to_coralogix" { ] }) inline_policy { - name = "${var.firehose_stream}-firehose" + name = "${var.firehose_stream}-firehose" policy = jsonencode({ - "Version" = "2012-10-17", + "Version" = "2012-10-17", "Statement" = [ { "Effect" = "Allow", @@ -149,9 +149,7 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" { count = var.logs_enable == true ? 1 : 0 dynamic "kinesis_source_configuration" { - for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [ - 1 - ] : [] + for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [1] : [] content { kinesis_stream_arn = var.kinesis_stream_arn role_arn = aws_iam_role.firehose_to_coralogix.arn From e797c5ff7a14905ead7e59e17a32d4876d2aa6f5 Mon Sep 17 00:00:00 2001 From: Ryan Tan Date: Tue, 5 Sep 2023 11:02:39 +0800 Subject: [PATCH 4/5] revert back to $firehose_stream usage --- modules/firehose/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/firehose/main.tf b/modules/firehose/main.tf index 780249b3..9631fadb 100644 --- a/modules/firehose/main.tf +++ b/modules/firehose/main.tf @@ -242,7 +242,7 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" { resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" { count = var.metric_enable == true ? 1 : 0 - name = "Coralogix-firehose_metric_policy" + name = "${var.firehose_stream}-metrics-policy" tags = local.tags policy = < Date: Tue, 5 Sep 2023 11:28:44 +0800 Subject: [PATCH 5/5] changelog --- modules/firehose/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/firehose/CHANGELOG.md b/modules/firehose/CHANGELOG.md index f81664f8..a7944612 100644 --- a/modules/firehose/CHANGELOG.md +++ b/modules/firehose/CHANGELOG.md @@ -3,4 +3,4 @@ ## firehose ### version / full date -* [Update/17-Aug-2023] fix duplicate IAM issue \ No newline at end of file +* [Update/5-Sep-2023] fix firehose policy management \ No newline at end of file