Merge branch 'coralogix:master' into master

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v1.0.107
+#### **firehose-logs & firehose-metrics**
+### 💡 Enhancements 
+- Add AP3 region to the list of regions
+- Added custom naming for global resources
+- Added ability to import global resources (s3 & iam)
+### 🛑 Breaking changes 🛑
+- For firehose-logs & firehose-metrics, Update variables: `coralogix_region` values regions from [Europe, Europe2, India, Singapore, US, US2] to [EU1, EU2, AP1, AP2, AP3, US1, US2]
+- Update variables: `private_key` renamed to `api_key` with type `string` instead of `any`.
+
+## v1.0.106
+#### **msk-data-stream**
+### 💡 Enhancements 
+- Update coralogix role from `arn:aws:iam::<account-id>:role/msk-access-<region>` to  `arn:aws:iam::<account-id>:role/coralogix-archive-<region>`
+- allow the module to run in AP3 region
+
+#### **coralogix-aws-shipper**
+### 💡 Enhancements 
+- Allow the module to be deployed in AP3
+
+#### **S3-archive**
+### 💡 Enhancements 
+- Allow the module to be deployed in ap-southeast-3 region
+
+
 ## v1.0.105
 #### **firehose-metrics**
 ### 💡 Enhancements 

examples/coralogix-aws-shipper/variables.tf

@@ -3,8 +3,8 @@ variable "coralogix_region" {
   description = "The Coralogix location region, possible options are [EU1, EU2, AP1, AP2, US1, US2]"
   type        = string
   validation {
-    condition     = contains(["EU1", "EU2", "AP1", "AP2", "US1", "US2", "Custom"], var.coralogix_region)
-    error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, US1, US2, Custom]."
+    condition     = contains(["EU1", "EU2", "AP1", "AP2", "AP3", "US1", "US2", "Custom"], var.coralogix_region)
+    error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, AP3, US1, US2, Custom]."
   }
 }
 

examples/firehose-logs/main.tf

@@ -1,7 +1,7 @@
 module "cloudwatch_firehose_logs_coralogix" {
   source                    = "coralogix/aws/coralogix//modules/firehose-logs"
   firehose_stream           = var.firehose_stream
-  private_key               = var.private_key
+  api_key                   = var.api_key
   coralogix_region          = var.coralogix_region
   integration_type_logs     = "Default"
   source_type_logs          = "DirectPut"

examples/firehose-logs/variables.tf

@@ -9,9 +9,9 @@ variable "coralogix_region" {
   description = "The region of the Coralogix account"
 }
 
-variable "private_key" {
+variable "api_key" {
   type        = string
-  description = "Coralogix account logs private key"
+  description = "Coralogix account api key"
   sensitive   = true
 }
 

examples/msk-data-stream/outputs.tf

@@ -9,3 +9,7 @@ output "security_group_id" {
 output "cluster_arn" {
   value = aws_msk_cluster.coralogix-msk-cluster.arn
 }
+
+output "msk_public_brokesrs" {
+  value = data.aws_msk_cluster.msk_public_brokers.bootstrap_brokers_public_sasl_iam
+}

examples/msk-data-stream/variables.tf

@@ -2,8 +2,8 @@ variable "aws_region" {
   type        = string
   description = "The AWS region that you want to create the MSK in, Must be the same as the AWS region where your coralogix account is set"
   validation {
-    condition     = contains(["eu-west-1", "eu-north-1", "ap-southeast-1", "ap-south-1", "us-east-2", "us-west-2", "custom"], var.aws_region)
-    error_message = "The aws region must be one of these values: [eu-west-1, eu-north-1, ap-southeast-1, ap-south-1, us-east-2, us-west-2, custom]."
+    condition     = contains(["eu-west-1", "eu-north-1", "ap-southeast-1", "ap-southeast-3", "ap-south-1", "us-east-2", "us-west-2", "custom"], var.aws_region)
+    error_message = "The aws region must be one of these values: [eu-west-1, eu-north-1, ap-southeast-1, ap-southeast-3, ap-south-1, us-east-2, us-west-2, custom]."
   }
 }
 
@@ -43,16 +43,3 @@ variable "custom_coralogix_arn" {
   default     = null
 
 }
-
-variable "coraloigx_roles_arn_mapping" {
-  type = map
-  default = {
-      "eu-west-1"      = "arn:aws:iam::625240141681:role/msk-access-eu1"
-      "eu-north-1"     = "arn:aws:iam::625240141681:role/msk-access-eu2"
-      "ap-southeast-1" = "arn:aws:iam::625240141681:role/msk-access-ap2"
-      "ap-south-1"     = "arn:aws:iam::625240141681:role/msk-access-ap1"
-      "us-east-2"      = "arn:aws:iam::625240141681:role/msk-access-us1"
-      "us-west-2"      = "arn:aws:iam::739076534691:role/msk-access-us2"
-      "custom"         = ""
-  }
-}

examples/s3-archive/varibales.tf → examples/s3-archive/variables.tf

@@ -3,8 +3,8 @@ variable "aws_region" {
   description = "The AWS region that you want to create the S3 bucket, Must be the same as the AWS region where your coralogix account is set"
   default     = ""
   validation {
-    condition     = contains(["eu-west-1", "eu-north-1", "ap-southeast-1", "ap-south-1", "us-east-2", "us-west-2", ""], var.aws_region)
-    error_message = "The aws region must be one of these values: [eu-west-1, eu-north-1, ap-southeast-1, ap-south-1, us-east-2, us-west-2]."
+    condition     = contains(["eu-west-1", "eu-north-1", "ap-southeast-1", "ap-southeast-3", "ap-south-1", "us-east-2", "us-west-2", ""], var.aws_region)
+    error_message = "The aws region must be one of these values: [eu-west-1, eu-north-1, ap-southeast-1, ap-southeast-3, ap-south-1, us-east-2, us-west-2]."
   }
 }
 

modules/coralogix-aws-shipper/README.md

@@ -41,7 +41,7 @@ If you want to avoid this issue, you can deploy in other ways:
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_coralogix_region"></a> [coralogix\_region](#input\_coralogix\_region) | The Coralogix location region, possible options are [`EU1`, `EU2`, `AP1`, `AP2`, `US1`, `US2`, `Custom`] | `string` | n/a | yes |
+| <a name="input_coralogix_region"></a> [coralogix\_region](#input\_coralogix\_region) | The Coralogix location region, possible options are [`EU1`, `EU2`, `AP1`, `AP2`, `AP3`, `US1`, `US2`, `Custom`] | `string` | n/a | yes |
 | <a name="input_custom_domain"></a> [custom_domain](#input\_custom\_domain) | If you choose a custom domain name for your private cluster, Coralogix will send telemetry from the specified address (e.g. custom.coralogix.com) there is no need to add `ingress.` to the domain .| `string` | n/a | no |
 | <a name="input_integration_type"></a> [integration_type](#input\_data\_type) | Choose the AWS service that you wish to integrate with Coralogix. Can be one of: S3, CloudTrail, VpcFlow, CloudWatch, S3Csv, SNS, SQS, Kinesis, CloudFront, MSK, Kafka, EcrScan. | `string` | n/a | yes |
 | <a name="input_api_key"></a> [api\_key](#input\_api_\_key) | The Coralogix Send Your Data - [API Key](https://coralogix.com/docs/send-your-data-api-key/) validates your authenticity. This value can be a direct Coralogix API Key or an AWS Secret Manager ARN containing the API Key.| `string` | n/a | yes |

modules/coralogix-aws-shipper/variables.tf

@@ -3,8 +3,8 @@ variable "coralogix_region" {
   description = "The Coralogix location region, possible options are [EU1, EU2, AP1, AP2, US1, US2]"
   type        = string
   validation {
-    condition     = contains(["EU1", "EU2", "AP1", "AP2", "US1", "US2", "Custom"], var.coralogix_region)
-    error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, US1, US2, Custom]."
+    condition     = contains(["EU1", "EU2", "AP1", "AP2", "AP3", "US1", "US2", "Custom"], var.coralogix_region)
+    error_message = "The coralogix region must be one of these values: [EU1, EU2, AP1, AP2, AP3, US1, US2, Custom]."
   }
 }
 

modules/firehose-logs/README.md

@@ -46,23 +46,25 @@ The application name and subsystem name by default is the firehose delivery stre
 
 # Coralogix account region
 The coralogix region variable accepts one of the following regions:
-* Europe
-* Europe2
-* India
-* Singapore
-* US
+* EU1
+* EU2
+* AP1
+* AP2
+* AP3
+* US1
 * US2
 
 ### Coralogix Regions & Description. 
 
-| Region    | Domain                 |  Endpoint                                       |
-|-----------|------------------------|---------------------------------------------------------|
-| Europe    | `coralogix.com`        | `https://firehose-ingress.coralogix.com/firehose`       |
-| Europe2   | `eu2.coralogix.com`    | `https://firehose-ingress.eu2.coralogix.com/firehose`   |
-| India     | `coralogix.in`         | `https://firehose-ingress.app.coralogix.in/firehose`    |
-| Singapore | `coralogixsg.com`      | `https://firehose-ingress.coralogixsg.com/firehose`     |
-| US        | `coralogix.us`         | `https://firehose-ingress.coralogix.us/firehose`        |
-| US2       | `cx498.coralogix.com`  | `https://firehose-ingress.cx498.coralogix.com/firehose` |
+| Region    | Domain                 |  Endpoint                                          |
+|-----------|------------------------|----------------------------------------------------|
+| EU1       | `coralogix.com`        | `https://ingress.coralogix.com/aws/firehose`       |
+| EU2       | `eu2.coralogix.com`    | `https://ingress.eu2.coralogix.com/aws/firehose`   |
+| AP1       | `coralogix.in`         | `https://ingress.app.coralogix.in/aws/firehose`    |
+| AP2       | `coralogixsg.com`      | `https://ingress.coralogixsg.com/aws/firehose`     |
+| AP3       | `ap3.coralogix.com`    | `https://ingress.ap3.coralogix.com/aws/firehose`   |
+| US1       | `coralogix.us`         | `https://ingress.coralogix.us/aws/firehose`        |
+| US2       | `cx498.coralogix.com`  | `https://ingress.cx498.coralogix.com/aws/firehose` |
 
 ### Custom Domain
 It is possible to pass a custom coralogix domain by using the `custom_domain` variable.
@@ -84,7 +86,7 @@ It is possible to pass a custom coralogix domain by using the `custom_domain` va
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_coralogix_region"></a> [coralogix\_region](#input\_coralogix\_region) | Coralogix account region: Europe, Europe2, India, Singapore, US, US2 [exact] | `any` | n/a | yes |
+| <a name="input_coralogix_region"></a> [coralogix\_region](#input\_coralogix\_region) | Coralogix account region: EU1, EU2, AP1, AP2, AP3, US1, US2 [exact] | `any` | n/a | yes |
 | <a name="input_private_key"></a> [private_key](#input\_private_key) | Coralogix account logs private key | `any` | n/a | yes |
 | <a name="input_firehose_stream"></a> [firehose\_stream](#input\_firehose\_stream) | AWS Kinesis firehose delivery stream name | `string` | n/a | yes |
 | <a name="input_application_name"></a> [application_name](#input\_application_name) | The name of your application in Coralogix | `string` | n/a | yes |

modules/firehose-logs/main.tf

@@ -25,8 +25,13 @@ locals {
     custom_endpoint          = local.endpoint_url
   }) : var.user_supplied_tags
 
-  # default namings
-  s3_logs_backup_bucket_name = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-logs"
+  # global resource referecing
+  s3_backup_bucket_arn  = var.existing_s3_backup != null ? one(data.aws_s3_bucket.exisiting_s3_bucket[*].arn) : one(aws_s3_bucket.new_s3_bucket[*].arn)
+  firehose_iam_role_arn = var.existing_firehose_iam != null ? one(data.aws_iam_role.existing_firehose_iam[*].arn) : one(aws_iam_role.new_firehose_iam[*].arn)
+
+  #new global resource namings
+  new_s3_backup_bucket_name = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-logs"
+  new_firehose_iam_name     = var.firehose_iam_custom_name != null ? var.firehose_iam_custom_name : "${var.firehose_stream}-firehose-logs-iam"
 }
 
 data "aws_caller_identity" "current_identity" {}
@@ -57,13 +62,20 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" {
   log_group_name = aws_cloudwatch_log_group.firehose_loggroup.name
 }
 
-resource "aws_s3_bucket" "firehose_bucket" {
-  tags   = merge(local.tags, { Name = local.s3_logs_backup_bucket_name })
-  bucket = local.s3_logs_backup_bucket_name
+data "aws_s3_bucket" "exisiting_s3_bucket" {
+  count  = var.existing_s3_backup != null ? 1 : 0
+  bucket = var.existing_s3_backup
+}
+
+resource "aws_s3_bucket" "new_s3_bucket" {
+  count  = var.existing_s3_backup != null ? 0 : 1
+  tags   = merge(local.tags, { Name = local.new_s3_backup_bucket_name })
+  bucket = local.new_s3_backup_bucket_name
 }
 
 resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
-  bucket = aws_s3_bucket.firehose_bucket.id
+  count  = var.existing_s3_backup != null ? 0 : 1
+  bucket = one(aws_s3_bucket.new_s3_bucket[*].id)
 
   block_public_acls       = true
   block_public_policy     = true
@@ -75,9 +87,15 @@ resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
 # Firehose Logs Stream
 ################################################################################
 
-resource "aws_iam_role" "firehose_to_coralogix" {
-  tags = local.tags
-  name = "${var.firehose_stream}-firehose-logs"
+data "aws_iam_role" "existing_firehose_iam" {
+  count = var.existing_firehose_iam != null ? 1 : 0
+  name  = var.existing_firehose_iam
+}
+
+resource "aws_iam_role" "new_firehose_iam" {
+  count = var.existing_firehose_iam != null ? 0 : 1
+  tags  = local.tags
+  name  = local.new_firehose_iam_name
   assume_role_policy = jsonencode({
     "Version" = "2012-10-17",
     "Statement" = [
@@ -91,7 +109,7 @@ resource "aws_iam_role" "firehose_to_coralogix" {
     ]
   })
   inline_policy {
-    name = "${var.firehose_stream}-firehose"
+    name = local.new_firehose_iam_name
     policy = jsonencode({
       "Version" = "2012-10-17",
       "Statement" = [
@@ -106,8 +124,8 @@ resource "aws_iam_role" "firehose_to_coralogix" {
             "s3:PutObject"
           ],
           "Resource" = [
-            aws_s3_bucket.firehose_bucket.arn,
-            "${aws_s3_bucket.firehose_bucket.arn}/*"
+            "${local.s3_backup_bucket_arn}",
+            "${local.s3_backup_bucket_arn}/*"
           ]
         },
         {
@@ -121,45 +139,64 @@ resource "aws_iam_role" "firehose_to_coralogix" {
           "Resource" = "arn:aws:kinesis:${data.aws_region.current_region.name}:${data.aws_caller_identity.current_identity.account_id}:stream/*"
         },
         {
-          "Effect" = "Allow",
-          "Action" = [
-            "*"
+          "Effect" : "Allow",
+          "Action" : [
+            "logs:PutLogEvents"
           ],
-          "Resource" = [
-            aws_cloudwatch_log_group.firehose_loggroup.arn
+          "Resource" : [
+            "${aws_cloudwatch_log_group.firehose_loggroup.arn}"
           ]
         }
       ]
     })
   }
 }
 
+# Add additional policies to the firehose IAM role
+resource "aws_iam_role_policy_attachment" "policy_attachment_firehose" {
+  count      = var.existing_firehose_iam != null ? 0 : 1
+  role       = one(aws_iam_role.new_firehose_iam[*].name)
+  policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisFirehoseFullAccess"
+}
+
+resource "aws_iam_role_policy_attachment" "policy_attachment_kinesis" {
+  count      = var.existing_firehose_iam != null ? 0 : 1
+  role       = one(aws_iam_role.new_firehose_iam[*].name)
+  policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisReadOnlyAccess"
+}
+
+resource "aws_iam_role_policy_attachment" "policy_attachment_cloudwatch" {
+  count      = var.existing_firehose_iam != null ? 0 : 1
+  role       = one(aws_iam_role.new_firehose_iam[*].name)
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
+}
+
 resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" {
   tags        = local.tags
-  name        = "${var.firehose_stream}-logs"
+  name        = var.firehose_stream
   destination = "http_endpoint"
 
   dynamic "kinesis_source_configuration" {
     for_each = var.source_type_logs == "KinesisStreamAsSource" && var.kinesis_stream_arn != null ? [1] : []
     content {
       kinesis_stream_arn = var.kinesis_stream_arn
-      role_arn           = aws_iam_role.firehose_to_coralogix.arn
+      role_arn           = local.firehose_iam_role_arn
     }
   }
 
   http_endpoint_configuration {
     url                = local.endpoint_url
     name               = "Coralogix"
-    access_key         = var.private_key
+    access_key         = var.api_key
     buffering_size     = 6
     buffering_interval = 60
     s3_backup_mode     = "FailedDataOnly"
-    role_arn           = aws_iam_role.firehose_to_coralogix.arn
+    role_arn           = local.firehose_iam_role_arn
     retry_duration     = 300
 
     s3_configuration {
-      role_arn           = aws_iam_role.firehose_to_coralogix.arn
-      bucket_arn         = aws_s3_bucket.firehose_bucket.arn
+      role_arn           = local.firehose_iam_role_arn
+      bucket_arn         = local.s3_backup_bucket_arn
       buffering_size     = 5
       buffering_interval = 300
       compression_format = "GZIP"
@@ -200,18 +237,3 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" {
     }
   }
 }
-
-resource "aws_iam_role_policy_attachment" "example_policy_attachment" {
-  role       = aws_iam_role.firehose_to_coralogix.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisFirehoseFullAccess"
-}
-
-resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" {
-  role       = aws_iam_role.firehose_to_coralogix.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisReadOnlyAccess"
-}
-
-resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" {
-  role       = aws_iam_role.firehose_to_coralogix.name
-  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-}