From aafbc10adcfdb77a7b303ded74c73808f6e10711 Mon Sep 17 00:00:00 2001 From: Sam Simpson Date: Thu, 11 Jul 2024 14:16:23 +0100 Subject: [PATCH 1/4] Manage aws-logging bucket Imported from infra-monitoring in govuk-aws --- terraform/deployments/vpc/aws_logging.tf | 159 ++++++++++++++++++ .../vpc/aws_logging_import.tf.json | 49 ++++++ terraform/deployments/vpc/outputs.tf | 15 ++ terraform/deployments/vpc/variables.tf | 12 ++ 4 files changed, 235 insertions(+) create mode 100644 terraform/deployments/vpc/aws_logging.tf create mode 100644 terraform/deployments/vpc/aws_logging_import.tf.json diff --git a/terraform/deployments/vpc/aws_logging.tf b/terraform/deployments/vpc/aws_logging.tf new file mode 100644 index 000000000..0ef46a836 --- /dev/null +++ b/terraform/deployments/vpc/aws_logging.tf @@ -0,0 +1,159 @@ +data "aws_elb_service_account" "main" {} + +data "aws_caller_identity" "current" {} + +data "aws_iam_policy_document" "s3_aws_logging" { + statement { + actions = [ "s3:PutObject" ] + effect = "Allow" + resources = [ "arn:aws:s3:::govuk-${var.govuk_environment}-aws-logging/*" ] + principals { + type = "AWS" + identifiers = [ data.aws_elb_service_account.main.arn ] + } + } +} + +data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_policy" { + statement { + actions = [ + "s3:GetReplicationConfiguration", + "s3:ListBucket" + ] + effect = "Allow" + resources = [ aws_s3_bucket.aws_logging.arn ] + } + statement { + actions = [ + "s3:GetObjectVersion", + "s3:GetObjectVersionAcl", + "s3:GetObjectVersionTagging" + ] + effect = "Allow" + resources = [ "${aws_s3_bucket.aws_logging.arn}/*" ] + } + statement { + actions = [ + "s3:ReplicateObject", + "s3:ReplicateDelete", + "s3:ReplicateTags", + "s3:GetObjectVersionTagging", + "s3:ObjectOwnerOverrideToBucketOwner" + ] + effect = "Allow" + resources = [ "arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}/*" ] + } +} + +data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_role" { + statement { + actions = [ "sts:AssumeRole" ] + effect = "Allow" + principals { + type = "Service" + identifiers = [ "s3.amazonaws.com" ] + } + } +} + +resource "aws_iam_policy" "govuk_aws_logging_replication_policy" { + name = "govuk-${var.govuk_environment}-aws-logging-bucket-replication-policy" + policy = data.aws_iam_policy_document.s3_govuk_aws_logging_replication_policy.json + description = "Allows replication of the aws-logging bucket" +} + +resource "aws_iam_role" "govuk_aws_logging_replication_role" { + name = "govuk-aws-logging-replication-role" + assume_role_policy = data.aws_iam_policy_document.s3_govuk_aws_logging_replication_role.json +} + +resource "aws_iam_role_policy_attachment" "govuk_aws_logging_replication_policy_attachment" { + role = aws_iam_role.govuk_aws_logging_replication_role.name + policy_arn = aws_iam_policy.govuk_aws_logging_replication_policy.arn +} + +# Create a bucket that allows AWS services to write to it +resource "aws_s3_bucket" "aws_logging" { + bucket = "govuk-${var.govuk_environment}-aws-logging" +} + +resource "aws_s3_bucket_policy" "aws_logging" { + bucket = aws_s3_bucket.aws_logging.id + policy = data.aws_iam_policy_document.s3_aws_logging.json +} + +resource "aws_s3_bucket_acl" "aws_logging" { + bucket = aws_s3_bucket.aws_logging.id + acl = "log-delivery-write" +} + +resource "aws_s3_bucket_lifecycle_configuration" "aws_logging" { + bucket = aws_s3_bucket.aws_logging.id + + rule { + id = "ExpireRule" + status = "Enabled" + + expiration { + days = 30 + } + noncurrent_version_expiration { + noncurrent_days = 1 + } + } +} + +resource "aws_s3_bucket_versioning" "aws_logging" { + bucket = aws_s3_bucket.aws_logging.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_replication_configuration" "aws_logging" { + bucket = aws_s3_bucket.aws_logging.id + role = aws_iam_role.govuk_aws_logging_replication_role.arn + + rule { + id = "govuk-aws-logging-elb-govuk-public-ckan-rule" + status = var.govuk_environment == "production" ? "Enabled" : "Disabled" + destination { + bucket = "arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}" + storage_class = "STANDARD" + account = var.cyber_slunk_aws_account_id + + access_control_translation { + owner = "Destination" + } + } + filter { + prefix = "elb/govuk-ckan-public-elb" + } + } +} + +# IAM role and policy for RDS Enhanced Monitoring + +data "aws_iam_policy_document" "rds_enhanced_monitoring" { + statement { + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "Service" + identifiers = ["monitoring.rds.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "rds_enhanced_monitoring" { + name = "rds-monitoring-role" + assume_role_policy = data.aws_iam_policy_document.rds_enhanced_monitoring.json +} + +resource "aws_iam_role_policy_attachment" "rds_enhanced_monitoring" { + role = aws_iam_role.rds_enhanced_monitoring.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole" +} diff --git a/terraform/deployments/vpc/aws_logging_import.tf.json b/terraform/deployments/vpc/aws_logging_import.tf.json new file mode 100644 index 000000000..2f1f5f1ce --- /dev/null +++ b/terraform/deployments/vpc/aws_logging_import.tf.json @@ -0,0 +1,49 @@ +{ + "//": "Generated by autoimports.py script", + "import": [ + { + "to": "aws_iam_policy.govuk_aws_logging_replication_policy", + "id": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/govuk-${var.govuk_environment}-aws-logging-bucket-replication-policy" + }, + { + "to": "aws_iam_role.govuk_aws_logging_replication_role", + "id": "govuk-aws-logging-replication-role" + }, + { + "to": "aws_iam_role_policy_attachment.govuk_aws_logging_replication_policy_attachment", + "id": "${aws_iam_role.govuk_aws_logging_replication_role.name}/${aws_iam_policy.govuk_aws_logging_replication_policy.arn}" + }, + { + "to": "aws_s3_bucket.aws_logging", + "id": "govuk-${var.govuk_environment}-aws-logging" + }, + { + "to": "aws_s3_bucket_policy.aws_logging", + "id": "${aws_s3_bucket.aws_logging.id}" + }, + { + "to": "aws_s3_bucket_acl.aws_logging", + "id": "${aws_s3_bucket.aws_logging.id}" + }, + { + "to": "aws_s3_bucket_lifecycle_configuration.aws_logging", + "id": "${aws_s3_bucket.aws_logging.id}" + }, + { + "to": "aws_s3_bucket_versioning.aws_logging", + "id": "${aws_s3_bucket.aws_logging.id}" + }, + { + "to": "aws_s3_bucket_replication_configuration.aws_logging", + "id": "${aws_s3_bucket.aws_logging.id}" + }, + { + "to": "aws_iam_role.rds_enhanced_monitoring", + "id": "rds-monitoring-role" + }, + { + "to": "aws_iam_role_policy_attachment.rds_enhanced_monitoring", + "id": "${aws_iam_role.rds_enhanced_monitoring.name}/arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole" + } + ] +} \ No newline at end of file diff --git a/terraform/deployments/vpc/outputs.tf b/terraform/deployments/vpc/outputs.tf index bd12a2793..c33ec2b9a 100644 --- a/terraform/deployments/vpc/outputs.tf +++ b/terraform/deployments/vpc/outputs.tf @@ -1 +1,16 @@ output "id" { value = aws_vpc.vpc.id } + +output "aws_logging_bucket_id" { + value = aws_s3_bucket.aws_logging.id + description = "Name of the AWS logging bucket" +} + +output "aws_logging_bucket_arn" { + value = aws_s3_bucket.aws_logging.arn + description = "ARN of the AWS logging bucket" +} + +output "rds_enhanced_monitoring_role_arn" { + description = "The ARN of the IAM role for RDS Enhanced Monitoring" + value = aws_iam_role.rds_enhanced_monitoring.arn +} diff --git a/terraform/deployments/vpc/variables.tf b/terraform/deployments/vpc/variables.tf index dbcead85b..ab269819f 100644 --- a/terraform/deployments/vpc/variables.tf +++ b/terraform/deployments/vpc/variables.tf @@ -24,3 +24,15 @@ variable "cluster_log_retention_in_days" { type = string description = "Number of days to retain Cloudwatch logs for" } + +variable "cyber_slunk_s3_bucket_name" { + type = string + description = "Bucket to store logs for ingestion by Splunk" + default = "central-pipeline-logging-prod-non-cw" +} + +variable "cyber_slunk_aws_account_id" { + type = string + description = "Account ID which holds the Splunk log bucket" + default = "885513274347" +} From a733359a8cec1b0daeb13c731060a82ce700dc14 Mon Sep 17 00:00:00 2001 From: Sam Simpson Date: Thu, 11 Jul 2024 15:06:51 +0100 Subject: [PATCH 2/4] Import gcp-logging bucket properly --- terraform/deployments/vpc/google_logging_bucket.tf | 1 - .../deployments/vpc/google_logging_bucket_import.tf.json | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 terraform/deployments/vpc/google_logging_bucket_import.tf.json diff --git a/terraform/deployments/vpc/google_logging_bucket.tf b/terraform/deployments/vpc/google_logging_bucket.tf index a17350a60..2a568f0ad 100644 --- a/terraform/deployments/vpc/google_logging_bucket.tf +++ b/terraform/deployments/vpc/google_logging_bucket.tf @@ -4,7 +4,6 @@ resource "google_storage_bucket" "google_logging" { name = "govuk-${var.govuk_environment}-gcp-logging" location = "eu" storage_class = "multi_regional" - project = data.google_project.project.id versioning { enabled = true diff --git a/terraform/deployments/vpc/google_logging_bucket_import.tf.json b/terraform/deployments/vpc/google_logging_bucket_import.tf.json new file mode 100644 index 000000000..cbd249876 --- /dev/null +++ b/terraform/deployments/vpc/google_logging_bucket_import.tf.json @@ -0,0 +1,9 @@ +{ + "//": "Generated by autoimports.py script", + "import": [ + { + "to": "google_storage_bucket.google_logging", + "id": "govuk-${var.govuk_environment}-gcp-logging" + } + ] +} \ No newline at end of file From b80dc2901b4e7019b397f69d950cc575c76f03ee Mon Sep 17 00:00:00 2001 From: Sam Simpson Date: Thu, 11 Jul 2024 15:09:30 +0100 Subject: [PATCH 3/4] Fix formatting in vpc module --- terraform/deployments/vpc/aws_logging.tf | 52 ++++++++++++------------ terraform/deployments/vpc/variables.tf | 8 ++-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/terraform/deployments/vpc/aws_logging.tf b/terraform/deployments/vpc/aws_logging.tf index 0ef46a836..17c97dd11 100644 --- a/terraform/deployments/vpc/aws_logging.tf +++ b/terraform/deployments/vpc/aws_logging.tf @@ -4,12 +4,12 @@ data "aws_caller_identity" "current" {} data "aws_iam_policy_document" "s3_aws_logging" { statement { - actions = [ "s3:PutObject" ] - effect = "Allow" - resources = [ "arn:aws:s3:::govuk-${var.govuk_environment}-aws-logging/*" ] + actions = ["s3:PutObject"] + effect = "Allow" + resources = ["arn:aws:s3:::govuk-${var.govuk_environment}-aws-logging/*"] principals { - type = "AWS" - identifiers = [ data.aws_elb_service_account.main.arn ] + type = "AWS" + identifiers = [data.aws_elb_service_account.main.arn] } } } @@ -20,8 +20,8 @@ data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_policy" { "s3:GetReplicationConfiguration", "s3:ListBucket" ] - effect = "Allow" - resources = [ aws_s3_bucket.aws_logging.arn ] + effect = "Allow" + resources = [aws_s3_bucket.aws_logging.arn] } statement { actions = [ @@ -29,8 +29,8 @@ data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_policy" { "s3:GetObjectVersionAcl", "s3:GetObjectVersionTagging" ] - effect = "Allow" - resources = [ "${aws_s3_bucket.aws_logging.arn}/*" ] + effect = "Allow" + resources = ["${aws_s3_bucket.aws_logging.arn}/*"] } statement { actions = [ @@ -40,18 +40,18 @@ data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_policy" { "s3:GetObjectVersionTagging", "s3:ObjectOwnerOverrideToBucketOwner" ] - effect = "Allow" - resources = [ "arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}/*" ] + effect = "Allow" + resources = ["arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}/*"] } } data "aws_iam_policy_document" "s3_govuk_aws_logging_replication_role" { statement { - actions = [ "sts:AssumeRole" ] - effect = "Allow" + actions = ["sts:AssumeRole"] + effect = "Allow" principals { - type = "Service" - identifiers = [ "s3.amazonaws.com" ] + type = "Service" + identifiers = ["s3.amazonaws.com"] } } } @@ -68,7 +68,7 @@ resource "aws_iam_role" "govuk_aws_logging_replication_role" { } resource "aws_iam_role_policy_attachment" "govuk_aws_logging_replication_policy_attachment" { - role = aws_iam_role.govuk_aws_logging_replication_role.name + role = aws_iam_role.govuk_aws_logging_replication_role.name policy_arn = aws_iam_policy.govuk_aws_logging_replication_policy.arn } @@ -84,14 +84,14 @@ resource "aws_s3_bucket_policy" "aws_logging" { resource "aws_s3_bucket_acl" "aws_logging" { bucket = aws_s3_bucket.aws_logging.id - acl = "log-delivery-write" + acl = "log-delivery-write" } resource "aws_s3_bucket_lifecycle_configuration" "aws_logging" { bucket = aws_s3_bucket.aws_logging.id rule { - id = "ExpireRule" + id = "ExpireRule" status = "Enabled" expiration { @@ -113,19 +113,19 @@ resource "aws_s3_bucket_versioning" "aws_logging" { resource "aws_s3_bucket_replication_configuration" "aws_logging" { bucket = aws_s3_bucket.aws_logging.id - role = aws_iam_role.govuk_aws_logging_replication_role.arn + role = aws_iam_role.govuk_aws_logging_replication_role.arn rule { - id = "govuk-aws-logging-elb-govuk-public-ckan-rule" + id = "govuk-aws-logging-elb-govuk-public-ckan-rule" status = var.govuk_environment == "production" ? "Enabled" : "Disabled" destination { - bucket = "arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}" - storage_class = "STANDARD" - account = var.cyber_slunk_aws_account_id + bucket = "arn:aws:s3:::${var.cyber_slunk_s3_bucket_name}" + storage_class = "STANDARD" + account = var.cyber_slunk_aws_account_id - access_control_translation { - owner = "Destination" - } + access_control_translation { + owner = "Destination" + } } filter { prefix = "elb/govuk-ckan-public-elb" diff --git a/terraform/deployments/vpc/variables.tf b/terraform/deployments/vpc/variables.tf index ab269819f..d5e7446c4 100644 --- a/terraform/deployments/vpc/variables.tf +++ b/terraform/deployments/vpc/variables.tf @@ -26,13 +26,13 @@ variable "cluster_log_retention_in_days" { } variable "cyber_slunk_s3_bucket_name" { - type = string + type = string description = "Bucket to store logs for ingestion by Splunk" - default = "central-pipeline-logging-prod-non-cw" + default = "central-pipeline-logging-prod-non-cw" } variable "cyber_slunk_aws_account_id" { - type = string + type = string description = "Account ID which holds the Splunk log bucket" - default = "885513274347" + default = "885513274347" } From 1e08b1829286ff3719cca0411c96f7a2d0f1314b Mon Sep 17 00:00:00 2001 From: Sam Simpson Date: Thu, 11 Jul 2024 16:06:53 +0100 Subject: [PATCH 4/4] Update gcp-logging bucket to reflect current lifecycle rule --- terraform/deployments/vpc/google_logging_bucket.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/deployments/vpc/google_logging_bucket.tf b/terraform/deployments/vpc/google_logging_bucket.tf index 2a568f0ad..29d4d5411 100644 --- a/terraform/deployments/vpc/google_logging_bucket.tf +++ b/terraform/deployments/vpc/google_logging_bucket.tf @@ -3,7 +3,7 @@ data "google_project" "project" {} resource "google_storage_bucket" "google_logging" { name = "govuk-${var.govuk_environment}-gcp-logging" location = "eu" - storage_class = "multi_regional" + storage_class = "MULTI_REGIONAL" versioning { enabled = true @@ -15,15 +15,15 @@ resource "google_storage_bucket" "google_logging" { } condition { - age = 30 + age = 30 + with_state = "ARCHIVED" } } } -resource "google_storage_bucket_acl" "google_logging" { +resource "google_storage_bucket_access_control" "google_logging" { bucket = google_storage_bucket.google_logging.name - role_entity = [ - "WRITER:group-cloud-storage-analytics@google.com", - ] + role = "WRITER" + entity = "group-cloud-storage-analytics@google.com" }