From e6c88ada33abad97b1eb3cb7c621c85e20237b67 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 7 Oct 2024 11:38:20 -0700 Subject: [PATCH 01/16] create service account helper module --- modules/google-service-account/README.md | 0 modules/google-service-account/main.tf | 8 ++++ modules/google-service-account/outputs.tf | 4 ++ modules/google-service-account/variables.tf | 49 +++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 modules/google-service-account/README.md create mode 100644 modules/google-service-account/main.tf create mode 100644 modules/google-service-account/outputs.tf create mode 100644 modules/google-service-account/variables.tf diff --git a/modules/google-service-account/README.md b/modules/google-service-account/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/google-service-account/main.tf b/modules/google-service-account/main.tf new file mode 100644 index 0000000..0e64011 --- /dev/null +++ b/modules/google-service-account/main.tf @@ -0,0 +1,8 @@ +resource "google_service_account" "this" { + account_id = var.account_id + create_ignore_already_exists = var.create_ignore_already_exists + disabled = var.disabled + display_name = var.display_name + description = var.description + project = var.project +} diff --git a/modules/google-service-account/outputs.tf b/modules/google-service-account/outputs.tf new file mode 100644 index 0000000..1dafa71 --- /dev/null +++ b/modules/google-service-account/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "Google Service Account." + value = google_service_account.this +} diff --git a/modules/google-service-account/variables.tf b/modules/google-service-account/variables.tf new file mode 100644 index 0000000..b1c4c0b --- /dev/null +++ b/modules/google-service-account/variables.tf @@ -0,0 +1,49 @@ +variable "account_id" { + description = "The account id that is used to generate the service account email address and a stable unique id. It is unique within a project, must be 6-30 characters long, and match the regular expression '[a-z]([-a-z0-9]*[a-z0-9])' to comply with RFC1035. Changing this forces a new service account to be created." + type = string + validation { + condition = ( + length(var.account_id >= 6) && + length(var.account_id <= 30) && + can(regex("[a-z]([-a-z0-9]*[a-z0-9])", var.account_id)) + ) + error_message = "Invalid service account ID." + } +} + +variable "create_ignore_already_exists" { + description = "If set to true, skip service account creation if a service account with the same email already exists. Defaults to false." + type = bool + default = false +} + +variable "disabled" { + description = "Whether a service account is disabled or not. Defaults to false. This field has no effect during creation. Must be set after creation to disable a service account." + type = bool + default = false +} + +variable "display_name" { + description = "The display name for the service account" + type = string + default = null +} + +variable "description" { + description = "A text description of the service account. Must be less than or equal to 256 UTF-8 bytes." + type = string + default = null + validation { + condition = ( + var.description == null || + length(var.description) <= 256 + ) + error_message = "Service account description must be less than 256 characters." + } +} + +variable "project" { + description = "The ID of the project that the service account will be created in. Defaults to the provider project configuration." + type = string + default = null +} From 6c0634ac660be8bee5f6faa75103a872377b8043 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 7 Oct 2024 12:22:31 -0700 Subject: [PATCH 02/16] fix validation --- modules/google-service-account/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/google-service-account/variables.tf b/modules/google-service-account/variables.tf index b1c4c0b..988d799 100644 --- a/modules/google-service-account/variables.tf +++ b/modules/google-service-account/variables.tf @@ -3,8 +3,8 @@ variable "account_id" { type = string validation { condition = ( - length(var.account_id >= 6) && - length(var.account_id <= 30) && + length(var.account_id) >= 6 && + length(var.account_id) <= 30 && can(regex("[a-z]([-a-z0-9]*[a-z0-9])", var.account_id)) ) error_message = "Invalid service account ID." From 81713c9d52f130df79b27c937cd16485657d65ee Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 7 Oct 2024 12:23:19 -0700 Subject: [PATCH 03/16] create service account role attachment helper module --- .../README.md | 0 .../main.tf | 5 ++++ .../outputs.tf | 4 +++ .../variables.tf | 27 +++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 modules/google-service-account-iam-binding/README.md create mode 100644 modules/google-service-account-iam-binding/main.tf create mode 100644 modules/google-service-account-iam-binding/outputs.tf create mode 100644 modules/google-service-account-iam-binding/variables.tf diff --git a/modules/google-service-account-iam-binding/README.md b/modules/google-service-account-iam-binding/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/google-service-account-iam-binding/main.tf b/modules/google-service-account-iam-binding/main.tf new file mode 100644 index 0000000..3a6f807 --- /dev/null +++ b/modules/google-service-account-iam-binding/main.tf @@ -0,0 +1,5 @@ +resource "google_service_account_iam_binding" "this" { + members = var.members + role = var.role + service_account_id = var.service_account_id +} diff --git a/modules/google-service-account-iam-binding/outputs.tf b/modules/google-service-account-iam-binding/outputs.tf new file mode 100644 index 0000000..b60436a --- /dev/null +++ b/modules/google-service-account-iam-binding/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "Attachment of a Google IAM role to a Service Account." + value = google_service_account_iam_binding.this +} diff --git a/modules/google-service-account-iam-binding/variables.tf b/modules/google-service-account-iam-binding/variables.tf new file mode 100644 index 0000000..b0cb9c2 --- /dev/null +++ b/modules/google-service-account-iam-binding/variables.tf @@ -0,0 +1,27 @@ +variable "members" { + description = "Identities that will be granted the privilege in role. See the [Google documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_service_account_iam#member/members) for a list of possible values." + type = list(string) + validation { + condition = alltrue([ + for m in var.members : ( + m == "allUsers" || + m == "allAuthenticatedUsers" || + startswith(m, "user:") || + startswith(m, "serviceAccount:") || + startswith(m, "group:") || + startswith(m, "domain:") + ) + ]) + error_message = "One or more 'member' values is invalid." + } +} + +variable "role" { + description = "The role that should be applied." + type = string +} + +variable "service_account_id" { + description = "The fully-qualified name of the service account to apply the policy to." + type = string +} From a05f890f4e2dd130a689a249a3518554c2b18a2b Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 8 Oct 2024 14:40:14 -0700 Subject: [PATCH 04/16] consolidate service account modules --- .../README.md | 0 modules/google-service-account-dsf/main.tf | 34 +++++++++++++++++++ modules/google-service-account-dsf/outputs.tf | 19 +++++++++++ .../variables.tf | 19 +++++++++++ .../main.tf | 5 --- .../outputs.tf | 4 --- .../variables.tf | 27 --------------- modules/google-service-account/README.md | 0 modules/google-service-account/main.tf | 8 ----- modules/google-service-account/outputs.tf | 4 --- 10 files changed, 72 insertions(+), 48 deletions(-) rename modules/{google-service-account-iam-binding => google-service-account-dsf}/README.md (100%) create mode 100644 modules/google-service-account-dsf/main.tf create mode 100644 modules/google-service-account-dsf/outputs.tf rename modules/{google-service-account => google-service-account-dsf}/variables.tf (71%) delete mode 100644 modules/google-service-account-iam-binding/main.tf delete mode 100644 modules/google-service-account-iam-binding/outputs.tf delete mode 100644 modules/google-service-account-iam-binding/variables.tf delete mode 100644 modules/google-service-account/README.md delete mode 100644 modules/google-service-account/main.tf delete mode 100644 modules/google-service-account/outputs.tf diff --git a/modules/google-service-account-iam-binding/README.md b/modules/google-service-account-dsf/README.md similarity index 100% rename from modules/google-service-account-iam-binding/README.md rename to modules/google-service-account-dsf/README.md diff --git a/modules/google-service-account-dsf/main.tf b/modules/google-service-account-dsf/main.tf new file mode 100644 index 0000000..24d05e2 --- /dev/null +++ b/modules/google-service-account-dsf/main.tf @@ -0,0 +1,34 @@ +resource "google_service_account" "account" { + account_id = var.account_id + create_ignore_already_exists = var.create_ignore_already_exists + disabled = var.disabled + display_name = var.display_name + description = var.description + project = var.project + + provisioner "local-exec" { + command = "sleep 20" + } +} + +resource "google_project_iam_member" "permissions" { + for_each = toset(var.project_roles) + + member = google_service_account.account.member + project = var.project + role = each.value +} + +resource "google_service_account_key" "key" { + count = var.auth_mechanism == "service_account" ? 1 : 0 + depends_on = [google_service_account.account] + + service_account_id = var.account_id +} + +resource "local_file" "service_account_key_file" { + count = var.auth_mechanism == "service_account" ? 1 : 0 + + filename = "${path.root}/${var.account_id}.json" + content = base64decode(google_service_account_key.key[0].private_key) +} diff --git a/modules/google-service-account-dsf/outputs.tf b/modules/google-service-account-dsf/outputs.tf new file mode 100644 index 0000000..7439ffb --- /dev/null +++ b/modules/google-service-account-dsf/outputs.tf @@ -0,0 +1,19 @@ +output "account" { + description = "Google Service Account." + value = google_service_account.account +} + +output "permissions" { + description = "Service Account permissions." + value = google_project_iam_member.permissions +} + +output "key" { + description = "Authentication key for the Service Account." + value = google_service_account_key.key +} + +output "key_file" { + description = "JSON key file containing the Service Account credentials." + value = local_file.service_account_key_file +} diff --git a/modules/google-service-account/variables.tf b/modules/google-service-account-dsf/variables.tf similarity index 71% rename from modules/google-service-account/variables.tf rename to modules/google-service-account-dsf/variables.tf index 988d799..904b495 100644 --- a/modules/google-service-account/variables.tf +++ b/modules/google-service-account-dsf/variables.tf @@ -11,6 +11,16 @@ variable "account_id" { } } +variable "auth_mechanism" { + description = "Authentication mechanism intended to be used by DSF to leverage this service account. Valid values are: service_account, default." + type = string + default = "service_account" + validation { + condition = contains(["service_account", "default"], var.auth_mechanism) + error_message = "Invalid service account auth_mechanism. Valid values are: service_account, default." + } +} + variable "create_ignore_already_exists" { description = "If set to true, skip service account creation if a service account with the same email already exists. Defaults to false." type = bool @@ -47,3 +57,12 @@ variable "project" { type = string default = null } + +variable "project_roles" { + description = "Roles to grant the Service Account in the specified project. Defaults to roles required for DSF to pull audit data." + type = list(string) + default = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} diff --git a/modules/google-service-account-iam-binding/main.tf b/modules/google-service-account-iam-binding/main.tf deleted file mode 100644 index 3a6f807..0000000 --- a/modules/google-service-account-iam-binding/main.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "google_service_account_iam_binding" "this" { - members = var.members - role = var.role - service_account_id = var.service_account_id -} diff --git a/modules/google-service-account-iam-binding/outputs.tf b/modules/google-service-account-iam-binding/outputs.tf deleted file mode 100644 index b60436a..0000000 --- a/modules/google-service-account-iam-binding/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "this" { - description = "Attachment of a Google IAM role to a Service Account." - value = google_service_account_iam_binding.this -} diff --git a/modules/google-service-account-iam-binding/variables.tf b/modules/google-service-account-iam-binding/variables.tf deleted file mode 100644 index b0cb9c2..0000000 --- a/modules/google-service-account-iam-binding/variables.tf +++ /dev/null @@ -1,27 +0,0 @@ -variable "members" { - description = "Identities that will be granted the privilege in role. See the [Google documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_service_account_iam#member/members) for a list of possible values." - type = list(string) - validation { - condition = alltrue([ - for m in var.members : ( - m == "allUsers" || - m == "allAuthenticatedUsers" || - startswith(m, "user:") || - startswith(m, "serviceAccount:") || - startswith(m, "group:") || - startswith(m, "domain:") - ) - ]) - error_message = "One or more 'member' values is invalid." - } -} - -variable "role" { - description = "The role that should be applied." - type = string -} - -variable "service_account_id" { - description = "The fully-qualified name of the service account to apply the policy to." - type = string -} diff --git a/modules/google-service-account/README.md b/modules/google-service-account/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/modules/google-service-account/main.tf b/modules/google-service-account/main.tf deleted file mode 100644 index 0e64011..0000000 --- a/modules/google-service-account/main.tf +++ /dev/null @@ -1,8 +0,0 @@ -resource "google_service_account" "this" { - account_id = var.account_id - create_ignore_already_exists = var.create_ignore_already_exists - disabled = var.disabled - display_name = var.display_name - description = var.description - project = var.project -} diff --git a/modules/google-service-account/outputs.tf b/modules/google-service-account/outputs.tf deleted file mode 100644 index 1dafa71..0000000 --- a/modules/google-service-account/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "this" { - description = "Google Service Account." - value = google_service_account.this -} From c2e60690bcda8bc88a5b32ee8a63272d0ca4c76b Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Wed, 9 Oct 2024 12:46:48 -0700 Subject: [PATCH 05/16] update key file output path --- modules/google-service-account-dsf/outputs.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/google-service-account-dsf/outputs.tf b/modules/google-service-account-dsf/outputs.tf index 7439ffb..36071ab 100644 --- a/modules/google-service-account-dsf/outputs.tf +++ b/modules/google-service-account-dsf/outputs.tf @@ -13,7 +13,7 @@ output "key" { value = google_service_account_key.key } -output "key_file" { +output "key_file_path" { description = "JSON key file containing the Service Account credentials." - value = local_file.service_account_key_file + value = "${path.root}/${var.account_id}" } From a2d38e736d5d5b8f046aa7d2be3d076b4de558e0 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Wed, 9 Oct 2024 12:46:56 -0700 Subject: [PATCH 06/16] add gcp cloud account module --- modules/dsfhub-gcp-cloud-account/README.md | 0 modules/dsfhub-gcp-cloud-account/main.tf | 22 ++++++++++ modules/dsfhub-gcp-cloud-account/outputs.tf | 4 ++ modules/dsfhub-gcp-cloud-account/variables.tf | 41 +++++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 modules/dsfhub-gcp-cloud-account/README.md create mode 100644 modules/dsfhub-gcp-cloud-account/main.tf create mode 100644 modules/dsfhub-gcp-cloud-account/outputs.tf create mode 100644 modules/dsfhub-gcp-cloud-account/variables.tf diff --git a/modules/dsfhub-gcp-cloud-account/README.md b/modules/dsfhub-gcp-cloud-account/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-cloud-account/main.tf b/modules/dsfhub-gcp-cloud-account/main.tf new file mode 100644 index 0000000..a0cb58a --- /dev/null +++ b/modules/dsfhub-gcp-cloud-account/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_cloud_account" "this" { + server_type = "GCP" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + gateway_id = var.gateway_id + + asset_connection { + auth_mechanism = var.auth_mechanism + reason = var.reason + key_file = var.auth_mechanism == "service_account" ? var.key_file : null + } +} diff --git a/modules/dsfhub-gcp-cloud-account/outputs.tf b/modules/dsfhub-gcp-cloud-account/outputs.tf new file mode 100644 index 0000000..185183e --- /dev/null +++ b/modules/dsfhub-gcp-cloud-account/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP cloud account asset." + value = dsfhub_cloud_account.this +} diff --git a/modules/dsfhub-gcp-cloud-account/variables.tf b/modules/dsfhub-gcp-cloud-account/variables.tf new file mode 100644 index 0000000..86415b6 --- /dev/null +++ b/modules/dsfhub-gcp-cloud-account/variables.tf @@ -0,0 +1,41 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier of the GCP account in the form ':' (e.g. my-service-account-name@my-project-id.iam.gserviceaccount.com:default-project-id-for-this-asset)." + type = string +} + +variable "auth_mechanism" { + description = "Specifies the auth mechanism used by the connection. Supported values: default, service_account." + type = string + default = "default" + validation { + condition = contains(["default", "service_account"], var.auth_mechanism) + error_message = "Invalid authentication mechanism. Supported values: default, service_account." + } +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "key_file" { + description = "Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth_mechanism is set to 'service_account'." + type = string + default = null +} + +variable "reason" { + description = "Used to differentiate connections that belong to the same asset" + type = string + default = "default" +} From 2b70b5383ecec3f131f704db9aad45ac9662a260 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Wed, 9 Oct 2024 12:47:05 -0700 Subject: [PATCH 07/16] add gcp pubsub module --- modules/dsfhub-gcp-pubsub/README.md | 0 modules/dsfhub-gcp-pubsub/main.tf | 27 +++++++++ modules/dsfhub-gcp-pubsub/outputs.tf | 4 ++ modules/dsfhub-gcp-pubsub/variables.tf | 78 ++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 modules/dsfhub-gcp-pubsub/README.md create mode 100644 modules/dsfhub-gcp-pubsub/main.tf create mode 100644 modules/dsfhub-gcp-pubsub/outputs.tf create mode 100644 modules/dsfhub-gcp-pubsub/variables.tf diff --git a/modules/dsfhub-gcp-pubsub/README.md b/modules/dsfhub-gcp-pubsub/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-pubsub/main.tf b/modules/dsfhub-gcp-pubsub/main.tf new file mode 100644 index 0000000..f0ffdc1 --- /dev/null +++ b/modules/dsfhub-gcp-pubsub/main.tf @@ -0,0 +1,27 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_log_aggregator" "this" { + server_type = "GCP PUBSUB" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + audit_pull_enabled = var.audit_pull_enabled + audit_type = var.audit_type + gateway_id = var.gateway_id + server_host_name = "pubsub.googleapis.com" + server_ip = "pubsub.googleapis.com" + server_port = "443" + + asset_connection { + auth_mechanism = var.auth_mechanism + reason = var.reason + key_file = var.auth_mechanism == "service_account" ? var.key_file : null + } +} diff --git a/modules/dsfhub-gcp-pubsub/outputs.tf b/modules/dsfhub-gcp-pubsub/outputs.tf new file mode 100644 index 0000000..a8945c8 --- /dev/null +++ b/modules/dsfhub-gcp-pubsub/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP PUBSUB asset." + value = dsfhub_log_aggregator.this +} diff --git a/modules/dsfhub-gcp-pubsub/variables.tf b/modules/dsfhub-gcp-pubsub/variables.tf new file mode 100644 index 0000000..7160ae1 --- /dev/null +++ b/modules/dsfhub-gcp-pubsub/variables.tf @@ -0,0 +1,78 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier of the Google PubSub Subscription in the form 'projects/{{project}}/subscriptions/{{name}}'." + type = string +} + +variable "audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "audit_type" { + description = "Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL_SLOW, MSSQL, POSTGRESQL, SPANNER." + type = string + default = null + validation { + condition = contains( + [ + "ALLOYDB_POSTGRESQL", + "BIGQUERY", + "BIGTABLE", + "MYSQL", + "MYSQL_SLOW", + "MSSQL", + "POSTGRESQL", + "SPANNER" + ], + var.audit_type + ) + error_message = "Invalid audit_type. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL_SLOW, MSSQL, POSTGRESQL, SPANNER." + } +} + +variable "auth_mechanism" { + description = "Specifies the auth mechanism used by the connection. Supported values: default, service_account." + type = string + default = "default" + validation { + condition = contains(["default", "service_account"], var.auth_mechanism) + error_message = "Invalid authentication mechanism. Supported values: default, service_account." + } +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "key_file" { + description = "Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth_mechanism is set to 'service_account'." + type = string + default = null +} + +variable "pubsub_subscription" { + description = "ID of the Google PubSub Subscription in the form 'projects/{{project}}/subscriptions/{{name}}'." + type = string + validation { + condition = can(regex("projects/.+/subscriptions/.+", var.pubsub_subscription)) + error_message = "Invalid pubsub subscription ID. Must be in the form 'projects/{{project}}/subscriptions/{{name}}'." + } +} + +variable "reason" { + description = "Used to differentiate connections that belong to the same asset" + type = string + default = "default" +} From d504715c92cb1eeeab3b81939eaf0fb0fb694003 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Wed, 9 Oct 2024 12:47:21 -0700 Subject: [PATCH 08/16] add gcp logging modules --- modules/google-logging-project-sink/README.md | 0 modules/google-logging-project-sink/main.tf | 18 +++++++ .../google-logging-project-sink/outputs.tf | 4 ++ .../google-logging-project-sink/variables.tf | 48 +++++++++++++++++++ modules/google-pubsub-subscription/README.md | 0 modules/google-pubsub-subscription/main.tf | 7 +++ modules/google-pubsub-subscription/outputs.tf | 4 ++ .../google-pubsub-subscription/variables.tf | 31 ++++++++++++ modules/google-pubsub-topic/README.md | 0 modules/google-pubsub-topic/main.tf | 4 ++ modules/google-pubsub-topic/outputs.tf | 4 ++ modules/google-pubsub-topic/variables.tf | 10 ++++ 12 files changed, 130 insertions(+) create mode 100644 modules/google-logging-project-sink/README.md create mode 100644 modules/google-logging-project-sink/main.tf create mode 100644 modules/google-logging-project-sink/outputs.tf create mode 100644 modules/google-logging-project-sink/variables.tf create mode 100644 modules/google-pubsub-subscription/README.md create mode 100644 modules/google-pubsub-subscription/main.tf create mode 100644 modules/google-pubsub-subscription/outputs.tf create mode 100644 modules/google-pubsub-subscription/variables.tf create mode 100644 modules/google-pubsub-topic/README.md create mode 100644 modules/google-pubsub-topic/main.tf create mode 100644 modules/google-pubsub-topic/outputs.tf create mode 100644 modules/google-pubsub-topic/variables.tf diff --git a/modules/google-logging-project-sink/README.md b/modules/google-logging-project-sink/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/google-logging-project-sink/main.tf b/modules/google-logging-project-sink/main.tf new file mode 100644 index 0000000..7e20c1d --- /dev/null +++ b/modules/google-logging-project-sink/main.tf @@ -0,0 +1,18 @@ +resource "google_logging_project_sink" "this" { + description = var.description + destination = "pubsub.googleapis.com/${var.pubsub_topic_id}" + filter = var.filter + name = var.name + project = var.project + + dynamic "exclusions" { + # If exclusions is not defined, do not create + for_each = var.exclusions != null ? [0] : [] + + content { + description = exclusions.value.description + filter = exclusions.value.filter + name = exclusions.value.name + } + } +} diff --git a/modules/google-logging-project-sink/outputs.tf b/modules/google-logging-project-sink/outputs.tf new file mode 100644 index 0000000..0717de9 --- /dev/null +++ b/modules/google-logging-project-sink/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP project level sink." + value = google_logging_project_sink.this +} diff --git a/modules/google-logging-project-sink/variables.tf b/modules/google-logging-project-sink/variables.tf new file mode 100644 index 0000000..e1aa200 --- /dev/null +++ b/modules/google-logging-project-sink/variables.tf @@ -0,0 +1,48 @@ +variable "description" { + description = "A description of this sink. The maximum length of the description is 8000 characters." + type = string + default = "" +} + +variable "exclusions" { + description = < Date: Wed, 9 Oct 2024 12:51:15 -0700 Subject: [PATCH 09/16] add pubsub_subscription field --- modules/dsfhub-gcp-pubsub/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dsfhub-gcp-pubsub/main.tf b/modules/dsfhub-gcp-pubsub/main.tf index f0ffdc1..16d50d3 100644 --- a/modules/dsfhub-gcp-pubsub/main.tf +++ b/modules/dsfhub-gcp-pubsub/main.tf @@ -15,6 +15,7 @@ resource "dsfhub_log_aggregator" "this" { audit_pull_enabled = var.audit_pull_enabled audit_type = var.audit_type gateway_id = var.gateway_id + pubsub_subscription = var.pubsub_subscription server_host_name = "pubsub.googleapis.com" server_ip = "pubsub.googleapis.com" server_port = "443" From 770cddfb19fcba8927ff5290a2734a87fefae633 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Sun, 13 Oct 2024 13:28:39 -0700 Subject: [PATCH 10/16] add bigquery example --- examples/onboard-gcp-bigquery/README.md | 14 ++ examples/onboard-gcp-bigquery/main.tf | 91 +++++++++++++ modules/dsfhub-gcp-bigquery/README.md | 0 modules/dsfhub-gcp-bigquery/main.tf | 23 ++++ modules/dsfhub-gcp-bigquery/outputs.tf | 4 + modules/dsfhub-gcp-bigquery/variables.tf | 46 +++++++ .../google-pubsub-subscription/variables.tf | 2 +- modules/google-pubsub-topic/variables.tf | 2 +- modules/google-service-account-dsf/README.md | 3 + .../google-service-account-dsf/variables.tf | 3 +- modules/onboard-gcp-pubsub/README.md | 2 + modules/onboard-gcp-pubsub/main.tf | 62 +++++++++ modules/onboard-gcp-pubsub/outputs.tf | 24 ++++ modules/onboard-gcp-pubsub/variables.tf | 128 ++++++++++++++++++ 14 files changed, 400 insertions(+), 4 deletions(-) create mode 100644 examples/onboard-gcp-bigquery/README.md create mode 100644 examples/onboard-gcp-bigquery/main.tf create mode 100644 modules/dsfhub-gcp-bigquery/README.md create mode 100644 modules/dsfhub-gcp-bigquery/main.tf create mode 100644 modules/dsfhub-gcp-bigquery/outputs.tf create mode 100644 modules/dsfhub-gcp-bigquery/variables.tf create mode 100644 modules/onboard-gcp-pubsub/README.md create mode 100644 modules/onboard-gcp-pubsub/main.tf create mode 100644 modules/onboard-gcp-pubsub/outputs.tf create mode 100644 modules/onboard-gcp-pubsub/variables.tf diff --git a/examples/onboard-gcp-bigquery/README.md b/examples/onboard-gcp-bigquery/README.md new file mode 100644 index 0000000..679297d --- /dev/null +++ b/examples/onboard-gcp-bigquery/README.md @@ -0,0 +1,14 @@ +# Onboard Google Cloud BigQuery example +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](hhttps://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/BigQuery-Onboarding-Steps_48367536.html). + +This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: +- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) +- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Prerequisites +### Service Account +A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway. + +### Google PubSub Subscription +A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + diff --git a/examples/onboard-gcp-bigquery/main.tf b/examples/onboard-gcp-bigquery/main.tf new file mode 100644 index 0000000..db93f83 --- /dev/null +++ b/examples/onboard-gcp-bigquery/main.tf @@ -0,0 +1,91 @@ +locals { + gcp_project_id = "my-gcp-project" + pubsub_subscription_name = "tf-bigquery-sub" + pubsub_topic_name = "tf-bigquery-topic" + service_account_name = "dsf-service-account" + sink_router_name = "tf-bigquery-sink" + + + admin_email = "test@example.com" + auth_mechanism = "default" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" +} + +################################################################################ +# Providers +################################################################################ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +provider "google" { + # Authenticated via "gcloud" CLI + project = local.gcp_project_id +} + +variable "dsfhub_host" {} # TF_VAR_dsfhub_host env variable +variable "dsfhub_token" {} # TF_VAR_dsfhub_token env variable + +provider "dsfhub" { + dsfhub_host = var.dsfhub_host + dsfhub_token = var.dsfhub_token +} + +################################################################################ +# Prerequisites +# 1. A service account with permissions to read from the PubSub subscription +# 2. A Google sink router, PubSub topic and subscription +################################################################################ +module "service-account" { + source = "../../modules/google-service-account-dsf" + + account_id = local.service_account_name + auth_mechanism = local.auth_mechanism + description = "BigQuery audit pull service account" + project = local.gcp_project_id + project_roles = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} + +module "gcp-pubsub" { + source = "../../modules/onboard-gcp-pubsub" + + gcp_pubsub_admin_email = local.admin_email + gcp_pubsub_audit_type = "BIGQUERY" + gcp_pubsub_auth_mechanism = local.auth_mechanism + gcp_pubsub_gateway_id = local.gateway_id + + project = local.gcp_project_id + + pubsub_subscription_name = local.pubsub_subscription_name + + pubsub_topic_name = local.pubsub_topic_name + + sink_router_description = "BigQuery sink" + sink_router_exclusions = null + sink_router_filter = < Date: Sun, 13 Oct 2024 13:33:34 -0700 Subject: [PATCH 11/16] add bigquery --- CHANGELOG.md | 6 ++++++ DSF_VERSION_COMPATABILITY.md | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f791df..265f4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.0.8 (TBD) + +### Features +- GCP account asset module +- Google Cloud BigQuery module + ## 1.0.7 (2024-10-03) ### Features diff --git a/DSF_VERSION_COMPATABILITY.md b/DSF_VERSION_COMPATABILITY.md index 2101f17..904eca2 100644 --- a/DSF_VERSION_COMPATABILITY.md +++ b/DSF_VERSION_COMPATABILITY.md @@ -99,5 +99,9 @@ The following table lists the DSF versions that each module is tested and mainta onboard-azure-sql-managed-instance 4.17+ + + onboard-gcp-bigquery + 4.17+ + \ No newline at end of file From d4a0f4e7a093405f2f48e386cb0bddbaf622964c Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Sun, 13 Oct 2024 20:34:18 +0000 Subject: [PATCH 12/16] Apply automatic changes --- examples/onboard-gcp-bigquery/main.tf | 48 ++++++++++++------------ modules/dsfhub-gcp-bigquery/main.tf | 26 ++++++------- modules/dsfhub-gcp-bigquery/outputs.tf | 2 +- modules/dsfhub-gcp-bigquery/variables.tf | 6 +-- modules/google-pubsub-topic/main.tf | 4 +- modules/onboard-gcp-pubsub/main.tf | 38 +++++++++---------- modules/onboard-gcp-pubsub/outputs.tf | 10 ++--- 7 files changed, 67 insertions(+), 67 deletions(-) diff --git a/examples/onboard-gcp-bigquery/main.tf b/examples/onboard-gcp-bigquery/main.tf index db93f83..cfee099 100644 --- a/examples/onboard-gcp-bigquery/main.tf +++ b/examples/onboard-gcp-bigquery/main.tf @@ -1,14 +1,14 @@ locals { - gcp_project_id = "my-gcp-project" + gcp_project_id = "my-gcp-project" pubsub_subscription_name = "tf-bigquery-sub" - pubsub_topic_name = "tf-bigquery-topic" - service_account_name = "dsf-service-account" - sink_router_name = "tf-bigquery-sink" + pubsub_topic_name = "tf-bigquery-topic" + service_account_name = "dsf-service-account" + sink_router_name = "tf-bigquery-sink" - admin_email = "test@example.com" + admin_email = "test@example.com" auth_mechanism = "default" - gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" } ################################################################################ @@ -43,10 +43,10 @@ provider "dsfhub" { module "service-account" { source = "../../modules/google-service-account-dsf" - account_id = local.service_account_name + account_id = local.service_account_name auth_mechanism = local.auth_mechanism - description = "BigQuery audit pull service account" - project = local.gcp_project_id + description = "BigQuery audit pull service account" + project = local.gcp_project_id project_roles = [ "roles/pubsub.subscriber", "roles/pubsub.viewer" @@ -56,23 +56,23 @@ module "service-account" { module "gcp-pubsub" { source = "../../modules/onboard-gcp-pubsub" - gcp_pubsub_admin_email = local.admin_email - gcp_pubsub_audit_type = "BIGQUERY" - gcp_pubsub_auth_mechanism = local.auth_mechanism - gcp_pubsub_gateway_id = local.gateway_id - - project = local.gcp_project_id - - pubsub_subscription_name = local.pubsub_subscription_name - - pubsub_topic_name = local.pubsub_topic_name - - sink_router_description = "BigQuery sink" - sink_router_exclusions = null - sink_router_filter = < Date: Sun, 13 Oct 2024 20:35:14 +0000 Subject: [PATCH 13/16] terraform-docs: automated action --- examples/onboard-gcp-bigquery/README.md | 33 +++++++++++ modules/dsfhub-gcp-bigquery/README.md | 40 +++++++++++++ modules/dsfhub-gcp-cloud-account/README.md | 39 +++++++++++++ modules/dsfhub-gcp-pubsub/README.md | 42 ++++++++++++++ modules/google-logging-project-sink/README.md | 38 ++++++++++++ modules/google-pubsub-subscription/README.md | 37 ++++++++++++ modules/google-pubsub-topic/README.md | 34 +++++++++++ modules/google-service-account-dsf/README.md | 48 +++++++++++++++ modules/onboard-gcp-pubsub/README.md | 58 +++++++++++++++++++ 9 files changed, 369 insertions(+) diff --git a/examples/onboard-gcp-bigquery/README.md b/examples/onboard-gcp-bigquery/README.md index 679297d..ecf4b93 100644 --- a/examples/onboard-gcp-bigquery/README.md +++ b/examples/onboard-gcp-bigquery/README.md @@ -12,3 +12,36 @@ A Google Service Account will need to be created with permissions to read from P ### Google PubSub Subscription A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-bigquery-1](#module\_gcp-bigquery-1) | ../../modules/dsfhub-gcp-bigquery | n/a | +| [gcp-pubsub](#module\_gcp-pubsub) | ../../modules/onboard-gcp-pubsub | n/a | +| [service-account](#module\_service-account) | ../../modules/google-service-account-dsf | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dsfhub\_host](#input\_dsfhub\_host) | n/a | `any` | n/a | yes | +| [dsfhub\_token](#input\_dsfhub\_token) | n/a | `any` | n/a | yes | + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/modules/dsfhub-gcp-bigquery/README.md b/modules/dsfhub-gcp-bigquery/README.md index e69de29..416b543 100644 --- a/modules/dsfhub-gcp-bigquery/README.md +++ b/modules/dsfhub-gcp-bigquery/README.md @@ -0,0 +1,40 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_data_source.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/data_source) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier for the BigQuery service in the form 'projects/{{project}}/bigquery'. | `string` | n/a | yes | +| [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this datasource is located. | `string` | `null` | no | +| [pubsub\_subscription](#input\_pubsub\_subscription) | ID of the Google PubSub Subscription containing the BigQuery audit logs in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP BIGQUERY asset. | + \ No newline at end of file diff --git a/modules/dsfhub-gcp-cloud-account/README.md b/modules/dsfhub-gcp-cloud-account/README.md index e69de29..659d013 100644 --- a/modules/dsfhub-gcp-cloud-account/README.md +++ b/modules/dsfhub-gcp-cloud-account/README.md @@ -0,0 +1,39 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_cloud_account.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/cloud_account) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier of the GCP account in the form ':' (e.g. my-service-account-name@my-project-id.iam.gserviceaccount.com:default-project-id-for-this-asset). | `string` | n/a | yes | +| [auth\_mechanism](#input\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [key\_file](#input\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no | +| [reason](#input\_reason) | Used to differentiate connections that belong to the same asset | `string` | `"default"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP cloud account asset. | + \ No newline at end of file diff --git a/modules/dsfhub-gcp-pubsub/README.md b/modules/dsfhub-gcp-pubsub/README.md index e69de29..24d2f87 100644 --- a/modules/dsfhub-gcp-pubsub/README.md +++ b/modules/dsfhub-gcp-pubsub/README.md @@ -0,0 +1,42 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_log_aggregator.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/log_aggregator) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier of the Google PubSub Subscription in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes | +| [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [audit\_type](#input\_audit\_type) | Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB\_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL\_SLOW, MSSQL, POSTGRESQL, SPANNER. | `string` | `null` | no | +| [auth\_mechanism](#input\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [key\_file](#input\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no | +| [pubsub\_subscription](#input\_pubsub\_subscription) | ID of the Google PubSub Subscription in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes | +| [reason](#input\_reason) | Used to differentiate connections that belong to the same asset | `string` | `"default"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP PUBSUB asset. | + \ No newline at end of file diff --git a/modules/google-logging-project-sink/README.md b/modules/google-logging-project-sink/README.md index e69de29..675606e 100644 --- a/modules/google-logging-project-sink/README.md +++ b/modules/google-logging-project-sink/README.md @@ -0,0 +1,38 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_logging_project_sink.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/logging_project_sink) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [description](#input\_description) | A description of this sink. The maximum length of the description is 8000 characters. | `string` | `""` | no | +| [exclusions](#input\_exclusions) | A list of exclusion objects as defined below.

exclusion:
- description: (Optional) A description of this exclusion.
- filter: An advanced logs filter that matches the log entries to be excluded.
- name: A client-assigned identifier for the exclusion filter. | 
list(
    object(
      {
        description = optional(string)
        filter      = string
        name        = string
      }
    )
  )
| `null` | no | +| [filter](#input\_filter) | The filter to apply when exporting logs. Only log entries that match the filter are exported. | `string` | n/a | yes | +| [name](#input\_name) | The name of the logging sink. | `string` | n/a | yes | +| [project](#input\_project) | The ID of the project to create the sink in. If omitted, the project associated with the provider is used. | `string` | `null` | no | +| [pubsub\_topic\_id](#input\_pubsub\_topic\_id) | ID of the PubSub topic to forward logs to, in the form of '/projects/[PROJECT\_ID]/topics/[TOPIC\_ID]'. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP project level sink. | + \ No newline at end of file diff --git a/modules/google-pubsub-subscription/README.md b/modules/google-pubsub-subscription/README.md index e69de29..81501a7 100644 --- a/modules/google-pubsub-subscription/README.md +++ b/modules/google-pubsub-subscription/README.md @@ -0,0 +1,37 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_subscription.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [message\_retention\_duration](#input\_message\_retention\_duration) | How long to retain unacknowledged messages in the subscription's backlog, from the moment a message is published. If retain\_acked\_messages is true, then this also configures the retention of acknowledged messages. Defaults to 7 days. | `string` | `"604800s"` | no | +| [name](#input\_name) | Name of the subscription. | `string` | n/a | yes | +| [project](#input\_project) | The ID of the project to create the subscription in. If omitted, the project associated with the provider is used. | `string` | `null` | no | +| [retain\_acked\_messages](#input\_retain\_acked\_messages) | Indicates whether to retain acknowledged messages. If true, then messages are not expunged from the subscription's backlog, even if they are acknowledged, until they fall out of the messageRetentionDuration window. Defaults to false. | `bool` | `false` | no | +| [topic](#input\_topic) | A reference to a Topic resource, of the form projects/{project}/topics/{{name}} (as in the id property of a google\_pubsub\_topic), or just a topic name if the topic is in the same project as the subscription. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | Google PubSub Subscription. | + \ No newline at end of file diff --git a/modules/google-pubsub-topic/README.md b/modules/google-pubsub-topic/README.md index e69de29..aa25adf 100644 --- a/modules/google-pubsub-topic/README.md +++ b/modules/google-pubsub-topic/README.md @@ -0,0 +1,34 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_topic.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [name](#input\_name) | Name of the topic. | `string` | n/a | yes | +| [project](#input\_project) | The ID of the project to create the topic in. If omitted, the project associated with the provider is used. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | Google PubSub Topic. | + \ No newline at end of file diff --git a/modules/google-service-account-dsf/README.md b/modules/google-service-account-dsf/README.md index 4b8c919..2ea3635 100644 --- a/modules/google-service-account-dsf/README.md +++ b/modules/google-service-account-dsf/README.md @@ -1,3 +1,51 @@ # google-service-account-dsf Creates a Google Service Account with permissiosn to be used for DSF audit pulls or discovery. Defaults to being granted the minimum permissions required to pull audit data from a PubSub subscription. If ``auth_mechanism = service_account`` is selected, will also create a JSON credentials file for the account that can be copied to an Agentless gateway. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | +| [local](#provider\_local) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_iam_member.permissions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_service_account.account](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | +| [google_service_account_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_key) | resource | +| [local_file.service_account_key_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_id](#input\_account\_id) | The account id that is used to generate the service account email address and a stable unique id. It is unique within a project, must be 6-30 characters long, and match the regular expression '[a-z]([-a-z0-9]*[a-z0-9])' to comply with RFC1035. Changing this forces a new service account to be created. | `string` | n/a | yes | +| [auth\_mechanism](#input\_auth\_mechanism) | Authentication mechanism intended to be used by DSF to leverage this service account. Valid values are: service\_account, default. | `string` | `"service_account"` | no | +| [create\_ignore\_already\_exists](#input\_create\_ignore\_already\_exists) | If set to true, skip service account creation if a service account with the same email already exists. Defaults to false. | `bool` | `false` | no | +| [description](#input\_description) | A text description of the service account. Must be less than or equal to 256 UTF-8 bytes. | `string` | `null` | no | +| [disabled](#input\_disabled) | Whether a service account is disabled or not. Defaults to false. This field has no effect during creation. Must be set after creation to disable a service account. | `bool` | `false` | no | +| [display\_name](#input\_display\_name) | The display name for the service account | `string` | `null` | no | +| [project](#input\_project) | The ID of the project that the service account will be created in. | `string` | n/a | yes | +| [project\_roles](#input\_project\_roles) | Roles to grant the Service Account in the specified project. Defaults to roles required for DSF to pull audit data. | `list(string)` | 
[
  "roles/pubsub.subscriber",
  "roles/pubsub.viewer"
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [account](#output\_account) | Google Service Account. | +| [key](#output\_key) | Authentication key for the Service Account. | +| [key\_file\_path](#output\_key\_file\_path) | JSON key file containing the Service Account credentials. | +| [permissions](#output\_permissions) | Service Account permissions. | + \ No newline at end of file diff --git a/modules/onboard-gcp-pubsub/README.md b/modules/onboard-gcp-pubsub/README.md index 26e5dd8..9b6c9c7 100644 --- a/modules/onboard-gcp-pubsub/README.md +++ b/modules/onboard-gcp-pubsub/README.md @@ -1,2 +1,60 @@ # onboard-gcp-pubsub Creates and onboards a Google PubSub Subscription to DSF Hub alongside a Logging Router and PubSub Topic that forward logs to the subscription. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-pubsub-asset](#module\_gcp-pubsub-asset) | ../dsfhub-gcp-pubsub | n/a | +| [pubsub-subscription](#module\_pubsub-subscription) | ../google-pubsub-subscription | n/a | +| [pubsub-topic](#module\_pubsub-topic) | ../google-pubsub-topic | n/a | +| [sink-router](#module\_sink-router) | ../google-logging-project-sink | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_topic_iam_binding.topic_binding](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_binding) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcp\_pubsub\_admin\_email](#input\_gcp\_pubsub\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [gcp\_pubsub\_audit\_pull\_enabled](#input\_gcp\_pubsub\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `null` | no | +| [gcp\_pubsub\_audit\_type](#input\_gcp\_pubsub\_audit\_type) | Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB\_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL\_SLOW, MSSQL, POSTGRESQL, SPANNER. | `string` | `null` | no | +| [gcp\_pubsub\_auth\_mechanism](#input\_gcp\_pubsub\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no | +| [gcp\_pubsub\_gateway\_id](#input\_gcp\_pubsub\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [gcp\_pubsub\_key\_file](#input\_gcp\_pubsub\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no | +| [gcp\_pubsub\_reason](#input\_gcp\_pubsub\_reason) | Used to differentiate connections that belong to the same asset | `string` | `"default"` | no | +| [project](#input\_project) | The ID of the project to create the resources in. If omitted, the project associated with the provider is used. | `string` | `null` | no | +| [pubsub\_subscription\_message\_retention\_duration](#input\_pubsub\_subscription\_message\_retention\_duration) | How long to retain unacknowledged messages in the subscription's backlog, from the moment a message is published. If retain\_acked\_messages is true, then this also configures the retention of acknowledged messages. Defaults to 7 days. | `string` | `"604800s"` | no | +| [pubsub\_subscription\_name](#input\_pubsub\_subscription\_name) | Name of the subscription. | `string` | n/a | yes | +| [pubsub\_subscription\_retain\_acked\_messages](#input\_pubsub\_subscription\_retain\_acked\_messages) | Indicates whether to retain acknowledged messages. If true, then messages are not expunged from the subscription's backlog, even if they are acknowledged, until they fall out of the messageRetentionDuration window. Defaults to false. | `bool` | `false` | no | +| [pubsub\_topic\_name](#input\_pubsub\_topic\_name) | Name of the topic. | `string` | n/a | yes | +| [sink\_router\_description](#input\_sink\_router\_description) | A description of this sink. The maximum length of the description is 8000 characters. | `string` | `""` | no | +| [sink\_router\_exclusions](#input\_sink\_router\_exclusions) | A list of exclusion objects as defined below.

exclusion:
- description: (Optional) A description of this exclusion.
- filter: An advanced logs filter that matches the log entries to be excluded.
- name: A client-assigned identifier for the exclusion filter. | 
list(
    object(
      {
        description = optional(string)
        filter      = string
        name        = string
      }
    )
  )
| `null` | no | +| [sink\_router\_filter](#input\_sink\_router\_filter) | The filter to apply when exporting logs. Only log entries that match the filter are exported. | `string` | n/a | yes | +| [sink\_router\_name](#input\_sink\_router\_name) | The name of the logging sink. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp-pubsub-asset](#output\_gcp-pubsub-asset) | GCP PUBSUB asset. | +| [pubsub-subscription](#output\_pubsub-subscription) | Google PubSub Subscription. | +| [pubsub-topic](#output\_pubsub-topic) | Google PubSub Topic. | +| [sink-router](#output\_sink-router) | Google Log Router. | +| [topic-binding](#output\_topic-binding) | Google PubSub Topic IAM Binding. | + \ No newline at end of file From e52fec7b1142332985d857456c2144d4e6b3c2e4 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 15 Oct 2024 09:53:03 -0700 Subject: [PATCH 14/16] resolve comments --- modules/dsfhub-gcp-bigquery/variables.tf | 2 +- modules/google-service-account-dsf/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dsfhub-gcp-bigquery/variables.tf b/modules/dsfhub-gcp-bigquery/variables.tf index be3b98b..91d9c66 100644 --- a/modules/dsfhub-gcp-bigquery/variables.tf +++ b/modules/dsfhub-gcp-bigquery/variables.tf @@ -31,7 +31,7 @@ variable "logs_destination_asset_id" { } variable "parent_asset_id" { - description = "The asset_id of the GCP asset representing the GCP account where this datasource is located." + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." type = string default = null } diff --git a/modules/google-service-account-dsf/README.md b/modules/google-service-account-dsf/README.md index 2ea3635..8448b96 100644 --- a/modules/google-service-account-dsf/README.md +++ b/modules/google-service-account-dsf/README.md @@ -1,5 +1,5 @@ # google-service-account-dsf -Creates a Google Service Account with permissiosn to be used for DSF audit pulls or discovery. Defaults to being granted the minimum permissions required to pull audit data from a PubSub subscription. If ``auth_mechanism = service_account`` is selected, will also create a JSON credentials file for the account that can be copied to an Agentless gateway. +Creates a Google Service Account with permissions to be used for DSF audit pulls or discovery. Defaults to being granted the minimum permissions required to pull audit data from a PubSub subscription. If ``auth_mechanism = service_account`` is selected, will also create a JSON credentials file for the account that can be copied to an Agentless gateway. From 9416d025637e76b7d5700a44640eb8e3eb82ee3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Oct 2024 16:54:30 +0000 Subject: [PATCH 15/16] terraform-docs: automated action --- modules/dsfhub-gcp-bigquery/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dsfhub-gcp-bigquery/README.md b/modules/dsfhub-gcp-bigquery/README.md index 416b543..848577b 100644 --- a/modules/dsfhub-gcp-bigquery/README.md +++ b/modules/dsfhub-gcp-bigquery/README.md @@ -29,7 +29,7 @@ No modules. | [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | | [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | | [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | -| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this datasource is located. | `string` | `null` | no | +| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | | [pubsub\_subscription](#input\_pubsub\_subscription) | ID of the Google PubSub Subscription containing the BigQuery audit logs in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes | ## Outputs From 7671c1ed56ed9accb1d337d72ce14d3b145ad50a Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 15 Oct 2024 12:22:08 -0700 Subject: [PATCH 16/16] update release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 265f4e8..c208f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 1.0.8 (TBD) +## 1.0.8 (2024-10-15) ### Features - GCP account asset module