From ba84648bed88ec9e56bcfe7c71db368cb38fa4ed Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Wed, 6 Nov 2024 13:58:32 -0800 Subject: [PATCH 01/30] add gcp mysql helper module --- modules/dsfhub-gcp-mysql/README.md | 0 modules/dsfhub-gcp-mysql/main.tf | 0 modules/dsfhub-gcp-mysql/outputs.tf | 4 +++ modules/dsfhub-gcp-mysql/variables.tf | 47 +++++++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 modules/dsfhub-gcp-mysql/README.md create mode 100644 modules/dsfhub-gcp-mysql/main.tf create mode 100644 modules/dsfhub-gcp-mysql/outputs.tf create mode 100644 modules/dsfhub-gcp-mysql/variables.tf diff --git a/modules/dsfhub-gcp-mysql/README.md b/modules/dsfhub-gcp-mysql/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-mysql/main.tf b/modules/dsfhub-gcp-mysql/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-mysql/outputs.tf b/modules/dsfhub-gcp-mysql/outputs.tf new file mode 100644 index 0000000..b6fad3b --- /dev/null +++ b/modules/dsfhub-gcp-mysql/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP MYSQL asset" + value = dsfhub_data_source.this +} diff --git a/modules/dsfhub-gcp-mysql/variables.tf b/modules/dsfhub-gcp-mysql/variables.tf new file mode 100644 index 0000000..78041ca --- /dev/null +++ b/modules/dsfhub-gcp-mysql/variables.tf @@ -0,0 +1,47 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier for the BigQuery service in the form 'projects/{{project}}/bigquery'." + type = string +} + +variable "audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "server_host_name" { + description = "Hostname (or IP if host is unknown) of the GCP MySQL instance" + type = string +} + +variable "server_ip" { + description = "IP address (or hostname if IP is unknown) of the GCP MySQL instance" + type = string +} From b94155e4efa5ef3f357a28799eb556e9343ba756 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Thu, 7 Nov 2024 09:25:57 -0800 Subject: [PATCH 02/30] add gcp sql instance module --- .../google-sql-database-instance/README.md | 0 modules/google-sql-database-instance/main.tf | 42 +++++++++ .../google-sql-database-instance/outputs.tf | 4 + .../google-sql-database-instance/variables.tf | 90 +++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 modules/google-sql-database-instance/README.md create mode 100644 modules/google-sql-database-instance/main.tf create mode 100644 modules/google-sql-database-instance/outputs.tf create mode 100644 modules/google-sql-database-instance/variables.tf diff --git a/modules/google-sql-database-instance/README.md b/modules/google-sql-database-instance/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/google-sql-database-instance/main.tf b/modules/google-sql-database-instance/main.tf new file mode 100644 index 0000000..75205d0 --- /dev/null +++ b/modules/google-sql-database-instance/main.tf @@ -0,0 +1,42 @@ +resource "google_sql_database_instance" "this" { + database_version = var.database_version + name = var.name + project = var.project + region = var.region + root_password = var.root_password + + settings { + tier = var.tier + + dynamic "database_flags" { + for_each = var.database_flags != null ? var.database_flags : [] + + content { + name = database_flags.value.name + value = database_flags.value.value + } + } + + ip_configuration { + dynamic "authorized_networks" { + for_each = var.authorized_networks != null ? var.authorized_networks : [] + + content { + expiration_time = authorized_networks.value.expiration_time + name = authorized_networks.value.name + value = authorized_networks.value.value + } + } + } + + dynamic "sql_server_audit_config" { + for_each = var.authorized_networks != null ? [0] : [] + + content { + bucket = var.sql_server_audit_config.bucket + upload_interval = var.sql_server_audit_config.upload_interval + retention_interval = var.sql_server_audit_config.retention_interval + } + } + } +} diff --git a/modules/google-sql-database-instance/outputs.tf b/modules/google-sql-database-instance/outputs.tf new file mode 100644 index 0000000..c58fe10 --- /dev/null +++ b/modules/google-sql-database-instance/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "Google SQL database instance" + value = google_sql_database_instance.this +} diff --git a/modules/google-sql-database-instance/variables.tf b/modules/google-sql-database-instance/variables.tf new file mode 100644 index 0000000..f40b003 --- /dev/null +++ b/modules/google-sql-database-instance/variables.tf @@ -0,0 +1,90 @@ +variable "authorized_networks" { + description = < Date: Thu, 7 Nov 2024 11:17:53 -0800 Subject: [PATCH 03/30] add gcp mysql onboarding module --- modules/dsfhub-gcp-mysql/main.tf | 22 ++++ modules/dsfhub-gcp-mysql/variables.tf | 2 +- .../google-sql-database-instance/variables.tf | 1 + modules/onboard-gcp-mysql/README.md | 10 ++ modules/onboard-gcp-mysql/main.tf | 27 +++++ modules/onboard-gcp-mysql/outputs.tf | 9 ++ modules/onboard-gcp-mysql/variables.tf | 103 ++++++++++++++++++ 7 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 modules/onboard-gcp-mysql/README.md create mode 100644 modules/onboard-gcp-mysql/main.tf create mode 100644 modules/onboard-gcp-mysql/outputs.tf create mode 100644 modules/onboard-gcp-mysql/variables.tf diff --git a/modules/dsfhub-gcp-mysql/main.tf b/modules/dsfhub-gcp-mysql/main.tf index e69de29..1943ebf 100644 --- a/modules/dsfhub-gcp-mysql/main.tf +++ b/modules/dsfhub-gcp-mysql/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_data_source" "this" { + server_type = "GCP MYSQL" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + audit_pull_enabled = var.audit_pull_enabled + gateway_id = var.gateway_id + logs_destination_asset_id = var.logs_destination_asset_id + parent_asset_id = var.parent_asset_id + server_host_name = var.server_host_name + server_ip = var.server_ip + +} diff --git a/modules/dsfhub-gcp-mysql/variables.tf b/modules/dsfhub-gcp-mysql/variables.tf index 78041ca..7babca4 100644 --- a/modules/dsfhub-gcp-mysql/variables.tf +++ b/modules/dsfhub-gcp-mysql/variables.tf @@ -9,7 +9,7 @@ variable "asset_display_name" { } variable "asset_id" { - description = "Unique identifier for the BigQuery service in the form 'projects/{{project}}/bigquery'." + description = "Unique identifier for the MySQL instance in the form '{project-id}:{instance-region}:{instance-name}'." type = string } diff --git a/modules/google-sql-database-instance/variables.tf b/modules/google-sql-database-instance/variables.tf index f40b003..120c56f 100644 --- a/modules/google-sql-database-instance/variables.tf +++ b/modules/google-sql-database-instance/variables.tf @@ -53,6 +53,7 @@ variable "name" { variable "project" { description = "The ID of the project that the service account will be created in." type = string + default = null } variable "region" { diff --git a/modules/onboard-gcp-mysql/README.md b/modules/onboard-gcp-mysql/README.md new file mode 100644 index 0000000..0cb5678 --- /dev/null +++ b/modules/onboard-gcp-mysql/README.md @@ -0,0 +1,10 @@ +# onboard-gcp-mysql + +Onboard Cloud SQL for MySQL to DSF Hub. + +## Notes +There are two prerequisites for using this module: +1. A Google Service Account with permissions to read from PubSub subscriptions. +2. A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF Hub. + +See the corresponding example for more details. diff --git a/modules/onboard-gcp-mysql/main.tf b/modules/onboard-gcp-mysql/main.tf new file mode 100644 index 0000000..8076fa2 --- /dev/null +++ b/modules/onboard-gcp-mysql/main.tf @@ -0,0 +1,27 @@ +module "gcp-mysql-instance" { + source = "../google-sql-database-instance" + + authorized_networks = var.instance_authorized_networks + database_flags = var.instance_database_flags + database_version = var.instance_database_version + name = var.instance_name + project = var.instance_project + region = var.instance_region + root_password = null + sql_server_audit_config = null + tier = var.instance_tier +} + +module "gcp-mysql-asset" { + source = "../dsfhub-gcp-mysql" + + admin_email = var.gcp_mysql_admin_email + asset_display_name = module.gcp-mysql-instance.this.name + asset_id = "${module.gcp-mysql-instance.this.project}:${module.gcp-mysql-instance.this.region}:${module.gcp-mysql-instance.this.name}" + audit_pull_enabled = var.gcp_mysql_audit_pull_enabled + gateway_id = var.gcp_mysql_gateway_id + logs_destination_asset_id = var.gcp_mysql_logs_destination_asset_id + parent_asset_id = var.gcp_mysql_parent_asset_id + server_host_name = module.gcp-mysql-instance.this.ip_address.0.ip_address + server_ip = module.gcp-mysql-instance.this.ip_address.0.ip_address +} diff --git a/modules/onboard-gcp-mysql/outputs.tf b/modules/onboard-gcp-mysql/outputs.tf new file mode 100644 index 0000000..2296af5 --- /dev/null +++ b/modules/onboard-gcp-mysql/outputs.tf @@ -0,0 +1,9 @@ +output "gcp-mysql-instance" { + description = "Google MySQL database instance" + value = module.gcp-mysql-instance.this +} + +output "gcp-mysql-asset" { + description = "GCP MYSQL asset" + value = module.gcp-mysql-asset.this +} diff --git a/modules/onboard-gcp-mysql/variables.tf b/modules/onboard-gcp-mysql/variables.tf new file mode 100644 index 0000000..7101af8 --- /dev/null +++ b/modules/onboard-gcp-mysql/variables.tf @@ -0,0 +1,103 @@ +variable "gcp_mysql_admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "gcp_mysql_audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gcp_mysql_gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "gcp_mysql_logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string +} + +variable "gcp_mysql_parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "instance_authorized_networks" { + description = < Date: Thu, 7 Nov 2024 11:39:15 -0800 Subject: [PATCH 04/30] fix typo in documentation URL --- examples/onboard-gcp-bigquery/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onboard-gcp-bigquery/README.md b/examples/onboard-gcp-bigquery/README.md index ecf4b93..af3d6f9 100644 --- a/examples/onboard-gcp-bigquery/README.md +++ b/examples/onboard-gcp-bigquery/README.md @@ -1,5 +1,5 @@ # Onboard Google Cloud BigQuery example -This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](hhttps://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/BigQuery-Onboarding-Steps_48367536.html). +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](https://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/BigQuery-Onboarding-Steps_48367536.html). This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: - [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) From 2a4fdf2cee18cec2d27b6ea807d47738323c0ac2 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Thu, 7 Nov 2024 15:49:40 -0800 Subject: [PATCH 05/30] add deletion protection --- modules/google-sql-database-instance/main.tf | 13 +++++++------ modules/google-sql-database-instance/variables.tf | 6 ++++++ modules/onboard-gcp-mysql/main.tf | 1 + modules/onboard-gcp-mysql/variables.tf | 6 ++++++ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/modules/google-sql-database-instance/main.tf b/modules/google-sql-database-instance/main.tf index 75205d0..cf699e2 100644 --- a/modules/google-sql-database-instance/main.tf +++ b/modules/google-sql-database-instance/main.tf @@ -1,9 +1,10 @@ resource "google_sql_database_instance" "this" { - database_version = var.database_version - name = var.name - project = var.project - region = var.region - root_password = var.root_password + database_version = var.database_version + deletion_protection = var.deletion_protection + name = var.name + project = var.project + region = var.region + root_password = var.root_password settings { tier = var.tier @@ -30,7 +31,7 @@ resource "google_sql_database_instance" "this" { } dynamic "sql_server_audit_config" { - for_each = var.authorized_networks != null ? [0] : [] + for_each = var.sql_server_audit_config != null ? [0] : [] content { bucket = var.sql_server_audit_config.bucket diff --git a/modules/google-sql-database-instance/variables.tf b/modules/google-sql-database-instance/variables.tf index 120c56f..cd1ede5 100644 --- a/modules/google-sql-database-instance/variables.tf +++ b/modules/google-sql-database-instance/variables.tf @@ -45,6 +45,12 @@ variable "database_version" { } } +variable "deletion_protection" { + description = "Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed." + type = bool + default = false +} + variable "name" { description = "The name of the instance." type = string diff --git a/modules/onboard-gcp-mysql/main.tf b/modules/onboard-gcp-mysql/main.tf index 8076fa2..4f18686 100644 --- a/modules/onboard-gcp-mysql/main.tf +++ b/modules/onboard-gcp-mysql/main.tf @@ -4,6 +4,7 @@ module "gcp-mysql-instance" { authorized_networks = var.instance_authorized_networks database_flags = var.instance_database_flags database_version = var.instance_database_version + deletion_protection = var.instance_deletion_protection name = var.instance_name project = var.instance_project region = var.instance_region diff --git a/modules/onboard-gcp-mysql/variables.tf b/modules/onboard-gcp-mysql/variables.tf index 7101af8..3326aa9 100644 --- a/modules/onboard-gcp-mysql/variables.tf +++ b/modules/onboard-gcp-mysql/variables.tf @@ -79,6 +79,12 @@ variable "instance_database_version" { } } +variable "instance_deletion_protection" { + description = "Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed." + type = bool + default = false +} + variable "instance_name" { description = "The name of the instance." type = string From e4482b8bb1c36df613e7518301bb77fd9030a089 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Fri, 8 Nov 2024 14:11:44 -0800 Subject: [PATCH 06/30] typos --- modules/google-logging-project-sink/main.tf | 2 +- modules/onboard-gcp-mysql/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/google-logging-project-sink/main.tf b/modules/google-logging-project-sink/main.tf index 7e20c1d..b70c0c4 100644 --- a/modules/google-logging-project-sink/main.tf +++ b/modules/google-logging-project-sink/main.tf @@ -7,7 +7,7 @@ resource "google_logging_project_sink" "this" { dynamic "exclusions" { # If exclusions is not defined, do not create - for_each = var.exclusions != null ? [0] : [] + for_each = var.exclusions != null ? var.exclusions : [] content { description = exclusions.value.description diff --git a/modules/onboard-gcp-mysql/variables.tf b/modules/onboard-gcp-mysql/variables.tf index 3326aa9..9a45200 100644 --- a/modules/onboard-gcp-mysql/variables.tf +++ b/modules/onboard-gcp-mysql/variables.tf @@ -62,7 +62,7 @@ variable "instance_database_flags" { }, { name = "general_log" - value = "On" + value = "on" } ] } From 5eb29d82ffe8bfa30a88065fa0b5ff56209cc654 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 12 Nov 2024 15:34:16 -0800 Subject: [PATCH 07/30] add gcp mysql examples --- examples/onboard-gcp-mysql/README.md | 13 ++ examples/onboard-gcp-mysql/main.tf | 262 ++++++++++++++++++++++++ modules/dsfhub-gcp-pubsub/main.tf | 1 + modules/dsfhub-gcp-pubsub/variables.tf | 8 +- modules/onboard-gcp-pubsub/main.tf | 1 + modules/onboard-gcp-pubsub/variables.tf | 8 +- 6 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 examples/onboard-gcp-mysql/README.md create mode 100644 examples/onboard-gcp-mysql/main.tf diff --git a/examples/onboard-gcp-mysql/README.md b/examples/onboard-gcp-mysql/README.md new file mode 100644 index 0000000..6e4d4ef --- /dev/null +++ b/examples/onboard-gcp-mysql/README.md @@ -0,0 +1,13 @@ +# Onboard Google Cloud SQL for MySQL example +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](https://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/Cloud-SQL-for-MySQL-Onboarding-Steps_48367584.html). + +This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: +- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) +- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Prerequisites +### Service Account +A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway. + +### Google PubSub Subscription +A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. diff --git a/examples/onboard-gcp-mysql/main.tf b/examples/onboard-gcp-mysql/main.tf new file mode 100644 index 0000000..a8b4618 --- /dev/null +++ b/examples/onboard-gcp-mysql/main.tf @@ -0,0 +1,262 @@ +# locals { +# admin_email = "test@example.com" +# gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" +# pubsub_auth_mechanism = "default" + +# gcp_mysql_instance_authorized_networks = [ +# { +# name = "local" +# value = "127.0.0.1" +# } +# ] +# gcp_project_id = "my-gcp-project" +# gcp_service_account_name = "dsf-service-account" + +# excluded_traffic_filter = [ +# { +# name = "exclude internal traffic" +# filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" +# } +# ] +# } + +locals { + admin_email = "matt.gill@thalesgroup.com" + gateway_id = "f2ba158f-6ead-42ef-be92-b10382df4ffa" + pubsub_auth_mechanism = "service_account" + + gcp_mysql_instance_authorized_networks = [ + { + name = "me" + value = "208.98.210.83" + } + ] + gcp_project_id = "cloudsql-postgres-audit" + gcp_service_account_name = "matt-tf-mysql-sa" + + excluded_traffic_filter = [ + { + description = "exclude internal traffic" + name = "dsf-exclusion" + filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" + } + ] +} + +################################################################################ +# Providers +################################################################################ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +provider "google" { + # Authenticated via "gcloud" CLI + project = local.gcp_project_id +} + +provider "dsfhub" {} + +################################################################################ +# Prerequisites +# 1. A service account with permissions to read from the PubSub subscription +# 2. A Google sink router, PubSub topic and subscription +################################################################################ +module "service-account" { + source = "../../modules/google-service-account-dsf" + + account_id = local.gcp_service_account_name + auth_mechanism = local.pubsub_auth_mechanism + description = "MySQL audit pull service account" + project = local.gcp_project_id + project_roles = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} + +# module "gcp-pubsub-1" { +# source = "../../modules/onboard-gcp-pubsub" + +# gcp_pubsub_admin_email = local.admin_email +# gcp_pubsub_audit_type = "MYSQL" +# gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism +# gcp_pubsub_gateway_id = local.gateway_id + +# project = local.gcp_project_id + +# pubsub_subscription_name = "tf-mysql-sub-1" + +# pubsub_topic_name = "tf-mysql-topic-1" + +# sink_router_description = "MySQL 8.0 sink" +# sink_router_exclusions = local.excluded_traffic_filter +# sink_router_filter = < Date: Tue, 12 Nov 2024 15:36:03 -0800 Subject: [PATCH 08/30] cleanup examples --- examples/onboard-gcp-mysql/main.tf | 190 +++++++++++++---------------- 1 file changed, 82 insertions(+), 108 deletions(-) diff --git a/examples/onboard-gcp-mysql/main.tf b/examples/onboard-gcp-mysql/main.tf index a8b4618..b2ca49c 100644 --- a/examples/onboard-gcp-mysql/main.tf +++ b/examples/onboard-gcp-mysql/main.tf @@ -1,44 +1,21 @@ -# locals { -# admin_email = "test@example.com" -# gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" -# pubsub_auth_mechanism = "default" - -# gcp_mysql_instance_authorized_networks = [ -# { -# name = "local" -# value = "127.0.0.1" -# } -# ] -# gcp_project_id = "my-gcp-project" -# gcp_service_account_name = "dsf-service-account" - -# excluded_traffic_filter = [ -# { -# name = "exclude internal traffic" -# filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" -# } -# ] -# } - locals { - admin_email = "matt.gill@thalesgroup.com" - gateway_id = "f2ba158f-6ead-42ef-be92-b10382df4ffa" - pubsub_auth_mechanism = "service_account" + admin_email = "test@example.com" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" + pubsub_auth_mechanism = "default" gcp_mysql_instance_authorized_networks = [ { - name = "me" - value = "208.98.210.83" + name = "local" + value = "127.0.0.1" } ] - gcp_project_id = "cloudsql-postgres-audit" - gcp_service_account_name = "matt-tf-mysql-sa" + gcp_project_id = "my-gcp-project" + gcp_service_account_name = "dsf-service-account" excluded_traffic_filter = [ { - description = "exclude internal traffic" - name = "dsf-exclusion" - filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" + name = "exclude internal traffic" + filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" } ] } @@ -79,29 +56,29 @@ module "service-account" { ] } -# module "gcp-pubsub-1" { -# source = "../../modules/onboard-gcp-pubsub" +module "gcp-pubsub-1" { + source = "../../modules/onboard-gcp-pubsub" -# gcp_pubsub_admin_email = local.admin_email -# gcp_pubsub_audit_type = "MYSQL" -# gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism -# gcp_pubsub_gateway_id = local.gateway_id + gcp_pubsub_admin_email = local.admin_email + gcp_pubsub_audit_type = "MYSQL" + gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism + gcp_pubsub_gateway_id = local.gateway_id -# project = local.gcp_project_id + project = local.gcp_project_id -# pubsub_subscription_name = "tf-mysql-sub-1" + pubsub_subscription_name = "tf-mysql-sub-1" -# pubsub_topic_name = "tf-mysql-topic-1" + pubsub_topic_name = "tf-mysql-topic-1" -# sink_router_description = "MySQL 8.0 sink" -# sink_router_exclusions = local.excluded_traffic_filter -# sink_router_filter = < Date: Tue, 12 Nov 2024 15:38:40 -0800 Subject: [PATCH 09/30] set required provider version --- modules/dsfhub-gcp-pubsub/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dsfhub-gcp-pubsub/main.tf b/modules/dsfhub-gcp-pubsub/main.tf index fcffbbb..f4ce784 100644 --- a/modules/dsfhub-gcp-pubsub/main.tf +++ b/modules/dsfhub-gcp-pubsub/main.tf @@ -2,6 +2,7 @@ terraform { required_providers { dsfhub = { source = "imperva/dsfhub" + version = ">= 1.3.5" } } } From a594819e57aa4284a068825aee3c8a9c401bab0a Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 12 Nov 2024 15:41:42 -0800 Subject: [PATCH 10/30] add gcp mysql module --- CHANGELOG.md | 1 + DSF_VERSION_COMPATABILITY.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3397bd6..2b80201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features - Aurora PostgreSQL CloudWatch with slow query auditing example +- Google Cloud SQL for MySQL module ## 1.0.8 (2024-10-15) diff --git a/DSF_VERSION_COMPATABILITY.md b/DSF_VERSION_COMPATABILITY.md index 904eca2..05570b0 100644 --- a/DSF_VERSION_COMPATABILITY.md +++ b/DSF_VERSION_COMPATABILITY.md @@ -103,5 +103,9 @@ The following table lists the DSF versions that each module is tested and mainta onboard-gcp-bigquery 4.17+ + + onboard-gcp-mysql + 4.17+ + \ No newline at end of file From d049febd7b2df0b1b3a5fc27be9b5323f5cad0d4 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 12 Nov 2024 23:42:23 +0000 Subject: [PATCH 11/30] Apply automatic changes --- modules/dsfhub-gcp-pubsub/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dsfhub-gcp-pubsub/main.tf b/modules/dsfhub-gcp-pubsub/main.tf index f4ce784..1621592 100644 --- a/modules/dsfhub-gcp-pubsub/main.tf +++ b/modules/dsfhub-gcp-pubsub/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { dsfhub = { - source = "imperva/dsfhub" + source = "imperva/dsfhub" version = ">= 1.3.5" } } From e50568a107a53ed9b68ad3dadc337015b95aca0c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 Nov 2024 23:43:17 +0000 Subject: [PATCH 12/30] terraform-docs: automated action --- examples/onboard-gcp-mysql/README.md | 35 ++++++++++++++ modules/dsfhub-gcp-mysql/README.md | 41 +++++++++++++++++ modules/dsfhub-gcp-pubsub/README.md | 7 ++- .../google-sql-database-instance/README.md | 42 +++++++++++++++++ modules/onboard-gcp-mysql/README.md | 46 +++++++++++++++++++ modules/onboard-gcp-pubsub/README.md | 1 + 6 files changed, 170 insertions(+), 2 deletions(-) diff --git a/examples/onboard-gcp-mysql/README.md b/examples/onboard-gcp-mysql/README.md index 6e4d4ef..ed9e377 100644 --- a/examples/onboard-gcp-mysql/README.md +++ b/examples/onboard-gcp-mysql/README.md @@ -11,3 +11,38 @@ A Google Service Account will need to be created with permissions to read from P ### Google PubSub Subscription A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-mysql-1](#module\_gcp-mysql-1) | ../../modules/onboard-gcp-mysql | n/a | +| [gcp-mysql-2](#module\_gcp-mysql-2) | ../../modules/onboard-gcp-mysql | n/a | +| [gcp-mysql-3](#module\_gcp-mysql-3) | ../../modules/onboard-gcp-mysql | n/a | +| [gcp-pubsub-1](#module\_gcp-pubsub-1) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-2-audit](#module\_gcp-pubsub-2-audit) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-2-slow-query](#module\_gcp-pubsub-2-slow-query) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-3](#module\_gcp-pubsub-3) | ../../modules/onboard-gcp-pubsub | n/a | +| [service-account](#module\_service-account) | ../../modules/google-service-account-dsf | n/a | + +## Resources + +No resources. + +## Inputs + +No inputs. + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/modules/dsfhub-gcp-mysql/README.md b/modules/dsfhub-gcp-mysql/README.md index e69de29..e28b167 100644 --- a/modules/dsfhub-gcp-mysql/README.md +++ b/modules/dsfhub-gcp-mysql/README.md @@ -0,0 +1,41 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_data_source.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/data_source) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier for the MySQL instance in the form '{project-id}:{instance-region}:{instance-name}'. | `string` | n/a | yes | +| [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [server\_host\_name](#input\_server\_host\_name) | Hostname (or IP if host is unknown) of the GCP MySQL instance | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | IP address (or hostname if IP is unknown) of the GCP MySQL instance | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP MYSQL asset | + \ No newline at end of file diff --git a/modules/dsfhub-gcp-pubsub/README.md b/modules/dsfhub-gcp-pubsub/README.md index 24d2f87..b9e3feb 100644 --- a/modules/dsfhub-gcp-pubsub/README.md +++ b/modules/dsfhub-gcp-pubsub/README.md @@ -1,13 +1,15 @@ ## Requirements -No requirements. +| Name | Version | +|------|---------| +| [dsfhub](#requirement\_dsfhub) | >= 1.3.5 | ## Providers | Name | Version | |------|---------| -| [dsfhub](#provider\_dsfhub) | n/a | +| [dsfhub](#provider\_dsfhub) | >= 1.3.5 | ## Modules @@ -29,6 +31,7 @@ No modules. | [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | | [audit\_type](#input\_audit\_type) | Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB\_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL\_SLOW, MSSQL, POSTGRESQL, SPANNER. | `string` | `null` | no | | [auth\_mechanism](#input\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no | +| [content\_type](#input\_content\_type) | Desired 'parent' asset 'Server Type' which is the one tha tuses this asset as a destination for logs. NOTE: The content\_type field will take precedence on the lookup for parent\_asset\_id field when checking which server is sending logs to this asset. | `string` | `null` | no | | [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | | [key\_file](#input\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no | | [pubsub\_subscription](#input\_pubsub\_subscription) | ID of the Google PubSub Subscription in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes | diff --git a/modules/google-sql-database-instance/README.md b/modules/google-sql-database-instance/README.md index e69de29..7a83f6a 100644 --- a/modules/google-sql-database-instance/README.md +++ b/modules/google-sql-database-instance/README.md @@ -0,0 +1,42 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_sql_database_instance.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [authorized\_networks](#input\_authorized\_networks) | A list of authorized network blocks as defined below.

authorized\_network:
- expiration\_time: (Optional) The RFC 3339 formatted date time string indicating when this whitelist expires.
- name: (Optional) A name for this whitelist entry.
- value: A CIDR notation IPv4 or IPv6 address that is allowed to access this instance. | 
list(
    object(
      {
        expiration_time = optional(string)
        name            = optional(string)
        value           = string
      }
    )
  )
| `null` | no | +| [database\_flags](#input\_database\_flags) | List of database flags to assign to the instance. | 
list(
    object(
      {
        name  = string
        value = string
      }
    )
  )
| `null` | no | +| [database\_version](#input\_database\_version) | The MySQL, PostgreSQL or SQL Server version to use. The full list of supported versions can be found at https://cloud.google.com/sql/docs/db-versions. | `string` | n/a | yes | +| [deletion\_protection](#input\_deletion\_protection) | Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed. | `bool` | `false` | no | +| [name](#input\_name) | The name of the instance. | `string` | n/a | yes | +| [project](#input\_project) | The ID of the project that the service account will be created in. | `string` | `null` | no | +| [region](#input\_region) | The region the instance will sit in. If a region is not provided in the resource definition, the provider region will be used instead. | `string` | `null` | no | +| [root\_password](#input\_root\_password) | Initial root password. Can be updated. Required for MS SQL Server. | `string` | `null` | no | +| [sql\_server\_audit\_config](#input\_sql\_server\_audit\_config) | A block describing a SQL Server audit configuration as described below.

- bucket: (Optional) The name of the destination bucket (e.g., gs://mybucket).
- upload\_interval: (Optional) How often to upload generated audit files. A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s".
- retention\_interval: (Optional) How long to keep generated audit files. A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". | 
object({
    bucket             = optional(string)
    upload_interval    = optional(string)
    retention_interval = optional(string)
  })
| `null` | no | +| [tier](#input\_tier) | The machine type to use. See [tiers](https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/tiers) for more details and supported versions | `string` | `"db-f1-micro"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | Google SQL database instance | + \ No newline at end of file diff --git a/modules/onboard-gcp-mysql/README.md b/modules/onboard-gcp-mysql/README.md index 0cb5678..c5848fb 100644 --- a/modules/onboard-gcp-mysql/README.md +++ b/modules/onboard-gcp-mysql/README.md @@ -8,3 +8,49 @@ There are two prerequisites for using this module: 2. A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF Hub. See the corresponding example for more details. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-mysql-asset](#module\_gcp-mysql-asset) | ../dsfhub-gcp-mysql | n/a | +| [gcp-mysql-instance](#module\_gcp-mysql-instance) | ../google-sql-database-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcp\_mysql\_admin\_email](#input\_gcp\_mysql\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [gcp\_mysql\_audit\_pull\_enabled](#input\_gcp\_mysql\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gcp\_mysql\_gateway\_id](#input\_gcp\_mysql\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [gcp\_mysql\_logs\_destination\_asset\_id](#input\_gcp\_mysql\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | n/a | yes | +| [gcp\_mysql\_parent\_asset\_id](#input\_gcp\_mysql\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [instance\_authorized\_networks](#input\_instance\_authorized\_networks) | A list of authorized network blocks as defined below.

authorized\_network:
- expiration\_time: (Optional) The RFC 3339 formatted date time string indicating when this whitelist expires.
- name: (Optional) A name for this whitelist entry.
- value: A CIDR notation IPv4 or IPv6 address that is allowed to access this instance. | 
list(
    object(
      {
        expiration_time = optional(string)
        name            = optional(string)
        value           = string
      }
    )
  )
| n/a | yes | +| [instance\_database\_flags](#input\_instance\_database\_flags) | List of database flags to assign to the instance. | 
list(
    object(
      {
        name  = string
        value = string
      }
    )
  )
| 
[
  {
    "name": "log_output",
    "value": "FILE"
  },
  {
    "name": "general_log",
    "value": "on"
  }
]
| no | +| [instance\_database\_version](#input\_instance\_database\_version) | The MySQL version to use. The full list of supported versions can be found at https://cloud.google.com/sql/docs/db-versions. | `string` | `"MYSQL_8_0"` | no | +| [instance\_deletion\_protection](#input\_instance\_deletion\_protection) | Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed. | `bool` | `false` | no | +| [instance\_name](#input\_instance\_name) | The name of the instance. | `string` | n/a | yes | +| [instance\_project](#input\_instance\_project) | The ID of the project that the service account will be created in. | `string` | `null` | no | +| [instance\_region](#input\_instance\_region) | The region the instance will sit in. If a region is not provided in the resource definition, the provider region will be used instead. | `string` | `null` | no | +| [instance\_tier](#input\_instance\_tier) | The machine type to use. See [tiers](https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/tiers) for more details and supported versions | `string` | `"db-f1-micro"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp-mysql-asset](#output\_gcp-mysql-asset) | GCP MYSQL asset | +| [gcp-mysql-instance](#output\_gcp-mysql-instance) | Google MySQL database instance | + \ No newline at end of file diff --git a/modules/onboard-gcp-pubsub/README.md b/modules/onboard-gcp-pubsub/README.md index 9b6c9c7..9c4725b 100644 --- a/modules/onboard-gcp-pubsub/README.md +++ b/modules/onboard-gcp-pubsub/README.md @@ -35,6 +35,7 @@ No requirements. | [gcp\_pubsub\_audit\_pull\_enabled](#input\_gcp\_pubsub\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `null` | no | | [gcp\_pubsub\_audit\_type](#input\_gcp\_pubsub\_audit\_type) | Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB\_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL\_SLOW, MSSQL, POSTGRESQL, SPANNER. | `string` | `null` | no | | [gcp\_pubsub\_auth\_mechanism](#input\_gcp\_pubsub\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no | +| [gcp\_pubsub\_content\_type](#input\_gcp\_pubsub\_content\_type) | Desired 'parent' asset 'Server Type' which is the one tha tuses this asset as a destination for logs. NOTE: The content\_type field will take precedence on the lookup for parent\_asset\_id field when checking which server is sending logs to this asset. | `string` | `null` | no | | [gcp\_pubsub\_gateway\_id](#input\_gcp\_pubsub\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | | [gcp\_pubsub\_key\_file](#input\_gcp\_pubsub\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no | | [gcp\_pubsub\_reason](#input\_gcp\_pubsub\_reason) | Used to differentiate connections that belong to the same asset | `string` | `"default"` | no | From c1cdd066a8a618b95ae3d68152746f9ec25070aa Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 12 Nov 2024 15:43:39 -0800 Subject: [PATCH 13/30] remove spaces from filter name --- examples/onboard-gcp-mysql/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onboard-gcp-mysql/main.tf b/examples/onboard-gcp-mysql/main.tf index b2ca49c..7efb94c 100644 --- a/examples/onboard-gcp-mysql/main.tf +++ b/examples/onboard-gcp-mysql/main.tf @@ -14,7 +14,7 @@ locals { excluded_traffic_filter = [ { - name = "exclude internal traffic" + name = "exclude-internal-traffic" filter = "textPayload:\"[root]\" OR \"__google_connectivity_prober\"" } ] From 4fac2e5115b06abf6f75df047ab20d1d82916c20 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Thu, 14 Nov 2024 12:10:44 -0800 Subject: [PATCH 14/30] refactor examples --- examples/onboard-gcp-mysql/main.tf | 166 ++++++++++++++++++----------- 1 file changed, 102 insertions(+), 64 deletions(-) diff --git a/examples/onboard-gcp-mysql/main.tf b/examples/onboard-gcp-mysql/main.tf index 7efb94c..2bccd3b 100644 --- a/examples/onboard-gcp-mysql/main.tf +++ b/examples/onboard-gcp-mysql/main.tf @@ -41,7 +41,7 @@ provider "dsfhub" {} ################################################################################ # Prerequisites # 1. A service account with permissions to read from the PubSub subscription -# 2. A Google sink router, PubSub topic and subscription +# 2. A Google sink router, PubSub topic and subscription (handled below) ################################################################################ module "service-account" { source = "../../modules/google-service-account-dsf" @@ -56,6 +56,13 @@ module "service-account" { ] } +################################################################################ +# GCP MySQL 8.0 +################################################################################ +locals { + gcp_mysql_1_instance_name = "tf-mysql-8" +} + module "gcp-pubsub-1" { source = "../../modules/onboard-gcp-pubsub" @@ -66,18 +73,49 @@ module "gcp-pubsub-1" { project = local.gcp_project_id - pubsub_subscription_name = "tf-mysql-sub-1" + pubsub_subscription_name = "${local.gcp_mysql_1_instance_name}-sub" - pubsub_topic_name = "tf-mysql-topic-1" + pubsub_topic_name = "${local.gcp_mysql_1_instance_name}-topic" sink_router_description = "MySQL 8.0 sink" sink_router_exclusions = local.excluded_traffic_filter sink_router_filter = < Date: Fri, 15 Nov 2024 13:02:12 -0800 Subject: [PATCH 15/30] update audit_type description --- modules/dsfhub-gcp-pubsub/variables.tf | 2 +- modules/onboard-gcp-pubsub/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dsfhub-gcp-pubsub/variables.tf b/modules/dsfhub-gcp-pubsub/variables.tf index 2b1c30d..a672aa7 100644 --- a/modules/dsfhub-gcp-pubsub/variables.tf +++ b/modules/dsfhub-gcp-pubsub/variables.tf @@ -20,7 +20,7 @@ variable "audit_pull_enabled" { } variable "audit_type" { - description = "Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL_SLOW, MSSQL, POSTGRESQL, SPANNER." + description = "Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, GCP_MYSQL_SLOW, MYSQL, MSSQL, POSTGRESQL, SPANNER." type = string default = null validation { diff --git a/modules/onboard-gcp-pubsub/variables.tf b/modules/onboard-gcp-pubsub/variables.tf index e2d86b8..72f3c9a 100644 --- a/modules/onboard-gcp-pubsub/variables.tf +++ b/modules/onboard-gcp-pubsub/variables.tf @@ -10,7 +10,7 @@ variable "gcp_pubsub_audit_pull_enabled" { } variable "gcp_pubsub_audit_type" { - description = "Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, MYSQL, MYSQL_SLOW, MSSQL, POSTGRESQL, SPANNER." + description = "Identifier for the type of audit data contained within the PubSub Subscription. Supported values: ALLOYDB_POSTGRESQL, BIGQUERY, BIGTABLE, GCP_MYSQL_SLOW, MYSQL, MSSQL, POSTGRESQL, SPANNER." type = string default = null validation { From c1df39aba7c5ef78c4636cf9d351b6ec930d3a40 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Fri, 15 Nov 2024 15:05:21 -0800 Subject: [PATCH 16/30] add gcp postgresql asset helper module --- modules/dsfhub-gcp-postgresql/README.md | 0 modules/dsfhub-gcp-postgresql/main.tf | 22 ++++++++++ modules/dsfhub-gcp-postgresql/outputs.tf | 4 ++ modules/dsfhub-gcp-postgresql/variables.tf | 47 ++++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 modules/dsfhub-gcp-postgresql/README.md create mode 100644 modules/dsfhub-gcp-postgresql/main.tf create mode 100644 modules/dsfhub-gcp-postgresql/outputs.tf create mode 100644 modules/dsfhub-gcp-postgresql/variables.tf diff --git a/modules/dsfhub-gcp-postgresql/README.md b/modules/dsfhub-gcp-postgresql/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-postgresql/main.tf b/modules/dsfhub-gcp-postgresql/main.tf new file mode 100644 index 0000000..c1a8804 --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_data_source" "this" { + server_type = "GCP POSTGRESQL" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + audit_pull_enabled = var.audit_pull_enabled + gateway_id = var.gateway_id + logs_destination_asset_id = var.logs_destination_asset_id + parent_asset_id = var.parent_asset_id + server_host_name = var.server_host_name + server_ip = var.server_ip + server_port = "5432" +} diff --git a/modules/dsfhub-gcp-postgresql/outputs.tf b/modules/dsfhub-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..18b138c --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP POSTGRESQL asset." + value = dsfhub_data_source.this +} diff --git a/modules/dsfhub-gcp-postgresql/variables.tf b/modules/dsfhub-gcp-postgresql/variables.tf new file mode 100644 index 0000000..21db9de --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/variables.tf @@ -0,0 +1,47 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier for the PostgreSQL instance in the form '{project-id}:{instance-region}:{instance-name}'." + type = string +} + +variable "audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "server_host_name" { + description = "Hostname (or IP if host is unknown) of the GCP PostgreSQL instance" + type = string +} + +variable "server_ip" { + description = "IP address (or hostname if IP is unknown) of the GCP PostgreSQL instance" + type = string +} From 90f2dab29ac9e4fbe1e4806f9d500c688dffd5e9 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Fri, 15 Nov 2024 15:06:48 -0800 Subject: [PATCH 17/30] add gcp postgres onboarding module --- modules/onboard-gcp-postgresql/README.md | 11 ++ modules/onboard-gcp-postgresql/main.tf | 28 +++++ modules/onboard-gcp-postgresql/outputs.tf | 0 modules/onboard-gcp-postgresql/variables.tf | 130 ++++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 modules/onboard-gcp-postgresql/README.md create mode 100644 modules/onboard-gcp-postgresql/main.tf create mode 100644 modules/onboard-gcp-postgresql/outputs.tf create mode 100644 modules/onboard-gcp-postgresql/variables.tf diff --git a/modules/onboard-gcp-postgresql/README.md b/modules/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..616eefa --- /dev/null +++ b/modules/onboard-gcp-postgresql/README.md @@ -0,0 +1,11 @@ +# onboard-gcp-postgresql + +Onboard Cloud SQL for PostgreSQL to DSF Hub. + +## Notes +There are three prerequisites for using this module: +1. A Google Service Account with permissions to read from PubSub subscriptions. +2. A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF Hub. +3. A method to create the 'pgaudit' extension on the postgres instance. + +See the corresponding example for more details. diff --git a/modules/onboard-gcp-postgresql/main.tf b/modules/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..94eb66a --- /dev/null +++ b/modules/onboard-gcp-postgresql/main.tf @@ -0,0 +1,28 @@ +module "gcp-postgresql-instance" { + source = "../google-sql-database-instance" + + authorized_networks = var.instance_authorized_networks + database_flags = var.instance_database_flags + database_version = var.instance_database_version + deletion_protection = var.instance_deletion_protection + name = var.instance_name + project = var.instance_project + region = var.instance_region + root_password = null + sql_server_audit_config = null + tier = var.instance_tier +} + +module "gcp-postgresql-asset" { + source = "../dsfhub-gcp-postgresql" + + admin_email = var.gcp_postgresql_admin_email + asset_display_name = module.gcp-postgresql-instance.this.name + asset_id = "${module.gcp-postgresql-instance.this.project}:${module.gcp-postgresql-instance.this.region}:${module.gcp-postgresql-instance.this.name}" + audit_pull_enabled = var.gcp_postgresql_audit_pull_enabled + gateway_id = var.gcp_postgresql_gateway_id + logs_destination_asset_id = var.gcp_postgresql_logs_destination_asset_id + parent_asset_id = var.gcp_postgresql_parent_asset_id + server_host_name = module.gcp-postgresql-instance.this.ip_address.0.ip_address + server_ip = module.gcp-postgresql-instance.this.ip_address.0.ip_address +} diff --git a/modules/onboard-gcp-postgresql/outputs.tf b/modules/onboard-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/onboard-gcp-postgresql/variables.tf b/modules/onboard-gcp-postgresql/variables.tf new file mode 100644 index 0000000..bf46677 --- /dev/null +++ b/modules/onboard-gcp-postgresql/variables.tf @@ -0,0 +1,130 @@ +variable "gcp_postgresql_admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "gcp_postgresql_audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gcp_postgresql_gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "gcp_postgresql_logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "gcp_postgresql_parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "instance_authorized_networks" { + description = < Date: Fri, 15 Nov 2024 15:34:20 -0800 Subject: [PATCH 18/30] add outputs --- modules/onboard-gcp-postgresql/outputs.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/onboard-gcp-postgresql/outputs.tf b/modules/onboard-gcp-postgresql/outputs.tf index e69de29..7791c86 100644 --- a/modules/onboard-gcp-postgresql/outputs.tf +++ b/modules/onboard-gcp-postgresql/outputs.tf @@ -0,0 +1,9 @@ +output "gcp-postgresql-instance" { + description = "Google PostgreSQL database instance" + value = module.gcp-postgresql-instance.this +} + +output "gcp-postgresql-asset" { + description = "GCP POSTGRESQL asset" + value = module.gcp-postgresql-asset.this +} From 7976eef7fb6fad0b1681b768e9a8c95aa5d08624 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 18 Nov 2024 12:09:25 -0800 Subject: [PATCH 19/30] add gcp postgres onboarding examples --- examples/onboard-gcp-postgresql/README.md | 16 + .../configure_database.sh | 38 ++ .../configure_database.sql | 11 + examples/onboard-gcp-postgresql/main.tf | 360 ++++++++++++++++++ modules/onboard-gcp-postgresql/variables.tf | 4 +- 5 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 examples/onboard-gcp-postgresql/README.md create mode 100755 examples/onboard-gcp-postgresql/configure_database.sh create mode 100644 examples/onboard-gcp-postgresql/configure_database.sql create mode 100644 examples/onboard-gcp-postgresql/main.tf diff --git a/examples/onboard-gcp-postgresql/README.md b/examples/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..cea5877 --- /dev/null +++ b/examples/onboard-gcp-postgresql/README.md @@ -0,0 +1,16 @@ +# Onboard Google Cloud SQL for PostgreSQL example +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](https://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/Cloud-SQL-for-PostgreSQL-Onboarding-Steps_48367600.html). + +This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: +- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) +- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Prerequisites +### Service Account +A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway. + +### Google PubSub Subscription +A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + +### Database Configuration +Part of the onboarding process involves connecting to your Google PostgreSQL instance and running SQL commands to create an extension. This module includes an example for how to connect to the instance from your local machine and create this. diff --git a/examples/onboard-gcp-postgresql/configure_database.sh b/examples/onboard-gcp-postgresql/configure_database.sh new file mode 100755 index 0000000..ff713bc --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Configures Google Postgresql database for auditing by connecting to the +# database using 'psql' +# +# Connection to the database uses PG environment variables +# See postgres documentation for more information: +# https://www.postgresql.org/docs/current/libpq-envars.html +################################################################################ + +# Settings +current_directory=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +sql_file="${current_directory}/configure_database.sql" + +# Functions +function is_pkg_installed { + local pkg="$1" + if ! command -v "${pkg}" &> /dev/null + then + echo "Package '${pkg}' is not installed." + echo "Install on MacOS: brew install libpq" + echo "Install on Ubuntu: apt-get install -y libpq-dev" + echo "Install on CentOS: yum install -y libpq" + echo "Exiting..." + exit 1 + else + return 0 + fi +} + +################################################################################ +is_pkg_installed "psql" +if [ ! -r "${sql_file}" ]; then + echo "Unable to read ${sql_file}" + echo "Exiting..." + exit 1 +else + psql --file="${sql_file}" +fi diff --git a/examples/onboard-gcp-postgresql/configure_database.sql b/examples/onboard-gcp-postgresql/configure_database.sql new file mode 100644 index 0000000..829e3e1 --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sql @@ -0,0 +1,11 @@ +DO +$$ +BEGIN + RAISE NOTICE 'Creating audit extension "pgaudit".'; + IF EXISTS (SELECT FROM pg_catalog.pg_extension WHERE extname = 'pgaudit') THEN + RAISE NOTICE 'Audit extension "pgaudit" already exists. Skipping.'; + ELSE + CREATE EXTENSION pgaudit; + END IF; +END +$$; diff --git a/examples/onboard-gcp-postgresql/main.tf b/examples/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..b22fb5a --- /dev/null +++ b/examples/onboard-gcp-postgresql/main.tf @@ -0,0 +1,360 @@ +locals { + admin_email = "test@example.com" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" + pubsub_auth_mechanism = "default" + + gcp_postgresql_instance_authorized_networks = [ + { + name = "local" + value = "127.0.0.1" + } + ] + gcp_project_id = "my-gcp-project" + gcp_service_account_name = "dsf-service-account" + + excluded_traffic_filter = [ + { + name = "exclude-cloudsqladmin-traffic" + filter = "textPayload:\"user=cloudsqladmin\"" + } + ] + postgresql_admin = "admin" + postgresql_admin_password = "Abcd1234" +} + +################################################################################ +# Providers +################################################################################ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +provider "google" { + # Authenticated via "gcloud" CLI + project = local.gcp_project_id +} + +provider "dsfhub" {} + +################################################################################ +# Prerequisites +# 1. A service account with permissions to read from the PubSub subscription +# 2. A Google sink router, PubSub topic and subscription (handled below) +# 3. A method to create 'pgaudit' extension on the postgresql instance. (handled +# below) +################################################################################ +module "service-account" { + source = "../../modules/google-service-account-dsf" + + account_id = local.gcp_service_account_name + auth_mechanism = local.pubsub_auth_mechanism + description = "PostgreSQL audit pull service account" + project = local.gcp_project_id + project_roles = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} + +################################################################################ +# GCP PostgreSQL 16 +################################################################################ +locals { + gcp_postgresql_1_instance_name = "tf-postgresql-16" +} + +module "gcp-pubsub-1" { + source = "../../modules/onboard-gcp-pubsub" + + gcp_pubsub_admin_email = local.admin_email + gcp_pubsub_audit_type = "POSTGRESQL" + gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism + gcp_pubsub_gateway_id = local.gateway_id + + project = local.gcp_project_id + + pubsub_subscription_name = "${local.gcp_postgresql_1_instance_name}-sub" + + pubsub_topic_name = "${local.gcp_postgresql_1_instance_name}-topic" + + sink_router_description = "PostgreSQL 16 sink" + sink_router_exclusions = local.excluded_traffic_filter + sink_router_filter = < Date: Mon, 18 Nov 2024 12:12:49 -0800 Subject: [PATCH 20/30] add gcp postgresql module --- CHANGELOG.md | 1 + DSF_VERSION_COMPATABILITY.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b80201..331bbb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Aurora PostgreSQL CloudWatch with slow query auditing example - Google Cloud SQL for MySQL module +- Google Cloud SQL for PostgreSQL module ## 1.0.8 (2024-10-15) diff --git a/DSF_VERSION_COMPATABILITY.md b/DSF_VERSION_COMPATABILITY.md index 05570b0..e2e5aca 100644 --- a/DSF_VERSION_COMPATABILITY.md +++ b/DSF_VERSION_COMPATABILITY.md @@ -107,5 +107,9 @@ The following table lists the DSF versions that each module is tested and mainta onboard-gcp-mysql 4.17+ + + onboard-gcp-postgresql + 4.17+ + \ No newline at end of file From aa1c1b3bc04119a8745b0f50032517e930a76a20 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Thu, 7 Nov 2024 11:17:53 -0800 Subject: [PATCH 21/30] add gcp mysql onboarding module --- modules/onboard-gcp-mysql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/onboard-gcp-mysql/README.md b/modules/onboard-gcp-mysql/README.md index c5848fb..684762c 100644 --- a/modules/onboard-gcp-mysql/README.md +++ b/modules/onboard-gcp-mysql/README.md @@ -53,4 +53,4 @@ No resources. |------|-------------| | [gcp-mysql-asset](#output\_gcp-mysql-asset) | GCP MYSQL asset | | [gcp-mysql-instance](#output\_gcp-mysql-instance) | Google MySQL database instance | - \ No newline at end of file + From 40d47b2d15ee68f06491e2d0a9e4f67cab29dc57 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Tue, 12 Nov 2024 15:34:16 -0800 Subject: [PATCH 22/30] add gcp mysql examples --- examples/onboard-gcp-mysql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onboard-gcp-mysql/README.md b/examples/onboard-gcp-mysql/README.md index ed9e377..2c30fd4 100644 --- a/examples/onboard-gcp-mysql/README.md +++ b/examples/onboard-gcp-mysql/README.md @@ -45,4 +45,4 @@ No inputs. ## Outputs No outputs. - \ No newline at end of file + From 8d7052f2a58dd4079de850c32f1c38814748af90 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Fri, 15 Nov 2024 15:05:21 -0800 Subject: [PATCH 23/30] add gcp postgresql asset helper module --- modules/dsfhub-gcp-postgresql/README.md | 0 modules/dsfhub-gcp-postgresql/main.tf | 22 ++++++++++ modules/dsfhub-gcp-postgresql/outputs.tf | 4 ++ modules/dsfhub-gcp-postgresql/variables.tf | 47 ++++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 modules/dsfhub-gcp-postgresql/README.md create mode 100644 modules/dsfhub-gcp-postgresql/main.tf create mode 100644 modules/dsfhub-gcp-postgresql/outputs.tf create mode 100644 modules/dsfhub-gcp-postgresql/variables.tf diff --git a/modules/dsfhub-gcp-postgresql/README.md b/modules/dsfhub-gcp-postgresql/README.md new file mode 100644 index 0000000..e69de29 diff --git a/modules/dsfhub-gcp-postgresql/main.tf b/modules/dsfhub-gcp-postgresql/main.tf new file mode 100644 index 0000000..c1a8804 --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_data_source" "this" { + server_type = "GCP POSTGRESQL" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + audit_pull_enabled = var.audit_pull_enabled + gateway_id = var.gateway_id + logs_destination_asset_id = var.logs_destination_asset_id + parent_asset_id = var.parent_asset_id + server_host_name = var.server_host_name + server_ip = var.server_ip + server_port = "5432" +} diff --git a/modules/dsfhub-gcp-postgresql/outputs.tf b/modules/dsfhub-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..18b138c --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP POSTGRESQL asset." + value = dsfhub_data_source.this +} diff --git a/modules/dsfhub-gcp-postgresql/variables.tf b/modules/dsfhub-gcp-postgresql/variables.tf new file mode 100644 index 0000000..21db9de --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/variables.tf @@ -0,0 +1,47 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier for the PostgreSQL instance in the form '{project-id}:{instance-region}:{instance-name}'." + type = string +} + +variable "audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "server_host_name" { + description = "Hostname (or IP if host is unknown) of the GCP PostgreSQL instance" + type = string +} + +variable "server_ip" { + description = "IP address (or hostname if IP is unknown) of the GCP PostgreSQL instance" + type = string +} From 7c6acdc00de4f38813928456abf45eb2261d1c17 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Fri, 15 Nov 2024 15:06:48 -0800 Subject: [PATCH 24/30] add gcp postgres onboarding module --- modules/onboard-gcp-postgresql/README.md | 11 ++ modules/onboard-gcp-postgresql/main.tf | 28 +++++ modules/onboard-gcp-postgresql/outputs.tf | 0 modules/onboard-gcp-postgresql/variables.tf | 130 ++++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 modules/onboard-gcp-postgresql/README.md create mode 100644 modules/onboard-gcp-postgresql/main.tf create mode 100644 modules/onboard-gcp-postgresql/outputs.tf create mode 100644 modules/onboard-gcp-postgresql/variables.tf diff --git a/modules/onboard-gcp-postgresql/README.md b/modules/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..616eefa --- /dev/null +++ b/modules/onboard-gcp-postgresql/README.md @@ -0,0 +1,11 @@ +# onboard-gcp-postgresql + +Onboard Cloud SQL for PostgreSQL to DSF Hub. + +## Notes +There are three prerequisites for using this module: +1. A Google Service Account with permissions to read from PubSub subscriptions. +2. A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF Hub. +3. A method to create the 'pgaudit' extension on the postgres instance. + +See the corresponding example for more details. diff --git a/modules/onboard-gcp-postgresql/main.tf b/modules/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..94eb66a --- /dev/null +++ b/modules/onboard-gcp-postgresql/main.tf @@ -0,0 +1,28 @@ +module "gcp-postgresql-instance" { + source = "../google-sql-database-instance" + + authorized_networks = var.instance_authorized_networks + database_flags = var.instance_database_flags + database_version = var.instance_database_version + deletion_protection = var.instance_deletion_protection + name = var.instance_name + project = var.instance_project + region = var.instance_region + root_password = null + sql_server_audit_config = null + tier = var.instance_tier +} + +module "gcp-postgresql-asset" { + source = "../dsfhub-gcp-postgresql" + + admin_email = var.gcp_postgresql_admin_email + asset_display_name = module.gcp-postgresql-instance.this.name + asset_id = "${module.gcp-postgresql-instance.this.project}:${module.gcp-postgresql-instance.this.region}:${module.gcp-postgresql-instance.this.name}" + audit_pull_enabled = var.gcp_postgresql_audit_pull_enabled + gateway_id = var.gcp_postgresql_gateway_id + logs_destination_asset_id = var.gcp_postgresql_logs_destination_asset_id + parent_asset_id = var.gcp_postgresql_parent_asset_id + server_host_name = module.gcp-postgresql-instance.this.ip_address.0.ip_address + server_ip = module.gcp-postgresql-instance.this.ip_address.0.ip_address +} diff --git a/modules/onboard-gcp-postgresql/outputs.tf b/modules/onboard-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/onboard-gcp-postgresql/variables.tf b/modules/onboard-gcp-postgresql/variables.tf new file mode 100644 index 0000000..bf46677 --- /dev/null +++ b/modules/onboard-gcp-postgresql/variables.tf @@ -0,0 +1,130 @@ +variable "gcp_postgresql_admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "gcp_postgresql_audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gcp_postgresql_gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "gcp_postgresql_logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "gcp_postgresql_parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "instance_authorized_networks" { + description = < Date: Fri, 15 Nov 2024 15:34:20 -0800 Subject: [PATCH 25/30] add outputs --- modules/onboard-gcp-postgresql/outputs.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/onboard-gcp-postgresql/outputs.tf b/modules/onboard-gcp-postgresql/outputs.tf index e69de29..7791c86 100644 --- a/modules/onboard-gcp-postgresql/outputs.tf +++ b/modules/onboard-gcp-postgresql/outputs.tf @@ -0,0 +1,9 @@ +output "gcp-postgresql-instance" { + description = "Google PostgreSQL database instance" + value = module.gcp-postgresql-instance.this +} + +output "gcp-postgresql-asset" { + description = "GCP POSTGRESQL asset" + value = module.gcp-postgresql-asset.this +} From 53de1c5011092540cc18bf00e048dfa9d503c940 Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 18 Nov 2024 12:09:25 -0800 Subject: [PATCH 26/30] add gcp postgres onboarding examples --- examples/onboard-gcp-postgresql/README.md | 16 + .../configure_database.sh | 38 ++ .../configure_database.sql | 11 + examples/onboard-gcp-postgresql/main.tf | 360 ++++++++++++++++++ modules/onboard-gcp-postgresql/variables.tf | 4 +- 5 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 examples/onboard-gcp-postgresql/README.md create mode 100755 examples/onboard-gcp-postgresql/configure_database.sh create mode 100644 examples/onboard-gcp-postgresql/configure_database.sql create mode 100644 examples/onboard-gcp-postgresql/main.tf diff --git a/examples/onboard-gcp-postgresql/README.md b/examples/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..cea5877 --- /dev/null +++ b/examples/onboard-gcp-postgresql/README.md @@ -0,0 +1,16 @@ +# Onboard Google Cloud SQL for PostgreSQL example +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](https://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/Cloud-SQL-for-PostgreSQL-Onboarding-Steps_48367600.html). + +This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: +- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) +- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Prerequisites +### Service Account +A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway. + +### Google PubSub Subscription +A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + +### Database Configuration +Part of the onboarding process involves connecting to your Google PostgreSQL instance and running SQL commands to create an extension. This module includes an example for how to connect to the instance from your local machine and create this. diff --git a/examples/onboard-gcp-postgresql/configure_database.sh b/examples/onboard-gcp-postgresql/configure_database.sh new file mode 100755 index 0000000..ff713bc --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Configures Google Postgresql database for auditing by connecting to the +# database using 'psql' +# +# Connection to the database uses PG environment variables +# See postgres documentation for more information: +# https://www.postgresql.org/docs/current/libpq-envars.html +################################################################################ + +# Settings +current_directory=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +sql_file="${current_directory}/configure_database.sql" + +# Functions +function is_pkg_installed { + local pkg="$1" + if ! command -v "${pkg}" &> /dev/null + then + echo "Package '${pkg}' is not installed." + echo "Install on MacOS: brew install libpq" + echo "Install on Ubuntu: apt-get install -y libpq-dev" + echo "Install on CentOS: yum install -y libpq" + echo "Exiting..." + exit 1 + else + return 0 + fi +} + +################################################################################ +is_pkg_installed "psql" +if [ ! -r "${sql_file}" ]; then + echo "Unable to read ${sql_file}" + echo "Exiting..." + exit 1 +else + psql --file="${sql_file}" +fi diff --git a/examples/onboard-gcp-postgresql/configure_database.sql b/examples/onboard-gcp-postgresql/configure_database.sql new file mode 100644 index 0000000..829e3e1 --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sql @@ -0,0 +1,11 @@ +DO +$$ +BEGIN + RAISE NOTICE 'Creating audit extension "pgaudit".'; + IF EXISTS (SELECT FROM pg_catalog.pg_extension WHERE extname = 'pgaudit') THEN + RAISE NOTICE 'Audit extension "pgaudit" already exists. Skipping.'; + ELSE + CREATE EXTENSION pgaudit; + END IF; +END +$$; diff --git a/examples/onboard-gcp-postgresql/main.tf b/examples/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..b22fb5a --- /dev/null +++ b/examples/onboard-gcp-postgresql/main.tf @@ -0,0 +1,360 @@ +locals { + admin_email = "test@example.com" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" + pubsub_auth_mechanism = "default" + + gcp_postgresql_instance_authorized_networks = [ + { + name = "local" + value = "127.0.0.1" + } + ] + gcp_project_id = "my-gcp-project" + gcp_service_account_name = "dsf-service-account" + + excluded_traffic_filter = [ + { + name = "exclude-cloudsqladmin-traffic" + filter = "textPayload:\"user=cloudsqladmin\"" + } + ] + postgresql_admin = "admin" + postgresql_admin_password = "Abcd1234" +} + +################################################################################ +# Providers +################################################################################ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +provider "google" { + # Authenticated via "gcloud" CLI + project = local.gcp_project_id +} + +provider "dsfhub" {} + +################################################################################ +# Prerequisites +# 1. A service account with permissions to read from the PubSub subscription +# 2. A Google sink router, PubSub topic and subscription (handled below) +# 3. A method to create 'pgaudit' extension on the postgresql instance. (handled +# below) +################################################################################ +module "service-account" { + source = "../../modules/google-service-account-dsf" + + account_id = local.gcp_service_account_name + auth_mechanism = local.pubsub_auth_mechanism + description = "PostgreSQL audit pull service account" + project = local.gcp_project_id + project_roles = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} + +################################################################################ +# GCP PostgreSQL 16 +################################################################################ +locals { + gcp_postgresql_1_instance_name = "tf-postgresql-16" +} + +module "gcp-pubsub-1" { + source = "../../modules/onboard-gcp-pubsub" + + gcp_pubsub_admin_email = local.admin_email + gcp_pubsub_audit_type = "POSTGRESQL" + gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism + gcp_pubsub_gateway_id = local.gateway_id + + project = local.gcp_project_id + + pubsub_subscription_name = "${local.gcp_postgresql_1_instance_name}-sub" + + pubsub_topic_name = "${local.gcp_postgresql_1_instance_name}-topic" + + sink_router_description = "PostgreSQL 16 sink" + sink_router_exclusions = local.excluded_traffic_filter + sink_router_filter = < Date: Mon, 18 Nov 2024 12:12:49 -0800 Subject: [PATCH 27/30] add gcp postgresql module --- CHANGELOG.md | 1 + DSF_VERSION_COMPATABILITY.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a895ef..e8b0f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Aurora PostgreSQL CloudWatch with slow query auditing example - Google Cloud SQL for MySQL module +- Google Cloud SQL for PostgreSQL module ### Bug Fixes - Modified Server Host Name of AWS RDS MS SQL SERVER Dsfhub assets diff --git a/DSF_VERSION_COMPATABILITY.md b/DSF_VERSION_COMPATABILITY.md index 05570b0..e2e5aca 100644 --- a/DSF_VERSION_COMPATABILITY.md +++ b/DSF_VERSION_COMPATABILITY.md @@ -107,5 +107,9 @@ The following table lists the DSF versions that each module is tested and mainta onboard-gcp-mysql 4.17+ + + onboard-gcp-postgresql + 4.17+ + \ No newline at end of file From 2d3743b11d97205fc43db811e2067c3dda8a5fdc Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 18 Nov 2024 12:23:37 -0800 Subject: [PATCH 28/30] fix changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331bbb7..e8b0f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ - Google Cloud SQL for MySQL module - Google Cloud SQL for PostgreSQL module +### Bug Fixes +- Modified Server Host Name of AWS RDS MS SQL SERVER Dsfhub assets + ## 1.0.8 (2024-10-15) ### Features From ff6f5f35b8a556affeb937ec3bbc2eb1fc1e8c2d Mon Sep 17 00:00:00 2001 From: mattJsonar Date: Mon, 18 Nov 2024 20:25:37 +0000 Subject: [PATCH 29/30] Apply automatic changes --- examples/onboard-gcp-postgresql/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/onboard-gcp-postgresql/main.tf b/examples/onboard-gcp-postgresql/main.tf index b22fb5a..e4dfadc 100644 --- a/examples/onboard-gcp-postgresql/main.tf +++ b/examples/onboard-gcp-postgresql/main.tf @@ -18,7 +18,7 @@ locals { filter = "textPayload:\"user=cloudsqladmin\"" } ] - postgresql_admin = "admin" + postgresql_admin = "admin" postgresql_admin_password = "Abcd1234" } @@ -296,7 +296,7 @@ module "gcp-pubsub-3" { gcp_pubsub_audit_type = "POSTGRESQL" gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism gcp_pubsub_gateway_id = local.gateway_id - + project = local.gcp_project_id pubsub_subscription_name = "${local.gcp_postgresql_3_instance_name_prefix}-sub" From c5270411e0f048c53088a778ecf01ebe70174487 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Nov 2024 20:26:45 +0000 Subject: [PATCH 30/30] terraform-docs: automated action --- examples/onboard-gcp-postgresql/README.md | 44 ++++++++++++++++++++++ modules/dsfhub-gcp-postgresql/README.md | 41 ++++++++++++++++++++ modules/onboard-gcp-postgresql/README.md | 46 +++++++++++++++++++++++ 3 files changed, 131 insertions(+) diff --git a/examples/onboard-gcp-postgresql/README.md b/examples/onboard-gcp-postgresql/README.md index cea5877..d7a27b9 100644 --- a/examples/onboard-gcp-postgresql/README.md +++ b/examples/onboard-gcp-postgresql/README.md @@ -14,3 +14,47 @@ A Google logging sink, PubSub topic, and PubSub subscription in addition to a GC ### Database Configuration Part of the onboarding process involves connecting to your Google PostgreSQL instance and running SQL commands to create an extension. This module includes an example for how to connect to the instance from your local machine and create this. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-postgresql-1](#module\_gcp-postgresql-1) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-postgresql-2](#module\_gcp-postgresql-2) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-postgresql-3](#module\_gcp-postgresql-3) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-pubsub-1](#module\_gcp-pubsub-1) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-2](#module\_gcp-pubsub-2) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-3](#module\_gcp-pubsub-3) | ../../modules/onboard-gcp-pubsub | n/a | +| [service-account](#module\_service-account) | ../../modules/google-service-account-dsf | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_sql_user.gcp-postgresql-admin-user-1](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [google_sql_user.gcp-postgresql-admin-user-2](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [google_sql_user.gcp-postgresql-admin-user-3](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [terraform_data.configure_database_1](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.configure_database_2](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.configure_database_3](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +No inputs. + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/modules/dsfhub-gcp-postgresql/README.md b/modules/dsfhub-gcp-postgresql/README.md index e69de29..87ec78b 100644 --- a/modules/dsfhub-gcp-postgresql/README.md +++ b/modules/dsfhub-gcp-postgresql/README.md @@ -0,0 +1,41 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_data_source.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/data_source) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier for the PostgreSQL instance in the form '{project-id}:{instance-region}:{instance-name}'. | `string` | n/a | yes | +| [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [server\_host\_name](#input\_server\_host\_name) | Hostname (or IP if host is unknown) of the GCP PostgreSQL instance | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | IP address (or hostname if IP is unknown) of the GCP PostgreSQL instance | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP POSTGRESQL asset. | + \ No newline at end of file diff --git a/modules/onboard-gcp-postgresql/README.md b/modules/onboard-gcp-postgresql/README.md index 616eefa..38349f7 100644 --- a/modules/onboard-gcp-postgresql/README.md +++ b/modules/onboard-gcp-postgresql/README.md @@ -9,3 +9,49 @@ There are three prerequisites for using this module: 3. A method to create the 'pgaudit' extension on the postgres instance. See the corresponding example for more details. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-postgresql-asset](#module\_gcp-postgresql-asset) | ../dsfhub-gcp-postgresql | n/a | +| [gcp-postgresql-instance](#module\_gcp-postgresql-instance) | ../google-sql-database-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcp\_postgresql\_admin\_email](#input\_gcp\_postgresql\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [gcp\_postgresql\_audit\_pull\_enabled](#input\_gcp\_postgresql\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gcp\_postgresql\_gateway\_id](#input\_gcp\_postgresql\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [gcp\_postgresql\_logs\_destination\_asset\_id](#input\_gcp\_postgresql\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [gcp\_postgresql\_parent\_asset\_id](#input\_gcp\_postgresql\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [instance\_authorized\_networks](#input\_instance\_authorized\_networks) | A list of authorized network blocks as defined below.

authorized\_network:
- expiration\_time: (Optional) The RFC 3339 formatted date time string indicating when this whitelist expires.
- name: (Optional) A name for this whitelist entry.
- value: A CIDR notation IPv4 or IPv6 address that is allowed to access this instance. | 
list(
    object(
      {
        expiration_time = optional(string)
        name            = optional(string)
        value           = string
      }
    )
  )
| n/a | yes | +| [instance\_database\_flags](#input\_instance\_database\_flags) | List of database flags to assign to the instance. | 
list(
    object(
      {
        name  = string
        value = string
      }
    )
  )
| 
[
  {
    "name": "cloudsql.enable_pgaudit",
    "value": "on"
  },
  {
    "name": "log_error_verbosity",
    "value": "verbose"
  },
  {
    "name": "log_connections",
    "value": "on"
  },
  {
    "name": "log_disconnections",
    "value": "on"
  },
  {
    "name": "log_hostname",
    "value": "on"
  },
  {
    "name": "pgaudit.log",
    "value": "all"
  },
  {
    "name": "log_line_prefix",
    "value": "SONAR_AUDIT=1|TIMESTAMP=%m|APPLICATION_NAME=%a|USER=%u|DATABASE=%d|REMOTE_HOST_AND_PORT=%r|SQL_STATE=%e|SESSION_ID=%c|SESSION_START=%s|PROCESS_ID=[%p]|VIRTUAL_TRANSACTION_ID=%v|TRANSACTION_ID=%x| "
  }
]
| no | +| [instance\_database\_version](#input\_instance\_database\_version) | The PostgreSQL version to use. The full list of supported versions can be found at https://cloud.google.com/sql/docs/db-versions. | `string` | `"POSTGRES_16"` | no | +| [instance\_deletion\_protection](#input\_instance\_deletion\_protection) | Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed. | `bool` | `false` | no | +| [instance\_name](#input\_instance\_name) | The name of the instance. | `string` | n/a | yes | +| [instance\_project](#input\_instance\_project) | The ID of the project that the service account will be created in. | `string` | `null` | no | +| [instance\_region](#input\_instance\_region) | The region the instance will sit in. If a region is not provided in the resource definition, the provider region will be used instead. | `string` | `null` | no | +| [instance\_tier](#input\_instance\_tier) | The machine type to use. See [tiers](https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/tiers) for more details and supported versions | `string` | `"db-perf-optimized-N-2"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp-postgresql-asset](#output\_gcp-postgresql-asset) | GCP POSTGRESQL asset | +| [gcp-postgresql-instance](#output\_gcp-postgresql-instance) | Google PostgreSQL database instance | + \ No newline at end of file